diff --git a/apps/client/src/server_types.ts b/apps/client/src/server_types.ts index 2aa5214052..34de9ecc43 100644 --- a/apps/client/src/server_types.ts +++ b/apps/client/src/server_types.ts @@ -23,4 +23,4 @@ export interface EntityChange { instanceId?: string | null; } -export type EntityType = "notes" | "branches" | "attributes" | "note_reordering" | "revisions" | "options" | "attachments" | "blobs" | "etapi_tokens" | "note_embeddings"; +export type EntityType = "notes" | "branches" | "attributes" | "note_reordering" | "revisions" | "options" | "attachments" | "blobs" | "etapi_tokens"; diff --git a/apps/client/src/services/froca_updater.ts b/apps/client/src/services/froca_updater.ts index 412d8d6cde..b528047ed2 100644 --- a/apps/client/src/services/froca_updater.ts +++ b/apps/client/src/services/froca_updater.ts @@ -35,7 +35,7 @@ async function processEntityChanges(entityChanges: EntityChange[]) { loadResults.addOption(attributeEntity.name); } else if (ec.entityName === "attachments") { processAttachment(loadResults, ec); - } else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens" || ec.entityName === "note_embeddings") { + } else if (ec.entityName === "blobs" || ec.entityName === "etapi_tokens") { // NOOP - these entities are handled at the backend level and don't require frontend processing } else { throw new Error(`Unknown entityName '${ec.entityName}'`); diff --git a/apps/client/src/services/load_results.ts b/apps/client/src/services/load_results.ts index 59d201f2be..5f9db2db7f 100644 --- a/apps/client/src/services/load_results.ts +++ b/apps/client/src/services/load_results.ts @@ -44,18 +44,7 @@ interface OptionRow {} interface NoteReorderingRow {} -interface NoteEmbeddingRow { - embedId: string; - noteId: string; - providerId: string; - modelId: string; - dimension: number; - version: number; - dateCreated: string; - utcDateCreated: string; - dateModified: string; - utcDateModified: string; -} + type EntityRowMappings = { notes: NoteRow; @@ -64,7 +53,6 @@ type EntityRowMappings = { options: OptionRow; revisions: RevisionRow; note_reordering: NoteReorderingRow; - note_embeddings: NoteEmbeddingRow; }; export type EntityRowNames = keyof EntityRowMappings; diff --git a/apps/client/src/translations/en/translation.json b/apps/client/src/translations/en/translation.json index c3f06f5da4..5d2e03ec5d 100644 --- a/apps/client/src/translations/en/translation.json +++ b/apps/client/src/translations/en/translation.json @@ -1124,10 +1124,8 @@ "layout-horizontal-description": "launcher bar is underneath the tab bar, the tab bar is now full width." }, "ai_llm": { - "embeddings_configuration": "Embeddings Configuration", "not_started": "Not started", - "title": "AI & Embedding Settings", - "embedding_statistics": "Embedding Statistics", + "title": "AI Settings", "processed_notes": "Processed Notes", "total_notes": "Total Notes", "progress": "Progress", @@ -1135,7 +1133,6 @@ "failed_notes": "Failed Notes", "last_processed": "Last Processed", "refresh_stats": "Refresh Statistics", - "no_failed_embeddings": "No failed embeddings found.", "enable_ai_features": "Enable AI/LLM features", "enable_ai_description": "Enable AI features like note summarization, content generation, and other LLM capabilities", "openai_tab": "OpenAI", @@ -1160,20 +1157,16 @@ "anthropic_api_key_description": "Your Anthropic API key for accessing Claude models", "default_model": "Default Model", "openai_model_description": "Examples: gpt-4o, gpt-4-turbo, gpt-3.5-turbo", - "embedding_model": "Embedding Model", - "openai_embedding_model_description": "Model used for generating embeddings (text-embedding-3-small recommended)", "base_url": "Base URL", "openai_url_description": "Default: https://api.openai.com/v1", "anthropic_settings": "Anthropic Settings", "anthropic_url_description": "Base URL for the Anthropic API (default: https://api.anthropic.com)", "anthropic_model_description": "Anthropic Claude models for chat completion", "voyage_settings": "Voyage AI Settings", - "voyage_api_key_description": "Your Voyage AI API key for accessing embeddings services", "ollama_settings": "Ollama Settings", "ollama_url_description": "URL for the Ollama API (default: http://localhost:11434)", "ollama_model_description": "Ollama model to use for chat completion", "anthropic_configuration": "Anthropic Configuration", - "voyage_embedding_model_description": "Voyage AI embedding models for text embeddings (voyage-2 recommended)", "voyage_configuration": "Voyage AI Configuration", "voyage_url_description": "Default: https://api.voyageai.com/v1", "ollama_configuration": "Ollama Configuration", @@ -1181,28 +1174,10 @@ "enable_ollama_description": "Enable Ollama for local AI model usage", "ollama_url": "Ollama URL", "ollama_model": "Ollama Model", - "ollama_embedding_model": "Embedding Model", - "ollama_embedding_model_description": "Specialized model for generating embeddings (vector representations)", "refresh_models": "Refresh Models", "refreshing_models": "Refreshing...", - "embedding_configuration": "Embeddings Configuration", - "embedding_default_provider": "Default Provider", - "embedding_default_provider_description": "Select the default provider used for generating note embeddings", - "embedding_provider_precedence": "Embedding Provider Precedence", - "embedding_providers_order": "Embedding Provider Order", - "embedding_providers_order_description": "Set the order of embedding providers in comma-separated format (e.g., \"openai,voyage,ollama,local\")", "enable_automatic_indexing": "Enable Automatic Indexing", - "enable_automatic_indexing_description": "Automatically generate embeddings for new and updated notes", - "embedding_auto_update_enabled": "Auto-update Embeddings", - "embedding_auto_update_enabled_description": "Automatically update embeddings when notes are modified", - "recreate_embeddings": "Recreate All Embeddings", - "recreate_embeddings_description": "Regenerate all note embeddings from scratch (may take a long time for large note collections)", - "recreate_embeddings_started": "Embeddings regeneration started. This may take a long time for large note collections.", - "recreate_embeddings_error": "Error starting embeddings regeneration. Check logs for details.", - "recreate_embeddings_confirm": "Are you sure you want to recreate all embeddings? This may take a long time for large note collections.", "rebuild_index": "Rebuild Index", - "rebuild_index_description": "Rebuild the vector search index for better performance (much faster than recreating embeddings)", - "rebuild_index_started": "Embedding index rebuild started. This may take several minutes.", "rebuild_index_error": "Error starting index rebuild. Check logs for details.", "note_title": "Note Title", "error": "Error", @@ -1212,43 +1187,16 @@ "partial": "{{ percentage }}% completed", "retry_queued": "Note queued for retry", "retry_failed": "Failed to queue note for retry", - "embedding_provider_precedence_description": "Comma-separated list of providers in order of precedence for embeddings search (e.g., 'openai,ollama,anthropic')", - "embedding_dimension_strategy": "Embedding Dimension Strategy", - "embedding_dimension_auto": "Auto (Recommended)", - "embedding_dimension_fixed": "Fixed", - "embedding_similarity_threshold": "Similarity Threshold", - "embedding_similarity_threshold_description": "Minimum similarity score for notes to be included in search results (0-1)", "max_notes_per_llm_query": "Max Notes Per Query", "max_notes_per_llm_query_description": "Maximum number of similar notes to include in AI context", - "embedding_dimension_strategy_description": "Choose how embeddings are handled. 'Native' preserves maximum information by adapting smaller vectors to match larger ones (recommended). 'Regenerate' creates new embeddings with the target model for specific search needs.", - "drag_providers_to_reorder": "Drag providers up or down to set your preferred order for embedding searches", "active_providers": "Active Providers", "disabled_providers": "Disabled Providers", "remove_provider": "Remove provider from search", "restore_provider": "Restore provider to search", - "embedding_generation_location": "Generation Location", - "embedding_generation_location_description": "Select where embedding generation should happen", - "embedding_generation_location_client": "Client/Server", - "embedding_generation_location_sync_server": "Sync Server", - "enable_auto_update_embeddings": "Auto-update Embeddings", - "enable_auto_update_embeddings_description": "Automatically update embeddings when notes are modified", - "auto_update_embeddings": "Auto-update Embeddings", - "auto_update_embeddings_desc": "Automatically update embeddings when notes are modified", "similarity_threshold": "Similarity Threshold", "similarity_threshold_description": "Minimum similarity score (0-1) for notes to be included in context for LLM queries", - "embedding_batch_size": "Batch Size", - "embedding_batch_size_description": "Number of notes to process in a single batch (1-50)", - "embedding_update_interval": "Update Interval (ms)", - "embedding_update_interval_description": "Time between processing batches of embeddings (in milliseconds)", - "embedding_default_dimension": "Default Dimension", - "embedding_default_dimension_description": "Default embedding vector dimension when creating new embeddings", - "reprocess_all_embeddings": "Reprocess All Embeddings", - "reprocess_all_embeddings_description": "Queue all notes for embedding processing. This may take some time depending on your number of notes.", - "reprocessing_embeddings": "Reprocessing...", - "reprocess_started": "Embedding reprocessing started in the background", - "reprocess_error": "Error starting embedding reprocessing", + "reprocess_index": "Rebuild Search Index", - "reprocess_index_description": "Optimize the search index for better performance. This uses existing embeddings without regenerating them (much faster than reprocessing all embeddings).", "reprocessing_index": "Rebuilding...", "reprocess_index_started": "Search index optimization started in the background", "reprocess_index_error": "Error rebuilding search index", @@ -1261,7 +1209,6 @@ "incomplete": "Incomplete ({{percentage}}%)", "complete": "Complete (100%)", "refreshing": "Refreshing...", - "stats_error": "Error fetching embedding statistics", "auto_refresh_notice": "Auto-refreshes every {{seconds}} seconds", "note_queued_for_retry": "Note queued for retry", "failed_to_retry_note": "Failed to retry note", @@ -1269,7 +1216,6 @@ "failed_to_retry_all": "Failed to retry notes", "ai_settings": "AI Settings", "api_key_tooltip": "API key for accessing the service", - "confirm_delete_embeddings": "Are you sure you want to delete all AI embeddings? This will remove all semantic search capabilities until notes are reindexed, which can take a significant amount of time.", "empty_key_warning": { "anthropic": "Anthropic API key is empty. Please enter a valid API key.", "openai": "OpenAI API key is empty. Please enter a valid API key.", @@ -1302,7 +1248,6 @@ "note_chat": "Note Chat", "notes_indexed": "{{ count }} note indexed", "notes_indexed_plural": "{{ count }} notes indexed", - "reset_embeddings": "Reset Embeddings", "sources": "Sources", "start_indexing": "Start Indexing", "use_advanced_context": "Use Advanced Context", @@ -1315,24 +1260,10 @@ }, "create_new_ai_chat": "Create new AI Chat", "configuration_warnings": "There are some issues with your AI configuration. Please check your settings.", - "embeddings_started": "Embedding generation started", - "embeddings_stopped": "Embedding generation stopped", - "embeddings_toggle_error": "Error toggling embeddings", - "local_embedding_description": "Uses local embedding models for offline text embedding generation", - "local_embedding_settings": "Local Embedding Settings", - "ollama_embedding_settings": "Ollama Embedding Settings", - "ollama_embedding_url_description": "URL for the Ollama API for embedding generation (default: http://localhost:11434)", - "openai_embedding_api_key_description": "Your OpenAI API key for embedding generation (can be different from chat API key)", - "openai_embedding_settings": "OpenAI Embedding Settings", - "openai_embedding_url_description": "Base URL for OpenAI embedding API (default: https://api.openai.com/v1)", - "selected_embedding_provider": "Selected Embedding Provider", - "selected_embedding_provider_description": "Choose the provider for generating note embeddings", "selected_provider": "Selected Provider", "selected_provider_description": "Choose the AI provider for chat and completion features", - "select_embedding_provider": "Select embedding provider...", "select_model": "Select model...", - "select_provider": "Select provider...", - "voyage_embedding_url_description": "Base URL for the Voyage AI embedding API (default: https://api.voyageai.com/v1)" + "select_provider": "Select provider..." }, "zoom_factor": { "title": "Zoom Factor (desktop build only)", diff --git a/apps/client/src/widgets/llm_chat/communication.ts b/apps/client/src/widgets/llm_chat/communication.ts index 614add7ada..ae231ca20d 100644 --- a/apps/client/src/widgets/llm_chat/communication.ts +++ b/apps/client/src/widgets/llm_chat/communication.ts @@ -258,9 +258,3 @@ export async function getDirectResponse(noteId: string, messageParams: any): Pro } } -/** - * Get embedding statistics - */ -export async function getEmbeddingStats(): Promise { - return server.get('llm/embeddings/stats'); -} diff --git a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts index 4565c0db24..be401031fd 100644 --- a/apps/client/src/widgets/llm_chat/llm_chat_panel.ts +++ b/apps/client/src/widgets/llm_chat/llm_chat_panel.ts @@ -11,7 +11,7 @@ import { TPL, addMessageToChat, showSources, hideSources, showLoadingIndicator, import { formatMarkdown } from "./utils.js"; import { createChatSession, checkSessionExists, setupStreamingResponse, getDirectResponse } from "./communication.js"; import { extractInChatToolSteps } from "./message_processor.js"; -import { validateEmbeddingProviders } from "./validation.js"; +import { validateProviders } from "./validation.js"; import type { MessageData, ToolExecutionStep, ChatData } from "./types.js"; import { formatCodeBlocks } from "../../services/syntax_highlight.js"; import { ClassicEditor, type CKTextEditor, type MentionFeed } from "@triliumnext/ckeditor5"; @@ -616,7 +616,7 @@ export default class LlmChatPanel extends BasicWidget { } // Check for any provider validation issues when refreshing - await validateEmbeddingProviders(this.validationWarning); + await validateProviders(this.validationWarning); // Get current note context if needed const currentActiveNoteId = appContext.tabManager.getActiveContext()?.note?.noteId || null; @@ -767,7 +767,7 @@ export default class LlmChatPanel extends BasicWidget { */ private async processUserMessage(content: string) { // Check for validation issues first - await validateEmbeddingProviders(this.validationWarning); + await validateProviders(this.validationWarning); // Make sure we have a valid session if (!this.noteId) { diff --git a/apps/client/src/widgets/llm_chat/validation.ts b/apps/client/src/widgets/llm_chat/validation.ts index 94977f63ae..60aac70030 100644 --- a/apps/client/src/widgets/llm_chat/validation.ts +++ b/apps/client/src/widgets/llm_chat/validation.ts @@ -2,12 +2,11 @@ * Validation functions for LLM Chat */ import options from "../../services/options.js"; -import { getEmbeddingStats } from "./communication.js"; /** - * Validate embedding providers configuration + * Validate providers configuration */ -export async function validateEmbeddingProviders(validationWarning: HTMLElement): Promise { +export async function validateProviders(validationWarning: HTMLElement): Promise { try { // Check if AI is enabled const aiEnabled = options.is('aiEnabled'); @@ -62,23 +61,8 @@ export async function validateEmbeddingProviders(validationWarning: HTMLElement) // Add checks for other providers as needed } - // Fetch embedding stats to check if there are any notes being processed - const embeddingStats = await getEmbeddingStats() as { - success: boolean, - stats: { - totalNotesCount: number; - embeddedNotesCount: number; - queuedNotesCount: number; - failedNotesCount: number; - lastProcessedDate: string | null; - percentComplete: number; - } - }; - const queuedNotes = embeddingStats?.stats?.queuedNotesCount || 0; - const hasEmbeddingsInQueue = queuedNotes > 0; - - // Show warning if there are configuration issues or embeddings in queue - if (configIssues.length > 0 || hasEmbeddingsInQueue) { + // Show warning if there are configuration issues + if (configIssues.length > 0) { let message = 'AI Provider Configuration Issues'; message += '
    '; @@ -87,11 +71,6 @@ export async function validateEmbeddingProviders(validationWarning: HTMLElement) for (const issue of configIssues) { message += `
  • ${issue}
    • `; } - - // Show warning about embeddings queue if applicable - if (hasEmbeddingsInQueue) { - message += `
  • Currently processing embeddings for ${queuedNotes} notes. Some AI features may produce incomplete results until processing completes.
    • `; - } message += '
'; message += '
Open AI Settings
'; @@ -103,7 +82,7 @@ export async function validateEmbeddingProviders(validationWarning: HTMLElement) validationWarning.style.display = 'none'; } } catch (error) { - console.error('Error validating embedding providers:', error); + console.error('Error validating providers:', error); validationWarning.style.display = 'none'; } } diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts index 9117268830..583f24f9c9 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/ai_settings_widget.ts @@ -4,16 +4,12 @@ import { t } from "../../../../services/i18n.js"; import type { OptionDefinitions, OptionMap } from "@triliumnext/commons"; import server from "../../../../services/server.js"; import toastService from "../../../../services/toast.js"; -import type { EmbeddingStats, FailedEmbeddingNotes } from "./interfaces.js"; import { ProviderService } from "./providers.js"; export default class AiSettingsWidget extends OptionsWidget { private ollamaModelsRefreshed = false; private openaiModelsRefreshed = false; private anthropicModelsRefreshed = false; - private statsRefreshInterval: NodeJS.Timeout | null = null; - private indexRebuildRefreshInterval: NodeJS.Timeout | null = null; - private readonly STATS_REFRESH_INTERVAL = 5000; // 5 seconds private providerService: ProviderService | null = null; doRender() { @@ -23,9 +19,6 @@ export default class AiSettingsWidget extends OptionsWidget { // Setup event handlers for options this.setupEventHandlers(); - this.refreshEmbeddingStats(); - this.fetchFailedEmbeddingNotes(); - return this.$widget; } @@ -57,26 +50,13 @@ export default class AiSettingsWidget extends OptionsWidget { const isEnabled = value === 'true'; if (isEnabled) { - // Start embedding generation - await server.post('llm/embeddings/start'); - toastService.showMessage(t("ai_llm.embeddings_started") || "Embedding generation started"); - - // Start polling for stats updates - this.refreshEmbeddingStats(); + toastService.showMessage(t("ai_llm.ai_enabled") || "AI features enabled"); } else { - // Stop embedding generation - await server.post('llm/embeddings/stop'); - toastService.showMessage(t("ai_llm.embeddings_stopped") || "Embedding generation stopped"); - - // Clear any active polling intervals - if (this.indexRebuildRefreshInterval) { - clearInterval(this.indexRebuildRefreshInterval); - this.indexRebuildRefreshInterval = null; - } + toastService.showMessage(t("ai_llm.ai_disabled") || "AI features disabled"); } } catch (error) { - console.error('Error toggling embeddings:', error); - toastService.showError(t("ai_llm.embeddings_toggle_error") || "Error toggling embeddings"); + console.error('Error toggling AI:', error); + toastService.showError(t("ai_llm.ai_toggle_error") || "Error toggling AI features"); } } @@ -102,7 +82,6 @@ export default class AiSettingsWidget extends OptionsWidget { this.setupChangeHandler('.openai-api-key', 'openaiApiKey', true); this.setupChangeHandler('.openai-base-url', 'openaiBaseUrl', true); this.setupChangeHandler('.openai-default-model', 'openaiDefaultModel'); - this.setupChangeHandler('.openai-embedding-model', 'openaiEmbeddingModel'); // Anthropic options this.setupChangeHandler('.anthropic-api-key', 'anthropicApiKey', true); @@ -111,18 +90,10 @@ export default class AiSettingsWidget extends OptionsWidget { // Voyage options this.setupChangeHandler('.voyage-api-key', 'voyageApiKey'); - this.setupChangeHandler('.voyage-embedding-model', 'voyageEmbeddingModel'); - this.setupChangeHandler('.voyage-embedding-base-url', 'voyageEmbeddingBaseUrl'); // Ollama options this.setupChangeHandler('.ollama-base-url', 'ollamaBaseUrl'); this.setupChangeHandler('.ollama-default-model', 'ollamaDefaultModel'); - this.setupChangeHandler('.ollama-embedding-model', 'ollamaEmbeddingModel'); - this.setupChangeHandler('.ollama-embedding-base-url', 'ollamaEmbeddingBaseUrl'); - - // Embedding-specific provider options - this.setupChangeHandler('.openai-embedding-api-key', 'openaiEmbeddingApiKey', true); - this.setupChangeHandler('.openai-embedding-base-url', 'openaiEmbeddingBaseUrl', true); const $refreshModels = this.$widget.find('.refresh-models'); $refreshModels.on('click', async () => { @@ -162,15 +133,6 @@ export default class AiSettingsWidget extends OptionsWidget { this.anthropicModelsRefreshed = await this.providerService?.refreshAnthropicModels(false, this.anthropicModelsRefreshed) || false; }); - // Embedding options event handlers - this.setupChangeHandler('.embedding-auto-update-enabled', 'embeddingAutoUpdateEnabled', false, true); - this.setupChangeHandler('.enable-automatic-indexing', 'enableAutomaticIndexing', false, true); - this.setupChangeHandler('.embedding-similarity-threshold', 'embeddingSimilarityThreshold'); - this.setupChangeHandler('.max-notes-per-llm-query', 'maxNotesPerLlmQuery'); - this.setupChangeHandler('.embedding-selected-provider', 'embeddingSelectedProvider', true); - this.setupChangeHandler('.embedding-dimension-strategy', 'embeddingDimensionStrategy'); - this.setupChangeHandler('.embedding-batch-size', 'embeddingBatchSize'); - this.setupChangeHandler('.embedding-update-interval', 'embeddingUpdateInterval'); // Add provider selection change handlers for dynamic settings visibility this.$widget.find('.ai-selected-provider').on('change', async () => { @@ -183,26 +145,13 @@ export default class AiSettingsWidget extends OptionsWidget { } }); - this.$widget.find('.embedding-selected-provider').on('change', async () => { - const selectedProvider = this.$widget.find('.embedding-selected-provider').val() as string; - this.$widget.find('.embedding-provider-settings').hide(); - if (selectedProvider) { - this.$widget.find(`.${selectedProvider}-embedding-provider-settings`).show(); - // Automatically fetch embedding models for the newly selected provider - await this.fetchModelsForProvider(selectedProvider, 'embedding'); - } - }); // Add base URL change handlers to trigger model fetching this.$widget.find('.openai-base-url').on('change', async () => { const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; if (selectedProvider === 'openai') { await this.fetchModelsForProvider('openai', 'chat'); } - if (selectedEmbeddingProvider === 'openai') { - await this.fetchModelsForProvider('openai', 'embedding'); - } }); this.$widget.find('.anthropic-base-url').on('change', async () => { @@ -214,25 +163,17 @@ export default class AiSettingsWidget extends OptionsWidget { this.$widget.find('.ollama-base-url').on('change', async () => { const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; if (selectedProvider === 'ollama') { await this.fetchModelsForProvider('ollama', 'chat'); } - if (selectedEmbeddingProvider === 'ollama') { - await this.fetchModelsForProvider('ollama', 'embedding'); - } }); // Add API key change handlers to trigger model fetching this.$widget.find('.openai-api-key').on('change', async () => { const selectedProvider = this.$widget.find('.ai-selected-provider').val() as string; - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; if (selectedProvider === 'openai') { await this.fetchModelsForProvider('openai', 'chat'); } - if (selectedEmbeddingProvider === 'openai') { - await this.fetchModelsForProvider('openai', 'embedding'); - } }); this.$widget.find('.anthropic-api-key').on('change', async () => { @@ -242,85 +183,6 @@ export default class AiSettingsWidget extends OptionsWidget { } }); - this.$widget.find('.voyage-api-key').on('change', async () => { - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - if (selectedEmbeddingProvider === 'voyage') { - // Voyage doesn't have dynamic model fetching yet, but we can add it here when implemented - console.log('Voyage API key changed - model fetching not yet implemented'); - } - }); - - // Add embedding base URL change handlers to trigger model fetching - this.$widget.find('.openai-embedding-base-url').on('change', async () => { - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - if (selectedEmbeddingProvider === 'openai') { - await this.fetchModelsForProvider('openai', 'embedding'); - } - }); - - this.$widget.find('.voyage-embedding-base-url').on('change', async () => { - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - if (selectedEmbeddingProvider === 'voyage') { - // Voyage doesn't have dynamic model fetching yet, but we can add it here when implemented - console.log('Voyage embedding base URL changed - model fetching not yet implemented'); - } - }); - - this.$widget.find('.ollama-embedding-base-url').on('change', async () => { - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - if (selectedEmbeddingProvider === 'ollama') { - await this.fetchModelsForProvider('ollama', 'embedding'); - } - }); - - // Add embedding API key change handlers to trigger model fetching - this.$widget.find('.openai-embedding-api-key').on('change', async () => { - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - if (selectedEmbeddingProvider === 'openai') { - await this.fetchModelsForProvider('openai', 'embedding'); - } - }); - - // No sortable behavior needed anymore - - // Embedding stats refresh button - const $refreshStats = this.$widget.find('.embedding-refresh-stats'); - $refreshStats.on('click', async () => { - await this.refreshEmbeddingStats(); - await this.fetchFailedEmbeddingNotes(); - }); - - // Recreate embeddings button - const $recreateEmbeddings = this.$widget.find('.recreate-embeddings'); - $recreateEmbeddings.on('click', async () => { - if (confirm(t("ai_llm.recreate_embeddings_confirm") || "Are you sure you want to recreate all embeddings? This may take a long time.")) { - try { - await server.post('llm/embeddings/reprocess'); - toastService.showMessage(t("ai_llm.recreate_embeddings_started")); - - // Start progress polling - this.pollIndexRebuildProgress(); - } catch (e) { - console.error('Error starting embeddings regeneration:', e); - toastService.showError(t("ai_llm.recreate_embeddings_error")); - } - } - }); - - // Rebuild index button - const $rebuildIndex = this.$widget.find('.rebuild-embeddings-index'); - $rebuildIndex.on('click', async () => { - try { - await server.post('llm/embeddings/rebuild-index'); - toastService.showMessage(t("ai_llm.rebuild_index_started")); - - // Start progress polling - this.pollIndexRebuildProgress(); - } catch (e) { - console.error('Error starting index rebuild:', e); - toastService.showError(t("ai_llm.rebuild_index_error")); - } - }); } /** @@ -360,30 +222,9 @@ export default class AiSettingsWidget extends OptionsWidget { } } - // Similar checks for embeddings - const embeddingWarnings: string[] = []; - const embeddingsEnabled = this.$widget.find('.enable-automatic-indexing').prop('checked'); - - if (embeddingsEnabled) { - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - - if (selectedEmbeddingProvider === 'openai' && !this.$widget.find('.openai-api-key').val()) { - embeddingWarnings.push(t("ai_llm.empty_key_warning.openai")); - } - - if (selectedEmbeddingProvider === 'voyage' && !this.$widget.find('.voyage-api-key').val()) { - embeddingWarnings.push(t("ai_llm.empty_key_warning.voyage")); - } - - if (selectedEmbeddingProvider === 'ollama' && !this.$widget.find('.ollama-embedding-base-url').val()) { - embeddingWarnings.push(t("ai_llm.empty_key_warning.ollama")); - } - } - // Combine all warnings const allWarnings = [ - ...providerWarnings, - ...embeddingWarnings + ...providerWarnings ]; // Show or hide warnings @@ -396,168 +237,6 @@ export default class AiSettingsWidget extends OptionsWidget { } } - /** - * Poll for index rebuild progress - */ - pollIndexRebuildProgress() { - if (this.indexRebuildRefreshInterval) { - clearInterval(this.indexRebuildRefreshInterval); - } - - // Set up polling interval for index rebuild progress - this.indexRebuildRefreshInterval = setInterval(async () => { - await this.refreshEmbeddingStats(); - }, this.STATS_REFRESH_INTERVAL); - - // Stop polling after 5 minutes to avoid indefinite polling - setTimeout(() => { - if (this.indexRebuildRefreshInterval) { - clearInterval(this.indexRebuildRefreshInterval); - this.indexRebuildRefreshInterval = null; - } - }, 5 * 60 * 1000); - } - - /** - * Refresh embedding statistics - */ - async refreshEmbeddingStats() { - if (!this.$widget) return; - - try { - const response = await server.get('llm/embeddings/stats'); - - if (response && response.success) { - const stats = response.stats; - - // Update stats display - this.$widget.find('.embedding-processed-notes').text(stats.embeddedNotesCount); - this.$widget.find('.embedding-total-notes').text(stats.totalNotesCount); - this.$widget.find('.embedding-queued-notes').text(stats.queuedNotesCount); - this.$widget.find('.embedding-failed-notes').text(stats.failedNotesCount); - - if (stats.lastProcessedDate) { - const date = new Date(stats.lastProcessedDate); - this.$widget.find('.embedding-last-processed').text(date.toLocaleString()); - } else { - this.$widget.find('.embedding-last-processed').text('-'); - } - - // Update progress bar - const $progressBar = this.$widget.find('.embedding-progress'); - const progressPercent = stats.percentComplete; - $progressBar.css('width', `${progressPercent}%`); - $progressBar.attr('aria-valuenow', progressPercent.toString()); - $progressBar.text(`${progressPercent}%`); - - // Update status text - let statusText; - if (stats.queuedNotesCount > 0) { - statusText = t("ai_llm.agent.processing", { percentage: progressPercent }); - } else if (stats.embeddedNotesCount === 0) { - statusText = t("ai_llm.not_started"); - } else if (stats.embeddedNotesCount === stats.totalNotesCount) { - statusText = t("ai_llm.complete"); - - // Clear polling interval if processing is complete - if (this.indexRebuildRefreshInterval) { - clearInterval(this.indexRebuildRefreshInterval); - this.indexRebuildRefreshInterval = null; - } - } else { - statusText = t("ai_llm.partial", { percentage: progressPercent }); - } - - this.$widget.find('.embedding-status-text').text(statusText); - } - } catch (e) { - console.error('Error fetching embedding stats:', e); - } - } - - /** - * Fetch failed embedding notes - */ - async fetchFailedEmbeddingNotes() { - if (!this.$widget) return; - - try { - const response = await server.get('llm/embeddings/failed'); - - if (response && response.success) { - const failedNotes = response.failedNotes || []; - const $failedNotesList = this.$widget.find('.embedding-failed-notes-list'); - - if (failedNotes.length === 0) { - $failedNotesList.html(`
${t("ai_llm.no_failed_embeddings")}
`); - return; - } - - // Create a table with failed notes - let html = ` - - - - - - - - - - - `; - - for (const note of failedNotes) { - const date = new Date(note.lastAttempt); - const isPermanent = note.isPermanent; - const noteTitle = note.title || note.noteId; - - html += ` - - - - - - - `; - } - - html += ` - -
${t("ai_llm.note_title")}${t("ai_llm.error")}${t("ai_llm.last_attempt")}${t("ai_llm.actions")}
${noteTitle}${note.error}${date.toLocaleString()} - -
- `; - - $failedNotesList.html(html); - - // Add event handlers for retry buttons - $failedNotesList.find('.retry-embedding').on('click', async function() { - const noteId = $(this).closest('tr').data('note-id'); - try { - await server.post('llm/embeddings/retry', { noteId }); - toastService.showMessage(t("ai_llm.retry_queued")); - // Remove this row or update status - $(this).closest('tr').remove(); - } catch (e) { - console.error('Error retrying embedding:', e); - toastService.showError(t("ai_llm.retry_failed")); - } - }); - - // Add event handlers for open note links - $failedNotesList.find('.open-note').on('click', function(e) { - e.preventDefault(); - const noteId = $(this).closest('tr').data('note-id'); - window.open(`#${noteId}`, '_blank'); - }); - } - } catch (e) { - console.error('Error fetching failed embedding notes:', e); - } - } /** * Helper to get display name for providers @@ -594,7 +273,7 @@ export default class AiSettingsWidget extends OptionsWidget { /** * Fetch models for a specific provider and model type */ - async fetchModelsForProvider(provider: string, modelType: 'chat' | 'embedding') { + async fetchModelsForProvider(provider: string, modelType: 'chat') { if (!this.providerService) return; try { @@ -629,12 +308,6 @@ export default class AiSettingsWidget extends OptionsWidget { this.$widget.find(`.${selectedAiProvider}-provider-settings`).show(); } - // Update embedding provider settings visibility - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - this.$widget.find('.embedding-provider-settings').hide(); - if (selectedEmbeddingProvider) { - this.$widget.find(`.${selectedEmbeddingProvider}-embedding-provider-settings`).show(); - } } /** @@ -653,7 +326,6 @@ export default class AiSettingsWidget extends OptionsWidget { this.$widget.find('.openai-api-key').val(options.openaiApiKey || ''); this.$widget.find('.openai-base-url').val(options.openaiBaseUrl || 'https://api.openai.com/v1'); this.setModelDropdownValue('.openai-default-model', options.openaiDefaultModel); - this.setModelDropdownValue('.openai-embedding-model', options.openaiEmbeddingModel); // Anthropic Section this.$widget.find('.anthropic-api-key').val(options.anthropicApiKey || ''); @@ -662,58 +334,26 @@ export default class AiSettingsWidget extends OptionsWidget { // Voyage Section this.$widget.find('.voyage-api-key').val(options.voyageApiKey || ''); - this.$widget.find('.voyage-embedding-base-url').val(options.voyageEmbeddingBaseUrl || 'https://api.voyageai.com/v1'); - this.setModelDropdownValue('.voyage-embedding-model', options.voyageEmbeddingModel); // Ollama Section this.$widget.find('.ollama-base-url').val(options.ollamaBaseUrl || 'http://localhost:11434'); - this.$widget.find('.ollama-embedding-base-url').val(options.ollamaEmbeddingBaseUrl || 'http://localhost:11434'); this.setModelDropdownValue('.ollama-default-model', options.ollamaDefaultModel); - this.setModelDropdownValue('.ollama-embedding-model', options.ollamaEmbeddingModel); - - // Embedding-specific provider options - this.$widget.find('.openai-embedding-api-key').val(options.openaiEmbeddingApiKey || ''); - this.$widget.find('.openai-embedding-base-url').val(options.openaiEmbeddingBaseUrl || 'https://api.openai.com/v1'); - - // Embedding Options - this.$widget.find('.embedding-selected-provider').val(options.embeddingSelectedProvider || 'openai'); - this.$widget.find('.embedding-auto-update-enabled').prop('checked', options.embeddingAutoUpdateEnabled !== 'false'); - this.$widget.find('.enable-automatic-indexing').prop('checked', options.enableAutomaticIndexing !== 'false'); - this.$widget.find('.embedding-similarity-threshold').val(options.embeddingSimilarityThreshold || '0.75'); - this.$widget.find('.max-notes-per-llm-query').val(options.maxNotesPerLlmQuery || '3'); - this.$widget.find('.embedding-dimension-strategy').val(options.embeddingDimensionStrategy || 'auto'); - this.$widget.find('.embedding-batch-size').val(options.embeddingBatchSize || '10'); - this.$widget.find('.embedding-update-interval').val(options.embeddingUpdateInterval || '5000'); // Show/hide provider settings based on selected providers this.updateProviderSettingsVisibility(); // Automatically fetch models for currently selected providers const selectedAiProvider = this.$widget.find('.ai-selected-provider').val() as string; - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; if (selectedAiProvider) { await this.fetchModelsForProvider(selectedAiProvider, 'chat'); } - if (selectedEmbeddingProvider) { - await this.fetchModelsForProvider(selectedEmbeddingProvider, 'embedding'); - } - // Display validation warnings this.displayValidationWarnings(); } cleanup() { - // Clear intervals - if (this.statsRefreshInterval) { - clearInterval(this.statsRefreshInterval); - this.statsRefreshInterval = null; - } - - if (this.indexRebuildRefreshInterval) { - clearInterval(this.indexRebuildRefreshInterval); - this.indexRebuildRefreshInterval = null; - } + // Cleanup method for widget } } diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/interfaces.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/interfaces.ts index 2a3326ced6..6af466cac0 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/interfaces.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/interfaces.ts @@ -11,34 +11,6 @@ export interface OllamaModelResponse { }>; } -// Interface for embedding statistics -export interface EmbeddingStats { - success: boolean; - stats: { - totalNotesCount: number; - embeddedNotesCount: number; - queuedNotesCount: number; - failedNotesCount: number; - lastProcessedDate: string | null; - percentComplete: number; - } -} - -// Interface for failed embedding notes -export interface FailedEmbeddingNotes { - success: boolean; - failedNotes: Array<{ - noteId: string; - title?: string; - operation: string; - attempts: number; - lastAttempt: string; - error: string; - failureType: string; - chunks: number; - isPermanent: boolean; - }>; -} export interface OpenAIModelResponse { success: boolean; @@ -47,11 +19,6 @@ export interface OpenAIModelResponse { name: string; type: string; }>; - embeddingModels: Array<{ - id: string; - name: string; - type: string; - }>; } export interface AnthropicModelResponse { @@ -61,9 +28,4 @@ export interface AnthropicModelResponse { name: string; type: string; }>; - embeddingModels: Array<{ - id: string; - name: string; - type: string; - }>; } \ No newline at end of file diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/providers.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/providers.ts index 281569bd6e..a88e53ecf7 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/providers.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/providers.ts @@ -6,21 +6,7 @@ import type { OpenAIModelResponse, AnthropicModelResponse, OllamaModelResponse } export class ProviderService { constructor(private $widget: JQuery) { - // Initialize Voyage models (since they don't have a dynamic refresh yet) - this.initializeVoyageModels(); - } - - /** - * Initialize Voyage models with default values and ensure proper selection - */ - private initializeVoyageModels() { - setTimeout(() => { - const $voyageModelSelect = this.$widget.find('.voyage-embedding-model'); - if ($voyageModelSelect.length > 0) { - const currentValue = $voyageModelSelect.val(); - this.ensureSelectedValue($voyageModelSelect, currentValue, 'voyageEmbeddingModel'); - } - }, 100); // Small delay to ensure the widget is fully initialized + // AI provider settings } /** @@ -95,29 +81,10 @@ export class ProviderService { this.ensureSelectedValue($chatModelSelect, currentChatValue, 'openaiDefaultModel'); } - // Update the embedding models dropdown - if (response.embeddingModels?.length > 0) { - const $embedModelSelect = this.$widget.find('.openai-embedding-model'); - const currentEmbedValue = $embedModelSelect.val(); - - // Clear existing options - $embedModelSelect.empty(); - - // Sort models by name - const sortedEmbedModels = [...response.embeddingModels].sort((a, b) => a.name.localeCompare(b.name)); - - // Add models to the dropdown - sortedEmbedModels.forEach(model => { - $embedModelSelect.append(``); - }); - - // Try to restore the previously selected value - this.ensureSelectedValue($embedModelSelect, currentEmbedValue, 'openaiEmbeddingModel'); - } if (showLoading) { // Show success message - const totalModels = (response.chatModels?.length || 0) + (response.embeddingModels?.length || 0); + const totalModels = (response.chatModels?.length || 0); toastService.showMessage(`${totalModels} OpenAI models found.`); } @@ -187,14 +154,9 @@ export class ProviderService { this.ensureSelectedValue($chatModelSelect, currentChatValue, 'anthropicDefaultModel'); } - // Handle embedding models if they exist - if (response.embeddingModels?.length > 0 && showLoading) { - toastService.showMessage(`Found ${response.embeddingModels.length} Anthropic embedding models.`); - } - if (showLoading) { // Show success message - const totalModels = (response.chatModels?.length || 0) + (response.embeddingModels?.length || 0); + const totalModels = (response.chatModels?.length || 0); toastService.showMessage(`${totalModels} Anthropic models found.`); } @@ -240,66 +202,13 @@ export class ProviderService { } try { - // Determine which URL to use based on the current context - // If we're in the embedding provider context, use the embedding base URL - // Otherwise, use the general base URL - const selectedAiProvider = this.$widget.find('.ai-selected-provider').val() as string; - const selectedEmbeddingProvider = this.$widget.find('.embedding-selected-provider').val() as string; - - let ollamaBaseUrl: string; - - // If embedding provider is Ollama and it's visible, use embedding URL - const $embeddingOllamaSettings = this.$widget.find('.ollama-embedding-provider-settings'); - if (selectedEmbeddingProvider === 'ollama' && $embeddingOllamaSettings.is(':visible')) { - ollamaBaseUrl = this.$widget.find('.ollama-embedding-base-url').val() as string; - } else { - ollamaBaseUrl = this.$widget.find('.ollama-base-url').val() as string; - } - + // Use the general Ollama base URL + const ollamaBaseUrl = this.$widget.find('.ollama-base-url').val() as string; + const response = await server.get(`llm/providers/ollama/models?baseUrl=${encodeURIComponent(ollamaBaseUrl)}`); if (response && response.success && response.models && response.models.length > 0) { - // Update both embedding model dropdowns - const $embedModelSelect = this.$widget.find('.ollama-embedding-model'); - const $chatEmbedModelSelect = this.$widget.find('.ollama-chat-embedding-model'); - - const currentValue = $embedModelSelect.val(); - const currentChatEmbedValue = $chatEmbedModelSelect.val(); - - // Prepare embedding models - const embeddingModels = response.models.filter(model => - model.name.includes('embed') || model.name.includes('bert')); - - const generalModels = response.models.filter(model => - !model.name.includes('embed') && !model.name.includes('bert')); - - // Update .ollama-embedding-model dropdown (embedding provider settings) - $embedModelSelect.empty(); - embeddingModels.forEach(model => { - $embedModelSelect.append(``); - }); - if (embeddingModels.length > 0) { - $embedModelSelect.append(``); - } - generalModels.forEach(model => { - $embedModelSelect.append(``); - }); - this.ensureSelectedValue($embedModelSelect, currentValue, 'ollamaEmbeddingModel'); - - // Update .ollama-chat-embedding-model dropdown (general Ollama provider settings) - $chatEmbedModelSelect.empty(); - embeddingModels.forEach(model => { - $chatEmbedModelSelect.append(``); - }); - if (embeddingModels.length > 0) { - $chatEmbedModelSelect.append(``); - } - generalModels.forEach(model => { - $chatEmbedModelSelect.append(``); - }); - this.ensureSelectedValue($chatEmbedModelSelect, currentChatEmbedValue, 'ollamaEmbeddingModel'); - - // Also update the LLM model dropdown + // Update the LLM model dropdown const $modelSelect = this.$widget.find('.ollama-default-model'); const currentModelValue = $modelSelect.val(); diff --git a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts index 25b014dc75..5dffb52c0a 100644 --- a/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts +++ b/apps/client/src/widgets/type_widgets/options/ai_settings/template.ts @@ -16,46 +16,7 @@ export const TPL = ` -
-

${t("ai_llm.embedding_statistics")}

-
-
-
-
-
${t("ai_llm.processed_notes")}: -
-
${t("ai_llm.total_notes")}: -
-
${t("ai_llm.progress")}: -
-
- -
-
${t("ai_llm.queued_notes")}: -
-
${t("ai_llm.failed_notes")}: -
-
${t("ai_llm.last_processed")}: -
-
-
-
-
-
0%
-
-
- -
-
- -
- -
${t("ai_llm.failed_notes")}
-
-
-
-
${t("ai_llm.no_failed_embeddings")}
-
-
-
-
+

${t("ai_llm.provider_configuration")}

@@ -171,188 +132,4 @@ export const TPL = `
- -
-

${t("ai_llm.embeddings_configuration")}

- -
- - -
${t("ai_llm.selected_embedding_provider_description")}
-
- - - - - - - - - - - - - -
- - -
${t("ai_llm.embedding_dimension_strategy_description")}
-
- -
- - -
${t("ai_llm.embedding_similarity_threshold_description")}
-
- -
- - -
${t("ai_llm.embedding_batch_size_description")}
-
- -
- - -
${t("ai_llm.embedding_update_interval_description")}
-
- -
- - -
${t("ai_llm.max_notes_per_llm_query_description")}
-
- -
- -
${t("ai_llm.enable_automatic_indexing_description")}
-
- -
- -
${t("ai_llm.embedding_auto_update_enabled_description")}
-
- - -
- -
${t("ai_llm.recreate_embeddings_description")}
-
- - -
- -
${t("ai_llm.rebuild_index_description")}
-
- - -
-
${t("ai_llm.embedding_providers_order")}
-
${t("ai_llm.embedding_providers_order_description")}
-
-
`; +`; diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 3ac3ee5c4d..a7bc00b42c 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -30,17 +30,8 @@ export default async function buildApp() { // Listen for database initialization event eventService.subscribe(eventService.DB_INITIALIZED, async () => { try { - log.info("Database initialized, setting up LLM features"); - - // Initialize embedding providers - const { initializeEmbeddings } = await import("./services/llm/embeddings/init.js"); - await initializeEmbeddings(); - - // Initialize the index service for LLM functionality - const { default: indexService } = await import("./services/llm/index_service.js"); - await indexService.initialize().catch(e => console.error("Failed to initialize index service:", e)); - - log.info("LLM features initialized successfully"); + log.info("Database initialized, LLM features available"); + log.info("LLM features ready"); } catch (error) { console.error("Error initializing LLM features:", error); } @@ -49,13 +40,7 @@ export default async function buildApp() { // Initialize LLM features only if database is already initialized if (sql_init.isDbInitialized()) { try { - // Initialize embedding providers - const { initializeEmbeddings } = await import("./services/llm/embeddings/init.js"); - await initializeEmbeddings(); - - // Initialize the index service for LLM functionality - const { default: indexService } = await import("./services/llm/index_service.js"); - await indexService.initialize().catch(e => console.error("Failed to initialize index service:", e)); + log.info("LLM features ready"); } catch (error) { console.error("Error initializing LLM features:", error); } diff --git a/apps/server/src/assets/db/schema.sql b/apps/server/src/assets/db/schema.sql index be21bcc3d4..07d924a915 100644 --- a/apps/server/src/assets/db/schema.sql +++ b/apps/server/src/assets/db/schema.sql @@ -146,47 +146,6 @@ CREATE INDEX IDX_notes_blobId on notes (blobId); CREATE INDEX IDX_revisions_blobId on revisions (blobId); CREATE INDEX IDX_attachments_blobId on attachments (blobId); -CREATE TABLE IF NOT EXISTS "note_embeddings" ( - "embedId" TEXT NOT NULL PRIMARY KEY, - "noteId" TEXT NOT NULL, - "providerId" TEXT NOT NULL, - "modelId" TEXT NOT NULL, - "dimension" INTEGER NOT NULL, - "embedding" BLOB NOT NULL, - "version" INTEGER NOT NULL DEFAULT 1, - "dateCreated" TEXT NOT NULL, - "utcDateCreated" TEXT NOT NULL, - "dateModified" TEXT NOT NULL, - "utcDateModified" TEXT NOT NULL -); - -CREATE INDEX "IDX_note_embeddings_noteId" ON "note_embeddings" ("noteId"); -CREATE INDEX "IDX_note_embeddings_providerId_modelId" ON "note_embeddings" ("providerId", "modelId"); - -CREATE TABLE IF NOT EXISTS "embedding_queue" ( - "noteId" TEXT NOT NULL PRIMARY KEY, - "operation" TEXT NOT NULL, - "dateQueued" TEXT NOT NULL, - "utcDateQueued" TEXT NOT NULL, - "priority" INTEGER NOT NULL DEFAULT 0, - "attempts" INTEGER NOT NULL DEFAULT 0, - "lastAttempt" TEXT NULL, - "error" TEXT NULL, - "failed" INTEGER NOT NULL DEFAULT 0, - "isProcessing" INTEGER NOT NULL DEFAULT 0 -); - -CREATE TABLE IF NOT EXISTS "embedding_providers" ( - "providerId" TEXT NOT NULL PRIMARY KEY, - "name" TEXT NOT NULL, - "isEnabled" INTEGER NOT NULL DEFAULT 0, - "priority" INTEGER NOT NULL DEFAULT 0, - "config" TEXT NOT NULL, - "dateCreated" TEXT NOT NULL, - "utcDateCreated" TEXT NOT NULL, - "dateModified" TEXT NOT NULL, - "utcDateModified" TEXT NOT NULL -); CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, diff --git a/apps/server/src/becca/becca-interface.ts b/apps/server/src/becca/becca-interface.ts index 4301b2b5e4..005a5cc520 100644 --- a/apps/server/src/becca/becca-interface.ts +++ b/apps/server/src/becca/becca-interface.ts @@ -12,7 +12,6 @@ import type { AttachmentRow, BlobRow, RevisionRow } from "@triliumnext/commons"; import BBlob from "./entities/bblob.js"; import BRecentNote from "./entities/brecent_note.js"; import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js"; -import type BNoteEmbedding from "./entities/bnote_embedding.js"; interface AttachmentOpts { includeContentLength?: boolean; @@ -33,7 +32,6 @@ export default class Becca { attributeIndex!: Record; options!: Record; etapiTokens!: Record; - noteEmbeddings!: Record; allNoteSetCache: NoteSet | null; @@ -50,7 +48,6 @@ export default class Becca { this.attributeIndex = {}; this.options = {}; this.etapiTokens = {}; - this.noteEmbeddings = {}; this.dirtyNoteSetCache(); diff --git a/apps/server/src/becca/becca_loader.ts b/apps/server/src/becca/becca_loader.ts index a968b14303..cef89b43f9 100644 --- a/apps/server/src/becca/becca_loader.ts +++ b/apps/server/src/becca/becca_loader.ts @@ -9,10 +9,9 @@ import BBranch from "./entities/bbranch.js"; import BAttribute from "./entities/battribute.js"; import BOption from "./entities/boption.js"; import BEtapiToken from "./entities/betapi_token.js"; -import BNoteEmbedding from "./entities/bnote_embedding.js"; import cls from "../services/cls.js"; import entityConstructor from "../becca/entity_constructor.js"; -import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow, NoteEmbeddingRow } from "@triliumnext/commons"; +import type { AttributeRow, BranchRow, EtapiTokenRow, NoteRow, OptionRow } from "@triliumnext/commons"; import type AbstractBeccaEntity from "./entities/abstract_becca_entity.js"; import ws from "../services/ws.js"; @@ -65,17 +64,6 @@ function load() { new BEtapiToken(row); } - try { - for (const row of sql.getRows(/*sql*/`SELECT embedId, noteId, providerId, modelId, dimension, embedding, version, dateCreated, dateModified, utcDateCreated, utcDateModified FROM note_embeddings`)) { - new BNoteEmbedding(row).init(); - } - } catch (e: unknown) { - if (e && typeof e === "object" && "message" in e && typeof e.message === "string" && e.message.includes("no such table")) { - // Can be ignored. - } else { - throw e; - } - } }); for (const noteId in becca.notes) { @@ -98,7 +86,7 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_CHANGE_SYNCED], ({ entity return; } - if (["notes", "branches", "attributes", "etapi_tokens", "options", "note_embeddings"].includes(entityName)) { + if (["notes", "branches", "attributes", "etapi_tokens", "options"].includes(entityName)) { const EntityClass = entityConstructor.getEntityFromEntityName(entityName); const primaryKeyName = EntityClass.primaryKeyName; @@ -156,8 +144,6 @@ eventService.subscribeBeccaLoader([eventService.ENTITY_DELETED, eventService.ENT attributeDeleted(entityId); } else if (entityName === "etapi_tokens") { etapiTokenDeleted(entityId); - } else if (entityName === "note_embeddings") { - noteEmbeddingDeleted(entityId); } }); @@ -293,9 +279,6 @@ function etapiTokenDeleted(etapiTokenId: string) { delete becca.etapiTokens[etapiTokenId]; } -function noteEmbeddingDeleted(embedId: string) { - delete becca.noteEmbeddings[embedId]; -} eventService.subscribeBeccaLoader(eventService.ENTER_PROTECTED_SESSION, () => { try { diff --git a/apps/server/src/becca/entities/bnote_embedding.ts b/apps/server/src/becca/entities/bnote_embedding.ts deleted file mode 100644 index c59a06e5e4..0000000000 --- a/apps/server/src/becca/entities/bnote_embedding.ts +++ /dev/null @@ -1,83 +0,0 @@ -import AbstractBeccaEntity from "./abstract_becca_entity.js"; -import dateUtils from "../../services/date_utils.js"; -import type { NoteEmbeddingRow } from "@triliumnext/commons"; - -/** - * Entity representing a note's vector embedding for semantic search and AI features - */ -class BNoteEmbedding extends AbstractBeccaEntity { - static get entityName() { - return "note_embeddings"; - } - static get primaryKeyName() { - return "embedId"; - } - static get hashedProperties() { - return ["embedId", "noteId", "providerId", "modelId", "dimension", "version"]; - } - - embedId!: string; - noteId!: string; - providerId!: string; - modelId!: string; - dimension!: number; - embedding!: Buffer; - version!: number; - - constructor(row?: NoteEmbeddingRow) { - super(); - - if (row) { - this.updateFromRow(row); - } - } - - init() { - if (this.embedId) { - this.becca.noteEmbeddings[this.embedId] = this; - } - } - - updateFromRow(row: NoteEmbeddingRow): void { - this.embedId = row.embedId; - this.noteId = row.noteId; - this.providerId = row.providerId; - this.modelId = row.modelId; - this.dimension = row.dimension; - this.embedding = row.embedding; - this.version = row.version; - this.dateCreated = row.dateCreated; - this.dateModified = row.dateModified; - this.utcDateCreated = row.utcDateCreated; - this.utcDateModified = row.utcDateModified; - - if (this.embedId) { - this.becca.noteEmbeddings[this.embedId] = this; - } - } - - override beforeSaving() { - super.beforeSaving(); - - this.dateModified = dateUtils.localNowDateTime(); - this.utcDateModified = dateUtils.utcNowDateTime(); - } - - getPojo(): NoteEmbeddingRow { - return { - embedId: this.embedId, - noteId: this.noteId, - providerId: this.providerId, - modelId: this.modelId, - dimension: this.dimension, - embedding: this.embedding, - version: this.version, - dateCreated: this.dateCreated!, - dateModified: this.dateModified!, - utcDateCreated: this.utcDateCreated, - utcDateModified: this.utcDateModified! - }; - } -} - -export default BNoteEmbedding; diff --git a/apps/server/src/becca/entity_constructor.ts b/apps/server/src/becca/entity_constructor.ts index 882f62492f..18f7a14c7e 100644 --- a/apps/server/src/becca/entity_constructor.ts +++ b/apps/server/src/becca/entity_constructor.ts @@ -6,7 +6,6 @@ import BBlob from "./entities/bblob.js"; import BBranch from "./entities/bbranch.js"; import BEtapiToken from "./entities/betapi_token.js"; import BNote from "./entities/bnote.js"; -import BNoteEmbedding from "./entities/bnote_embedding.js"; import BOption from "./entities/boption.js"; import BRecentNote from "./entities/brecent_note.js"; import BRevision from "./entities/brevision.js"; @@ -20,7 +19,6 @@ const ENTITY_NAME_TO_ENTITY: Record & EntityClass> branches: BBranch, etapi_tokens: BEtapiToken, notes: BNote, - note_embeddings: BNoteEmbedding, options: BOption, recent_notes: BRecentNote, revisions: BRevision diff --git a/apps/server/src/etapi/metrics.ts b/apps/server/src/etapi/metrics.ts index 2fc9e6e5f6..972a6e4e7a 100644 --- a/apps/server/src/etapi/metrics.ts +++ b/apps/server/src/etapi/metrics.ts @@ -26,8 +26,6 @@ interface MetricsData { totalBlobs: number; totalEtapiTokens: number; totalRecentNotes: number; - totalEmbeddings: number; - totalEmbeddingProviders: number; }; noteTypes: Record; attachmentTypes: Record; @@ -88,8 +86,6 @@ function formatPrometheusMetrics(data: MetricsData): string { addMetric('trilium_blobs_total', data.database.totalBlobs, 'Total number of blob records'); addMetric('trilium_etapi_tokens_total', data.database.totalEtapiTokens, 'Number of active ETAPI tokens'); addMetric('trilium_recent_notes_total', data.database.totalRecentNotes, 'Number of recent notes tracked'); - addMetric('trilium_embeddings_total', data.database.totalEmbeddings, 'Number of note embeddings'); - addMetric('trilium_embedding_providers_total', data.database.totalEmbeddingProviders, 'Number of embedding providers'); // Note types for (const [type, count] of Object.entries(data.noteTypes)) { @@ -155,15 +151,6 @@ function collectMetrics(): MetricsData { const totalEtapiTokens = sql.getValue("SELECT COUNT(*) FROM etapi_tokens WHERE isDeleted = 0"); const totalRecentNotes = sql.getValue("SELECT COUNT(*) FROM recent_notes"); - // Embedding-related metrics (these tables might not exist in older versions) - let totalEmbeddings = 0; - let totalEmbeddingProviders = 0; - try { - totalEmbeddings = sql.getValue("SELECT COUNT(*) FROM note_embeddings"); - totalEmbeddingProviders = sql.getValue("SELECT COUNT(*) FROM embedding_providers"); - } catch (e) { - // Tables don't exist, keep defaults - } const database = { totalNotes, @@ -179,8 +166,6 @@ function collectMetrics(): MetricsData { totalBlobs, totalEtapiTokens, totalRecentNotes, - totalEmbeddings, - totalEmbeddingProviders }; // Note types breakdown diff --git a/apps/server/src/migrations/migrations.ts b/apps/server/src/migrations/migrations.ts index 1ac414afee..4743d92867 100644 --- a/apps/server/src/migrations/migrations.ts +++ b/apps/server/src/migrations/migrations.ts @@ -6,6 +6,19 @@ // Migrations should be kept in descending order, so the latest migration is first. const MIGRATIONS: (SqlMigration | JsMigration)[] = [ + // Remove embedding tables since LLM embedding functionality has been removed + { + version: 232, + sql: /*sql*/` + -- Remove LLM embedding tables and data + DROP TABLE IF EXISTS "note_embeddings"; + DROP TABLE IF EXISTS "embedding_queue"; + DROP TABLE IF EXISTS "embedding_providers"; + + -- Remove embedding-related entity changes + DELETE FROM entity_changes WHERE entityName IN ('note_embeddings', 'embedding_queue', 'embedding_providers'); + ` + }, // Session store { version: 231, diff --git a/apps/server/src/routes/api/anthropic.ts b/apps/server/src/routes/api/anthropic.ts index 900a0b0845..95f8af4eac 100644 --- a/apps/server/src/routes/api/anthropic.ts +++ b/apps/server/src/routes/api/anthropic.ts @@ -48,17 +48,6 @@ interface AnthropicModel { * type: string * type: * type: string - * embeddingModels: - * type: array - * items: - * type: object - * properties: - * id: - * type: string - * name: - * type: string - * type: - * type: string * '500': * description: Error listing models * security: @@ -90,14 +79,10 @@ async function listModels(req: Request, res: Response) { type: 'chat' })); - // Anthropic doesn't currently have embedding models - const embeddingModels: AnthropicModel[] = []; - // Return the models list return { success: true, - chatModels, - embeddingModels + chatModels }; } catch (error: any) { log.error(`Error listing Anthropic models: ${error.message || 'Unknown error'}`); diff --git a/apps/server/src/routes/api/embeddings.ts b/apps/server/src/routes/api/embeddings.ts deleted file mode 100644 index 226231ea2f..0000000000 --- a/apps/server/src/routes/api/embeddings.ts +++ /dev/null @@ -1,843 +0,0 @@ -import options from "../../services/options.js"; -import vectorStore from "../../services/llm/embeddings/index.js"; -import providerManager from "../../services/llm/providers/providers.js"; -import indexService from "../../services/llm/index_service.js"; -import becca from "../../becca/becca.js"; -import type { Request, Response } from "express"; -import log from "../../services/log.js"; -import sql from "../../services/sql.js"; - -/** - * @swagger - * /api/llm/embeddings/similar/{noteId}: - * get: - * summary: Find similar notes based on a given note ID - * operationId: embeddings-similar-by-note - * parameters: - * - name: noteId - * in: path - * required: true - * schema: - * type: string - * - name: providerId - * in: query - * required: false - * schema: - * type: string - * default: openai - * description: Embedding provider ID - * - name: modelId - * in: query - * required: false - * schema: - * type: string - * default: text-embedding-3-small - * description: Embedding model ID - * - name: limit - * in: query - * required: false - * schema: - * type: integer - * default: 10 - * description: Maximum number of similar notes to return - * - name: threshold - * in: query - * required: false - * schema: - * type: number - * format: float - * default: 0.7 - * description: Similarity threshold (0.0-1.0) - * responses: - * '200': - * description: List of similar notes - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * similarNotes: - * type: array - * items: - * type: object - * properties: - * noteId: - * type: string - * title: - * type: string - * similarity: - * type: number - * format: float - * '400': - * description: Invalid request parameters - * '404': - * description: Note not found - * security: - * - session: [] - * tags: ["llm"] - */ -async function findSimilarNotes(req: Request, res: Response) { - const noteId = req.params.noteId; - const providerId = req.query.providerId as string || 'openai'; - const modelId = req.query.modelId as string || 'text-embedding-3-small'; - const limit = parseInt(req.query.limit as string || '10', 10); - const threshold = parseFloat(req.query.threshold as string || '0.7'); - - if (!noteId) { - return [400, { - success: false, - message: "Note ID is required" - }]; - } - - const embedding = await vectorStore.getEmbeddingForNote(noteId, providerId, modelId); - - if (!embedding) { - // If no embedding exists for this note yet, generate one - const note = becca.getNote(noteId); - if (!note) { - return [404, { - success: false, - message: "Note not found" - }]; - } - - const context = await vectorStore.getNoteEmbeddingContext(noteId); - const provider = providerManager.getEmbeddingProvider(providerId); - - if (!provider) { - return [400, { - success: false, - message: `Embedding provider '${providerId}' not found` - }]; - } - - const newEmbedding = await provider.generateNoteEmbeddings(context); - await vectorStore.storeNoteEmbedding(noteId, providerId, modelId, newEmbedding); - - const similarNotes = await vectorStore.findSimilarNotes( - newEmbedding, providerId, modelId, limit, threshold - ); - - return { - success: true, - similarNotes - }; - } - - const similarNotes = await vectorStore.findSimilarNotes( - embedding.embedding, providerId, modelId, limit, threshold - ); - - return { - success: true, - similarNotes - }; -} - -/** - * @swagger - * /api/llm/embeddings/search: - * post: - * summary: Search for notes similar to provided text - * operationId: embeddings-search-by-text - * parameters: - * - name: providerId - * in: query - * required: false - * schema: - * type: string - * default: openai - * description: Embedding provider ID - * - name: modelId - * in: query - * required: false - * schema: - * type: string - * default: text-embedding-3-small - * description: Embedding model ID - * - name: limit - * in: query - * required: false - * schema: - * type: integer - * default: 10 - * description: Maximum number of similar notes to return - * - name: threshold - * in: query - * required: false - * schema: - * type: number - * format: float - * default: 0.7 - * description: Similarity threshold (0.0-1.0) - * requestBody: - * required: true - * content: - * application/json: - * schema: - * type: object - * properties: - * text: - * type: string - * description: Text to search with - * responses: - * '200': - * description: List of similar notes - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * similarNotes: - * type: array - * items: - * type: object - * properties: - * noteId: - * type: string - * title: - * type: string - * similarity: - * type: number - * format: float - * '400': - * description: Invalid request parameters - * security: - * - session: [] - * tags: ["llm"] - */ -async function searchByText(req: Request, res: Response) { - const { text } = req.body; - const providerId = req.query.providerId as string || 'openai'; - const modelId = req.query.modelId as string || 'text-embedding-3-small'; - const limit = parseInt(req.query.limit as string || '10', 10); - const threshold = parseFloat(req.query.threshold as string || '0.7'); - - if (!text) { - return [400, { - success: false, - message: "Search text is required" - }]; - } - - const provider = providerManager.getEmbeddingProvider(providerId); - - if (!provider) { - return [400, { - success: false, - message: `Embedding provider '${providerId}' not found` - }]; - } - - // Generate embedding for the search text - const embedding = await provider.generateEmbeddings(text); - - // Find similar notes - const similarNotes = await vectorStore.findSimilarNotes( - embedding, providerId, modelId, limit, threshold - ); - - return { - success: true, - similarNotes - }; -} - -/** - * @swagger - * /api/llm/embeddings/providers: - * get: - * summary: Get available embedding providers - * operationId: embeddings-get-providers - * responses: - * '200': - * description: List of available embedding providers - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * providers: - * type: array - * items: - * type: object - * properties: - * id: - * type: string - * name: - * type: string - * isEnabled: - * type: boolean - * priority: - * type: integer - * config: - * type: object - * security: - * - session: [] - * tags: ["llm"] - */ -async function getProviders(req: Request, res: Response) { - const providerConfigs = await providerManager.getEmbeddingProviderConfigs(); - - return { - success: true, - providers: providerConfigs - }; -} - -/** - * @swagger - * /api/llm/embeddings/providers/{providerId}: - * patch: - * summary: Update embedding provider configuration - * operationId: embeddings-update-provider - * parameters: - * - name: providerId - * in: path - * required: true - * schema: - * type: string - * description: Provider ID to update - * requestBody: - * required: true - * content: - * application/json: - * schema: - * type: object - * properties: - * enabled: - * type: boolean - * description: Whether provider is enabled - * priority: - * type: integer - * description: Priority order (lower is higher priority) - * config: - * type: object - * description: Provider-specific configuration - * responses: - * '200': - * description: Provider updated successfully - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * '400': - * description: Invalid provider ID or configuration - * security: - * - session: [] - * tags: ["llm"] - */ -async function updateProvider(req: Request, res: Response) { - const { providerId } = req.params; - const { isEnabled, priority, config } = req.body; - - const success = await providerManager.updateEmbeddingProviderConfig( - providerId, isEnabled, priority - ); - - if (!success) { - return [404, { - success: false, - message: "Provider not found" - }]; - } - - return { - success: true - }; -} - -/** - * @swagger - * /api/llm/embeddings/reprocess: - * post: - * summary: Reprocess embeddings for all notes - * operationId: embeddings-reprocess-all - * requestBody: - * required: true - * content: - * application/json: - * schema: - * type: object - * properties: - * providerId: - * type: string - * description: Provider ID to use for reprocessing - * modelId: - * type: string - * description: Model ID to use for reprocessing - * forceReprocess: - * type: boolean - * description: Whether to reprocess notes that already have embeddings - * responses: - * '200': - * description: Reprocessing started - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * jobId: - * type: string - * message: - * type: string - * '400': - * description: Invalid provider ID or configuration - * security: - * - session: [] - * tags: ["llm"] - */ -async function reprocessAllNotes(req: Request, res: Response) { - // Import cls - const cls = (await import("../../services/cls.js")).default; - - // Start the reprocessing operation in the background - setTimeout(async () => { - try { - // Wrap the operation in cls.init to ensure proper context - cls.init(async () => { - await indexService.reprocessAllNotes(); - log.info("Embedding reprocessing completed successfully"); - }); - } catch (error: any) { - log.error(`Error during background embedding reprocessing: ${error.message || "Unknown error"}`); - } - }, 0); - - // Return the response data - return { - success: true, - message: "Embedding reprocessing started in the background" - }; -} - -/** - * @swagger - * /api/llm/embeddings/queue-status: - * get: - * summary: Get status of the embedding processing queue - * operationId: embeddings-queue-status - * parameters: - * - name: jobId - * in: query - * required: false - * schema: - * type: string - * description: Optional job ID to get status for a specific processing job - * responses: - * '200': - * description: Queue status information - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * status: - * type: string - * enum: [idle, processing, paused] - * progress: - * type: number - * format: float - * description: Progress percentage (0-100) - * details: - * type: object - * security: - * - session: [] - * tags: ["llm"] - */ -async function getQueueStatus(req: Request, res: Response) { - // Use the imported sql instead of requiring it - const queueCount = await sql.getValue( - "SELECT COUNT(*) FROM embedding_queue" - ); - - const failedCount = await sql.getValue( - "SELECT COUNT(*) FROM embedding_queue WHERE attempts > 0" - ); - - const totalEmbeddingsCount = await sql.getValue( - "SELECT COUNT(*) FROM note_embeddings" - ); - - return { - success: true, - status: { - queueCount, - failedCount, - totalEmbeddingsCount - } - }; -} - -/** - * @swagger - * /api/llm/embeddings/stats: - * get: - * summary: Get embedding statistics - * operationId: embeddings-stats - * responses: - * '200': - * description: Embedding statistics - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * stats: - * type: object - * properties: - * totalEmbeddings: - * type: integer - * providers: - * type: object - * modelCounts: - * type: object - * lastUpdated: - * type: string - * format: date-time - * security: - * - session: [] - * tags: ["llm"] - */ -async function getEmbeddingStats(req: Request, res: Response) { - const stats = await vectorStore.getEmbeddingStats(); - - return { - success: true, - stats - }; -} - -/** - * @swagger - * /api/llm/embeddings/failed: - * get: - * summary: Get list of notes that failed embedding generation - * operationId: embeddings-failed-notes - * responses: - * '200': - * description: List of failed notes - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * failedNotes: - * type: array - * items: - * type: object - * properties: - * noteId: - * type: string - * title: - * type: string - * error: - * type: string - * failedAt: - * type: string - * format: date-time - * security: - * - session: [] - * tags: ["llm"] - */ -async function getFailedNotes(req: Request, res: Response) { - const limit = parseInt(req.query.limit as string || '100', 10); - const failedNotes = await vectorStore.getFailedEmbeddingNotes(limit); - - // No need to fetch note titles here anymore as they're already included in the response - return { - success: true, - failedNotes: failedNotes - }; -} - -/** - * @swagger - * /api/llm/embeddings/retry/{noteId}: - * post: - * summary: Retry generating embeddings for a failed note - * operationId: embeddings-retry-note - * parameters: - * - name: noteId - * in: path - * required: true - * schema: - * type: string - * description: Note ID to retry - * - name: providerId - * in: query - * required: false - * schema: - * type: string - * description: Provider ID to use (defaults to configured default) - * - name: modelId - * in: query - * required: false - * schema: - * type: string - * description: Model ID to use (defaults to provider default) - * responses: - * '200': - * description: Retry result - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * message: - * type: string - * '400': - * description: Invalid request - * '404': - * description: Note not found - * security: - * - session: [] - * tags: ["llm"] - */ -async function retryFailedNote(req: Request, res: Response) { - const { noteId } = req.params; - - if (!noteId) { - return [400, { - success: false, - message: "Note ID is required" - }]; - } - - const success = await vectorStore.retryFailedEmbedding(noteId); - - if (!success) { - return [404, { - success: false, - message: "Failed note not found or note is not marked as failed" - }]; - } - - return { - success: true, - message: "Note queued for retry" - }; -} - -/** - * @swagger - * /api/llm/embeddings/retry-all-failed: - * post: - * summary: Retry generating embeddings for all failed notes - * operationId: embeddings-retry-all-failed - * requestBody: - * required: false - * content: - * application/json: - * schema: - * type: object - * properties: - * providerId: - * type: string - * description: Provider ID to use (defaults to configured default) - * modelId: - * type: string - * description: Model ID to use (defaults to provider default) - * responses: - * '200': - * description: Retry started - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * message: - * type: string - * jobId: - * type: string - * security: - * - session: [] - * tags: ["llm"] - */ -async function retryAllFailedNotes(req: Request, res: Response) { - const count = await vectorStore.retryAllFailedEmbeddings(); - - return { - success: true, - message: `${count} failed notes queued for retry` - }; -} - -/** - * @swagger - * /api/llm/embeddings/rebuild-index: - * post: - * summary: Rebuild the vector store index - * operationId: embeddings-rebuild-index - * responses: - * '200': - * description: Rebuild started - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * message: - * type: string - * jobId: - * type: string - * security: - * - session: [] - * tags: ["llm"] - */ -async function rebuildIndex(req: Request, res: Response) { - // Start the index rebuilding operation in the background - setTimeout(async () => { - try { - await indexService.startFullIndexing(true); - log.info("Index rebuilding completed successfully"); - } catch (error: any) { - log.error(`Error during background index rebuilding: ${error.message || "Unknown error"}`); - } - }, 0); - - // Return the response data - return { - success: true, - message: "Index rebuilding started in the background" - }; -} - -/** - * @swagger - * /api/llm/embeddings/index-rebuild-status: - * get: - * summary: Get status of the vector index rebuild operation - * operationId: embeddings-rebuild-status - * parameters: - * - name: jobId - * in: query - * required: false - * schema: - * type: string - * description: Optional job ID to get status for a specific rebuild job - * responses: - * '200': - * description: Rebuild status information - * content: - * application/json: - * schema: - * type: object - * properties: - * success: - * type: boolean - * status: - * type: string - * enum: [idle, in_progress, completed, failed] - * progress: - * type: number - * format: float - * description: Progress percentage (0-100) - * message: - * type: string - * details: - * type: object - * properties: - * startTime: - * type: string - * format: date-time - * processed: - * type: integer - * total: - * type: integer - * security: - * - session: [] - * tags: ["llm"] - */ -async function getIndexRebuildStatus(req: Request, res: Response) { - const status = indexService.getIndexRebuildStatus(); - - return { - success: true, - status - }; -} - -/** - * Start embedding generation when AI is enabled - */ -async function startEmbeddings(req: Request, res: Response) { - try { - log.info("Starting embedding generation system"); - - // Initialize the index service if not already initialized - await indexService.initialize(); - - // Start automatic indexing - await indexService.startEmbeddingGeneration(); - - return { - success: true, - message: "Embedding generation started" - }; - } catch (error: any) { - log.error(`Error starting embeddings: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to start embeddings: ${error.message || 'Unknown error'}`); - } -} - -/** - * Stop embedding generation when AI is disabled - */ -async function stopEmbeddings(req: Request, res: Response) { - try { - log.info("Stopping embedding generation system"); - - // Stop automatic indexing - await indexService.stopEmbeddingGeneration(); - - return { - success: true, - message: "Embedding generation stopped" - }; - } catch (error: any) { - log.error(`Error stopping embeddings: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to stop embeddings: ${error.message || 'Unknown error'}`); - } -} - -export default { - findSimilarNotes, - searchByText, - getProviders, - updateProvider, - reprocessAllNotes, - getQueueStatus, - getEmbeddingStats, - getFailedNotes, - retryFailedNote, - retryAllFailedNotes, - rebuildIndex, - getIndexRebuildStatus, - startEmbeddings, - stopEmbeddings -}; diff --git a/apps/server/src/routes/api/llm.ts b/apps/server/src/routes/api/llm.ts index f586b85d6d..658b44c930 100644 --- a/apps/server/src/routes/api/llm.ts +++ b/apps/server/src/routes/api/llm.ts @@ -2,8 +2,6 @@ import type { Request, Response } from "express"; import log from "../../services/log.js"; import options from "../../services/options.js"; -// Import the index service for knowledge base management -import indexService from "../../services/llm/index_service.js"; import restChatService from "../../services/llm/rest_chat_service.js"; import chatStorageService from '../../services/llm/chat_storage_service.js'; @@ -371,400 +369,13 @@ async function sendMessage(req: Request, res: Response) { return restChatService.handleSendMessage(req, res); } -/** - * @swagger - * /api/llm/indexes/stats: - * get: - * summary: Get stats about the LLM knowledge base indexing status - * operationId: llm-index-stats - * responses: - * '200': - * description: Index stats successfully retrieved - * security: - * - session: [] - * tags: ["llm"] - */ -async function getIndexStats(req: Request, res: Response) { - try { - // Check if AI is enabled - const aiEnabled = await options.getOptionBool('aiEnabled'); - if (!aiEnabled) { - return { - success: false, - message: "AI features are disabled" - }; - } - // Return indexing stats - const stats = await indexService.getIndexingStats(); - return { - success: true, - ...stats - }; - } catch (error: any) { - log.error(`Error getting index stats: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to get index stats: ${error.message || 'Unknown error'}`); - } -} -/** - * @swagger - * /api/llm/indexes: - * post: - * summary: Start or continue indexing the knowledge base - * operationId: llm-start-indexing - * requestBody: - * required: false - * content: - * application/json: - * schema: - * type: object - * properties: - * force: - * type: boolean - * description: Whether to force reindexing of all notes - * responses: - * '200': - * description: Indexing started successfully - * security: - * - session: [] - * tags: ["llm"] - */ -async function startIndexing(req: Request, res: Response) { - try { - // Check if AI is enabled - const aiEnabled = await options.getOptionBool('aiEnabled'); - if (!aiEnabled) { - return { - success: false, - message: "AI features are disabled" - }; - } - const { force = false } = req.body; - // Start indexing - await indexService.startFullIndexing(force); - return { - success: true, - message: "Indexing started" - }; - } catch (error: any) { - log.error(`Error starting indexing: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to start indexing: ${error.message || 'Unknown error'}`); - } -} -/** - * @swagger - * /api/llm/indexes/failed: - * get: - * summary: Get list of notes that failed to index - * operationId: llm-failed-indexes - * parameters: - * - name: limit - * in: query - * required: false - * schema: - * type: integer - * default: 100 - * responses: - * '200': - * description: Failed indexes successfully retrieved - * security: - * - session: [] - * tags: ["llm"] - */ -async function getFailedIndexes(req: Request, res: Response) { - try { - // Check if AI is enabled - const aiEnabled = await options.getOptionBool('aiEnabled'); - if (!aiEnabled) { - return { - success: false, - message: "AI features are disabled" - }; - } - - const limit = parseInt(req.query.limit as string || "100", 10); - - // Get failed indexes - const failed = await indexService.getFailedIndexes(limit); - - return { - success: true, - failed - }; - } catch (error: any) { - log.error(`Error getting failed indexes: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to get failed indexes: ${error.message || 'Unknown error'}`); - } -} -/** - * @swagger - * /api/llm/indexes/notes/{noteId}: - * put: - * summary: Retry indexing a specific note that previously failed - * operationId: llm-retry-index - * parameters: - * - name: noteId - * in: path - * required: true - * schema: - * type: string - * responses: - * '200': - * description: Index retry successfully initiated - * security: - * - session: [] - * tags: ["llm"] - */ -async function retryFailedIndex(req: Request, res: Response) { - try { - // Check if AI is enabled - const aiEnabled = await options.getOptionBool('aiEnabled'); - if (!aiEnabled) { - return { - success: false, - message: "AI features are disabled" - }; - } - - const { noteId } = req.params; - - // Retry indexing the note - const result = await indexService.retryFailedNote(noteId); - - return { - success: true, - message: result ? "Note queued for indexing" : "Failed to queue note for indexing" - }; - } catch (error: any) { - log.error(`Error retrying failed index: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to retry index: ${error.message || 'Unknown error'}`); - } -} - -/** - * @swagger - * /api/llm/indexes/failed: - * put: - * summary: Retry indexing all failed notes - * operationId: llm-retry-all-indexes - * responses: - * '200': - * description: Retry of all failed indexes successfully initiated - * security: - * - session: [] - * tags: ["llm"] - */ -async function retryAllFailedIndexes(req: Request, res: Response) { - try { - // Check if AI is enabled - const aiEnabled = await options.getOptionBool('aiEnabled'); - if (!aiEnabled) { - return { - success: false, - message: "AI features are disabled" - }; - } - - // Retry all failed notes - const count = await indexService.retryAllFailedNotes(); - - return { - success: true, - message: `${count} notes queued for reprocessing` - }; - } catch (error: any) { - log.error(`Error retrying all failed indexes: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to retry all indexes: ${error.message || 'Unknown error'}`); - } -} - -/** - * @swagger - * /api/llm/indexes/notes/similar: - * get: - * summary: Find notes similar to a query string - * operationId: llm-find-similar-notes - * parameters: - * - name: query - * in: query - * required: true - * schema: - * type: string - * - name: contextNoteId - * in: query - * required: false - * schema: - * type: string - * - name: limit - * in: query - * required: false - * schema: - * type: integer - * default: 5 - * responses: - * '200': - * description: Similar notes found successfully - * security: - * - session: [] - * tags: ["llm"] - */ -async function findSimilarNotes(req: Request, res: Response) { - try { - // Check if AI is enabled - const aiEnabled = await options.getOptionBool('aiEnabled'); - if (!aiEnabled) { - return { - success: false, - message: "AI features are disabled" - }; - } - - const query = req.query.query as string; - const contextNoteId = req.query.contextNoteId as string | undefined; - const limit = parseInt(req.query.limit as string || "5", 10); - - if (!query) { - return { - success: false, - message: "Query is required" - }; - } - - // Find similar notes - const similar = await indexService.findSimilarNotes(query, contextNoteId, limit); - - return { - success: true, - similar - }; - } catch (error: any) { - log.error(`Error finding similar notes: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to find similar notes: ${error.message || 'Unknown error'}`); - } -} - -/** - * @swagger - * /api/llm/indexes/context: - * get: - * summary: Generate context for an LLM query based on the knowledge base - * operationId: llm-generate-context - * parameters: - * - name: query - * in: query - * required: true - * schema: - * type: string - * - name: contextNoteId - * in: query - * required: false - * schema: - * type: string - * - name: depth - * in: query - * required: false - * schema: - * type: integer - * default: 2 - * responses: - * '200': - * description: Context generated successfully - * security: - * - session: [] - * tags: ["llm"] - */ -async function generateQueryContext(req: Request, res: Response) { - try { - // Check if AI is enabled - const aiEnabled = await options.getOptionBool('aiEnabled'); - if (!aiEnabled) { - return { - success: false, - message: "AI features are disabled" - }; - } - - const query = req.query.query as string; - const contextNoteId = req.query.contextNoteId as string | undefined; - const depth = parseInt(req.query.depth as string || "2", 10); - - if (!query) { - return { - success: false, - message: "Query is required" - }; - } - - // Generate context - const context = await indexService.generateQueryContext(query, contextNoteId, depth); - - return { - success: true, - context - }; - } catch (error: any) { - log.error(`Error generating query context: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to generate query context: ${error.message || 'Unknown error'}`); - } -} - -/** - * @swagger - * /api/llm/indexes/notes/{noteId}: - * post: - * summary: Index a specific note for LLM knowledge base - * operationId: llm-index-note - * parameters: - * - name: noteId - * in: path - * required: true - * schema: - * type: string - * responses: - * '200': - * description: Note indexed successfully - * security: - * - session: [] - * tags: ["llm"] - */ -async function indexNote(req: Request, res: Response) { - try { - // Check if AI is enabled - const aiEnabled = await options.getOptionBool('aiEnabled'); - if (!aiEnabled) { - return { - success: false, - message: "AI features are disabled" - }; - } - - const { noteId } = req.params; - - if (!noteId) { - return { - success: false, - message: "Note ID is required" - }; - } - - // Index the note - const result = await indexService.generateNoteIndex(noteId); - - return { - success: true, - message: result ? "Note indexed successfully" : "Failed to index note" - }; - } catch (error: any) { - log.error(`Error indexing note: ${error.message || 'Unknown error'}`); - throw new Error(`Failed to index note: ${error.message || 'Unknown error'}`); - } -} /** * @swagger @@ -936,15 +547,5 @@ export default { listSessions, deleteSession, sendMessage, - streamMessage, - - // Knowledge base index management - getIndexStats, - startIndexing, - getFailedIndexes, - retryFailedIndex, - retryAllFailedIndexes, - findSimilarNotes, - generateQueryContext, - indexNote + streamMessage }; diff --git a/apps/server/src/routes/api/metrics.ts b/apps/server/src/routes/api/metrics.ts index e2428374bc..54310174d0 100644 --- a/apps/server/src/routes/api/metrics.ts +++ b/apps/server/src/routes/api/metrics.ts @@ -99,12 +99,6 @@ type MetricsData = ReturnType; * totalRecentNotes: * type: integer * example: 50 - * totalEmbeddings: - * type: integer - * example: 123 - * totalEmbeddingProviders: - * type: integer - * example: 2 * noteTypes: * type: object * additionalProperties: diff --git a/apps/server/src/routes/api/openai.ts b/apps/server/src/routes/api/openai.ts index 84efad2ca1..9c3d2382bd 100644 --- a/apps/server/src/routes/api/openai.ts +++ b/apps/server/src/routes/api/openai.ts @@ -40,17 +40,6 @@ import OpenAI from "openai"; * type: string * type: * type: string - * embeddingModels: - * type: array - * items: - * type: object - * properties: - * id: - * type: string - * name: - * type: string - * type: - * type: string * '500': * description: Error listing models * security: @@ -82,8 +71,7 @@ async function listModels(req: Request, res: Response) { // Filter and categorize models const allModels = response.data || []; - // Include all models as chat models, without filtering by specific model names - // This allows models from providers like OpenRouter to be displayed + // Include all models as chat models, excluding embedding models const chatModels = allModels .filter((model) => // Exclude models that are explicitly for embeddings @@ -96,23 +84,10 @@ async function listModels(req: Request, res: Response) { type: 'chat' })); - const embeddingModels = allModels - .filter((model) => - // Only include embedding-specific models - model.id.includes('embedding') || - model.id.includes('embed') - ) - .map((model) => ({ - id: model.id, - name: model.id, - type: 'embedding' - })); - // Return the models list return { success: true, - chatModels, - embeddingModels + chatModels }; } catch (error: any) { log.error(`Error listing OpenAI models: ${error.message || 'Unknown error'}`); diff --git a/apps/server/src/routes/api/options.ts b/apps/server/src/routes/api/options.ts index 022b4514e4..f481b24a45 100644 --- a/apps/server/src/routes/api/options.ts +++ b/apps/server/src/routes/api/options.ts @@ -100,30 +100,11 @@ const ALLOWED_OPTIONS = new Set([ "openaiApiKey", "openaiBaseUrl", "openaiDefaultModel", - "openaiEmbeddingModel", - "openaiEmbeddingApiKey", - "openaiEmbeddingBaseUrl", "anthropicApiKey", "anthropicBaseUrl", "anthropicDefaultModel", - "voyageApiKey", - "voyageEmbeddingModel", - "voyageEmbeddingBaseUrl", "ollamaBaseUrl", "ollamaDefaultModel", - "ollamaEmbeddingModel", - "ollamaEmbeddingBaseUrl", - "embeddingAutoUpdateEnabled", - "embeddingDimensionStrategy", - "embeddingSelectedProvider", - "embeddingSimilarityThreshold", - "embeddingBatchSize", - "embeddingUpdateInterval", - "enableAutomaticIndexing", - "maxNotesPerLlmQuery", - - // Embedding options - "embeddingDefaultDimension", "mfaEnabled", "mfaMethod" ]); diff --git a/apps/server/src/routes/routes.ts b/apps/server/src/routes/routes.ts index 73beb28e23..39ad6d4eb6 100644 --- a/apps/server/src/routes/routes.ts +++ b/apps/server/src/routes/routes.ts @@ -54,7 +54,6 @@ import relationMapApiRoute from "./api/relation-map.js"; import otherRoute from "./api/other.js"; import metricsRoute from "./api/metrics.js"; import shareRoutes from "../share/routes.js"; -import embeddingsRoute from "./api/embeddings.js"; import ollamaRoute from "./api/ollama.js"; import openaiRoute from "./api/openai.js"; import anthropicRoute from "./api/anthropic.js"; @@ -377,31 +376,7 @@ function register(app: express.Application) { asyncApiRoute(PST, "/api/llm/chat/:chatNoteId/messages", llmRoute.sendMessage); asyncApiRoute(PST, "/api/llm/chat/:chatNoteId/messages/stream", llmRoute.streamMessage); - // LLM index management endpoints - reorganized for REST principles - asyncApiRoute(GET, "/api/llm/indexes/stats", llmRoute.getIndexStats); - asyncApiRoute(PST, "/api/llm/indexes", llmRoute.startIndexing); // Create index process - asyncApiRoute(GET, "/api/llm/indexes/failed", llmRoute.getFailedIndexes); - asyncApiRoute(PUT, "/api/llm/indexes/notes/:noteId", llmRoute.retryFailedIndex); // Update index for note - asyncApiRoute(PUT, "/api/llm/indexes/failed", llmRoute.retryAllFailedIndexes); // Update all failed indexes - asyncApiRoute(GET, "/api/llm/indexes/notes/similar", llmRoute.findSimilarNotes); // Get similar notes - asyncApiRoute(GET, "/api/llm/indexes/context", llmRoute.generateQueryContext); // Get context - asyncApiRoute(PST, "/api/llm/indexes/notes/:noteId", llmRoute.indexNote); // Create index for specific note - - // LLM embeddings endpoints - asyncApiRoute(GET, "/api/llm/embeddings/similar/:noteId", embeddingsRoute.findSimilarNotes); - asyncApiRoute(PST, "/api/llm/embeddings/search", embeddingsRoute.searchByText); - asyncApiRoute(GET, "/api/llm/embeddings/providers", embeddingsRoute.getProviders); - asyncApiRoute(PATCH, "/api/llm/embeddings/providers/:providerId", embeddingsRoute.updateProvider); - asyncApiRoute(PST, "/api/llm/embeddings/reprocess", embeddingsRoute.reprocessAllNotes); - asyncApiRoute(GET, "/api/llm/embeddings/queue-status", embeddingsRoute.getQueueStatus); - asyncApiRoute(GET, "/api/llm/embeddings/stats", embeddingsRoute.getEmbeddingStats); - asyncApiRoute(GET, "/api/llm/embeddings/failed", embeddingsRoute.getFailedNotes); - asyncApiRoute(PST, "/api/llm/embeddings/retry/:noteId", embeddingsRoute.retryFailedNote); - asyncApiRoute(PST, "/api/llm/embeddings/retry-all-failed", embeddingsRoute.retryAllFailedNotes); - asyncApiRoute(PST, "/api/llm/embeddings/rebuild-index", embeddingsRoute.rebuildIndex); - asyncApiRoute(GET, "/api/llm/embeddings/index-rebuild-status", embeddingsRoute.getIndexRebuildStatus); - asyncApiRoute(PST, "/api/llm/embeddings/start", embeddingsRoute.startEmbeddings); - asyncApiRoute(PST, "/api/llm/embeddings/stop", embeddingsRoute.stopEmbeddings); + // LLM provider endpoints - moved under /api/llm/providers hierarchy asyncApiRoute(GET, "/api/llm/providers/ollama/models", ollamaRoute.listModels); diff --git a/apps/server/src/services/consistency_checks.ts b/apps/server/src/services/consistency_checks.ts index 8022c74df4..ec78505728 100644 --- a/apps/server/src/services/consistency_checks.ts +++ b/apps/server/src/services/consistency_checks.ts @@ -799,7 +799,6 @@ class ConsistencyChecks { this.runEntityChangeChecks("attributes", "attributeId"); this.runEntityChangeChecks("etapi_tokens", "etapiTokenId"); this.runEntityChangeChecks("options", "name"); - this.runEntityChangeChecks("note_embeddings", "embedId"); } findWronglyNamedAttributes() { diff --git a/apps/server/src/services/entity_changes.ts b/apps/server/src/services/entity_changes.ts index a22ecb11c2..66c2613ced 100644 --- a/apps/server/src/services/entity_changes.ts +++ b/apps/server/src/services/entity_changes.ts @@ -188,7 +188,6 @@ function fillAllEntityChanges() { fillEntityChanges("attributes", "attributeId"); fillEntityChanges("etapi_tokens", "etapiTokenId"); fillEntityChanges("options", "name", "WHERE isSynced = 1"); - fillEntityChanges("note_embeddings", "embedId"); }); } diff --git a/apps/server/src/services/erase.ts b/apps/server/src/services/erase.ts index 61b3f7ce8e..d5f4d2b1d1 100644 --- a/apps/server/src/services/erase.ts +++ b/apps/server/src/services/erase.ts @@ -28,10 +28,6 @@ function eraseNotes(noteIdsToErase: string[]) { eraseRevisions(revisionIdsToErase); - // Erase embeddings related to the deleted notes - const embeddingIdsToErase = sql.getManyRows<{ embedId: string }>(`SELECT embedId FROM note_embeddings WHERE noteId IN (???)`, noteIdsToErase).map((row) => row.embedId); - - eraseEmbeddings(embeddingIdsToErase); log.info(`Erased notes: ${JSON.stringify(noteIdsToErase)}`); } @@ -156,12 +152,6 @@ function eraseNotesWithDeleteId(deleteId: string) { const attachmentIdsToErase = sql.getColumn("SELECT attachmentId FROM attachments WHERE isDeleted = 1 AND deleteId = ?", [deleteId]); eraseAttachments(attachmentIdsToErase); - // Find and erase embeddings for deleted notes - const deletedNoteIds = sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 1 AND deleteId = ?", [deleteId]); - if (deletedNoteIds.length > 0) { - const embeddingIdsToErase = sql.getColumn("SELECT embedId FROM note_embeddings WHERE noteId IN (???)", deletedNoteIds); - eraseEmbeddings(embeddingIdsToErase); - } eraseUnusedBlobs(); } @@ -185,16 +175,6 @@ function eraseScheduledAttachments(eraseUnusedAttachmentsAfterSeconds: number | eraseAttachments(attachmentIdsToErase); } -function eraseEmbeddings(embedIdsToErase: string[]) { - if (embedIdsToErase.length === 0) { - return; - } - - sql.executeMany(`DELETE FROM note_embeddings WHERE embedId IN (???)`, embedIdsToErase); - setEntityChangesAsErased(sql.getManyRows(`SELECT * FROM entity_changes WHERE entityName = 'note_embeddings' AND entityId IN (???)`, embedIdsToErase)); - - log.info(`Erased embeddings: ${JSON.stringify(embedIdsToErase)}`); -} export function startScheduledCleanup() { sqlInit.dbReady.then(() => { diff --git a/apps/server/src/services/llm/ai_service_manager.ts b/apps/server/src/services/llm/ai_service_manager.ts index 394523d8fd..cbb0a87552 100644 --- a/apps/server/src/services/llm/ai_service_manager.ts +++ b/apps/server/src/services/llm/ai_service_manager.ts @@ -5,8 +5,6 @@ import { AnthropicService } from './providers/anthropic_service.js'; import { ContextExtractor } from './context/index.js'; import agentTools from './context_extractors/index.js'; import contextService from './context/services/context_service.js'; -import { getEmbeddingProvider, getEnabledEmbeddingProviders } from './providers/providers.js'; -import indexService from './index_service.js'; import log from '../log.js'; import { OllamaService } from './providers/ollama_service.js'; import { OpenAIService } from './providers/openai_service.js'; @@ -22,7 +20,6 @@ import type { NoteSearchResult } from './interfaces/context_interfaces.js'; // Import new configuration system import { getSelectedProvider, - getSelectedEmbeddingProvider, parseModelIdentifier, isAIEnabled, getDefaultModelForProvider, @@ -307,10 +304,11 @@ export class AIServiceManager implements IAIServiceManager { /** * Get the index service for managing knowledge base indexing - * @returns The index service instance + * @returns null since index service has been removed */ getIndexService() { - return indexService; + log.info('Index service has been removed - returning null'); + return null; } /** @@ -333,10 +331,11 @@ export class AIServiceManager implements IAIServiceManager { /** * Get the vector search tool for semantic similarity search + * Returns null since vector search has been removed */ getVectorSearchTool() { - const tools = agentTools.getTools(); - return tools.vectorSearch; + log.info('Vector search has been removed - getVectorSearchTool returning null'); + return null; } /** @@ -455,8 +454,7 @@ export class AIServiceManager implements IAIServiceManager { return; } - // Initialize index service - await this.getIndexService().initialize(); + // Index service has been removed - no initialization needed // Tools are already initialized in the constructor // No need to initialize them again @@ -648,7 +646,6 @@ export class AIServiceManager implements IAIServiceManager { name: provider, capabilities: { chat: true, - embeddings: provider !== 'anthropic', // Anthropic doesn't have embeddings streaming: true, functionCalling: provider === 'openai' // Only OpenAI has function calling }, @@ -676,7 +673,6 @@ export class AIServiceManager implements IAIServiceManager { const aiRelatedOptions = [ 'aiEnabled', 'aiSelectedProvider', - 'embeddingSelectedProvider', 'openaiApiKey', 'openaiBaseUrl', 'openaiDefaultModel', @@ -684,8 +680,7 @@ export class AIServiceManager implements IAIServiceManager { 'anthropicBaseUrl', 'anthropicDefaultModel', 'ollamaBaseUrl', - 'ollamaDefaultModel', - 'voyageApiKey' + 'ollamaDefaultModel' ]; eventService.subscribe(['entityChanged'], async ({ entityName, entity }) => { @@ -697,15 +692,11 @@ export class AIServiceManager implements IAIServiceManager { const isEnabled = entity.value === 'true'; if (isEnabled) { - log.info('AI features enabled, initializing AI service and embeddings'); + log.info('AI features enabled, initializing AI service'); // Initialize the AI service await this.initialize(); - // Initialize embeddings through index service - await indexService.startEmbeddingGeneration(); } else { - log.info('AI features disabled, stopping embeddings and clearing providers'); - // Stop embeddings through index service - await indexService.stopEmbeddingGeneration(); + log.info('AI features disabled, clearing providers'); // Clear chat providers this.services = {}; } @@ -730,10 +721,6 @@ export class AIServiceManager implements IAIServiceManager { // Clear existing chat providers (they will be recreated on-demand) this.services = {}; - // Clear embedding providers (they will be recreated on-demand when needed) - const providerManager = await import('./providers/providers.js'); - providerManager.clearAllEmbeddingProviders(); - log.info('LLM services recreated successfully'); } catch (error) { log.error(`Error recreating LLM services: ${this.handleError(error)}`); @@ -770,10 +757,6 @@ export default { async generateChatCompletion(messages: Message[], options: ChatCompletionOptions = {}): Promise { return getInstance().generateChatCompletion(messages, options); }, - // Add validateEmbeddingProviders method - async validateEmbeddingProviders(): Promise { - return getInstance().validateConfiguration(); - }, // Context and index related methods getContextExtractor() { return getInstance().getContextExtractor(); diff --git a/apps/server/src/services/llm/chat/handlers/context_handler.ts b/apps/server/src/services/llm/chat/handlers/context_handler.ts index c5af211164..fccfd3e5a7 100644 --- a/apps/server/src/services/llm/chat/handlers/context_handler.ts +++ b/apps/server/src/services/llm/chat/handlers/context_handler.ts @@ -1,20 +1,19 @@ /** * Handler for LLM context management + * Uses TriliumNext's native search service for powerful note discovery */ import log from "../../../log.js"; import becca from "../../../../becca/becca.js"; -import vectorStore from "../../embeddings/index.js"; -import providerManager from "../../providers/providers.js"; import contextService from "../../context/services/context_service.js"; +import searchService from "../../../search/services/search.js"; import type { NoteSource } from "../../interfaces/chat_session.js"; -import { SEARCH_CONSTANTS } from '../../constants/search_constants.js'; /** * Handles context management for LLM chat */ export class ContextHandler { /** - * Find relevant notes based on search query + * Find relevant notes based on search query using TriliumNext's search service * @param content The search content * @param contextNoteId Optional note ID for context * @param limit Maximum number of results to return @@ -27,106 +26,182 @@ export class ContextHandler { return []; } - // Check if embeddings are available - const enabledProviders = await providerManager.getEnabledEmbeddingProviders(); - if (enabledProviders.length === 0) { - log.info("No embedding providers available, can't find relevant notes"); - return []; - } + log.info(`Finding relevant notes for query: "${content.substring(0, 50)}..." using TriliumNext search`); - // Get the embedding for the query - const provider = enabledProviders[0]; - const embedding = await provider.generateEmbeddings(content); + const sources: NoteSource[] = []; - let results; if (contextNoteId) { - // For branch context, get notes specifically from that branch + // For branch context, get notes specifically from that branch and related notes const contextNote = becca.notes[contextNoteId]; if (!contextNote) { return []; } - const sql = require("../../../../services/sql.js").default; - const childBranches = await sql.getRows(` - SELECT branches.* FROM branches - WHERE branches.parentNoteId = ? - AND branches.isDeleted = 0 - `, [contextNoteId]); - - const childNoteIds = childBranches.map((branch: any) => branch.noteId); - - // Include the context note itself - childNoteIds.push(contextNoteId); - - // Find similar notes in this context - results = []; - - for (const noteId of childNoteIds) { - const noteEmbedding = await vectorStore.getEmbeddingForNote( - noteId, - provider.name, - provider.getConfig().model - ); - - if (noteEmbedding) { - const similarity = vectorStore.cosineSimilarity( - embedding, - noteEmbedding.embedding - ); - - if (similarity > SEARCH_CONSTANTS.VECTOR_SEARCH.EXACT_MATCH_THRESHOLD) { - results.push({ - noteId, - similarity - }); - } - } - } - - // Sort by similarity - results.sort((a, b) => b.similarity - a.similarity); - results = results.slice(0, limit); + const relevantNotes = this.findNotesInContext(contextNote, content, limit); + sources.push(...relevantNotes); } else { - // General search across all notes - results = await vectorStore.findSimilarNotes( - embedding, - provider.name, - provider.getConfig().model, - limit - ); + // General search across all notes using TriliumNext's search service + const relevantNotes = this.findNotesBySearch(content, limit); + sources.push(...relevantNotes); } - // Format the results - const sources: NoteSource[] = []; + log.info(`Found ${sources.length} relevant notes using TriliumNext search`); + return sources.slice(0, limit); + } catch (error: any) { + log.error(`Error finding relevant notes: ${error.message}`); + return []; + } + } + + /** + * Find notes in the context of a specific note (children, siblings, linked notes) + */ + private static findNotesInContext(contextNote: any, searchQuery: string, limit: number): NoteSource[] { + const sources: NoteSource[] = []; + const processedNoteIds = new Set(); - for (const result of results) { - const note = becca.notes[result.noteId]; - if (!note) continue; + // Add the context note itself (high priority) + sources.push(this.createNoteSource(contextNote, 1.0)); + processedNoteIds.add(contextNote.noteId); - let noteContent: string | undefined = undefined; - if (note.type === 'text') { - const content = note.getContent(); - // Handle both string and Buffer types - noteContent = typeof content === 'string' ? content : - content instanceof Buffer ? content.toString('utf8') : undefined; + // Get child notes (search within children) + try { + const childQuery = `note.childOf.noteId = "${contextNote.noteId}" ${searchQuery}`; + const childSearchResults = searchService.searchNotes(childQuery, { includeArchivedNotes: false }); + + for (const childNote of childSearchResults.slice(0, Math.floor(limit / 2))) { + if (!processedNoteIds.has(childNote.noteId)) { + sources.push(this.createNoteSource(childNote, 0.8)); + processedNoteIds.add(childNote.noteId); + } + } + } catch (error) { + log.info(`Child search failed, falling back to direct children: ${error}`); + // Fallback to direct child enumeration + const childNotes = contextNote.getChildNotes(); + for (const child of childNotes.slice(0, Math.floor(limit / 2))) { + if (!processedNoteIds.has(child.noteId) && !child.isDeleted) { + sources.push(this.createNoteSource(child, 0.8)); + processedNoteIds.add(child.noteId); } + } + } + + // Get related notes (through relations) + const relatedNotes = this.getRelatedNotes(contextNote); + for (const related of relatedNotes.slice(0, Math.floor(limit / 2))) { + if (!processedNoteIds.has(related.noteId) && !related.isDeleted) { + sources.push(this.createNoteSource(related, 0.6)); + processedNoteIds.add(related.noteId); + } + } - sources.push({ - noteId: result.noteId, - title: note.title, - content: noteContent, - similarity: result.similarity, - branchId: note.getBranches()[0]?.branchId + // Fill remaining slots with broader search if needed + if (sources.length < limit) { + try { + const remainingSlots = limit - sources.length; + const broadSearchResults = searchService.searchNotes(searchQuery, { + includeArchivedNotes: false, + limit: remainingSlots * 2 // Get more to filter out duplicates }); + + for (const note of broadSearchResults.slice(0, remainingSlots)) { + if (!processedNoteIds.has(note.noteId)) { + sources.push(this.createNoteSource(note, 0.5)); + processedNoteIds.add(note.noteId); + } + } + } catch (error) { + log.error(`Broad search failed: ${error}`); + } + } + + return sources.slice(0, limit); + } + + /** + * Find notes by search across all notes using TriliumNext's search service + */ + private static findNotesBySearch(searchQuery: string, limit: number): NoteSource[] { + try { + log.info(`Performing global search for: "${searchQuery}"`); + + // Use TriliumNext's search service for powerful note discovery + const searchResults = searchService.searchNotes(searchQuery, { + includeArchivedNotes: false, + fastSearch: false // Use full search for better results + }); + + log.info(`Global search found ${searchResults.length} notes`); + + // Convert search results to NoteSource format + const sources: NoteSource[] = []; + const limitedResults = searchResults.slice(0, limit); + + for (let i = 0; i < limitedResults.length; i++) { + const note = limitedResults[i]; + // Calculate similarity score based on position (first results are more relevant) + const similarity = Math.max(0.1, 1.0 - (i / limitedResults.length) * 0.8); + sources.push(this.createNoteSource(note, similarity)); } return sources; - } catch (error: any) { - log.error(`Error finding relevant notes: ${error.message}`); + } catch (error) { + log.error(`Error in global search: ${error}`); + // Fallback to empty results rather than crashing return []; } } + + /** + * Get notes related through attributes/relations + */ + private static getRelatedNotes(note: any): any[] { + const relatedNotes: any[] = []; + + // Get notes this note points to via relations + const outgoingRelations = note.getOwnedAttributes().filter((attr: any) => attr.type === 'relation'); + for (const relation of outgoingRelations) { + const targetNote = becca.notes[relation.value]; + if (targetNote && !targetNote.isDeleted) { + relatedNotes.push(targetNote); + } + } + + // Get notes that point to this note via relations + const incomingRelations = note.getTargetRelations(); + for (const relation of incomingRelations) { + const sourceNote = relation.getNote(); + if (sourceNote && !sourceNote.isDeleted) { + relatedNotes.push(sourceNote); + } + } + + return relatedNotes; + } + + /** + * Create a NoteSource object from a note + */ + private static createNoteSource(note: any, similarity: number): NoteSource { + let noteContent: string | undefined = undefined; + if (note.type === 'text') { + const content = note.getContent(); + // Handle both string and Buffer types + noteContent = typeof content === 'string' ? content : + content instanceof Buffer ? content.toString('utf8') : undefined; + } + + return { + noteId: note.noteId, + title: note.title, + content: noteContent, + similarity: similarity, + branchId: note.getBranches()[0]?.branchId + }; + } + /** * Process enhanced context using the context service * @param query Query to process @@ -165,4 +240,4 @@ export class ContextHandler { })) }; } -} +} \ No newline at end of file diff --git a/apps/server/src/services/llm/config/configuration_helpers.ts b/apps/server/src/services/llm/config/configuration_helpers.ts index 2635cc35f1..d48504bd2a 100644 --- a/apps/server/src/services/llm/config/configuration_helpers.ts +++ b/apps/server/src/services/llm/config/configuration_helpers.ts @@ -1,5 +1,6 @@ import configurationManager from './configuration_manager.js'; import optionService from '../../options.js'; +import log from '../../log.js'; import type { ProviderType, ModelIdentifier, @@ -19,13 +20,6 @@ export async function getSelectedProvider(): Promise { return providerOption as ProviderType || null; } -/** - * Get the selected embedding provider - */ -export async function getSelectedEmbeddingProvider(): Promise { - const providerOption = optionService.getOption('embeddingSelectedProvider'); - return providerOption || null; -} /** * Parse a model identifier (handles "provider:model" format) diff --git a/apps/server/src/services/llm/config/configuration_manager.ts b/apps/server/src/services/llm/config/configuration_manager.ts index 1e082db417..8dfd8df05d 100644 --- a/apps/server/src/services/llm/config/configuration_manager.ts +++ b/apps/server/src/services/llm/config/configuration_manager.ts @@ -3,11 +3,9 @@ import log from '../../log.js'; import type { AIConfig, ProviderPrecedenceConfig, - EmbeddingProviderPrecedenceConfig, ModelIdentifier, ModelConfig, ProviderType, - EmbeddingProviderType, ConfigValidationResult, ProviderSettings, OpenAISettings, @@ -51,7 +49,6 @@ export class ConfigurationManager { const config: AIConfig = { enabled: await this.getAIEnabled(), selectedProvider: await this.getSelectedProvider(), - selectedEmbeddingProvider: await this.getSelectedEmbeddingProvider(), defaultModels: await this.getDefaultModels(), providerSettings: await this.getProviderSettings() }; @@ -78,18 +75,6 @@ export class ConfigurationManager { } } - /** - * Get the selected embedding provider - */ - public async getSelectedEmbeddingProvider(): Promise { - try { - const selectedProvider = options.getOption('embeddingSelectedProvider'); - return selectedProvider as EmbeddingProviderType || null; - } catch (error) { - log.error(`Error getting selected embedding provider: ${error}`); - return null; - } - } /** * Parse model identifier with optional provider prefix @@ -269,10 +254,6 @@ export class ConfigurationManager { } } - // Validate selected embedding provider - if (!config.selectedEmbeddingProvider) { - result.warnings.push('No embedding provider selected'); - } } catch (error) { result.errors.push(`Configuration validation error: ${error}`); @@ -334,7 +315,6 @@ export class ConfigurationManager { return { enabled: false, selectedProvider: null, - selectedEmbeddingProvider: null, defaultModels: { openai: undefined, anthropic: undefined, diff --git a/apps/server/src/services/llm/constants/embedding_constants.ts b/apps/server/src/services/llm/constants/embedding_constants.ts deleted file mode 100644 index 07d3f83a3b..0000000000 --- a/apps/server/src/services/llm/constants/embedding_constants.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const EMBEDDING_CONSTANTS = { - exactTitleMatch: 0.3, - titleContainsQuery: 0.2, - partialTitleMatch: 0.1, - sameType: 0.05, - attributeMatch: 0.05, - recentlyCreated: 0.05, - recentlyModified: 0.05 -}; diff --git a/apps/server/src/services/llm/constants/provider_constants.ts b/apps/server/src/services/llm/constants/provider_constants.ts index b54454374e..65b5b47499 100644 --- a/apps/server/src/services/llm/constants/provider_constants.ts +++ b/apps/server/src/services/llm/constants/provider_constants.ts @@ -1,24 +1,17 @@ +/** + * Configuration constants for LLM providers + */ export const PROVIDER_CONSTANTS = { ANTHROPIC: { - API_VERSION: '2023-06-01', - BETA_VERSION: 'messages-2023-12-15', BASE_URL: 'https://api.anthropic.com', - DEFAULT_MODEL: 'claude-3-haiku-20240307', - // Model mapping for simplified model names to their full versions - MODEL_MAPPING: { - 'claude-3.7-sonnet': 'claude-3-7-sonnet-20250219', - 'claude-3.5-sonnet': 'claude-3-5-sonnet-20241022', - 'claude-3.5-haiku': 'claude-3-5-haiku-20241022', - 'claude-3-opus': 'claude-3-opus-20240229', - 'claude-3-sonnet': 'claude-3-sonnet-20240229', - 'claude-3-haiku': 'claude-3-haiku-20240307', - 'claude-2': 'claude-2.1' - }, - // These are the currently available models from Anthropic + DEFAULT_MODEL: 'claude-3-5-sonnet-20241022', + API_VERSION: '2023-06-01', + BETA_VERSION: undefined, + CONTEXT_WINDOW: 200000, AVAILABLE_MODELS: [ { - id: 'claude-3-7-sonnet-20250219', - name: 'Claude 3.7 Sonnet', + id: 'claude-3-5-sonnet-20250106', + name: 'Claude 3.5 Sonnet (New)', description: 'Most intelligent model with hybrid reasoning capabilities', maxTokens: 8192 }, @@ -64,12 +57,7 @@ export const PROVIDER_CONSTANTS = { OPENAI: { BASE_URL: 'https://api.openai.com/v1', DEFAULT_MODEL: 'gpt-3.5-turbo', - DEFAULT_EMBEDDING_MODEL: 'text-embedding-ada-002', CONTEXT_WINDOW: 16000, - EMBEDDING_DIMENSIONS: { - ADA: 1536, - DEFAULT: 1536 - }, AVAILABLE_MODELS: [ { id: 'gpt-4o', @@ -132,51 +120,6 @@ export const LLM_CONSTANTS = { DEFAULT: 6000 }, - // Embedding dimensions (verify these with your actual models) - EMBEDDING_DIMENSIONS: { - OLLAMA: { - DEFAULT: 384, - NOMIC: 768, - MISTRAL: 1024 - }, - OPENAI: { - ADA: 1536, - DEFAULT: 1536 - }, - ANTHROPIC: { - CLAUDE: 1024, - DEFAULT: 1024 - }, - VOYAGE: { - DEFAULT: 1024 - } - }, - - // Model-specific embedding dimensions for Ollama models - OLLAMA_MODEL_DIMENSIONS: { - "llama3": 8192, - "llama3.1": 8192, - "mistral": 8192, - "nomic": 768, - "mxbai": 1024, - "nomic-embed-text": 768, - "mxbai-embed-large": 1024, - "default": 384 - }, - - // Model-specific context windows for Ollama models - OLLAMA_MODEL_CONTEXT_WINDOWS: { - "llama3": 8192, - "llama3.1": 8192, - "llama3.2": 8192, - "mistral": 8192, - "nomic": 32768, - "mxbai": 32768, - "nomic-embed-text": 32768, - "mxbai-embed-large": 32768, - "default": 8192 - }, - // Batch size configuration BATCH_SIZE: { OPENAI: 10, // OpenAI can handle larger batches efficiently @@ -189,8 +132,7 @@ export const LLM_CONSTANTS = { CHUNKING: { DEFAULT_SIZE: 1500, OLLAMA_SIZE: 1000, - DEFAULT_OVERLAP: 100, - MAX_SIZE_FOR_SINGLE_EMBEDDING: 5000 + DEFAULT_OVERLAP: 100 }, // Search/similarity thresholds diff --git a/apps/server/src/services/llm/constants/search_constants.ts b/apps/server/src/services/llm/constants/search_constants.ts index bc16899611..d7e57a9766 100644 --- a/apps/server/src/services/llm/constants/search_constants.ts +++ b/apps/server/src/services/llm/constants/search_constants.ts @@ -1,16 +1,4 @@ export const SEARCH_CONSTANTS = { - // Vector search parameters - VECTOR_SEARCH: { - DEFAULT_MAX_RESULTS: 10, - DEFAULT_THRESHOLD: 0.6, - SIMILARITY_THRESHOLD: { - COSINE: 0.6, - HYBRID: 0.3, - DIM_AWARE: 0.1 - }, - EXACT_MATCH_THRESHOLD: 0.65 - }, - // Context extraction parameters CONTEXT: { CONTENT_LENGTH: { @@ -40,7 +28,6 @@ export const SEARCH_CONSTANTS = { TEMPERATURE: { DEFAULT: 0.7, RELATIONSHIP_TOOL: 0.4, - VECTOR_SEARCH: 0.3, QUERY_PROCESSOR: 0.3 }, @@ -49,7 +36,6 @@ export const SEARCH_CONSTANTS = { DEFAULT_NOTE_SUMMARY_LENGTH: 500, DEFAULT_MAX_TOKENS: 4096, RELATIONSHIP_TOOL_MAX_TOKENS: 50, - VECTOR_SEARCH_MAX_TOKENS: 500, QUERY_PROCESSOR_MAX_TOKENS: 300, MIN_STRING_LENGTH: 3 }, @@ -87,51 +73,3 @@ export const MODEL_CAPABILITIES = { } }; -// Embedding processing constants -export const EMBEDDING_PROCESSING = { - MAX_TOTAL_PROCESSING_TIME: 5 * 60 * 1000, // 5 minutes - MAX_CHUNK_RETRY_ATTEMPTS: 2, - DEFAULT_MAX_CHUNK_PROCESSING_TIME: 60 * 1000, // 1 minute - OLLAMA_MAX_CHUNK_PROCESSING_TIME: 120 * 1000, // 2 minutes - DEFAULT_EMBEDDING_UPDATE_INTERVAL: 200 -}; - -// Provider-specific embedding capabilities -export const PROVIDER_EMBEDDING_CAPABILITIES = { - VOYAGE: { - MODELS: { - 'voyage-large-2': { - contextWidth: 8192, - dimension: 1536 - }, - 'voyage-2': { - contextWidth: 8192, - dimension: 1024 - }, - 'voyage-lite-02': { - contextWidth: 8192, - dimension: 768 - }, - 'default': { - contextWidth: 8192, - dimension: 1024 - } - } - }, - OPENAI: { - MODELS: { - 'text-embedding-3-small': { - dimension: 1536, - contextWindow: 8191 - }, - 'text-embedding-3-large': { - dimension: 3072, - contextWindow: 8191 - }, - 'default': { - dimension: 1536, - contextWindow: 8192 - } - } - } -}; diff --git a/apps/server/src/services/llm/context/modules/cache_manager.ts b/apps/server/src/services/llm/context/modules/cache_manager.ts index 540a23a19d..8886593f9c 100644 --- a/apps/server/src/services/llm/context/modules/cache_manager.ts +++ b/apps/server/src/services/llm/context/modules/cache_manager.ts @@ -6,7 +6,7 @@ import type { ICacheManager, CachedNoteData, CachedQueryResults } from '../../in * Provides a centralized caching system to avoid redundant operations */ export class CacheManager implements ICacheManager { - // Cache for recently used context to avoid repeated embedding lookups + // Cache for recently used context to avoid repeated lookups private noteDataCache = new Map>(); // Cache for recently used queries diff --git a/apps/server/src/services/llm/context/modules/context_formatter.ts b/apps/server/src/services/llm/context/modules/context_formatter.ts index 80f1addaa0..58198c9049 100644 --- a/apps/server/src/services/llm/context/modules/context_formatter.ts +++ b/apps/server/src/services/llm/context/modules/context_formatter.ts @@ -47,7 +47,7 @@ export class ContextFormatter implements IContextFormatter { let modelName = providerId; // Look up model capabilities - const modelCapabilities = await modelCapabilitiesService.getModelCapabilities(modelName); + const modelCapabilities = await modelCapabilitiesService.getChatModelCapabilities(modelName); // Calculate available context size for this conversation const availableContextSize = calculateAvailableContextSize( diff --git a/apps/server/src/services/llm/context/modules/provider_manager.ts b/apps/server/src/services/llm/context/modules/provider_manager.ts deleted file mode 100644 index 56c6437c03..0000000000 --- a/apps/server/src/services/llm/context/modules/provider_manager.ts +++ /dev/null @@ -1,83 +0,0 @@ -import log from '../../../log.js'; -import { getEmbeddingProvider, getEnabledEmbeddingProviders } from '../../providers/providers.js'; -import { getSelectedEmbeddingProvider as getSelectedEmbeddingProviderName } from '../../config/configuration_helpers.js'; - -/** - * Manages embedding providers for context services - */ -export class ProviderManager { - /** - * Get the selected embedding provider based on user settings - * Uses the single provider selection approach - * - * @returns The selected embedding provider or null if none available - */ - async getSelectedEmbeddingProvider(): Promise { - try { - // Get the selected embedding provider - const selectedProvider = await getSelectedEmbeddingProviderName(); - - if (selectedProvider) { - const provider = await getEmbeddingProvider(selectedProvider); - if (provider) { - log.info(`Using selected embedding provider: ${selectedProvider}`); - return provider; - } - log.info(`Selected embedding provider ${selectedProvider} is not available`); - } - - // If no provider is selected or available, try any enabled provider - const providers = await getEnabledEmbeddingProviders(); - if (providers.length > 0) { - log.info(`Using available embedding provider: ${providers[0].name}`); - return providers[0]; - } - - // Last resort is local provider - log.info('Using local embedding provider as fallback'); - return await getEmbeddingProvider('local'); - } catch (error) { - log.error(`Error getting preferred embedding provider: ${error}`); - return null; - } - } - - /** - * Generate embeddings for a text query - * - * @param query - The text query to embed - * @returns The generated embedding or null if failed - */ - async generateQueryEmbedding(query: string): Promise { - try { - // Get the preferred embedding provider - const provider = await this.getSelectedEmbeddingProvider(); - if (!provider) { - log.error('No embedding provider available'); - return null; - } - - // Generate the embedding - const embedding = await provider.generateEmbeddings(query); - - if (embedding) { - // Add the original query as a property to the embedding - // This is used for title matching in the vector search - Object.defineProperty(embedding, 'originalQuery', { - value: query, - writable: false, - enumerable: true, - configurable: false - }); - } - - return embedding; - } catch (error) { - log.error(`Error generating query embedding: ${error}`); - return null; - } - } -} - -// Export singleton instance -export default new ProviderManager(); diff --git a/apps/server/src/services/llm/context/services/context_service.ts b/apps/server/src/services/llm/context/services/context_service.ts index b4d4fd613c..9bffd88f37 100644 --- a/apps/server/src/services/llm/context/services/context_service.ts +++ b/apps/server/src/services/llm/context/services/context_service.ts @@ -11,9 +11,7 @@ */ import log from '../../../log.js'; -import providerManager from '../modules/provider_manager.js'; import cacheManager from '../modules/cache_manager.js'; -import vectorSearchService from './vector_search_service.js'; import queryProcessor from './query_processor.js'; import contextFormatter from '../modules/context_formatter.js'; import aiServiceManager from '../../ai_service_manager.js'; @@ -57,17 +55,11 @@ export class ContextService { this.initPromise = (async () => { try { - // Initialize provider - const provider = await providerManager.getSelectedEmbeddingProvider(); - if (!provider) { - throw new Error(`No embedding provider available. Could not initialize context service.`); - } - // Agent tools are already initialized in the AIServiceManager constructor // No need to initialize them again this.initialized = true; - log.info(`Context service initialized with provider: ${provider.name}`); + log.info(`Context service initialized`); } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); log.error(`Failed to initialize context service: ${errorMessage}`); @@ -178,59 +170,47 @@ export class ContextService { } } - // Step 3: Find relevant notes using vector search - const allResults = new Map(); + // Step 3: Find relevant notes using traditional search + log.info("Using traditional search for note discovery"); - for (const query of searchQueries) { + // Use fallback context based on the context note if provided + if (contextNoteId) { try { - log.info(`Searching for: "${query.substring(0, 50)}..."`); - - // Use the unified vector search service - const results = await vectorSearchService.findRelevantNotes( - query, - contextNoteId, - { - maxResults: maxResults, - summarizeContent: summarizeContent, - llmService: summarizeContent ? llmService : null - } - ); - - log.info(`Found ${results.length} results for query "${query.substring(0, 30)}..."`); - - // Combine results, avoiding duplicates - for (const result of results) { - if (!allResults.has(result.noteId)) { - allResults.set(result.noteId, result); - } else { - // If note already exists, update similarity to max of both values - const existing = allResults.get(result.noteId); - if (existing && result.similarity > existing.similarity) { - existing.similarity = result.similarity; - allResults.set(result.noteId, existing); - } + const becca = (await import('../../../../becca/becca.js')).default; + const contextNote = becca.getNote(contextNoteId); + if (contextNote) { + const content = await this.contextExtractor.getNoteContent(contextNoteId); + relevantNotes = [{ + noteId: contextNoteId, + title: contextNote.title, + similarity: 1.0, + content: content || "" + }]; + + // Add child notes as additional context + const childNotes = contextNote.getChildNotes().slice(0, maxResults - 1); + for (const child of childNotes) { + const childContent = await this.contextExtractor.getNoteContent(child.noteId); + relevantNotes.push({ + noteId: child.noteId, + title: child.title, + similarity: 0.8, + content: childContent || "" + }); } } } catch (error) { - log.error(`Error searching for query "${query}": ${error}`); + log.error(`Error accessing context note: ${error}`); } } - // Convert to array and sort by similarity - relevantNotes = Array.from(allResults.values()) - .sort((a, b) => b.similarity - a.similarity) - .slice(0, maxResults); - log.info(`Final combined results: ${relevantNotes.length} relevant notes`); // Step 4: Build context from the notes - const provider = await providerManager.getSelectedEmbeddingProvider(); - const providerId = provider?.name || 'default'; - const context = await contextFormatter.buildContextFromNotes( relevantNotes, userQuestion, - providerId + 'default' ); // Step 5: Add agent tools context if requested @@ -332,15 +312,10 @@ export class ContextService { llmService?: LLMServiceInterface | null } = {} ): Promise { - return vectorSearchService.findRelevantNotes( - query, - contextNoteId, - { - maxResults: options.maxResults, - summarizeContent: options.summarize, - llmService: options.llmService - } - ); + // Vector search has been removed - return empty results + // The LLM will rely on tool calls for context gathering + log.info(`Vector search disabled - findRelevantNotes returning empty results for query: ${query}`); + return []; } } diff --git a/apps/server/src/services/llm/context/services/index.ts b/apps/server/src/services/llm/context/services/index.ts index 8ce9f9c7fb..3556c69183 100644 --- a/apps/server/src/services/llm/context/services/index.ts +++ b/apps/server/src/services/llm/context/services/index.ts @@ -5,23 +5,19 @@ * consolidated from previously overlapping implementations: * * - ContextService: Main entry point for context extraction operations - * - VectorSearchService: Unified semantic search functionality * - QueryProcessor: Query enhancement and decomposition */ import contextService from './context_service.js'; -import vectorSearchService from './vector_search_service.js'; import queryProcessor from './query_processor.js'; export { contextService, - vectorSearchService, queryProcessor }; // Export types export type { ContextOptions } from './context_service.js'; -export type { VectorSearchOptions } from './vector_search_service.js'; export type { SubQuery, DecomposedQuery } from './query_processor.js'; // Default export for backwards compatibility diff --git a/apps/server/src/services/llm/context/services/vector_search_service.ts b/apps/server/src/services/llm/context/services/vector_search_service.ts deleted file mode 100644 index 98c7b993cf..0000000000 --- a/apps/server/src/services/llm/context/services/vector_search_service.ts +++ /dev/null @@ -1,464 +0,0 @@ -/** - * Unified Vector Search Service - * - * Consolidates functionality from: - * - semantic_search.ts - * - vector_search_stage.ts - * - * This service provides a central interface for all vector search operations, - * supporting both full and summarized note context extraction. - */ - -import * as vectorStore from '../../embeddings/index.js'; -import { cosineSimilarity } from '../../embeddings/index.js'; -import log from '../../../log.js'; -import becca from '../../../../becca/becca.js'; -import providerManager from '../modules/provider_manager.js'; -import cacheManager from '../modules/cache_manager.js'; -import type { NoteSearchResult } from '../../interfaces/context_interfaces.js'; -import type { LLMServiceInterface } from '../../interfaces/agent_tool_interfaces.js'; -import { SEARCH_CONSTANTS } from '../../constants/search_constants.js'; -import { isNoteExcludedFromAI } from '../../utils/ai_exclusion_utils.js'; - -export interface VectorSearchOptions { - maxResults?: number; - threshold?: number; - useEnhancedQueries?: boolean; - summarizeContent?: boolean; - llmService?: LLMServiceInterface | null; -} - -export class VectorSearchService { - private contextExtractor: any; - - constructor() { - // Lazy load the context extractor to avoid circular dependencies - import('../index.js').then(module => { - this.contextExtractor = new module.ContextExtractor(); - }); - } - - /** - * Find notes that are semantically relevant to a query - * - * @param query - The search query - * @param contextNoteId - Optional note ID to restrict search to a branch - * @param options - Search options including result limit and summarization preference - * @returns Array of relevant notes with similarity scores - */ - async findRelevantNotes( - query: string, - contextNoteId: string | null = null, - options: VectorSearchOptions = {} - ): Promise { - const { - maxResults = SEARCH_CONSTANTS.VECTOR_SEARCH.DEFAULT_MAX_RESULTS, - threshold = SEARCH_CONSTANTS.VECTOR_SEARCH.DEFAULT_THRESHOLD, - useEnhancedQueries = false, - summarizeContent = false, - llmService = null - } = options; - - log.info(`VectorSearchService: Finding relevant notes for "${query}"`); - log.info(`Parameters: contextNoteId=${contextNoteId || 'global'}, maxResults=${maxResults}, summarize=${summarizeContent}`); - - try { - // Check cache first - const cacheKey = `find:${query}:${contextNoteId || 'all'}:${maxResults}:${summarizeContent}`; - const cached = cacheManager.getQueryResults(cacheKey); - if (cached && Array.isArray(cached)) { - log.info(`VectorSearchService: Returning ${cached.length} cached results`); - return cached; - } - - // Get embedding for query - const queryEmbedding = await providerManager.generateQueryEmbedding(query); - if (!queryEmbedding) { - log.error('Failed to generate query embedding'); - return []; - } - - // Get provider information - const provider = await providerManager.getSelectedEmbeddingProvider(); - if (!provider) { - log.error('No embedding provider available'); - return []; - } - - // Find similar notes based on embeddings - let noteResults: { noteId: string, similarity: number }[] = []; - - // If contextNoteId is provided, search only within that branch - if (contextNoteId) { - noteResults = await this.findNotesInBranch( - queryEmbedding, - contextNoteId, - maxResults - ); - } else { - // Otherwise search across all notes with embeddings - noteResults = await vectorStore.findSimilarNotes( - queryEmbedding, - provider.name, - provider.getConfig().model || '', - maxResults - ); - } - - // Ensure context extractor is loaded - if (!this.contextExtractor) { - const module = await import('../index.js'); - this.contextExtractor = new module.ContextExtractor(); - } - - // Get note details for results - const enrichedResults = await Promise.all( - noteResults.map(async result => { - const note = becca.getNote(result.noteId); - if (!note) { - return null; - } - - // Check if this note is excluded from AI features - if (isNoteExcludedFromAI(note)) { - return null; // Skip this note if it has the AI exclusion label - } - - // Get note content - full or summarized based on option - let content: string | null = null; - - if (summarizeContent) { - content = await this.getSummarizedNoteContent(result.noteId, llmService); - } else { - content = await this.contextExtractor.getNoteContent(result.noteId); - } - - // Adjust similarity score based on content quality - let adjustedSimilarity = result.similarity; - - // Penalize notes with empty or minimal content - if (!content || content.trim().length <= 10) { - adjustedSimilarity *= 0.2; - } - // Slightly boost notes with substantial content - else if (content.length > 100) { - adjustedSimilarity = Math.min(1.0, adjustedSimilarity * 1.1); - } - - // Get primary parent note ID - const parentNotes = note.getParentNotes(); - const parentId = parentNotes.length > 0 ? parentNotes[0].noteId : undefined; - - // Create parent chain for context - const parentPath = await this.getParentPath(result.noteId); - - return { - noteId: result.noteId, - title: note.title, - content, - similarity: adjustedSimilarity, - parentId, - parentPath - }; - }) - ); - - // Filter out null results and notes with very low similarity - const filteredResults = enrichedResults.filter(result => - result !== null && result.similarity > threshold - ) as NoteSearchResult[]; - - // Sort results by adjusted similarity - filteredResults.sort((a, b) => b.similarity - a.similarity); - - // Limit to requested number of results - const limitedResults = filteredResults.slice(0, maxResults); - - // Cache results - cacheManager.storeQueryResults(cacheKey, limitedResults); - - log.info(`VectorSearchService: Found ${limitedResults.length} relevant notes`); - return limitedResults; - } catch (error) { - log.error(`Error finding relevant notes: ${error}`); - return []; - } - } - - /** - * Get a summarized version of note content - * - * @param noteId - The note ID to summarize - * @param llmService - Optional LLM service for summarization - * @returns Summarized content or full content if summarization fails - */ - private async getSummarizedNoteContent( - noteId: string, - llmService: LLMServiceInterface | null - ): Promise { - try { - // Get the full content first - const fullContent = await this.contextExtractor.getNoteContent(noteId); - if (!fullContent || fullContent.length < 500) { - // Don't summarize short content - return fullContent; - } - - // Check if we have an LLM service for summarization - if (!llmService) { - // If no LLM service, truncate the content instead - return fullContent.substring(0, 500) + "..."; - } - - // Check cache for summarized content - const cacheKey = `summary:${noteId}:${fullContent.length}`; - const cached = cacheManager.getNoteData(noteId, cacheKey); - if (cached) { - return cached as string; - } - - const note = becca.getNote(noteId); - if (!note) return null; - - // Prepare a summarization prompt - const messages = [ - { - role: "system" as const, - content: "Summarize the following note content concisely while preserving key information. Keep your summary to about 20% of the original length." - }, - { - role: "user" as const, - content: `Note title: ${note.title}\n\nContent:\n${fullContent}` - } - ]; - - // Request summarization with safeguards to prevent recursion - const result = await llmService.generateChatCompletion(messages, { - temperature: SEARCH_CONSTANTS.TEMPERATURE.VECTOR_SEARCH, - maxTokens: SEARCH_CONSTANTS.LIMITS.VECTOR_SEARCH_MAX_TOKENS, - // Use any to bypass type checking for these special options - // that are recognized by the LLM service but not in the interface - ...(({ - bypassFormatter: true, - bypassContextProcessing: true, - enableTools: false - } as any)) - }); - - const summary = result.text; - - // Cache the summarization result - cacheManager.storeNoteData(noteId, cacheKey, summary); - - return summary; - } catch (error) { - log.error(`Error summarizing note content: ${error}`); - // Fall back to getting the full content - return this.contextExtractor.getNoteContent(noteId); - } - } - - /** - * Find notes in a specific branch (subtree) that are relevant to a query - * - * @param embedding - The query embedding - * @param contextNoteId - Root note ID of the branch - * @param limit - Maximum results to return - * @returns Array of note IDs with similarity scores - */ - private async findNotesInBranch( - embedding: Float32Array, - contextNoteId: string, - limit = SEARCH_CONSTANTS.CONTEXT.MAX_SIMILAR_NOTES - ): Promise<{ noteId: string, similarity: number }[]> { - try { - // Get all notes in the subtree - const noteIds = await this.getSubtreeNoteIds(contextNoteId); - - if (noteIds.length === 0) { - return []; - } - - // Get provider information - const provider = await providerManager.getSelectedEmbeddingProvider(); - if (!provider) { - log.error('No embedding provider available'); - return []; - } - - // Get model configuration - const model = provider.getConfig().model || ''; - const providerName = provider.name; - - // Get embeddings for all notes in the branch - const results: { noteId: string, similarity: number }[] = []; - - for (const noteId of noteIds) { - try { - // Check if this note is excluded from AI features - const note = becca.getNote(noteId); - if (!note || isNoteExcludedFromAI(note)) { - continue; // Skip this note if it doesn't exist or has the AI exclusion label - } - - // Get note embedding - const embeddingResult = await vectorStore.getEmbeddingForNote( - noteId, - providerName, - model - ); - - if (embeddingResult && embeddingResult.embedding) { - // Calculate similarity - const similarity = cosineSimilarity(embedding, embeddingResult.embedding); - results.push({ noteId, similarity }); - } - } catch (error) { - log.error(`Error processing note ${noteId} for branch search: ${error}`); - } - } - - // Sort by similarity and return top results - return results - .sort((a, b) => b.similarity - a.similarity) - .slice(0, limit); - } catch (error) { - log.error(`Error in branch search: ${error}`); - return []; - } - } - - /** - * Get all note IDs in a subtree (branch) - * - * @param rootNoteId - The root note ID of the branch - * @returns Array of note IDs in the subtree - */ - private async getSubtreeNoteIds(rootNoteId: string): Promise { - try { - const note = becca.getNote(rootNoteId); - if (!note) return []; - - const noteIds = new Set([rootNoteId]); - const processChildNotes = async (noteId: string) => { - const childNotes = becca.getNote(noteId)?.getChildNotes() || []; - for (const childNote of childNotes) { - if (!noteIds.has(childNote.noteId)) { - noteIds.add(childNote.noteId); - await processChildNotes(childNote.noteId); - } - } - }; - - await processChildNotes(rootNoteId); - return Array.from(noteIds); - } catch (error) { - log.error(`Error getting subtree note IDs: ${error}`); - return []; - } - } - - /** - * Get the parent path for a note (for additional context) - * - * @param noteId - The note ID to get the parent path for - * @returns String representation of the parent path - */ - private async getParentPath(noteId: string): Promise { - try { - const note = becca.getNote(noteId); - if (!note) return ''; - - const path: string[] = []; - const parentNotes = note.getParentNotes(); - let currentNote = parentNotes.length > 0 ? parentNotes[0] : null; - - // Build path up to the maximum parent depth - let level = 0; - while (currentNote && level < SEARCH_CONSTANTS.CONTEXT.MAX_PARENT_DEPTH) { - path.unshift(currentNote.title); - const grandParents = currentNote.getParentNotes(); - currentNote = grandParents.length > 0 ? grandParents[0] : null; - level++; - } - - return path.join(' > '); - } catch (error) { - log.error(`Error getting parent path: ${error}`); - return ''; - } - } - - /** - * Find notes that are semantically relevant to multiple queries - * Combines results from multiple queries, deduplicates them, and returns the most relevant ones - * - * @param queries - Array of search queries - * @param contextNoteId - Optional note ID to restrict search to a branch - * @param options - Search options including result limit and summarization preference - * @returns Array of relevant notes with similarity scores, deduplicated and sorted - */ - async findRelevantNotesMultiQuery( - queries: string[], - contextNoteId: string | null = null, - options: VectorSearchOptions = {} - ): Promise { - if (!queries || queries.length === 0) { - log.info('No queries provided to findRelevantNotesMultiQuery'); - return []; - } - - log.info(`VectorSearchService: Finding relevant notes for ${queries.length} queries`); - log.info(`Multi-query parameters: contextNoteId=${contextNoteId || 'global'}, queries=${JSON.stringify(queries.map(q => q.substring(0, 20) + '...'))}`); - - try { - // Create a Map to deduplicate results across queries - const allResults = new Map(); - - // For each query, adjust maxResults to avoid getting too many total results - const adjustedMaxResults = options.maxResults ? - Math.ceil(options.maxResults / queries.length) : - Math.ceil(SEARCH_CONSTANTS.VECTOR_SEARCH.DEFAULT_MAX_RESULTS / queries.length); - - // Search for each query and combine results - for (const query of queries) { - try { - const queryOptions = { - ...options, - maxResults: adjustedMaxResults, - useEnhancedQueries: false // We're already using enhanced queries - }; - - const results = await this.findRelevantNotes(query, contextNoteId, queryOptions); - - // Merge results, keeping the highest similarity score for duplicates - for (const note of results) { - if (!allResults.has(note.noteId) || - (allResults.has(note.noteId) && note.similarity > (allResults.get(note.noteId)?.similarity || 0))) { - allResults.set(note.noteId, note); - } - } - - log.info(`Found ${results.length} results for query: "${query.substring(0, 30)}..."`); - } catch (error) { - log.error(`Error searching for query "${query}": ${error}`); - } - } - - // Convert map to array and sort by similarity - const combinedResults = Array.from(allResults.values()) - .sort((a, b) => b.similarity - a.similarity) - .slice(0, options.maxResults || SEARCH_CONSTANTS.VECTOR_SEARCH.DEFAULT_MAX_RESULTS); - - log.info(`VectorSearchService: Found ${combinedResults.length} total deduplicated results across ${queries.length} queries`); - - return combinedResults; - } catch (error) { - log.error(`Error in findRelevantNotesMultiQuery: ${error}`); - return []; - } - } -} - -// Export a singleton instance -export default new VectorSearchService(); diff --git a/apps/server/src/services/llm/context_extractors/index.ts b/apps/server/src/services/llm/context_extractors/index.ts index 9b97f38a3c..81a29eec49 100644 --- a/apps/server/src/services/llm/context_extractors/index.ts +++ b/apps/server/src/services/llm/context_extractors/index.ts @@ -7,7 +7,6 @@ import { ContextualThinkingTool } from './contextual_thinking_tool.js'; import { NoteNavigatorTool } from './note_navigator_tool.js'; import { QueryDecompositionTool } from './query_decomposition_tool.js'; -import { VectorSearchTool } from './vector_search_tool.js'; // Import services needed for initialization import contextService from '../context/services/context_service.js'; @@ -17,8 +16,7 @@ import log from '../../log.js'; import type { IContextualThinkingTool, INoteNavigatorTool, - IQueryDecompositionTool, - IVectorSearchTool + IQueryDecompositionTool } from '../interfaces/agent_tool_interfaces.js'; /** @@ -27,7 +25,6 @@ import type { * Manages and provides access to all available agent tools. */ export class AgentToolsManager { - private vectorSearchTool: VectorSearchTool | null = null; private noteNavigatorTool: NoteNavigatorTool | null = null; private queryDecompositionTool: QueryDecompositionTool | null = null; private contextualThinkingTool: ContextualThinkingTool | null = null; @@ -52,16 +49,10 @@ export class AgentToolsManager { } // Create tool instances - this.vectorSearchTool = new VectorSearchTool(); this.noteNavigatorTool = new NoteNavigatorTool(); this.queryDecompositionTool = new QueryDecompositionTool(); this.contextualThinkingTool = new ContextualThinkingTool(); - // Set context service in the vector search tool - if (this.vectorSearchTool) { - this.vectorSearchTool.setContextService(contextService); - } - this.initialized = true; log.info("Agent tools initialized successfully"); } catch (error) { @@ -75,11 +66,6 @@ export class AgentToolsManager { */ getAllTools() { return [ - { - name: "vector_search", - description: "Searches your notes for semantically similar content", - function: this.vectorSearchTool?.search.bind(this.vectorSearchTool) - }, { name: "navigate_to_note", description: "Navigates to a specific note", @@ -103,7 +89,6 @@ export class AgentToolsManager { */ getTools() { return { - vectorSearch: this.vectorSearchTool as IVectorSearchTool, noteNavigator: this.noteNavigatorTool as INoteNavigatorTool, queryDecomposition: this.queryDecompositionTool as IQueryDecompositionTool, contextualThinking: this.contextualThinkingTool as IContextualThinkingTool @@ -117,7 +102,6 @@ export default agentTools; // Export all tools for direct import if needed export { - VectorSearchTool, NoteNavigatorTool, QueryDecompositionTool, ContextualThinkingTool diff --git a/apps/server/src/services/llm/context_extractors/vector_search_tool.ts b/apps/server/src/services/llm/context_extractors/vector_search_tool.ts deleted file mode 100644 index a9f962e8f6..0000000000 --- a/apps/server/src/services/llm/context_extractors/vector_search_tool.ts +++ /dev/null @@ -1,218 +0,0 @@ -/** - * Vector Search Tool - * - * This tool enables the LLM agent to perform semantic vector-based searches - * over the content in the notes database. It handles: - * - Finding semantically related notes to a query - * - Extracting relevant sections from notes - * - Providing relevant context for LLM to generate accurate responses - * - * Updated to use the consolidated VectorSearchService - */ - -import log from '../../log.js'; -import type { ContextService } from '../context/services/context_service.js'; -import vectorSearchService from '../context/services/vector_search_service.js'; - -export interface VectorSearchResult { - noteId: string; - title: string; - contentPreview: string; - similarity: number; - parentId?: string; - dateCreated?: string; - dateModified?: string; -} - -export interface SearchResultItem { - noteId: string; - noteTitle: string; - contentPreview: string; - similarity: number; - parentId?: string; - dateCreated?: string; - dateModified?: string; -} - -export interface VectorSearchOptions { - limit?: number; - threshold?: number; - includeContent?: boolean; - summarize?: boolean; -} - -// Define a type for the context service -export interface IVectorContextService { - findRelevantNotes?: (query: string, contextNoteId: string | null, options: Record) => Promise; -} - -export class VectorSearchTool { - private contextService: IVectorContextService | null = null; - private maxResults: number = 5; - - constructor() { - log.info('VectorSearchTool initialized using consolidated VectorSearchService'); - } - - /** - * Set the context service for performing vector searches - */ - setContextService(contextService: IVectorContextService): void { - this.contextService = contextService; - log.info('Context service set in VectorSearchTool'); - } - - /** - * Perform a vector search for related notes - */ - async search( - query: string, - contextNoteId?: string, - searchOptions: VectorSearchOptions = {} - ): Promise { - try { - // Set more aggressive defaults to return more content - const options = { - maxResults: searchOptions.limit || 15, // Increased from default - threshold: searchOptions.threshold || 0.5, // Lower threshold to include more results - includeContent: searchOptions.includeContent !== undefined ? searchOptions.includeContent : true, - summarizeContent: searchOptions.summarize || false, - ...searchOptions - }; - - log.info(`Vector search: "${query.substring(0, 50)}..." with limit=${options.maxResults}, threshold=${options.threshold}`); - - // Use the consolidated vector search service - const searchResults = await vectorSearchService.findRelevantNotes( - query, - contextNoteId || null, - { - maxResults: options.maxResults, - threshold: options.threshold, - summarizeContent: options.summarizeContent - } - ); - - log.info(`Vector search found ${searchResults.length} relevant notes`); - - // Format results to match the expected VectorSearchResult interface - return searchResults.map(note => ({ - noteId: note.noteId, - title: note.title, - contentPreview: note.content - ? (options.summarizeContent - // Don't truncate already summarized content - ? note.content - // Only truncate non-summarized content - : (note.content.length > 200 - ? note.content.substring(0, 200) + '...' - : note.content)) - : 'No content available', - similarity: note.similarity, - parentId: note.parentId - })); - } catch (error) { - log.error(`Vector search error: ${error}`); - return []; - } - } - - /** - * Search for notes that are semantically related to the query - */ - async searchNotes(query: string, options: { - parentNoteId?: string, - maxResults?: number, - similarityThreshold?: number, - summarize?: boolean - } = {}): Promise { - try { - // Set defaults - const maxResults = options.maxResults || this.maxResults; - const threshold = options.similarityThreshold || 0.6; - const parentNoteId = options.parentNoteId || null; - const summarize = options.summarize || false; - - // Use the consolidated vector search service - const results = await vectorSearchService.findRelevantNotes( - query, - parentNoteId, - { - maxResults, - threshold, - summarizeContent: summarize - } - ); - - // Format results to match the expected interface - return results.map(result => ({ - noteId: result.noteId, - title: result.title, - contentPreview: result.content - ? (summarize - // Don't truncate already summarized content - ? result.content - // Only truncate non-summarized content - : (result.content.length > 200 - ? result.content.substring(0, 200) + '...' - : result.content)) - : 'No content available', - similarity: result.similarity, - parentId: result.parentId - })); - } catch (error) { - log.error(`Error in vector search: ${error}`); - return []; - } - } - - /** - * Search for content chunks that are semantically related to the query - */ - async searchContentChunks(query: string, options: { - noteId?: string, - maxResults?: number, - similarityThreshold?: number, - summarize?: boolean - } = {}): Promise { - try { - // For now, use the same implementation as searchNotes, - // but in the future we'll implement chunk-based search - return this.searchNotes(query, { - parentNoteId: options.noteId, - maxResults: options.maxResults, - similarityThreshold: options.similarityThreshold, - summarize: options.summarize - }); - } catch (error) { - log.error(`Error in vector chunk search: ${error}`); - return []; - } - } - - /** - * Elaborate on why certain results were returned for a query - */ - explainResults(query: string, results: VectorSearchResult[]): string { - if (!query || !results || results.length === 0) { - return "No results to explain."; - } - - let explanation = `For query "${query}", I found these semantically related notes:\n\n`; - - results.forEach((result, index) => { - explanation += `${index + 1}. "${result.title}" (similarity: ${(result.similarity * 100).toFixed(1)}%)\n`; - explanation += ` Preview: ${result.contentPreview.substring(0, 150)}...\n`; - - if (index < results.length - 1) { - explanation += "\n"; - } - }); - - explanation += "\nThese results were found based on semantic similarity rather than just keyword matching."; - - return explanation; - } -} - -export default new VectorSearchTool(); diff --git a/apps/server/src/services/llm/embeddings/base_embeddings.ts b/apps/server/src/services/llm/embeddings/base_embeddings.ts deleted file mode 100644 index 0332bbf512..0000000000 --- a/apps/server/src/services/llm/embeddings/base_embeddings.ts +++ /dev/null @@ -1,438 +0,0 @@ -import { NormalizationStatus } from './embeddings_interface.js'; -import type { NoteEmbeddingContext } from './embeddings_interface.js'; -import log from "../../log.js"; -import { LLM_CONSTANTS } from "../constants/provider_constants.js"; -import options from "../../options.js"; -import { isBatchSizeError as checkBatchSizeError } from '../interfaces/error_interfaces.js'; -import type { EmbeddingModelInfo } from '../interfaces/embedding_interfaces.js'; - -export interface EmbeddingConfig { - model: string; - dimension: number; - type: 'float32' | 'float64'; - apiKey?: string; - baseUrl?: string; - batchSize?: number; - contextWidth?: number; - normalizationStatus?: NormalizationStatus; -} - -/** - * Base class for embedding providers that implements common functionality - */ -export abstract class BaseEmbeddingProvider { - protected model: string; - protected dimension: number; - protected type: 'float32' | 'float64'; - protected maxBatchSize: number = 100; - protected apiKey?: string; - protected baseUrl: string; - protected name: string = 'base'; - protected modelInfoCache = new Map(); - protected config: EmbeddingConfig; - - constructor(config: EmbeddingConfig) { - this.model = config.model; - this.dimension = config.dimension; - this.type = config.type; - this.apiKey = config.apiKey; - this.baseUrl = config.baseUrl || ''; - this.config = config; - - // If batch size is specified, use it as maxBatchSize - if (config.batchSize) { - this.maxBatchSize = config.batchSize; - } - } - - getConfig(): EmbeddingConfig { - return { ...this.config }; - } - - /** - * Get the normalization status of this provider - * Default implementation returns the status from config if available, - * otherwise returns UNKNOWN status - */ - getNormalizationStatus(): NormalizationStatus { - return this.config.normalizationStatus || NormalizationStatus.UNKNOWN; - } - - getDimension(): number { - return this.config.dimension; - } - - async initialize(): Promise { - // Default implementation does nothing - return; - } - - /** - * Generate embeddings for a single text - */ - abstract generateEmbeddings(text: string): Promise; - - /** - * Get the appropriate batch size for this provider - * Override in provider implementations if needed - */ - protected async getBatchSize(): Promise { - // Try to get the user-configured batch size - let configuredBatchSize: number | null = null; - - try { - const batchSizeStr = await options.getOption('embeddingBatchSize'); - if (batchSizeStr) { - configuredBatchSize = parseInt(batchSizeStr, 10); - } - } catch (error) { - log.error(`Error getting batch size from options: ${error}`); - } - - // If user has configured a specific batch size, use that - if (configuredBatchSize && !isNaN(configuredBatchSize) && configuredBatchSize > 0) { - return configuredBatchSize; - } - - // Otherwise use the provider-specific default from constants - return this.config.batchSize || - LLM_CONSTANTS.BATCH_SIZE[this.name.toUpperCase() as keyof typeof LLM_CONSTANTS.BATCH_SIZE] || - LLM_CONSTANTS.BATCH_SIZE.DEFAULT; - } - - /** - * Process a batch of texts with adaptive handling - * This method will try to process the batch and reduce batch size if encountering errors - */ - protected async processWithAdaptiveBatch( - items: T[], - processFn: (batch: T[]) => Promise, - isBatchSizeError: (error: unknown) => boolean - ): Promise { - const results: R[] = []; - const failures: { index: number, error: string }[] = []; - let currentBatchSize = await this.getBatchSize(); - let lastError: Error | null = null; - - // Process items in batches - for (let i = 0; i < items.length;) { - const batch = items.slice(i, i + currentBatchSize); - - try { - // Process the current batch - const batchResults = await processFn(batch); - results.push(...batchResults); - i += batch.length; - } - catch (error) { - lastError = error as Error; - const errorMessage = (lastError as Error).message || 'Unknown error'; - - // Check if this is a batch size related error - if (isBatchSizeError(error) && currentBatchSize > 1) { - // Reduce batch size and retry - const newBatchSize = Math.max(1, Math.floor(currentBatchSize / 2)); - console.warn(`Batch size error detected, reducing batch size from ${currentBatchSize} to ${newBatchSize}: ${errorMessage}`); - currentBatchSize = newBatchSize; - } - else if (currentBatchSize === 1) { - // If we're already at batch size 1, we can't reduce further, so log the error and skip this item - log.error(`Error processing item at index ${i} with batch size 1: ${errorMessage}`); - failures.push({ index: i, error: errorMessage }); - i++; // Move to the next item - } - else { - // For other errors, retry with a smaller batch size as a precaution - const newBatchSize = Math.max(1, Math.floor(currentBatchSize / 2)); - console.warn(`Error processing batch, reducing batch size from ${currentBatchSize} to ${newBatchSize} as a precaution: ${errorMessage}`); - currentBatchSize = newBatchSize; - } - } - } - - // If all items failed and we have a last error, throw it - if (results.length === 0 && failures.length > 0 && lastError) { - throw lastError; - } - - // If some items failed but others succeeded, log the summary - if (failures.length > 0) { - console.warn(`Processed ${results.length} items successfully, but ${failures.length} items failed`); - } - - return results; - } - - /** - * Detect if an error is related to batch size limits - * Override in provider-specific implementations - */ - protected isBatchSizeError(error: unknown): boolean { - return checkBatchSizeError(error); - } - - /** - * Generate embeddings for multiple texts - * Default implementation processes texts one by one - */ - async generateBatchEmbeddings(texts: string[]): Promise { - if (texts.length === 0) { - return []; - } - - try { - return await this.processWithAdaptiveBatch( - texts, - async (batch) => { - const batchResults = await Promise.all( - batch.map(text => this.generateEmbeddings(text)) - ); - return batchResults; - }, - this.isBatchSizeError.bind(this) - ); - } - catch (error) { - const errorMessage = (error as Error).message || "Unknown error"; - log.error(`Batch embedding error for provider ${this.name}: ${errorMessage}`); - throw new Error(`${this.name} batch embedding error: ${errorMessage}`); - } - } - - /** - * Generate embeddings for a note with its context - */ - async generateNoteEmbeddings(context: NoteEmbeddingContext): Promise { - const text = [context.title || "", context.content || ""].filter(Boolean).join(" "); - return this.generateEmbeddings(text); - } - - /** - * Generate embeddings for multiple notes with their contexts - */ - async generateBatchNoteEmbeddings(contexts: NoteEmbeddingContext[]): Promise { - if (contexts.length === 0) { - return []; - } - - try { - return await this.processWithAdaptiveBatch( - contexts, - async (batch) => { - const batchResults = await Promise.all( - batch.map(context => this.generateNoteEmbeddings(context)) - ); - return batchResults; - }, - this.isBatchSizeError.bind(this) - ); - } - catch (error) { - const errorMessage = (error as Error).message || "Unknown error"; - log.error(`Batch note embedding error for provider ${this.name}: ${errorMessage}`); - throw new Error(`${this.name} batch note embedding error: ${errorMessage}`); - } - } - - /** - * Cleans and normalizes text for embeddings by removing excessive whitespace - */ - private cleanText(text: string): string { - return text.replace(/\s+/g, ' ').trim(); - } - - /** - * Generates a rich text representation of a note's context for embedding - */ - protected generateNoteContextText(context: NoteEmbeddingContext): string { - // Build a relationship-focused summary first - const relationshipSummary: string[] = []; - - // Summarize the note's place in the hierarchy - if (context.parentTitles.length > 0) { - relationshipSummary.push(`This note is a child of: ${context.parentTitles.map(t => this.cleanText(t)).join(', ')}.`); - } - - if (context.childTitles.length > 0) { - relationshipSummary.push(`This note has children: ${context.childTitles.map(t => this.cleanText(t)).join(', ')}.`); - } - - // Emphasize relationships with other notes - if (context.relatedNotes && context.relatedNotes.length > 0) { - // Group by relation type for better understanding - const relationsByType: Record = {}; - for (const rel of context.relatedNotes) { - if (!relationsByType[rel.relationName]) { - relationsByType[rel.relationName] = []; - } - relationsByType[rel.relationName].push(this.cleanText(rel.targetTitle)); - } - - for (const [relType, targets] of Object.entries(relationsByType)) { - relationshipSummary.push(`This note has ${relType} relationship with: ${targets.join(', ')}.`); - } - } - - // Emphasize backlinks for bidirectional relationships - if (context.backlinks && context.backlinks.length > 0) { - // Group by relation type - const backlinksByType: Record = {}; - for (const link of context.backlinks) { - if (!backlinksByType[link.relationName]) { - backlinksByType[link.relationName] = []; - } - backlinksByType[link.relationName].push(this.cleanText(link.sourceTitle)); - } - - for (const [relType, sources] of Object.entries(backlinksByType)) { - relationshipSummary.push(`This note is ${relType} of: ${sources.join(', ')}.`); - } - } - - // Emphasize templates/inheritance - if (context.templateTitles && context.templateTitles.length > 0) { - relationshipSummary.push(`This note inherits from: ${context.templateTitles.map(t => this.cleanText(t)).join(', ')}.`); - } - - // Start with core note information - let result = - `Title: ${this.cleanText(context.title)}\n` + - `Type: ${context.type}\n` + - `MIME: ${context.mime}\n`; - - // Add the relationship summary at the beginning for emphasis - if (relationshipSummary.length > 0) { - result += `Relationships: ${relationshipSummary.join(' ')}\n`; - } - - // Continue with dates - result += - `Created: ${context.dateCreated}\n` + - `Modified: ${context.dateModified}\n`; - - // Add attributes in a concise format - if (context.attributes.length > 0) { - result += 'Attributes: '; - const attributeTexts = context.attributes.map(attr => - `${attr.type}:${attr.name}=${this.cleanText(attr.value)}` - ); - result += attributeTexts.join('; ') + '\n'; - } - - // Add important label values concisely - if (context.labelValues && Object.keys(context.labelValues).length > 0) { - result += 'Labels: '; - const labelTexts = Object.entries(context.labelValues).map(([name, value]) => - `${name}=${this.cleanText(value)}` - ); - result += labelTexts.join('; ') + '\n'; - } - - // Parents, children, templates, relations, and backlinks are now handled in the relationship summary - // But we'll include them again in a more structured format for organization - - if (context.parentTitles.length > 0) { - result += `Parents: ${context.parentTitles.map(t => this.cleanText(t)).join('; ')}\n`; - } - - if (context.childTitles.length > 0) { - result += `Children: ${context.childTitles.map(t => this.cleanText(t)).join('; ')}\n`; - } - - if (context.templateTitles && context.templateTitles.length > 0) { - result += `Templates: ${context.templateTitles.map(t => this.cleanText(t)).join('; ')}\n`; - } - - if (context.relatedNotes && context.relatedNotes.length > 0) { - result += 'Related: '; - const relatedTexts = context.relatedNotes.map(rel => - `${rel.relationName}→${this.cleanText(rel.targetTitle)}` - ); - result += relatedTexts.join('; ') + '\n'; - } - - if (context.backlinks && context.backlinks.length > 0) { - result += 'Referenced By: '; - const backlinkTexts = context.backlinks.map(link => - `${this.cleanText(link.sourceTitle)}→${link.relationName}` - ); - result += backlinkTexts.join('; ') + '\n'; - } - - // Add attachments concisely - if (context.attachments.length > 0) { - result += 'Attachments: '; - const attachmentTexts = context.attachments.map(att => - `${this.cleanText(att.title)}(${att.mime})` - ); - result += attachmentTexts.join('; ') + '\n'; - } - - // Add content (already cleaned in getNoteEmbeddingContext) - result += `Content: ${context.content}`; - - return result; - } - - /** - * Process a batch of items with automatic retries and batch size adjustment - */ - protected async processBatchWithRetries( - items: T[], - processFn: (batch: T[]) => Promise, - isBatchSizeError: (error: unknown) => boolean = this.isBatchSizeError.bind(this) - ): Promise { - const results: Float32Array[] = []; - const failures: { index: number, error: string }[] = []; - let currentBatchSize = await this.getBatchSize(); - let lastError: Error | null = null; - - // Process items in batches - for (let i = 0; i < items.length;) { - const batch = items.slice(i, i + currentBatchSize); - - try { - // Process the current batch - const batchResults = await processFn(batch); - results.push(...batchResults); - i += batch.length; - } - catch (error) { - lastError = error as Error; - const errorMessage = lastError.message || 'Unknown error'; - - // Check if this is a batch size related error - if (isBatchSizeError(error) && currentBatchSize > 1) { - // Reduce batch size and retry - const newBatchSize = Math.max(1, Math.floor(currentBatchSize / 2)); - console.warn(`Batch size error detected, reducing batch size from ${currentBatchSize} to ${newBatchSize}: ${errorMessage}`); - currentBatchSize = newBatchSize; - } - else if (currentBatchSize === 1) { - // If we're already at batch size 1, we can't reduce further, so log the error and skip this item - console.error(`Error processing item at index ${i} with batch size 1: ${errorMessage}`); - failures.push({ index: i, error: errorMessage }); - i++; // Move to the next item - } - else { - // For other errors, retry with a smaller batch size as a precaution - const newBatchSize = Math.max(1, Math.floor(currentBatchSize / 2)); - console.warn(`Error processing batch, reducing batch size from ${currentBatchSize} to ${newBatchSize} as a precaution: ${errorMessage}`); - currentBatchSize = newBatchSize; - } - } - } - - // If all items failed and we have a last error, throw it - if (results.length === 0 && failures.length > 0 && lastError) { - throw lastError; - } - - // If some items failed but others succeeded, log the summary - if (failures.length > 0) { - console.warn(`Processed ${results.length} items successfully, but ${failures.length} items failed`); - } - - return results; - } -} diff --git a/apps/server/src/services/llm/embeddings/chunking/chunking_interface.ts b/apps/server/src/services/llm/embeddings/chunking/chunking_interface.ts deleted file mode 100644 index d2515ac024..0000000000 --- a/apps/server/src/services/llm/embeddings/chunking/chunking_interface.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { NoteEmbeddingContext } from "../types.js"; -import type { EmbeddingProvider } from "../embeddings_interface.js"; - -/** - * Interface for chunking operations - */ -export interface ChunkingOperations { - /** - * Process a large note by breaking it into chunks and creating embeddings for each chunk - */ - processNoteWithChunking( - noteId: string, - provider: EmbeddingProvider, - context: NoteEmbeddingContext - ): Promise; -} - -/** - * Get the chunking operations instance - * This function is implemented to break circular dependencies - */ -export async function getChunkingOperations(): Promise { - const chunking = await import('./chunking_processor.js'); - return chunking; -} diff --git a/apps/server/src/services/llm/embeddings/chunking/chunking_processor.ts b/apps/server/src/services/llm/embeddings/chunking/chunking_processor.ts deleted file mode 100644 index 60e1267fad..0000000000 --- a/apps/server/src/services/llm/embeddings/chunking/chunking_processor.ts +++ /dev/null @@ -1,477 +0,0 @@ -import log from "../../../log.js"; -import dateUtils from "../../../date_utils.js"; -import sql from "../../../sql.js"; -import becca from "../../../../becca/becca.js"; -import cls from "../../../../services/cls.js"; -import type { NoteEmbeddingContext } from "../types.js"; -import type { EmbeddingProvider } from "../embeddings_interface.js"; -import type { EmbeddingConfig } from "../embeddings_interface.js"; -import { LLM_CONSTANTS } from "../../../llm/constants/provider_constants.js"; -import { EMBEDDING_PROCESSING } from '../../constants/search_constants.js'; - -// Define error categories for better handling -const ERROR_CATEGORIES = { - // Temporary errors that should be retried - TEMPORARY: { - patterns: [ - 'timeout', 'connection', 'network', 'rate limit', 'try again', - 'service unavailable', 'too many requests', 'server error', - 'gateway', 'temporarily', 'overloaded' - ] - }, - // Permanent errors that should not be retried - PERMANENT: { - patterns: [ - 'invalid request', 'invalid content', 'not found', 'unsupported model', - 'invalid model', 'content policy', 'forbidden', 'unauthorized', - 'token limit', 'context length', 'too long', 'content violation' - ] - } -}; - -// Maximum time (in milliseconds) allowed for the entire chunking process -const MAX_TOTAL_PROCESSING_TIME = EMBEDDING_PROCESSING.MAX_TOTAL_PROCESSING_TIME; - -// Maximum number of retry attempts per chunk -const MAX_CHUNK_RETRY_ATTEMPTS = EMBEDDING_PROCESSING.MAX_CHUNK_RETRY_ATTEMPTS; - -// Maximum time per chunk processing (to prevent individual chunks from hanging) -const DEFAULT_MAX_CHUNK_PROCESSING_TIME = EMBEDDING_PROCESSING.DEFAULT_MAX_CHUNK_PROCESSING_TIME; -const OLLAMA_MAX_CHUNK_PROCESSING_TIME = EMBEDDING_PROCESSING.OLLAMA_MAX_CHUNK_PROCESSING_TIME; - -/** - * Interface for chunks from the chunking process - */ -interface ContentChunk { - content: string; - index: number; - metadata?: Record; -} - -/** - * Categorize an error as temporary or permanent based on its message - * @param errorMessage - The error message to categorize - * @returns 'temporary', 'permanent', or 'unknown' - */ -function categorizeError(errorMessage: string): 'temporary' | 'permanent' | 'unknown' { - const lowerCaseMessage = errorMessage.toLowerCase(); - - // Check for temporary error patterns - for (const pattern of ERROR_CATEGORIES.TEMPORARY.patterns) { - if (lowerCaseMessage.includes(pattern.toLowerCase())) { - return 'temporary'; - } - } - - // Check for permanent error patterns - for (const pattern of ERROR_CATEGORIES.PERMANENT.patterns) { - if (lowerCaseMessage.includes(pattern.toLowerCase())) { - return 'permanent'; - } - } - - // Default to unknown - return 'unknown'; -} - -/** - * Process a chunk with a timeout to prevent hanging - * @param provider - The embedding provider - * @param chunk - The chunk to process - * @param timeoutMs - Timeout in milliseconds - * @returns The generated embedding - */ -async function processChunkWithTimeout( - provider: EmbeddingProvider, - chunk: { content: string }, - timeoutMs: number -): Promise { - // Create a promise that rejects after the timeout - const timeoutPromise = new Promise((_, reject) => { - setTimeout(() => { - reject(new Error(`Chunk processing timed out after ${timeoutMs}ms`)); - }, timeoutMs); - }); - - // Create the actual processing promise - const processingPromise = provider.generateEmbeddings(chunk.content); - - // Race the two promises - whichever completes/rejects first wins - return Promise.race([processingPromise, timeoutPromise]); -} - -/** - * Process a large note by breaking it into chunks and creating embeddings for each chunk - * This provides more detailed and focused embeddings for different parts of large notes - * - * @param noteId - The ID of the note to process - * @param provider - The embedding provider to use - * @param context - The note context data - */ -export async function processNoteWithChunking( - noteId: string, - provider: EmbeddingProvider, - context: NoteEmbeddingContext -): Promise { - // Track the overall start time - const startTime = Date.now(); - - try { - // Get the context extractor dynamically to avoid circular dependencies - const { ContextExtractor } = await import('../../context/index.js'); - const contextExtractor = new ContextExtractor(); - - // Get note from becca - const note = becca.notes[noteId]; - if (!note) { - throw new Error(`Note ${noteId} not found in Becca cache`); - } - - // Use semantic chunking for better boundaries - const chunks = await contextExtractor.semanticChunking( - context.content, - note.title, - noteId, - { - // Adjust chunk size based on provider using constants - maxChunkSize: provider.name === 'ollama' ? - LLM_CONSTANTS.CHUNKING.OLLAMA_SIZE : - LLM_CONSTANTS.CHUNKING.DEFAULT_SIZE, - respectBoundaries: true - } - ); - - if (!chunks || chunks.length === 0) { - // Fall back to single embedding if chunking fails - await cls.init(async () => { - const embedding = await provider.generateEmbeddings(context.content); - const config = provider.getConfig(); - - // Use dynamic import instead of static import - const storage = await import('../storage.js'); - await storage.storeNoteEmbedding(noteId, provider.name, config.model, embedding); - }); - - log.info(`Generated single embedding for note ${noteId} (${note.title}) since chunking failed`); - return; - } - - // Generate and store embeddings for each chunk - const config = provider.getConfig(); - - // Delete existing embeddings first to avoid duplicates - // Use dynamic import - const storage = await import('../storage.js'); - await storage.deleteNoteEmbeddings(noteId, provider.name, config.model); - - // Track successful and failed chunks in memory during this processing run - let successfulChunks = 0; - let failedChunks = 0; - const totalChunks = chunks.length; - const failedChunkDetails: { - index: number, - error: string, - category: 'temporary' | 'permanent' | 'unknown', - attempts: number - }[] = []; - const retryQueue: { - index: number, - chunk: any, - attempts: number - }[] = []; - - log.info(`Processing ${chunks.length} chunks for note ${noteId} (${note.title})`); - - // Get the current time to prevent duplicate processing from timeouts - const processingStartTime = Date.now(); - const processingId = `${noteId}-${processingStartTime}`; - log.info(`Starting processing run ${processingId}`); - - // Process each chunk with a delay based on provider to avoid rate limits - for (let i = 0; i < chunks.length; i++) { - // Check if we've exceeded the overall time limit - if (Date.now() - startTime > MAX_TOTAL_PROCESSING_TIME) { - log.info(`Exceeded maximum processing time (${MAX_TOTAL_PROCESSING_TIME}ms) for note ${noteId}, stopping after ${i} chunks`); - - // Mark remaining chunks as failed due to timeout - for (let j = i; j < chunks.length; j++) { - failedChunks++; - failedChunkDetails.push({ - index: j + 1, - error: "Processing timeout - exceeded total allowed time", - category: 'temporary', - attempts: 1 - }); - } - - // Break the loop, we'll handle this as partial success if some chunks succeeded - break; - } - - const chunk = chunks[i]; - try { - // Generate embedding for this chunk's content with a timeout - await cls.init(async () => { - const embedding = await processChunkWithTimeout( - provider, - chunk, - provider.name === 'ollama' ? OLLAMA_MAX_CHUNK_PROCESSING_TIME : DEFAULT_MAX_CHUNK_PROCESSING_TIME - ); - - // Store with chunk information in a unique ID format - const chunkIdSuffix = `${i + 1}_of_${chunks.length}`; - await storage.storeNoteEmbedding( - noteId, - provider.name, - config.model, - embedding - ); - }); - - successfulChunks++; - - // Small delay between chunks to avoid rate limits - longer for Ollama - if (i < chunks.length - 1) { - await new Promise(resolve => setTimeout(resolve, - provider.name === 'ollama' ? 2000 : 100)); - } - } catch (error: any) { - const errorMessage = error.message || 'Unknown error'; - const errorCategory = categorizeError(errorMessage); - - // Track the failure for this specific chunk - failedChunks++; - failedChunkDetails.push({ - index: i + 1, - error: errorMessage, - category: errorCategory, - attempts: 1 - }); - - // Only add to retry queue if not a permanent error - if (errorCategory !== 'permanent') { - retryQueue.push({ - index: i, - chunk: chunk, - attempts: 1 - }); - } else { - log.info(`Chunk ${i + 1} for note ${noteId} has permanent error, skipping retries: ${errorMessage}`); - } - - log.error(`Error processing chunk ${i + 1} for note ${noteId} (${errorCategory} error): ${errorMessage}`); - } - } - - // Set a time limit for the retry phase - const retryStartTime = Date.now(); - const MAX_RETRY_TIME = 2 * 60 * 1000; // 2 minutes for all retries - - // Retry failed chunks with exponential backoff, but only those that aren't permanent errors - if (retryQueue.length > 0 && retryQueue.length < chunks.length) { - log.info(`Retrying ${retryQueue.length} failed chunks for note ${noteId}`); - - for (let j = 0; j < retryQueue.length; j++) { - // Check if we've exceeded the retry time limit - if (Date.now() - retryStartTime > MAX_RETRY_TIME) { - log.info(`Exceeded maximum retry time (${MAX_RETRY_TIME}ms) for note ${noteId}, stopping after ${j} retries`); - break; - } - - const item = retryQueue[j]; - - // Skip if we've already reached the max retry attempts for this chunk - if (item.attempts >= MAX_CHUNK_RETRY_ATTEMPTS) { - log.info(`Skipping chunk ${item.index + 1} for note ${noteId} as it reached maximum retry attempts (${MAX_CHUNK_RETRY_ATTEMPTS})`); - continue; - } - - try { - // Wait longer for retries with exponential backoff - await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(1.5, j))); - - // Retry the embedding with timeout using cls.init - await cls.init(async () => { - const embedding = await processChunkWithTimeout( - provider, - item.chunk, - provider.name === 'ollama' ? OLLAMA_MAX_CHUNK_PROCESSING_TIME : DEFAULT_MAX_CHUNK_PROCESSING_TIME - ); - - // Store with unique ID that indicates it was a retry - const chunkIdSuffix = `${item.index + 1}_of_${chunks.length}`; - const storage = await import('../storage.js'); - await storage.storeNoteEmbedding( - noteId, - provider.name, - config.model, - embedding - ); - }); - - // Update counters - successfulChunks++; - failedChunks--; - - // Remove from failedChunkDetails - const detailIndex = failedChunkDetails.findIndex(d => d.index === item.index + 1); - if (detailIndex >= 0) { - failedChunkDetails.splice(detailIndex, 1); - } - - log.info(`Successfully retried chunk ${item.index + 1} for note ${noteId} on attempt ${item.attempts + 1}`); - } catch (error: any) { - const errorMessage = error.message || 'Unknown error'; - const errorCategory = categorizeError(errorMessage); - - // Update failure record with new attempt count - const detailIndex = failedChunkDetails.findIndex(d => d.index === item.index + 1); - if (detailIndex >= 0) { - failedChunkDetails[detailIndex].attempts++; - failedChunkDetails[detailIndex].error = errorMessage; - failedChunkDetails[detailIndex].category = errorCategory; - } - - log.error(`Retry failed for chunk ${item.index + 1} of note ${noteId} (${errorCategory} error): ${errorMessage}`); - - // For timeout errors, mark as permanent to avoid further retries - if (errorMessage.includes('timed out')) { - if (detailIndex >= 0) { - failedChunkDetails[detailIndex].category = 'permanent'; - } - log.info(`Chunk ${item.index + 1} for note ${noteId} timed out, marking as permanent failure`); - } - // Add to retry queue again only if it's not a permanent error and hasn't reached the max attempts - else if (errorCategory !== 'permanent' && item.attempts + 1 < MAX_CHUNK_RETRY_ATTEMPTS) { - // If we're still below MAX_CHUNK_RETRY_ATTEMPTS, we'll try again in the next cycle - item.attempts++; - } else if (errorCategory === 'permanent') { - log.info(`Chunk ${item.index + 1} for note ${noteId} will not be retried further due to permanent error`); - } else { - log.info(`Chunk ${item.index + 1} for note ${noteId} reached maximum retry attempts (${MAX_CHUNK_RETRY_ATTEMPTS})`); - } - } - } - } - - // Log information about the processed chunks - if (successfulChunks > 0) { - log.info(`[${processingId}] Generated ${successfulChunks} chunk embeddings for note ${noteId} (${note.title})`); - } - - if (failedChunks > 0) { - // Count permanent vs temporary errors - const permanentErrors = failedChunkDetails.filter(d => d.category === 'permanent').length; - const temporaryErrors = failedChunkDetails.filter(d => d.category === 'temporary').length; - const unknownErrors = failedChunkDetails.filter(d => d.category === 'unknown').length; - - log.info(`[${processingId}] Failed to generate ${failedChunks} chunk embeddings for note ${noteId} (${note.title}). ` + - `Permanent: ${permanentErrors}, Temporary: ${temporaryErrors}, Unknown: ${unknownErrors}`); - } - - // Calculate the failure ratio - const failureRatio = failedChunks / totalChunks; - - // If no chunks were successfully processed, or if more than 50% failed, mark the entire note as failed - if (successfulChunks === 0 || failureRatio > 0.5) { - // Check if all failures are permanent - const allPermanent = failedChunkDetails.every(d => d.category === 'permanent'); - const errorType = allPermanent ? 'permanent' : (failureRatio > 0.5 ? 'too_many_failures' : 'all_failed'); - - // Mark this note as failed in the embedding_queue table with a permanent error status - const now = dateUtils.utcNowDateTime(); - const errorSummary = `Note embedding failed: ${failedChunks}/${totalChunks} chunks failed (${errorType}). First error: ${failedChunkDetails[0]?.error}`; - - await sql.execute(` - UPDATE embedding_queue - SET error = ?, lastAttempt = ?, attempts = 999 - WHERE noteId = ? - `, [errorSummary, now, noteId]); - - throw new Error(errorSummary); - } - - // If some chunks failed but others succeeded, log a warning but consider the processing complete - // The note will be removed from the queue, but we'll store error information - if (failedChunks > 0 && successfulChunks > 0) { - // Create detailed error summary - const permanentErrors = failedChunkDetails.filter(d => d.category === 'permanent').length; - const temporaryErrors = failedChunkDetails.filter(d => d.category === 'temporary').length; - const unknownErrors = failedChunkDetails.filter(d => d.category === 'unknown').length; - - const errorSummary = `Note processed partially: ${successfulChunks}/${totalChunks} chunks succeeded, ` + - `${failedChunks}/${totalChunks} failed (${permanentErrors} permanent, ${temporaryErrors} temporary, ${unknownErrors} unknown)`; - log.info(errorSummary); - - // Store a summary in the error field of embedding_queue - // This is just for informational purposes - the note will be removed from the queue - const now = dateUtils.utcNowDateTime(); - await sql.execute(` - UPDATE embedding_queue - SET error = ?, lastAttempt = ? - WHERE noteId = ? - `, [errorSummary, now, noteId]); - } - - // Track total processing time - const totalTime = Date.now() - startTime; - log.info(`[${processingId}] Total processing time for note ${noteId}: ${totalTime}ms`); - - } catch (error: any) { - log.error(`Error in chunked embedding process for note ${noteId}: ${error.message || 'Unknown error'}`); - throw error; - } -} - -/** - * Process a chunk with retry logic to handle errors - * @param index - The chunk index for tracking - * @param chunk - The content chunk - * @param provider - The embedding provider - * @param noteId - ID of the note being processed - * @param config - Embedding configuration - * @param startTime - When the overall process started - * @param storage - The storage module - * @param maxTimePerChunk - Max time per chunk processing - * @param retryAttempt - Current retry attempt number - */ -async function processChunkWithRetry( - index: number, - chunk: ContentChunk, - provider: EmbeddingProvider, - noteId: string, - config: EmbeddingConfig, - startTime: number, - storage: typeof import('../storage.js'), - maxTimePerChunk: number, - retryAttempt = 0 -): Promise { - try { - // Try to generate embedding with timeout - const embedding = await processChunkWithTimeout(provider, chunk, maxTimePerChunk); - - // Store the embedding with the chunk ID - const chunkId = `${noteId}_chunk${index}`; - await storage.storeNoteEmbedding(chunkId, provider.name, config.model, embedding); - - return true; - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : String(error); - const category = categorizeError(errorMessage); - - // If we haven't exceeded the retry limit and it's a temporary error, retry - if (retryAttempt < MAX_CHUNK_RETRY_ATTEMPTS && (category === 'temporary' || category === 'unknown')) { - // Exponential backoff - const delayMs = Math.min(1000 * Math.pow(2, retryAttempt), 15000); - log.info(`Retrying chunk ${index} after ${delayMs}ms (attempt ${retryAttempt + 1}/${MAX_CHUNK_RETRY_ATTEMPTS})`); - await new Promise(resolve => setTimeout(resolve, delayMs)); - - return processChunkWithRetry( - index, chunk, provider, noteId, config, startTime, storage, maxTimePerChunk, retryAttempt + 1 - ); - } else { - log.error(`Failed to process chunk ${index} after ${retryAttempt + 1} attempts: ${errorMessage}`); - return false; - } - } -} diff --git a/apps/server/src/services/llm/embeddings/content_processing.ts b/apps/server/src/services/llm/embeddings/content_processing.ts deleted file mode 100644 index cb540dab39..0000000000 --- a/apps/server/src/services/llm/embeddings/content_processing.ts +++ /dev/null @@ -1,327 +0,0 @@ -import becca from "../../../becca/becca.js"; -import type { NoteEmbeddingContext } from "./types.js"; -import sanitizeHtml from "sanitize-html"; -import type BNote from "../../../becca/entities/bnote.js"; - -/** - * Clean note content by removing HTML tags and normalizing whitespace - */ -export async function cleanNoteContent(content: string, type: string, mime: string): Promise { - if (!content) return ''; - - // If it's HTML content, remove HTML tags - if ((type === 'text' && mime === 'text/html') || content.includes('
') || content.includes('

')) { - // Use sanitizeHtml to remove all HTML tags - content = sanitizeHtml(content, { - allowedTags: [], - allowedAttributes: {}, - textFilter: (text) => { - // Normalize the text, removing excessive whitespace - return text.replace(/\s+/g, ' '); - } - }); - } - - // Additional cleanup for any remaining HTML entities - content = content - .replace(/ /g, ' ') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/&/g, '&'); - - // Normalize whitespace (replace multiple spaces/newlines with single space) - content = content.replace(/\s+/g, ' '); - - // Trim the content - content = content.trim(); - - // Import constants directly - const { LLM_CONSTANTS } = await import('../constants/provider_constants.js'); - // Truncate if extremely long - if (content.length > LLM_CONSTANTS.CONTENT.MAX_TOTAL_CONTENT_LENGTH) { - content = content.substring(0, LLM_CONSTANTS.CONTENT.MAX_TOTAL_CONTENT_LENGTH) + ' [content truncated]'; - } - - return content; -} - -/** - * Extract content from different note types - */ -export function extractStructuredContent(content: string, type: string, mime: string): string { - try { - if (!content) return ''; - - // Special handling based on note type - switch (type) { - case 'mindMap': - case 'relationMap': - case 'canvas': - if (mime === 'application/json') { - const jsonContent = JSON.parse(content); - - if (type === 'canvas') { - // Extract text elements from canvas - if (jsonContent.elements && Array.isArray(jsonContent.elements)) { - const texts = jsonContent.elements - .filter((element: any) => element.type === 'text' && element.text) - .map((element: any) => element.text); - return texts.join('\n'); - } - } - else if (type === 'mindMap') { - // Extract node text from mind map - const extractMindMapNodes = (node: any): string[] => { - let texts: string[] = []; - if (node.text) { - texts.push(node.text); - } - if (node.children && Array.isArray(node.children)) { - for (const child of node.children) { - texts = texts.concat(extractMindMapNodes(child)); - } - } - return texts; - }; - - if (jsonContent.root) { - return extractMindMapNodes(jsonContent.root).join('\n'); - } - } - else if (type === 'relationMap') { - // Extract relation map entities and connections - let result = ''; - - if (jsonContent.notes && Array.isArray(jsonContent.notes)) { - result += 'Notes: ' + jsonContent.notes - .map((note: any) => note.title || note.name) - .filter(Boolean) - .join(', ') + '\n'; - } - - if (jsonContent.relations && Array.isArray(jsonContent.relations)) { - result += 'Relations: ' + jsonContent.relations - .map((rel: any) => { - const sourceNote = jsonContent.notes.find((n: any) => n.noteId === rel.sourceNoteId); - const targetNote = jsonContent.notes.find((n: any) => n.noteId === rel.targetNoteId); - const source = sourceNote ? (sourceNote.title || sourceNote.name) : 'unknown'; - const target = targetNote ? (targetNote.title || targetNote.name) : 'unknown'; - return `${source} → ${rel.name || ''} → ${target}`; - }) - .join('; '); - } - - return result; - } - } - return JSON.stringify(content); - - case 'mermaid': - // Return mermaid diagrams as-is (they're human-readable) - return content; - - case 'geoMap': - if (mime === 'application/json') { - const jsonContent = JSON.parse(content); - let result = ''; - - if (jsonContent.markers && Array.isArray(jsonContent.markers)) { - result += jsonContent.markers - .map((marker: any) => { - return `Location: ${marker.title || ''} (${marker.lat}, ${marker.lng})${marker.description ? ' - ' + marker.description : ''}`; - }) - .join('\n'); - } - - return result || JSON.stringify(content); - } - return JSON.stringify(content); - - case 'file': - case 'image': - // For files and images, just return a placeholder - return `[${type} attachment]`; - - default: - return content; - } - } - catch (error) { - console.error(`Error extracting content from ${type} note:`, error); - return content; - } -} - -/** - * Gets context for a note to be embedded - */ -export async function getNoteEmbeddingContext(noteId: string): Promise { - const note = becca.getNote(noteId); - - if (!note) { - throw new Error(`Note ${noteId} not found`); - } - - // Get parent note titles - const parentNotes = note.getParentNotes(); - const parentTitles = parentNotes.map(note => note.title); - - // Get child note titles - const childNotes = note.getChildNotes(); - const childTitles = childNotes.map(note => note.title); - - // Get all attributes (not just owned ones) - const attributes = note.getAttributes().map(attr => ({ - type: attr.type, - name: attr.name, - value: attr.value - })); - - // Get backlinks (notes that reference this note through relations) - const targetRelations = note.getTargetRelations(); - const backlinks = targetRelations - .map(relation => { - const sourceNote = relation.getNote(); - if (sourceNote && sourceNote.type !== 'search') { // Filter out search notes - return { - sourceNoteId: sourceNote.noteId, - sourceTitle: sourceNote.title, - relationName: relation.name - }; - } - return null; - }) - .filter((item): item is { sourceNoteId: string; sourceTitle: string; relationName: string } => item !== null); - - // Get related notes through relations - const relations = note.getRelations(); - const relatedNotes = relations - .map(relation => { - const targetNote = relation.targetNote; - if (targetNote) { - return { - targetNoteId: targetNote.noteId, - targetTitle: targetNote.title, - relationName: relation.name - }; - } - return null; - }) - .filter((item): item is { targetNoteId: string; targetTitle: string; relationName: string } => item !== null); - - // Extract important labels that might affect semantics - const labelValues: Record = {}; - const labels = note.getLabels(); - for (const label of labels) { - // Skip CSS and UI-related labels that don't affect semantics - if (!label.name.startsWith('css') && - !label.name.startsWith('workspace') && - !label.name.startsWith('hide') && - !label.name.startsWith('collapsed')) { - labelValues[label.name] = label.value; - } - } - - // Get attachments - const attachments = note.getAttachments().map(att => ({ - title: att.title, - mime: att.mime - })); - - // Get content - let content = ""; - - try { - // Use the enhanced context extractor for improved content extraction - // We're using a dynamic import to avoid circular dependencies - const { ContextExtractor } = await import('../../llm/context/index.js'); - const contextExtractor = new ContextExtractor(); - - // Get the content using the enhanced formatNoteContent method in context extractor - const noteContent = await contextExtractor.getNoteContent(noteId); - - if (noteContent) { - content = noteContent; - - // For large content, consider chunking or summarization - if (content.length > 10000) { - // Large content handling options: - - // Option 1: Use our summarization feature - const summary = await contextExtractor.getNoteSummary(noteId); - if (summary) { - content = summary; - } - - // Option 2: Alternative approach - use the first chunk if summarization fails - if (content.length > 10000) { - const chunks = await contextExtractor.getChunkedNoteContent(noteId); - if (chunks && chunks.length > 0) { - // Use the first chunk (most relevant/beginning) - content = chunks[0]; - } - } - } - } else { - // Fallback to original method if context extractor fails - const rawContent = String(await note.getContent() || ""); - - // Process the content based on note type to extract meaningful text - if (note.type === 'text' || note.type === 'code') { - content = rawContent; - } else if (['canvas', 'mindMap', 'relationMap', 'mermaid', 'geoMap'].includes(note.type)) { - // Process structured content types - content = extractStructuredContent(rawContent, note.type, note.mime); - } else if (note.type === 'image' || note.type === 'file') { - content = `[${note.type} attachment: ${note.mime}]`; - } - - // Clean the content to remove HTML tags and normalize whitespace - content = await cleanNoteContent(content, note.type, note.mime); - } - } catch (err) { - console.error(`Error getting content for note ${noteId}:`, err); - content = `[Error extracting content]`; - - // Try fallback to original method - try { - const rawContent = String(await note.getContent() || ""); - if (note.type === 'text' || note.type === 'code') { - content = rawContent; - } else if (['canvas', 'mindMap', 'relationMap', 'mermaid', 'geoMap'].includes(note.type)) { - content = extractStructuredContent(rawContent, note.type, note.mime); - } - content = await cleanNoteContent(content, note.type, note.mime); - } catch (fallbackErr) { - console.error(`Fallback content extraction also failed for note ${noteId}:`, fallbackErr); - } - } - - // Get template/inheritance relationships - // This is from FNote.getNotesToInheritAttributesFrom - recreating similar logic for BNote - const templateRelations = note.getRelations('template').concat(note.getRelations('inherit')); - const templateTitles = templateRelations - .map(rel => rel.targetNote) - .filter((note): note is BNote => note !== undefined) - .map(templateNote => templateNote.title); - - return { - noteId: note.noteId, - title: note.title, - content: content, - type: note.type, - mime: note.mime, - dateCreated: note.dateCreated || "", - dateModified: note.dateModified || "", - attributes, - parentTitles, - childTitles, - attachments, - backlinks, - relatedNotes, - labelValues, - templateTitles - }; -} diff --git a/apps/server/src/services/llm/embeddings/embeddings_interface.ts b/apps/server/src/services/llm/embeddings/embeddings_interface.ts deleted file mode 100644 index 1db6c50992..0000000000 --- a/apps/server/src/services/llm/embeddings/embeddings_interface.ts +++ /dev/null @@ -1,140 +0,0 @@ -import type { NoteType, AttributeType } from "@triliumnext/commons"; - -/** - * Represents the context of a note that will be embedded - */ -export interface NoteEmbeddingContext { - noteId: string; - title: string; - content: string; - type: NoteType; - mime: string; - dateCreated: string; - dateModified: string; - attributes: { - type: AttributeType; - name: string; - value: string; - }[]; - parentTitles: string[]; - childTitles: string[]; - attachments: { - title: string; - mime: string; - }[]; - backlinks?: Backlink[]; - relatedNotes?: RelatedNote[]; - labelValues?: Record; - templateTitles?: string[]; -} - -export interface Backlink { - sourceNoteId: string; - sourceTitle: string; - relationName: string; -} - -export interface RelatedNote { - targetNoteId: string; - targetTitle: string; - relationName: string; -} - -/** - * Information about an embedding model's capabilities - */ -export interface EmbeddingModelInfo { - dimension: number; - contextWindow: number; - /** - * Whether the model guarantees normalized vectors (unit length) - */ - guaranteesNormalization: boolean; -} - -/** - * Normalization status of a provider's embeddings - */ -export enum NormalizationStatus { - /** - * Provider guarantees all embeddings are normalized to unit vectors - */ - GUARANTEED = 'guaranteed', - - /** - * Provider does not guarantee normalization, but embeddings are usually normalized - */ - USUALLY = 'usually', - - /** - * Provider does not guarantee normalization, embeddings must be normalized before use - */ - NEVER = 'never', - - /** - * Normalization status is unknown and should be checked at runtime - */ - UNKNOWN = 'unknown' -} - -/** - * Configuration for how embeddings should be generated - */ -export interface EmbeddingConfig { - model: string; - dimension: number; - type: 'float32' | 'float64'; - /** - * Whether embeddings should be normalized before use - * If true, normalization will always be applied - * If false, normalization depends on provider's status - */ - normalize?: boolean; - /** - * The normalization status of this provider - */ - normalizationStatus?: NormalizationStatus; - batchSize?: number; - contextWindowSize?: number; - apiKey?: string; - baseUrl?: string; -} - -/** - * Core interface that all embedding providers must implement - */ -export interface EmbeddingProvider { - name: string; - getConfig(): EmbeddingConfig; - - /** - * Returns information about the normalization status of this provider - */ - getNormalizationStatus(): NormalizationStatus; - - /** - * Verify that embeddings are properly normalized - * @returns true if embeddings are properly normalized - */ - verifyNormalization?(sample?: Float32Array): Promise; - - /** - * Generate embeddings for a single piece of text - */ - generateEmbeddings(text: string): Promise; - - /** - * Generate embeddings for multiple pieces of text in batch - */ - generateBatchEmbeddings(texts: string[]): Promise; - - /** - * Generate embeddings for a note with its full context - */ - generateNoteEmbeddings(context: NoteEmbeddingContext): Promise; - - /** - * Generate embeddings for multiple notes with their contexts in batch - */ - generateBatchNoteEmbeddings(contexts: NoteEmbeddingContext[]): Promise; -} diff --git a/apps/server/src/services/llm/embeddings/events.ts b/apps/server/src/services/llm/embeddings/events.ts deleted file mode 100644 index 2b8eac7d98..0000000000 --- a/apps/server/src/services/llm/embeddings/events.ts +++ /dev/null @@ -1,112 +0,0 @@ -import sql from "../../../services/sql.js"; -import log from "../../../services/log.js"; -import options from "../../../services/options.js"; -import cls from "../../../services/cls.js"; -import { processEmbeddingQueue, queueNoteForEmbedding } from "./queue.js"; -import eventService from "../../../services/events.js"; -import becca from "../../../becca/becca.js"; - -// Add mutex to prevent concurrent processing -let isProcessingEmbeddings = false; - -// Store interval reference for cleanup -let backgroundProcessingInterval: NodeJS.Timeout | null = null; - -/** - * Setup event listeners for embedding-related events - */ -export function setupEmbeddingEventListeners() { - // Listen for note content changes - eventService.subscribe(eventService.NOTE_CONTENT_CHANGE, ({ entity }) => { - if (entity && entity.noteId) { - queueNoteForEmbedding(entity.noteId); - } - }); - - // Listen for new notes - eventService.subscribe(eventService.ENTITY_CREATED, ({ entityName, entity }) => { - if (entityName === "notes" && entity && entity.noteId) { - queueNoteForEmbedding(entity.noteId); - } - }); - - // Listen for note title changes - eventService.subscribe(eventService.NOTE_TITLE_CHANGED, ({ noteId }) => { - if (noteId) { - queueNoteForEmbedding(noteId); - } - }); - - // Listen for note deletions - eventService.subscribe(eventService.ENTITY_DELETED, ({ entityName, entityId }) => { - if (entityName === "notes" && entityId) { - queueNoteForEmbedding(entityId, 'DELETE'); - } - }); - - // Listen for attribute changes that might affect context - eventService.subscribe(eventService.ENTITY_CHANGED, ({ entityName, entity }) => { - if (entityName === "attributes" && entity && entity.noteId) { - queueNoteForEmbedding(entity.noteId); - } - }); -} - -/** - * Setup background processing of the embedding queue - */ -export async function setupEmbeddingBackgroundProcessing() { - // Clear any existing interval - if (backgroundProcessingInterval) { - clearInterval(backgroundProcessingInterval); - backgroundProcessingInterval = null; - } - - const interval = parseInt(await options.getOption('embeddingUpdateInterval') || '200', 10); - - backgroundProcessingInterval = setInterval(async () => { - try { - // Skip if already processing - if (isProcessingEmbeddings) { - return; - } - - // Set mutex - isProcessingEmbeddings = true; - - // Wrap in cls.init to ensure proper context - cls.init(async () => { - await processEmbeddingQueue(); - }); - } catch (error: any) { - log.error(`Error in background embedding processing: ${error.message || 'Unknown error'}`); - } finally { - // Always release the mutex - isProcessingEmbeddings = false; - } - }, interval); -} - -/** - * Stop background processing of the embedding queue - */ -export function stopEmbeddingBackgroundProcessing() { - if (backgroundProcessingInterval) { - clearInterval(backgroundProcessingInterval); - backgroundProcessingInterval = null; - log.info("Embedding background processing stopped"); - } -} - -/** - * Initialize embeddings system - */ -export async function initEmbeddings() { - if (await options.getOptionBool('aiEnabled')) { - setupEmbeddingEventListeners(); - await setupEmbeddingBackgroundProcessing(); - log.info("Embeddings system initialized"); - } else { - log.info("Embeddings system disabled"); - } -} diff --git a/apps/server/src/services/llm/embeddings/index.ts b/apps/server/src/services/llm/embeddings/index.ts deleted file mode 100644 index c4be44a2ec..0000000000 --- a/apps/server/src/services/llm/embeddings/index.ts +++ /dev/null @@ -1,112 +0,0 @@ -// Re-export all modules for easy access -import * as vectorUtils from './vector_utils.js'; -import * as storage from './storage.js'; -import * as contentProcessing from './content_processing.js'; -import * as queue from './queue.js'; -// Import chunking dynamically to prevent circular dependencies -// import * as chunking from './chunking.js'; -import * as events from './events.js'; -import * as stats from './stats.js'; -import * as indexOperations from './index_operations.js'; -import { getChunkingOperations } from './chunking/chunking_interface.js'; -import type { NoteEmbeddingContext } from './types.js'; - -// Export types -export * from './types.js'; - -// Maintain backward compatibility by exposing all functions at the top level -export const { - cosineSimilarity, - embeddingToBuffer, - bufferToEmbedding, - adaptEmbeddingDimensions, - enhancedCosineSimilarity, - selectOptimalEmbedding -} = vectorUtils; - -export const { - storeNoteEmbedding, - getEmbeddingForNote, - findSimilarNotes, - deleteNoteEmbeddings -} = storage; - -export const { - getNoteEmbeddingContext, - cleanNoteContent, - extractStructuredContent -} = contentProcessing; - -export const { - queueNoteForEmbedding, - getFailedEmbeddingNotes, - retryFailedEmbedding, - retryAllFailedEmbeddings, - processEmbeddingQueue -} = queue; - -// Export chunking function using the interface to break circular dependencies -export const processNoteWithChunking = async ( - noteId: string, - provider: any, - context: NoteEmbeddingContext -): Promise => { - const chunkingOps = await getChunkingOperations(); - return chunkingOps.processNoteWithChunking(noteId, provider, context); -}; - -export const { - setupEmbeddingEventListeners, - setupEmbeddingBackgroundProcessing, - stopEmbeddingBackgroundProcessing, - initEmbeddings -} = events; - -export const { - getEmbeddingStats, - cleanupEmbeddings -} = stats; - -export const { - rebuildSearchIndex -} = indexOperations; - -// Default export for backward compatibility -export default { - // Vector utils - cosineSimilarity: vectorUtils.cosineSimilarity, - embeddingToBuffer: vectorUtils.embeddingToBuffer, - bufferToEmbedding: vectorUtils.bufferToEmbedding, - - // Storage - storeNoteEmbedding: storage.storeNoteEmbedding, - getEmbeddingForNote: storage.getEmbeddingForNote, - findSimilarNotes: storage.findSimilarNotes, - deleteNoteEmbeddings: storage.deleteNoteEmbeddings, - - // Content processing - getNoteEmbeddingContext: contentProcessing.getNoteEmbeddingContext, - - // Queue management - queueNoteForEmbedding: queue.queueNoteForEmbedding, - processEmbeddingQueue: queue.processEmbeddingQueue, - getFailedEmbeddingNotes: queue.getFailedEmbeddingNotes, - retryFailedEmbedding: queue.retryFailedEmbedding, - retryAllFailedEmbeddings: queue.retryAllFailedEmbeddings, - - // Chunking - use the dynamic wrapper - processNoteWithChunking, - - // Event handling - setupEmbeddingEventListeners: events.setupEmbeddingEventListeners, - setupEmbeddingBackgroundProcessing: events.setupEmbeddingBackgroundProcessing, - stopEmbeddingBackgroundProcessing: events.stopEmbeddingBackgroundProcessing, - initEmbeddings: events.initEmbeddings, - - // Stats and maintenance - getEmbeddingStats: stats.getEmbeddingStats, - cleanupEmbeddings: stats.cleanupEmbeddings, - - // Index operations - rebuildSearchIndex: indexOperations.rebuildSearchIndex -}; diff --git a/apps/server/src/services/llm/embeddings/index_operations.ts b/apps/server/src/services/llm/embeddings/index_operations.ts deleted file mode 100644 index 19fdd4bb17..0000000000 --- a/apps/server/src/services/llm/embeddings/index_operations.ts +++ /dev/null @@ -1,107 +0,0 @@ -import sql from "../../../services/sql.js"; -import log from "../../../services/log.js"; -import dateUtils from "../../../services/date_utils.js"; -import { bufferToEmbedding } from "./vector_utils.js"; -import indexService from "../index_service.js"; - -/** - * Rebuilds the search index structure without regenerating embeddings. - * This optimizes the existing embeddings for faster searches. - * - * @returns The number of embeddings processed - */ -export async function rebuildSearchIndex(): Promise { - log.info("Starting search index rebuild"); - const startTime = Date.now(); - - try { - // 1. Get count of all existing embeddings to track progress - const totalEmbeddings = await sql.getValue( - "SELECT COUNT(*) FROM note_embeddings" - ) as number; - - if (totalEmbeddings === 0) { - log.info("No embeddings found to rebuild index for"); - return 0; - } - - log.info(`Found ${totalEmbeddings} embeddings to process`); - - // 2. Process embeddings in batches to avoid memory issues - const batchSize = 100; - let processed = 0; - - // Get unique provider/model combinations - const providerModels = await sql.getRows( - "SELECT DISTINCT providerId, modelId FROM note_embeddings" - ) as {providerId: string, modelId: string}[]; - - // Process each provider/model combination - for (const {providerId, modelId} of providerModels) { - log.info(`Processing embeddings for provider: ${providerId}, model: ${modelId}`); - - // Get embeddings for this provider/model in batches - let offset = 0; - while (true) { - const embeddings = await sql.getRows(` - SELECT embedId, noteId, dimension, embedding, dateModified - FROM note_embeddings - WHERE providerId = ? AND modelId = ? - ORDER BY noteId - LIMIT ? OFFSET ?`, - [providerId, modelId, batchSize, offset] - ) as any[]; - - if (embeddings.length === 0) { - break; - } - - // Process this batch of embeddings - for (const embedding of embeddings) { - try { - // Convert buffer to embedding for processing - const vector = bufferToEmbedding(embedding.embedding, embedding.dimension); - - // Optimize this embedding (in a real system, this might involve: - // - Adding to an optimized index structure - // - Normalizing vectors - // - Updating index metadata - // For this implementation, we'll just "touch" the record to simulate optimization) - await sql.execute(` - UPDATE note_embeddings - SET dateModified = ?, utcDateModified = ? - WHERE embedId = ?`, - [dateUtils.localNowDateTime(), dateUtils.utcNowDateTime(), embedding.embedId] - ); - - processed++; - - // Update progress every 10 embeddings - if (processed % 10 === 0) { - indexService.updateIndexRebuildProgress(10); - - // Log progress every 100 embeddings - if (processed % 100 === 0) { - const percent = Math.round((processed / totalEmbeddings) * 100); - log.info(`Index rebuild progress: ${percent}% (${processed}/${totalEmbeddings})`); - } - } - } catch (error: any) { - log.error(`Error processing embedding ${embedding.embedId}: ${error.message || "Unknown error"}`); - } - } - - offset += embeddings.length; - } - } - - // 3. Finalize - could involve additional optimization steps - const duration = Math.round((Date.now() - startTime) / 1000); - log.info(`Index rebuild completed: processed ${processed} embeddings in ${duration} seconds`); - - return processed; - } catch (error: any) { - log.error(`Error during index rebuild: ${error.message || "Unknown error"}`); - throw error; - } -} diff --git a/apps/server/src/services/llm/embeddings/init.ts b/apps/server/src/services/llm/embeddings/init.ts deleted file mode 100644 index 75d664e43d..0000000000 --- a/apps/server/src/services/llm/embeddings/init.ts +++ /dev/null @@ -1,67 +0,0 @@ -import log from "../../log.js"; -import options from "../../options.js"; -import { initEmbeddings } from "./index.js"; -import providerManager from "../providers/providers.js"; -import sqlInit from "../../sql_init.js"; -import sql from "../../sql.js"; -import { validateProviders, logValidationResults, hasWorkingEmbeddingProviders } from "../provider_validation.js"; - -/** - * Reset any stuck embedding queue items that were left in processing state - * from a previous server shutdown - */ -async function resetStuckEmbeddingQueue() { - try { - const stuckCount = await sql.getValue( - "SELECT COUNT(*) FROM embedding_queue WHERE isProcessing = 1" - ) as number; - - if (stuckCount > 0) { - log.info(`Resetting ${stuckCount} stuck items in embedding queue from previous shutdown`); - - await sql.execute( - "UPDATE embedding_queue SET isProcessing = 0 WHERE isProcessing = 1" - ); - } - } catch (error: any) { - log.error(`Error resetting stuck embedding queue: ${error.message || error}`); - } -} - -/** - * Initialize the embedding system - */ -export async function initializeEmbeddings() { - try { - log.info("Initializing embedding system..."); - - // Check if the database is initialized before proceeding - if (!sqlInit.isDbInitialized()) { - log.info("Skipping embedding system initialization as database is not initialized yet."); - return; - } - - // Reset any stuck embedding queue items from previous server shutdown - await resetStuckEmbeddingQueue(); - - // Start the embedding system if AI is enabled - if (await options.getOptionBool('aiEnabled')) { - // Validate providers before starting the embedding system - log.info("Validating AI providers before starting embedding system..."); - const validation = await validateProviders(); - logValidationResults(validation); - - if (await hasWorkingEmbeddingProviders()) { - // Embedding providers will be created on-demand when needed - await initEmbeddings(); - log.info("Embedding system initialized successfully."); - } else { - log.info("Embedding system not started: No working embedding providers found. Please configure at least one AI provider (OpenAI, Ollama, or Voyage) to use embedding features."); - } - } else { - log.info("Embedding system disabled (AI features are turned off)."); - } - } catch (error: any) { - log.error(`Error initializing embedding system: ${error.message || error}`); - } -} diff --git a/apps/server/src/services/llm/embeddings/providers/local.ts b/apps/server/src/services/llm/embeddings/providers/local.ts deleted file mode 100644 index 0bfd3b4c47..0000000000 --- a/apps/server/src/services/llm/embeddings/providers/local.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { BaseEmbeddingProvider } from "../base_embeddings.js"; -import type { EmbeddingConfig } from "../embeddings_interface.js"; -import crypto from "crypto"; - -/** - * Local embedding provider implementation - * - * This is a fallback provider that generates simple deterministic embeddings - * using cryptographic hashing. These are not semantic vectors but can be used - * for exact matches when no other providers are available. - */ -export class LocalEmbeddingProvider extends BaseEmbeddingProvider { - override name = "local"; - - constructor(config: EmbeddingConfig) { - super(config); - } - - /** - * Generate a simple embedding by hashing the text - */ - async generateEmbeddings(text: string): Promise { - const dimension = this.config.dimension || 384; - const result = new Float32Array(dimension); - - // Generate a hash of the input text - const hash = crypto.createHash('sha256').update(text).digest(); - - // Use the hash to seed a deterministic PRNG - let seed = 0; - for (let i = 0; i < hash.length; i += 4) { - seed = (seed * 65536 + hash.readUInt32LE(i % (hash.length - 3))) >>> 0; - } - - // Generate pseudo-random but deterministic values for the embedding - for (let i = 0; i < dimension; i++) { - // Generate next pseudo-random number - seed = (seed * 1664525 + 1013904223) >>> 0; - - // Convert to a float between -1 and 1 - result[i] = (seed / 2147483648) - 1; - } - - // Normalize the vector - let magnitude = 0; - for (let i = 0; i < dimension; i++) { - magnitude += result[i] * result[i]; - } - - magnitude = Math.sqrt(magnitude); - if (magnitude > 0) { - for (let i = 0; i < dimension; i++) { - result[i] /= magnitude; - } - } - - return result; - } - - /** - * Generate embeddings for multiple texts - */ - override async generateBatchEmbeddings(texts: string[]): Promise { - const results: Float32Array[] = []; - - for (const text of texts) { - const embedding = await this.generateEmbeddings(text); - results.push(embedding); - } - - return results; - } -} diff --git a/apps/server/src/services/llm/embeddings/providers/ollama.ts b/apps/server/src/services/llm/embeddings/providers/ollama.ts deleted file mode 100644 index fab2f7f9d3..0000000000 --- a/apps/server/src/services/llm/embeddings/providers/ollama.ts +++ /dev/null @@ -1,324 +0,0 @@ -import log from "../../../log.js"; -import { BaseEmbeddingProvider } from "../base_embeddings.js"; -import type { EmbeddingConfig } from "../embeddings_interface.js"; -import { NormalizationStatus } from "../embeddings_interface.js"; -import { LLM_CONSTANTS } from "../../constants/provider_constants.js"; -import type { EmbeddingModelInfo } from "../../interfaces/embedding_interfaces.js"; -import { Ollama } from "ollama"; - -/** - * Ollama embedding provider implementation using the official Ollama client - */ -export class OllamaEmbeddingProvider extends BaseEmbeddingProvider { - override name = "ollama"; - private client: Ollama | null = null; - - constructor(config: EmbeddingConfig) { - super(config); - } - - /** - * Get the Ollama client instance - */ - private getClient(): Ollama { - if (!this.client) { - this.client = new Ollama({ host: this.baseUrl }); - } - return this.client; - } - - /** - * Initialize the provider by detecting model capabilities - */ - override async initialize(): Promise { - const modelName = this.config.model || "llama3"; - try { - // Detect model capabilities - const modelInfo = await this.getModelInfo(modelName); - - // Update the config dimension - this.config.dimension = modelInfo.dimension; - - log.info(`Ollama model ${modelName} initialized with dimension ${this.config.dimension} and context window ${modelInfo.contextWidth}`); - } catch (error: any) { - log.error(`Error initializing Ollama provider: ${error.message}`); - } - } - - /** - * Fetch detailed model information from Ollama API - * @param modelName The name of the model to fetch information for - */ - private async fetchModelCapabilities(modelName: string): Promise { - try { - const client = this.getClient(); - - // Get model info using the client's show method - const modelData = await client.show({ model: modelName }); - - if (modelData && modelData.parameters) { - const params = modelData.parameters as any; - // Extract context length from parameters (different models might use different parameter names) - const contextWindow = params.context_length || - params.num_ctx || - params.context_window || - (LLM_CONSTANTS.OLLAMA_MODEL_CONTEXT_WINDOWS as Record).default; - - // Some models might provide embedding dimensions - const embeddingDimension = params.embedding_length || params.dim || null; - - log.info(`Fetched Ollama model info for ${modelName}: context window ${contextWindow}`); - - return { - name: modelName, - dimension: embeddingDimension || 0, // We'll detect this separately if not provided - contextWidth: contextWindow, - type: 'float32' - }; - } - } catch (error: any) { - log.info(`Could not fetch model info from Ollama API: ${error.message}. Will try embedding test.`); - // We'll fall back to embedding test if this fails - } - - return null; - } - - /** - * Get model information by probing the API - */ - async getModelInfo(modelName: string): Promise { - // Check cache first - if (this.modelInfoCache.has(modelName)) { - return this.modelInfoCache.get(modelName)!; - } - - // Try to fetch model capabilities from API - const apiModelInfo = await this.fetchModelCapabilities(modelName); - if (apiModelInfo) { - // If we have context window but no embedding dimension, we need to detect the dimension - if (apiModelInfo.contextWidth && !apiModelInfo.dimension) { - try { - // Detect dimension with a test embedding - const dimension = await this.detectEmbeddingDimension(modelName); - apiModelInfo.dimension = dimension; - } catch (error) { - // If dimension detection fails, fall back to defaults - const baseModelName = modelName.split(':')[0]; - apiModelInfo.dimension = (LLM_CONSTANTS.OLLAMA_MODEL_DIMENSIONS as Record)[baseModelName] || - (LLM_CONSTANTS.OLLAMA_MODEL_DIMENSIONS as Record).default; - } - } - - // Cache and return the API-provided info - this.modelInfoCache.set(modelName, apiModelInfo); - this.config.dimension = apiModelInfo.dimension; - return apiModelInfo; - } - - // If API info fetch fails, fall back to test embedding - try { - const dimension = await this.detectEmbeddingDimension(modelName); - const baseModelName = modelName.split(':')[0]; - const contextWindow = (LLM_CONSTANTS.OLLAMA_MODEL_CONTEXT_WINDOWS as Record)[baseModelName] || - (LLM_CONSTANTS.OLLAMA_MODEL_CONTEXT_WINDOWS as Record).default; - - const modelInfo: EmbeddingModelInfo = { - name: modelName, - dimension, - contextWidth: contextWindow, - type: 'float32' - }; - this.modelInfoCache.set(modelName, modelInfo); - this.config.dimension = dimension; - - log.info(`Detected Ollama model ${modelName} with dimension ${dimension} (context: ${contextWindow})`); - return modelInfo; - } catch (error: any) { - log.error(`Error detecting Ollama model capabilities: ${error.message}`); - - // If all detection fails, use defaults based on model name - const baseModelName = modelName.split(':')[0]; - const dimension = (LLM_CONSTANTS.OLLAMA_MODEL_DIMENSIONS as Record)[baseModelName] || - (LLM_CONSTANTS.OLLAMA_MODEL_DIMENSIONS as Record).default; - const contextWindow = (LLM_CONSTANTS.OLLAMA_MODEL_CONTEXT_WINDOWS as Record)[baseModelName] || - (LLM_CONSTANTS.OLLAMA_MODEL_CONTEXT_WINDOWS as Record).default; - - log.info(`Using default parameters for model ${modelName}: dimension ${dimension}, context ${contextWindow}`); - - const modelInfo: EmbeddingModelInfo = { - name: modelName, - dimension, - contextWidth: contextWindow, - type: 'float32' - }; - this.modelInfoCache.set(modelName, modelInfo); - this.config.dimension = dimension; - - return modelInfo; - } - } - - /** - * Detect embedding dimension by making a test API call - */ - private async detectEmbeddingDimension(modelName: string): Promise { - try { - const client = this.getClient(); - const embedResponse = await client.embeddings({ - model: modelName, - prompt: "Test" - }); - - if (embedResponse && Array.isArray(embedResponse.embedding)) { - return embedResponse.embedding.length; - } else { - throw new Error("Could not detect embedding dimensions"); - } - } catch (error) { - throw new Error(`Failed to detect embedding dimensions: ${error}`); - } - } - - /** - * Get the current embedding dimension - */ - override getDimension(): number { - return this.config.dimension; - } - - /** - * Generate embeddings for a single text - */ - async generateEmbeddings(text: string): Promise { - // Handle empty text - if (!text.trim()) { - return new Float32Array(this.config.dimension); - } - - // Configuration for retries - const maxRetries = 3; - let retryCount = 0; - let lastError: any = null; - - while (retryCount <= maxRetries) { - try { - const modelName = this.config.model || "llama3"; - - // Ensure we have model info - const modelInfo = await this.getModelInfo(modelName); - - // Trim text if it might exceed context window (rough character estimate) - // This is a simplistic approach - ideally we'd count tokens properly - const charLimit = (modelInfo.contextWidth || 8192) * 4; // Rough estimate: avg 4 chars per token - const trimmedText = text.length > charLimit ? text.substring(0, charLimit) : text; - - const client = this.getClient(); - const response = await client.embeddings({ - model: modelName, - prompt: trimmedText - }); - - if (response && Array.isArray(response.embedding)) { - // Success! Return the embedding - return new Float32Array(response.embedding); - } else { - throw new Error("Unexpected response structure from Ollama API"); - } - } catch (error: any) { - lastError = error; - // Only retry on timeout or connection errors - const errorMessage = error.message || "Unknown error"; - const isTimeoutError = errorMessage.includes('timeout') || - errorMessage.includes('socket hang up') || - errorMessage.includes('ECONNREFUSED') || - errorMessage.includes('ECONNRESET') || - errorMessage.includes('AbortError') || - errorMessage.includes('NetworkError'); - - if (isTimeoutError && retryCount < maxRetries) { - // Exponential backoff with jitter - const delay = Math.min(Math.pow(2, retryCount) * 1000 + Math.random() * 1000, 15000); - log.info(`Ollama embedding timeout, retrying in ${Math.round(delay/1000)}s (attempt ${retryCount + 1}/${maxRetries})`); - await new Promise(resolve => setTimeout(resolve, delay)); - retryCount++; - } else { - // Non-retryable error or max retries exceeded - const errorMessage = error.message || "Unknown error"; - log.error(`Ollama embedding error: ${errorMessage}`); - throw new Error(`Ollama embedding error: ${errorMessage}`); - } - } - } - - // If we get here, we've exceeded our retry limit - const errorMessage = lastError.message || "Unknown error"; - log.error(`Ollama embedding error after ${maxRetries} retries: ${errorMessage}`); - throw new Error(`Ollama embedding error after ${maxRetries} retries: ${errorMessage}`); - } - - /** - * More specific implementation of batch size error detection for Ollama - */ - protected override isBatchSizeError(error: any): boolean { - const errorMessage = error?.message || ''; - const ollamaBatchSizeErrorPatterns = [ - 'context length', 'token limit', 'out of memory', - 'too large', 'overloaded', 'prompt too long', - 'too many tokens', 'maximum size' - ]; - - return ollamaBatchSizeErrorPatterns.some(pattern => - errorMessage.toLowerCase().includes(pattern.toLowerCase()) - ); - } - - /** - * Generate embeddings for multiple texts - * - * Note: Ollama API doesn't support batch embedding, so we process them sequentially - * but using the adaptive batch processor to handle rate limits and retries - */ - override async generateBatchEmbeddings(texts: string[]): Promise { - if (texts.length === 0) { - return []; - } - - try { - return await this.processWithAdaptiveBatch( - texts, - async (batch) => { - const results: Float32Array[] = []; - - // For Ollama, we have to process one at a time - for (const text of batch) { - // Skip empty texts - if (!text.trim()) { - results.push(new Float32Array(this.config.dimension)); - continue; - } - - const embedding = await this.generateEmbeddings(text); - results.push(embedding); - } - - return results; - }, - this.isBatchSizeError - ); - } - catch (error: any) { - const errorMessage = error.message || "Unknown error"; - log.error(`Ollama batch embedding error: ${errorMessage}`); - throw new Error(`Ollama batch embedding error: ${errorMessage}`); - } - } - - /** - * Returns the normalization status for Ollama embeddings - * Ollama embeddings are not guaranteed to be normalized - */ - override getNormalizationStatus(): NormalizationStatus { - return NormalizationStatus.NEVER; // Be conservative and always normalize - } -} diff --git a/apps/server/src/services/llm/embeddings/providers/openai.ts b/apps/server/src/services/llm/embeddings/providers/openai.ts deleted file mode 100644 index 22e51fe8b6..0000000000 --- a/apps/server/src/services/llm/embeddings/providers/openai.ts +++ /dev/null @@ -1,318 +0,0 @@ -import log from "../../../log.js"; -import { BaseEmbeddingProvider } from "../base_embeddings.js"; -import type { EmbeddingConfig } from "../embeddings_interface.js"; -import { NormalizationStatus } from "../embeddings_interface.js"; -import { LLM_CONSTANTS } from "../../constants/provider_constants.js"; -import type { EmbeddingModelInfo } from "../../interfaces/embedding_interfaces.js"; -import OpenAI from "openai"; -import { PROVIDER_EMBEDDING_CAPABILITIES } from '../../constants/search_constants.js'; - -/** - * OpenAI embedding provider implementation using the official SDK - */ -export class OpenAIEmbeddingProvider extends BaseEmbeddingProvider { - override name = "openai"; - private client: OpenAI | null = null; - - constructor(config: EmbeddingConfig) { - super(config); - this.initClient(); - } - - /** - * Initialize the OpenAI client - */ - private initClient() { - if (this.apiKey) { - this.client = new OpenAI({ - apiKey: this.apiKey, - baseURL: this.baseUrl - }); - } - } - - /** - * Initialize the provider by detecting model capabilities - */ - override async initialize(): Promise { - const modelName = this.config.model || "text-embedding-3-small"; - try { - // Initialize client if needed - if (!this.client && this.apiKey) { - this.initClient(); - } - - // Detect model capabilities - const modelInfo = await this.getModelInfo(modelName); - - // Update the config dimension - this.config.dimension = modelInfo.dimension; - - log.info(`OpenAI model ${modelName} initialized with dimension ${this.config.dimension} and context window ${modelInfo.contextWidth}`); - } catch (error: any) { - log.error(`Error initializing OpenAI provider: ${error.message}`); - } - } - - /** - * Fetch model information from the OpenAI API - */ - private async fetchModelCapabilities(modelName: string): Promise { - if (!this.client) { - return null; - } - - try { - // Get model details using the SDK - const model = await this.client.models.retrieve(modelName); - - if (model) { - // Different model families may have different ways of exposing context window - let contextWindow = 0; - let dimension = 0; - - // Extract context window if available from the response - const modelData = model as any; - - if (modelData.context_window) { - contextWindow = modelData.context_window; - } else if (modelData.limits && modelData.limits.context_window) { - contextWindow = modelData.limits.context_window; - } else if (modelData.limits && modelData.limits.context_length) { - contextWindow = modelData.limits.context_length; - } - - // Extract embedding dimensions if available - if (modelData.dimensions) { - dimension = modelData.dimensions; - } else if (modelData.embedding_dimension) { - dimension = modelData.embedding_dimension; - } - - // If we didn't get all the info, use defaults for missing values - if (!contextWindow) { - // Set contextWindow based on model name patterns - if (modelName.includes('embedding-3')) { - contextWindow = PROVIDER_EMBEDDING_CAPABILITIES.OPENAI.MODELS['text-embedding-3-small'].contextWindow; - } else { - contextWindow = PROVIDER_EMBEDDING_CAPABILITIES.OPENAI.MODELS.default.contextWindow; - } - } - - if (!dimension) { - // Set default dimensions based on model name patterns - if (modelName.includes('ada') || modelName.includes('embedding-ada')) { - dimension = LLM_CONSTANTS.EMBEDDING_DIMENSIONS.OPENAI.ADA; - } else if (modelName.includes('embedding-3-small')) { - dimension = PROVIDER_EMBEDDING_CAPABILITIES.OPENAI.MODELS['text-embedding-3-small'].dimension; - } else if (modelName.includes('embedding-3-large')) { - dimension = PROVIDER_EMBEDDING_CAPABILITIES.OPENAI.MODELS['text-embedding-3-large'].dimension; - } else { - dimension = PROVIDER_EMBEDDING_CAPABILITIES.OPENAI.MODELS.default.dimension; - } - } - - log.info(`Fetched OpenAI model info for ${modelName}: context window ${contextWindow}, dimension ${dimension}`); - - return { - name: modelName, - dimension, - contextWidth: contextWindow, - type: 'float32' - }; - } - } catch (error: any) { - log.info(`Could not fetch model info from OpenAI API: ${error.message}. Will try embedding test.`); - } - - return null; - } - - /** - * Get model information including embedding dimensions - */ - async getModelInfo(modelName: string): Promise { - // Check cache first - if (this.modelInfoCache.has(modelName)) { - return this.modelInfoCache.get(modelName)!; - } - - // Try to fetch model capabilities from API - const apiModelInfo = await this.fetchModelCapabilities(modelName); - if (apiModelInfo) { - // Cache and return the API-provided info - this.modelInfoCache.set(modelName, apiModelInfo); - this.config.dimension = apiModelInfo.dimension; - return apiModelInfo; - } - - // If API info fetch fails, try to detect embedding dimension with a test call - try { - const testEmbedding = await this.generateEmbeddings("Test"); - const dimension = testEmbedding.length; - - // Use default context window - let contextWindow = PROVIDER_EMBEDDING_CAPABILITIES.OPENAI.MODELS.default.contextWindow; - - const modelInfo: EmbeddingModelInfo = { - name: modelName, - dimension, - contextWidth: contextWindow, - type: 'float32' - }; - this.modelInfoCache.set(modelName, modelInfo); - this.config.dimension = dimension; - - log.info(`Detected OpenAI model ${modelName} with dimension ${dimension} (context: ${contextWindow})`); - return modelInfo; - } catch (error: any) { - // If detection fails, use defaults - const dimension = PROVIDER_EMBEDDING_CAPABILITIES.OPENAI.MODELS.default.dimension; - const contextWindow = PROVIDER_EMBEDDING_CAPABILITIES.OPENAI.MODELS.default.contextWindow; - - log.info(`Using default parameters for OpenAI model ${modelName}: dimension ${dimension}, context ${contextWindow}`); - - const modelInfo: EmbeddingModelInfo = { - name: modelName, - dimension, - contextWidth: contextWindow, - type: 'float32' - }; - this.modelInfoCache.set(modelName, modelInfo); - this.config.dimension = dimension; - - return modelInfo; - } - } - - /** - * Generate embeddings for a single text - */ - async generateEmbeddings(text: string): Promise { - try { - if (!text.trim()) { - return new Float32Array(this.config.dimension); - } - - if (!this.client) { - this.initClient(); - if (!this.client) { - throw new Error("OpenAI client initialization failed"); - } - } - - const response = await this.client.embeddings.create({ - model: this.config.model || "text-embedding-3-small", - input: text, - encoding_format: "float" - }); - - if (response && response.data && response.data[0] && response.data[0].embedding) { - return new Float32Array(response.data[0].embedding); - } else { - throw new Error("Unexpected response structure from OpenAI API"); - } - } catch (error: any) { - const errorMessage = error.message || "Unknown error"; - log.error(`OpenAI embedding error: ${errorMessage}`); - throw new Error(`OpenAI embedding error: ${errorMessage}`); - } - } - - /** - * More specific implementation of batch size error detection for OpenAI - */ - protected override isBatchSizeError(error: any): boolean { - const errorMessage = error?.message || ''; - const openAIBatchSizeErrorPatterns = [ - 'batch size', 'too many inputs', 'context length exceeded', - 'maximum context length', 'token limit', 'rate limit exceeded', - 'tokens in the messages', 'reduce the length', 'too long' - ]; - - return openAIBatchSizeErrorPatterns.some(pattern => - errorMessage.toLowerCase().includes(pattern.toLowerCase()) - ); - } - - /** - * Custom implementation for batched OpenAI embeddings - */ - async generateBatchEmbeddingsWithAPI(texts: string[]): Promise { - if (texts.length === 0) { - return []; - } - - if (!this.client) { - this.initClient(); - if (!this.client) { - throw new Error("OpenAI client initialization failed"); - } - } - - const response = await this.client.embeddings.create({ - model: this.config.model || "text-embedding-3-small", - input: texts, - encoding_format: "float" - }); - - if (response && response.data) { - // Sort the embeddings by index to ensure they match the input order - const sortedEmbeddings = response.data - .sort((a, b) => a.index - b.index) - .map(item => new Float32Array(item.embedding)); - - return sortedEmbeddings; - } else { - throw new Error("Unexpected response structure from OpenAI API"); - } - } - - /** - * Generate embeddings for multiple texts in a single batch - * OpenAI API supports batch embedding, so we implement a custom version - */ - override async generateBatchEmbeddings(texts: string[]): Promise { - if (texts.length === 0) { - return []; - } - - try { - return await this.processWithAdaptiveBatch( - texts, - async (batch) => { - // Filter out empty texts and use the API batch functionality - const filteredBatch = batch.filter(text => text.trim().length > 0); - - if (filteredBatch.length === 0) { - // If all texts are empty after filtering, return empty embeddings - return batch.map(() => new Float32Array(this.config.dimension)); - } - - if (filteredBatch.length === 1) { - // If only one text, use the single embedding endpoint - const embedding = await this.generateEmbeddings(filteredBatch[0]); - return [embedding]; - } - - // Use the batch API endpoint - return this.generateBatchEmbeddingsWithAPI(filteredBatch); - }, - this.isBatchSizeError - ); - } - catch (error: any) { - const errorMessage = error.message || "Unknown error"; - log.error(`OpenAI batch embedding error: ${errorMessage}`); - throw new Error(`OpenAI batch embedding error: ${errorMessage}`); - } - } - - /** - * Returns the normalization status for OpenAI embeddings - * OpenAI embeddings are guaranteed to be normalized to unit length - */ - override getNormalizationStatus(): NormalizationStatus { - return NormalizationStatus.GUARANTEED; - } -} diff --git a/apps/server/src/services/llm/embeddings/providers/voyage.ts b/apps/server/src/services/llm/embeddings/providers/voyage.ts deleted file mode 100644 index 8572cbf6ae..0000000000 --- a/apps/server/src/services/llm/embeddings/providers/voyage.ts +++ /dev/null @@ -1,285 +0,0 @@ -import log from "../../../log.js"; -import { BaseEmbeddingProvider } from "../base_embeddings.js"; -import type { EmbeddingConfig } from "../embeddings_interface.js"; -import { NormalizationStatus } from "../embeddings_interface.js"; -import { LLM_CONSTANTS } from "../../constants/provider_constants.js"; -import { PROVIDER_EMBEDDING_CAPABILITIES } from "../../constants/search_constants.js"; -import type { EmbeddingModelInfo } from "../../interfaces/embedding_interfaces.js"; - -// Use constants from the central constants file -const VOYAGE_MODEL_CONTEXT_WINDOWS = PROVIDER_EMBEDDING_CAPABILITIES.VOYAGE.MODELS; -const VOYAGE_MODEL_DIMENSIONS = Object.entries(PROVIDER_EMBEDDING_CAPABILITIES.VOYAGE.MODELS).reduce((acc, [key, value]) => { - acc[key] = value.dimension; - return acc; -}, {} as Record); - -/** - * Voyage AI embedding provider implementation - */ -export class VoyageEmbeddingProvider extends BaseEmbeddingProvider { - override name = "voyage"; - - constructor(config: EmbeddingConfig) { - super(config); - - // Set default base URL if not provided - if (!this.baseUrl) { - this.baseUrl = "https://api.voyageai.com/v1"; - } - } - - /** - * Initialize the provider by detecting model capabilities - */ - override async initialize(): Promise { - const modelName = this.config.model || "voyage-2"; - try { - // Detect model capabilities - const modelInfo = await this.getModelInfo(modelName); - - // Update the config dimension - this.config.dimension = modelInfo.dimension; - - log.info(`Voyage AI model ${modelName} initialized with dimension ${this.config.dimension} and context window ${modelInfo.contextWidth}`); - } catch (error: any) { - log.error(`Error initializing Voyage AI provider: ${error.message}`); - } - } - - /** - * Try to determine Voyage AI model capabilities - */ - private async fetchModelCapabilities(modelName: string): Promise { - try { - // Find the closest matching model - const modelMapKey = Object.keys(PROVIDER_EMBEDDING_CAPABILITIES.VOYAGE.MODELS).find( - model => modelName.startsWith(model) - ) || "default"; - - // Use as keyof to tell TypeScript this is a valid key - const modelInfo = PROVIDER_EMBEDDING_CAPABILITIES.VOYAGE.MODELS[modelMapKey as keyof typeof PROVIDER_EMBEDDING_CAPABILITIES.VOYAGE.MODELS]; - - return { - dimension: modelInfo.dimension, - contextWidth: modelInfo.contextWidth, - name: modelName, - type: 'float32' - }; - } catch (error) { - log.info(`Could not determine capabilities for Voyage AI model ${modelName}: ${error}`); - return null; - } - } - - /** - * Get model information including embedding dimensions - */ - async getModelInfo(modelName: string): Promise { - // Check cache first - if (this.modelInfoCache.has(modelName)) { - return this.modelInfoCache.get(modelName)!; - } - - // Try to determine model capabilities - const capabilities = await this.fetchModelCapabilities(modelName); - const defaults = PROVIDER_EMBEDDING_CAPABILITIES.VOYAGE.MODELS.default; - const contextWindow = capabilities?.contextWidth || defaults.contextWidth; - const knownDimension = capabilities?.dimension || defaults.dimension; - - // For Voyage, we can use known dimensions or detect with a test call - try { - if (knownDimension) { - // Use known dimension - const modelInfo: EmbeddingModelInfo = { - dimension: knownDimension, - contextWidth: contextWindow, - name: modelName, - type: 'float32' - }; - - this.modelInfoCache.set(modelName, modelInfo); - this.config.dimension = knownDimension; - - log.info(`Using known parameters for Voyage AI model ${modelName}: dimension ${knownDimension}, context ${contextWindow}`); - return modelInfo; - } else { - // Detect dimension with a test embedding as fallback - const testEmbedding = await this.generateEmbeddings("Test"); - const dimension = testEmbedding.length; - - // Set model info based on the model name, detected dimension, and reasonable defaults - if (modelName.includes('voyage-2')) { - return { - dimension: dimension || 1024, - contextWidth: 8192, - name: modelName, - type: 'float32' - }; - } else if (modelName.includes('voyage-lite-02')) { - return { - dimension: dimension || 768, - contextWidth: 8192, - name: modelName, - type: 'float32' - }; - } else { - // Default for other Voyage models - return { - dimension: dimension || 1024, - contextWidth: 8192, - name: modelName, - type: 'float32' - }; - } - } - } catch (error: any) { - log.info(`Could not fetch model info from Voyage AI API: ${error.message}. Using defaults.`); - - // Use default parameters if everything else fails - const defaultModelInfo: EmbeddingModelInfo = { - dimension: 1024, // Default for Voyage models - contextWidth: 8192, - name: modelName, - type: 'float32' - }; - - this.modelInfoCache.set(modelName, defaultModelInfo); - this.config.dimension = defaultModelInfo.dimension; - return defaultModelInfo; - } - } - - /** - * Generate embeddings for a single text - */ - async generateEmbeddings(text: string): Promise { - try { - if (!text.trim()) { - return new Float32Array(this.config.dimension); - } - - // Get model info to check context window - const modelName = this.config.model || "voyage-2"; - const modelInfo = await this.getModelInfo(modelName); - - // Trim text if it might exceed context window (rough character estimate) - const charLimit = (modelInfo.contextWidth || PROVIDER_EMBEDDING_CAPABILITIES.VOYAGE.MODELS.default.contextWidth) * 4; // Rough estimate: avg 4 chars per token - const trimmedText = text.length > charLimit ? text.substring(0, charLimit) : text; - - const response = await fetch(`${this.baseUrl}/embeddings`, { - method: 'POST', - headers: { - "Content-Type": "application/json", - "Authorization": `Bearer ${this.apiKey}` - }, - body: JSON.stringify({ - model: modelName, - input: trimmedText, - input_type: "text", - truncation: true - }) - }); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.error?.message || `HTTP error ${response.status}`); - } - - const data = await response.json(); - if (data && data.data && data.data[0] && data.data[0].embedding) { - return new Float32Array(data.data[0].embedding); - } else { - throw new Error("Unexpected response structure from Voyage AI API"); - } - } catch (error: any) { - const errorMessage = error.message || "Unknown error"; - log.error(`Voyage AI embedding error: ${errorMessage}`); - throw new Error(`Voyage AI embedding error: ${errorMessage}`); - } - } - - /** - * More specific implementation of batch size error detection for Voyage AI - */ - protected override isBatchSizeError(error: any): boolean { - const errorMessage = error?.message || ''; - const voyageBatchSizeErrorPatterns = [ - 'batch size', 'too many inputs', 'context length exceeded', - 'token limit', 'rate limit', 'limit exceeded', - 'too long', 'request too large', 'content too large' - ]; - - return voyageBatchSizeErrorPatterns.some(pattern => - errorMessage.toLowerCase().includes(pattern.toLowerCase()) - ); - } - - /** - * Generate embeddings for multiple texts in a single batch - */ - override async generateBatchEmbeddings(texts: string[]): Promise { - if (texts.length === 0) { - return []; - } - - try { - return await this.processWithAdaptiveBatch( - texts, - async (batch) => { - if (batch.length === 0) return []; - if (batch.length === 1) { - return [await this.generateEmbeddings(batch[0])]; - } - - // For Voyage AI, we can batch embeddings - const modelName = this.config.model || "voyage-2"; - - // Filter out empty texts - const validBatch = batch.map(text => text.trim() || " "); - - const response = await fetch(`${this.baseUrl}/embeddings`, { - method: 'POST', - headers: { - "Content-Type": "application/json", - "Authorization": `Bearer ${this.apiKey}` - }, - body: JSON.stringify({ - model: modelName, - input: validBatch, - input_type: "text", - truncation: true - }) - }); - - if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - throw new Error(errorData.error?.message || `HTTP error ${response.status}`); - } - - const data = await response.json(); - if (data && data.data && Array.isArray(data.data)) { - return data.data.map((item: any) => - new Float32Array(item.embedding || []) - ); - } else { - throw new Error("Unexpected response structure from Voyage AI batch API"); - } - }, - this.isBatchSizeError - ); - } - catch (error: any) { - const errorMessage = error.message || "Unknown error"; - log.error(`Voyage AI batch embedding error: ${errorMessage}`); - throw new Error(`Voyage AI batch embedding error: ${errorMessage}`); - } - } - - /** - * Returns the normalization status for Voyage embeddings - * Voyage embeddings are generally normalized by the API - */ - override getNormalizationStatus(): NormalizationStatus { - return NormalizationStatus.GUARANTEED; - } -} diff --git a/apps/server/src/services/llm/embeddings/queue.ts b/apps/server/src/services/llm/embeddings/queue.ts deleted file mode 100644 index b002513c53..0000000000 --- a/apps/server/src/services/llm/embeddings/queue.ts +++ /dev/null @@ -1,397 +0,0 @@ -import sql from "../../../services/sql.js"; -import dateUtils from "../../../services/date_utils.js"; -import log from "../../../services/log.js"; -import becca from "../../../becca/becca.js"; -import options from "../../../services/options.js"; -import { getEnabledEmbeddingProviders } from "../providers/providers.js"; -import { getNoteEmbeddingContext } from "./content_processing.js"; -import { deleteNoteEmbeddings } from "./storage.js"; -import type { QueueItem } from "./types.js"; -import { getChunkingOperations } from "./chunking/chunking_interface.js"; -import indexService from '../index_service.js'; -import { isNoteExcludedFromAIById } from "../utils/ai_exclusion_utils.js"; - -// Track which notes are currently being processed -const notesInProcess = new Set(); - -interface FailedItemRow { - noteId: string; - operation: string; - attempts: number; - lastAttempt: string; - error: string | null; - failed: number; -} - -interface FailedItemWithTitle extends FailedItemRow { - title?: string; - failureType: 'chunks' | 'full'; - isPermanent: boolean; -} - -/** - * Queues a note for embedding update - */ -export async function queueNoteForEmbedding(noteId: string, operation = 'UPDATE') { - const now = dateUtils.localNowDateTime(); - const utcNow = dateUtils.utcNowDateTime(); - - try { - // Check if note is already in queue and whether it's marked as permanently failed - const queueInfo = await sql.getRow( - "SELECT 1 as exists_flag, failed, isProcessing FROM embedding_queue WHERE noteId = ?", - [noteId] - ) as {exists_flag: number, failed: number, isProcessing: number} | null; - - if (queueInfo) { - // If the note is currently being processed, don't change its status - if (queueInfo.isProcessing === 1) { - log.info(`Note ${noteId} is currently being processed, skipping queue update`); - return; - } - - // Only update if not permanently failed - if (queueInfo.failed !== 1) { - // Update existing queue entry but preserve the failed status - await sql.execute(` - UPDATE embedding_queue - SET operation = ?, dateQueued = ?, utcDateQueued = ?, attempts = 0, error = NULL - WHERE noteId = ?`, - [operation, now, utcNow, noteId] - ); - } else { - // Note is marked as permanently failed, don't update - log.info(`Note ${noteId} is marked as permanently failed, skipping automatic re-queue`); - } - } else { - // Add new queue entry - await sql.execute(` - INSERT INTO embedding_queue - (noteId, operation, dateQueued, utcDateQueued, failed, isProcessing) - VALUES (?, ?, ?, ?, 0, 0)`, - [noteId, operation, now, utcNow] - ); - } - } catch (error: any) { - // If there's a race condition where multiple events try to queue the same note simultaneously, - // one of them will succeed and others will fail with UNIQUE constraint violation. - // We can safely ignore this specific error since the note is already queued. - if (error.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' && error.message.includes('UNIQUE constraint failed: embedding_queue.noteId')) { - log.info(`Note ${noteId} was already queued by another process, ignoring duplicate queue request`); - return; - } - // Rethrow any other errors - throw error; - } -} - -/** - * Get notes that have failed embedding generation - * - * @param limit - Maximum number of failed notes to return - * @returns List of failed notes with their error information - */ -export async function getFailedEmbeddingNotes(limit: number = 100): Promise { - // Get notes with failed embedding attempts or permanently failed flag - const failedQueueItems = sql.getRows(` - SELECT noteId, operation, attempts, lastAttempt, error, failed - FROM embedding_queue - WHERE attempts > 0 OR failed = 1 - ORDER BY failed DESC, attempts DESC, lastAttempt DESC - LIMIT ?`, - [limit] - ); - - // Add titles to the failed notes - const failedNotesWithTitles: FailedItemWithTitle[] = []; - for (const item of failedQueueItems) { - const note = becca.getNote(item.noteId); - if (note) { - // Check if this is a chunking error (contains the word "chunks") - const isChunkFailure = item.error && item.error.toLowerCase().includes('chunk'); - const isPermanentFailure = item.failed === 1; - - failedNotesWithTitles.push({ - ...item, - title: note.title, - failureType: isChunkFailure ? 'chunks' : 'full', - isPermanent: isPermanentFailure - }); - } else { - failedNotesWithTitles.push({ - ...item, - failureType: 'full', - isPermanent: item.failed === 1 - }); - } - } - - // Sort by latest attempt - failedNotesWithTitles.sort((a, b) => { - if (a.lastAttempt && b.lastAttempt) { - return b.lastAttempt.localeCompare(a.lastAttempt); - } - return 0; - }); - - // Limit to the specified number - return failedNotesWithTitles.slice(0, limit); -} - -/** - * Retry a specific failed note embedding - */ -export async function retryFailedEmbedding(noteId: string): Promise { - const now = dateUtils.localNowDateTime(); - const utcNow = dateUtils.utcNowDateTime(); - - // Check if the note is in the embedding queue and has failed or has attempts - const existsInQueue = await sql.getValue( - "SELECT 1 FROM embedding_queue WHERE noteId = ? AND (failed = 1 OR attempts > 0)", - [noteId] - ); - - if (existsInQueue) { - // Reset the note in the queue - await sql.execute(` - UPDATE embedding_queue - SET attempts = 0, error = NULL, failed = 0, dateQueued = ?, utcDateQueued = ?, priority = 10 - WHERE noteId = ?`, - [now, utcNow, noteId] - ); - return true; - } - - return false; -} - -/** - * Retry all failed embeddings - * - * @returns Number of notes queued for retry - */ -export async function retryAllFailedEmbeddings(): Promise { - const now = dateUtils.localNowDateTime(); - const utcNow = dateUtils.utcNowDateTime(); - - // Get count of all failed notes in queue (either with failed=1 or attempts>0) - const failedCount = await sql.getValue( - "SELECT COUNT(*) FROM embedding_queue WHERE failed = 1 OR attempts > 0" - ) as number; - - if (failedCount > 0) { - // Reset all failed notes in the queue - await sql.execute(` - UPDATE embedding_queue - SET attempts = 0, error = NULL, failed = 0, dateQueued = ?, utcDateQueued = ?, priority = 10 - WHERE failed = 1 OR attempts > 0`, - [now, utcNow] - ); - } - - return failedCount; -} - -/** - * Process the embedding queue - */ -export async function processEmbeddingQueue() { - if (!(await options.getOptionBool('aiEnabled'))) { - return; - } - - // Check if this instance should process embeddings - const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; - const isSyncServer = await indexService.isSyncServerForEmbeddings(); - const shouldProcessEmbeddings = embeddingLocation === 'client' || isSyncServer; - - if (!shouldProcessEmbeddings) { - // This instance is not configured to process embeddings - return; - } - - const batchSize = parseInt(await options.getOption('embeddingBatchSize') || '10', 10); - const enabledProviders = await getEnabledEmbeddingProviders(); - - if (enabledProviders.length === 0) { - return; - } - - // Get notes from queue (excluding failed ones and those being processed) - const notes = await sql.getRows(` - SELECT noteId, operation, attempts - FROM embedding_queue - WHERE failed = 0 AND isProcessing = 0 - ORDER BY priority DESC, utcDateQueued ASC - LIMIT ?`, - [batchSize] - ); - - if (notes.length === 0) { - return; - } - - // Track successfully processed notes count for progress reporting - let processedCount = 0; - - for (const note of notes) { - const noteData = note as unknown as QueueItem; - const noteId = noteData.noteId; - - // Double-check that this note isn't already being processed - if (notesInProcess.has(noteId)) { - //log.info(`Note ${noteId} is already being processed by another thread, skipping`); - continue; - } - - try { - // Mark the note as being processed - notesInProcess.add(noteId); - await sql.execute( - "UPDATE embedding_queue SET isProcessing = 1 WHERE noteId = ?", - [noteId] - ); - - // Skip if note no longer exists - if (!becca.getNote(noteId)) { - await sql.execute( - "DELETE FROM embedding_queue WHERE noteId = ?", - [noteId] - ); - await deleteNoteEmbeddings(noteId); - continue; - } - - // Check if this note is excluded from AI features - if (isNoteExcludedFromAIById(noteId)) { - log.info(`Note ${noteId} excluded from AI features, removing from embedding queue`); - await sql.execute( - "DELETE FROM embedding_queue WHERE noteId = ?", - [noteId] - ); - await deleteNoteEmbeddings(noteId); // Also remove any existing embeddings - continue; - } - - if (noteData.operation === 'DELETE') { - await deleteNoteEmbeddings(noteId); - await sql.execute( - "DELETE FROM embedding_queue WHERE noteId = ?", - [noteId] - ); - continue; - } - - - // Get note context for embedding - const context = await getNoteEmbeddingContext(noteId); - - // Check if we should use chunking for large content - const useChunking = context.content.length > 5000; - - // Track provider successes and failures - let allProvidersFailed = true; - let allProvidersSucceeded = true; - - // Process with each enabled provider - for (const provider of enabledProviders) { - try { - if (useChunking) { - // Process large notes using chunking - const chunkingOps = await getChunkingOperations(); - await chunkingOps.processNoteWithChunking(noteId, provider, context); - allProvidersFailed = false; - } else { - // Standard approach: Generate a single embedding for the whole note - const embedding = await provider.generateNoteEmbeddings(context); - - // Store embedding - const config = provider.getConfig(); - await import('./storage.js').then(storage => { - return storage.storeNoteEmbedding( - noteId, - provider.name, - config.model, - embedding - ); - }); - - // At least one provider succeeded - allProvidersFailed = false; - } - } catch (providerError: any) { - // This provider failed - allProvidersSucceeded = false; - log.error(`Error generating embedding with provider ${provider.name} for note ${noteId}: ${providerError.message || 'Unknown error'}`); - } - } - - if (!allProvidersFailed) { - // At least one provider succeeded, remove from queue - await sql.execute( - "DELETE FROM embedding_queue WHERE noteId = ?", - [noteId] - ); - - // Count as successfully processed - processedCount++; - } else { - // If all providers failed, mark as failed but keep in queue - await sql.execute(` - UPDATE embedding_queue - SET attempts = attempts + 1, - lastAttempt = ?, - error = ?, - isProcessing = 0 - WHERE noteId = ?`, - [dateUtils.utcNowDateTime(), "All providers failed to generate embeddings", noteId] - ); - - // Mark as permanently failed if too many attempts - if (noteData.attempts + 1 >= 3) { - log.error(`Marked note ${noteId} as permanently failed after multiple embedding attempts`); - - // Set the failed flag but keep the actual attempts count - await sql.execute(` - UPDATE embedding_queue - SET failed = 1 - WHERE noteId = ? - `, [noteId]); - } - } - } catch (error: any) { - // Update attempt count and log error - await sql.execute(` - UPDATE embedding_queue - SET attempts = attempts + 1, - lastAttempt = ?, - error = ?, - isProcessing = 0 - WHERE noteId = ?`, - [dateUtils.utcNowDateTime(), error.message || 'Unknown error', noteId] - ); - - log.error(`Error processing embedding for note ${noteId}: ${error.message || 'Unknown error'}`); - - // Mark as permanently failed if too many attempts - if (noteData.attempts + 1 >= 3) { - log.error(`Marked note ${noteId} as permanently failed after multiple embedding attempts`); - - // Set the failed flag but keep the actual attempts count - await sql.execute(` - UPDATE embedding_queue - SET failed = 1 - WHERE noteId = ? - `, [noteId]); - } - } finally { - // Always clean up the processing status in the in-memory set - notesInProcess.delete(noteId); - } - } - - // Update the index rebuild progress if any notes were processed - if (processedCount > 0) { - indexService.updateIndexRebuildProgress(processedCount); - } -} diff --git a/apps/server/src/services/llm/embeddings/stats.ts b/apps/server/src/services/llm/embeddings/stats.ts deleted file mode 100644 index a8b594723c..0000000000 --- a/apps/server/src/services/llm/embeddings/stats.ts +++ /dev/null @@ -1,64 +0,0 @@ -import sql from "../../../services/sql.js"; -import log from "../../../services/log.js"; - -/** - * Get current embedding statistics - */ -export async function getEmbeddingStats() { - const totalNotesCount = await sql.getValue( - "SELECT COUNT(*) FROM notes WHERE isDeleted = 0" - ) as number; - - const embeddedNotesCount = await sql.getValue( - "SELECT COUNT(DISTINCT noteId) FROM note_embeddings" - ) as number; - - const queuedNotesCount = await sql.getValue( - "SELECT COUNT(*) FROM embedding_queue WHERE failed = 0" - ) as number; - - const failedNotesCount = await sql.getValue( - "SELECT COUNT(*) FROM embedding_queue WHERE failed = 1" - ) as number; - - // Get the last processing time by checking the most recent embedding - const lastProcessedDate = await sql.getValue( - "SELECT utcDateCreated FROM note_embeddings ORDER BY utcDateCreated DESC LIMIT 1" - ) as string | null || null; - - // Calculate the actual completion percentage - // When reprocessing, we need to consider notes in the queue as not completed yet - // We calculate the percentage of notes that are embedded and NOT in the queue - - // First, get the count of notes that are both in the embeddings table and queue - const notesInQueueWithEmbeddings = await sql.getValue(` - SELECT COUNT(DISTINCT eq.noteId) - FROM embedding_queue eq - JOIN note_embeddings ne ON eq.noteId = ne.noteId - `) as number; - - // The number of notes with valid, up-to-date embeddings - const upToDateEmbeddings = embeddedNotesCount - notesInQueueWithEmbeddings; - - // Calculate the percentage of notes that are properly embedded - const percentComplete = totalNotesCount > 0 - ? Math.round((upToDateEmbeddings / (totalNotesCount - failedNotesCount)) * 100) - : 0; - - return { - totalNotesCount, - embeddedNotesCount, - queuedNotesCount, - failedNotesCount, - lastProcessedDate, - percentComplete: Math.max(0, Math.min(100, percentComplete)) // Ensure between 0-100 - }; -} - -/** - * Cleanup function to remove stale or unused embeddings - */ -export function cleanupEmbeddings() { - // Implementation can be added later when needed - // For example, removing embeddings for deleted notes, etc. -} diff --git a/apps/server/src/services/llm/embeddings/storage.ts b/apps/server/src/services/llm/embeddings/storage.ts deleted file mode 100644 index bbd1eba9c8..0000000000 --- a/apps/server/src/services/llm/embeddings/storage.ts +++ /dev/null @@ -1,544 +0,0 @@ -import sql from '../../sql.js' -import { randomString } from "../../../services/utils.js"; -import dateUtils from "../../../services/date_utils.js"; -import log from "../../log.js"; -import { embeddingToBuffer, bufferToEmbedding, cosineSimilarity, enhancedCosineSimilarity, selectOptimalEmbedding, adaptEmbeddingDimensions } from "./vector_utils.js"; -import type { EmbeddingResult } from "./types.js"; -import entityChangesService from "../../../services/entity_changes.js"; -import type { EntityChange } from "../../../services/entity_changes_interface.js"; -import { EMBEDDING_CONSTANTS } from "../constants/embedding_constants.js"; -import { SEARCH_CONSTANTS } from '../constants/search_constants.js'; -import type { NoteEmbeddingContext } from "./embeddings_interface.js"; -import becca from "../../../becca/becca.js"; -import { isNoteExcludedFromAIById } from "../utils/ai_exclusion_utils.js"; -import { getSelectedEmbeddingProvider } from '../config/configuration_helpers.js'; - -interface Similarity { - noteId: string; - similarity: number; - contentType: string; - bonuses?: Record; // Optional for debugging -} - -/** - * Creates or updates an embedding for a note - */ -export async function storeNoteEmbedding( - noteId: string, - providerId: string, - modelId: string, - embedding: Float32Array -): Promise { - const dimension = embedding.length; - const embeddingBlob = embeddingToBuffer(embedding); - const now = dateUtils.localNowDateTime(); - const utcNow = dateUtils.utcNowDateTime(); - - // Check if an embedding already exists for this note and provider/model - const existingEmbed = await getEmbeddingForNote(noteId, providerId, modelId); - let embedId; - - if (existingEmbed) { - // Update existing embedding - embedId = existingEmbed.embedId; - await sql.execute(` - UPDATE note_embeddings - SET embedding = ?, dimension = ?, version = version + 1, - dateModified = ?, utcDateModified = ? - WHERE embedId = ?`, - [embeddingBlob, dimension, now, utcNow, embedId] - ); - } else { - // Create new embedding - embedId = randomString(16); - await sql.execute(` - INSERT INTO note_embeddings - (embedId, noteId, providerId, modelId, dimension, embedding, - dateCreated, utcDateCreated, dateModified, utcDateModified) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - [embedId, noteId, providerId, modelId, dimension, embeddingBlob, - now, utcNow, now, utcNow] - ); - } - - // Create entity change record for syncing - interface EmbeddingRow { - embedId: string; - noteId: string; - providerId: string; - modelId: string; - dimension: number; - version: number; - dateCreated: string; - utcDateCreated: string; - dateModified: string; - utcDateModified: string; - } - - const row = await sql.getRow(` - SELECT embedId, noteId, providerId, modelId, dimension, version, - dateCreated, utcDateCreated, dateModified, utcDateModified - FROM note_embeddings - WHERE embedId = ?`, - [embedId] - ); - - if (row) { - // Skip the actual embedding data for the hash since it's large - const ec: EntityChange = { - entityName: "note_embeddings", - entityId: embedId, - hash: `${row.noteId}|${row.providerId}|${row.modelId}|${row.dimension}|${row.version}|${row.utcDateModified}`, - utcDateChanged: row.utcDateModified, - isSynced: true, - isErased: false - }; - - entityChangesService.putEntityChange(ec); - } - - return embedId; -} - -/** - * Retrieves embedding for a specific note - */ -export async function getEmbeddingForNote(noteId: string, providerId: string, modelId: string): Promise { - const row = await sql.getRow(` - SELECT embedId, noteId, providerId, modelId, dimension, embedding, version, - dateCreated, utcDateCreated, dateModified, utcDateModified - FROM note_embeddings - WHERE noteId = ? AND providerId = ? AND modelId = ?`, - [noteId, providerId, modelId] - ); - - if (!row) { - return null; - } - - // Need to cast row to any as it doesn't have type information - const rowData = row as any; - - return { - ...rowData, - embedding: bufferToEmbedding(rowData.embedding, rowData.dimension) - }; -} - -// Create an interface that represents the embedding row from the database -interface EmbeddingRow { - embedId: string; - noteId: string; - providerId: string; - modelId: string; - dimension: number; - embedding: Buffer; - title?: string; - type?: string; - mime?: string; - isDeleted?: number; -} - -// Interface for enhanced embedding with query model information -interface EnhancedEmbeddingRow extends EmbeddingRow { - queryProviderId: string; - queryModelId: string; -} - -/** - * Finds similar notes based on vector similarity - */ -export async function findSimilarNotes( - embedding: Float32Array, - providerId: string, - modelId: string, - limit = SEARCH_CONSTANTS.VECTOR_SEARCH.DEFAULT_MAX_RESULTS, - threshold?: number, // Made optional to use constants - useFallback = true // Whether to try other providers if no embeddings found -): Promise<{noteId: string, similarity: number, contentType?: string}[]> { - // Import constants dynamically to avoid circular dependencies - const llmModule = await import('../../../routes/api/llm.js'); - // Use default threshold if not provided - const actualThreshold = threshold || SEARCH_CONSTANTS.VECTOR_SEARCH.EXACT_MATCH_THRESHOLD; - - try { - log.info(`Finding similar notes with provider: ${providerId}, model: ${modelId}, dimension: ${embedding.length}, threshold: ${actualThreshold}`); - - // First try to find embeddings for the exact provider and model - const embeddings = await sql.getRows(` - SELECT ne.embedId, ne.noteId, ne.providerId, ne.modelId, ne.dimension, ne.embedding, - n.isDeleted, n.title, n.type, n.mime - FROM note_embeddings ne - JOIN notes n ON ne.noteId = n.noteId - WHERE ne.providerId = ? AND ne.modelId = ? AND n.isDeleted = 0 - `, [providerId, modelId]) as EmbeddingRow[]; - - if (embeddings && embeddings.length > 0) { - log.info(`Found ${embeddings.length} embeddings for provider ${providerId}, model ${modelId}`); - - // Add query model information to each embedding for cross-model comparison - const enhancedEmbeddings: EnhancedEmbeddingRow[] = embeddings.map(e => { - return { - embedId: e.embedId, - noteId: e.noteId, - providerId: e.providerId, - modelId: e.modelId, - dimension: e.dimension, - embedding: e.embedding, - title: e.title, - type: e.type, - mime: e.mime, - isDeleted: e.isDeleted, - queryProviderId: providerId, - queryModelId: modelId - }; - }); - - return await processEmbeddings(embedding, enhancedEmbeddings, actualThreshold, limit); - } - - // If no embeddings found and fallback is allowed, try other providers - if (useFallback) { - log.info(`No embeddings found for ${providerId}/${modelId}, trying fallback providers`); - - // Define the type for embedding metadata - interface EmbeddingMetadata { - providerId: string; - modelId: string; - count: number; - dimension: number; - } - - // Get all available embedding metadata - const availableEmbeddings = await sql.getRows(` - SELECT DISTINCT providerId, modelId, COUNT(*) as count, dimension - FROM note_embeddings - GROUP BY providerId, modelId - ORDER BY dimension DESC, count DESC - `) as EmbeddingMetadata[]; - - if (availableEmbeddings.length > 0) { - log.info(`Available embeddings: ${JSON.stringify(availableEmbeddings.map(e => ({ - providerId: e.providerId, - modelId: e.modelId, - count: e.count, - dimension: e.dimension - })))}`); - - // Import the vector utils - const { selectOptimalEmbedding } = await import('./vector_utils.js'); - - // Get user dimension strategy preference - const options = (await import('../../options.js')).default; - const dimensionStrategy = await options.getOption('embeddingDimensionStrategy') || 'native'; - log.info(`Using embedding dimension strategy: ${dimensionStrategy}`); - - // Find the best alternative based on highest dimension for 'native' strategy - if (dimensionStrategy === 'native') { - const bestAlternative = selectOptimalEmbedding(availableEmbeddings); - - if (bestAlternative) { - log.info(`Using highest-dimension fallback: ${bestAlternative.providerId}/${bestAlternative.modelId} (${bestAlternative.dimension}D)`); - - // Get embeddings for this provider/model - const alternativeEmbeddings = await sql.getRows(` - SELECT ne.embedId, ne.noteId, ne.providerId, ne.modelId, ne.dimension, ne.embedding, - n.isDeleted, n.title, n.type, n.mime - FROM note_embeddings ne - JOIN notes n ON ne.noteId = n.noteId - WHERE ne.providerId = ? AND ne.modelId = ? AND n.isDeleted = 0 - `, [bestAlternative.providerId, bestAlternative.modelId]) as EmbeddingRow[]; - - if (alternativeEmbeddings && alternativeEmbeddings.length > 0) { - // Add query model information to each embedding for cross-model comparison - const enhancedEmbeddings: EnhancedEmbeddingRow[] = alternativeEmbeddings.map(e => { - return { - embedId: e.embedId, - noteId: e.noteId, - providerId: e.providerId, - modelId: e.modelId, - dimension: e.dimension, - embedding: e.embedding, - title: e.title, - type: e.type, - mime: e.mime, - isDeleted: e.isDeleted, - queryProviderId: providerId, - queryModelId: modelId - }; - }); - - return await processEmbeddings(embedding, enhancedEmbeddings, actualThreshold, limit); - } - } - } else { - // Try providers using the new configuration system - if (useFallback) { - log.info('No embeddings found for specified provider, trying fallback providers...'); - - // Use the new configuration system - no string parsing! - const selectedProvider = await getSelectedEmbeddingProvider(); - const preferredProviders = selectedProvider ? [selectedProvider] : []; - - log.info(`Using selected provider: ${selectedProvider || 'none'}`); - - // Try providers in precedence order - for (const provider of preferredProviders) { - const providerEmbeddings = availableEmbeddings.filter(e => e.providerId === provider); - - if (providerEmbeddings.length > 0) { - // Choose the model with the most embeddings - const bestModel = providerEmbeddings.sort((a, b) => b.count - a.count)[0]; - log.info(`Found fallback provider: ${provider}, model: ${bestModel.modelId}, dimension: ${bestModel.dimension}`); - - // The 'regenerate' strategy would go here if needed - // We're no longer supporting the 'adapt' strategy - } - } - } - } - } - - log.info('No suitable fallback embeddings found, returning empty results'); - } - - return []; - } catch (error) { - log.error(`Error finding similar notes: ${error}`); - return []; - } -} - -// Helper function to process embeddings and calculate similarities -async function processEmbeddings(queryEmbedding: Float32Array, embeddings: any[], threshold: number, limit: number) { - const { - enhancedCosineSimilarity, - bufferToEmbedding, - ContentType, - PerformanceProfile, - detectContentType, - vectorDebugConfig - } = await import('./vector_utils.js'); - - // Store original debug settings but keep debug disabled - const originalDebugEnabled = vectorDebugConfig.enabled; - const originalLogLevel = vectorDebugConfig.logLevel; - - // Keep debug disabled for normal operation - vectorDebugConfig.enabled = false; - vectorDebugConfig.recordStats = false; - - const options = (await import('../../options.js')).default; - - // Define weighting factors with defaults that can be overridden by settings - interface SimilarityWeights { - exactTitleMatch: number; - titleContainsQuery: number; - partialTitleMatch: number; - // Add more weights as needed - examples: - sameType?: number; - attributeMatch?: number; - recentlyCreated?: number; - recentlyModified?: number; - } - - // Default weights that match our previous hardcoded values - const defaultWeights: SimilarityWeights = { - exactTitleMatch: 0.3, - titleContainsQuery: 0.2, - partialTitleMatch: 0.1, - sameType: 0.05, - attributeMatch: 0.05, - recentlyCreated: 0.05, - recentlyModified: 0.05 - }; - - // Get weights from options if they exist - const weights: SimilarityWeights = { ...defaultWeights }; - try { - const customWeightsJSON = EMBEDDING_CONSTANTS; - if (customWeightsJSON) { - try { - const customWeights = EMBEDDING_CONSTANTS; - // Override defaults with any custom weights - Object.assign(weights, customWeights); - log.info(`Using custom similarity weights: ${JSON.stringify(weights)}`); - } catch (e) { - log.error(`Error parsing custom similarity weights: ${e}`); - } - } - } catch (e) { - // Use defaults if no custom weights - } - - /** - * Calculate similarity bonuses based on various factors - */ - function calculateSimilarityBonuses( - embedding: any, - note: any, - queryText: string, - weights: SimilarityWeights - ): { bonuses: Record, totalBonus: number } { - const bonuses: Record = {}; - - // Skip if we don't have query text - if (!queryText || !note.title) { - return { bonuses, totalBonus: 0 }; - } - - const titleLower = note.title.toLowerCase(); - const queryLower = queryText.toLowerCase(); - - // 1. Exact title match - if (titleLower === queryLower) { - bonuses.exactTitleMatch = weights.exactTitleMatch; - } - // 2. Title contains the entire query - else if (titleLower.includes(queryLower)) { - bonuses.titleContainsQuery = weights.titleContainsQuery; - } - // 3. Partial term matching - else { - // Split query into terms and check if title contains them - const queryTerms = queryLower.split(/\s+/).filter((term: string) => term.length > 2); - let matchCount = 0; - - for (const term of queryTerms) { - if (titleLower.includes(term)) { - matchCount++; - } - } - - if (matchCount > 0 && queryTerms.length > 0) { - // Calculate proportion of matching terms and apply a scaled bonus - const matchProportion = matchCount / queryTerms.length; - bonuses.partialTitleMatch = weights.partialTitleMatch * matchProportion; - } - } - - // 4. Add more factors as needed here - // Example: Same note type bonus - // if (note.type && weights.sameType) { - // // Note: This would need to be compared with the query context to be meaningful - // // For now, this is a placeholder for demonstration - // bonuses.sameType = weights.sameType; - // } - - // Calculate total bonus - const totalBonus = Object.values(bonuses).reduce((sum, bonus) => sum + bonus, 0); - - return { bonuses, totalBonus }; - } - - const similarities: Similarity[] = []; - - try { - // Try to extract the original query text if it was added to the metadata - // This will help us determine title matches - const queryText = queryEmbedding.hasOwnProperty('originalQuery') - ? (queryEmbedding as any).originalQuery - : ''; - - for (const e of embeddings) { - // Check if this note is excluded from AI features - if (isNoteExcludedFromAIById(e.noteId)) { - continue; // Skip this note if it has the AI exclusion label - } - - const embVector = bufferToEmbedding(e.embedding, e.dimension); - - // Detect content type from mime type if available - let contentType = ContentType.GENERAL_TEXT; - if (e.mime) { - contentType = detectContentType(e.mime); - // Debug logging removed to avoid console spam - } - - // Select performance profile based on embedding size and use case - // For most similarity searches, BALANCED is a good default - const performanceProfile = PerformanceProfile.BALANCED; - - // Determine if this is cross-model comparison - const isCrossModel = e.providerId !== e.queryProviderId || e.modelId !== e.queryModelId; - - // Calculate similarity with content-aware parameters - let similarity = enhancedCosineSimilarity( - queryEmbedding, - embVector, - true, // normalize vectors to ensure consistent comparison - e.queryModelId, // source model ID - e.modelId, // target model ID - contentType, // content-specific padding strategy - performanceProfile - ); - - // Calculate and apply similarity bonuses - const { bonuses, totalBonus } = calculateSimilarityBonuses( - queryEmbedding, - e, - queryText, - weights - ); - - if (totalBonus > 0) { - similarity += totalBonus; - - // Log significant bonuses for debugging - const significantBonuses = Object.entries(bonuses) - .filter(([_, value]) => value >= 0.05) - .map(([key, value]) => `${key}: +${value.toFixed(2)}`) - .join(', '); - - if (significantBonuses) { - log.info(`Added bonuses for note "${e.title}" (${e.noteId}): ${significantBonuses}`); - } - - // Cap similarity at 1.0 to maintain expected range - similarity = Math.min(similarity, 1.0); - } - - if (similarity >= threshold) { - similarities.push({ - noteId: e.noteId, - similarity: similarity, - contentType: contentType.toString(), - // Optionally include bonuses for debugging/analysis - // bonuses: bonuses - }); - } - } - - return similarities - .sort((a, b) => b.similarity - a.similarity) - .slice(0, limit); - } finally { - // Restore original debug settings - vectorDebugConfig.enabled = originalDebugEnabled; - vectorDebugConfig.logLevel = originalLogLevel; - } -} - -/** - * Delete embeddings for a note - * - * @param noteId - The ID of the note - * @param providerId - Optional provider ID to delete embeddings only for a specific provider - * @param modelId - Optional model ID to delete embeddings only for a specific model - */ -export async function deleteNoteEmbeddings(noteId: string, providerId?: string, modelId?: string) { - let query = "DELETE FROM note_embeddings WHERE noteId = ?"; - const params: any[] = [noteId]; - - if (providerId) { - query += " AND providerId = ?"; - params.push(providerId); - - if (modelId) { - query += " AND modelId = ?"; - params.push(modelId); - } - } - - await sql.execute(query, params); -} diff --git a/apps/server/src/services/llm/embeddings/types.ts b/apps/server/src/services/llm/embeddings/types.ts deleted file mode 100644 index 49a79484e2..0000000000 --- a/apps/server/src/services/llm/embeddings/types.ts +++ /dev/null @@ -1,29 +0,0 @@ -import type { NoteEmbeddingContext } from "./embeddings_interface.js"; - -/** - * Type definition for embedding result - */ -export interface EmbeddingResult { - embedId: string; - noteId: string; - providerId: string; - modelId: string; - dimension: number; - embedding: Float32Array; - version: number; - dateCreated: string; - utcDateCreated: string; - dateModified: string; - utcDateModified: string; -} - -/** - * Type for queue item - */ -export interface QueueItem { - noteId: string; - operation: string; - attempts: number; -} - -export type { NoteEmbeddingContext }; diff --git a/apps/server/src/services/llm/embeddings/vector_utils.ts b/apps/server/src/services/llm/embeddings/vector_utils.ts deleted file mode 100644 index 73037cfaca..0000000000 --- a/apps/server/src/services/llm/embeddings/vector_utils.ts +++ /dev/null @@ -1,886 +0,0 @@ -import { SEARCH_CONSTANTS } from '../constants/search_constants.js'; - -/** - * Computes the cosine similarity between two vectors - * If dimensions don't match, automatically adapts using the enhanced approach - * @param normalize Optional flag to normalize vectors before comparison (default: false) - * @param sourceModel Optional identifier for the source model - * @param targetModel Optional identifier for the target model - * @param contentType Optional content type for strategy selection - * @param performanceProfile Optional performance profile - */ -export function cosineSimilarity( - a: Float32Array, - b: Float32Array, - normalize: boolean = false, - sourceModel?: string, - targetModel?: string, - contentType?: ContentType, - performanceProfile?: PerformanceProfile -): number { - // Use the enhanced approach that preserves more information - return enhancedCosineSimilarity(a, b, normalize, sourceModel, targetModel, contentType, performanceProfile); -} - -/** - * Enhanced cosine similarity that adaptively handles different dimensions - * Instead of truncating larger embeddings, it pads smaller ones to preserve information - * @param normalize Optional flag to normalize vectors before comparison (default: false) - * @param sourceModel Optional identifier for the source model - * @param targetModel Optional identifier for the target model - * @param contentType Optional content type for strategy selection - * @param performanceProfile Optional performance profile - */ -export function enhancedCosineSimilarity( - a: Float32Array, - b: Float32Array, - normalize: boolean = false, - sourceModel?: string, - targetModel?: string, - contentType?: ContentType, - performanceProfile?: PerformanceProfile -): number { - // If normalization is requested, normalize vectors first - if (normalize) { - a = normalizeVector(a); - b = normalizeVector(b); - } - - // If dimensions match, use standard calculation - if (a.length === b.length) { - return standardCosineSimilarity(a, b); - } - - // Log dimension adaptation - debugLog(`Dimension mismatch: ${a.length} vs ${b.length}. Adapting dimensions...`, 'info'); - - // Determine if models are different - const isCrossModelComparison = sourceModel !== targetModel && - sourceModel !== undefined && - targetModel !== undefined; - - // Context for strategy selection - const context: StrategySelectionContext = { - contentType: contentType || ContentType.GENERAL_TEXT, - performanceProfile: performanceProfile || PerformanceProfile.BALANCED, - sourceDimension: a.length, - targetDimension: b.length, - sourceModel, - targetModel, - isCrossModelComparison - }; - - // Select the optimal strategy based on context - let adaptOptions: AdaptationOptions; - - if (a.length > b.length) { - // Pad b to match a's dimensions - debugLog(`Adapting embedding B (${b.length}D) to match A (${a.length}D)`, 'debug'); - - // Get optimal strategy - adaptOptions = selectOptimalPaddingStrategy(context); - const adaptedB = adaptEmbeddingDimensions(b, a.length, adaptOptions); - - // Record stats - recordAdaptationStats({ - operation: 'dimension_adaptation', - sourceModel: targetModel, - targetModel: sourceModel, - sourceDimension: b.length, - targetDimension: a.length, - strategy: adaptOptions.strategy - }); - - return standardCosineSimilarity(a, adaptedB); - } else { - // Pad a to match b's dimensions - debugLog(`Adapting embedding A (${a.length}D) to match B (${b.length}D)`, 'debug'); - - // Get optimal strategy - adaptOptions = selectOptimalPaddingStrategy(context); - const adaptedA = adaptEmbeddingDimensions(a, b.length, adaptOptions); - - // Record stats - recordAdaptationStats({ - operation: 'dimension_adaptation', - sourceModel: sourceModel, - targetModel: targetModel, - sourceDimension: a.length, - targetDimension: b.length, - strategy: adaptOptions.strategy - }); - - return standardCosineSimilarity(adaptedA, b); - } -} - -/** - * Normalizes a vector to unit length - * @param vector The vector to normalize - * @returns A new normalized vector - */ -export function normalizeVector(vector: Float32Array): Float32Array { - let magnitude = 0; - for (let i = 0; i < vector.length; i++) { - magnitude += vector[i] * vector[i]; - } - - magnitude = Math.sqrt(magnitude); - - // If vector is already normalized or is a zero vector, return a copy - if (magnitude === 0 || Math.abs(magnitude - 1.0) < 1e-6) { - return new Float32Array(vector); - } - - // Create a new normalized vector - const normalized = new Float32Array(vector.length); - for (let i = 0; i < vector.length; i++) { - normalized[i] = vector[i] / magnitude; - } - - return normalized; -} - -/** - * Standard cosine similarity for same-dimension vectors - */ -function standardCosineSimilarity(a: Float32Array, b: Float32Array): number { - let dotProduct = 0; - let aMagnitude = 0; - let bMagnitude = 0; - - for (let i = 0; i < a.length; i++) { - dotProduct += a[i] * b[i]; - aMagnitude += a[i] * a[i]; - bMagnitude += b[i] * b[i]; - } - - aMagnitude = Math.sqrt(aMagnitude); - bMagnitude = Math.sqrt(bMagnitude); - - if (aMagnitude === 0 || bMagnitude === 0) { - return 0; - } - - return dotProduct / (aMagnitude * bMagnitude); -} - -/** - * Identifies the optimal embedding when multiple are available - * Prioritizes higher-dimensional embeddings as they contain more information - */ -export function selectOptimalEmbedding(embeddings: Array<{ - providerId: string; - modelId: string; - dimension: number; - count?: number; -}>): {providerId: string; modelId: string; dimension: number} | null { - if (!embeddings || embeddings.length === 0) return null; - - // First prioritize by dimension (higher is better) - let optimal = embeddings.reduce((best, current) => - current.dimension > best.dimension ? current : best, - embeddings[0] - ); - - return optimal; -} - -/** - * Padding strategy options for dimension adaptation - */ -export enum PaddingStrategy { - ZERO = 'zero', // Simple zero padding (default) - MEAN = 'mean', // Padding with mean value of source embedding - GAUSSIAN = 'gaussian', // Padding with Gaussian noise based on source statistics - MIRROR = 'mirror' // Mirroring existing values for padding -} - -/** - * Configuration for embedding adaptation - */ -export interface AdaptationOptions { - strategy: PaddingStrategy; - seed?: number; // Seed for random number generation (gaussian) - variance?: number; // Variance for gaussian noise (default: 0.01) - normalize?: boolean; // Whether to normalize after adaptation -} - -/** - * Adapts an embedding to match target dimensions with configurable strategies - * - * @param sourceEmbedding The original embedding - * @param targetDimension The desired dimension - * @param options Configuration options for the adaptation - * @returns A new embedding with the target dimensions - */ -export function adaptEmbeddingDimensions( - sourceEmbedding: Float32Array, - targetDimension: number, - options: AdaptationOptions = { strategy: PaddingStrategy.ZERO, normalize: true } -): Float32Array { - const sourceDimension = sourceEmbedding.length; - - // If dimensions already match, return a copy of the original - if (sourceDimension === targetDimension) { - return new Float32Array(sourceEmbedding); - } - - // Create a new embedding with target dimensions - const adaptedEmbedding = new Float32Array(targetDimension); - - if (sourceDimension < targetDimension) { - // Copy all source values first - adaptedEmbedding.set(sourceEmbedding); - - // Apply the selected padding strategy - switch (options.strategy) { - case PaddingStrategy.ZERO: - // Zero padding is already done by default - break; - - case PaddingStrategy.MEAN: - // Calculate mean of source embedding - let sum = 0; - for (let i = 0; i < sourceDimension; i++) { - sum += sourceEmbedding[i]; - } - const mean = sum / sourceDimension; - - // Fill remaining dimensions with mean value - for (let i = sourceDimension; i < targetDimension; i++) { - adaptedEmbedding[i] = mean; - } - break; - - case PaddingStrategy.GAUSSIAN: - // Calculate mean and standard deviation of source embedding - let meanSum = 0; - for (let i = 0; i < sourceDimension; i++) { - meanSum += sourceEmbedding[i]; - } - const meanValue = meanSum / sourceDimension; - - let varianceSum = 0; - for (let i = 0; i < sourceDimension; i++) { - varianceSum += Math.pow(sourceEmbedding[i] - meanValue, 2); - } - const variance = options.variance ?? Math.min(0.01, varianceSum / sourceDimension); - const stdDev = Math.sqrt(variance); - - // Fill remaining dimensions with Gaussian noise - for (let i = sourceDimension; i < targetDimension; i++) { - // Box-Muller transform for Gaussian distribution - const u1 = Math.random(); - const u2 = Math.random(); - const z0 = Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2); - - adaptedEmbedding[i] = meanValue + stdDev * z0; - } - break; - - case PaddingStrategy.MIRROR: - // Mirror existing values for padding - for (let i = sourceDimension; i < targetDimension; i++) { - // Cycle through source values in reverse order - const mirrorIndex = sourceDimension - 1 - ((i - sourceDimension) % sourceDimension); - adaptedEmbedding[i] = sourceEmbedding[mirrorIndex]; - } - break; - - default: - // Default to zero padding - break; - } - } else { - // If source is larger, truncate to target dimension - for (let i = 0; i < targetDimension; i++) { - adaptedEmbedding[i] = sourceEmbedding[i]; - } - } - - // Normalize the adapted embedding if requested - if (options.normalize) { - return normalizeVector(adaptedEmbedding); - } - - return adaptedEmbedding; -} - -/** - * Converts embedding Float32Array to Buffer for storage in SQLite - */ -export function embeddingToBuffer(embedding: Float32Array): Buffer { - return Buffer.from(embedding.buffer); -} - -/** - * Converts Buffer from SQLite back to Float32Array - */ -export function bufferToEmbedding(buffer: Buffer, dimension: number): Float32Array { - return new Float32Array(buffer.buffer, buffer.byteOffset, dimension); -} - -/** - * Similarity metric options - */ -export enum SimilarityMetric { - COSINE = 'cosine', // Standard cosine similarity - DOT_PRODUCT = 'dot_product', // Simple dot product (assumes normalized vectors) - HYBRID = 'hybrid', // Dot product + cosine hybrid - DIM_AWARE = 'dimension_aware', // Dimension-aware similarity that factors in dimension differences - ENSEMBLE = 'ensemble' // Combined score from multiple metrics -} - -/** - * Configuration for similarity calculation - */ -export interface SimilarityOptions { - metric: SimilarityMetric; - normalize?: boolean; - ensembleWeights?: {[key in SimilarityMetric]?: number}; - dimensionPenalty?: number; // Penalty factor for dimension differences (0 to 1) - sourceModel?: string; // Source model identifier - targetModel?: string; // Target model identifier - contentType?: ContentType; // Type of content being compared - performanceProfile?: PerformanceProfile; // Performance requirements -} - -/** - * Computes similarity between two vectors using the specified metric - * @param a First vector - * @param b Second vector - * @param options Similarity calculation options - */ -export function computeSimilarity( - a: Float32Array, - b: Float32Array, - options: SimilarityOptions = { metric: SimilarityMetric.COSINE } -): number { - // Apply normalization if requested - const normalize = options.normalize ?? false; - - switch (options.metric) { - case SimilarityMetric.COSINE: - return cosineSimilarity( - a, b, normalize, - options.sourceModel, options.targetModel, - options.contentType, options.performanceProfile - ); - - case SimilarityMetric.DOT_PRODUCT: - // Dot product assumes normalized vectors for proper similarity measurement - const aNorm = normalize ? normalizeVector(a) : a; - const bNorm = normalize ? normalizeVector(b) : b; - return computeDotProduct(aNorm, bNorm, options); - - case SimilarityMetric.HYBRID: - // Hybrid approach combines dot product with cosine similarity - // More robust against small perturbations while maintaining angle sensitivity - return hybridSimilarity(a, b, normalize, options); - - case SimilarityMetric.DIM_AWARE: - // Dimension-aware similarity that factors in dimension differences - return dimensionAwareSimilarity( - a, b, normalize, - options.dimensionPenalty ?? 0.1, - options.contentType, - options.performanceProfile - ); - - case SimilarityMetric.ENSEMBLE: - // Ensemble scoring combines multiple metrics with weights - return ensembleSimilarity(a, b, options); - - default: - // Default to cosine similarity - return cosineSimilarity( - a, b, normalize, - options.sourceModel, options.targetModel, - options.contentType, options.performanceProfile - ); - } -} - -/** - * Computes dot product between two vectors - */ -export function computeDotProduct( - a: Float32Array, - b: Float32Array, - options?: Pick -): number { - // Adapt dimensions if needed - if (a.length !== b.length) { - // Create context for strategy selection if dimensions don't match - if (options) { - const context: StrategySelectionContext = { - contentType: options.contentType || ContentType.GENERAL_TEXT, - performanceProfile: options.performanceProfile || PerformanceProfile.BALANCED, - sourceDimension: a.length, - targetDimension: b.length, - sourceModel: options.sourceModel, - targetModel: options.targetModel, - isCrossModelComparison: options.sourceModel !== options.targetModel && - options.sourceModel !== undefined && - options.targetModel !== undefined - }; - - if (a.length > b.length) { - const adaptOptions = selectOptimalPaddingStrategy(context); - b = adaptEmbeddingDimensions(b, a.length, adaptOptions); - } else { - const adaptOptions = selectOptimalPaddingStrategy(context); - a = adaptEmbeddingDimensions(a, b.length, adaptOptions); - } - } else { - // Default behavior without options - if (a.length > b.length) { - b = adaptEmbeddingDimensions(b, a.length); - } else { - a = adaptEmbeddingDimensions(a, b.length); - } - } - } - - let dotProduct = 0; - for (let i = 0; i < a.length; i++) { - dotProduct += a[i] * b[i]; - } - - return dotProduct; -} - -/** - * Hybrid similarity combines dot product and cosine similarity - * Provides robustness against small perturbations while maintaining angle sensitivity - */ -export function hybridSimilarity( - a: Float32Array, - b: Float32Array, - normalize: boolean = false, - options?: Pick -): number { - // Get cosine similarity with full options - const cosine = cosineSimilarity( - a, b, normalize, - options?.sourceModel, options?.targetModel, - options?.contentType, options?.performanceProfile - ); - - // For dot product, we should always normalize - const aNorm = normalize ? a : normalizeVector(a); - const bNorm = normalize ? b : normalizeVector(b); - - // If dimensions don't match, adapt with optimal strategy - let adaptedA = aNorm; - let adaptedB = bNorm; - - if (aNorm.length !== bNorm.length) { - // Use optimal padding strategy - if (options) { - const context: StrategySelectionContext = { - contentType: options.contentType || ContentType.GENERAL_TEXT, - performanceProfile: options.performanceProfile || PerformanceProfile.BALANCED, - sourceDimension: aNorm.length, - targetDimension: bNorm.length, - sourceModel: options.sourceModel, - targetModel: options.targetModel, - isCrossModelComparison: options.sourceModel !== options.targetModel && - options.sourceModel !== undefined && - options.targetModel !== undefined - }; - - if (aNorm.length < bNorm.length) { - const adaptOptions = selectOptimalPaddingStrategy(context); - adaptedA = adaptEmbeddingDimensions(aNorm, bNorm.length, adaptOptions); - } else { - const adaptOptions = selectOptimalPaddingStrategy(context); - adaptedB = adaptEmbeddingDimensions(bNorm, aNorm.length, adaptOptions); - } - } else { - // Default behavior - adaptedA = aNorm.length < bNorm.length ? adaptEmbeddingDimensions(aNorm, bNorm.length) : aNorm; - adaptedB = bNorm.length < aNorm.length ? adaptEmbeddingDimensions(bNorm, aNorm.length) : bNorm; - } - } - - // Compute dot product (should be similar to cosine for normalized vectors) - const dot = computeDotProduct(adaptedA, adaptedB, options); - - // Return weighted average - giving more weight to cosine - return 0.7 * cosine + 0.3 * dot; -} - -/** - * Dimension-aware similarity that factors in dimension differences - * @param dimensionPenalty Penalty factor for dimension differences (0 to 1) - */ -export function dimensionAwareSimilarity( - a: Float32Array, - b: Float32Array, - normalize: boolean = false, - dimensionPenalty: number = 0.1, - contentType?: ContentType, - performanceProfile?: PerformanceProfile -): number { - // Basic cosine similarity with content type information - const cosine = cosineSimilarity(a, b, normalize, undefined, undefined, contentType, performanceProfile); - - // If dimensions match, return standard cosine - if (a.length === b.length) { - return cosine; - } - - // Calculate dimension penalty - // This penalizes vectors with very different dimensions - const dimRatio = Math.min(a.length, b.length) / Math.max(a.length, b.length); - const penalty = 1 - dimensionPenalty * (1 - dimRatio); - - // Apply penalty to similarity score - return cosine * penalty; -} - -/** - * Ensemble similarity combines multiple metrics with weights - */ -export function ensembleSimilarity( - a: Float32Array, - b: Float32Array, - options: SimilarityOptions -): number { - // Default weights if not provided - const weights = options.ensembleWeights ?? { - [SimilarityMetric.COSINE]: SEARCH_CONSTANTS.VECTOR_SEARCH.SIMILARITY_THRESHOLD.COSINE, - [SimilarityMetric.HYBRID]: SEARCH_CONSTANTS.VECTOR_SEARCH.SIMILARITY_THRESHOLD.HYBRID, - [SimilarityMetric.DIM_AWARE]: SEARCH_CONSTANTS.VECTOR_SEARCH.SIMILARITY_THRESHOLD.DIM_AWARE - }; - - let totalWeight = 0; - let weightedSum = 0; - - // Compute each metric and apply weight - for (const [metricStr, weight] of Object.entries(weights)) { - const metric = metricStr as SimilarityMetric; - if (weight && weight > 0) { - // Skip the ensemble itself to avoid recursion - if (metric !== SimilarityMetric.ENSEMBLE) { - const similarity = computeSimilarity(a, b, { - metric, - normalize: options.normalize - }); - - weightedSum += similarity * weight; - totalWeight += weight; - } - } - } - - // Normalize by total weight - return totalWeight > 0 ? weightedSum / totalWeight : cosineSimilarity(a, b, options.normalize); -} - -/** - * Debug configuration for vector operations - */ -export interface DebugConfig { - enabled: boolean; - logLevel: 'info' | 'debug' | 'warning' | 'error'; - recordStats: boolean; -} - -/** - * Global debug configuration, can be modified at runtime - */ -export const vectorDebugConfig: DebugConfig = { - enabled: false, - logLevel: 'info', - recordStats: false -}; - -/** - * Statistics collected during vector operations - */ -export interface AdaptationStats { - timestamp: number; - operation: string; - sourceModel?: string; - targetModel?: string; - sourceDimension: number; - targetDimension: number; - strategy: string; - similarity?: number; -} - -// Collection of adaptation statistics for quality auditing -export const adaptationStats: AdaptationStats[] = []; - -/** - * Log a message if debugging is enabled - */ -function debugLog( - message: string, - level: 'info' | 'debug' | 'warning' | 'error' = 'info' -): void { - if (vectorDebugConfig.enabled) { - const levelOrder = { 'debug': 0, 'info': 1, 'warning': 2, 'error': 3 }; - - if (levelOrder[level] >= levelOrder[vectorDebugConfig.logLevel]) { - const prefix = `[VectorUtils:${level.toUpperCase()}]`; - - switch (level) { - case 'error': - console.error(prefix, message); - break; - case 'warning': - console.warn(prefix, message); - break; - case 'debug': - console.debug(prefix, message); - break; - default: - console.log(prefix, message); - } - } - } -} - -/** - * Record adaptation statistics if enabled - */ -function recordAdaptationStats(stats: Omit): void { - if (vectorDebugConfig.enabled && vectorDebugConfig.recordStats) { - adaptationStats.push({ - ...stats, - timestamp: Date.now() - }); - - // Keep only the last 1000 stats to prevent memory issues - if (adaptationStats.length > 1000) { - adaptationStats.shift(); - } - } -} - -/** - * Content types for embedding adaptation strategy selection - */ -export enum ContentType { - GENERAL_TEXT = 'general_text', - CODE = 'code', - STRUCTURED_DATA = 'structured_data', - MATHEMATICAL = 'mathematical', - MIXED = 'mixed' -} - -/** - * Performance profile for selecting adaptation strategy - */ -export enum PerformanceProfile { - MAXIMUM_QUALITY = 'maximum_quality', // Prioritize similarity quality over speed - BALANCED = 'balanced', // Balance quality and performance - MAXIMUM_SPEED = 'maximum_speed' // Prioritize speed over quality -} - -/** - * Context for selecting the optimal padding strategy - */ -export interface StrategySelectionContext { - contentType?: ContentType; // Type of content being compared - performanceProfile?: PerformanceProfile; // Performance requirements - sourceDimension: number; // Source embedding dimension - targetDimension: number; // Target embedding dimension - sourceModel?: string; // Source model identifier - targetModel?: string; // Target model identifier - isHighPrecisionRequired?: boolean; // Whether high precision is needed - isCrossModelComparison?: boolean; // Whether comparing across different models - dimensionRatio?: number; // Custom dimension ratio threshold -} - -/** - * Selects the optimal padding strategy based on content type and performance considerations - * @param context Selection context parameters - * @returns The most appropriate padding strategy and options - */ -export function selectOptimalPaddingStrategy( - context: StrategySelectionContext -): AdaptationOptions { - const { - contentType = ContentType.GENERAL_TEXT, - performanceProfile = PerformanceProfile.BALANCED, - sourceDimension, - targetDimension, - isHighPrecisionRequired = false, - isCrossModelComparison = false - } = context; - - // Calculate dimension ratio - const dimRatio = Math.min(sourceDimension, targetDimension) / - Math.max(sourceDimension, targetDimension); - - // Default options - const options: AdaptationOptions = { - strategy: PaddingStrategy.ZERO, - normalize: true - }; - - // Significant dimension difference detection - const hasSignificantDimDifference = dimRatio < (context.dimensionRatio || 0.5); - - // Select strategy based on content type - switch (contentType) { - case ContentType.CODE: - // Code benefits from structural patterns - options.strategy = PaddingStrategy.MIRROR; - break; - - case ContentType.STRUCTURED_DATA: - // Structured data works well with mean-value padding - options.strategy = PaddingStrategy.MEAN; - break; - - case ContentType.MATHEMATICAL: - // Mathematical content benefits from gaussian noise to maintain statistical properties - options.strategy = PaddingStrategy.GAUSSIAN; - options.variance = 0.005; // Lower variance for mathematical precision - break; - - case ContentType.MIXED: - // For mixed content, choose based on performance profile - if (performanceProfile === PerformanceProfile.MAXIMUM_QUALITY) { - options.strategy = PaddingStrategy.GAUSSIAN; - } else if (performanceProfile === PerformanceProfile.MAXIMUM_SPEED) { - options.strategy = PaddingStrategy.ZERO; - } else { - options.strategy = PaddingStrategy.MEAN; - } - break; - - case ContentType.GENERAL_TEXT: - default: - // For general text, base decision on other factors - if (isHighPrecisionRequired) { - options.strategy = PaddingStrategy.GAUSSIAN; - } else if (isCrossModelComparison) { - options.strategy = PaddingStrategy.MEAN; - } else { - options.strategy = PaddingStrategy.ZERO; - } - break; - } - - // Override based on performance profile if we have significant dimension differences - if (hasSignificantDimDifference) { - // For extreme dimension differences, specialized handling - if (performanceProfile === PerformanceProfile.MAXIMUM_QUALITY) { - // For quality, use gaussian noise for better statistical matching - options.strategy = PaddingStrategy.GAUSSIAN; - // Adjust variance based on dimension ratio - options.variance = Math.min(0.01, 0.02 * dimRatio); - - // Log the significant dimension adaptation - debugLog(`Significant dimension difference detected: ${sourceDimension} vs ${targetDimension}. ` + - `Ratio: ${dimRatio.toFixed(2)}. Using Gaussian strategy.`, 'warning'); - } else if (performanceProfile === PerformanceProfile.MAXIMUM_SPEED) { - // For speed, stick with zero padding - options.strategy = PaddingStrategy.ZERO; - } - } - - // Always use zero padding for trivial dimension differences - // (e.g. 1536 vs 1537) for performance reasons - if (Math.abs(sourceDimension - targetDimension) <= 5) { - options.strategy = PaddingStrategy.ZERO; - } - - // Log the selected strategy - debugLog(`Selected padding strategy: ${options.strategy} for ` + - `content type: ${contentType}, performance profile: ${performanceProfile}`, 'debug'); - - return options; -} - -/** - * Helper function to determine content type from note context - * @param context The note context information - * @returns The detected content type - */ -export function detectContentType(mime: string, content?: string): ContentType { - // Detect based on mime type - if (mime.includes('code') || - mime.includes('javascript') || - mime.includes('typescript') || - mime.includes('python') || - mime.includes('java') || - mime.includes('c++') || - mime.includes('json')) { - return ContentType.CODE; - } - - if (mime.includes('xml') || - mime.includes('csv') || - mime.includes('sql') || - mime.endsWith('+json')) { - return ContentType.STRUCTURED_DATA; - } - - if (mime.includes('latex') || - mime.includes('mathml') || - mime.includes('tex')) { - return ContentType.MATHEMATICAL; - } - - // If we have content, we can do deeper analysis - if (content) { - // Detect code by looking for common patterns - const codePatterns = [ - /function\s+\w+\s*\(.*\)\s*{/, // JavaScript/TypeScript function - /def\s+\w+\s*\(.*\):/, // Python function - /class\s+\w+(\s+extends\s+\w+)?(\s+implements\s+\w+)?\s*{/, // Java/TypeScript class - /import\s+.*\s+from\s+['"]/, // JS/TS import - /^\s*```\w+/m // Markdown code block - ]; - - if (codePatterns.some(pattern => pattern.test(content))) { - return ContentType.CODE; - } - - // Detect structured data - const structuredPatterns = [ - /^\s*[{\[]/, // JSON-like start - /^\s*<\?xml/, // XML declaration - /^\s*<[a-z]+>/i, // HTML/XML tag - /^\s*(\w+,)+\w+$/m, // CSV-like - /CREATE\s+TABLE|SELECT\s+.*\s+FROM/i // SQL - ]; - - if (structuredPatterns.some(pattern => pattern.test(content))) { - return ContentType.STRUCTURED_DATA; - } - - // Detect mathematical content - const mathPatterns = [ - /\$\$.*\$\$/s, // LaTeX block - /\\begin{equation}/, // LaTeX equation environment - /\\sum|\\int|\\frac|\\sqrt/, // Common LaTeX math commands - ]; - - if (mathPatterns.some(pattern => pattern.test(content))) { - return ContentType.MATHEMATICAL; - } - - // Check for mixed content - const hasMixedContent = - (codePatterns.some(pattern => pattern.test(content)) && - content.split(/\s+/).length > 100) || // Code and substantial text - (content.includes('```') && - content.replace(/```.*?```/gs, '').length > 200); // Markdown with code blocks and text - - if (hasMixedContent) { - return ContentType.MIXED; - } - } - - // Default to general text - return ContentType.GENERAL_TEXT; -} diff --git a/apps/server/src/services/llm/index_service.ts b/apps/server/src/services/llm/index_service.ts deleted file mode 100644 index 6887fbc364..0000000000 --- a/apps/server/src/services/llm/index_service.ts +++ /dev/null @@ -1,970 +0,0 @@ -/** - * LLM Index Service - * - * Centralized service for managing knowledge base indexing for LLM features. - * This service coordinates: - * - Note embedding generation and management - * - Smart context retrieval for LLM queries - * - Progressive indexing of the knowledge base - * - Optimization of the semantic search capabilities - */ - -import log from "../log.js"; -import options from "../options.js"; -import becca from "../../becca/becca.js"; -import beccaLoader from "../../becca/becca_loader.js"; -import vectorStore from "./embeddings/index.js"; -import providerManager from "./providers/providers.js"; -import { ContextExtractor } from "./context/index.js"; -import eventService from "../events.js"; -import sql from "../sql.js"; -import sqlInit from "../sql_init.js"; -import { CONTEXT_PROMPTS } from './constants/llm_prompt_constants.js'; -import { SEARCH_CONSTANTS } from './constants/search_constants.js'; -import { isNoteExcludedFromAI } from "./utils/ai_exclusion_utils.js"; -import { hasWorkingEmbeddingProviders } from "./provider_validation.js"; - -export class IndexService { - private initialized = false; - private indexingInProgress = false; - private contextExtractor = new ContextExtractor(); - private automaticIndexingInterval?: NodeJS.Timeout; - - // Index rebuilding tracking - private indexRebuildInProgress = false; - private indexRebuildProgress = 0; - private indexRebuildTotal = 0; - private indexRebuildCurrent = 0; - - // Configuration - private defaultQueryDepth = SEARCH_CONSTANTS.HIERARCHY.DEFAULT_QUERY_DEPTH; - private maxNotesPerQuery = SEARCH_CONSTANTS.HIERARCHY.MAX_NOTES_PER_QUERY; - private defaultSimilarityThreshold = SEARCH_CONSTANTS.VECTOR_SEARCH.EXACT_MATCH_THRESHOLD; - private indexUpdateInterval = 3600000; // 1 hour in milliseconds - - /** - * Initialize the index service - */ - async initialize() { - if (this.initialized) return; - - // Setup event listeners for note changes - this.setupEventListeners(); - - // Setup automatic indexing if enabled - if (await options.getOptionBool('embeddingAutoUpdateEnabled')) { - this.setupAutomaticIndexing(); - } - - this.initialized = true; - log.info("Index service initialized"); - } - - /** - * Setup event listeners for index updates - */ - private setupEventListeners() { - // Listen for note content changes - eventService.subscribe(eventService.NOTE_CONTENT_CHANGE, async ({ entity }) => { - if (entity && entity.noteId) { - // Always queue notes for indexing, but the actual processing will depend on configuration - await this.queueNoteForIndexing(entity.noteId); - } - }); - - // Listen for new notes - eventService.subscribe(eventService.ENTITY_CREATED, async ({ entityName, entity }) => { - if (entityName === "notes" && entity && entity.noteId) { - await this.queueNoteForIndexing(entity.noteId); - } - }); - - // Listen for note title changes - eventService.subscribe(eventService.NOTE_TITLE_CHANGED, async ({ noteId }) => { - if (noteId) { - await this.queueNoteForIndexing(noteId); - } - }); - - // Listen for changes in AI settings - eventService.subscribe(eventService.ENTITY_CHANGED, async ({ entityName, entity }) => { - if (entityName === "options" && entity && entity.name) { - if (entity.name.startsWith('ai') || entity.name.startsWith('embedding')) { - log.info("AI settings changed, updating index service configuration"); - await this.updateConfiguration(); - } - } - }); - } - - /** - * Set up automatic indexing of notes - */ - private setupAutomaticIndexing() { - // Clear existing interval if any - if (this.automaticIndexingInterval) { - clearInterval(this.automaticIndexingInterval); - } - - // Create new interval - this.automaticIndexingInterval = setInterval(async () => { - try { - if (!this.indexingInProgress) { - await this.runBatchIndexing(50); // Processing logic handles sync server checks - } - } catch (error: any) { - log.error(`Error in automatic indexing: ${error.message || "Unknown error"}`); - } - }, this.indexUpdateInterval); - - log.info("Automatic indexing scheduled"); - } - - /** - * Update service configuration from options - */ - private async updateConfiguration() { - try { - // Update indexing interval - const intervalMs = parseInt(await options.getOption('embeddingUpdateInterval') || '3600000', 10); - this.indexUpdateInterval = intervalMs; - - // Check if this instance should process embeddings - const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; - const isSyncServer = await this.isSyncServerForEmbeddings(); - const shouldProcessEmbeddings = embeddingLocation === 'client' || isSyncServer; - - // Update automatic indexing setting - const autoIndexing = await options.getOptionBool('embeddingAutoUpdateEnabled'); - if (autoIndexing && shouldProcessEmbeddings && !this.automaticIndexingInterval) { - this.setupAutomaticIndexing(); - log.info(`Index service: Automatic indexing enabled, processing embeddings ${isSyncServer ? 'as sync server' : 'as client'}`); - } else if (autoIndexing && !shouldProcessEmbeddings && this.automaticIndexingInterval) { - clearInterval(this.automaticIndexingInterval); - this.automaticIndexingInterval = undefined; - log.info("Index service: Automatic indexing disabled for this instance based on configuration"); - } else if (!autoIndexing && this.automaticIndexingInterval) { - clearInterval(this.automaticIndexingInterval); - this.automaticIndexingInterval = undefined; - } - - // Update similarity threshold - const similarityThreshold = await options.getOption('embeddingSimilarityThreshold'); - this.defaultSimilarityThreshold = parseFloat(similarityThreshold || '0.65'); - - // Update max notes per query - const maxNotesPerQuery = await options.getOption('maxNotesPerLlmQuery'); - this.maxNotesPerQuery = parseInt(maxNotesPerQuery || '10', 10); - - log.info("Index service configuration updated"); - } catch (error: any) { - log.error(`Error updating index service configuration: ${error.message || "Unknown error"}`); - } - } - - /** - * Queue a note for indexing - */ - async queueNoteForIndexing(noteId: string, priority = false) { - if (!this.initialized) { - await this.initialize(); - } - - try { - // Always queue notes for indexing, regardless of where embedding generation happens - // The actual processing will be determined when the queue is processed - await vectorStore.queueNoteForEmbedding(noteId, 'UPDATE'); - return true; - } catch (error: any) { - log.error(`Error queueing note ${noteId} for indexing: ${error.message || "Unknown error"}`); - return false; - } - } - - /** - * Start full knowledge base indexing - * @param force - Whether to force reindexing of all notes - */ - async startFullIndexing(force = false) { - if (!this.initialized) { - await this.initialize(); - } - - if (this.indexingInProgress) { - throw new Error("Indexing already in progress"); - } - - try { - // Check if this instance should process embeddings - const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; - const isSyncServer = await this.isSyncServerForEmbeddings(); - const shouldProcessEmbeddings = embeddingLocation === 'client' || isSyncServer; - - if (!shouldProcessEmbeddings) { - // This instance is not configured to process embeddings - log.info("Skipping full indexing as this instance is not configured to process embeddings"); - return false; - } - - this.indexingInProgress = true; - this.indexRebuildInProgress = true; - this.indexRebuildProgress = 0; - this.indexRebuildCurrent = 0; - - // Reset index rebuild progress - const totalEmbeddings = await sql.getValue("SELECT COUNT(*) FROM note_embeddings") as number; - - if (totalEmbeddings === 0) { - // If there are no embeddings yet, we need to create them first - const totalNotes = await sql.getValue("SELECT COUNT(*) FROM notes WHERE isDeleted = 0") as number; - this.indexRebuildTotal = totalNotes; - - log.info("No embeddings found, starting full embedding generation first"); - await this.reprocessAllNotes(); - log.info("Full embedding generation initiated"); - } else { - // For index rebuild, use the number of embeddings as the total - this.indexRebuildTotal = totalEmbeddings; - - if (force) { - // Use the new rebuildSearchIndex function that doesn't regenerate embeddings - log.info("Starting forced index rebuild without regenerating embeddings"); - setTimeout(async () => { - try { - await vectorStore.rebuildSearchIndex(); - this.indexRebuildInProgress = false; - this.indexRebuildProgress = 100; - log.info("Index rebuild completed successfully"); - } catch (error: any) { - log.error(`Error during index rebuild: ${error.message || "Unknown error"}`); - this.indexRebuildInProgress = false; - } - }, 0); - } else { - // Check current stats - const stats = await vectorStore.getEmbeddingStats(); - - // Only start indexing if we're below 90% completion or if embeddings exist but need optimization - if (stats.percentComplete < 90) { - log.info("Embedding coverage below 90%, starting full embedding generation"); - await this.reprocessAllNotes(); - log.info("Full embedding generation initiated"); - } else { - log.info(`Embedding coverage at ${stats.percentComplete}%, starting index optimization`); - setTimeout(async () => { - try { - await vectorStore.rebuildSearchIndex(); - this.indexRebuildInProgress = false; - this.indexRebuildProgress = 100; - log.info("Index optimization completed successfully"); - } catch (error: any) { - log.error(`Error during index optimization: ${error.message || "Unknown error"}`); - this.indexRebuildInProgress = false; - } - }, 0); - } - } - } - - return true; - } catch (error: any) { - log.error(`Error starting full indexing: ${error.message || "Unknown error"}`); - this.indexRebuildInProgress = false; - return false; - } finally { - this.indexingInProgress = false; - } - } - - /** - * Update index rebuild progress - * @param processed - Number of notes processed - */ - updateIndexRebuildProgress(processed: number) { - if (!this.indexRebuildInProgress) return; - - this.indexRebuildCurrent += processed; - - if (this.indexRebuildTotal > 0) { - this.indexRebuildProgress = Math.min( - Math.round((this.indexRebuildCurrent / this.indexRebuildTotal) * 100), - 100 - ); - } - - if (this.indexRebuildCurrent >= this.indexRebuildTotal) { - this.indexRebuildInProgress = false; - this.indexRebuildProgress = 100; - } - } - - /** - * Get the current index rebuild progress - */ - getIndexRebuildStatus() { - return { - inProgress: this.indexRebuildInProgress, - progress: this.indexRebuildProgress, - total: this.indexRebuildTotal, - current: this.indexRebuildCurrent - }; - } - - /** - * Run a batch indexing job for a limited number of notes - * @param batchSize - Maximum number of notes to process - */ - async runBatchIndexing(batchSize = 20) { - if (!this.initialized) { - await this.initialize(); - } - - if (this.indexingInProgress) { - return false; - } - - try { - this.indexingInProgress = true; - - // Check if this instance should process embeddings - const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; - const isSyncServer = await this.isSyncServerForEmbeddings(); - const shouldProcessEmbeddings = embeddingLocation === 'client' || isSyncServer; - - if (!shouldProcessEmbeddings) { - // This instance is not configured to process embeddings - return false; - } - - // Process the embedding queue (batch size is controlled by embeddingBatchSize option) - await vectorStore.processEmbeddingQueue(); - - return true; - } catch (error: any) { - log.error(`Error in batch indexing: ${error.message || "Unknown error"}`); - return false; - } finally { - this.indexingInProgress = false; - } - } - - /** - * Get the current indexing statistics - */ - async getIndexingStats() { - if (!this.initialized) { - await this.initialize(); - } - - try { - const stats = await vectorStore.getEmbeddingStats(); - - return { - ...stats, - isIndexing: this.indexingInProgress, - automaticIndexingEnabled: !!this.automaticIndexingInterval - }; - } catch (error: any) { - log.error(`Error getting indexing stats: ${error.message || "Unknown error"}`); - return { - totalNotesCount: 0, - embeddedNotesCount: 0, - queuedNotesCount: 0, - failedNotesCount: 0, - percentComplete: 0, - isIndexing: this.indexingInProgress, - automaticIndexingEnabled: !!this.automaticIndexingInterval, - error: error.message || "Unknown error" - }; - } - } - - /** - * Get information about failed embedding attempts - */ - async getFailedIndexes(limit = 100) { - if (!this.initialized) { - await this.initialize(); - } - - try { - return await vectorStore.getFailedEmbeddingNotes(limit); - } catch (error: any) { - log.error(`Error getting failed indexes: ${error.message || "Unknown error"}`); - return []; - } - } - - /** - * Retry indexing a specific note that previously failed - */ - async retryFailedNote(noteId: string) { - if (!this.initialized) { - await this.initialize(); - } - - try { - return await vectorStore.retryFailedEmbedding(noteId); - } catch (error: any) { - log.error(`Error retrying failed note ${noteId}: ${error.message || "Unknown error"}`); - return false; - } - } - - /** - * Retry all failed indexing operations - */ - async retryAllFailedNotes() { - if (!this.initialized) { - await this.initialize(); - } - - try { - const count = await vectorStore.retryAllFailedEmbeddings(); - log.info(`Queued ${count} failed notes for retry`); - return count; - } catch (error: any) { - log.error(`Error retrying all failed notes: ${error.message || "Unknown error"}`); - return 0; - } - } - - /** - * Find semantically similar notes to a given query - * @param query - Text query to find similar notes for - * @param contextNoteId - Optional note ID to restrict search to a branch - * @param limit - Maximum number of results to return - */ - async findSimilarNotes( - query: string, - contextNoteId?: string, - limit = 10 - ) { - if (!this.initialized) { - await this.initialize(); - } - - try { - // Get the selected embedding provider on-demand - const selectedEmbeddingProvider = await options.getOption('embeddingSelectedProvider'); - const provider = selectedEmbeddingProvider - ? await providerManager.getOrCreateEmbeddingProvider(selectedEmbeddingProvider) - : (await providerManager.getEnabledEmbeddingProviders())[0]; - - if (!provider) { - throw new Error("No embedding provider available"); - } - - log.info(`Searching with embedding provider: ${provider.name}, model: ${provider.getConfig().model}`); - - // Generate embedding for the query - const embedding = await provider.generateEmbeddings(query); - log.info(`Generated embedding for query: "${query}" (${embedding.length} dimensions)`); - - // Add the original query as a property to the embedding - // This is used for title matching in the vector search - Object.defineProperty(embedding, 'originalQuery', { - value: query, - writable: false, - enumerable: true, - configurable: false - }); - - // Store query text in a global cache for possible regeneration with different providers - // Use a type declaration to avoid TypeScript errors - interface CustomGlobal { - recentEmbeddingQueries?: Record; - } - const globalWithCache = global as unknown as CustomGlobal; - - if (!globalWithCache.recentEmbeddingQueries) { - globalWithCache.recentEmbeddingQueries = {}; - } - - // Use a substring of the embedding as a key (full embedding is too large) - const embeddingKey = embedding.toString().substring(0, 100); - globalWithCache.recentEmbeddingQueries[embeddingKey] = query; - - // Limit cache size to prevent memory leaks (keep max 50 recent queries) - const keys = Object.keys(globalWithCache.recentEmbeddingQueries); - if (keys.length > 50) { - delete globalWithCache.recentEmbeddingQueries[keys[0]]; - } - - // Get Note IDs to search, optionally filtered by branch - let similarNotes: { noteId: string; title: string; similarity: number; contentType?: string }[] = []; - - // Check if we need to restrict search to a specific branch - if (contextNoteId) { - const note = becca.getNote(contextNoteId); - if (!note) { - throw new Error(`Context note ${contextNoteId} not found`); - } - - // Get all note IDs in the branch - const branchNoteIds = new Set(); - const collectNoteIds = (noteId: string) => { - branchNoteIds.add(noteId); - const note = becca.getNote(noteId); - if (note) { - for (const childNote of note.getChildNotes()) { - if (!branchNoteIds.has(childNote.noteId)) { - collectNoteIds(childNote.noteId); - } - } - } - }; - - collectNoteIds(contextNoteId); - - // Get embeddings for all notes in the branch - const config = provider.getConfig(); - - // Import the ContentType detection from vector utils - const { ContentType, detectContentType, cosineSimilarity } = await import('./embeddings/vector_utils.js'); - - for (const noteId of branchNoteIds) { - const noteEmbedding = await vectorStore.getEmbeddingForNote( - noteId, - provider.name, - config.model - ); - - if (noteEmbedding) { - // Get the note to determine its content type - const note = becca.getNote(noteId); - if (note) { - // Detect content type from mime type - const contentType = detectContentType(note.mime, ''); - - // Use content-aware similarity calculation - const similarity = cosineSimilarity( - embedding, - noteEmbedding.embedding, - true, // normalize - config.model, // source model - noteEmbedding.providerId, // target model (use providerId) - contentType, // content type for padding strategy - undefined // use default BALANCED performance profile - ); - - if (similarity >= this.defaultSimilarityThreshold) { - similarNotes.push({ - noteId, - title: note.title, - similarity, - contentType: contentType.toString() - }); - } - } - } - } - - // Sort by similarity and return top results - return similarNotes - .sort((a, b) => b.similarity - a.similarity) - .slice(0, limit); - } else { - // Search across all notes - const config = provider.getConfig(); - const results = await vectorStore.findSimilarNotes( - embedding, - provider.name, - config.model, - limit, - this.defaultSimilarityThreshold - ); - - // Enhance results with note titles - similarNotes = results.map(result => { - const note = becca.getNote(result.noteId); - return { - noteId: result.noteId, - title: note ? note.title : 'Unknown Note', - similarity: result.similarity, - contentType: result.contentType - }; - }); - - return similarNotes; - } - } catch (error: any) { - log.error(`Error finding similar notes: ${error.message || "Unknown error"}`); - return []; - } - } - - /** - * Generate context for an LLM query based on relevance to the user's question - * @param query - The user's question - * @param contextNoteId - Optional ID of a note to use as context root - * @param depth - Depth of context to include (1-4) - */ - async generateQueryContext( - query: string, - contextNoteId?: string, - depth = 2 - ) { - if (!this.initialized) { - await this.initialize(); - } - - try { - // Get embedding providers on-demand - const providers = await providerManager.getEnabledEmbeddingProviders(); - if (providers.length === 0) { - return "I don't have access to your note embeddings. Please configure an embedding provider in your AI settings."; - } - - // Find similar notes to the query - const similarNotes = await this.findSimilarNotes( - query, - contextNoteId, - this.maxNotesPerQuery - ); - - if (similarNotes.length === 0) { - return CONTEXT_PROMPTS.INDEX_NO_NOTES_CONTEXT; - } - - // Build context from the similar notes - let context = `I found some relevant information in your notes that may help answer: "${query}"\n\n`; - - for (const note of similarNotes) { - const noteObj = becca.getNote(note.noteId); - if (!noteObj) continue; - - context += `## ${noteObj.title}\n`; - - // Add parent context for better understanding - const parents = noteObj.getParentNotes(); - if (parents.length > 0) { - context += `Path: ${parents.map(p => p.title).join(' > ')}\n`; - } - - // Add content based on depth - if (depth >= 2) { - const content = await this.contextExtractor.getNoteContent(note.noteId); - if (content) { - // For larger content, use summary - if (content.length > 2000) { - const summary = await this.contextExtractor.summarizeContent(content, noteObj.title); - context += `${summary}\n[Content summarized due to length]\n\n`; - } else { - context += `${content}\n\n`; - } - } - } - - // Add child note titles for more context if depth >= 3 - if (depth >= 3) { - const childNotes = noteObj.getChildNotes(); - if (childNotes.length > 0) { - context += `Child notes: ${childNotes.slice(0, 5).map(n => n.title).join(', ')}`; - if (childNotes.length > 5) { - context += ` and ${childNotes.length - 5} more`; - } - context += `\n\n`; - } - } - - // Add attribute context for even deeper understanding if depth >= 4 - if (depth >= 4) { - const attributes = noteObj.getOwnedAttributes(); - if (attributes.length > 0) { - const relevantAttrs = attributes.filter(a => - !a.name.startsWith('_') && !a.name.startsWith('child:') && !a.name.startsWith('relation:') - ); - - if (relevantAttrs.length > 0) { - context += `Attributes: ${relevantAttrs.map(a => - `${a.type === 'label' ? '#' : '~'}${a.name}${a.value ? '=' + a.value : ''}` - ).join(', ')}\n\n`; - } - } - } - } - - // Add instructions about how to reference the notes - context += "When referring to information from these notes in your response, please cite them by their titles " + - "(e.g., \"According to your note on [Title]...\"). If the information doesn't contain what you need, " + - "just say so and use your general knowledge instead."; - - return context; - } catch (error: any) { - log.error(`Error generating query context: ${error.message || "Unknown error"}`); - return "I'm an AI assistant helping with your Trilium notes. I encountered an error while retrieving context from your notes, but I'll try to assist based on general knowledge."; - } - } - - /** - * Check if this instance is a sync server and should generate embeddings - */ - async isSyncServerForEmbeddings() { - // Check if this is a sync server (no syncServerHost means this is a sync server) - const syncServerHost = await options.getOption('syncServerHost'); - const isSyncServer = !syncServerHost; - - // Check if embedding generation should happen on the sync server - const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; - const shouldGenerateOnSyncServer = embeddingLocation === 'sync_server'; - - return isSyncServer && shouldGenerateOnSyncServer; - } - - /** - * Generate a comprehensive index entry for a note - * This prepares all metadata and contexts for optimal LLM retrieval - */ - async generateNoteIndex(noteId: string) { - if (!this.initialized) { - await this.initialize(); - } - - try { - const note = becca.getNote(noteId); - if (!note) { - throw new Error(`Note ${noteId} not found`); - } - - // Check if this note is excluded from AI features - if (isNoteExcludedFromAI(note)) { - log.info(`Note ${noteId} (${note.title}) excluded from AI indexing due to exclusion label`); - return true; // Return true to indicate successful handling (exclusion is intentional) - } - - // Check where embedding generation should happen - const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; - - // If embedding generation should happen on the sync server and we're not the sync server, - // just queue the note for embedding but don't generate it - const isSyncServer = await this.isSyncServerForEmbeddings(); - const shouldSkipGeneration = embeddingLocation === 'sync_server' && !isSyncServer; - - if (shouldSkipGeneration) { - // We're not the sync server, so just queue the note for embedding - // The sync server will handle the actual embedding generation - log.info(`Note ${noteId} queued for embedding generation on sync server`); - return true; - } - - // Get complete note context for indexing - const context = await vectorStore.getNoteEmbeddingContext(noteId); - - // Generate embedding with the selected provider - const selectedEmbeddingProvider = await options.getOption('embeddingSelectedProvider'); - const provider = selectedEmbeddingProvider - ? await providerManager.getOrCreateEmbeddingProvider(selectedEmbeddingProvider) - : (await providerManager.getEnabledEmbeddingProviders())[0]; - - if (provider) { - try { - const embedding = await provider.generateNoteEmbeddings(context); - if (embedding) { - const config = provider.getConfig(); - await vectorStore.storeNoteEmbedding( - noteId, - provider.name, - config.model, - embedding - ); - } - } catch (error: any) { - log.error(`Error generating embedding with provider ${provider.name} for note ${noteId}: ${error.message || "Unknown error"}`); - } - } - - return true; - } catch (error: any) { - log.error(`Error generating note index for ${noteId}: ${error.message || "Unknown error"}`); - return false; - } - } - - /** - * Start embedding generation (called when AI is enabled) - */ - async startEmbeddingGeneration() { - try { - log.info("Starting embedding generation system"); - - const aiEnabled = options.getOptionOrNull('aiEnabled') === "true"; - if (!aiEnabled) { - log.error("Cannot start embedding generation - AI features are disabled"); - throw new Error("AI features must be enabled first"); - } - - // Re-initialize if needed - if (!this.initialized) { - await this.initialize(); - } - - // Check if this instance should process embeddings - const embeddingLocation = await options.getOption('embeddingGenerationLocation') || 'client'; - const isSyncServer = await this.isSyncServerForEmbeddings(); - const shouldProcessEmbeddings = embeddingLocation === 'client' || isSyncServer; - - if (!shouldProcessEmbeddings) { - log.info("This instance is not configured to process embeddings"); - return; - } - - // Get embedding providers (will be created on-demand when needed) - const providers = await providerManager.getEnabledEmbeddingProviders(); - if (providers.length === 0) { - log.info("No embedding providers configured, but continuing initialization"); - } else { - log.info(`Found ${providers.length} embedding providers: ${providers.map(p => p.name).join(', ')}`); - } - - // Setup automatic indexing if enabled - if (await options.getOptionBool('embeddingAutoUpdateEnabled')) { - this.setupAutomaticIndexing(); - log.info(`Automatic embedding indexing started ${isSyncServer ? 'as sync server' : 'as client'}`); - } - - // Start background processing of the embedding queue - const { setupEmbeddingBackgroundProcessing } = await import('./embeddings/events.js'); - await setupEmbeddingBackgroundProcessing(); - - // Re-initialize event listeners - this.setupEventListeners(); - - // Queue notes that don't have embeddings for current providers - await this.queueNotesForMissingEmbeddings(); - - // Start processing the queue immediately - await this.runBatchIndexing(20); - - log.info("Embedding generation started successfully"); - } catch (error: any) { - log.error(`Error starting embedding generation: ${error.message || "Unknown error"}`); - throw error; - } - } - - - - /** - * Queue notes that don't have embeddings for current provider settings - */ - async queueNotesForMissingEmbeddings() { - try { - // Wait for becca to be fully loaded before accessing notes - await beccaLoader.beccaLoaded; - - // Get all non-deleted notes - const allNotes = Object.values(becca.notes).filter(note => !note.isDeleted); - - // Get enabled providers - const providers = await providerManager.getEnabledEmbeddingProviders(); - if (providers.length === 0) { - return; - } - - let queuedCount = 0; - let excludedCount = 0; - - // Process notes in batches to avoid overwhelming the system - const batchSize = 100; - for (let i = 0; i < allNotes.length; i += batchSize) { - const batch = allNotes.slice(i, i + batchSize); - - for (const note of batch) { - try { - // Skip notes excluded from AI - if (isNoteExcludedFromAI(note)) { - excludedCount++; - continue; - } - - // Check if note needs embeddings for any enabled provider - let needsEmbedding = false; - - for (const provider of providers) { - const config = provider.getConfig(); - const existingEmbedding = await vectorStore.getEmbeddingForNote( - note.noteId, - provider.name, - config.model - ); - - if (!existingEmbedding) { - needsEmbedding = true; - break; - } - } - - if (needsEmbedding) { - await vectorStore.queueNoteForEmbedding(note.noteId, 'UPDATE'); - queuedCount++; - } - } catch (error: any) { - log.error(`Error checking embeddings for note ${note.noteId}: ${error.message || 'Unknown error'}`); - } - } - - } - } catch (error: any) { - log.error(`Error queuing notes for missing embeddings: ${error.message || 'Unknown error'}`); - } - } - - /** - * Reprocess all notes to update embeddings - */ - async reprocessAllNotes() { - if (!this.initialized) { - await this.initialize(); - } - - try { - // Get all non-deleted note IDs - const noteIds = await sql.getColumn("SELECT noteId FROM notes WHERE isDeleted = 0"); - - // Process each note ID - for (const noteId of noteIds) { - await vectorStore.queueNoteForEmbedding(noteId as string, 'UPDATE'); - } - } catch (error: any) { - log.error(`Error reprocessing all notes: ${error.message || 'Unknown error'}`); - throw error; - } - } - - /** - * Stop embedding generation (called when AI is disabled) - */ - async stopEmbeddingGeneration() { - try { - log.info("Stopping embedding generation system"); - - // Clear automatic indexing interval - if (this.automaticIndexingInterval) { - clearInterval(this.automaticIndexingInterval); - this.automaticIndexingInterval = undefined; - log.info("Automatic indexing stopped"); - } - - // Stop the background processing from embeddings/events.ts - const { stopEmbeddingBackgroundProcessing } = await import('./embeddings/events.js'); - stopEmbeddingBackgroundProcessing(); - - // Clear all embedding providers to clean up resources - providerManager.clearAllEmbeddingProviders(); - - // Mark as not indexing - this.indexingInProgress = false; - this.indexRebuildInProgress = false; - - log.info("Embedding generation stopped successfully"); - } catch (error: any) { - log.error(`Error stopping embedding generation: ${error.message || "Unknown error"}`); - throw error; - } - } -} - -// Create singleton instance -const indexService = new IndexService(); -export default indexService; diff --git a/apps/server/src/services/llm/interfaces/agent_tool_interfaces.ts b/apps/server/src/services/llm/interfaces/agent_tool_interfaces.ts index 38051437e6..3558060c1f 100644 --- a/apps/server/src/services/llm/interfaces/agent_tool_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/agent_tool_interfaces.ts @@ -1,5 +1,10 @@ import type { ChatResponse } from '../ai_interface.js'; -import type { VectorSearchResult } from '../context_extractors/vector_search_tool.js'; +// VectorSearchResult type definition moved here since vector search tool was removed +export interface VectorSearchResult { + searchResults: Array; + totalResults: number; + executionTime: number; +} import type { NoteInfo, NotePathInfo, NoteHierarchyLevel } from '../context_extractors/note_navigator_tool.js'; import type { DecomposedQuery, SubQuery } from '../context_extractors/query_decomposition_tool.js'; import type { ThinkingProcess, ThinkingStep } from '../context_extractors/contextual_thinking_tool.js'; diff --git a/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts b/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts index 52736cbd5e..6fffbce70f 100644 --- a/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/ai_service_interfaces.ts @@ -7,7 +7,6 @@ export interface ProviderMetadata { name: string; capabilities: { chat: boolean; - embeddings: boolean; streaming: boolean; functionCalling?: boolean; }; diff --git a/apps/server/src/services/llm/interfaces/configuration_interfaces.ts b/apps/server/src/services/llm/interfaces/configuration_interfaces.ts index 6adcac9777..15bb88f0a1 100644 --- a/apps/server/src/services/llm/interfaces/configuration_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/configuration_interfaces.ts @@ -21,13 +21,6 @@ export interface ModelConfig { capabilities?: ModelCapabilities; } -/** - * Embedding provider precedence configuration - */ -export interface EmbeddingProviderPrecedenceConfig { - providers: EmbeddingProviderType[]; - defaultProvider?: EmbeddingProviderType; -} /** * Model capabilities @@ -47,7 +40,6 @@ export interface ModelCapabilities { export interface AIConfig { enabled: boolean; selectedProvider: ProviderType | null; - selectedEmbeddingProvider: EmbeddingProviderType | null; defaultModels: Record; providerSettings: ProviderSettings; } @@ -84,10 +76,6 @@ export interface OllamaSettings { */ export type ProviderType = 'openai' | 'anthropic' | 'ollama'; -/** - * Valid embedding provider types - */ -export type EmbeddingProviderType = 'openai' | 'voyage' | 'ollama' | 'local'; /** * Model identifier with provider prefix (e.g., "openai:gpt-4" or "ollama:llama2") diff --git a/apps/server/src/services/llm/interfaces/context_interfaces.ts b/apps/server/src/services/llm/interfaces/context_interfaces.ts index a23a374dbf..46d2994e64 100644 --- a/apps/server/src/services/llm/interfaces/context_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/context_interfaces.ts @@ -60,7 +60,6 @@ export interface IContextFormatter { */ export interface ILLMService { sendMessage(message: string, options?: Record): Promise; - generateEmbedding?(text: string): Promise; streamMessage?(message: string, callback: (text: string) => void, options?: Record): Promise; } diff --git a/apps/server/src/services/llm/interfaces/embedding_interfaces.ts b/apps/server/src/services/llm/interfaces/embedding_interfaces.ts deleted file mode 100644 index eaecd4cf98..0000000000 --- a/apps/server/src/services/llm/interfaces/embedding_interfaces.ts +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Interface for embedding provider configuration - */ -export interface EmbeddingProviderConfig { - name: string; - model: string; - dimension: number; - type: 'float32' | 'int8' | 'uint8' | 'float16'; - enabled?: boolean; - priority?: number; - baseUrl?: string; - apiKey?: string; - contextWidth?: number; - batchSize?: number; -} - -/** - * Interface for embedding model information - */ -export interface EmbeddingModelInfo { - name: string; - dimension: number; - contextWidth?: number; - maxBatchSize?: number; - tokenizer?: string; - type: 'float32' | 'int8' | 'uint8' | 'float16'; -} - -/** - * Interface for embedding provider - */ -export interface EmbeddingProvider { - getName(): string; - getModel(): string; - getDimension(): number; - getType(): 'float32' | 'int8' | 'uint8' | 'float16'; - isEnabled(): boolean; - getPriority(): number; - getMaxBatchSize(): number; - generateEmbedding(text: string): Promise; - generateBatchEmbeddings(texts: string[]): Promise; - initialize(): Promise; -} - -/** - * Interface for embedding process result - */ -export interface EmbeddingProcessResult { - noteId: string; - title: string; - success: boolean; - message?: string; - error?: Error; - chunks?: number; -} - -/** - * Interface for embedding queue item - */ -export interface EmbeddingQueueItem { - id: number; - noteId: string; - status: 'pending' | 'processing' | 'completed' | 'failed' | 'retrying'; - provider: string; - model: string; - dimension: number; - type: string; - attempts: number; - lastAttempt: string | null; - dateCreated: string; - dateCompleted: string | null; - error: string | null; - chunks: number; -} - -/** - * Interface for embedding batch processing - */ -export interface EmbeddingBatch { - texts: string[]; - noteIds: string[]; - indexes: number[]; -} - -/** - * Interface for embedding search result - */ -export interface EmbeddingSearchResult { - noteId: string; - similarity: number; - title?: string; - content?: string; - parentId?: string; - parentTitle?: string; - dateCreated?: string; - dateModified?: string; -} - -/** - * Interface for embedding chunk - */ -export interface EmbeddingChunk { - id: number; - noteId: string; - content: string; - embedding: Float32Array | Int8Array | Uint8Array; - metadata?: Record; -} diff --git a/apps/server/src/services/llm/interfaces/error_interfaces.ts b/apps/server/src/services/llm/interfaces/error_interfaces.ts index 542b497f57..54c6c3fc90 100644 --- a/apps/server/src/services/llm/interfaces/error_interfaces.ts +++ b/apps/server/src/services/llm/interfaces/error_interfaces.ts @@ -36,16 +36,6 @@ export interface OllamaError extends LLMServiceError { code?: string; } -/** - * Embedding-specific error interface - */ -export interface EmbeddingError extends LLMServiceError { - provider: string; - model?: string; - batchSize?: number; - isRetryable: boolean; -} - /** * Guard function to check if an error is a specific type of error */ diff --git a/apps/server/src/services/llm/model_capabilities_service.ts b/apps/server/src/services/llm/model_capabilities_service.ts index c327ebc9db..0174103195 100644 --- a/apps/server/src/services/llm/model_capabilities_service.ts +++ b/apps/server/src/services/llm/model_capabilities_service.ts @@ -1,159 +1,85 @@ import log from '../log.js'; import type { ModelCapabilities } from './interfaces/model_capabilities.js'; -import { MODEL_CAPABILITIES, DEFAULT_MODEL_CAPABILITIES } from './interfaces/model_capabilities.js'; +import { DEFAULT_MODEL_CAPABILITIES } from './interfaces/model_capabilities.js'; +import { MODEL_CAPABILITIES } from './constants/search_constants.js'; import aiServiceManager from './ai_service_manager.js'; -import { getEmbeddingProvider } from './providers/providers.js'; -import type { BaseEmbeddingProvider } from './embeddings/base_embeddings.js'; -import type { EmbeddingModelInfo } from './interfaces/embedding_interfaces.js'; - -// Define a type for embedding providers that might have the getModelInfo method -interface EmbeddingProviderWithModelInfo { - getModelInfo?: (modelName: string) => Promise; -} /** * Service for fetching and caching model capabilities + * Handles chat model capabilities */ export class ModelCapabilitiesService { // Cache model capabilities private capabilitiesCache: Map = new Map(); - constructor() { - // Initialize cache with known models - this.initializeCache(); - } - - /** - * Initialize the cache with known model capabilities - */ - private initializeCache() { - // Add all predefined model capabilities to cache - for (const [model, capabilities] of Object.entries(MODEL_CAPABILITIES)) { - this.capabilitiesCache.set(model, { - ...DEFAULT_MODEL_CAPABILITIES, - ...capabilities - }); - } - } - /** - * Get model capabilities, fetching from provider if needed - * - * @param modelName Full model name (with or without provider prefix) - * @returns Model capabilities + * Get capabilities for a chat model */ - async getModelCapabilities(modelName: string): Promise { - // Handle provider-prefixed model names (e.g., "openai:gpt-4") - let provider = 'default'; - let baseModelName = modelName; - - if (modelName.includes(':')) { - const parts = modelName.split(':'); - provider = parts[0]; - baseModelName = parts[1]; - } - + async getChatModelCapabilities(modelName: string): Promise { // Check cache first - const cacheKey = baseModelName; - if (this.capabilitiesCache.has(cacheKey)) { - return this.capabilitiesCache.get(cacheKey)!; + const cached = this.capabilitiesCache.get(`chat:${modelName}`); + if (cached) { + return cached; } - // Fetch from provider if possible - try { - // Get provider service - const providerService = aiServiceManager.getService(provider); - - if (providerService && typeof (providerService as any).getModelCapabilities === 'function') { - // If provider supports direct capability fetching, use it - const capabilities = await (providerService as any).getModelCapabilities(baseModelName); - - if (capabilities) { - // Merge with defaults and cache - const fullCapabilities = { - ...DEFAULT_MODEL_CAPABILITIES, - ...capabilities - }; - - this.capabilitiesCache.set(cacheKey, fullCapabilities); - log.info(`Fetched capabilities for ${modelName}: context window ${fullCapabilities.contextWindowTokens} tokens`); - - return fullCapabilities; - } - } - - // Try to fetch from embedding provider if available - const embeddingProvider = getEmbeddingProvider(provider); + // Get from static definitions or service + const capabilities = await this.fetchChatModelCapabilities(modelName); - if (embeddingProvider) { - try { - // Cast to a type that might have getModelInfo method - const providerWithModelInfo = embeddingProvider as unknown as EmbeddingProviderWithModelInfo; + // Cache the result + this.capabilitiesCache.set(`chat:${modelName}`, capabilities); - if (providerWithModelInfo.getModelInfo) { - const modelInfo = await providerWithModelInfo.getModelInfo(baseModelName); - - if (modelInfo && modelInfo.contextWidth) { - // Convert to our capabilities format - const capabilities: ModelCapabilities = { - ...DEFAULT_MODEL_CAPABILITIES, - contextWindowTokens: modelInfo.contextWidth, - contextWindowChars: modelInfo.contextWidth * 4 // Rough estimate: 4 chars per token - }; - - this.capabilitiesCache.set(cacheKey, capabilities); - log.info(`Derived capabilities for ${modelName} from embedding provider: context window ${capabilities.contextWindowTokens} tokens`); - - return capabilities; - } - } - } catch (error) { - log.info(`Could not get model info from embedding provider for ${modelName}: ${error}`); - } - } - } catch (error) { - log.error(`Error fetching model capabilities for ${modelName}: ${error}`); - } + return capabilities; + } - // If we get here, try to find a similar model in our predefined list - for (const knownModel of Object.keys(MODEL_CAPABILITIES)) { - // Check if the model name contains this known model (e.g., "gpt-4-1106-preview" contains "gpt-4") - if (baseModelName.includes(knownModel)) { - const capabilities = { + /** + * Fetch chat model capabilities from AI service or static definitions + */ + private async fetchChatModelCapabilities(modelName: string): Promise { + try { + // Try to get from static definitions first + const staticCapabilities = MODEL_CAPABILITIES[modelName.toLowerCase()]; + if (staticCapabilities) { + log.info(`Using static capabilities for chat model: ${modelName}`); + // Merge partial capabilities with defaults + return { ...DEFAULT_MODEL_CAPABILITIES, - ...MODEL_CAPABILITIES[knownModel] + ...staticCapabilities }; - - this.capabilitiesCache.set(cacheKey, capabilities); - log.info(`Using similar model (${knownModel}) capabilities for ${modelName}`); - - return capabilities; } - } - // Fall back to defaults if nothing else works - log.info(`Using default capabilities for unknown model ${modelName}`); - this.capabilitiesCache.set(cacheKey, DEFAULT_MODEL_CAPABILITIES); + // AI service doesn't have getModelCapabilities method + // Use default capabilities instead + log.info(`AI service doesn't support model capabilities - using defaults for model: ${modelName}`); - return DEFAULT_MODEL_CAPABILITIES; + // Fallback to default capabilities + log.info(`Using default capabilities for chat model: ${modelName}`); + return DEFAULT_MODEL_CAPABILITIES; + } catch (error) { + log.error(`Error fetching capabilities for chat model ${modelName}: ${error}`); + return DEFAULT_MODEL_CAPABILITIES; + } } /** - * Update model capabilities in the cache - * - * @param modelName Model name - * @param capabilities Capabilities to update + * Clear capabilities cache */ - updateModelCapabilities(modelName: string, capabilities: Partial) { - const currentCapabilities = this.capabilitiesCache.get(modelName) || DEFAULT_MODEL_CAPABILITIES; + clearCache(): void { + this.capabilitiesCache.clear(); + log.info('Model capabilities cache cleared'); + } - this.capabilitiesCache.set(modelName, { - ...currentCapabilities, - ...capabilities - }); + /** + * Get all cached capabilities + */ + getCachedCapabilities(): Record { + const result: Record = {}; + for (const [key, value] of this.capabilitiesCache.entries()) { + result[key] = value; + } + return result; } } -// Create and export singleton instance -const modelCapabilitiesService = new ModelCapabilitiesService(); +// Export singleton instance +export const modelCapabilitiesService = new ModelCapabilitiesService(); export default modelCapabilitiesService; diff --git a/apps/server/src/services/llm/pipeline/chat_pipeline.ts b/apps/server/src/services/llm/pipeline/chat_pipeline.ts index 50671a8091..60c5df87c0 100644 --- a/apps/server/src/services/llm/pipeline/chat_pipeline.ts +++ b/apps/server/src/services/llm/pipeline/chat_pipeline.ts @@ -8,7 +8,7 @@ import { ModelSelectionStage } from './stages/model_selection_stage.js'; import { LLMCompletionStage } from './stages/llm_completion_stage.js'; import { ResponseProcessingStage } from './stages/response_processing_stage.js'; import { ToolCallingStage } from './stages/tool_calling_stage.js'; -import { VectorSearchStage } from './stages/vector_search_stage.js'; +// Traditional search is used instead of vector search import toolRegistry from '../tools/tool_registry.js'; import toolInitializer from '../tools/tool_initializer.js'; import log from '../../log.js'; @@ -29,7 +29,7 @@ export class ChatPipeline { llmCompletion: LLMCompletionStage; responseProcessing: ResponseProcessingStage; toolCalling: ToolCallingStage; - vectorSearch: VectorSearchStage; + // traditional search is used instead of vector search }; config: ChatPipelineConfig; @@ -50,7 +50,7 @@ export class ChatPipeline { llmCompletion: new LLMCompletionStage(), responseProcessing: new ResponseProcessingStage(), toolCalling: new ToolCallingStage(), - vectorSearch: new VectorSearchStage() + // traditional search is used instead of vector search }; // Set default configuration values @@ -198,27 +198,20 @@ export class ChatPipeline { log.info('No LLM service available for query decomposition, using original query'); } - // STAGE 3: Execute vector similarity search with decomposed queries + // STAGE 3: Vector search has been removed - skip semantic search const vectorSearchStartTime = Date.now(); - log.info(`========== STAGE 3: VECTOR SEARCH ==========`); - log.info('Using VectorSearchStage pipeline component to find relevant notes'); - log.info(`Searching with ${searchQueries.length} queries from decomposition`); - - // Use the vectorSearchStage with multiple queries - const vectorSearchResult = await this.stages.vectorSearch.execute({ - query: userQuery, // Original query as fallback - queries: searchQueries, // All decomposed queries - noteId: input.noteId || 'global', - options: { - maxResults: SEARCH_CONSTANTS.CONTEXT.MAX_SIMILAR_NOTES, - useEnhancedQueries: false, // We're already using enhanced queries from decomposition - threshold: SEARCH_CONSTANTS.VECTOR_SEARCH.DEFAULT_THRESHOLD, - llmService: llmService || undefined - } - }); + log.info(`========== STAGE 3: VECTOR SEARCH (DISABLED) ==========`); + log.info('Vector search has been removed - LLM will rely on tool calls for context'); + + // Create empty vector search result since vector search is disabled + const vectorSearchResult = { + searchResults: [], + totalResults: 0, + executionTime: Date.now() - vectorSearchStartTime + }; - this.updateStageMetrics('vectorSearch', vectorSearchStartTime); - log.info(`Vector search found ${vectorSearchResult.searchResults.length} relevant notes across ${searchQueries.length} queries`); + // Skip metrics update for disabled vector search functionality + log.info(`Vector search disabled - using tool-based context extraction instead`); // Extract context from search results log.info(`========== SEMANTIC CONTEXT EXTRACTION ==========`); @@ -314,7 +307,7 @@ export class ChatPipeline { // Forward to callback with original chunk data in case it contains additional information streamCallback(processedChunk.text, processedChunk.done, chunk); - + // Mark that we have streamed content to prevent duplication hasStreamedContent = true; }); @@ -906,6 +899,12 @@ export class ChatPipeline { const executionTime = Date.now() - startTime; const metrics = this.metrics.stageMetrics[stageName]; + // Guard against undefined metrics (e.g., for removed stages) + if (!metrics) { + log.info(`WARNING: Attempted to update metrics for unknown stage: ${stageName}`); + return; + } + metrics.totalExecutions++; metrics.averageExecutionTime = (metrics.averageExecutionTime * (metrics.totalExecutions - 1) + executionTime) / diff --git a/apps/server/src/services/llm/pipeline/stages/semantic_context_extraction_stage.ts b/apps/server/src/services/llm/pipeline/stages/semantic_context_extraction_stage.ts index 1395106637..03a8f2b733 100644 --- a/apps/server/src/services/llm/pipeline/stages/semantic_context_extraction_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/semantic_context_extraction_stage.ts @@ -1,70 +1,27 @@ import { BasePipelineStage } from '../pipeline_stage.js'; import type { SemanticContextExtractionInput } from '../interfaces.js'; -import aiServiceManager from '../../ai_service_manager.js'; import log from '../../../log.js'; -import { VectorSearchStage } from './vector_search_stage.js'; -import contextFormatter from '../../context/modules/context_formatter.js'; -import providerManager from '../../context/modules/provider_manager.js'; -import type { NoteSearchResult } from '../../interfaces/context_interfaces.js'; -import type { Message } from '../../ai_interface.js'; -import { SEARCH_CONSTANTS } from "../../constants/search_constants.js"; /** * Pipeline stage for extracting semantic context from notes - * This uses the new VectorSearchStage to find relevant content + * Since vector search has been removed, this now returns empty context + * and relies on other context extraction methods */ export class SemanticContextExtractionStage extends BasePipelineStage { - private vectorSearchStage: VectorSearchStage; - constructor() { super('SemanticContextExtraction'); - this.vectorSearchStage = new VectorSearchStage(); } /** * Extract semantic context based on a query + * Returns empty context since vector search has been removed */ protected async process(input: SemanticContextExtractionInput): Promise<{ context: string }> { - const { noteId, query, maxResults = 5, messages = [] } = input; - log.info(`Extracting semantic context from note ${noteId}, query: ${query?.substring(0, 50)}...`); - - try { - // Step 1: Use vector search stage to find relevant notes - const vectorSearchResult = await this.vectorSearchStage.execute({ - query, - noteId, - options: { - maxResults, - useEnhancedQueries: true, - threshold: SEARCH_CONSTANTS.VECTOR_SEARCH.DEFAULT_THRESHOLD, - llmService: undefined // Let the vectorSearchStage use the default service - } - }); - - log.info(`Vector search found ${vectorSearchResult.searchResults.length} relevant notes`); - - // If no results, return empty context - if (vectorSearchResult.searchResults.length === 0) { - log.info(`No relevant notes found for context extraction`); - return { context: "" }; - } - - // Step 2: Format search results into a context string - const provider = await providerManager.getSelectedEmbeddingProvider(); - const providerId = provider?.name || 'default'; - - const context = await contextFormatter.buildContextFromNotes( - vectorSearchResult.searchResults, - query, - providerId, - messages - ); + const { noteId, query } = input; + log.info(`Semantic context extraction disabled - vector search has been removed. Using tool-based context instead for note ${noteId}`); - log.info(`Built context of ${context.length} chars from ${vectorSearchResult.searchResults.length} notes`); - return { context }; - } catch (error) { - log.error(`Error extracting semantic context: ${error}`); - return { context: "" }; - } + // Return empty context since we no longer use vector search + // The LLM will rely on tool calls for context gathering + return { context: "" }; } -} +} \ No newline at end of file diff --git a/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts b/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts index f988a63949..8299f8fd64 100644 --- a/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts +++ b/apps/server/src/services/llm/pipeline/stages/tool_calling_stage.ts @@ -37,11 +37,7 @@ interface ToolValidationResult { export class ToolCallingStage extends BasePipelineStage { constructor() { super('ToolCalling'); - - // Preload the vectorSearchTool to ensure it's available when needed - this.preloadVectorSearchTool().catch(error => { - log.error(`Error preloading vector search tool: ${error.message}`); - }); + // Vector search tool has been removed - no preloading needed } /** @@ -498,13 +494,13 @@ export class ToolCallingStage extends BasePipelineStage { - const aiServiceManager = (await import('../../ai_service_manager.js')).default; - - try { - log.info(`Getting dependency '${dependencyType}' for tool '${toolName}'`); - - // Check for specific dependency types - if (dependencyType === 'vectorSearchTool') { - // Try to get the existing vector search tool - let vectorSearchTool = aiServiceManager.getVectorSearchTool(); - - if (vectorSearchTool) { - log.info(`Found existing vectorSearchTool dependency`); - return vectorSearchTool; - } - - // No existing tool, try to initialize it - log.info(`Dependency '${dependencyType}' not found, attempting initialization`); - - // Get agent tools manager and initialize it - const agentTools = aiServiceManager.getAgentTools(); - if (agentTools && typeof agentTools.initialize === 'function') { - try { - // Force initialization to ensure it runs even if previously marked as initialized - await agentTools.initialize(true); - } catch (initError: unknown) { - const errorMessage = initError instanceof Error ? initError.message : String(initError); - log.error(`Failed to initialize agent tools: ${errorMessage}`); - return null; - } - } else { - log.error('Agent tools manager not available'); - return null; - } - - // Try getting the vector search tool again after initialization - vectorSearchTool = aiServiceManager.getVectorSearchTool(); - - if (vectorSearchTool) { - log.info('Successfully created vectorSearchTool dependency'); - return vectorSearchTool; - } else { - log.error('Failed to create vectorSearchTool dependency after initialization'); - return null; - } - } - - // Add more dependency types as needed - - // Unknown dependency type - log.error(`Unknown dependency type: ${dependencyType}`); - return null; - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : String(error); - log.error(`Error getting or creating dependency '${dependencyType}': ${errorMessage}`); - return null; - } - } /** * Validate a tool before execution @@ -614,50 +545,9 @@ export class ToolCallingStage extends BasePipelineStage name.includes('search')); guidance += `AVAILABLE SEARCH TOOLS: ${searchTools.join(', ')}\n`; - guidance += "TRY VECTOR SEARCH: For conceptual matches, use 'vector_search' with a query parameter.\n"; + guidance += "TRY SEARCH NOTES: For semantic matches, use 'search_notes' with a query parameter.\n"; guidance += "EXAMPLE: { \"query\": \"your search terms here\" }\n"; } else if (errorMessage.includes('missing required parameter')) { // Provide parameter guidance based on the tool name - if (toolName === 'vector_search') { - guidance += "REQUIRED PARAMETERS: The 'vector_search' tool requires a 'query' parameter.\n"; + if (toolName === 'search_notes') { + guidance += "REQUIRED PARAMETERS: The 'search_notes' tool requires a 'query' parameter.\n"; guidance += "EXAMPLE: { \"query\": \"your search terms here\" }\n"; } else if (toolName === 'keyword_search') { guidance += "REQUIRED PARAMETERS: The 'keyword_search' tool requires a 'query' parameter.\n"; @@ -719,9 +609,9 @@ export class ToolCallingStage extends BasePipelineStage { - try { - log.info(`Preloading vector search tool...`); - - // Get the agent tools and initialize them if needed - const agentTools = aiServiceManager.getAgentTools(); - if (agentTools && typeof agentTools.initialize === 'function') { - await agentTools.initialize(true); - } - - // Check if the vector search tool is available - const vectorSearchTool = aiServiceManager.getVectorSearchTool(); - if (!(vectorSearchTool && typeof vectorSearchTool.searchNotes === 'function')) { - log.error(`Vector search tool not available after initialization`); - } - } catch (error: unknown) { - const errorMessage = error instanceof Error ? error.message : String(error); - log.error(`Failed to preload vector search tool: ${errorMessage}`); - } - } } diff --git a/apps/server/src/services/llm/pipeline/stages/vector_search_stage.ts b/apps/server/src/services/llm/pipeline/stages/vector_search_stage.ts deleted file mode 100644 index 306b5079d4..0000000000 --- a/apps/server/src/services/llm/pipeline/stages/vector_search_stage.ts +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Vector Search Stage - * - * Part of the chat pipeline that handles finding semantically relevant notes - * using vector similarity search. - */ - -import log from '../../../log.js'; -import vectorSearchService from '../../context/services/vector_search_service.js'; -import type { NoteSearchResult } from '../../interfaces/context_interfaces.js'; -import type { LLMServiceInterface } from '../../interfaces/agent_tool_interfaces.js'; -import { SEARCH_CONSTANTS } from '../../constants/search_constants.js'; - -export interface VectorSearchInput { - query: string; - queries?: string[]; - noteId?: string; - options?: { - maxResults?: number; - threshold?: number; - useEnhancedQueries?: boolean; - llmService?: LLMServiceInterface; - }; -} - -export interface VectorSearchOutput { - searchResults: NoteSearchResult[]; - originalQuery: string; - noteId: string; -} - -/** - * Pipeline stage for performing vector-based semantic search - */ -export class VectorSearchStage { - constructor() { - log.info('VectorSearchStage initialized'); - } - - /** - * Execute vector search to find relevant notes - */ - async execute(input: VectorSearchInput): Promise { - const { - query, - queries = [], - noteId = 'global', - options = {} - } = input; - - const { - maxResults = SEARCH_CONSTANTS.VECTOR_SEARCH.DEFAULT_MAX_RESULTS, - threshold = SEARCH_CONSTANTS.VECTOR_SEARCH.DEFAULT_THRESHOLD, - useEnhancedQueries = false, - llmService = undefined - } = options; - - // If queries array is provided, use multi-query search - if (queries && queries.length > 0) { - log.info(`VectorSearchStage: Searching with ${queries.length} queries`); - log.info(`Parameters: noteId=${noteId}, maxResults=${maxResults}, threshold=${threshold}`); - - try { - // Use the new multi-query method - const searchResults = await vectorSearchService.findRelevantNotesMultiQuery( - queries, - noteId === 'global' ? null : noteId, - { - maxResults, - threshold, - llmService: llmService || null - } - ); - - log.info(`VectorSearchStage: Found ${searchResults.length} relevant notes from multi-query search`); - - return { - searchResults, - originalQuery: query, - noteId - }; - } catch (error) { - log.error(`Error in vector search stage multi-query: ${error}`); - // Return empty results on error - return { - searchResults: [], - originalQuery: query, - noteId - }; - } - } - - // Fallback to single query search - log.info(`VectorSearchStage: Searching for "${query.substring(0, 50)}..."`); - log.info(`Parameters: noteId=${noteId}, maxResults=${maxResults}, threshold=${threshold}`); - - try { - // Find relevant notes using vector search service - const searchResults = await vectorSearchService.findRelevantNotes( - query, - noteId === 'global' ? null : noteId, - { - maxResults, - threshold, - llmService: llmService || null - } - ); - - log.info(`VectorSearchStage: Found ${searchResults.length} relevant notes`); - - return { - searchResults, - originalQuery: query, - noteId - }; - } catch (error) { - log.error(`Error in vector search stage: ${error}`); - // Return empty results on error - return { - searchResults: [], - originalQuery: query, - noteId - }; - } - } -} diff --git a/apps/server/src/services/llm/provider_validation.ts b/apps/server/src/services/llm/provider_validation.ts index 6a94153e84..95d5a8b7ea 100644 --- a/apps/server/src/services/llm/provider_validation.ts +++ b/apps/server/src/services/llm/provider_validation.ts @@ -1,17 +1,15 @@ /** * Provider Validation Service - * - * Validates AI provider configurations before initializing the embedding system. + * + * Validates AI provider configurations before initializing the chat system. * This prevents startup errors when AI is enabled but providers are misconfigured. */ import log from "../log.js"; import options from "../options.js"; -import type { EmbeddingProvider } from "./embeddings/embeddings_interface.js"; export interface ProviderValidationResult { hasValidProviders: boolean; - validEmbeddingProviders: EmbeddingProvider[]; validChatProviders: string[]; errors: string[]; warnings: string[]; @@ -23,73 +21,35 @@ export interface ProviderValidationResult { export async function validateProviders(): Promise { const result: ProviderValidationResult = { hasValidProviders: false, - validEmbeddingProviders: [], validChatProviders: [], errors: [], warnings: [] }; - try { - // Check if AI is enabled - const aiEnabled = await options.getOptionBool('aiEnabled'); - if (!aiEnabled) { - result.warnings.push("AI features are disabled"); - return result; - } + log.info("Starting provider validation..."); - // Check configuration only - don't create providers - await checkEmbeddingProviderConfigs(result); - await checkChatProviderConfigs(result); + // Check if AI is enabled + const aiEnabled = await options.getOptionBool('aiEnabled'); + if (!aiEnabled) { + log.info("AI is disabled, skipping provider validation"); + return result; + } - // Determine if we have any valid providers based on configuration - result.hasValidProviders = result.validChatProviders.length > 0; + // Check chat provider configurations + await checkChatProviderConfigs(result); - if (!result.hasValidProviders) { - result.errors.push("No valid AI providers are configured"); - } + // Update overall validation status + result.hasValidProviders = result.validChatProviders.length > 0; - } catch (error: any) { - result.errors.push(`Error during provider validation: ${error.message || 'Unknown error'}`); + if (result.hasValidProviders) { + log.info(`Provider validation successful. Valid chat providers: ${result.validChatProviders.join(', ')}`); + } else { + log.info("No valid providers found"); } return result; } -/** - * Check embedding provider configurations without creating providers - */ -async function checkEmbeddingProviderConfigs(result: ProviderValidationResult): Promise { - try { - // Check OpenAI embedding configuration - const openaiApiKey = await options.getOption('openaiApiKey'); - const openaiBaseUrl = await options.getOption('openaiBaseUrl'); - if (openaiApiKey || openaiBaseUrl) { - if (!openaiApiKey) { - result.warnings.push("OpenAI embedding: No API key (may work with compatible endpoints)"); - } - log.info("OpenAI embedding provider configuration available"); - } - - // Check Ollama embedding configuration - const ollamaEmbeddingBaseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); - if (ollamaEmbeddingBaseUrl) { - log.info("Ollama embedding provider configuration available"); - } - - // Check Voyage embedding configuration - const voyageApiKey = await options.getOption('voyageApiKey' as any); - if (voyageApiKey) { - log.info("Voyage embedding provider configuration available"); - } - - // Local provider is always available - log.info("Local embedding provider available as fallback"); - - } catch (error: any) { - result.errors.push(`Error checking embedding provider configs: ${error.message || 'Unknown error'}`); - } -} - /** * Check chat provider configurations without creating providers */ @@ -98,28 +58,24 @@ async function checkChatProviderConfigs(result: ProviderValidationResult): Promi // Check OpenAI chat provider const openaiApiKey = await options.getOption('openaiApiKey'); const openaiBaseUrl = await options.getOption('openaiBaseUrl'); - + if (openaiApiKey || openaiBaseUrl) { - if (!openaiApiKey) { - result.warnings.push("OpenAI chat: No API key (may work with compatible endpoints)"); - } result.validChatProviders.push('openai'); + log.info("OpenAI chat provider configuration available"); } // Check Anthropic chat provider const anthropicApiKey = await options.getOption('anthropicApiKey'); if (anthropicApiKey) { result.validChatProviders.push('anthropic'); + log.info("Anthropic chat provider configuration available"); } // Check Ollama chat provider const ollamaBaseUrl = await options.getOption('ollamaBaseUrl'); if (ollamaBaseUrl) { result.validChatProviders.push('ollama'); - } - - if (result.validChatProviders.length === 0) { - result.warnings.push("No chat providers configured. Please configure at least one provider."); + log.info("Ollama chat provider configuration available"); } } catch (error: any) { @@ -127,51 +83,6 @@ async function checkChatProviderConfigs(result: ProviderValidationResult): Promi } } - -/** - * Check if any chat providers are configured - */ -export async function hasWorkingChatProviders(): Promise { - const validation = await validateProviders(); - return validation.validChatProviders.length > 0; -} - -/** - * Check if any embedding providers are configured (simplified) - */ -export async function hasWorkingEmbeddingProviders(): Promise { - if (!(await options.getOptionBool('aiEnabled'))) { - return false; - } - - // Check if any embedding provider is configured - const openaiKey = await options.getOption('openaiApiKey'); - const openaiBaseUrl = await options.getOption('openaiBaseUrl'); - const ollamaUrl = await options.getOption('ollamaEmbeddingBaseUrl'); - const voyageKey = await options.getOption('voyageApiKey' as any); - - // Local provider is always available as fallback - return !!(openaiKey || openaiBaseUrl || ollamaUrl || voyageKey) || true; -} - -/** - * Log validation results in a user-friendly way - */ -export function logValidationResults(validation: ProviderValidationResult): void { - if (validation.hasValidProviders) { - log.info(`AI provider validation passed: ${validation.validEmbeddingProviders.length} embedding providers, ${validation.validChatProviders.length} chat providers`); - - if (validation.validEmbeddingProviders.length > 0) { - log.info(`Working embedding providers: ${validation.validEmbeddingProviders.map(p => p.name).join(', ')}`); - } - - if (validation.validChatProviders.length > 0) { - log.info(`Working chat providers: ${validation.validChatProviders.join(', ')}`); - } - } else { - log.info("AI provider validation failed: No working providers found"); - } - - validation.warnings.forEach(warning => log.info(`Provider validation: ${warning}`)); - validation.errors.forEach(error => log.error(`Provider validation: ${error}`)); -} \ No newline at end of file +export default { + validateProviders +}; diff --git a/apps/server/src/services/llm/providers/providers.ts b/apps/server/src/services/llm/providers/providers.ts index dae8b34a02..8575fa6aa6 100644 --- a/apps/server/src/services/llm/providers/providers.ts +++ b/apps/server/src/services/llm/providers/providers.ts @@ -1,13 +1,5 @@ import options from "../../options.js"; import log from "../../log.js"; -import sql from "../../sql.js"; -import dateUtils from "../../date_utils.js"; -import { randomString } from "../../utils.js"; -import type { EmbeddingProvider, EmbeddingConfig } from "../embeddings/embeddings_interface.js"; -import { NormalizationStatus } from "../embeddings/embeddings_interface.js"; -import { OpenAIEmbeddingProvider } from "../embeddings/providers/openai.js"; -import { OllamaEmbeddingProvider } from "../embeddings/providers/ollama.js"; -import { VoyageEmbeddingProvider } from "../embeddings/providers/voyage.js"; import type { OptionDefinitions } from "@triliumnext/commons"; import type { ChatCompletionOptions } from '../ai_interface.js'; import type { OpenAIOptions, AnthropicOptions, OllamaOptions, ModelMetadata } from './provider_options.js'; @@ -19,347 +11,6 @@ import { import { PROVIDER_CONSTANTS } from '../constants/provider_constants.js'; import { SEARCH_CONSTANTS, MODEL_CAPABILITIES } from '../constants/search_constants.js'; -/** - * Simple local embedding provider implementation - * This avoids the need to import a separate file which might not exist - */ -class SimpleLocalEmbeddingProvider implements EmbeddingProvider { - name = "local"; - config: EmbeddingConfig; - - constructor(config: EmbeddingConfig) { - this.config = config; - } - - getConfig(): EmbeddingConfig { - return this.config; - } - - /** - * Returns the normalization status of the local provider - * Local provider does not guarantee normalization - */ - getNormalizationStatus(): NormalizationStatus { - return NormalizationStatus.NEVER; // Simple embedding does not normalize vectors - } - - async generateEmbeddings(text: string): Promise { - // Create deterministic embeddings based on text content - const result = new Float32Array(this.config.dimension || 384); - - // Simple hash-based approach - for (let i = 0; i < result.length; i++) { - // Use character codes and position to generate values between -1 and 1 - const charSum = Array.from(text).reduce((sum, char, idx) => - sum + char.charCodeAt(0) * Math.sin(idx * 0.1), 0); - result[i] = Math.sin(i * 0.1 + charSum * 0.01); - } - - return result; - } - - async generateBatchEmbeddings(texts: string[]): Promise { - return Promise.all(texts.map(text => this.generateEmbeddings(text))); - } - - async generateNoteEmbeddings(context: any): Promise { - // Combine text from context - const text = (context.title || "") + " " + (context.content || ""); - return this.generateEmbeddings(text); - } - - async generateBatchNoteEmbeddings(contexts: any[]): Promise { - return Promise.all(contexts.map(context => this.generateNoteEmbeddings(context))); - } -} - -const providers = new Map(); - -// Cache to track which provider errors have been logged -const loggedProviderErrors = new Set(); - -/** - * Register a new embedding provider - */ -export function registerEmbeddingProvider(provider: EmbeddingProvider) { - providers.set(provider.name, provider); - log.info(`Registered embedding provider: ${provider.name}`); -} - -/** - * Unregister an embedding provider - */ -export function unregisterEmbeddingProvider(name: string): boolean { - const existed = providers.has(name); - if (existed) { - providers.delete(name); - log.info(`Unregistered embedding provider: ${name}`); - } - return existed; -} - -/** - * Clear all embedding providers - */ -export function clearAllEmbeddingProviders(): void { - const providerNames = Array.from(providers.keys()); - providers.clear(); - if (providerNames.length > 0) { - log.info(`Cleared all embedding providers: ${providerNames.join(', ')}`); - } -} - -/** - * Get all registered embedding providers - */ -export function getEmbeddingProviders(): EmbeddingProvider[] { - return Array.from(providers.values()); -} - -/** - * Get a specific embedding provider by name - */ -export function getEmbeddingProvider(name: string): EmbeddingProvider | undefined { - return providers.get(name); -} - -/** - * Get or create a specific embedding provider with inline validation - */ -export async function getOrCreateEmbeddingProvider(providerName: string): Promise { - // Return existing provider if already created and valid - const existing = providers.get(providerName); - if (existing) { - return existing; - } - - // Create and validate provider on-demand - try { - let provider: EmbeddingProvider | null = null; - - switch (providerName) { - case 'ollama': { - const baseUrl = await options.getOption('ollamaEmbeddingBaseUrl'); - if (!baseUrl) return null; - - const model = await options.getOption('ollamaEmbeddingModel'); - provider = new OllamaEmbeddingProvider({ - model, - dimension: 768, - type: 'float32', - baseUrl - }); - - // Validate by initializing (if provider supports it) - if ('initialize' in provider && typeof provider.initialize === 'function') { - await provider.initialize(); - } - break; - } - - case 'openai': { - const apiKey = await options.getOption('openaiApiKey'); - const baseUrl = await options.getOption('openaiBaseUrl'); - if (!apiKey && !baseUrl) return null; - - const model = await options.getOption('openaiEmbeddingModel'); - provider = new OpenAIEmbeddingProvider({ - model, - dimension: 1536, - type: 'float32', - apiKey: apiKey || '', - baseUrl: baseUrl || 'https://api.openai.com/v1' - }); - - if (!apiKey) { - log.info('OpenAI embedding provider created without API key for compatible endpoints'); - } - break; - } - - case 'voyage': { - const apiKey = await options.getOption('voyageApiKey' as any); - if (!apiKey) return null; - - const model = await options.getOption('voyageEmbeddingModel') || 'voyage-2'; - provider = new VoyageEmbeddingProvider({ - model, - dimension: 1024, - type: 'float32', - apiKey, - baseUrl: 'https://api.voyageai.com/v1' - }); - break; - } - - case 'local': { - provider = new SimpleLocalEmbeddingProvider({ - model: 'local', - dimension: 384, - type: 'float32' - }); - break; - } - - default: - return null; - } - - if (provider) { - registerEmbeddingProvider(provider); - log.info(`Created and validated ${providerName} embedding provider`); - return provider; - } - } catch (error: any) { - log.error(`Failed to create ${providerName} embedding provider: ${error.message || 'Unknown error'}`); - } - - return null; -} - -/** - * Get all enabled embedding providers for the specified feature - */ -export async function getEnabledEmbeddingProviders(feature: 'embeddings' | 'chat' = 'embeddings'): Promise { - if (!(await options.getOptionBool('aiEnabled'))) { - return []; - } - - const result: EmbeddingProvider[] = []; - - // Get the selected provider for the feature - const selectedProvider = feature === 'embeddings' - ? await options.getOption('embeddingSelectedProvider') - : await options.getOption('aiSelectedProvider'); - - // Try to get or create the specific selected provider - const provider = await getOrCreateEmbeddingProvider(selectedProvider); - if (!provider) { - throw new Error(`Failed to create selected embedding provider: ${selectedProvider}. Please check your configuration.`); - } - result.push(provider); - - - // Always ensure local provider as fallback - const localProvider = await getOrCreateEmbeddingProvider('local'); - if (localProvider && !result.some(p => p.name === 'local')) { - result.push(localProvider); - } - - return result; -} - -/** - * Create a new embedding provider configuration in the database - */ -export async function createEmbeddingProviderConfig( - name: string, - config: EmbeddingConfig, - priority = 0 -): Promise { - const providerId = randomString(16); - const now = dateUtils.localNowDateTime(); - const utcNow = dateUtils.utcNowDateTime(); - - await sql.execute(` - INSERT INTO embedding_providers - (providerId, name, priority, config, - dateCreated, utcDateCreated, dateModified, utcDateModified) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - [providerId, name, priority, JSON.stringify(config), - now, utcNow, now, utcNow] - ); - - return providerId; -} - -/** - * Update an existing embedding provider configuration - */ -export async function updateEmbeddingProviderConfig( - providerId: string, - priority?: number, - config?: EmbeddingConfig -): Promise { - const now = dateUtils.localNowDateTime(); - const utcNow = dateUtils.utcNowDateTime(); - - // Get existing provider - const provider = await sql.getRow( - "SELECT * FROM embedding_providers WHERE providerId = ?", - [providerId] - ); - - if (!provider) { - return false; - } - - // Build update query parts - const updates: string[] = []; - const params: any[] = []; - - if (priority !== undefined) { - updates.push("priority = ?"); - params.push(priority); - } - - if (config) { - updates.push("config = ?"); - params.push(JSON.stringify(config)); - } - - if (updates.length === 0) { - return true; // Nothing to update - } - - updates.push("dateModified = ?"); - updates.push("utcDateModified = ?"); - params.push(now, utcNow); - - params.push(providerId); - - // Execute update - await sql.execute( - `UPDATE embedding_providers SET ${updates.join(", ")} WHERE providerId = ?`, - params - ); - - return true; -} - -/** - * Delete an embedding provider configuration - */ -export async function deleteEmbeddingProviderConfig(providerId: string): Promise { - const result = await sql.execute( - "DELETE FROM embedding_providers WHERE providerId = ?", - [providerId] - ); - - return result.changes > 0; -} - -/** - * Get all embedding provider configurations from the database - */ -export async function getEmbeddingProviderConfigs() { - return await sql.getRows("SELECT * FROM embedding_providers ORDER BY priority DESC"); -} - -export default { - registerEmbeddingProvider, - unregisterEmbeddingProvider, - clearAllEmbeddingProviders, - getEmbeddingProviders, - getEmbeddingProvider, - getEnabledEmbeddingProviders, - getOrCreateEmbeddingProvider, - createEmbeddingProviderConfig, - updateEmbeddingProviderConfig, - deleteEmbeddingProviderConfig, - getEmbeddingProviderConfigs -}; - /** * Get OpenAI provider options from chat options and configuration * Updated to use provider metadata approach @@ -598,4 +249,4 @@ async function getOllamaModelContextWindow(modelName: string): Promise { log.info(`Error getting context window for model ${modelName}: ${error}`); return MODEL_CAPABILITIES['default'].contextWindowTokens; // Default fallback } -} +} \ No newline at end of file diff --git a/apps/server/src/services/llm/tools/note_summarization_tool.ts b/apps/server/src/services/llm/tools/note_summarization_tool.ts index 8fa5d39d8e..6a4bc42f57 100644 --- a/apps/server/src/services/llm/tools/note_summarization_tool.ts +++ b/apps/server/src/services/llm/tools/note_summarization_tool.ts @@ -130,8 +130,8 @@ export class NoteSummarizationTool implements ToolHandler { { role: 'system', content: 'You are a skilled summarizer. Create concise, accurate summaries while preserving the key information.' }, { role: 'user', content: prompt } ], { - temperature: SEARCH_CONSTANTS.TEMPERATURE.VECTOR_SEARCH, // Lower temperature for more focused summaries - maxTokens: SEARCH_CONSTANTS.LIMITS.VECTOR_SEARCH_MAX_TOKENS // Enough tokens for the summary + temperature: SEARCH_CONSTANTS.TEMPERATURE.QUERY_PROCESSOR, // Lower temperature for more focused summaries + maxTokens: SEARCH_CONSTANTS.LIMITS.DEFAULT_MAX_TOKENS // Enough tokens for the summary }); const summaryDuration = Date.now() - summaryStartTime; diff --git a/apps/server/src/services/llm/tools/relationship_tool.ts b/apps/server/src/services/llm/tools/relationship_tool.ts index d0a91c98d3..9466eb42d6 100644 --- a/apps/server/src/services/llm/tools/relationship_tool.ts +++ b/apps/server/src/services/llm/tools/relationship_tool.ts @@ -10,7 +10,24 @@ import becca from '../../../becca/becca.js'; import attributes from '../../attributes.js'; import aiServiceManager from '../ai_service_manager.js'; import { SEARCH_CONSTANTS } from '../constants/search_constants.js'; -import type { Backlink, RelatedNote } from '../embeddings/embeddings_interface.js'; +import searchService from '../../search/services/search.js'; +// Define types locally for relationship tool +interface Backlink { + noteId: string; + title: string; + relationName: string; + sourceNoteId: string; + sourceTitle: string; +} + +interface RelatedNote { + noteId: string; + title: string; + similarity: number; + relationName: string; + targetNoteId: string; + targetTitle: string; +} interface Suggestion { targetNoteId: string; @@ -195,6 +212,9 @@ export class RelationshipTool implements ToolHandler { if (targetNote) { outgoingRelations.push({ + noteId: targetNote.noteId, + title: targetNote.title, + similarity: 1.0, relationName: attr.name, targetNoteId: targetNote.noteId, targetTitle: targetNote.title @@ -215,6 +235,8 @@ export class RelationshipTool implements ToolHandler { if (sourceOfRelation && !sourceOfRelation.isDeleted) { incomingRelations.push({ + noteId: sourceOfRelation.noteId, + title: sourceOfRelation.title, relationName: attr.name, sourceNoteId: sourceOfRelation.noteId, sourceTitle: sourceOfRelation.title @@ -244,51 +266,87 @@ export class RelationshipTool implements ToolHandler { } /** - * Find related notes using vector similarity + * Find related notes using TriliumNext's search service */ private async findRelatedNotes(sourceNote: any, limit: number): Promise { try { - // Get the vector search tool from the AI service manager - const vectorSearchTool = aiServiceManager.getVectorSearchTool(); + log.info(`Using TriliumNext search to find notes related to "${sourceNote.title}"`); - if (!vectorSearchTool) { - log.error('Vector search tool not available'); - return { - success: false, - message: 'Vector search capability not available' - }; + // Get note content for search + const content = sourceNote.getContent(); + const title = sourceNote.title; + + // Create search queries from the note title and content + const searchQueries = [title]; + + // Extract key terms from content if available + if (content && typeof content === 'string') { + // Extract meaningful words from content (filter out common words) + const contentWords = content + .toLowerCase() + .split(/\s+/) + .filter(word => word.length > 3) + .filter(word => !/^(the|and|but|for|are|from|they|been|have|this|that|with|will|when|where|what|how)$/.test(word)) + .slice(0, 10); // Take first 10 meaningful words + + if (contentWords.length > 0) { + searchQueries.push(contentWords.join(' ')); + } } - log.info(`Using vector search to find notes related to "${sourceNote.title}"`); + // Execute searches and combine results + const searchStartTime = Date.now(); + const allResults = new Map(); + let searchDuration = 0; - // Get note content for semantic search - const content = await sourceNote.getContent(); - const title = sourceNote.title; + for (const query of searchQueries) { + try { + const results = searchService.searchNotes(query, { + includeArchivedNotes: false, + fastSearch: false // Use full search for better results + }); - // Use both title and content for search - const searchQuery = title + (content && typeof content === 'string' ? ': ' + content.substring(0, 500) : ''); + // Add results to our map (avoiding duplicates) + for (const note of results.slice(0, limit * 2)) { // Get more to account for duplicates + if (note.noteId !== sourceNote.noteId && !note.isDeleted) { + allResults.set(note.noteId, { + noteId: note.noteId, + title: note.title, + similarity: 0.8 // Base similarity for search results + }); + } + } + } catch (error) { + log.error(`Search query failed: ${query} - ${error}`); + } + } - // Execute the search - const searchStartTime = Date.now(); - const results = await vectorSearchTool.searchNotes(searchQuery, { - maxResults: limit + 1 // Add 1 to account for the source note itself - }); - const searchDuration = Date.now() - searchStartTime; + searchDuration = Date.now() - searchStartTime; + + // Also add notes that are directly related via attributes + const directlyRelatedNotes = this.getDirectlyRelatedNotes(sourceNote); + for (const note of directlyRelatedNotes) { + if (!allResults.has(note.noteId)) { + allResults.set(note.noteId, { + noteId: note.noteId, + title: note.title, + similarity: 1.0 // Higher similarity for directly related notes + }); + } + } + + const relatedNotes = Array.from(allResults.values()) + .sort((a, b) => b.similarity - a.similarity) // Sort by similarity + .slice(0, limit); - // Filter out the source note from results - const filteredResults = results.filter(note => note.noteId !== sourceNote.noteId); - log.info(`Found ${filteredResults.length} related notes in ${searchDuration}ms`); + log.info(`Found ${relatedNotes.length} related notes in ${searchDuration}ms`); return { success: true, noteId: sourceNote.noteId, title: sourceNote.title, - relatedNotes: filteredResults.slice(0, limit).map(note => ({ - noteId: note.noteId, - title: note.title, - similarity: Math.round(note.similarity * 100) / 100 - })), - message: `Found ${filteredResults.length} notes semantically related to "${sourceNote.title}"` + relatedNotes: relatedNotes, + message: `Found ${relatedNotes.length} notes related to "${sourceNote.title}" using search and relationship analysis` }; } catch (error: any) { log.error(`Error finding related notes: ${error.message || String(error)}`); @@ -296,6 +354,55 @@ export class RelationshipTool implements ToolHandler { } } + /** + * Get notes that are directly related through attributes/relations + */ + private getDirectlyRelatedNotes(sourceNote: any): any[] { + const relatedNotes: any[] = []; + + try { + // Get outgoing relations + const outgoingAttributes = sourceNote.getAttributes().filter((attr: any) => attr.type === 'relation'); + for (const attr of outgoingAttributes) { + const targetNote = becca.notes[attr.value]; + if (targetNote && !targetNote.isDeleted) { + relatedNotes.push(targetNote); + } + } + + // Get incoming relations + const incomingRelations = sourceNote.getTargetRelations(); + for (const attr of incomingRelations) { + if (attr.type === 'relation') { + const sourceOfRelation = attr.getNote(); + if (sourceOfRelation && !sourceOfRelation.isDeleted) { + relatedNotes.push(sourceOfRelation); + } + } + } + + // Get parent and child notes + const parentNotes = sourceNote.getParentNotes(); + for (const parent of parentNotes) { + if (!parent.isDeleted) { + relatedNotes.push(parent); + } + } + + const childNotes = sourceNote.getChildNotes(); + for (const child of childNotes) { + if (!child.isDeleted) { + relatedNotes.push(child); + } + } + + } catch (error) { + log.error(`Error getting directly related notes: ${error}`); + } + + return relatedNotes; + } + /** * Suggest possible relationships based on content analysis */ diff --git a/apps/server/src/services/options_init.ts b/apps/server/src/services/options_init.ts index 5311a67f00..4126ed321e 100644 --- a/apps/server/src/services/options_init.ts +++ b/apps/server/src/services/options_init.ts @@ -196,38 +196,19 @@ const defaultOptions: DefaultOption[] = [ { name: "aiEnabled", value: "false", isSynced: true }, { name: "openaiApiKey", value: "", isSynced: false }, { name: "openaiDefaultModel", value: "", isSynced: true }, - { name: "openaiEmbeddingModel", value: "", isSynced: true }, { name: "openaiBaseUrl", value: "https://api.openai.com/v1", isSynced: true }, { name: "anthropicApiKey", value: "", isSynced: false }, { name: "anthropicDefaultModel", value: "", isSynced: true }, - { name: "voyageEmbeddingModel", value: "", isSynced: true }, { name: "voyageApiKey", value: "", isSynced: false }, { name: "anthropicBaseUrl", value: "https://api.anthropic.com/v1", isSynced: true }, { name: "ollamaEnabled", value: "false", isSynced: true }, { name: "ollamaDefaultModel", value: "", isSynced: true }, { name: "ollamaBaseUrl", value: "http://localhost:11434", isSynced: true }, - { name: "ollamaEmbeddingModel", value: "", isSynced: true }, - { name: "embeddingAutoUpdateEnabled", value: "true", isSynced: true }, - - // Embedding-specific provider options - { name: "openaiEmbeddingApiKey", value: "", isSynced: false }, - { name: "openaiEmbeddingBaseUrl", value: "https://api.openai.com/v1", isSynced: true }, - { name: "voyageEmbeddingBaseUrl", value: "https://api.voyageai.com/v1", isSynced: true }, - { name: "ollamaEmbeddingBaseUrl", value: "http://localhost:11434", isSynced: true }, // Adding missing AI options { name: "aiTemperature", value: "0.7", isSynced: true }, { name: "aiSystemPrompt", value: "", isSynced: true }, { name: "aiSelectedProvider", value: "openai", isSynced: true }, - { name: "embeddingDimensionStrategy", value: "auto", isSynced: true }, - { name: "embeddingSelectedProvider", value: "openai", isSynced: true }, - { name: "embeddingSimilarityThreshold", value: "0.75", isSynced: true }, - { name: "enableAutomaticIndexing", value: "true", isSynced: true }, - { name: "maxNotesPerLlmQuery", value: "3", isSynced: true }, - { name: "embeddingBatchSize", value: "10", isSynced: true }, - { name: "embeddingUpdateInterval", value: "5000", isSynced: true }, - { name: "embeddingDefaultDimension", value: "1536", isSynced: true }, - { name: "embeddingGenerationLocation", value: "client", isSynced: true }, ]; /** diff --git a/apps/server/src/services/sync.ts b/apps/server/src/services/sync.ts index f852a567f0..a26fabf822 100644 --- a/apps/server/src/services/sync.ts +++ b/apps/server/src/services/sync.ts @@ -364,14 +364,6 @@ function getEntityChangeRow(entityChange: EntityChange) { } } - // Special handling for note_embeddings embedding field - if (entityName === "note_embeddings") { - // Cast to any to access the embedding property - const row = entityRow as any; - if (row.embedding && Buffer.isBuffer(row.embedding)) { - row.embedding = row.embedding.toString("base64"); - } - } return entityRow; } diff --git a/apps/server/src/services/sync_update.ts b/apps/server/src/services/sync_update.ts index 19edbf7596..a22ba67173 100644 --- a/apps/server/src/services/sync_update.ts +++ b/apps/server/src/services/sync_update.ts @@ -54,9 +54,7 @@ function updateEntity(remoteEC: EntityChange, remoteEntityRow: EntityRow | undef const updated = remoteEC.entityName === "note_reordering" ? updateNoteReordering(remoteEC, remoteEntityRow, instanceId) - : (remoteEC.entityName === "note_embeddings" - ? updateNoteEmbedding(remoteEC, remoteEntityRow, instanceId, updateContext) - : updateNormalEntity(remoteEC, remoteEntityRow, instanceId, updateContext)); + : updateNormalEntity(remoteEC, remoteEntityRow, instanceId, updateContext); if (updated) { if (remoteEntityRow?.isDeleted) { @@ -145,78 +143,11 @@ function updateNoteReordering(remoteEC: EntityChange, remoteEntityRow: EntityRow return true; } -function updateNoteEmbedding(remoteEC: EntityChange, remoteEntityRow: EntityRow | undefined, instanceId: string, updateContext: UpdateContext) { - if (remoteEC.isErased) { - eraseEntity(remoteEC); - updateContext.erased++; - return true; - } - - if (!remoteEntityRow) { - log.error(`Entity ${remoteEC.entityName} ${remoteEC.entityId} not found in sync update.`); - return false; - } - - interface NoteEmbeddingRow { - embedId: string; - noteId: string; - providerId: string; - modelId: string; - dimension: number; - embedding: Buffer; - version: number; - dateCreated: string; - utcDateCreated: string; - dateModified: string; - utcDateModified: string; - } - - // Cast remoteEntityRow to include required embedding properties - const typedRemoteEntityRow = remoteEntityRow as unknown as NoteEmbeddingRow; - - // Convert embedding from base64 string to Buffer if needed - if (typedRemoteEntityRow.embedding && typeof typedRemoteEntityRow.embedding === "string") { - typedRemoteEntityRow.embedding = Buffer.from(typedRemoteEntityRow.embedding, "base64"); - } - - const localEntityRow = sql.getRow(`SELECT * FROM note_embeddings WHERE embedId = ?`, [remoteEC.entityId]); - - if (localEntityRow) { - // We already have this embedding, check if we need to update it - if (localEntityRow.utcDateModified >= typedRemoteEntityRow.utcDateModified) { - // Local is newer or same, no need to update - entityChangesService.putEntityChangeWithInstanceId(remoteEC, instanceId); - return true; - } else { - // Remote is newer, update local - sql.replace("note_embeddings", remoteEntityRow); - - if (!updateContext.updated[remoteEC.entityName]) { - updateContext.updated[remoteEC.entityName] = []; - } - updateContext.updated[remoteEC.entityName].push(remoteEC.entityId); - - entityChangesService.putEntityChangeWithInstanceId(remoteEC, instanceId); - return true; - } - } else { - // We don't have this embedding, insert it - sql.replace("note_embeddings", remoteEntityRow); - - if (!updateContext.updated[remoteEC.entityName]) { - updateContext.updated[remoteEC.entityName] = []; - } - updateContext.updated[remoteEC.entityName].push(remoteEC.entityId); - - entityChangesService.putEntityChangeWithInstanceId(remoteEC, instanceId); - return true; - } -} function eraseEntity(entityChange: EntityChange) { const { entityName, entityId } = entityChange; - const entityNames = ["notes", "branches", "attributes", "revisions", "attachments", "blobs", "note_embeddings"]; + const entityNames = ["notes", "branches", "attributes", "revisions", "attachments", "blobs"]; if (!entityNames.includes(entityName)) { log.error(`Cannot erase ${entityName} '${entityId}'.`); diff --git a/apps/server/src/services/ws.ts b/apps/server/src/services/ws.ts index 6a211c5728..ad1aafe0f3 100644 --- a/apps/server/src/services/ws.ts +++ b/apps/server/src/services/ws.ts @@ -203,13 +203,6 @@ function fillInAdditionalProperties(entityChange: EntityChange) { WHERE attachmentId = ?`, [entityChange.entityId] ); - } else if (entityChange.entityName === "note_embeddings") { - // Note embeddings are backend-only entities for AI/vector search - // Frontend doesn't need the full embedding data (which is large binary data) - // Just ensure entity is marked as handled - actual sync happens at database level - if (!entityChange.isErased) { - entityChange.entity = { embedId: entityChange.entityId }; - } } if (entityChange.entity instanceof AbstractBeccaEntity) { @@ -228,7 +221,6 @@ const ORDERING: Record = { attachments: 3, notes: 1, options: 0, - note_embeddings: 3 }; function sendPing(client: WebSocket, entityChangeIds = []) { diff --git a/packages/commons/src/lib/options_interface.ts b/packages/commons/src/lib/options_interface.ts index 77c3b2a68a..ad781b65cf 100644 --- a/packages/commons/src/lib/options_interface.ts +++ b/packages/commons/src/lib/options_interface.ts @@ -131,35 +131,17 @@ export interface OptionDefinitions extends KeyboardShortcutsOptions