feat: Vector Storage Model Isolation with Automatic Migration

.github/workflows/e2e-tests.yml

@@ -0,0 +1,190 @@
+name: E2E Tests (Real Databases)
+
+on:
+  workflow_dispatch:  # Manual trigger only for E2E tests
+  pull_request:
+    branches: [ main, dev ]
+    paths:
+      - 'lightrag/kg/postgres_impl.py'
+      - 'lightrag/kg/qdrant_impl.py'
+      - 'tests/test_e2e_*.py'
+
+jobs:
+  e2e-postgres:
+    name: E2E PostgreSQL Tests
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: ankane/pgvector:latest
+        env:
+          POSTGRES_USER: lightrag
+          POSTGRES_PASSWORD: lightrag_test_password
+          POSTGRES_DB: lightrag_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U lightrag"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache pip packages
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-e2e-${{ hashFiles('**/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-e2e-
+          ${{ runner.os }}-pip-
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[api]"
+        pip install pytest pytest-asyncio asyncpg numpy qdrant-client
+
+    - name: Wait for PostgreSQL
+      run: |
+        timeout 30 bash -c 'until pg_isready -h localhost -p 5432 -U lightrag; do sleep 1; done'
+
+    - name: Setup pgvector extension
+      env:
+        PGPASSWORD: lightrag_test_password
+      run: |
+        psql -h localhost -U lightrag -d lightrag_test -c "CREATE EXTENSION IF NOT EXISTS vector;"
+        psql -h localhost -U lightrag -d lightrag_test -c "SELECT extname, extversion FROM pg_extension WHERE extname = 'vector';"
+
+    - name: Run PostgreSQL E2E tests
+      env:
+        POSTGRES_HOST: localhost
+        POSTGRES_PORT: 5432
+        POSTGRES_USER: lightrag
+        POSTGRES_PASSWORD: lightrag_test_password
+        POSTGRES_DATABASE: lightrag_test
+      run: |
+        pytest tests/test_e2e_multi_instance.py -k "postgres" -v --tb=short -s
+      timeout-minutes: 20
+
+    - name: Upload PostgreSQL test results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: e2e-postgres-results-py${{ matrix.python-version }}
+        path: |
+          .pytest_cache/
+          test-results.xml
+        retention-days: 7
+
+  e2e-qdrant:
+    name: E2E Qdrant Tests
+    runs-on: ubuntu-latest
+
+    services:
+      qdrant:
+        image: qdrant/qdrant:latest
+        ports:
+          - 6333:6333
+          - 6334:6334
+
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache pip packages
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-e2e-${{ hashFiles('**/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-e2e-
+          ${{ runner.os }}-pip-
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[api]"
+        pip install pytest pytest-asyncio qdrant-client numpy
+
+    - name: Wait for Qdrant
+      run: |
+        echo "Waiting for Qdrant to be ready..."
+        for i in {1..60}; do
+          if curl -s http://localhost:6333 > /dev/null 2>&1; then
+            echo "Qdrant is ready!"
+            break
+          fi
+          echo "Attempt $i/60: Qdrant not ready yet, waiting..."
+          sleep 1
+        done
+        # Final check
+        if ! curl -s http://localhost:6333 > /dev/null 2>&1; then
+          echo "ERROR: Qdrant failed to start after 60 seconds"
+          exit 1
+        fi
+
+    - name: Verify Qdrant connection
+      run: |
+        echo "Verifying Qdrant API..."
+        curl -X GET "http://localhost:6333/collections" -H "Content-Type: application/json"
+        echo ""
+        echo "Qdrant is accessible and ready for testing"
+
+    - name: Run Qdrant E2E tests
+      env:
+        QDRANT_URL: http://localhost:6333
+        QDRANT_API_KEY: ""
+      run: |
+        pytest tests/test_e2e_multi_instance.py -k "qdrant" -v --tb=short -s
+      timeout-minutes: 15
+
+    - name: Upload Qdrant test results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: e2e-qdrant-results-py${{ matrix.python-version }}
+        path: |
+          .pytest_cache/
+          test-results.xml
+        retention-days: 7
+
+  e2e-summary:
+    name: E2E Test Summary
+    runs-on: ubuntu-latest
+    needs: [e2e-postgres, e2e-qdrant]
+    if: always()
+
+    steps:
+    - name: Check test results
+      run: |
+        echo "## E2E Test Summary" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "### PostgreSQL E2E Tests" >> $GITHUB_STEP_SUMMARY
+        echo "Status: ${{ needs.e2e-postgres.result }}" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "### Qdrant E2E Tests" >> $GITHUB_STEP_SUMMARY
+        echo "Status: ${{ needs.e2e-qdrant.result }}" >> $GITHUB_STEP_SUMMARY
+
+    - name: Fail if any test failed
+      if: needs.e2e-postgres.result != 'success' || needs.e2e-qdrant.result != 'success'
+      run: exit 1

.github/workflows/feature-tests.yml

@@ -0,0 +1,74 @@
+name: Feature Branch Tests
+
+on:
+  workflow_dispatch:  # Allow manual trigger
+  push:
+    branches:
+      - 'feature/**'
+  pull_request:
+    branches: [ main, dev ]
+
+jobs:
+  migration-tests:
+    name: Vector Storage Migration Tests
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Cache pip packages
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e ".[api]"
+        pip install pytest pytest-asyncio
+
+    - name: Run Qdrant migration tests
+      run: |
+        pytest tests/test_qdrant_migration.py -v --tb=short
+      continue-on-error: false
+
+    - name: Run PostgreSQL migration tests
+      run: |
+        pytest tests/test_postgres_migration.py -v --tb=short
+      continue-on-error: false
+
+    - name: Run all unit tests (if exists)
+      run: |
+        # Run EmbeddingFunc tests
+        pytest tests/ -k "embedding" -v --tb=short || true
+      continue-on-error: true
+
+    - name: Upload test results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: migration-test-results-py${{ matrix.python-version }}
+        path: |
+          .pytest_cache/
+          test-results.xml
+        retention-days: 7
+
+    - name: Test Summary
+      if: always()
+      run: |
+        echo "## Test Summary" >> $GITHUB_STEP_SUMMARY
+        echo "- Python: ${{ matrix.python-version }}" >> $GITHUB_STEP_SUMMARY
+        echo "- Branch: ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY
+        echo "- Commit: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY