Merge commit from fork

eastandwestwind · daveqnet · eastandwestwind · commit 59903c195e2f · 2025-09-03T19:32:22.000+02:00
* formatting * slowapi to use same redis connection pool as app * feat: temporary modification to spin up two local webservers * adds connection_url_encoded field to Redis config * unencode redis pw for slowapi * actually using unencoded pass * reverse logic to add rate limiting middleware * feat: add enhanced nginx logging * refactor rate limiting into 2 shared buckets, add validation for insecure rate limit headers * explicit annotations for rate limiting buckets * adds missing annotations to endpoints * adds annotations to health endpoints * refactor: make nginx optional in nox -s dev * refactor: restructure docker-compose and nox dev session for single/nginx modes * fix: re-add mistaken removed env vars from fides service * fix: revert other changes unrelated to nginx * Revert "adds annotations to health endpoints" This reverts commit af5659c. * Revert "adds missing annotations to endpoints" This reverts commit f69150d. * Revert "explicit annotations for rate limiting buckets" This reverts commit d849d82. * remove public req rate limit env var, implement shared bucket * address CR feedback, notably adding custom exception handler for rate limit hit, and adding clean_ip logic to better handle various use cases * use existing lib to help parse ip * restricts ip validation to only one ip address and possibly a port, no handling of special characters * Adds changelog * adds safe and unsafe ip header extraction such that we do not throw errors from the key_func * fix improts * update env var usage --------- Co-authored-by: Dave Quinlan <83430497+daveqnet@users.noreply.github.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -64,6 +64,7 @@ Changes can also be flagged with a GitHub label for tracking purposes. The URL o
 
 ### Security
 - Added stricter rate limiting to authentication endpoints to mitigate against brute force attacks. [CVE-2025-57815](https://github.com/ethyca/fides/security/advisories/GHSA-7q62-r88r-j5gw)
+- Adds Redis-driven rate limiting across all endpoints [CVE-2025-57816](https://github.com/ethyca/fides/security/advisories/GHSA-fq34-xw6c-fphf)
 
 ## [2.68.0](https://github.com/ethyca/fides/compare/2.67.2...2.68.0)
 
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -221,6 +221,100 @@ services:
       - CELERY_BROKER_URL=redis://:redispassword@redis:6379/0
       - CELERY_RESULT_BACKEND=redis://:redispassword@redis:6379/0
 
+  # Cluster services for nginx load balancing
+  fides-1:
+    container_name: fides-1
+    image: ethyca/fides:local
+    command: uvicorn --host 0.0.0.0 --port 8080 --reload --reload-dir src --reload-dir data --reload-include='*.yml' fides.api.main:app
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://0.0.0.0:8080/health"]
+      interval: 20s
+      timeout: 5s
+      retries: 10
+    ports:
+      - "8081:8080"
+    depends_on:
+      fides-db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    expose:
+      - 8080
+    env_file:
+      - .env
+    environment:
+      FIDES__CONFIG_PATH: ${FIDES__CONFIG_PATH:-/fides/.fides/fides.toml}
+      FIDES__CLI__ANALYTICS_ID: ${FIDES__CLI__ANALYTICS_ID-}
+      FIDES__CLI__SERVER_HOST: "fides"
+      FIDES__CLI__SERVER_PORT: "8080"
+      FIDES__DATABASE__SERVER: "fides-db"
+      FIDES__DEV_MODE: "True"
+      FIDES__LOGGING__COLORIZE: "True"
+      FIDES__USER__ANALYTICS_OPT_OUT: "True"
+      FIDES__SECURITY__BASTION_SERVER_HOST: ${FIDES__SECURITY__BASTION_SERVER_HOST-}
+      FIDES__SECURITY__BASTION_SERVER_SSH_USERNAME: ${FIDES__SECURITY__BASTION_SERVER_SSH_USERNAME-}
+      FIDES__SECURITY__BASTION_SERVER_SSH_PRIVATE_KEY: ${FIDES__SECURITY__BASTION_SERVER_SSH_PRIVATE_KEY-}
+      SAAS_OP_SERVICE_ACCOUNT_TOKEN: ${SAAS_OP_SERVICE_ACCOUNT_TOKEN-}
+      SAAS_SECRETS_OP_VAULT_ID: ${SAAS_SECRETS_OP_VAULT_ID-}
+    volumes:
+      - type: bind
+        source: .
+        target: /fides
+        read_only: False
+
+  fides-2:
+    container_name: fides-2
+    image: ethyca/fides:local
+    command: uvicorn --host 0.0.0.0 --port 8080 --reload --reload-dir src --reload-dir data --reload-include='*.yml' fides.api.main:app
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://0.0.0.0:8080/health"]
+      interval: 20s
+      timeout: 5s
+      retries: 10
+    ports:
+      - "8082:8080"
+    depends_on:
+      fides-db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    expose:
+      - 8080
+    env_file:
+      - .env
+    environment:
+      FIDES__CONFIG_PATH: ${FIDES__CONFIG_PATH:-/fides/.fides/fides.toml}
+      FIDES__CLI__ANALYTICS_ID: ${FIDES__CLI__ANALYTICS_ID-}
+      FIDES__CLI__SERVER_HOST: "fides"
+      FIDES__CLI__SERVER_PORT: "8080"
+      FIDES__DATABASE__SERVER: "fides-db"
+      FIDES__DEV_MODE: "True"
+      FIDES__LOGGING__COLORIZE: "True"
+      FIDES__USER__ANALYTICS_OPT_OUT: "True"
+      FIDES__SECURITY__BASTION_SERVER_HOST: ${FIDES__SECURITY__BASTION_SERVER_HOST-}
+      FIDES__SECURITY__BASTION_SERVER_SSH_USERNAME: ${FIDES__SECURITY__BASTION_SERVER_SSH_USERNAME-}
+      FIDES__SECURITY__BASTION_SERVER_SSH_PRIVATE_KEY: ${FIDES__SECURITY__BASTION_SERVER_SSH_PRIVATE_KEY-}
+      SAAS_OP_SERVICE_ACCOUNT_TOKEN: ${SAAS_OP_SERVICE_ACCOUNT_TOKEN-}
+      SAAS_SECRETS_OP_VAULT_ID: ${SAAS_SECRETS_OP_VAULT_ID-}
+    volumes:
+      - type: bind
+        source: .
+        target: /fides
+        read_only: False
+
+  fides-proxy:
+    container_name: fides-proxy
+    image: nginx:latest
+    ports:
+      - "8083:8080"
+    volumes:
+      - ./docker/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      fides-1:
+        condition: service_healthy
+      fides-2:
+        condition: service_healthy
+
 volumes:
   postgres: null
 
diff --git a/docker/nginx/nginx.conf b/docker/nginx/nginx.conf
@@ -0,0 +1,24 @@
+events {}
+
+http {
+
+  log_format upstream_log '$remote_addr - $remote_user [$time_local] '
+                        '"$request" $status $body_bytes_sent '
+                        '"$http_referer" "$http_user_agent" '
+                        'upstream_addr="$upstream_addr" '
+                        'upstream_status="$upstream_status"';
+
+  upstream fides {
+    server fides-1:8080;
+    server fides-2:8080;
+  }
+
+  server {
+    listen 8080;
+    access_log /dev/stdout upstream_log;
+
+    location / {
+      proxy_pass http://fides;
+    }
+  }
+}
diff --git a/noxfiles/dev_nox.py b/noxfiles/dev_nox.py
@@ -54,6 +54,7 @@ def dev(session: Session) -> None:
         - workers-all = Run all available Fides workers (see below)
         - flower = Run Flower monitoring dashboard for Celery
         - child = Run a Fides child node
+        - nginx = Run two Fides webservers with nginx load balancer proxy
         - <datastore(s)> = Run a test datastore (e.g. 'mssql', 'mongodb')
 
     To run specific workers only, use any of the following posargs:
@@ -111,7 +112,20 @@ def dev(session: Session) -> None:
 
     open_shell = "shell" in session.posargs
     remote_debug = "remote_debug" in session.posargs
-    if not datastores:
+    use_nginx = "nginx" in session.posargs
+
+    if use_nginx:
+        # Run two Fides webservers with nginx load balancer proxy
+        session.run(
+            "docker",
+            "compose",
+            "up",
+            "fides-1",
+            "fides-2",
+            "fides-proxy",
+            external=True,
+        )
+    elif not datastores:
         if open_shell:
             session.run(*START_APP, external=True)
             session.log("~~Remember to login with `fides user login`!~~")
diff --git a/src/fides/api/api/v1/endpoints/dsr_package_link.py b/src/fides/api/api/v1/endpoints/dsr_package_link.py
@@ -21,7 +21,7 @@
 from fides.api.schemas.storage.storage import StorageType
 from fides.api.service.storage.streaming.s3 import S3StorageClient
 from fides.api.util.api_router import APIRouter
-from fides.api.util.endpoint_utils import fides_limiter
+from fides.api.util.rate_limit import fides_limiter
 from fides.common.api.v1.urn_registry import PRIVACY_CENTER_DSR_PACKAGE, V1_URL_PREFIX
 from fides.config import CONFIG
 
@@ -62,7 +62,7 @@ def raise_error(status_code: int, detail: str) -> None:
     PRIVACY_CENTER_DSR_PACKAGE,
     status_code=HTTP_302_FOUND,
 )
-@fides_limiter.limit(CONFIG.security.public_request_rate_limit)
+@fides_limiter.limit(CONFIG.security.request_rate_limit)
 def get_access_results_urls(
     privacy_request_id: str,
     token: str,
diff --git a/src/fides/api/api/v1/endpoints/oauth_endpoints.py b/src/fides/api/api/v1/endpoints/oauth_endpoints.py
@@ -37,7 +37,7 @@
 )
 from fides.api.util.api_router import APIRouter
 from fides.api.util.connection_util import connection_status
-from fides.api.util.endpoint_utils import fides_limiter
+from fides.api.util.rate_limit import fides_limiter
 from fides.common.api.scope_registry import (
     CLIENT_CREATE,
     CLIENT_DELETE,
diff --git a/src/fides/api/api/v1/endpoints/user_endpoints.py b/src/fides/api/api/v1/endpoints/user_endpoints.py
@@ -59,7 +59,7 @@
 )
 from fides.api.service.deps import get_user_service
 from fides.api.util.api_router import APIRouter
-from fides.api.util.endpoint_utils import fides_limiter
+from fides.api.util.rate_limit import fides_limiter
 from fides.common.api.scope_registry import (
     SCOPE_REGISTRY,
     SYSTEM_MANAGER_DELETE,
diff --git a/src/fides/api/app_setup.py b/src/fides/api/app_setup.py
@@ -48,9 +48,13 @@
 from fides.api.util.api_router import APIRouter
 from fides.api.util.cache import get_cache
 from fides.api.util.consent_util import create_default_tcf_purpose_overrides_on_startup
-from fides.api.util.endpoint_utils import fides_limiter
 from fides.api.util.errors import FidesError
 from fides.api.util.logger import setup as setup_logging
+from fides.api.util.rate_limit import (
+    RateLimitIPValidationMiddleware,
+    fides_limiter,
+    is_rate_limit_enabled,
+)
 from fides.config import CONFIG
 from fides.config.config_proxy import ConfigProxy
 
@@ -88,7 +92,17 @@ def create_fides_app(
     for handler in ExceptionHandlers.get_handlers():
         # Starlette bug causing this to fail mypy
         fastapi_app.add_exception_handler(RedisNotConfigured, handler)  # type: ignore
-    fastapi_app.add_middleware(SlowAPIMiddleware)
+
+    if is_rate_limit_enabled:
+        # Validate header before SlowAPI processes the request
+        fastapi_app.add_middleware(RateLimitIPValidationMiddleware)
+        # Required for default rate limiting to work
+        fastapi_app.add_middleware(SlowAPIMiddleware)
+    else:
+        logger.warning(
+            "Rate limiting client IPs is disabled because the FIDES__SECURITY__RATE_LIMIT_CLIENT_IP_HEADER env var is not configured."
+        )
+
     fastapi_app.add_middleware(
         GZipMiddleware, minimum_size=1000, compresslevel=5
     )  # minimum_size is in bytes
diff --git a/src/fides/api/main.py b/src/fides/api/main.py
@@ -18,6 +18,8 @@
 from fideslog.sdk.python.event import AnalyticsEvent
 from loguru import logger
 from pyinstrument import Profiler
+from slowapi import _rate_limit_exceeded_handler
+from slowapi.errors import RateLimitExceeded
 from starlette.background import BackgroundTask
 from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
 from uvicorn import Config, Server
@@ -60,6 +62,7 @@
 )
 from fides.api.util.endpoint_utils import API_PREFIX
 from fides.api.util.logger import _log_exception
+from fides.api.util.rate_limit import safe_rate_limit_key
 from fides.cli.utils import FIDES_ASCII_ART
 from fides.config import CONFIG, check_required_webserver_config_values
 
@@ -388,3 +391,22 @@ async def request_validation_exception_handler(
             "detail": jsonable_encoder(exc.errors(), exclude={"input", "url", "ctx"})
         },
     )
+
+
+@app.exception_handler(RateLimitExceeded)
+async def rate_limit_handler(request: Request, exc: RateLimitExceeded) -> Response:
+    """Log rate limit violations and delegate to default handler."""
+    client_ip = safe_rate_limit_key(
+        request
+    )  # non exception-raising, falls back to source IP
+
+    # Log the rate limit event
+    logger.warning(
+        "Rate limit exceeded - IP: %s, Path: %s, Method: %s",
+        client_ip,
+        request.url.path,
+        request.method,
+    )
+
+    # Use the default handler to generate the proper response
+    return _rate_limit_exceeded_handler(request, exc)
diff --git a/src/fides/api/util/endpoint_utils.py b/src/fides/api/util/endpoint_utils.py
@@ -5,8 +5,6 @@
 
 from fastapi import HTTPException
 from fideslang import FidesModelType
-from slowapi import Limiter
-from slowapi.util import get_remote_address  # type: ignore
 from sqlalchemy.ext.asyncio import AsyncSession
 from starlette.status import HTTP_400_BAD_REQUEST
 
@@ -23,7 +21,6 @@
     ORGANIZATION,
     SYSTEM,
 )
-from fides.config import CONFIG
 
 from fides.api.models.sql_models import (  # type: ignore[attr-defined] # isort: skip
     ModelWithDefaultField,
@@ -44,16 +41,6 @@
     "system": SYSTEM,
 }
 
-# Used for rate limiting with Slow API
-# Decorate individual routes to deviate from the default rate limits
-fides_limiter = Limiter(
-    default_limits=[CONFIG.security.request_rate_limit],
-    headers_enabled=True,
-    key_prefix=CONFIG.security.rate_limit_prefix,
-    key_func=get_remote_address,
-    retry_after="http-date",
-)
-
 
 async def forbid_if_editing_is_default(
     sql_model: Base,
diff --git a/src/fides/api/util/rate_limit.py b/src/fides/api/util/rate_limit.py
diff --git a/src/fides/config/redis_settings.py b/src/fides/config/redis_settings.py
diff --git a/src/fides/config/security_settings.py b/src/fides/config/security_settings.py
diff --git a/tests/api/util/test_rate_limit.py b/tests/api/util/test_rate_limit.py
diff --git a/tests/ctl/core/config/test_security_settings.py b/tests/ctl/core/config/test_security_settings.py

Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,7 @@`
`37`	`37`	`)`
`38`	`38`	`from fides.api.util.api_router import APIRouter`
`39`	`39`	`from fides.api.util.connection_util import connection_status`
`40`		`-from fides.api.util.endpoint_utils import fides_limiter`
	`40`	`+from fides.api.util.rate_limit import fides_limiter`
`41`	`41`	`from fides.common.api.scope_registry import (`
`42`	`42`	`CLIENT_CREATE,`
`43`	`43`	`CLIENT_DELETE,`
Original file line number	Diff line number	Diff line change
`@@ -59,7 +59,7 @@`
`59`	`59`	`)`
`60`	`60`	`from fides.api.service.deps import get_user_service`
`61`	`61`	`from fides.api.util.api_router import APIRouter`
`62`		`-from fides.api.util.endpoint_utils import fides_limiter`
	`62`	`+from fides.api.util.rate_limit import fides_limiter`
`63`	`63`	`from fides.common.api.scope_registry import (`
`64`	`64`	`SCOPE_REGISTRY,`
`65`	`65`	`SYSTEM_MANAGER_DELETE,`