diff --git a/Makefile b/Makefile index 39c92a121a5..6750db18446 100644 --- a/Makefile +++ b/Makefile @@ -7,18 +7,18 @@ target: dev: pip install --upgrade pip pre-commit poetry @$(MAKE) dev-version-plugin - poetry install --extras "all redis datamasking" + poetry install --extras "all redis datamasking valkey" pre-commit install dev-quality-code: pip install --upgrade pip pre-commit poetry @$(MAKE) dev-version-plugin - poetry install --extras "all redis datamasking" + poetry install --extras "all redis datamasking valkey" pre-commit install dev-gitpod: pip install --upgrade pip poetry - poetry install --extras "all redis datamasking" + poetry install --extras "all redis datamasking valkey" pre-commit install format-check: diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/cache.py b/aws_lambda_powertools/utilities/idempotency/persistence/cache.py new file mode 100644 index 00000000000..fcd2c37c7c1 --- /dev/null +++ b/aws_lambda_powertools/utilities/idempotency/persistence/cache.py @@ -0,0 +1,11 @@ +from aws_lambda_powertools.utilities.idempotency.persistence.redis import ( + CacheClientProtocol, + CacheConnection, + CachePersistenceLayer, +) + +__all__ = [ + "CacheClientProtocol", + "CachePersistenceLayer", + "CacheConnection", +] diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py index 7f27566cc24..d1c490ee0f3 100644 --- a/aws_lambda_powertools/utilities/idempotency/persistence/redis.py +++ b/aws_lambda_powertools/utilities/idempotency/persistence/redis.py @@ -8,6 +8,7 @@ from typing import Any, Literal, Protocol import redis +from typing_extensions import TypeAlias, deprecated from aws_lambda_powertools.utilities.idempotency import BasePersistenceLayer from aws_lambda_powertools.utilities.idempotency.exceptions import ( @@ -25,11 +26,12 @@ logger = logging.getLogger(__name__) +@deprecated("RedisPersistenceLayer will be removed in v4.0.0. Please use CacheProtocol instead.") class RedisClientProtocol(Protocol): """ - Protocol class defining the interface for a Redis client. + Protocol class defining the interface for a Cache client. - This protocol outlines the expected behavior of a Redis client, allowing for + This protocol outlines the expected behavior of a Cache client, allowing for standardization among different implementations and allowing customers to extend it in their own implementation. @@ -78,6 +80,7 @@ def delete(self, keys: bytes | str | memoryview) -> Any: raise NotImplementedError +@deprecated("RedisConnection will be removed in v4.0.0. Please use CacheConnection instead.") class RedisConnection: def __init__( self, @@ -85,32 +88,32 @@ def __init__( host: str = "", port: int = 6379, username: str = "", - password: str = "", # nosec - password for Redis connection + password: str = "", # nosec - password for Cache connection db_index: int = 0, mode: Literal["standalone", "cluster"] = "standalone", ssl: bool = True, ) -> None: """ - Initialize Redis connection which will be used in Redis persistence_store to support Idempotency + Initialize Cache connection which will be used in Cache persistence_store to support Idempotency Parameters ---------- host: str, optional - Redis host + Cache host port: int, optional: default 6379 - Redis port + Cache port username: str, optional - Redis username + Cache username password: str, optional - Redis password + Cache password url: str, optional - Redis connection string, using url will override the host/port in the previous parameters + Cache connection string, using url will override the host/port in the previous parameters db_index: int, optional: default 0 - Redis db index + Cache db index mode: str, Literal["standalone","cluster"] - set Redis client mode, choose from standalone/cluster. The default is standalone + set Cache client mode, choose from standalone/cluster. The default is standalone ssl: bool, optional: default True - set whether to use ssl for Redis connection + set whether to use ssl for Cache connection Example -------- @@ -122,13 +125,13 @@ def __init__( from aws_lambda_powertools.utilities.idempotency import ( idempotent, ) - from aws_lambda_powertools.utilities.idempotency.persistence.redis import ( - RedisCachePersistenceLayer, + from aws_lambda_powertools.utilities.idempotency.persistence.cache import ( + CachePersistenceLayer, ) from aws_lambda_powertools.utilities.typing import LambdaContext - persistence_layer = RedisCachePersistenceLayer(host="localhost", port=6379) + persistence_layer = CachePersistenceLayer(host="localhost", port=6379) @dataclass @@ -181,15 +184,15 @@ def _init_client(self) -> RedisClientProtocol: try: if self.url: - logger.debug(f"Using URL format to connect to Redis: {self.host}") + logger.debug(f"Using URL format to connect to Cache: {self.host}") return client.from_url(url=self.url) else: - # Redis in cluster mode doesn't support db parameter + # Cache in cluster mode doesn't support db parameter extra_param_connection: dict[str, Any] = {} if self.mode != "cluster": extra_param_connection = {"db": self.db_index} - logger.debug(f"Using arguments to connect to Redis: {self.host}") + logger.debug(f"Using arguments to connect to Cache: {self.host}") return client( host=self.host, port=self.port, @@ -200,10 +203,11 @@ def _init_client(self) -> RedisClientProtocol: **extra_param_connection, ) except redis.exceptions.ConnectionError as exc: - logger.debug(f"Cannot connect in Redis: {self.host}") - raise IdempotencyPersistenceConnectionError("Could not to connect to Redis", exc) from exc + logger.debug(f"Cannot connect to Cache endpoint: {self.host}") + raise IdempotencyPersistenceConnectionError("Could not to connect to Cache endpoint", exc) from exc +@deprecated("RedisCachePersistenceLayer will be removed in v4.0.0. Please use CachePersistenceLayer instead.") class RedisCachePersistenceLayer(BasePersistenceLayer): def __init__( self, @@ -211,7 +215,7 @@ def __init__( host: str = "", port: int = 6379, username: str = "", - password: str = "", # nosec - password for Redis connection + password: str = "", # nosec - password for Cache connection db_index: int = 0, mode: Literal["standalone", "cluster"] = "standalone", ssl: bool = True, @@ -223,39 +227,39 @@ def __init__( validation_key_attr: str = "validation", ): """ - Initialize the Redis Persistence Layer + Initialize the Cache Persistence Layer Parameters ---------- host: str, optional - Redis host + Cache host port: int, optional: default 6379 - Redis port + Cache port username: str, optional - Redis username + Cache username password: str, optional - Redis password + Cache password url: str, optional - Redis connection string, using url will override the host/port in the previous parameters + Cache connection string, using url will override the host/port in the previous parameters db_index: int, optional: default 0 - Redis db index + Cache db index mode: str, Literal["standalone","cluster"] - set Redis client mode, choose from standalone/cluster + set Cache client mode, choose from standalone/cluster ssl: bool, optional: default True - set whether to use ssl for Redis connection - client: RedisClientProtocol, optional - Bring your own Redis client that follows RedisClientProtocol. + set whether to use ssl for Cache connection + client: CacheClientProtocol, optional + Bring your own Cache client that follows CacheClientProtocol. If provided, all other connection configuration options will be ignored expiry_attr: str, optional - Redis json attribute name for expiry timestamp, by default "expiration" + Cache json attribute name for expiry timestamp, by default "expiration" in_progress_expiry_attr: str, optional - Redis json attribute name for in-progress expiry timestamp, by default "in_progress_expiration" + Cache json attribute name for in-progress expiry timestamp, by default "in_progress_expiration" status_attr: str, optional - Redis json attribute name for status, by default "status" + Cache json attribute name for status, by default "status" data_attr: str, optional - Redis json attribute name for response data, by default "data" + Cache json attribute name for response data, by default "data" validation_key_attr: str, optional - Redis json attribute name for hashed representation of the parts of the event used for validation + Cache json attribute name for hashed representation of the parts of the event used for validation Examples -------- @@ -266,8 +270,8 @@ def __init__( idempotent, ) - from aws_lambda_powertools.utilities.idempotency.persistence.redis import ( - RedisCachePersistenceLayer, + from aws_lambda_powertools.utilities.idempotency.persistence.cache import ( + CachePersistenceLayer, ) client = redis.Redis( @@ -275,7 +279,7 @@ def __init__( port="6379", decode_responses=True, ) - persistence_layer = RedisCachePersistenceLayer(client=client) + persistence_layer = CachePersistenceLayer(client=client) @idempotent(persistence_store=persistence_layer) def lambda_handler(event: dict, context: LambdaContext): @@ -288,7 +292,7 @@ def lambda_handler(event: dict, context: LambdaContext): ``` """ - # Initialize Redis client with Redis config if no client is passed in + # Initialize Cache client with cache config if no client is passed in if client is None: self.client = RedisConnection( host=host, @@ -330,11 +334,11 @@ def _item_to_data_record(self, idempotency_key: str, item: dict[str, Any]) -> Da in_progress_expiry_timestamp=in_progress_expiry_timestamp, response_data=str(item.get(self.data_attr)), payload_hash=str(item.get(self.validation_key_attr)), - expiry_timestamp=item.get("expiration", None), + expiry_timestamp=item.get("expiration"), ) def _get_record(self, idempotency_key) -> DataRecord: - # See: https://redis.io/commands/get/ + # See: https://valkey.io/valkey-glide/python/core/#glide.async_commands.CoreCommands.set response = self.client.get(idempotency_key) # key not found @@ -384,25 +388,25 @@ def _put_in_progress_record(self, data_record: DataRecord) -> None: # The idempotency key does not exist: # - first time that this invocation key is used # - previous invocation with the same key was deleted due to TTL - # - SET see https://redis.io/commands/set/ + # - SET see https://valkey.io/valkey-glide/python/core/#glide.async_commands.CoreCommands.set - logger.debug(f"Putting record on Redis for idempotency key: {data_record.idempotency_key}") + logger.debug(f"Putting record on Cache for idempotency key: {data_record.idempotency_key}") encoded_item = self._json_serializer(item["mapping"]) ttl = self._get_expiry_second(expiry_timestamp=data_record.expiry_timestamp) - redis_response = self.client.set(name=data_record.idempotency_key, value=encoded_item, ex=ttl, nx=True) + cache_response = self.client.set(name=data_record.idempotency_key, value=encoded_item, ex=ttl, nx=True) - # If redis_response is True, the Redis SET operation was successful and the idempotency key was not + # If cache_response is True, the Cache SET operation was successful and the idempotency key was not # previously set. This indicates that we can safely proceed to the handler execution phase. # Most invocations should successfully proceed past this point. - if redis_response: + if cache_response: return - # If redis_response is None, it indicates an existing record in Redis for the given idempotency key. + # If cache_response is None, it indicates an existing record in Cache for the given idempotency key. # This could be due to: # - An active idempotency record from a previous invocation that has not yet expired. # - An orphan record where a previous invocation has timed out. - # - An expired idempotency record that has not been deleted by Redis. + # - An expired idempotency record that has not been deleted by Cache. # In any case, we proceed to retrieve the record for further inspection. idempotency_record = self._get_record(data_record.idempotency_key) @@ -427,7 +431,7 @@ def _put_in_progress_record(self, data_record: DataRecord) -> None: # Reaching this point indicates that the idempotency record found is an orphan record. An orphan record is # one that is neither completed nor in-progress within its expected time frame. It may result from a - # previous invocation that has timed out or an expired record that has yet to be cleaned up by Redis. + # previous invocation that has timed out or an expired record that has yet to be cleaned up by Cache. # We raise an error to handle this exceptional scenario appropriately. raise IdempotencyPersistenceConsistencyError @@ -435,24 +439,22 @@ def _put_in_progress_record(self, data_record: DataRecord) -> None: # Handle an orphan record by attempting to acquire a lock, which by default lasts for 10 seconds. # The purpose of acquiring the lock is to prevent race conditions with other processes that might # also be trying to handle the same orphan record. Once the lock is acquired, we set a new value - # for the idempotency record in Redis with the appropriate time-to-live (TTL). + # for the idempotency record in Cache with the appropriate time-to-live (TTL). with self._acquire_lock(name=item["name"]): self.client.set(name=item["name"], value=encoded_item, ex=ttl) # Not removing the lock here serves as a safeguard against race conditions, # preventing another operation from mistakenly treating this record as an orphan while the # current operation is still in progress. - except (redis.exceptions.RedisError, redis.exceptions.RedisClusterException) as e: - raise e except Exception as e: - logger.debug(f"encountered non-Redis exception: {e}") - raise e + logger.debug(f"An error occurred: {e}") + raise @contextmanager def _acquire_lock(self, name: str): """ Attempt to acquire a lock for a specified resource name, with a default timeout. - This context manager attempts to set a lock using Redis to prevent concurrent + This context manager attempts to set a lock using Cache to prevent concurrent access to a resource identified by 'name'. It uses the 'nx' flag to ensure that the lock is only set if it does not already exist, thereby enforcing mutual exclusion. """ @@ -496,9 +498,9 @@ def _update_record(self, data_record: DataRecord) -> None: def _delete_record(self, data_record: DataRecord) -> None: """ - Deletes the idempotency record associated with a given DataRecord from Redis. + Deletes the idempotency record associated with a given DataRecord from Cache. This function is designed to be called after a Lambda handler invocation has completed processing. - It ensures that the idempotency key associated with the DataRecord is removed from Redis to + It ensures that the idempotency key associated with the DataRecord is removed from Cache to prevent future conflicts and to maintain the idempotency integrity. Note: it is essential that the idempotency key is not empty, as that would indicate the Lambda @@ -506,5 +508,10 @@ def _delete_record(self, data_record: DataRecord) -> None: """ logger.debug(f"Deleting record for idempotency key: {data_record.idempotency_key}") - # See: https://redis.io/commands/del/ + # See: https://valkey.io/valkey-glide/python/core/#glide.async_commands.CoreCommands.delete self.client.delete(data_record.idempotency_key) + + +CachePersistenceLayer: TypeAlias = RedisCachePersistenceLayer +CacheClientProtocol: TypeAlias = RedisClientProtocol +CacheConnection: TypeAlias = RedisConnection diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 97ffd38903b..7786813b9e4 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -12,7 +12,7 @@ The idempotency utility allows you to retry operations within a time window with * Produces the previous successful result when a function is called repeatedly with the same idempotency key * Choose your idempotency key from one or more fields, or entire payload * Safeguard concurrent requests, timeouts, missing idempotency keys, and payload tampering -* Support for Amazon DynamoDB, Redis, bring your own persistence layer, and in-memory caching +* Support for Amazon DynamoDB, Valkey, Redis OSS, or any Redis-compatible cache as the persistence layer ## Terminology @@ -82,7 +82,7 @@ To start, you'll need: --- - [Amazon DynamoDB](#dynamodb-table) or [Redis](#redis-database) + [Amazon DynamoDB](#dynamodb-table) or [Valkey/Redis OSS/Redis compatible](#cache-database) * :simple-awslambda:{ .lg .middle } **AWS Lambda function** @@ -139,13 +139,13 @@ You **can** use a single DynamoDB table for all functions annotated with Idempot * **Old boto3 versions can increase costs**. For cost optimization, we use a conditional `PutItem` to always lock a new idempotency record. If locking fails, it means we already have an idempotency record saving us an additional `GetItem` call. However, this is only supported in boto3 `1.26.194` and higher _([June 30th 2023](https://aws.amazon.com/about-aws/whats-new/2023/06/amazon-dynamodb-cost-failed-conditional-writes/){target="_blank"})_. -#### Redis database +#### Cache database -We recommend you start with a Redis compatible management services such as [Amazon ElastiCache for Redis](https://aws.amazon.com/elasticache/redis/){target="_blank"} or [Amazon MemoryDB for Redis](https://aws.amazon.com/memorydb/){target="_blank"}. +We recommend starting with a managed cache service, such as [Amazon ElastiCache for Valkey and for Redis OSS](https://aws.amazon.com/elasticache/redis/){target="_blank"} or [Amazon MemoryDB](https://aws.amazon.com/memorydb/){target="_blank"}. In both services, you'll need to configure [VPC access](https://docs.aws.amazon.com/lambda/latest/dg/configuration-vpc.html){target="_blank"} to your AWS Lambda. -##### Redis IaC examples +##### Cache configuration === "AWS CloudFormation example" @@ -160,7 +160,7 @@ In both services, you'll need to configure [VPC access](https://docs.aws.amazon. 1. Replace the Security Group ID and Subnet ID to match your VPC settings. 2. Replace the Security Group ID and Subnet ID to match your VPC settings. -Once setup, you can find a quick start and advanced examples for Redis in [the persistent layers section](#redispersistencelayer). +Once setup, you can find a quick start and advanced examples for Cache in [the persistent layers section](#cachepersistencelayer). @@ -464,17 +464,22 @@ You can customize the attribute names during initialization: | **sort_key_attr** | | | Sort key of the table (if table is configured with a sort key). | | **static_pk_value** | | `idempotency#{LAMBDA_FUNCTION_NAME}` | Static value to use as the partition key. Only used when **sort_key_attr** is set. | -#### RedisPersistenceLayer +#### CachePersistenceLayer -!!! info "We recommend Redis version 7 or higher for optimal performance." +The `CachePersistenceLayer` enables you to use Valkey, Redis OSS, or any Redis-compatible cache as the persistence layer for idempotency state. -For simple setups, initialize `RedisCachePersistenceLayer` with your Redis endpoint and port to connect. +We recommend using [`valkey-glide`](https://pypi.org/project/valkey-glide/){target="_blank"} for Valkey or [`redis`](https://pypi.org/project/redis/){target="_blank"} for Redis. However, any Redis OSS-compatible client should work. -For security, we enforce SSL connections by default; to disable it, set `ssl=False`. +For simple setups, initialize `CachePersistenceLayer` with your Cache endpoint and port to connect. Note that for security, we enforce SSL connections by default; to disable it, set `ssl=False`. -=== "Redis quick start" - ```python title="getting_started_with_idempotency_redis_config.py" hl_lines="8-10 14 27" - --8<-- "examples/idempotency/src/getting_started_with_idempotency_redis_config.py" +=== "Cache quick start" + ```python title="getting_started_with_idempotency_cache_config.py" hl_lines="8-10 14 27" + --8<-- "examples/idempotency/src/getting_started_with_idempotency_cache_config.py" + ``` + +=== "Using an existing Valkey Glide client" + ```python title="getting_started_with_idempotency_valkey_client.py" hl_lines="5 10-12 16-22 24 37" + --8<-- "examples/idempotency/src/getting_started_with_idempotency_valkey_client.py" ``` === "Using an existing Redis client" @@ -488,23 +493,23 @@ For security, we enforce SSL connections by default; to disable it, set `ssl=Fal --8<-- "examples/idempotency/src/getting_started_with_idempotency_payload.json" ``` -##### Redis SSL connections +##### Cache SSL connections We recommend using AWS Secrets Manager to store and rotate certificates safely, and the [Parameters feature](./parameters.md){target="_blank"} to fetch and cache optimally. -For advanced configurations, we recommend using an existing Redis client for optimal compatibility like SSL certificates and timeout. +For advanced configurations, we recommend using an existing Valkey client for optimal compatibility like SSL certificates and timeout. === "Advanced configuration using AWS Secrets" - ```python title="using_redis_client_with_aws_secrets.py" hl_lines="9-11 13 15 25" - --8<-- "examples/idempotency/src/using_redis_client_with_aws_secrets.py" + ```python title="using_cache_client_with_aws_secrets.py" hl_lines="5 9-11 13 15 18 19 23" + --8<-- "examples/idempotency/src/using_cache_client_with_aws_secrets.py" ``` 1. JSON stored: ```json { - "REDIS_ENDPOINT": "127.0.0.1", - "REDIS_PORT": "6379", - "REDIS_PASSWORD": "redis-secret" + "CACHE_HOST": "127.0.0.1", + "CACHE_PORT": "6379", + "CACHE_PASSWORD": "cache-secret" } ``` @@ -516,16 +521,16 @@ For advanced configurations, we recommend using an existing Redis client for opt 1. JSON stored: ```json { - "REDIS_ENDPOINT": "127.0.0.1", - "REDIS_PORT": "6379", - "REDIS_PASSWORD": "redis-secret" + "CACHE_HOST": "127.0.0.1", + "CACHE_PORT": "6379", + "CACHE_PASSWORD": "cache-secret" } ``` - 2. redis_user.crt file stored in the "certs" directory of your Lambda function - 3. redis_user_private.key file stored in the "certs" directory of your Lambda function - 4. redis_ca.pem file stored in the "certs" directory of your Lambda function + 2. cache_user.crt file stored in the "certs" directory of your Lambda function + 3. cache_user_private.key file stored in the "certs" directory of your Lambda function + 4. cache_ca.pem file stored in the "certs" directory of your Lambda function -##### Redis attributes +##### Cache attributes You can customize the attribute names during initialization: @@ -811,28 +816,28 @@ sequenceDiagram Optional idempotency key -#### Race condition with Redis +#### Race condition with Cache
```mermaid graph TD; - A(Existing orphan record in redis)-->A1; + A(Existing orphan record in cache)-->A1; A1[Two Lambda invoke at same time]-->B1[Lambda handler1]; - B1-->B2[Fetch from Redis]; + B1-->B2[Fetch from Cache]; B2-->B3[Handler1 got orphan record]; B3-->B4[Handler1 acquired lock]; B4-->B5[Handler1 overwrite orphan record] B5-->B6[Handler1 continue to execution]; A1-->C1[Lambda handler2]; - C1-->C2[Fetch from Redis]; + C1-->C2[Fetch from Cache]; C2-->C3[Handler2 got orphan record]; C3-->C4[Handler2 failed to acquire lock]; - C4-->C5[Handler2 wait and fetch from Redis]; + C4-->C5[Handler2 wait and fetch from Cache]; C5-->C6[Handler2 return without executing]; B6-->D(Lambda handler executed only once); C6-->D; ``` -Race condition with Redis +Race condition with Cache
## Advanced diff --git a/examples/idempotency/src/getting_started_with_idempotency_redis_config.py b/examples/idempotency/src/getting_started_with_idempotency_cache_config.py similarity index 77% rename from examples/idempotency/src/getting_started_with_idempotency_redis_config.py rename to examples/idempotency/src/getting_started_with_idempotency_cache_config.py index f3917042b28..5425bbee9d3 100644 --- a/examples/idempotency/src/getting_started_with_idempotency_redis_config.py +++ b/examples/idempotency/src/getting_started_with_idempotency_cache_config.py @@ -5,13 +5,13 @@ from aws_lambda_powertools.utilities.idempotency import ( idempotent, ) -from aws_lambda_powertools.utilities.idempotency.persistence.redis import ( - RedisCachePersistenceLayer, +from aws_lambda_powertools.utilities.idempotency.persistence.cache import ( + CachePersistenceLayer, ) from aws_lambda_powertools.utilities.typing import LambdaContext -redis_endpoint = os.getenv("REDIS_CLUSTER_ENDPOINT", "localhost") -persistence_layer = RedisCachePersistenceLayer(host=redis_endpoint, port=6379) +redis_endpoint = os.getenv("CACHE_CLUSTER_ENDPOINT", "localhost") +persistence_layer = CachePersistenceLayer(host=redis_endpoint, port=6379) @dataclass @@ -34,7 +34,7 @@ def lambda_handler(event: dict, context: LambdaContext): "statusCode": 200, } except Exception as exc: - raise PaymentError(f"Error creating payment {str(exc)}") + raise PaymentError(f"Error creating payment {str(exc)}") from exc def create_subscription_payment(event: dict) -> Payment: diff --git a/examples/idempotency/src/getting_started_with_idempotency_redis_client.py b/examples/idempotency/src/getting_started_with_idempotency_redis_client.py index 24dfe1be117..ac2a20587e8 100644 --- a/examples/idempotency/src/getting_started_with_idempotency_redis_client.py +++ b/examples/idempotency/src/getting_started_with_idempotency_redis_client.py @@ -7,21 +7,21 @@ from aws_lambda_powertools.utilities.idempotency import ( idempotent, ) -from aws_lambda_powertools.utilities.idempotency.persistence.redis import ( - RedisCachePersistenceLayer, +from aws_lambda_powertools.utilities.idempotency.persistence.cache import ( + CachePersistenceLayer, ) from aws_lambda_powertools.utilities.typing import LambdaContext -redis_endpoint = os.getenv("REDIS_CLUSTER_ENDPOINT", "localhost") +cache_endpoint = os.getenv("CACHE_CLUSTER_ENDPOINT", "localhost") client = Redis( - host=redis_endpoint, + host=cache_endpoint, port=6379, socket_connect_timeout=5, socket_timeout=5, max_connections=1000, ) -persistence_layer = RedisCachePersistenceLayer(client=client) +persistence_layer = CachePersistenceLayer(client=client) @dataclass diff --git a/examples/idempotency/src/getting_started_with_idempotency_valkey_client.py b/examples/idempotency/src/getting_started_with_idempotency_valkey_client.py new file mode 100644 index 00000000000..11a0c710bac --- /dev/null +++ b/examples/idempotency/src/getting_started_with_idempotency_valkey_client.py @@ -0,0 +1,53 @@ +import os +from dataclasses import dataclass, field +from uuid import uuid4 + +from glide import GlideClient, GlideClientConfiguration, NodeAddress + +from aws_lambda_powertools.utilities.idempotency import ( + idempotent, +) +from aws_lambda_powertools.utilities.idempotency.persistence.cache import ( + CachePersistenceLayer, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +cache_endpoint = os.getenv("CACHE_CLUSTER_ENDPOINT", "localhost") +client_config = GlideClientConfiguration( + addresses=[ + NodeAddress( + host="localhost", + port=6379, + ), + ], +) +client = GlideClient.create(config=client_config) + +persistence_layer = CachePersistenceLayer(client=client) # type: ignore[arg-type] + + +@dataclass +class Payment: + user_id: str + product_id: str + payment_id: str = field(default_factory=lambda: f"{uuid4()}") + + +class PaymentError(Exception): ... + + +@idempotent(persistence_store=persistence_layer) +def lambda_handler(event: dict, context: LambdaContext): + try: + payment: Payment = create_subscription_payment(event) + return { + "payment_id": payment.payment_id, + "message": "success", + "statusCode": 200, + } + except Exception as exc: + raise PaymentError(f"Error creating payment {str(exc)}") + + +def create_subscription_payment(event: dict) -> Payment: + return Payment(**event) diff --git a/examples/idempotency/src/using_cache_client_with_aws_secrets.py b/examples/idempotency/src/using_cache_client_with_aws_secrets.py new file mode 100644 index 00000000000..84eb039168e --- /dev/null +++ b/examples/idempotency/src/using_cache_client_with_aws_secrets.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import Any + +from glide import BackoffStrategy, GlideClient, GlideClientConfiguration, NodeAddress, ServerCredentials + +from aws_lambda_powertools.utilities import parameters +from aws_lambda_powertools.utilities.idempotency import IdempotencyConfig, idempotent +from aws_lambda_powertools.utilities.idempotency.persistence.cache import ( + CachePersistenceLayer, +) + +cache_values: dict[str, Any] = parameters.get_secret("cache_info", transform="json") # (1)! + +client_config = GlideClientConfiguration( + addresses=[ + NodeAddress( + host=cache_values.get("CACHE_HOST", "localhost"), + port=cache_values.get("CACHE_PORT", 6379), + ), + ], + credentials=ServerCredentials( + password=cache_values.get("CACHE_PASSWORD", ""), + ), + request_timeout=10, + use_tls=True, + reconnect_strategy=BackoffStrategy(num_of_retries=10, factor=2, exponent_base=1), +) +valkey_client = GlideClient.create(config=client_config) + +persistence_layer = CachePersistenceLayer(client=valkey_client) # type: ignore[arg-type] +config = IdempotencyConfig( + expires_after_seconds=2 * 60, # 2 minutes +) + + +@idempotent(config=config, persistence_store=persistence_layer) +def lambda_handler(event, context): + return {"message": "Hello"} diff --git a/examples/idempotency/src/using_redis_client_with_aws_secrets.py b/examples/idempotency/src/using_redis_client_with_aws_secrets.py deleted file mode 100644 index ee9e6d78c45..00000000000 --- a/examples/idempotency/src/using_redis_client_with_aws_secrets.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from redis import Redis - -from aws_lambda_powertools.utilities import parameters -from aws_lambda_powertools.utilities.idempotency import IdempotencyConfig, idempotent -from aws_lambda_powertools.utilities.idempotency.persistence.redis import ( - RedisCachePersistenceLayer, -) - -redis_values: dict[str, Any] = parameters.get_secret("redis_info", transform="json") # (1)! - -redis_client = Redis( - host=redis_values.get("REDIS_HOST", "localhost"), - port=redis_values.get("REDIS_PORT", 6379), - password=redis_values.get("REDIS_PASSWORD"), - decode_responses=True, - socket_timeout=10.0, - ssl=True, - retry_on_timeout=True, -) - -persistence_layer = RedisCachePersistenceLayer(client=redis_client) -config = IdempotencyConfig( - expires_after_seconds=2 * 60, # 2 minutes -) - - -@idempotent(config=config, persistence_store=persistence_layer) -def lambda_handler(event, context): - return {"message": "Hello"} diff --git a/examples/idempotency/src/using_redis_client_with_local_certs.py b/examples/idempotency/src/using_redis_client_with_local_certs.py index 2b6a5892c5b..844f5b37e7d 100644 --- a/examples/idempotency/src/using_redis_client_with_local_certs.py +++ b/examples/idempotency/src/using_redis_client_with_local_certs.py @@ -7,27 +7,27 @@ from aws_lambda_powertools.shared.functions import abs_lambda_path from aws_lambda_powertools.utilities import parameters from aws_lambda_powertools.utilities.idempotency import IdempotencyConfig, idempotent -from aws_lambda_powertools.utilities.idempotency.persistence.redis import ( - RedisCachePersistenceLayer, +from aws_lambda_powertools.utilities.idempotency.persistence.cache import ( + CachePersistenceLayer, ) -redis_values: dict[str, Any] = parameters.get_secret("redis_info", transform="json") # (1)! +cache_values: dict[str, Any] = parameters.get_secret("cache_info", transform="json") # (1)! redis_client = Redis( - host=redis_values.get("REDIS_HOST", "localhost"), - port=redis_values.get("REDIS_PORT", 6379), - password=redis_values.get("REDIS_PASSWORD"), + host=cache_values.get("REDIS_HOST", "localhost"), + port=cache_values.get("REDIS_PORT", 6379), + password=cache_values.get("REDIS_PASSWORD"), decode_responses=True, socket_timeout=10.0, ssl=True, retry_on_timeout=True, - ssl_certfile=f"{abs_lambda_path()}/certs/redis_user.crt", # (2)! - ssl_keyfile=f"{abs_lambda_path()}/certs/redis_user_private.key", # (3)! - ssl_ca_certs=f"{abs_lambda_path()}/certs/redis_ca.pem", # (4)! + ssl_certfile=f"{abs_lambda_path()}/certs/cache_user.crt", # (2)! + ssl_keyfile=f"{abs_lambda_path()}/certs/cache_user_private.key", # (3)! + ssl_ca_certs=f"{abs_lambda_path()}/certs/cache_ca.pem", # (4)! ) -persistence_layer = RedisCachePersistenceLayer(client=redis_client) +persistence_layer = CachePersistenceLayer(client=redis_client) config = IdempotencyConfig( expires_after_seconds=2 * 60, # 2 minutes ) diff --git a/examples/idempotency/templates/cfn_redis_serverless.yaml b/examples/idempotency/templates/cfn_redis_serverless.yaml index 8ce9d67f3cb..8def8774909 100644 --- a/examples/idempotency/templates/cfn_redis_serverless.yaml +++ b/examples/idempotency/templates/cfn_redis_serverless.yaml @@ -2,30 +2,30 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 Resources: - RedisServerlessIdempotency: + CacheServerlessIdempotency: Type: AWS::ElastiCache::ServerlessCache Properties: Engine: redis ServerlessCacheName: redis-cache SecurityGroupIds: # (1)! - - security-{your_sg_id} + - sg-07d998809154f9d88 SubnetIds: - subnet-{your_subnet_id_1} - subnet-{your_subnet_id_2} - HelloWorldFunction: + IdempotencyFunction: Type: AWS::Serverless::Function Properties: - Runtime: python3.12 + Runtime: python3.13 Handler: app.py VpcConfig: # (1)! SecurityGroupIds: - - security-{your_sg_id} + - sg-07d998809154f9d88 SubnetIds: - subnet-{your_subnet_id_1} - subnet-{your_subnet_id_2} Environment: Variables: POWERTOOLS_SERVICE_NAME: sample - REDIS_HOST: !GetAtt RedisServerlessIdempotency.Endpoint.Address - REDIS_PORT: !GetAtt RedisServerlessIdempotency.Endpoint.Port + CACHE_HOST: !GetAtt CacheServerlessIdempotency.Endpoint.Address + CACHE_PORT: !GetAtt CacheServerlessIdempotency.Endpoint.Port diff --git a/poetry.lock b/poetry.lock index 0d04ebfd466..5cbff8cb6d2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -62,7 +62,7 @@ files = [ {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, ] -markers = {main = "extra == \"redis\" and python_full_version < \"3.11.3\"", dev = "python_full_version < \"3.11.3\""} +markers = {main = "python_version < \"3.11\" and (extra == \"redis\" or extra == \"valkey\") or extra == \"redis\" and python_full_version < \"3.11.3\"", dev = "python_full_version < \"3.11.3\""} [[package]] name = "attrs" @@ -4506,6 +4506,55 @@ brotli = ["brotli (==1.0.9) ; os_name != \"nt\" and python_version < \"3\" and p secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress ; python_version == \"2.7\"", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "valkey-glide" +version = "1.3.5" +description = "An open source Valkey client library that supports Valkey and Redis open source 6.2, 7.0, 7.2 and 8.0." +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"valkey\"" +files = [ + {file = "valkey_glide-1.3.5-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:5335e1c3799b80d61e88369b9d56ea1f9af7f366dad1e0e50678312c09f9230f"}, + {file = "valkey_glide-1.3.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90a7752309213244df0a9c97484f29dc1133628ba07efbf5581d3709a66d98ba"}, + {file = "valkey_glide-1.3.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf01955e0035a5fb0c07e08a1090129b883f3c7820d85151d8549ad4ddc62064"}, + {file = "valkey_glide-1.3.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebf9014a26a94a15b765b9f9d76252aedb2cf811ca9cbaad822e7ccec7e1269d"}, + {file = "valkey_glide-1.3.5-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:df2c5be1782a6cac667983af87734dd9fb89f089820e51beb9a2c593d546ecfe"}, + {file = "valkey_glide-1.3.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0cd7a2066c5785b599798ff59ac6c8cdbc3a25262ca789853e7ecb0c75ff20d9"}, + {file = "valkey_glide-1.3.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d0c019f1605329dabb5f33925b3d9c600feff568b413cdab7304331588455e7"}, + {file = "valkey_glide-1.3.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b55c2cbed655ae249e05f7ae9d048d30a69a0017ec57dfcc0282aaacdfe894f6"}, + {file = "valkey_glide-1.3.5-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:6046204c9c527d6f07fd8af1babec5661cfc66fa01bec8672f87e283847c9fd8"}, + {file = "valkey_glide-1.3.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6e6c28da4423d0800dfd5ff44f5432ecd1652ac6fa29d91e50eade4c2fe11d09"}, + {file = "valkey_glide-1.3.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d321c8e27e894fb0eab96f2d1d8c37b4f31ea1a55d67ae087f9127357bd3291"}, + {file = "valkey_glide-1.3.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd69d89cd6c486e2a0c559787e64007f17522e7728d41d063371584c850524e7"}, + {file = "valkey_glide-1.3.5-cp313-cp313-macosx_10_7_x86_64.whl", hash = "sha256:6eaca572835fad1af1e2b480346540f541443ff841c50eaaadf33dfeb5a6fe1d"}, + {file = "valkey_glide-1.3.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:58d2115f10eb7ae385be82bf470613978bc60f9b85c48f887061834774d3efa3"}, + {file = "valkey_glide-1.3.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44e1bff6cf0fefc099a3b49dd7da6924a96324333f05409c80b792c9137427e8"}, + {file = "valkey_glide-1.3.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74c2f5aea2ca9aefefc00045b53ce9153ca3a03a6584cc8c2c075c97ac863a0f"}, + {file = "valkey_glide-1.3.5-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:f9aaf1bac93c0ee24f995fd4d90bd17685909be686f6a92eb20e0b7be1de7155"}, + {file = "valkey_glide-1.3.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d5913caf32ebea4682910e07ea1cc6382d1fe594c764cbd4b7112095f8bfe5c4"}, + {file = "valkey_glide-1.3.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd1a5e92c2d5b6c804edaaef48493f75d1cef94bb4309b2f7f7ee308b70f836a"}, + {file = "valkey_glide-1.3.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca3618dd6ba1a27e1a479b95fa5f70f4e50383431c8a1ebd01673a2444446c42"}, + {file = "valkey_glide-1.3.5-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:8aac8a48c34ab8c1885150ad25e4f4b296ae1ba4e405914c6b64ac6034ae6df7"}, + {file = "valkey_glide-1.3.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:a2d6f41163e19ef3277bf4861a754514e440b93930c0a5b63fe20f563543b4ce"}, + {file = "valkey_glide-1.3.5-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b613271b2a9ee60c534b2f8bc9eeeffef94600417f10d45ee0400b9f5ebfebd"}, + {file = "valkey_glide-1.3.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:703adff16c1ac6ffd1ae9da492502a412179786b779e45da4f0f5e5665de0fb9"}, + {file = "valkey_glide-1.3.5-pp311-pypy311_pp73-macosx_10_7_x86_64.whl", hash = "sha256:56b4abb66b43974e3fac1a070d9bb24dea6fc2ed24b412093920bfd502af2a8a"}, + {file = "valkey_glide-1.3.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:02ade5c2c1a4b8841011fb220ec126ee884569701934dd2be59d342bfea6cd6d"}, + {file = "valkey_glide-1.3.5-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c540d627da49f1be714deb0774f0114293d425f90ce58faa52895c7036e67a9a"}, + {file = "valkey_glide-1.3.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45a40c2e6d0fba5656f1e3c42251e983ab89e9362455697bf5dd663b9513207f"}, + {file = "valkey_glide-1.3.5-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:71ae003951b00b2d2000050b989eb2c585c89b5b884b4652e0cc7831d09f9373"}, + {file = "valkey_glide-1.3.5-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:183ae871bf30de26ca2e4f46b4823e32cd3051656c4c3330921c7c48171bd979"}, + {file = "valkey_glide-1.3.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8cd894bc91ea6081f30acf4d609b3e6a00e120d32db3de70997c77a1feb3872d"}, + {file = "valkey_glide-1.3.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6ee8e7b787ec6362256bce0ebb004fb93d3be24314c7b3f15d2e6552fca79fc"}, + {file = "valkey_glide-1.3.5.tar.gz", hash = "sha256:4751e6975f24a4f2e788102578ecf7bc4f4ff2f997efc400f547b983b9ec18ac"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.2", markers = "python_version < \"3.11\""} +protobuf = ">=3.20" +typing-extensions = {version = ">=4.8.0", markers = "python_version < \"3.11\""} + [[package]] name = "verspec" version = "0.1.0" @@ -4732,8 +4781,9 @@ parser = ["pydantic"] redis = ["redis"] tracer = ["aws-xray-sdk"] validation = ["fastjsonschema"] +valkey = ["valkey-glide"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0.0" -content-hash = "72b33753c75cfbbe28ccbea9377e187a87476ffe320ebaf6da1b696afd279c3c" +content-hash = "d296c8362cabbfd44a198f70de109db9fec0f82ec8ef990fbcaf17d5750c9b1e" diff --git a/pyproject.toml b/pyproject.toml index cf106482a1b..4e9cddf2c8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ pydantic = { version = "^2.4.0", optional = true } pydantic-settings = {version = "^2.6.1", optional = true} boto3 = { version = "^1.34.32", optional = true } redis = { version = ">=4.4,<7.0", optional = true } +valkey-glide = { version = ">=1.3.5,<2.0", optional = true } aws-encryption-sdk = { version = ">=3.1.1,<5.0.0", optional = true } jsonpath-ng = { version = "^1.6.0", optional = true } datadog-lambda = { version = "^6.106.0", optional = true } @@ -61,6 +62,7 @@ parser = ["pydantic"] validation = ["fastjsonschema"] tracer = ["aws-xray-sdk"] redis = ["redis"] +valkey = ["valkey-glide"] all = [ "pydantic", "pydantic-settings",