Skip to content

Commit f7d3064

Browse files
tiangolotiangolo
authored
👷 Update CI with Pulumi, Helm, Kubernetes, enable AWS Load Balancer, ingress-nginx, HTTPS (#129)
1 parent 02a812f commit f7d3064

10 files changed

+243
-11
lines changed

‎.github/workflows/deploy.yml

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ on:
66
release:
77
types:
88
- created
9+
workflow_dispatch:
10+
inputs:
11+
debug_enabled:
12+
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
13+
required: false
14+
default: 'false'
915

1016
jobs:
1117
deploy:
@@ -33,10 +39,22 @@ jobs:
3339
- run: pip install -r requirements.txt
3440
working-directory: infra
3541
- uses: pulumi/actions@v5
42+
id: pulumi
3643
with:
3744
command: up
3845
# stack-name: org-name/stack-name # When using an individual account, only use stack-name.
39-
stack-name: staging
46+
# staging on push, production on release
47+
stack-name: ${{ github.event_name == 'release' && 'production' || 'staging' }}
4048
work-dir: infra
4149
env:
4250
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
51+
- name: Setup tmate session
52+
uses: mxschmitt/action-tmate@v3
53+
if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
54+
with:
55+
limit-access-to-actor: true
56+
env:
57+
CLUSTER_NAME: ${{ steps.pulumi.outputs.cluster_name }}
58+
KUBECONFIG_CONTENT: ${{ steps.pulumi.outputs.kubeconfig }}
59+
CLOUDFLARE_API_TOKEN_DNS: ${{ secrets.CLOUDFLARE_API_TOKEN_DNS }}
60+
DEPLOYMENT_ENV: ${{ github.event_name == 'release' && 'production' || 'staging' }}

‎infra/__main__.py

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414
eks_node_instance_type = config.get("eksNodeInstanceType", "t3.medium")
1515
vpc_network_cidr = config.get("vpcNetworkCidr", "10.0.0.0/16")
1616

17-
aws_loadbalancer_name = "aws-load-balancer-controller"
17+
aws_load_balancer_name = "aws-load-balancer-controller"
18+
19+
### AWS Resources ###
1820

1921
# Role generated automatically by AWS from permission set from AWS IAM Identity Center
2022
roles = aws.iam.get_roles(name_regex="FastAPILabsPowerUserK8s")
@@ -85,7 +87,7 @@
8587
# Ref: https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.8/deploy/installation/
8688
aws_lb_controller_policy_content = (
8789
Path(__file__)
88-
.parent.joinpath(f"{aws_loadbalancer_name}-config/iam-policy.json")
90+
.parent.joinpath(f"{aws_load_balancer_name}-config/iam-policy.json")
8991
.read_text()
9092
)
9193

@@ -94,13 +96,13 @@
9496
# Ref: https://docs.aws.amazon.com/eks/latest/userguide/associate-service-account-role.html
9597

9698

97-
service_account_name = f"system:serviceaccount:kube-system:{aws_loadbalancer_name}"
99+
service_account_name = f"system:serviceaccount:kube-system:{aws_load_balancer_name}"
98100
oidc_url = eks_cluster.core.apply(lambda x: x.oidc_provider and x.oidc_provider.url)
99101
oidc_arn = eks_cluster.core.apply(lambda x: x.oidc_provider and x.oidc_provider.arn)
100102

101103

102104
aws_lb_controller_role = aws.iam.Role(
103-
f"{aws_loadbalancer_name}-role",
105+
f"{aws_load_balancer_name}-role",
104106
assume_role_policy=pulumi.Output.json_dumps(
105107
{
106108
"Version": "2012-10-17",
@@ -128,35 +130,66 @@
128130
)
129131

130132
aws_lb_controller_policy = aws.iam.Policy(
131-
f"{aws_loadbalancer_name}-policy",
133+
f"{aws_load_balancer_name}-policy",
132134
policy=aws_lb_controller_policy_content,
133135
)
134136

135137
# Attach IAM Policy to IAM Role
136138
aws.iam.PolicyAttachment(
137-
f"{aws_loadbalancer_name}-attachment",
139+
f"{aws_load_balancer_name}-attachment",
138140
policy_arn=aws_lb_controller_policy.arn,
139141
roles=[aws_lb_controller_role.name],
140142
)
141143

144+
### Kubernetes Resources ###
145+
142146
provider = k8s.Provider("provider", kubeconfig=eks_cluster.kubeconfig)
143147

144-
service_account = k8s.core.v1.ServiceAccount(
145-
f"{aws_loadbalancer_name}-sa",
148+
aws_load_balancer_service_account = k8s.core.v1.ServiceAccount(
149+
f"{aws_load_balancer_name}-sa",
146150
metadata={
147-
"name": aws_loadbalancer_name,
151+
"name": aws_load_balancer_name,
148152
"namespace": "kube-system",
149153
"labels": {
150154
"app.kubernetes.io/component": "controller",
151-
"app.kubernetes.io/name": aws_loadbalancer_name,
155+
"app.kubernetes.io/name": aws_load_balancer_name,
152156
},
153157
"annotations": {"eks.amazonaws.com/role-arn": aws_lb_controller_role.arn},
154158
},
159+
opts=pulumi.ResourceOptions(provider=provider),
155160
)
156161

162+
cluster_name = eks_cluster.core.apply(lambda x: x.cluster.name)
163+
164+
# TODO: Fix this
165+
166+
# error: 1 error occurred:
167+
# * Helm release "kube-system/aws-load-balancer-controller-bf4de232" was created, but failed to initialize completely. Use Helm CLI to investigate: failed to become available within allocated timeout. Error: Helm Release kube-system/aws-load-balancer-controller-bf4de232: client rate limiter Wait returned an error: context deadline exceeded
168+
169+
# aws_load_balancer_controller = k8s.helm.v3.Release(
170+
# aws_load_balancer_name,
171+
# k8s.helm.v3.ReleaseArgs(
172+
# chart="aws-load-balancer-controller",
173+
# version="1.8.1",
174+
# repository_opts=k8s.helm.v3.RepositoryOptsArgs(
175+
# repo="https://aws.github.io/eks-charts"
176+
# ),
177+
# namespace="kube-system",
178+
# values={
179+
# "clusterName": cluster_name,
180+
# "serviceAccount": {
181+
# "create": False,
182+
# "name": aws_load_balancer_service_account.metadata["name"],
183+
# },
184+
# },
185+
# ),
186+
# opts=pulumi.ResourceOptions(provider=provider),
187+
# )
188+
157189

158190
# Export values to use elsewhere
159191
pulumi.export("kubeconfig", eks_cluster.kubeconfig)
192+
pulumi.export("cluster_name", cluster_name)
160193
pulumi.export("vpc_id", eks_vpc.vpc_id)
161194
pulumi.export("k8s_role_arn", k8s_role_arn)
162195
pulumi.export("aws_lb_controller_policy", aws_lb_controller_policy.arn)

‎infra/deploy-helm.sh

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
CLUSTER_NAME=${CLUSTER_NAME:?}
6+
KUBECONFIG_CONTENT=${KUBECONFIG_CONTENT:?}
7+
AWS_LOAD_BALANCER_CONTROLLER_VERSION="1.8.1"
8+
CERT_MANAGER_VERSION="1.15.1"
9+
INGRESS_NGINX_VERSION="4.10.1"
10+
11+
echo "Configure Kubeconfig"
12+
echo "${KUBECONFIG_CONTENT}" > kubeconfig.json
13+
chmod 600 kubeconfig.json
14+
export KUBECONFIG=kubeconfig.json
15+
16+
# Enable debugging
17+
set -x
18+
19+
echo "Add Helm repo: eks, for AWS Load Balancer Controller"
20+
helm repo add eks https://aws.github.io/eks-charts --force-update
21+
22+
echo "Add Helm repo: jetstack, for Cert Manager"
23+
helm repo add jetstack https://charts.jetstack.io --force-update
24+
25+
echo "Add Helm repo: ingress-nginx, for Ingress Controller"
26+
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx --force-update
27+
28+
echo "Update Helm repos"
29+
helm repo update
30+
31+
echo "Install AWS Load Balancer Controller"
32+
helm upgrade --install aws-load-balancer-controller eks/aws-load-balancer-controller \
33+
--namespace kube-system \
34+
--version "${AWS_LOAD_BALANCER_CONTROLLER_VERSION}" \
35+
--set clusterName="${CLUSTER_NAME}" \
36+
--set serviceAccount.create=false \
37+
--set serviceAccount.name=aws-load-balancer-controller
38+
39+
echo "Install Cert Manager"
40+
helm upgrade --install cert-manager jetstack/cert-manager \
41+
--namespace cert-manager \
42+
--create-namespace \
43+
--version "${CERT_MANAGER_VERSION}" \
44+
--set crds.enabled=true
45+
46+
echo "Install Ingress Nginx Controller"
47+
helm upgrade --install ingress-nginx-external ingress-nginx/ingress-nginx \
48+
--namespace ingress-nginx-external \
49+
--create-namespace \
50+
--version "${INGRESS_NGINX_VERSION}" \
51+
-f helm-values/ingress-nginx-external-values.yaml
52+
53+
LOAD_BALANCER_HOSTHAME=$(kubectl -n ingress-nginx-external get service ingress-nginx-external-controller --output jsonpath='{.status.loadBalancer.ingress[0].hostname}')
54+
55+
echo "Add DNS record for Load Balancer before continuing:"
56+
echo "$LOAD_BALANCER_HOSTHAME"
57+
58+
echo "Remove kubeconfig.json with by running:"
59+
echo "rm kubeconfig.json"

‎infra/deploy-kubectl.sh

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
CLOUDFLARE_API_TOKEN_DNS=${CLOUDFLARE_API_TOKEN_DNS:?}
6+
DEPLOY_ENVIRONMENT=${DEPLOY_ENVIRONMENT:-staging}
7+
8+
echo "Configure Kubeconfig"
9+
echo "${KUBECONFIG_CONTENT}" > kubeconfig.json
10+
chmod 600 kubeconfig.json
11+
export KUBECONFIG=kubeconfig.json
12+
13+
# Enable debugging
14+
set -x
15+
16+
echo "Add Cloudflare token secret"
17+
envsubst < k8s/cert-manager-cloudflare-token.yaml | kubectl apply -f -
18+
19+
echo "Add Cert Manager issuer prod"
20+
kubectl apply -f k8s/cert-manager-issuer-prod.yaml
21+
22+
echo "Add Cert Manager issuer staging"
23+
kubectl apply -f k8s/cert-manager-issuer-staging.yaml
24+
25+
echo "Add wildcard cert"
26+
kubectl apply -f "k8s/cert-manager-wildcard-cert-${DEPLOY_ENVIRONMENT}.yaml"
27+
28+
echo "Remove kubeconfig.json with by running:"
29+
echo "rm kubeconfig.json"

‎infra/helm-values/ingress-nginx-external-values.yaml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
controller:
2+
ingressClassResource:
3+
name: external
4+
extraArgs:
5+
# Sync with cert-manager-wildcard-cert.yaml
6+
default-ssl-certificate: cert-manager/fastapicloud-tls
7+
service:
8+
annotations:
9+
service.beta.kubernetes.io/aws-load-balancer-type: external
10+
service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
11+
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip

‎infra/k8s/cert-manager-cloudflare-token.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: v1
2+
kind: Secret
3+
metadata:
4+
name: cloudflare-api-token-secret
5+
namespace: cert-manager
6+
type: Opaque
7+
stringData:
8+
api-token: ${CLOUDFLARE_API_TOKEN_DNS}

‎infra/k8s/cert-manager-issuer-prod.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Issuer
3+
metadata:
4+
name: letsencrypt-prod
5+
namespace: cert-manager
6+
spec:
7+
acme:
8+
# The ACME server URL
9+
server: https://acme-v02.api.letsencrypt.org/directory
10+
# Email address used for ACME registration
11+
email: [email protected]
12+
# Name of a secret used to store the ACME account private key
13+
privateKeySecretRef:
14+
name: letsencrypt-prod
15+
# Enable the DNS-01 challenge provider with Cloudflare API token
16+
solvers:
17+
- dns01:
18+
cloudflare:
19+
apiTokenSecretRef:
20+
name: cloudflare-api-token-secret
21+
key: api-token

‎infra/k8s/cert-manager-issuer-staging.yaml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Issuer
3+
metadata:
4+
name: letsencrypt-staging
5+
namespace: cert-manager
6+
spec:
7+
acme:
8+
# The ACME server URL
9+
server: https://acme-staging-v02.api.letsencrypt.org/directory
10+
# Email address used for ACME registration
11+
email: [email protected]
12+
# Name of a secret used to store the ACME account private key
13+
privateKeySecretRef:
14+
name: letsencrypt-staging
15+
# Enable the DNS-01 challenge provider with Cloudflare API token
16+
solvers:
17+
- dns01:
18+
cloudflare:
19+
apiTokenSecretRef:
20+
name: cloudflare-api-token-secret
21+
key: api-token

‎infra/k8s/cert-manager-wildcard-cert-production.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
name: fastapicloud-com
5+
namespace: cert-manager
6+
7+
spec:
8+
# Sync with ingress-nginx-external-values.yaml
9+
secretName: fastapicloud-tls
10+
dnsNames:
11+
- fastapicloud.com
12+
- "*.fastapicloud.com"
13+
issuerRef:
14+
name: letsencrypt-prod
15+
kind: Issuer
16+
group: cert-manager.io

‎infra/k8s/cert-manager-wildcard-cert-staging.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
name: fastapicloud-work
5+
namespace: cert-manager
6+
7+
spec:
8+
# Sync with ingress-nginx-external-values.yaml
9+
secretName: fastapicloud-tls
10+
dnsNames:
11+
- fastapicloud.work
12+
- "*.fastapicloud.work"
13+
issuerRef:
14+
name: letsencrypt-prod
15+
kind: Issuer
16+
group: cert-manager.io

0 commit comments

Comments
 (0)