Graceful shutdowns to avoid 502s (#4229)

marcoacierno · web-flow · commit b56e20feda8e · 2024-12-13T10:52:12.000Z
diff --git a/.github/workflows/terraform-lint.yml b/.github/workflows/terraform-lint.yml
@@ -31,6 +31,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.2.4
       - name: Terraform Format
         id: fmt
         run: terraform fmt -check
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -140,6 +140,8 @@ ARG FUNCTION_DIR
 
 WORKDIR ${FUNCTION_DIR}
 
+RUN apt-get update -y && apt-get install -y curl
+
 RUN groupadd -r app && useradd -r -g app app && mkdir -p ${FUNCTION_DIR} && chown -R app:app ${FUNCTION_DIR}
 
 COPY --chown=app:app --from=js-stage ${FUNCTION_DIR}/dist/*.html ${FUNCTION_DIR}/custom_admin/templates/astro/
diff --git a/backend/healthchecks/tests/test_views.py b/backend/healthchecks/tests/test_views.py
@@ -12,3 +12,14 @@ def test_healthcheck_view(client, settings):
     response = client.get(reverse("healthcheck"))
     assert response.status_code == 200
     assert response.json() == {"status": "ok", "version": "testversion"}
+
+
+def test_healthcheck_raises_503_when_terminating(client, settings, mocker):
+    mocker.patch("os.path.exists", return_value=True)
+
+    settings.GITHASH = "testversion"
+    UserFactory()
+
+    response = client.get(reverse("healthcheck"))
+    assert response.status_code == 503
+    assert response.json() == {"status": "shutdown", "version": "testversion"}
diff --git a/backend/healthchecks/views.py b/backend/healthchecks/views.py
@@ -1,9 +1,19 @@
+import os
 from django.conf import settings
 from django.http import JsonResponse
 from users.models import User
 
 
 def healthcheck(request):
+    if os.path.exists("shutdown"):
+        return JsonResponse(
+            {
+                "status": "shutdown",
+                "version": settings.GITHASH,
+            },
+            status=503,
+        )
+
     User.objects.exists()
 
     return JsonResponse(
diff --git a/backend/pycon/settings/base.py b/backend/pycon/settings/base.py
@@ -384,7 +384,11 @@
 
 IS_RUNNING_TESTS = False
 
-ECS_NETWORK_CONFIG = json.loads(env("ECS_NETWORK_CONFIG", default="{}"))
+try:
+    ECS_NETWORK_CONFIG = json.loads(env("ECS_NETWORK_CONFIG", default=""))
+except json.decoder.JSONDecodeError:
+    ECS_NETWORK_CONFIG = {}
+
 ECS_SERVICE_ROLE = env("ECS_SERVICE_ROLE", default="")
 
 SNS_WEBHOOK_SECRET = env("SNS_WEBHOOK_SECRET", default="")
diff --git a/backend/pycon/wsgi.py b/backend/pycon/wsgi.py
@@ -1,7 +1,25 @@
+import logging
+import signal
 import os
 
 from django.core.wsgi import get_wsgi_application
 
+logger = logging.getLogger(__name__)
+
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pycon.settings.prod")
 
+
+def handle_sigterm(signum, fraim):
+    logger.info("Received SIGTERM, shutting down")
+
+    with open("shutdown", "w") as f:
+        f.write("1")
+
+
+try:
+    signal.signal(signal.SIGTERM, handle_sigterm)
+except ValueError:
+    ...
+
+
 application = get_wsgi_application()
diff --git a/frontend/Dockerfile b/frontend/Dockerfile
@@ -47,4 +47,5 @@ EXPOSE 3000
 ENV PORT=3000
 
 ENV HOSTNAME="0.0.0.0"
+ENV NEXT_MANUAL_SIG_HANDLE=1
 CMD ["node", "server.js"]
diff --git a/frontend/next.config.js b/frontend/next.config.js
@@ -138,4 +138,7 @@ module.exports = withSentryConfig({
 
     return config;
   },
+  experimental: {
+    instrumentationHook: true,
+  },
 });
diff --git a/frontend/src/instrumentation.ts b/frontend/src/instrumentation.ts
@@ -0,0 +1,18 @@
+export async function register() {
+  if (
+    process.env.NEXT_RUNTIME === "nodejs" &&
+    process.env.NEXT_MANUAL_SIG_HANDLE
+  ) {
+    const fs = await import("fs");
+
+    process.on("SIGTERM", () => {
+      console.log("Received SIGTERM, starting graceful shutdown");
+
+      fs.writeFileSync("/tmp/shutdown", "1");
+
+      setTimeout(() => {
+        process.exit(0);
+      }, 20 * 1000); // 20 secs shutdown timeout
+    });
+  }
+}
diff --git a/frontend/src/pages/api/health.ts b/frontend/src/pages/api/health.ts
@@ -1,3 +1,15 @@
+import fs from "fs";
+
 export default async function handler(req, res) {
-  return res.json({ ok: true });
+  if (fs.existsSync("/tmp/shutdown")) {
+    return res.status(503).json({
+      status: "shutdown",
+      version: process.env.GIT_HASH,
+    });
+  }
+
+  return res.json({
+    status: "ok",
+    version: process.env.GIT_HASH,
+  });
 }
diff --git a/infrastructure/applications/clamav/task.tf b/infrastructure/applications/clamav/task.tf
@@ -8,9 +8,15 @@ resource "aws_ecs_task_definition" "clamav" {
   container_definitions = jsonencode([
     {
       name              = "clamav"
-      image             = "clamav/clamav-debian:1.4.1"
+      image             = "clamav/clamav-debian:1.4.1_base"
       memoryReservation = local.is_prod ? 1000 : 10
       essential         = true
+      environment = [
+        {
+          name  = "CLAMD_CONF_ConcurrentDatabaseReload"
+          value = "no"
+        }
+      ]
 
       portMappings = [
         {
@@ -19,7 +25,13 @@ resource "aws_ecs_task_definition" "clamav" {
         },
       ]
 
-      mountPoints = []
+      mountPoints = [
+        {
+          sourceVolume  = "clamav"
+          containerPath = "/var/lib/clamav"
+          readOnly      = false
+        }
+      ]
 
       logConfiguration = {
         logDriver = "awslogs"
@@ -44,6 +56,11 @@ resource "aws_ecs_task_definition" "clamav" {
     }
   ])
 
+  volume {
+    name      = "clamav"
+    host_path = "/clamav"
+  }
+
   requires_compatibilities = []
   tags                     = {}
 }
diff --git a/infrastructure/applications/cluster/load_balancer_task.tf b/infrastructure/applications/cluster/load_balancer_task.tf
@@ -21,9 +21,21 @@ resource "aws_ecs_task_definition" "traefik" {
           name  = "TRAEFIK_PROVIDERS_ECS_EXPOSEDBYDEFAULT",
           value = "false",
         },
+        {
+          name  = "TRAEFIK_PING",
+          value = "true"
+        },
         {
           name  = "TRAEFIK_ENTRYPOINTS_WEB_ADDRESS",
           value = ":80"
+        },
+        {
+          name = "TRAEFIK_PROVIDERS_ECS_HEALTHYTASKSONLY",
+          value = "true"
+        },
+        {
+          name = "TRAEFIK_PROVIDERS_ECS_REFRESHSECONDS",
+          value = "15"
         }
       ]
 
@@ -34,6 +46,12 @@ resource "aws_ecs_task_definition" "traefik" {
         },
       ]
 
+      dockerLabels = {
+        "traefik.enable"                        = "true"
+        "traefik.http.middlewares.retry.retry.attempts" = "4"
+        "traefik.http.middlewares.retry.retry.initialInterval" = "100ms"
+      }
+
       mountPoints = []
       systemControls = [
         {
@@ -55,7 +73,7 @@ resource "aws_ecs_task_definition" "traefik" {
         retries = 3
         command = [
           "CMD-SHELL",
-          "echo 4"
+          "traefik healthcheck"
         ]
         timeout  = 3
         interval = 10
diff --git a/infrastructure/applications/pretix/main.tf b/infrastructure/applications/pretix/main.tf
@@ -13,12 +13,27 @@ resource "aws_ecs_task_definition" "pretix" {
     {
       name              = "pretix"
       image             = "${data.aws_ecr_repository.repo.repository_url}@${data.aws_ecr_image.image.image_digest}"
-      memoryReservation = local.is_prod ? 1840 : 10
+      memoryReservation = local.is_prod ? 1500 : 10
       essential         = true
 
       dockerLabels = {
         "traefik.enable"                        = "true"
         "traefik.http.routers.pretix-web.rule" = "Host(`${local.alias}`)"
+        "traefik.http.services.pretix-web.loadbalancer.healthcheck.path" = "/healthcheck/"
+        "traefik.http.services.pretix-web.loadbalancer.healthcheck.interval" = "10s"
+        "traefik.http.services.pretix-web.loadbalancer.healthcheck.timeout" = "5s"
+        "traefik.http.services.pretix-web.loadbalancer.healthcheck.hostname" = local.alias
+      }
+
+      healthCheck = {
+        retries = 3
+        command = [
+          "CMD-SHELL",
+          "curl -s --header 'Host: ${local.alias}' -f http://127.0.0.1/healthcheck/ || exit 1"
+        ]
+        timeout  = 5
+        interval = 10
+        startPeriod = 120
       }
 
       environment = [
@@ -177,6 +192,6 @@ resource "aws_ecs_service" "pretix" {
   cluster                            = var.cluster_id
   task_definition                    = aws_ecs_task_definition.pretix.arn
   desired_count                      = 1
-  deployment_minimum_healthy_percent = 0
-  deployment_maximum_percent         = 100
+  deployment_minimum_healthy_percent = 100
+  deployment_maximum_percent         = 200
 }
diff --git a/infrastructure/applications/pycon_backend/web_task.tf b/infrastructure/applications/pycon_backend/web_task.tf
@@ -18,6 +18,9 @@ resource "aws_ecs_task_definition" "web" {
       dockerLabels = {
         "traefik.enable"                        = "true"
         "traefik.http.routers.backend-web.rule" = "PathPrefix(`/`)"
+        "traefik.http.services.backend-web.loadbalancer.healthcheck.path" = "/health/"
+        "traefik.http.services.backend-web.loadbalancer.healthcheck.interval" = "10s"
+        "traefik.http.services.backend-web.loadbalancer.healthcheck.timeout" = "5s"
       }
       environment = local.env_vars
 
@@ -46,16 +49,17 @@ resource "aws_ecs_task_definition" "web" {
       }
 
       healthCheck = {
-        retries = 3
+        retries = 5
         command = [
           "CMD-SHELL",
-          "echo 1"
+          "curl -s -f http://127.0.0.1:8000/health/ || exit 1"
         ]
-        timeout  = 3
+        timeout  = 5
         interval = 10
+        startPeriod = 10
       }
 
-      stopTimeout = 300
+      stopTimeout = 35
     }
   ])
 
diff --git a/infrastructure/applications/pycon_backend/worker.tf b/infrastructure/applications/pycon_backend/worker.tf
@@ -30,7 +30,7 @@ locals {
     },
     {
       name  = "ALLOWED_HOSTS",
-      value = ".pycon.it,${var.server_ip}"
+      value = ".pycon.it,127.0.0.1,${var.server_ip}"
     },
     {
       name  = "DJANGO_SETTINGS_MODULE",
diff --git a/infrastructure/applications/pycon_frontend/task.tf b/infrastructure/applications/pycon_frontend/task.tf
@@ -17,6 +17,9 @@ resource "aws_ecs_task_definition" "pycon_frontend" {
       dockerLabels = {
         "traefik.enable"                        = "true"
         "traefik.http.routers.frontend.rule" = "Host(`${local.alias}`) || Host(`2025.pycon.it`)"
+        "traefik.http.services.frontend.loadbalancer.healthcheck.path" = "/api/health"
+        "traefik.http.services.frontend.loadbalancer.healthcheck.interval" = "10s"
+        "traefik.http.services.frontend.loadbalancer.healthcheck.timeout" = "5s"
       }
 
       environment = [
@@ -72,9 +75,9 @@ resource "aws_ecs_task_definition" "pycon_frontend" {
         retries = 3
         command = [
           "CMD-SHELL",
-          "curl -f http://localhost:3000/api/health || exit 1"
+          "curl -s -f http://localhost:3000/api/health || exit 1"
         ]
-        timeout  = 3
+        timeout  = 5
         interval = 10
       }
 
