Forward fluent-bit logs to CloudWatch; import fluentbit DNS record

pulumi/Pulumi.dev.yaml

@@ -1,3 +1,7 @@
 config:
   observability:posthog_api_key:
     secure: AAABAACLeD5lasJAmY66NyJXtacSmTSMj/PiXtmBNIHeBfLx2HA3mhTzyWkPZnD9j8MCYPbtnjJiWeZBzOROWVKEcKpuysV/FV5CDoHCJg==
+  observability:cloudflare_zone_id:
+    secure: AAABAKragv0vFq2i/lBhwJRTkD/wjW8jefzGy6Mq5A4eZubZLEeh4cSFESB+M3Fv34TvYNxJpFlT208EMqUQLw==
+  cloudflare:apiToken:
+    secure: AAABAJfLM7HTgF++SR/ps+pkQQFMNxc0XyRidPcCJKD2nzpc9mRnqdEguDnJlKKwtStygHkRT95D/6n568y+TUf/hkGut6P5

pulumi/Pulumi.prod.yaml

@@ -201,6 +201,10 @@ config:
       selection_type: all_monitors
       statusiq_role: super_admin
       user_role: super_admin
+  observability:cloudflare_zone_id:
+    secure: AAABAFkTz7RxaV86Kw6RQ+XJ9O1orS7QTUgkRDIwvoE4kXYdQGWJ2i6zj9XqLoevXb3PgOGnNmv550aMA/H+zA==
+  cloudflare:apiToken:
+    secure: AAABAJ/XXYzFsIlhvWdl0FwFnUWHUTKIyJMAFNicpTtxUlWcOurQC9Y1O5qu+hjL/DmMRoV2c0+KdBy00YUzvIWf0HWuyIZk
 
 # The "api" kind seems to have some bugs in the Terraform provider, but this is what such a
 # configuration should look like:

pulumi/Pulumi.stage.yaml

@@ -1,3 +1,7 @@
-config: 
+config:
   observability:posthog_api_key:
     secure: AAABADNVbsoTmx0hogPjFb+Egd5TX7Wheactt3JgEv21j1G+OJSjHF+CUpY/w9qTS3KEw4IdoYUuufBKX2sJXudMUYFAEc8m8o2rg56eGw==
+  observability:cloudflare_zone_id:
+    secure: AAABALYXXhMfqqRW1FbwgqDZIsD7VwJ+AGDcqv4RiMDfi7cadVqba5L1esVND3ieXJdpE/qwcnN1AIUfBe6Zpw==
+  cloudflare:apiToken:
+    secure: AAABAB5n/gkdIZz7ZKyeu00kWDwDrpPr0Wl/FD449u+/WrX88r8nk+FHj/GTjyQL/GURB9TrMRzJ2as9ceKoWfOO6Z4+24Yk

pulumi/__main__.py

@@ -9,7 +9,11 @@
 of any of those larger infrastructure patterns.
 """
 
+import pulumi
+import pulumi_cloudflare as cloudflare
 import tb_pulumi
+import tb_pulumi.cloudwatch
+import tb_pulumi.iam
 import tb_pulumi.fargate
 import tb_pulumi.network
 import tb_pulumi.secrets
@@ -36,6 +40,17 @@
         **psm_opts,
     )
 
+    logdest_opts = resources.get('tb:cloudwatch:LogDestination', {})
+    logdests = {
+        logdest_name: tb_pulumi.cloudwatch.LogDestination(
+            f'{project.name_prefix}-logdest-{logdest_name}',
+            project=project,
+            app_name=logdest_name,
+            **logdest_config,
+        )
+        for logdest_name, logdest_config in logdest_opts.items()
+    }
+
     vpc_config = resources.get('tb:network:MultiCidrVpc', {}).get('fluentbit', {})
     vpc_fluentbit = tb_pulumi.network.MultiCidrVpc(
         f'{project.name_prefix}-vpc-fluentbit',
@@ -49,8 +64,20 @@
             project=project,
             subnets=vpc_fluentbit.resources.get('subnets', []),
             **cluster_config,
+            opts=pulumi.ResourceOptions(depends_on=[vpc_fluentbit]),
         )
         for cluster_name, cluster_config in resources.get(
             'tb:fargate:AutoscalingFargateCluster'
         ).items()
     }
+
+    cloudflare_zone_id = project.pulumi_config.require_secret('cloudflare_zone_id')
+    fluent_bit_dns = cloudflare.DnsRecord(
+        f'{project.name_prefix}-dns-fluentbit',
+        name='fluentbit' if project.stack == 'prod' else f'fluentbit-{project.stack}',
+        content=ecs_clusters['fluentbit'].resources['load_balancers']['fluentbit-http'].dns_name,
+        ttl=60,
+        type='CNAME',
+        zone_id=cloudflare_zone_id,
+        opts=pulumi.ResourceOptions(depends_on=[*ecs_clusters.values()]),
+    )

pulumi/config.dev.yaml

@@ -1,5 +1,7 @@
 ---
 
+.fluentbit_image: &FLUENTBIT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:426154b20a1b0d005f9f6029836a5902e5b5b228edd9688686f10d373e72a5b2
+
 config:
   build_site24x7: False
   build_tbpulumi: True
@@ -10,6 +12,14 @@ resources:
       secret_names:
         - posthog_api_key
 
+  tb:cloudwatch:LogDestination:
+    observability:
+      log_group:
+        retention_in_days: 7
+      log_streams:
+        untagged: untagged
+      org_name: tb
+
   tb:network:MultiCidrVpc:
     fluentbit:
       # The observability project has all of 10.202.0.0/16 assigned to it, but let's not soak all
@@ -30,15 +40,15 @@ resources:
         - secretsmanager
       additional_routes:
         - destination_cidr_block: 10.2.0.0/16  # mailstrom-dev
-          vpc_peering_connection_id: pcx-0d2027442f0e54ca4
-  
+          vpc_peering_connection_id: pcx-04d7e54008cd9326c
+
   tb:fargate:AutoscalingFargateCluster:
     fluentbit:
       cluster: {}
 
       container_security_groups:
-        fluentbit:
-          fluentbit-http:
+        fluentbit: # Service
+          fluentbit-http: # Load Balancer
             rules:
               ingress:
                 - description: Allow traffic from the load balancer to the container
@@ -61,7 +71,7 @@ resources:
       ssm_params: {}
 
       task_definitions:
-        fluentbit:
+        fluentbit: # Service
           container_definitions:
             - name: fluentbit
               environment:
@@ -78,13 +88,13 @@ resources:
               secrets:
                 - name: POSTHOG_API_KEY
                   valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/dev/posthog_api_key-e3UEK4
-              image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:fdd1b4748cfaee29553ee2c83fcaa428b68ba8e88c2791e1626e282b48127b9d
+              image: *FLUENTBIT_IMAGE
               logConfiguration:
                 logDriver: awslogs
                 options:
-                  awslogs-group: observability-dev-fargate-fluentbit-loggroup-fluentbit
+                  awslogs-group: /tb/dev/observability
                   awslogs-region: eu-central-1
-                  awslogs-stream-prefix: observability/dev/fluentbit/
+                  awslogs-stream-prefix: 'ecs'
               portMappings:
                 - containerPort: 1337
                   protocol: tcp
@@ -99,7 +109,7 @@ resources:
             - FARGATE
 
       load_balancer_security_groups:
-        fluentbit-http:
+        fluentbit-http: # Load Balancer
           description: Governs access to the fluent-bit-http load balancer in dev
           rules:
             ingress:
@@ -144,24 +154,31 @@ resources:
           ip_address_type: ipv4
 
       listeners:
-        fluentbit-http:
-          stalwart-metrics:
+        fluentbit-http: # Load Balancer
+          stalwart-metrics: # Target
             # This cert is for fluentbit-dev.tb.pro
             certificate_arn: arn:aws:acm:eu-central-1:768512802988:certificate/04dd0573-a3cc-4c19-b483-a868876c63b0
             port: 443
             protocol: HTTPS
 
       services:
-        fluentbit:
+        fluentbit: # Service
           assign_public_ip: yes
-          container_name: fluentbit
+          container_name: fluentbit  # Name from container definition
           container_port: 1337
           load_balancer: fluentbit-http
           service:
             desired_count: 2
-          target: stalwart-metrics
+          targets:
+            - container_name: fluentbit
+              container_port: 1337
+              target_name: stalwart-metrics
+
+      extra_policies:
+        fluentbit:
+          - arn:aws:iam::768512802988:policy/observability-dev-observability-logs-write-access
 
       autoscalers:
         fluentbit:
-          min_capacity: 2
-          max_capacity: 4
+          min_capacity: 1
+          max_capacity: 1

pulumi/config.prod.yaml

@@ -1,5 +1,7 @@
 ---
 
+.fluentbit_image: &FLUENTBIT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:426154b20a1b0d005f9f6029836a5902e5b5b228edd9688686f10d373e72a5b2
+
 config:
   build_site24x7: True
   build_tbpulumi: True
@@ -11,6 +13,14 @@ resources:
       secret_names:
         - posthog_api_key
 
+  tb:cloudwatch:LogDestination:
+    observability:
+      log_group:
+        retention_in_days: 3
+      log_streams:
+        untagged: untagged
+      org_name: tb
+
   tb:network:MultiCidrVpc:
     fluentbit:
       # The observability project has all of 10.200.0.0/16 assigned to it, but let's not soak all
@@ -79,13 +89,13 @@ resources:
               secrets:
                 - name: POSTHOG_API_KEY
                   valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/prod/posthog_api_key-pVtqmp
-              image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:aa968a499d7e
+              image: *FLUENTBIT_IMAGE
               logConfiguration:
                 logDriver: awslogs
                 options:
-                  awslogs-group: observability-prod-fargate-fluentbit-loggroup-fluentbit
+                  awslogs-group: /tb/prod/observability
                   awslogs-region: eu-central-1
-                  awslogs-stream-prefix: observability/prod/fluentbit/
+                  awslogs-stream-prefix: 'ecs'
               portMappings:
                 - containerPort: 1337
                   protocol: tcp
@@ -160,7 +170,10 @@ resources:
           load_balancer: fluentbit-http
           service:
             desired_count: 2
-          target: stalwart-metrics
+          targets:
+            - container_name: fluentbit
+              container_port: 1337
+              target_name: stalwart-metrics
 
       autoscalers:
         fluentbit:

pulumi/config.stage.yaml

@@ -1,5 +1,7 @@
 ---
 
+.fluentbit_image: &FLUENTBIT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:426154b20a1b0d005f9f6029836a5902e5b5b228edd9688686f10d373e72a5b2
+
 config:
   build_site24x7: False
   build_tbpulumi: True
@@ -10,6 +12,14 @@ resources:
       secret_names:
         - posthog_api_key
 
+  tb:cloudwatch:LogDestination:
+    observability:
+      log_group:
+        retention_in_days: 7
+      log_streams:
+        untagged: untagged
+      org_name: tb
+
   tb:network:MultiCidrVpc:
     fluentbit:
       # The observability project has all of 10.201.0.0/16 assigned to it, but let's not soak all
@@ -78,13 +88,13 @@ resources:
               secrets:
                 - name: POSTHOG_API_KEY
                   valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/stage/posthog_api_key-3xsHYd
-              image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:fdd1b4748cfaee29553ee2c83fcaa428b68ba8e88c2791e1626e282b48127b9d
+              image: *FLUENTBIT_IMAGE
               logConfiguration:
                 logDriver: awslogs
                 options:
-                  awslogs-group: observability-stage-fargate-fluentbit-loggroup-fluentbit
+                  awslogs-group: /tb/stage/observability
                   awslogs-region: eu-central-1
-                  awslogs-stream-prefix: observability/stage/fluentbit/
+                  awslogs-stream-prefix: 'ecs'
               portMappings:
                 - containerPort: 1337
                   hostPort: 1337
@@ -160,7 +170,10 @@ resources:
           load_balancer: fluentbit-http
           service:
             desired_count: 2
-          target: stalwart-metrics
+          targets:
+            - container_name: fluentbit
+              container_port: 1337
+              target_name: stalwart-metrics
 
       autoscalers:
         fluentbit:

pulumi/requirements.txt

@@ -1,3 +1,4 @@
 requests>=2.32.5
-tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@main
+pulumi_cloudflare>=6.14.0,<7
+tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@v0.0.18
 sdks/site24x7