diff --git a/pulumi/Pulumi.dev.yaml b/pulumi/Pulumi.dev.yaml index 6a30ae2..caf11ab 100644 --- a/pulumi/Pulumi.dev.yaml +++ b/pulumi/Pulumi.dev.yaml @@ -1,3 +1,7 @@ config: observability:posthog_api_key: secure: AAABAACLeD5lasJAmY66NyJXtacSmTSMj/PiXtmBNIHeBfLx2HA3mhTzyWkPZnD9j8MCYPbtnjJiWeZBzOROWVKEcKpuysV/FV5CDoHCJg== + observability:cloudflare_zone_id: + secure: AAABAKragv0vFq2i/lBhwJRTkD/wjW8jefzGy6Mq5A4eZubZLEeh4cSFESB+M3Fv34TvYNxJpFlT208EMqUQLw== + cloudflare:apiToken: + secure: AAABAJfLM7HTgF++SR/ps+pkQQFMNxc0XyRidPcCJKD2nzpc9mRnqdEguDnJlKKwtStygHkRT95D/6n568y+TUf/hkGut6P5 diff --git a/pulumi/Pulumi.prod.yaml b/pulumi/Pulumi.prod.yaml index 682a5ab..c49f704 100644 --- a/pulumi/Pulumi.prod.yaml +++ b/pulumi/Pulumi.prod.yaml @@ -201,6 +201,10 @@ config: selection_type: all_monitors statusiq_role: super_admin user_role: super_admin + observability:cloudflare_zone_id: + secure: AAABAFkTz7RxaV86Kw6RQ+XJ9O1orS7QTUgkRDIwvoE4kXYdQGWJ2i6zj9XqLoevXb3PgOGnNmv550aMA/H+zA== + cloudflare:apiToken: + secure: AAABAJ/XXYzFsIlhvWdl0FwFnUWHUTKIyJMAFNicpTtxUlWcOurQC9Y1O5qu+hjL/DmMRoV2c0+KdBy00YUzvIWf0HWuyIZk # The "api" kind seems to have some bugs in the Terraform provider, but this is what such a # configuration should look like: diff --git a/pulumi/Pulumi.stage.yaml b/pulumi/Pulumi.stage.yaml index a3b9132..5c0d806 100644 --- a/pulumi/Pulumi.stage.yaml +++ b/pulumi/Pulumi.stage.yaml @@ -1,3 +1,7 @@ -config: +config: observability:posthog_api_key: secure: AAABADNVbsoTmx0hogPjFb+Egd5TX7Wheactt3JgEv21j1G+OJSjHF+CUpY/w9qTS3KEw4IdoYUuufBKX2sJXudMUYFAEc8m8o2rg56eGw== + observability:cloudflare_zone_id: + secure: AAABALYXXhMfqqRW1FbwgqDZIsD7VwJ+AGDcqv4RiMDfi7cadVqba5L1esVND3ieXJdpE/qwcnN1AIUfBe6Zpw== + cloudflare:apiToken: + secure: AAABAB5n/gkdIZz7ZKyeu00kWDwDrpPr0Wl/FD449u+/WrX88r8nk+FHj/GTjyQL/GURB9TrMRzJ2as9ceKoWfOO6Z4+24Yk diff --git a/pulumi/__main__.py b/pulumi/__main__.py index f84bcb1..74ddf9b 100644 --- a/pulumi/__main__.py +++ b/pulumi/__main__.py @@ -9,7 +9,11 @@ of any of those larger infrastructure patterns. """ +import pulumi +import pulumi_cloudflare as cloudflare import tb_pulumi +import tb_pulumi.cloudwatch +import tb_pulumi.iam import tb_pulumi.fargate import tb_pulumi.network import tb_pulumi.secrets @@ -36,6 +40,17 @@ **psm_opts, ) + logdest_opts = resources.get('tb:cloudwatch:LogDestination', {}) + logdests = { + logdest_name: tb_pulumi.cloudwatch.LogDestination( + f'{project.name_prefix}-logdest-{logdest_name}', + project=project, + app_name=logdest_name, + **logdest_config, + ) + for logdest_name, logdest_config in logdest_opts.items() + } + vpc_config = resources.get('tb:network:MultiCidrVpc', {}).get('fluentbit', {}) vpc_fluentbit = tb_pulumi.network.MultiCidrVpc( f'{project.name_prefix}-vpc-fluentbit', @@ -49,8 +64,20 @@ project=project, subnets=vpc_fluentbit.resources.get('subnets', []), **cluster_config, + opts=pulumi.ResourceOptions(depends_on=[vpc_fluentbit]), ) for cluster_name, cluster_config in resources.get( 'tb:fargate:AutoscalingFargateCluster' ).items() } + + cloudflare_zone_id = project.pulumi_config.require_secret('cloudflare_zone_id') + fluent_bit_dns = cloudflare.DnsRecord( + f'{project.name_prefix}-dns-fluentbit', + name='fluentbit' if project.stack == 'prod' else f'fluentbit-{project.stack}', + content=ecs_clusters['fluentbit'].resources['load_balancers']['fluentbit-http'].dns_name, + ttl=60, + type='CNAME', + zone_id=cloudflare_zone_id, + opts=pulumi.ResourceOptions(depends_on=[*ecs_clusters.values()]), + ) diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index e29a67a..da13d53 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -1,5 +1,7 @@ --- +.fluentbit_image: &FLUENTBIT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:426154b20a1b0d005f9f6029836a5902e5b5b228edd9688686f10d373e72a5b2 + config: build_site24x7: False build_tbpulumi: True @@ -10,6 +12,14 @@ resources: secret_names: - posthog_api_key + tb:cloudwatch:LogDestination: + observability: + log_group: + retention_in_days: 7 + log_streams: + untagged: untagged + org_name: tb + tb:network:MultiCidrVpc: fluentbit: # The observability project has all of 10.202.0.0/16 assigned to it, but let's not soak all @@ -30,15 +40,15 @@ resources: - secretsmanager additional_routes: - destination_cidr_block: 10.2.0.0/16 # mailstrom-dev - vpc_peering_connection_id: pcx-0d2027442f0e54ca4 - + vpc_peering_connection_id: pcx-04d7e54008cd9326c + tb:fargate:AutoscalingFargateCluster: fluentbit: cluster: {} container_security_groups: - fluentbit: - fluentbit-http: + fluentbit: # Service + fluentbit-http: # Load Balancer rules: ingress: - description: Allow traffic from the load balancer to the container @@ -61,7 +71,7 @@ resources: ssm_params: {} task_definitions: - fluentbit: + fluentbit: # Service container_definitions: - name: fluentbit environment: @@ -78,13 +88,13 @@ resources: secrets: - name: POSTHOG_API_KEY valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/dev/posthog_api_key-e3UEK4 - image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:fdd1b4748cfaee29553ee2c83fcaa428b68ba8e88c2791e1626e282b48127b9d + image: *FLUENTBIT_IMAGE logConfiguration: logDriver: awslogs options: - awslogs-group: observability-dev-fargate-fluentbit-loggroup-fluentbit + awslogs-group: /tb/dev/observability awslogs-region: eu-central-1 - awslogs-stream-prefix: observability/dev/fluentbit/ + awslogs-stream-prefix: 'ecs' portMappings: - containerPort: 1337 protocol: tcp @@ -99,7 +109,7 @@ resources: - FARGATE load_balancer_security_groups: - fluentbit-http: + fluentbit-http: # Load Balancer description: Governs access to the fluent-bit-http load balancer in dev rules: ingress: @@ -144,24 +154,31 @@ resources: ip_address_type: ipv4 listeners: - fluentbit-http: - stalwart-metrics: + fluentbit-http: # Load Balancer + stalwart-metrics: # Target # This cert is for fluentbit-dev.tb.pro certificate_arn: arn:aws:acm:eu-central-1:768512802988:certificate/04dd0573-a3cc-4c19-b483-a868876c63b0 port: 443 protocol: HTTPS services: - fluentbit: + fluentbit: # Service assign_public_ip: yes - container_name: fluentbit + container_name: fluentbit # Name from container definition container_port: 1337 load_balancer: fluentbit-http service: desired_count: 2 - target: stalwart-metrics + targets: + - container_name: fluentbit + container_port: 1337 + target_name: stalwart-metrics + + extra_policies: + fluentbit: + - arn:aws:iam::768512802988:policy/observability-dev-observability-logs-write-access autoscalers: fluentbit: - min_capacity: 2 - max_capacity: 4 \ No newline at end of file + min_capacity: 1 + max_capacity: 1 \ No newline at end of file diff --git a/pulumi/config.prod.yaml b/pulumi/config.prod.yaml index 333a236..e3024be 100644 --- a/pulumi/config.prod.yaml +++ b/pulumi/config.prod.yaml @@ -1,5 +1,7 @@ --- +.fluentbit_image: &FLUENTBIT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:426154b20a1b0d005f9f6029836a5902e5b5b228edd9688686f10d373e72a5b2 + config: build_site24x7: True build_tbpulumi: True @@ -11,6 +13,14 @@ resources: secret_names: - posthog_api_key + tb:cloudwatch:LogDestination: + observability: + log_group: + retention_in_days: 3 + log_streams: + untagged: untagged + org_name: tb + tb:network:MultiCidrVpc: fluentbit: # The observability project has all of 10.200.0.0/16 assigned to it, but let's not soak all @@ -79,13 +89,13 @@ resources: secrets: - name: POSTHOG_API_KEY valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/prod/posthog_api_key-pVtqmp - image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:aa968a499d7e + image: *FLUENTBIT_IMAGE logConfiguration: logDriver: awslogs options: - awslogs-group: observability-prod-fargate-fluentbit-loggroup-fluentbit + awslogs-group: /tb/prod/observability awslogs-region: eu-central-1 - awslogs-stream-prefix: observability/prod/fluentbit/ + awslogs-stream-prefix: 'ecs' portMappings: - containerPort: 1337 protocol: tcp @@ -160,7 +170,10 @@ resources: load_balancer: fluentbit-http service: desired_count: 2 - target: stalwart-metrics + targets: + - container_name: fluentbit + container_port: 1337 + target_name: stalwart-metrics autoscalers: fluentbit: diff --git a/pulumi/config.stage.yaml b/pulumi/config.stage.yaml index 9fe4881..0b06952 100644 --- a/pulumi/config.stage.yaml +++ b/pulumi/config.stage.yaml @@ -1,5 +1,7 @@ --- +.fluentbit_image: &FLUENTBIT_IMAGE 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:426154b20a1b0d005f9f6029836a5902e5b5b228edd9688686f10d373e72a5b2 + config: build_site24x7: False build_tbpulumi: True @@ -10,6 +12,14 @@ resources: secret_names: - posthog_api_key + tb:cloudwatch:LogDestination: + observability: + log_group: + retention_in_days: 7 + log_streams: + untagged: untagged + org_name: tb + tb:network:MultiCidrVpc: fluentbit: # The observability project has all of 10.201.0.0/16 assigned to it, but let's not soak all @@ -78,13 +88,13 @@ resources: secrets: - name: POSTHOG_API_KEY valueFrom: arn:aws:secretsmanager:eu-central-1:768512802988:secret:observability/stage/posthog_api_key-3xsHYd - image: 768512802988.dkr.ecr.eu-central-1.amazonaws.com/thunderbird/fluent-bit:fdd1b4748cfaee29553ee2c83fcaa428b68ba8e88c2791e1626e282b48127b9d + image: *FLUENTBIT_IMAGE logConfiguration: logDriver: awslogs options: - awslogs-group: observability-stage-fargate-fluentbit-loggroup-fluentbit + awslogs-group: /tb/stage/observability awslogs-region: eu-central-1 - awslogs-stream-prefix: observability/stage/fluentbit/ + awslogs-stream-prefix: 'ecs' portMappings: - containerPort: 1337 hostPort: 1337 @@ -160,7 +170,10 @@ resources: load_balancer: fluentbit-http service: desired_count: 2 - target: stalwart-metrics + targets: + - container_name: fluentbit + container_port: 1337 + target_name: stalwart-metrics autoscalers: fluentbit: diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt index 3573c82..e964f86 100644 --- a/pulumi/requirements.txt +++ b/pulumi/requirements.txt @@ -1,3 +1,4 @@ requests>=2.32.5 -tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@main +pulumi_cloudflare>=6.14.0,<7 +tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@v0.0.18 sdks/site24x7