Ship Stalwart logs to CloudWatch Logs

pulumi/__main__.py

@@ -25,6 +25,17 @@
     **psm_opts,
 )
 
+logdest_opts = resources.get('tb:cloudwatch:LogDestination', {})
+logdests = {
+    logdest_name: tb_pulumi.cloudwatch.LogDestination(
+        f'{project.name_prefix}-logdest-{logdest_name}',
+        app_name=logdest_name,
+        project=project,
+        **logdest_config,
+    )
+    for logdest_name, logdest_config in logdest_opts.items()
+}
+
 # Build out some private network space
 vpc_opts = resources['tb:network:MultiTierVpc']['vpc']
 vpc = tb_pulumi.network.MultiTierVpc(
@@ -71,6 +82,7 @@ def __stalwart_cluster(jumphost_rules: list[dict]):
     return stalwart.StalwartCluster(
         f'{project.name_prefix}-stalwart',
         project=project,
+        log_group_arn=logdests['stalwart'].resources['iam_policies']['write'].arn,
         private_subnets=vpc.resources['private_subnets'],
         public_subnets=vpc.resources['public_subnets'],
         node_additional_ingress_rules=jumphost_rules,

pulumi/bootstrap/bootstrap.py

@@ -10,13 +10,19 @@
 BOOTSTRAP_DIR = '/opt/stalwart-bootstrap'
 BOOTSTRAP_LOG = '/var/log/stalwart-bootstrap.log'
 INSTANCE_TAGS = {}
+
 # Map of template files to target files
 TEMPLATE_MAP = {
+    'fluent-bit.service.j2': '/usr/lib/systemd/system/fluent-bit.service',
+    'fluent-bit.yaml.j2': '/etc/fluent-bit/fluent-bit.yaml',
+    'journald.conf.j2': '/etc/systemd/journald.conf',
     'stalwart.toml.j2': '/opt/stalwart/etc/config.toml',
     'thundermail.service.j2': '/usr/lib/systemd/system/thundermail.service',
 }
 # Map of template variable to EC2 tags
 TEMPLATE_VALUE_TAG_MAP = {
+    'env': 'environment',
+    'function': 'postboot.stalwart.function',
     'https_paths': 'postboot.stalwart.https_paths',
     'node_services': 'postboot.stalwart.node_services',
     'node_id': 'postboot.stalwart.node_id',

pulumi/bootstrap/templates/fluent-bit.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Fluent Bit
+Documentation=https://docs.fluentbit.io/manual/
+Requires=network.target
+After=network.target
+
+[Service]
+Type=simple
+Environment="ENV={{ env }}"
+ExecStart=/opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yaml
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

pulumi/bootstrap/templates/fluent-bit.yaml.j2

@@ -0,0 +1,41 @@
+---
+
+service:
+  flush: 1
+  grace: 5
+  daemon: no
+  dns.mode: UDP
+  hot_reload: on
+  log_level: info
+  storage.path: /fluent-bit/buffers
+  storage.backlog.flush_on_shutdown: on
+  storage.keep.rejected: on
+  storage.rejected.path: /fluent-bit/dlq
+
+pipeline:
+  inputs:
+    - name: systemd
+      tag: cloudwatch.stalwart.{{ function }}
+      db: /opt/fluent-bit/thundermail.cursor
+      systemd_filter: _SYSTEMD_UNIT=thundermail.service
+
+  filters: []
+
+  outputs:
+    # Send logs onward to CloudWatch. Log groups by the given name must pre-exist, and this service
+    # must have sufficient IAM permissions to post events to these log streams. If these log streams
+    # do not exist, this service must have permission to create them.
+    - name: cloudwatch_logs
+      match: cloudwatch.stalwart.mail
+      log_group_name: /tb/${ENV}/stalwart
+      log_stream_name: mail
+      region: eu-central-1
+      log_key: MESSAGE
+
+    - name: cloudwatch_logs
+      match: cloudwatch.stalwart.api
+      log_group_name: /tb/${ENV}/stalwart
+      log_stream_name: api
+      region: eu-central-1
+      log_key: MESSAGE
+

pulumi/bootstrap/templates/journald.conf.j2

@@ -0,0 +1,6 @@
+[Journal]
+{% if env == 'prod' %}
+MaxRetentionSec=3day
+{% else %}
+MaxRetentionSec=7day
+{% endif %}

pulumi/config.dev.yaml

@@ -8,6 +8,15 @@ resources:
         - stalwart.postboot.keycloak_backend
       recovery_window_in_days: 0
 
+  tb:cloudwatch:LogDestination:
+    stalwart:
+      log_group:
+        retention_in_days: 7
+      log_streams:
+        api: api
+        mail: mail
+      org_name: tb
+
   tb:network:MultiTierVpc:
     vpc:
       cidr_block: 10.2.0.0/16
@@ -43,18 +52,18 @@ resources:
       additional_routes:
         private:
           - destination_cidr_block: 10.202.0.0/22  # observability-dev
-            vpc_peering_connection_id: pcx-0d2027442f0e54ca4
+            vpc_peering_connection_id: pcx-04d7e54008cd9326c
         public: []
       endpoint_interfaces:
         - secretsmanager
 
-  # tb:ec2:SshableInstance: {}
+  tb:ec2:SshableInstance: {}
   # Fill out this template to build an SSH bastion
-  tb:ec2:SshableInstance:
-    bastion:
-      ssh_keypair_name: mailstrom-dev
-      source_cidrs:
-        - 10.2.0.0/16  # Internal access
+  # tb:ec2:SshableInstance:
+  #   bastion:
+  #     ssh_keypair_name: mailstrom-dev
+  #     source_cidrs:
+  #       - 10.2.0.0/16  # Internal access
 
   tb:mailstrom:StalwartCluster:
     thundermail:
@@ -81,6 +90,7 @@ resources:
       nodes:
         "0": # Must be a unique, stringified integer
           disable_api_termination: True
+          function: 'mail'
           ignore_ami_changes: True
           ignore_user_data_changes: True
           instance_type: t3.micro
@@ -99,6 +109,7 @@ resources:
           storage_capacity: 20
         "50":
           disable_api_termination: True
+          function: 'api'
           ignore_ami_changes: True
           ignore_user_data_changes: True
           instance_type: t3.micro

pulumi/config.prod.yaml

@@ -7,6 +7,15 @@ resources:
         - stalwart.postboot.keycloak_backend
         - stalwart.postboot.postgresql_backend
 
+  tb:cloudwatch:LogDestination:
+    stalwart:
+      log_group:
+        retention_in_days: 3
+      log_streams:
+        api: api
+        mail: mail
+      org_name: tb
+
   tb:network:MultiTierVpc:
     vpc:
       cidr_block: 10.0.0.0/16
@@ -93,6 +102,7 @@ resources:
       nodes:
         "0": # Must be a unique, stringified integer
           disable_api_termination: True
+          function: 'mail'
           ignore_ami_changes: True
           ignore_user_data_changes: True
           instance_type: t3a.large
@@ -110,6 +120,7 @@ resources:
           storage_capacity: 20
         "1": # Must be a unique, stringified integer
           disable_api_termination: True
+          function: 'mail'
           ignore_ami_changes: True
           ignore_user_data_changes: True
           instance_type: t3a.large
@@ -127,6 +138,7 @@ resources:
           storage_capacity: 20
         "50":
           disable_api_termination: True
+          function: 'api'
           ignore_ami_changes: True
           ignore_user_data_changes: True
           instance_type: t3.micro

pulumi/config.stage.yaml

@@ -7,6 +7,15 @@ resources:
         - stalwart.postboot.postgresql_backend
         - stalwart.postboot.keycloak_backend
 
+  tb:cloudwatch:LogDestination:
+    stalwart:
+      log_group:
+        retention_in_days: 7
+      log_streams:
+        api: api
+        mail: mail
+      org_name: tb
+
   tb:network:MultiTierVpc:
     vpc:
       cidr_block: 10.1.0.0/16
@@ -108,6 +117,7 @@ resources:
         #   subnet: subnet-07ade1ed35462907d  # eu-central-1a
         "1":
           disable_api_termination: True
+          function: 'mail'
           ignore_ami_changes: True
           ignore_user_data_changes: True
           instance_type: t3a.large
@@ -126,6 +136,7 @@ resources:
           subnet: subnet-07712b990eb0d17c0  # eu-central-1b
         "50":
           disable_api_termination: True
+          function: 'api'
           ignore_ami_changes: True
           ignore_user_data_changes: True
           instance_type: t3.micro

pulumi/requirements.txt

@@ -1,4 +1,4 @@
 Jinja2>=3.1,<4.0
 pulumi_cloudflare==6.6.0
-tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@v0.0.16
+tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@v0.0.18
 toml>=0.10.2,<0.11

pulumi/stalwart/__init__.py

@@ -274,6 +274,7 @@ def __init__(
         self,
         name: str,
         project: tb_pulumi.ThunderbirdPulumiProject,
+        log_group_arn: str,
         private_subnets: list[aws.ec2.Subnet],
         public_subnets: list[aws.ec2.Subnet],
         https_features: list = [],
@@ -343,8 +344,16 @@ def __init__(
         s3_bucket, s3_secret, s3_policy = stalwart_s3.s3(self=self)
 
         # Build an IAM role with a policy to enable node bootstrapping
-        profile_policy, role, profile_postboot_attachment, profile_s3_attachment, profile = stalwart_iam.iam(
+        (
+            profile_policy,
+            role,
+            profile_postboot_attachment,
+            profile_s3_attachment,
+            profile_logwrite_attachment,
+            profile,
+        ) = stalwart_iam.iam(
             self,
+            log_group_arn=log_group_arn,
             s3_policy=s3_policy,
         )
 
@@ -463,6 +472,7 @@ def __init__(
                 'spam_filter_secret': config_secrets['spam_filter'],
                 'node_profile': profile,
                 'node_profile_policy': profile_policy,
+                'node_profile_logwrite_attachment': profile_logwrite_attachment,
                 'node_profile_postboot_policy_attachment': profile_postboot_attachment,
                 'node_profile_s3_policy_attachment': profile_s3_attachment,
                 'node_sgs': self.node_sgs,
@@ -671,6 +681,7 @@ def node(
         depends_on: list = [],
         disable_api_stop: bool = False,
         disable_api_termination: bool = False,
+        function: str = 'unknown',
         ignore_ami_changes: bool = True,
         ignore_user_data_changes: bool = True,
         instance_type: str = 't3.micro',
@@ -694,6 +705,9 @@ def node(
             False.
         :type disable_api_termination: bool, optional
 
+        :param function: This becomes the ``postboot.stalwart.function`` tag on the instance and the ``function``
+            variable inside of postboot templates.
+
         :param ignore_ami_changes: When True, changes to the instance's AMI will not be applied. This prevents unwanted
             rebuilding of cluster nodes, potentially causing downtime. Set to False if the AMI has changed and you
             intend on rebuilding the node. Defaults to True.
@@ -749,6 +763,7 @@ def node(
         postboot_tags = {
             'postboot.stalwart.aws_region': self.project.aws_region,
             'postboot.stalwart.env': self.project.stack,
+            'postboot.stalwart.function': function,
             'postboot.stalwart.https_paths': ','.join(https_paths),
             'postboot.stalwart.image': self.stalwart_image,
             'postboot.stalwart.node_services': node_services_tag,
@@ -810,6 +825,9 @@ def user_data(self):
         archive_file_base = './bootstrap'
         archive_files = [
             'bootstrap.py',
+            'templates/fluent-bit.service.j2',
+            'templates/fluent-bit.yaml.j2',
+            'templates/journald.conf.j2',
             'templates/ports.j2',
             'templates/stalwart.toml.j2',
             'templates/thundermail.service.j2',

pulumi/stalwart/iam.py

@@ -5,11 +5,12 @@
 import json
 import pulumi_aws as aws
 
-from tb_pulumi.constants import ASSUME_ROLE_POLICY, IAM_POLICY_DOCUMENT
+from tb_pulumi.constants import ASSUME_ROLE_POLICY
 
 
 def iam(
     self,
+    log_group_arn: str,
     s3_policy: aws.iam.Policy,
 ) -> tuple[
     aws.iam.Policy, aws.iam.Role, aws.iam.RolePolicyAttachment, aws.iam.RolePolicyAttachment, aws.iam.InstanceProfile
@@ -32,14 +33,18 @@ def iam(
             + f':secret:mailstrom/{self.project.stack}/stalwart.postboot.*'
         ),
     ]
-    profile_postboot_policy_doc = IAM_POLICY_DOCUMENT.copy()
-    profile_postboot_policy_doc['Statement'][0].update(
-        {
-            'Sid': 'AllowPostbootSecretAccess',
-            'Action': ['secretsmanager:GetSecretValue'],
-            'Resource': bootstrap_secret_arns,
-        }
-    )
+    profile_postboot_policy_doc = {
+        'Version': '2012-10-17',
+        'Statement': [
+            {
+                'Sid': 'AllowPostbootSecretAccess',
+                'Effect': 'Allow',
+                'Action': ['secretsmanager:GetSecretValue'],
+                'Resource': bootstrap_secret_arns,
+            }
+        ],
+    }
+
     profile_policy = aws.iam.Policy(
         f'{self.name}-policy-nodeprofile',
         path='/',
@@ -64,7 +69,19 @@ def iam(
         role=role.name,
         policy_arn=s3_policy.arn,
     )
+    profile_logwrite_attachment = aws.iam.RolePolicyAttachment(
+        f'{self.name}-rpa-nodeprofile-logs',
+        role=role.name,
+        policy_arn=log_group_arn,
+    )
 
     profile = aws.iam.InstanceProfile(f'{self.name}-ip-nodeprofile', name=f'{self.name}-nodeprofile', role=role.name)
 
-    return profile_policy, role, profile_postboot_attachment, profile_s3_attachment, profile
+    return (
+        profile_policy,
+        role,
+        profile_postboot_attachment,
+        profile_s3_attachment,
+        profile_logwrite_attachment,
+        profile,
+    )