From 87e1c136cec7aa2621e829edec5ff6f9aea0dddd Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Fri, 3 Apr 2026 13:54:38 -0600 Subject: [PATCH 01/12] Add log destination, remove bastion in dev --- pulumi/__main__.py | 11 +++++++++++ pulumi/config.dev.yaml | 21 +++++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/pulumi/__main__.py b/pulumi/__main__.py index 426e7b6..295becf 100644 --- a/pulumi/__main__.py +++ b/pulumi/__main__.py @@ -25,6 +25,17 @@ **psm_opts, ) +logdest_opts = resources.get('tb:cloudwatch:LogDestination', {}) +logdests = { + logdest_name: tb_pulumi.cloudwatch.LogDestination( + f'{project.name_prefix}-logdest-{logdest_name}', + app_name=logdest_name, + project=project, + **logdest_config, + ) + for logdest_name, logdest_config in logdest_opts.items() +} + # Build out some private network space vpc_opts = resources['tb:network:MultiTierVpc']['vpc'] vpc = tb_pulumi.network.MultiTierVpc( diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index 6bfad6d..b849b52 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -8,6 +8,15 @@ resources: - stalwart.postboot.keycloak_backend recovery_window_in_days: 0 + tb:cloudwatch:LogDestination: + stalwart: + log_group: + retention_in_days: 7 + log_streams: + stalwart-api-logs: api + stalwart-mail-logs: mail + org_name: tb + tb:network:MultiTierVpc: vpc: cidr_block: 10.2.0.0/16 @@ -48,13 +57,13 @@ resources: endpoint_interfaces: - secretsmanager - # tb:ec2:SshableInstance: {} + tb:ec2:SshableInstance: {} # Fill out this template to build an SSH bastion - tb:ec2:SshableInstance: - bastion: - ssh_keypair_name: mailstrom-dev - source_cidrs: - - 10.2.0.0/16 # Internal access + # tb:ec2:SshableInstance: + # bastion: + # ssh_keypair_name: mailstrom-dev + # source_cidrs: + # - 10.2.0.0/16 # Internal access tb:mailstrom:StalwartCluster: thundermail: From 39c4d804fe93125adf022cb1916061a98b93d258 Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Mon, 6 Apr 2026 14:37:52 -0600 Subject: [PATCH 02/12] Install fluent-bit to ship logs --- pulumi/bootstrap/bootstrap.py | 3 ++ pulumi/bootstrap/templates/fluent-bit.yaml.j2 | 25 ++++++++++++++++ pulumi/config.dev.yaml | 2 ++ pulumi/requirements.txt | 2 +- pulumi/stalwart/__init__.py | 6 ++++ pulumi/stalwart_instance_user_data.sh.j2 | 29 ++++++++++++++++--- 6 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 pulumi/bootstrap/templates/fluent-bit.yaml.j2 diff --git a/pulumi/bootstrap/bootstrap.py b/pulumi/bootstrap/bootstrap.py index 8a67f18..167070e 100644 --- a/pulumi/bootstrap/bootstrap.py +++ b/pulumi/bootstrap/bootstrap.py @@ -10,13 +10,16 @@ BOOTSTRAP_DIR = '/opt/stalwart-bootstrap' BOOTSTRAP_LOG = '/var/log/stalwart-bootstrap.log' INSTANCE_TAGS = {} + # Map of template files to target files TEMPLATE_MAP = { + 'fluent-bit.yaml.j2': '/etc/fluent-bit/fluent-bit.yaml', 'stalwart.toml.j2': '/opt/stalwart/etc/config.toml', 'thundermail.service.j2': '/usr/lib/systemd/system/thundermail.service', } # Map of template variable to EC2 tags TEMPLATE_VALUE_TAG_MAP = { + 'function': 'postboot.stalwart.function', 'https_paths': 'postboot.stalwart.https_paths', 'node_services': 'postboot.stalwart.node_services', 'node_id': 'postboot.stalwart.node_id', diff --git a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 new file mode 100644 index 0000000..fd65d6f --- /dev/null +++ b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 @@ -0,0 +1,25 @@ +--- + +service: + flush: 1 + grace: 5 + daemon: no + dns.mode: UDP + hot_reload: on + log_level: info + storage.path: /fluent-bit/buffers + storage.backlog.flush_on_shutdown: on + storage.keep.rejected: on + storage.rejected.path: /fluent-bit/dlq + +pipeline: + inputs: + - name: systemd + tag: cloudwatch.stalwart.{{ function }} + systemd_filter: _SYSTEMD_UNIT=thundermail.service + + filters: [] + + outputs: + - name: forward + match: '*' diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index b849b52..165291f 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -90,6 +90,7 @@ resources: nodes: "0": # Must be a unique, stringified integer disable_api_termination: True + function: 'mail' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3.micro @@ -108,6 +109,7 @@ resources: storage_capacity: 20 "50": disable_api_termination: True + function: 'api' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3.micro diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt index 6606c2b..6b3e795 100644 --- a/pulumi/requirements.txt +++ b/pulumi/requirements.txt @@ -1,4 +1,4 @@ Jinja2>=3.1,<4.0 pulumi_cloudflare==6.6.0 -tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@v0.0.16 +tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@main toml>=0.10.2,<0.11 diff --git a/pulumi/stalwart/__init__.py b/pulumi/stalwart/__init__.py index 119d3ce..75ca158 100644 --- a/pulumi/stalwart/__init__.py +++ b/pulumi/stalwart/__init__.py @@ -671,6 +671,7 @@ def node( depends_on: list = [], disable_api_stop: bool = False, disable_api_termination: bool = False, + function: str = 'unknown', ignore_ami_changes: bool = True, ignore_user_data_changes: bool = True, instance_type: str = 't3.micro', @@ -694,6 +695,9 @@ def node( False. :type disable_api_termination: bool, optional + :param function: This becomes the ``postboot.stalwart.function`` tag on the instance and the ``function`` + variable inside of postboot templates. + :param ignore_ami_changes: When True, changes to the instance's AMI will not be applied. This prevents unwanted rebuilding of cluster nodes, potentially causing downtime. Set to False if the AMI has changed and you intend on rebuilding the node. Defaults to True. @@ -749,6 +753,7 @@ def node( postboot_tags = { 'postboot.stalwart.aws_region': self.project.aws_region, 'postboot.stalwart.env': self.project.stack, + 'postboot.stalwart.function': function, 'postboot.stalwart.https_paths': ','.join(https_paths), 'postboot.stalwart.image': self.stalwart_image, 'postboot.stalwart.node_services': node_services_tag, @@ -810,6 +815,7 @@ def user_data(self): archive_file_base = './bootstrap' archive_files = [ 'bootstrap.py', + 'templates/fluent-bit.yaml.j2', 'templates/ports.j2', 'templates/stalwart.toml.j2', 'templates/thundermail.service.j2', diff --git a/pulumi/stalwart_instance_user_data.sh.j2 b/pulumi/stalwart_instance_user_data.sh.j2 index d572ec4..94885f4 100644 --- a/pulumi/stalwart_instance_user_data.sh.j2 +++ b/pulumi/stalwart_instance_user_data.sh.j2 @@ -8,30 +8,51 @@ set -x set -e +# Places data get stored BOOTSTRAP_DIR=/opt/stalwart-bootstrap BOOTSTRAP_TBZ=/root/bootstrap.tbz STALWART_DIR=/opt/stalwart +# Install the fluent-bit repo for Amazon Linux +echo '[fluent-bit] +name = Fluent Bit +baseurl = https://packages.fluentbit.io/amazonlinux/2023/ +gpgcheck=1 +gpgkey=https://packages.fluentbit.io/fluentbit.key +enabled=1' > /etc/yum.repos.d/fluent-bit.repo + +# Update system, install dependencies dnf update -y -dnf install -y bzip2 docker python3.12 +dnf install -y bzip2 docker fluent-bit python3.12 -mkdir -p $STALWART_DIR/etc +# Delete the default fluent-bit config; we'll template a new one in Phase 2 +rm -f /etc/fluent-bit/fluent-bit.conf + +# Make sure fluent-bit uses the right config file +sed -i 's/fluent-bit.conf/fluent-bit.yaml/' /usr/lib/systemd/system/fluent-bit.service +# Set up Stalwart config directory and virtual environment +mkdir -p $STALWART_DIR/etc python3.12 -m ensurepip pip3.12 install virtualenv /usr/local/bin/virtualenv -p python3.12 $BOOTSTRAP_DIR +# Make sure Docker always runs systemctl start docker systemctl enable docker +# Install Bootstrap Phase 2 echo '{{ bootstrap_tbz_base64 }}' | base64 -d > $BOOTSTRAP_TBZ tar -xvf $BOOTSTRAP_TBZ -C $BOOTSTRAP_DIR +# Run Bootstrap Phase 2 source $BOOTSTRAP_DIR/bin/activate pip install -r $BOOTSTRAP_DIR/requirements.txt python $BOOTSTRAP_DIR/bootstrap.py +# Ensure all our services are online with current configs systemctl daemon-reload +systemctl enable fluent-bit +systemctl restart fluent-bit systemctl enable thundermail -systemctl start thundermail - +systemctl restart thundermail From da7db96a8a6cfa530885fafc4cb0015c27f876b2 Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Tue, 7 Apr 2026 14:26:06 -0600 Subject: [PATCH 03/12] Rebuild dev, add env to templates --- pulumi/bootstrap/bootstrap.py | 1 + pulumi/bootstrap/templates/fluent-bit.yaml.j2 | 6 ++++++ pulumi/config.dev.yaml | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pulumi/bootstrap/bootstrap.py b/pulumi/bootstrap/bootstrap.py index 167070e..345f296 100644 --- a/pulumi/bootstrap/bootstrap.py +++ b/pulumi/bootstrap/bootstrap.py @@ -19,6 +19,7 @@ } # Map of template variable to EC2 tags TEMPLATE_VALUE_TAG_MAP = { + 'env': 'environment', 'function': 'postboot.stalwart.function', 'https_paths': 'postboot.stalwart.https_paths', 'node_services': 'postboot.stalwart.node_services', diff --git a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 index fd65d6f..6822cbf 100644 --- a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 +++ b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 @@ -23,3 +23,9 @@ pipeline: outputs: - name: forward match: '*' +{%- if env == 'prod' %} + host: fluentbit.tb.pro +{%- else %} + host: {{ 'fluentbit-' + env + '.tb.pro' }} +{%- endif %} + port: 24224 diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index 165291f..8f3d0c2 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -52,7 +52,7 @@ resources: additional_routes: private: - destination_cidr_block: 10.202.0.0/22 # observability-dev - vpc_peering_connection_id: pcx-0d2027442f0e54ca4 + vpc_peering_connection_id: pcx-04d7e54008cd9326c public: [] endpoint_interfaces: - secretsmanager From 68408cdf881fe4c0cf7c71c78e8a06da53fdd6da Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Thu, 9 Apr 2026 09:31:56 -0600 Subject: [PATCH 04/12] Remove log destination (obsv should have it) --- pulumi/config.dev.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index 8f3d0c2..17da1e5 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -8,15 +8,6 @@ resources: - stalwart.postboot.keycloak_backend recovery_window_in_days: 0 - tb:cloudwatch:LogDestination: - stalwart: - log_group: - retention_in_days: 7 - log_streams: - stalwart-api-logs: api - stalwart-mail-logs: mail - org_name: tb - tb:network:MultiTierVpc: vpc: cidr_block: 10.2.0.0/16 From 91d7de96e74b5926f3780ee135dcb890cb874619 Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Thu, 9 Apr 2026 12:50:06 -0600 Subject: [PATCH 05/12] Build Stalwart log group; give instances write access --- pulumi/__main__.py | 1 + pulumi/config.dev.yaml | 9 +++++++++ pulumi/stalwart/__init__.py | 12 +++++++++++- pulumi/stalwart/iam.py | 35 ++++++++++++++++++++++++++--------- 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/pulumi/__main__.py b/pulumi/__main__.py index 295becf..86e407c 100644 --- a/pulumi/__main__.py +++ b/pulumi/__main__.py @@ -82,6 +82,7 @@ def __stalwart_cluster(jumphost_rules: list[dict]): return stalwart.StalwartCluster( f'{project.name_prefix}-stalwart', project=project, + log_group_arn=logdests['stalwart'].resources['iam_policies']['write'].arn, private_subnets=vpc.resources['private_subnets'], public_subnets=vpc.resources['public_subnets'], node_additional_ingress_rules=jumphost_rules, diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index 17da1e5..cfff08c 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -8,6 +8,15 @@ resources: - stalwart.postboot.keycloak_backend recovery_window_in_days: 0 + tb:cloudwatch:LogDestination: + stalwart: + log_group: + retention_in_days: 7 + log_streams: + api: api + mail: mail + org_name: tb + tb:network:MultiTierVpc: vpc: cidr_block: 10.2.0.0/16 diff --git a/pulumi/stalwart/__init__.py b/pulumi/stalwart/__init__.py index 75ca158..2c42bd1 100644 --- a/pulumi/stalwart/__init__.py +++ b/pulumi/stalwart/__init__.py @@ -274,6 +274,7 @@ def __init__( self, name: str, project: tb_pulumi.ThunderbirdPulumiProject, + log_group_arn: str, private_subnets: list[aws.ec2.Subnet], public_subnets: list[aws.ec2.Subnet], https_features: list = [], @@ -343,8 +344,16 @@ def __init__( s3_bucket, s3_secret, s3_policy = stalwart_s3.s3(self=self) # Build an IAM role with a policy to enable node bootstrapping - profile_policy, role, profile_postboot_attachment, profile_s3_attachment, profile = stalwart_iam.iam( + ( + profile_policy, + role, + profile_postboot_attachment, + profile_s3_attachment, + profile_logwrite_attachment, + profile, + ) = stalwart_iam.iam( self, + log_group_arn=log_group_arn, s3_policy=s3_policy, ) @@ -463,6 +472,7 @@ def __init__( 'spam_filter_secret': config_secrets['spam_filter'], 'node_profile': profile, 'node_profile_policy': profile_policy, + 'node_profile_logwrite_attachment': profile_logwrite_attachment, 'node_profile_postboot_policy_attachment': profile_postboot_attachment, 'node_profile_s3_policy_attachment': profile_s3_attachment, 'node_sgs': self.node_sgs, diff --git a/pulumi/stalwart/iam.py b/pulumi/stalwart/iam.py index 92d598b..963e51e 100644 --- a/pulumi/stalwart/iam.py +++ b/pulumi/stalwart/iam.py @@ -10,6 +10,7 @@ def iam( self, + log_group_arn: str, s3_policy: aws.iam.Policy, ) -> tuple[ aws.iam.Policy, aws.iam.Role, aws.iam.RolePolicyAttachment, aws.iam.RolePolicyAttachment, aws.iam.InstanceProfile @@ -32,14 +33,18 @@ def iam( + f':secret:mailstrom/{self.project.stack}/stalwart.postboot.*' ), ] - profile_postboot_policy_doc = IAM_POLICY_DOCUMENT.copy() - profile_postboot_policy_doc['Statement'][0].update( - { - 'Sid': 'AllowPostbootSecretAccess', - 'Action': ['secretsmanager:GetSecretValue'], - 'Resource': bootstrap_secret_arns, - } - ) + profile_postboot_policy_doc = { + 'Version': '2012-10-17', + 'Statement': [ + { + 'Sid': 'AllowPostbootSecretAccess', + 'Effect': 'Allow', + 'Action': ['secretsmanager:GetSecretValue'], + 'Resource': bootstrap_secret_arns, + } + ], + } + profile_policy = aws.iam.Policy( f'{self.name}-policy-nodeprofile', path='/', @@ -64,7 +69,19 @@ def iam( role=role.name, policy_arn=s3_policy.arn, ) + profile_logwrite_attachment = aws.iam.RolePolicyAttachment( + f'{self.name}-rpa-nodeprofile-logs', + role=role.name, + policy_arn=log_group_arn, + ) profile = aws.iam.InstanceProfile(f'{self.name}-ip-nodeprofile', name=f'{self.name}-nodeprofile', role=role.name) - return profile_policy, role, profile_postboot_attachment, profile_s3_attachment, profile + return ( + profile_policy, + role, + profile_postboot_attachment, + profile_s3_attachment, + profile_logwrite_attachment, + profile, + ) From 0ee694dd93979e3ce92281a65676d7744c34c93e Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Thu, 9 Apr 2026 13:52:16 -0600 Subject: [PATCH 06/12] Ship logs directly to CWLogs --- pulumi/bootstrap/bootstrap.py | 1 + .../bootstrap/templates/fluent-bit.service.j2 | 14 +++++++++ pulumi/bootstrap/templates/fluent-bit.yaml.j2 | 30 ++++++++++++++----- pulumi/config.dev.yaml | 1 + pulumi/requirements.txt | 3 +- pulumi/stalwart/__init__.py | 1 + pulumi/stalwart_instance_user_data.sh.j2 | 3 -- 7 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 pulumi/bootstrap/templates/fluent-bit.service.j2 diff --git a/pulumi/bootstrap/bootstrap.py b/pulumi/bootstrap/bootstrap.py index 345f296..01dda5d 100644 --- a/pulumi/bootstrap/bootstrap.py +++ b/pulumi/bootstrap/bootstrap.py @@ -13,6 +13,7 @@ # Map of template files to target files TEMPLATE_MAP = { + 'fluent-bit.service.j2': '/usr/lib/systemd/system/fluent-bit.service', 'fluent-bit.yaml.j2': '/etc/fluent-bit/fluent-bit.yaml', 'stalwart.toml.j2': '/opt/stalwart/etc/config.toml', 'thundermail.service.j2': '/usr/lib/systemd/system/thundermail.service', diff --git a/pulumi/bootstrap/templates/fluent-bit.service.j2 b/pulumi/bootstrap/templates/fluent-bit.service.j2 new file mode 100644 index 0000000..13a7957 --- /dev/null +++ b/pulumi/bootstrap/templates/fluent-bit.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Fluent Bit +Documentation=https://docs.fluentbit.io/manual/ +Requires=network.target +After=network.target + +[Service] +Type=simple +Environment="ENV={{ env }}" +ExecStart=/opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yaml +Restart=always + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 index 6822cbf..73f4383 100644 --- a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 +++ b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 @@ -21,11 +21,25 @@ pipeline: filters: [] outputs: - - name: forward - match: '*' -{%- if env == 'prod' %} - host: fluentbit.tb.pro -{%- else %} - host: {{ 'fluentbit-' + env + '.tb.pro' }} -{%- endif %} - port: 24224 + # Send logs onward to CloudWatch. Log groups by the derived name must pre-exist, and this + # service must have sufficient IAM permissions to create log streams and post events to them. + - name: cloudwatch_logs + match: cloudwatch.stalwart.mail + log_group_name: /tb/${ENV}/stalwart + log_stream_name: mail + region: eu-central-1 + log_key: MESSAGE + + - name: cloudwatch_logs + match: cloudwatch.stalwart.api + log_group_name: /tb/${ENV}/stalwart + log_stream_name: api + region: eu-central-1 + log_key: MESSAGE + + - name: cloudwatch_logs + match: cloudwatch.untagged + log_group_name: /tb/${ENV}/stalwart + log_stream_name: untagged + region: eu-central-1 + log_key: MESSAGE \ No newline at end of file diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index cfff08c..e37e8b1 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -15,6 +15,7 @@ resources: log_streams: api: api mail: mail + untagged: untagged org_name: tb tb:network:MultiTierVpc: diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt index 6b3e795..72aabe9 100644 --- a/pulumi/requirements.txt +++ b/pulumi/requirements.txt @@ -1,4 +1,5 @@ Jinja2>=3.1,<4.0 pulumi_cloudflare==6.6.0 -tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@main +# tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@main +-e /home/rjung/workspace/thunderbird/pulumi toml>=0.10.2,<0.11 diff --git a/pulumi/stalwart/__init__.py b/pulumi/stalwart/__init__.py index 2c42bd1..c573db2 100644 --- a/pulumi/stalwart/__init__.py +++ b/pulumi/stalwart/__init__.py @@ -825,6 +825,7 @@ def user_data(self): archive_file_base = './bootstrap' archive_files = [ 'bootstrap.py', + 'templates/fluent-bit.service.j2', 'templates/fluent-bit.yaml.j2', 'templates/ports.j2', 'templates/stalwart.toml.j2', diff --git a/pulumi/stalwart_instance_user_data.sh.j2 b/pulumi/stalwart_instance_user_data.sh.j2 index 94885f4..fba71ed 100644 --- a/pulumi/stalwart_instance_user_data.sh.j2 +++ b/pulumi/stalwart_instance_user_data.sh.j2 @@ -28,9 +28,6 @@ dnf install -y bzip2 docker fluent-bit python3.12 # Delete the default fluent-bit config; we'll template a new one in Phase 2 rm -f /etc/fluent-bit/fluent-bit.conf -# Make sure fluent-bit uses the right config file -sed -i 's/fluent-bit.conf/fluent-bit.yaml/' /usr/lib/systemd/system/fluent-bit.service - # Set up Stalwart config directory and virtual environment mkdir -p $STALWART_DIR/etc python3.12 -m ensurepip From 04b5a4fc3a114f826e6d6bf5ed0c9964865e7047 Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Thu, 9 Apr 2026 14:07:53 -0600 Subject: [PATCH 07/12] Set up stage and prod with logging --- pulumi/config.prod.yaml | 13 +++++++++++++ pulumi/config.stage.yaml | 12 ++++++++++++ pulumi/requirements.txt | 3 +-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/pulumi/config.prod.yaml b/pulumi/config.prod.yaml index 614351d..57a447d 100644 --- a/pulumi/config.prod.yaml +++ b/pulumi/config.prod.yaml @@ -7,6 +7,16 @@ resources: - stalwart.postboot.keycloak_backend - stalwart.postboot.postgresql_backend + tb:cloudwatch:LogDestination: + stalwart: + log_group: + retention_in_days: 7 + log_streams: + api: api + mail: mail + untagged: untagged + org_name: tb + tb:network:MultiTierVpc: vpc: cidr_block: 10.0.0.0/16 @@ -93,6 +103,7 @@ resources: nodes: "0": # Must be a unique, stringified integer disable_api_termination: True + function: 'mail' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3a.large @@ -110,6 +121,7 @@ resources: storage_capacity: 20 "1": # Must be a unique, stringified integer disable_api_termination: True + function: 'mail' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3a.large @@ -127,6 +139,7 @@ resources: storage_capacity: 20 "50": disable_api_termination: True + function: 'api' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3.micro diff --git a/pulumi/config.stage.yaml b/pulumi/config.stage.yaml index 4cccea7..1019360 100644 --- a/pulumi/config.stage.yaml +++ b/pulumi/config.stage.yaml @@ -7,6 +7,16 @@ resources: - stalwart.postboot.postgresql_backend - stalwart.postboot.keycloak_backend + tb:cloudwatch:LogDestination: + stalwart: + log_group: + retention_in_days: 7 + log_streams: + api: api + mail: mail + untagged: untagged + org_name: tb + tb:network:MultiTierVpc: vpc: cidr_block: 10.1.0.0/16 @@ -108,6 +118,7 @@ resources: # subnet: subnet-07ade1ed35462907d # eu-central-1a "1": disable_api_termination: True + function: 'mail' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3a.large @@ -126,6 +137,7 @@ resources: subnet: subnet-07712b990eb0d17c0 # eu-central-1b "50": disable_api_termination: True + function: 'api' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3.micro diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt index 72aabe9..e066bcd 100644 --- a/pulumi/requirements.txt +++ b/pulumi/requirements.txt @@ -1,5 +1,4 @@ Jinja2>=3.1,<4.0 pulumi_cloudflare==6.6.0 -# tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@main --e /home/rjung/workspace/thunderbird/pulumi +tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@v0.0.18 toml>=0.10.2,<0.11 From 40b5734e73e84f67b15a8cc6ac964376a0fd88bd Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Thu, 9 Apr 2026 14:09:47 -0600 Subject: [PATCH 08/12] Remove 'untagged' refs --- pulumi/bootstrap/templates/fluent-bit.yaml.j2 | 8 +------- pulumi/config.dev.yaml | 1 - pulumi/config.prod.yaml | 1 - pulumi/config.stage.yaml | 1 - 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 index 73f4383..b5e7053 100644 --- a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 +++ b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 @@ -36,10 +36,4 @@ pipeline: log_stream_name: api region: eu-central-1 log_key: MESSAGE - - - name: cloudwatch_logs - match: cloudwatch.untagged - log_group_name: /tb/${ENV}/stalwart - log_stream_name: untagged - region: eu-central-1 - log_key: MESSAGE \ No newline at end of file + \ No newline at end of file diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index e37e8b1..cfff08c 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -15,7 +15,6 @@ resources: log_streams: api: api mail: mail - untagged: untagged org_name: tb tb:network:MultiTierVpc: diff --git a/pulumi/config.prod.yaml b/pulumi/config.prod.yaml index 57a447d..c3f30ba 100644 --- a/pulumi/config.prod.yaml +++ b/pulumi/config.prod.yaml @@ -14,7 +14,6 @@ resources: log_streams: api: api mail: mail - untagged: untagged org_name: tb tb:network:MultiTierVpc: diff --git a/pulumi/config.stage.yaml b/pulumi/config.stage.yaml index 1019360..5449369 100644 --- a/pulumi/config.stage.yaml +++ b/pulumi/config.stage.yaml @@ -14,7 +14,6 @@ resources: log_streams: api: api mail: mail - untagged: untagged org_name: tb tb:network:MultiTierVpc: From 411ac503060e2356e919a648ad4cc4a8cb49b9bc Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Thu, 9 Apr 2026 14:19:08 -0600 Subject: [PATCH 09/12] Commentary and line --- pulumi/bootstrap/templates/fluent-bit.yaml.j2 | 5 +++-- pulumi/stalwart/iam.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 index b5e7053..b2c467f 100644 --- a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 +++ b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 @@ -21,8 +21,9 @@ pipeline: filters: [] outputs: - # Send logs onward to CloudWatch. Log groups by the derived name must pre-exist, and this - # service must have sufficient IAM permissions to create log streams and post events to them. + # Send logs onward to CloudWatch. Log groups by the given name must pre-exist, and this service + # must have sufficient IAM permissions to post events to these log streams. If these log streams + # do not exist, this service must have permission to create them. - name: cloudwatch_logs match: cloudwatch.stalwart.mail log_group_name: /tb/${ENV}/stalwart diff --git a/pulumi/stalwart/iam.py b/pulumi/stalwart/iam.py index 963e51e..c1fb6e2 100644 --- a/pulumi/stalwart/iam.py +++ b/pulumi/stalwart/iam.py @@ -5,7 +5,7 @@ import json import pulumi_aws as aws -from tb_pulumi.constants import ASSUME_ROLE_POLICY, IAM_POLICY_DOCUMENT +from tb_pulumi.constants import ASSUME_ROLE_POLICY def iam( From ece6194b1d365efdf8c0bf37fd306eecc4afa318 Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Thu, 9 Apr 2026 14:33:56 -0600 Subject: [PATCH 10/12] Reduce log retention in prod --- pulumi/config.prod.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulumi/config.prod.yaml b/pulumi/config.prod.yaml index c3f30ba..3ebd940 100644 --- a/pulumi/config.prod.yaml +++ b/pulumi/config.prod.yaml @@ -10,7 +10,7 @@ resources: tb:cloudwatch:LogDestination: stalwart: log_group: - retention_in_days: 7 + retention_in_days: 3 log_streams: api: api mail: mail From 9429886e4d9d8b29fa4309245129e287b6354d0d Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Fri, 10 Apr 2026 07:53:12 -0600 Subject: [PATCH 11/12] Add journald cursor storage for fluent-bit --- pulumi/bootstrap/templates/fluent-bit.yaml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 index b2c467f..f6fde76 100644 --- a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 +++ b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 @@ -16,6 +16,7 @@ pipeline: inputs: - name: systemd tag: cloudwatch.stalwart.{{ function }} + db: /opt/fluent-bit/thundermail.cursor systemd_filter: _SYSTEMD_UNIT=thundermail.service filters: [] From 9e6b55037381833e5e439ad4580f90684739a3a4 Mon Sep 17 00:00:00 2001 From: Ryan Jung Date: Mon, 13 Apr 2026 10:21:25 -0600 Subject: [PATCH 12/12] Set log retention on journald as well --- pulumi/bootstrap/bootstrap.py | 1 + pulumi/bootstrap/templates/journald.conf.j2 | 6 ++++++ pulumi/stalwart/__init__.py | 1 + pulumi/stalwart_instance_user_data.sh.j2 | 1 + 4 files changed, 9 insertions(+) create mode 100644 pulumi/bootstrap/templates/journald.conf.j2 diff --git a/pulumi/bootstrap/bootstrap.py b/pulumi/bootstrap/bootstrap.py index 01dda5d..a405618 100644 --- a/pulumi/bootstrap/bootstrap.py +++ b/pulumi/bootstrap/bootstrap.py @@ -15,6 +15,7 @@ TEMPLATE_MAP = { 'fluent-bit.service.j2': '/usr/lib/systemd/system/fluent-bit.service', 'fluent-bit.yaml.j2': '/etc/fluent-bit/fluent-bit.yaml', + 'journald.conf.j2': '/etc/systemd/journald.conf', 'stalwart.toml.j2': '/opt/stalwart/etc/config.toml', 'thundermail.service.j2': '/usr/lib/systemd/system/thundermail.service', } diff --git a/pulumi/bootstrap/templates/journald.conf.j2 b/pulumi/bootstrap/templates/journald.conf.j2 new file mode 100644 index 0000000..44feee9 --- /dev/null +++ b/pulumi/bootstrap/templates/journald.conf.j2 @@ -0,0 +1,6 @@ +[Journal] +{% if env == 'prod' %} +MaxRetentionSec=3day +{% else %} +MaxRetentionSec=7day +{% endif %} \ No newline at end of file diff --git a/pulumi/stalwart/__init__.py b/pulumi/stalwart/__init__.py index c573db2..f5aab42 100644 --- a/pulumi/stalwart/__init__.py +++ b/pulumi/stalwart/__init__.py @@ -827,6 +827,7 @@ def user_data(self): 'bootstrap.py', 'templates/fluent-bit.service.j2', 'templates/fluent-bit.yaml.j2', + 'templates/journald.conf.j2', 'templates/ports.j2', 'templates/stalwart.toml.j2', 'templates/thundermail.service.j2', diff --git a/pulumi/stalwart_instance_user_data.sh.j2 b/pulumi/stalwart_instance_user_data.sh.j2 index fba71ed..b65b6d4 100644 --- a/pulumi/stalwart_instance_user_data.sh.j2 +++ b/pulumi/stalwart_instance_user_data.sh.j2 @@ -49,6 +49,7 @@ python $BOOTSTRAP_DIR/bootstrap.py # Ensure all our services are online with current configs systemctl daemon-reload +systemctl restart systemd-journald systemctl enable fluent-bit systemctl restart fluent-bit systemctl enable thundermail