diff --git a/pulumi/__main__.py b/pulumi/__main__.py index 426e7b6..86e407c 100644 --- a/pulumi/__main__.py +++ b/pulumi/__main__.py @@ -25,6 +25,17 @@ **psm_opts, ) +logdest_opts = resources.get('tb:cloudwatch:LogDestination', {}) +logdests = { + logdest_name: tb_pulumi.cloudwatch.LogDestination( + f'{project.name_prefix}-logdest-{logdest_name}', + app_name=logdest_name, + project=project, + **logdest_config, + ) + for logdest_name, logdest_config in logdest_opts.items() +} + # Build out some private network space vpc_opts = resources['tb:network:MultiTierVpc']['vpc'] vpc = tb_pulumi.network.MultiTierVpc( @@ -71,6 +82,7 @@ def __stalwart_cluster(jumphost_rules: list[dict]): return stalwart.StalwartCluster( f'{project.name_prefix}-stalwart', project=project, + log_group_arn=logdests['stalwart'].resources['iam_policies']['write'].arn, private_subnets=vpc.resources['private_subnets'], public_subnets=vpc.resources['public_subnets'], node_additional_ingress_rules=jumphost_rules, diff --git a/pulumi/bootstrap/bootstrap.py b/pulumi/bootstrap/bootstrap.py index 8a67f18..a405618 100644 --- a/pulumi/bootstrap/bootstrap.py +++ b/pulumi/bootstrap/bootstrap.py @@ -10,13 +10,19 @@ BOOTSTRAP_DIR = '/opt/stalwart-bootstrap' BOOTSTRAP_LOG = '/var/log/stalwart-bootstrap.log' INSTANCE_TAGS = {} + # Map of template files to target files TEMPLATE_MAP = { + 'fluent-bit.service.j2': '/usr/lib/systemd/system/fluent-bit.service', + 'fluent-bit.yaml.j2': '/etc/fluent-bit/fluent-bit.yaml', + 'journald.conf.j2': '/etc/systemd/journald.conf', 'stalwart.toml.j2': '/opt/stalwart/etc/config.toml', 'thundermail.service.j2': '/usr/lib/systemd/system/thundermail.service', } # Map of template variable to EC2 tags TEMPLATE_VALUE_TAG_MAP = { + 'env': 'environment', + 'function': 'postboot.stalwart.function', 'https_paths': 'postboot.stalwart.https_paths', 'node_services': 'postboot.stalwart.node_services', 'node_id': 'postboot.stalwart.node_id', diff --git a/pulumi/bootstrap/templates/fluent-bit.service.j2 b/pulumi/bootstrap/templates/fluent-bit.service.j2 new file mode 100644 index 0000000..13a7957 --- /dev/null +++ b/pulumi/bootstrap/templates/fluent-bit.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Fluent Bit +Documentation=https://docs.fluentbit.io/manual/ +Requires=network.target +After=network.target + +[Service] +Type=simple +Environment="ENV={{ env }}" +ExecStart=/opt/fluent-bit/bin/fluent-bit -c /etc/fluent-bit/fluent-bit.yaml +Restart=always + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pulumi/bootstrap/templates/fluent-bit.yaml.j2 b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 new file mode 100644 index 0000000..f6fde76 --- /dev/null +++ b/pulumi/bootstrap/templates/fluent-bit.yaml.j2 @@ -0,0 +1,41 @@ +--- + +service: + flush: 1 + grace: 5 + daemon: no + dns.mode: UDP + hot_reload: on + log_level: info + storage.path: /fluent-bit/buffers + storage.backlog.flush_on_shutdown: on + storage.keep.rejected: on + storage.rejected.path: /fluent-bit/dlq + +pipeline: + inputs: + - name: systemd + tag: cloudwatch.stalwart.{{ function }} + db: /opt/fluent-bit/thundermail.cursor + systemd_filter: _SYSTEMD_UNIT=thundermail.service + + filters: [] + + outputs: + # Send logs onward to CloudWatch. Log groups by the given name must pre-exist, and this service + # must have sufficient IAM permissions to post events to these log streams. If these log streams + # do not exist, this service must have permission to create them. + - name: cloudwatch_logs + match: cloudwatch.stalwart.mail + log_group_name: /tb/${ENV}/stalwart + log_stream_name: mail + region: eu-central-1 + log_key: MESSAGE + + - name: cloudwatch_logs + match: cloudwatch.stalwart.api + log_group_name: /tb/${ENV}/stalwart + log_stream_name: api + region: eu-central-1 + log_key: MESSAGE + \ No newline at end of file diff --git a/pulumi/bootstrap/templates/journald.conf.j2 b/pulumi/bootstrap/templates/journald.conf.j2 new file mode 100644 index 0000000..44feee9 --- /dev/null +++ b/pulumi/bootstrap/templates/journald.conf.j2 @@ -0,0 +1,6 @@ +[Journal] +{% if env == 'prod' %} +MaxRetentionSec=3day +{% else %} +MaxRetentionSec=7day +{% endif %} \ No newline at end of file diff --git a/pulumi/config.dev.yaml b/pulumi/config.dev.yaml index 6bfad6d..cfff08c 100644 --- a/pulumi/config.dev.yaml +++ b/pulumi/config.dev.yaml @@ -8,6 +8,15 @@ resources: - stalwart.postboot.keycloak_backend recovery_window_in_days: 0 + tb:cloudwatch:LogDestination: + stalwart: + log_group: + retention_in_days: 7 + log_streams: + api: api + mail: mail + org_name: tb + tb:network:MultiTierVpc: vpc: cidr_block: 10.2.0.0/16 @@ -43,18 +52,18 @@ resources: additional_routes: private: - destination_cidr_block: 10.202.0.0/22 # observability-dev - vpc_peering_connection_id: pcx-0d2027442f0e54ca4 + vpc_peering_connection_id: pcx-04d7e54008cd9326c public: [] endpoint_interfaces: - secretsmanager - # tb:ec2:SshableInstance: {} + tb:ec2:SshableInstance: {} # Fill out this template to build an SSH bastion - tb:ec2:SshableInstance: - bastion: - ssh_keypair_name: mailstrom-dev - source_cidrs: - - 10.2.0.0/16 # Internal access + # tb:ec2:SshableInstance: + # bastion: + # ssh_keypair_name: mailstrom-dev + # source_cidrs: + # - 10.2.0.0/16 # Internal access tb:mailstrom:StalwartCluster: thundermail: @@ -81,6 +90,7 @@ resources: nodes: "0": # Must be a unique, stringified integer disable_api_termination: True + function: 'mail' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3.micro @@ -99,6 +109,7 @@ resources: storage_capacity: 20 "50": disable_api_termination: True + function: 'api' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3.micro diff --git a/pulumi/config.prod.yaml b/pulumi/config.prod.yaml index 614351d..3ebd940 100644 --- a/pulumi/config.prod.yaml +++ b/pulumi/config.prod.yaml @@ -7,6 +7,15 @@ resources: - stalwart.postboot.keycloak_backend - stalwart.postboot.postgresql_backend + tb:cloudwatch:LogDestination: + stalwart: + log_group: + retention_in_days: 3 + log_streams: + api: api + mail: mail + org_name: tb + tb:network:MultiTierVpc: vpc: cidr_block: 10.0.0.0/16 @@ -93,6 +102,7 @@ resources: nodes: "0": # Must be a unique, stringified integer disable_api_termination: True + function: 'mail' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3a.large @@ -110,6 +120,7 @@ resources: storage_capacity: 20 "1": # Must be a unique, stringified integer disable_api_termination: True + function: 'mail' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3a.large @@ -127,6 +138,7 @@ resources: storage_capacity: 20 "50": disable_api_termination: True + function: 'api' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3.micro diff --git a/pulumi/config.stage.yaml b/pulumi/config.stage.yaml index 4cccea7..5449369 100644 --- a/pulumi/config.stage.yaml +++ b/pulumi/config.stage.yaml @@ -7,6 +7,15 @@ resources: - stalwart.postboot.postgresql_backend - stalwart.postboot.keycloak_backend + tb:cloudwatch:LogDestination: + stalwart: + log_group: + retention_in_days: 7 + log_streams: + api: api + mail: mail + org_name: tb + tb:network:MultiTierVpc: vpc: cidr_block: 10.1.0.0/16 @@ -108,6 +117,7 @@ resources: # subnet: subnet-07ade1ed35462907d # eu-central-1a "1": disable_api_termination: True + function: 'mail' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3a.large @@ -126,6 +136,7 @@ resources: subnet: subnet-07712b990eb0d17c0 # eu-central-1b "50": disable_api_termination: True + function: 'api' ignore_ami_changes: True ignore_user_data_changes: True instance_type: t3.micro diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt index 6606c2b..e066bcd 100644 --- a/pulumi/requirements.txt +++ b/pulumi/requirements.txt @@ -1,4 +1,4 @@ Jinja2>=3.1,<4.0 pulumi_cloudflare==6.6.0 -tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@v0.0.16 +tb_pulumi @ git+https://github.com/thunderbird/pulumi.git@v0.0.18 toml>=0.10.2,<0.11 diff --git a/pulumi/stalwart/__init__.py b/pulumi/stalwart/__init__.py index 119d3ce..f5aab42 100644 --- a/pulumi/stalwart/__init__.py +++ b/pulumi/stalwart/__init__.py @@ -274,6 +274,7 @@ def __init__( self, name: str, project: tb_pulumi.ThunderbirdPulumiProject, + log_group_arn: str, private_subnets: list[aws.ec2.Subnet], public_subnets: list[aws.ec2.Subnet], https_features: list = [], @@ -343,8 +344,16 @@ def __init__( s3_bucket, s3_secret, s3_policy = stalwart_s3.s3(self=self) # Build an IAM role with a policy to enable node bootstrapping - profile_policy, role, profile_postboot_attachment, profile_s3_attachment, profile = stalwart_iam.iam( + ( + profile_policy, + role, + profile_postboot_attachment, + profile_s3_attachment, + profile_logwrite_attachment, + profile, + ) = stalwart_iam.iam( self, + log_group_arn=log_group_arn, s3_policy=s3_policy, ) @@ -463,6 +472,7 @@ def __init__( 'spam_filter_secret': config_secrets['spam_filter'], 'node_profile': profile, 'node_profile_policy': profile_policy, + 'node_profile_logwrite_attachment': profile_logwrite_attachment, 'node_profile_postboot_policy_attachment': profile_postboot_attachment, 'node_profile_s3_policy_attachment': profile_s3_attachment, 'node_sgs': self.node_sgs, @@ -671,6 +681,7 @@ def node( depends_on: list = [], disable_api_stop: bool = False, disable_api_termination: bool = False, + function: str = 'unknown', ignore_ami_changes: bool = True, ignore_user_data_changes: bool = True, instance_type: str = 't3.micro', @@ -694,6 +705,9 @@ def node( False. :type disable_api_termination: bool, optional + :param function: This becomes the ``postboot.stalwart.function`` tag on the instance and the ``function`` + variable inside of postboot templates. + :param ignore_ami_changes: When True, changes to the instance's AMI will not be applied. This prevents unwanted rebuilding of cluster nodes, potentially causing downtime. Set to False if the AMI has changed and you intend on rebuilding the node. Defaults to True. @@ -749,6 +763,7 @@ def node( postboot_tags = { 'postboot.stalwart.aws_region': self.project.aws_region, 'postboot.stalwart.env': self.project.stack, + 'postboot.stalwart.function': function, 'postboot.stalwart.https_paths': ','.join(https_paths), 'postboot.stalwart.image': self.stalwart_image, 'postboot.stalwart.node_services': node_services_tag, @@ -810,6 +825,9 @@ def user_data(self): archive_file_base = './bootstrap' archive_files = [ 'bootstrap.py', + 'templates/fluent-bit.service.j2', + 'templates/fluent-bit.yaml.j2', + 'templates/journald.conf.j2', 'templates/ports.j2', 'templates/stalwart.toml.j2', 'templates/thundermail.service.j2', diff --git a/pulumi/stalwart/iam.py b/pulumi/stalwart/iam.py index 92d598b..c1fb6e2 100644 --- a/pulumi/stalwart/iam.py +++ b/pulumi/stalwart/iam.py @@ -5,11 +5,12 @@ import json import pulumi_aws as aws -from tb_pulumi.constants import ASSUME_ROLE_POLICY, IAM_POLICY_DOCUMENT +from tb_pulumi.constants import ASSUME_ROLE_POLICY def iam( self, + log_group_arn: str, s3_policy: aws.iam.Policy, ) -> tuple[ aws.iam.Policy, aws.iam.Role, aws.iam.RolePolicyAttachment, aws.iam.RolePolicyAttachment, aws.iam.InstanceProfile @@ -32,14 +33,18 @@ def iam( + f':secret:mailstrom/{self.project.stack}/stalwart.postboot.*' ), ] - profile_postboot_policy_doc = IAM_POLICY_DOCUMENT.copy() - profile_postboot_policy_doc['Statement'][0].update( - { - 'Sid': 'AllowPostbootSecretAccess', - 'Action': ['secretsmanager:GetSecretValue'], - 'Resource': bootstrap_secret_arns, - } - ) + profile_postboot_policy_doc = { + 'Version': '2012-10-17', + 'Statement': [ + { + 'Sid': 'AllowPostbootSecretAccess', + 'Effect': 'Allow', + 'Action': ['secretsmanager:GetSecretValue'], + 'Resource': bootstrap_secret_arns, + } + ], + } + profile_policy = aws.iam.Policy( f'{self.name}-policy-nodeprofile', path='/', @@ -64,7 +69,19 @@ def iam( role=role.name, policy_arn=s3_policy.arn, ) + profile_logwrite_attachment = aws.iam.RolePolicyAttachment( + f'{self.name}-rpa-nodeprofile-logs', + role=role.name, + policy_arn=log_group_arn, + ) profile = aws.iam.InstanceProfile(f'{self.name}-ip-nodeprofile', name=f'{self.name}-nodeprofile', role=role.name) - return profile_policy, role, profile_postboot_attachment, profile_s3_attachment, profile + return ( + profile_policy, + role, + profile_postboot_attachment, + profile_s3_attachment, + profile_logwrite_attachment, + profile, + ) diff --git a/pulumi/stalwart_instance_user_data.sh.j2 b/pulumi/stalwart_instance_user_data.sh.j2 index d572ec4..b65b6d4 100644 --- a/pulumi/stalwart_instance_user_data.sh.j2 +++ b/pulumi/stalwart_instance_user_data.sh.j2 @@ -8,30 +8,49 @@ set -x set -e +# Places data get stored BOOTSTRAP_DIR=/opt/stalwart-bootstrap BOOTSTRAP_TBZ=/root/bootstrap.tbz STALWART_DIR=/opt/stalwart +# Install the fluent-bit repo for Amazon Linux +echo '[fluent-bit] +name = Fluent Bit +baseurl = https://packages.fluentbit.io/amazonlinux/2023/ +gpgcheck=1 +gpgkey=https://packages.fluentbit.io/fluentbit.key +enabled=1' > /etc/yum.repos.d/fluent-bit.repo + +# Update system, install dependencies dnf update -y -dnf install -y bzip2 docker python3.12 +dnf install -y bzip2 docker fluent-bit python3.12 -mkdir -p $STALWART_DIR/etc +# Delete the default fluent-bit config; we'll template a new one in Phase 2 +rm -f /etc/fluent-bit/fluent-bit.conf +# Set up Stalwart config directory and virtual environment +mkdir -p $STALWART_DIR/etc python3.12 -m ensurepip pip3.12 install virtualenv /usr/local/bin/virtualenv -p python3.12 $BOOTSTRAP_DIR +# Make sure Docker always runs systemctl start docker systemctl enable docker +# Install Bootstrap Phase 2 echo '{{ bootstrap_tbz_base64 }}' | base64 -d > $BOOTSTRAP_TBZ tar -xvf $BOOTSTRAP_TBZ -C $BOOTSTRAP_DIR +# Run Bootstrap Phase 2 source $BOOTSTRAP_DIR/bin/activate pip install -r $BOOTSTRAP_DIR/requirements.txt python $BOOTSTRAP_DIR/bootstrap.py +# Ensure all our services are online with current configs systemctl daemon-reload +systemctl restart systemd-journald +systemctl enable fluent-bit +systemctl restart fluent-bit systemctl enable thundermail -systemctl start thundermail - +systemctl restart thundermail