docker-compose.yml

networks:
  ingestion-net:
  processing-net:
  serving-net:

services:

  kafka:
    image: confluentinc/cp-kafka:7.7.0
    hostname: kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:29092,PLAINTEXT_HOST://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      KAFKA_LOG_RETENTION_HOURS: 24
      CLUSTER_ID: "fraud-platform-cluster-01"
    networks: [ingestion-net]
    healthcheck:
      test: ["CMD", "kafka-broker-api-versions", "--bootstrap-server", "localhost:29092"]
      interval: 15s
      timeout: 10s
      retries: 10
      start_period: 30s
    restart: unless-stopped

  schema-registry:
    image: confluentinc/cp-schema-registry:7.7.0
    hostname: schema-registry
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:29092
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    networks: [ingestion-net]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 20s
    restart: unless-stopped

  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    depends_on:
      kafka:
        condition: service_healthy
      schema-registry:
        condition: service_healthy
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: fraud-platform
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092
      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry:8081
    networks: [ingestion-net]
    restart: unless-stopped

  topic-init:
    image: confluentinc/cp-kafka:7.7.0
    depends_on:
      kafka:
        condition: service_healthy
    volumes:
      - ./phase1-ingestion/init_topics.sh:/init_topics.sh
    command: ["bash", "/init_topics.sh"]
    networks: [ingestion-net]

  transaction-producer:
    build: ./phase1-ingestion/producer
    depends_on:
      schema-registry:
        condition: service_healthy
      topic-init:
        condition: service_completed_successfully
    env_file: .env
    networks: [ingestion-net]
    restart: unless-stopped

  # ── Phase 2: Lakehouse ────────────────────────────────────

  minio:
    image: minio/minio:latest
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
    command: server /data --console-address ":9001"
    volumes:
      - minio-data:/data
    networks: [processing-net]
    healthcheck:
      test: ["CMD", "mc", "ready", "local"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s
    restart: unless-stopped

  minio-init:
    image: minio/mc:latest
    depends_on:
      minio:
        condition: service_healthy
    env_file: .env
    volumes:
      - ./infra/minio/init_buckets.sh:/init_buckets.sh
    entrypoint: ["/bin/sh", "/init_buckets.sh"]
    networks: [processing-net]

  spark-master:
    image: apache/spark:3.5.0
    command: ["/opt/spark/bin/spark-class", "org.apache.spark.deploy.master.Master"]
    ports:
      - "7077:7077"
      - "8090:8080"
    environment:
      SPARK_MASTER_PORT: 7077
      SPARK_MASTER_WEBUI_PORT: 8080
    networks: [processing-net]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080"]
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 20s
    restart: unless-stopped

  spark-worker:
    image: apache/spark:3.5.0
    command:
      - "/opt/spark/bin/spark-class"
      - "org.apache.spark.deploy.worker.Worker"
      - "spark://spark-master:7077"
    depends_on:
      spark-master:
        condition: service_healthy
    environment:
      SPARK_WORKER_MEMORY: 2g
      SPARK_WORKER_CORES: "2"
    networks: [processing-net]
    restart: unless-stopped

  spark-streaming:
    build: ./phase2-streaming
    depends_on:
      spark-master:
        condition: service_healthy
      minio-init:
        condition: service_completed_successfully
      schema-registry:
        condition: service_healthy
    env_file: .env
    networks: [ingestion-net, processing-net]
    restart: unless-stopped

  # ── Phase 3: Analytics Layer ──────────────────────────────

  postgres:
    image: postgres:15
    hostname: postgres
    ports:
      - "5432:5432"
    environment:
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_DB: ${POSTGRES_DB}
    volumes:
      - postgres-data:/var/lib/postgresql/data
    networks: [serving-net]
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 15s
    restart: unless-stopped

  airflow:
    build: ./phase3-transforms/airflow
    depends_on:
      postgres:
        condition: service_healthy
    ports:
      - "8082:8080"
      - "8099:8099"
    environment:
      AIRFLOW__CORE__EXECUTOR: LocalExecutor
      AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}
      AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY}
      AIRFLOW__CORE__LOAD_EXAMPLES: "false"
      AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW_FERNET_KEY}
      AIRFLOW_ADMIN_USER: ${AIRFLOW_ADMIN_USER}
      AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD}
    volumes:
      - ./phase3-transforms/airflow/dags:/opt/airflow/dags
      - airflow-logs:/opt/airflow/logs
      - ./phase3-transforms/dbt_project:/opt/dbt
      - ./phase4-ml/serving:/opt/ml
      - ./phase4-ml/features.py:/opt/ml/features.py
    networks: [serving-net, processing-net]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s
    restart: unless-stopped

  trino:
    image: trinodb/trino:435
    hostname: trino
    ports:
      - "8085:8080"
    volumes:
      - ./infra/trino/catalog:/etc/trino/catalog
      - ./infra/trino/jvm.config:/etc/trino/jvm.config
      - ./infra/trino/config.properties:/etc/trino/config.properties
    networks: [processing-net, serving-net]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/v1/info"]
      interval: 15s
      timeout: 5s
      retries: 10
      start_period: 60s
    restart: unless-stopped

  trino-init:
    image: trinodb/trino:435
    depends_on:
      trino:
        condition: service_healthy
    volumes:
      - ./infra/trino/init.sql:/init.sql
    entrypoint: ["/bin/sh", "-c"]
    command:
      - |
        trino --server http://trino:8080 --file /init.sql || true
    networks: [processing-net]

  dbt-runner:
    build: ./phase3-transforms/dbt_project
    depends_on:
      trino:
        condition: service_healthy
      minio-init:
        condition: service_completed_successfully
    env_file: .env
    volumes:
      - ./phase3-transforms/dbt_project:/opt/dbt
    networks: [processing-net, serving-net]

  # ── Phase 4: ML Deployment ────────────────────────────────

  mlflow:
    image: ghcr.io/mlflow/mlflow:v2.10.0
    ports:
      - "5000:5000"
    environment:
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
      MLFLOW_S3_ENDPOINT_URL: ${MINIO_ENDPOINT}
    command: >
      mlflow server
      --host 0.0.0.0
      --port 5000
      --backend-store-uri sqlite:////mlflow/mlflow.db
      --default-artifact-root s3://fraud-platform/mlflow
    volumes:
      - mlflow-data:/mlflow
    depends_on:
      minio-init:
        condition: service_completed_successfully
    networks: [serving-net, processing-net]
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:5000/health')"]
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 30s
    restart: unless-stopped

  redis:
    image: redis:7-alpine
    ports:
      - "6379:6379"
    networks: [serving-net]
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 5s
      retries: 5
    restart: unless-stopped

  ml-training:
    build:
      context: ./phase4-ml
      dockerfile: training/Dockerfile
    depends_on:
      mlflow:
        condition: service_healthy
      trino:
        condition: service_healthy
    env_file: .env
    environment:
      TRINO_HOST: trino
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
      MLFLOW_S3_ENDPOINT_URL: ${MINIO_ENDPOINT}
    networks: [serving-net, processing-net]
    profiles: ["training"]

  fraud-api:
    build:
      context: ./phase4-ml
      dockerfile: serving/Dockerfile
    ports:
      - "8000:8000"
    depends_on:
      mlflow:
        condition: service_healthy
      redis:
        condition: service_healthy
    env_file: .env
    environment:
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
      MLFLOW_S3_ENDPOINT_URL: ${MINIO_ENDPOINT}
    networks: [serving-net, processing-net]
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 30s
    restart: unless-stopped

  # ── Phase 5: Analytics & Hardening ───────────────────────

  kafka-exporter:
    image: danielqsj/kafka-exporter:latest
    command: ["--kafka.server=kafka:29092"]
    depends_on:
      kafka:
        condition: service_healthy
    networks: [ingestion-net, serving-net]
    restart: unless-stopped

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus-data:/prometheus
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.path=/prometheus"
      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
      - "--web.console.templates=/usr/share/prometheus/consoles"
    networks: [serving-net]
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:9090/-/healthy"]
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 15s
    restart: unless-stopped

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin123}
      GF_USERS_ALLOW_SIGN_UP: "false"
    volumes:
      - grafana-data:/var/lib/grafana
      - ./phase5-analytics/grafana/provisioning:/etc/grafana/provisioning
      - ./phase5-analytics/grafana/dashboards:/var/lib/grafana/dashboards
    depends_on:
      prometheus:
        condition: service_healthy
    networks: [serving-net]
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:3000/api/health"]
      interval: 15s
      timeout: 5s
      retries: 5
      start_period: 30s
    restart: unless-stopped

  superset:
    image: apache/superset:latest
    ports:
      - "8088:8088"
    environment:
      SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY:-superset-secret-change-in-prod}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_HOST: postgres
      POSTGRES_DB: ${POSTGRES_DB}
      SUPERSET_ADMIN_USER: ${SUPERSET_ADMIN_USER:-admin}
      SUPERSET_ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD:-admin123}
    volumes:
      - ./phase5-analytics/superset/superset_config.py:/app/pythonpath/superset_config.py
      - ./phase5-analytics/superset/init_superset.sh:/init_superset.sh
      - superset-data:/app/superset_home
    entrypoint: ["/bin/bash", "/init_superset.sh"]
    depends_on:
      postgres:
        condition: service_healthy
    networks: [serving-net, processing-net]
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost:8088/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 90s
    restart: unless-stopped

volumes:
  minio-data:
  postgres-data:
  airflow-logs:
  mlflow-data:
  prometheus-data:
  grafana-data:
  superset-data: