diff --git a/morph-node/.env_mpt b/morph-node/.env_mpt new file mode 100644 index 0000000..1defda6 --- /dev/null +++ b/morph-node/.env_mpt @@ -0,0 +1,7 @@ +# MPT specific overrides (loaded after base env to override values) +GETH_ENTRYPOINT_FILE=./entrypoint-geth-mpt.sh +MPT_FORK_TIME=2000000000000 + +# MPT snapshot names +HOODI_MPT_SNAPSHOT_NAME=snapshot-20260211-1 +MAINNET_MPT_SNAPSHOT_NAME=snapshot-20260211-1 diff --git a/morph-node/Makefile b/morph-node/Makefile index b38471c..2aef9ad 100644 --- a/morph-node/Makefile +++ b/morph-node/Makefile @@ -12,6 +12,8 @@ JWT_SECRET_FILE_HOLESKY := $(JWT_SECRET_FILE) JWT_SECRET_FILE_HOODI := $(JWT_SECRET_FILE) +include .env_mpt + generate-jwt: @[ -f $(JWT_SECRET_FILE_MAINNET) ] || (echo "Generating $(JWT_SECRET_FILE_MAINNET)..." && openssl rand -hex 32 > $(JWT_SECRET_FILE_MAINNET) && echo "$(JWT_SECRET_FILE_MAINNET) created.") @@ -31,6 +33,12 @@ run-holesky-node: generate-jwt-holesky run-hoodi-node: generate-jwt-hoodi docker-compose --env-file .env_hoodi up node & +run-hoodi-mpt-node: generate-jwt-hoodi + docker-compose --env-file .env_hoodi --env-file .env_mpt up node & + +run-mainnet-mpt-node: generate-jwt + docker-compose --env-file .env --env-file .env_mpt up node & + stop-node: docker stop morph-node morph-geth @@ -47,6 +55,12 @@ run-holesky-validator: generate-jwt-holesky run-hoodi-validator: generate-jwt-hoodi docker-compose --env-file .env_hoodi up validator & +run-hoodi-mpt-validator: generate-jwt-hoodi + docker-compose --env-file .env_hoodi --env-file .env_mpt up validator & + +run-mainnet-mpt-validator: generate-jwt + docker-compose --env-file .env --env-file .env_mpt up validator & + stop-validator: docker stop validator-node morph-geth @@ -93,6 +107,11 @@ download-and-decompress-hoodi-snapshot: download-and-decompress-mainnet-snapshot: $(call download-and-decompress,$(MAINNET_SNAPSHOT_NAME),https://snapshot.morphl2.io/mainnet) +download-and-decompress-hoodi-mpt-snapshot: + $(call download-and-decompress,$(HOODI_MPT_SNAPSHOT_NAME),https://snapshot.morphl2.io/hoodi) + +download-and-decompress-mainnet-mpt-snapshot: + $(call download-and-decompress,$(MAINNET_MPT_SNAPSHOT_NAME),https://snapshot.morphl2.io/mainnet) diff --git a/morph-node/docker-compose.yml b/morph-node/docker-compose.yml index 72027f8..3225d7f 100644 --- a/morph-node/docker-compose.yml +++ b/morph-node/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.8' services: geth: container_name: morph-geth - image: ghcr.io/morph-l2/go-ethereum:2.1.1 + image: ghcr.io/morph-l2/go-ethereum:2.1.2 restart: unless-stopped ports: - "8545:8545" @@ -26,7 +26,7 @@ services: depends_on: geth: condition: service_started - image: ghcr.io/morph-l2/node:0.4.10 + image: ghcr.io/morph-l2/node:0.4.11 restart: unless-stopped ports: - "26656" @@ -53,7 +53,7 @@ services: depends_on: geth: condition: service_started - image: ghcr.io/morph-l2/node:0.4.10 + image: ghcr.io/morph-l2/node:0.4.11 ports: - "26660" environment: diff --git a/morph-node/entrypoint-geth-mpt.sh b/morph-node/entrypoint-geth-mpt.sh new file mode 100644 index 0000000..ecaf768 --- /dev/null +++ b/morph-node/entrypoint-geth-mpt.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +if [ ! -f /jwt-secret.txt ]; then + echo "Error: jwt-secret.txt not found. Please create it before starting the service." + exit 1 +fi + +MORPH_FLAG=${MORPH_FLAG:-"morph"} + +COMMAND="geth \ +--$MORPH_FLAG \ +--morph-mpt +--datadir="./db" \ +--verbosity=3 \ +--http \ +--http.corsdomain="*" \ +--http.vhosts="*" \ +--http.addr=0.0.0.0 \ +--http.port=8545 \ +--http.api=web3,debug,eth,txpool,net,morph,engine,admin \ +--ws \ +--ws.addr=0.0.0.0 \ +--ws.port=8546 \ +--ws.origins="*" \ +--ws.api=web3,debug,eth,txpool,net,morph,engine,admin \ +--authrpc.addr=0.0.0.0 \ +--authrpc.port=8551 \ +--authrpc.vhosts="*" \ +--authrpc.jwtsecret="./jwt-secret.txt" \ +--gcmode=archive \ +--log.filename=./db/geth.log \ +--metrics \ +--metrics.addr=0.0.0.0" + +eval $COMMAND diff --git a/ops/snapshot/README.md b/ops/snapshot/README.md new file mode 100644 index 0000000..32737a2 --- /dev/null +++ b/ops/snapshot/README.md @@ -0,0 +1,151 @@ +# Snapshot Automation + +> 中文版请见 [README.zh.md](./README.zh.md) + +Automatically creates a node snapshot every two weeks and syncs the relevant parameters to README.md for users to download. + +## Background + +Manually creating snapshots is error-prone and tedious. This solution automates the entire process using a server-side cron job and the GitHub REST API — no GitHub Actions or git CLI required. + +## Directory Structure + +``` +run-morph-node/ +├── README.md # snapshot table is updated here +└── ops/snapshot/ + ├── README.md # this document + ├── README.zh.md # Chinese version + ├── snapshot_make.py # entry point: stop → snapshot → upload → restart → update README + ├── update_metadata.py # fetches indexer API data and orchestrates the full update flow + ├── update_readme.py # pure table-update logic (imported by update_metadata.py) + └── metrics_server.py # persistent HTTP server exposing metrics on :6060/metrics +``` + +## Workflow + +``` +Server cron job (1st and 15th of each month) + │ + ▼ + ops/snapshot/snapshot_make.py + [1] stop morph-node, morph-geth + [2] create snapshot (tar geth + node data) + [3] upload to S3 + [4] restart morph-geth → wait for RPC → collect base_height + [5] restart morph-node + [6] call update_metadata.py + │ BASE_HEIGHT, SNAPSHOT_NAME + ▼ + python3 update_metadata.py + ┌─────────────────────────────────────────────────────┐ + │ 1. call internal explorer-indexer API: │ + │ GET /v1/batch/l1_msg_start_height/ │ + │ GET /v1/batch/derivation_start_height/│ + │ 2. fetch README.md content via GitHub API │ + │ 3. insert new snapshot row at top of table │ + │ 4. create branch + push updated file via GitHub API │ + │ 5. open PR via GitHub API │ + └─────────────────────────────────────────────────────┘ +``` + +## Triggers + +| Method | Description | +|---|---| +| Scheduled | Server cron job on the 1st and 15th of each month | +| Manual | SSH into the server and run `snapshot_make.py` directly | + +## Multi-environment Support + +| Environment | Indexer API (internal) | +|---|---| +| mainnet | `explorer-indexer.morphl2.io` | +| hoodi | `explorer-indexer-hoodi.morphl2.io` | +| holesky | `explorer-indexer-holesky.morphl2.io` | + +Each environment has its own node server with its own cron job. S3 paths and README table sections are automatically scoped by environment. + +## Deployment + +### 1. Clone the Repository on the Node Server + +```bash +git clone https://github.com/morphl2/run-morph-node.git /data/run-morph-node +``` + +### 2. Create the Environment File + +Copy the template into the same directory and fill in the values: + +```bash +cd /data/run-morph-node/ops/snapshot +cp snapshot.env.example snapshot.env +# edit snapshot.env and fill in GH_TOKEN, S3_BUCKET, ENVIRONMENT, etc. +``` + +For multiple environments or snapshot types, use separate files: + +```bash +cp snapshot.env.example snapshot-hoodi.env +cp snapshot.env.example snapshot-mainnet-mpt.env +``` + +All available variables are documented in [`snapshot.env.example`](./snapshot.env.example). These files must **not** be committed to git (add `*.env` to `.gitignore`). + +Also recommended: enable **"Automatically delete head branches"** under repo Settings → General. Branches will be deleted automatically after a PR is merged. + +### 3. Configure the Cron Job + +Add one entry per environment / snapshot type: + +```bash +crontab -e +``` + +```cron +REPO=/data/run-morph-node/ops/snapshot + +# mainnet standard snapshot (uses default snapshot.env) +0 2 1,15 * * python3 $REPO/snapshot_make.py >> /var/log/snapshot-mainnet.log 2>&1 + +# mainnet mpt-snapshot +0 3 1,15 * * ENV_FILE=$REPO/snapshot-mainnet-mpt.env python3 $REPO/snapshot_make.py >> /var/log/snapshot-mainnet-mpt.log 2>&1 + +# hoodi +0 2 1,15 * * ENV_FILE=$REPO/snapshot-hoodi.env python3 $REPO/snapshot_make.py >> /var/log/snapshot-hoodi.log 2>&1 +``` + +### 4. Start the Metrics Server + +Run `metrics_server.py` as a persistent pm2 process so it survives server reboots: + +```bash +pm2 startup # register pm2 itself as a system startup service (run once) +pm2 start python3 --name morph-snapshot-metrics -- /data/run-morph-node/ops/snapshot/metrics_server.py +pm2 save +``` + +Once running, the metrics endpoint is available at `http://:6060/metrics`. + +Exposed metrics: + +| Metric | Type | Description | +|---|---|---| +| `morph_snapshot_readme_update_status` | gauge | 1 = success, 0 = failure | +| `morph_snapshot_readme_update_timestamp_seconds` | gauge | Unix timestamp of the last run | + +Labels: `environment` (mainnet / hoodi / holesky), `snapshot` (snapshot name) + +> Default metrics file path: `/tmp/morph_snapshot_metrics.prom` +> Override via the `METRICS_FILE` environment variable — applies to both `update_readme.py` and `metrics_server.py`. + +## Key Design Decisions + +- **`base_height` is collected after geth restarts**: querying the RPC after the snapshot is created and geth is started alone gives the actual block state of the snapshot, which is more accurate than querying before the stop. `morph-node` is started only after the height is confirmed. +- **Fallback recovery on failure**: if the snapshot or upload fails, a fallback step in `snapshot_make.py` attempts to restart both processes to avoid prolonged service interruption. +- **No GitHub Actions or git CLI required**: `update_metadata.py` uses the GitHub REST API directly — the server only needs Python 3. The `GH_TOKEN` is the only credential needed. +- **New entries are inserted at the top of the table**: the latest snapshot always appears in the first row for quick access. +- **Changes are merged via PR, not direct push**: a new branch is created and a PR is opened, preserving review opportunity and preventing automated scripts from writing directly to the main branch. + + diff --git a/ops/snapshot/README.zh.md b/ops/snapshot/README.zh.md new file mode 100644 index 0000000..604a938 --- /dev/null +++ b/ops/snapshot/README.zh.md @@ -0,0 +1,151 @@ +# Snapshot 自动化 + +> English version: [README.md](./README.md) + +每两周自动制作一次节点 snapshot，并将相关参数同步到 README.md 供用户下载使用。 + +## 背景 + +手动制作 snapshot 流程繁琐且容易遗漏，本方案通过服务器 cron 定时任务和 GitHub REST API 将全流程自动化，无需 GitHub Actions 或 git CLI。 + +## 目录结构 + +``` +run-morph-node/ +├── README.md # snapshot 表格在此更新 +└── ops/snapshot/ + ├── README.md # 英文文档 + ├── README.zh.md # 本文档 + ├── snapshot.env.example # 环境变量模板（每个环境复制一份填写） + ├── snapshot_make.py # 入口：停服 → 制作 → 上传 → 重启 → 更新 README + ├── update_metadata.py # 查询 indexer API 并编排完整更新流程 + ├── update_readme.py # 纯表格更新逻辑（由 update_metadata.py 调用） + └── metrics_server.py # 常驻 HTTP server，在 :6060/metrics 暴露 metrics +``` + +## 完整流程 + +``` +服务器 cron 定时任务（每月 1 日 / 15 日） + │ + ▼ + ops/snapshot/snapshot_make.py + [1] 停止 morph-node、morph-geth + [2] 制作快照（tar geth + node 数据） + [3] 上传至 S3 + [4] 重启 morph-geth → 等待 RPC 就绪 → 采集 base_height + [5] 重启 morph-node + [6] 调用 update_metadata.py + │ BASE_HEIGHT, SNAPSHOT_NAME + ▼ + python3 update_metadata.py + ┌─────────────────────────────────────────────────────┐ + │ 1. 调用内网 explorer-indexer API： │ + │ GET /v1/batch/l1_msg_start_height/ │ + │ GET /v1/batch/derivation_start_height/│ + │ 2. 通过 GitHub API 获取 README.md 当前内容 │ + │ 3. 在内存中插入新快照记录到表格顶部 │ + │ 4. 通过 GitHub API 建新分支并推送更新后的文件 │ + │ 5. 通过 GitHub API 开启 PR │ + └─────────────────────────────────────────────────────┘ +``` + +## 触发方式 + +| 方式 | 说明 | +|---|---| +| 定时 | 服务器 cron，每月 1 日和 15 日自动执行 | +| 手动 | SSH 登录服务器，直接执行 `snapshot_make.py` | + +## 多环境支持 + +| 环境 | Indexer API（内网） | +|---|---| +| mainnet | `explorer-indexer.morphl2.io` | +| hoodi | `explorer-indexer-hoodi.morphl2.io` | +| holesky | `explorer-indexer-holesky.morphl2.io` | + +每个环境 / 快照类型有独立的 env 文件，通过 `ENV_FILE` 环境变量指定。S3 路径和 README 表格自动按环境区分。 + +## 部署步骤 + +### 1. 在节点服务器上克隆仓库 + +```bash +git clone https://github.com/morph-l2/run-morph-node.git /data/run-morph-node +``` + +### 2. 创建环境变量文件 + +在脚本同级目录复制模板并填写对应值： + +```bash +cd /data/run-morph-node/ops/snapshot +cp snapshot.env.example snapshot.env +# 编辑 snapshot.env，填写 GH_TOKEN、S3_BUCKET、ENVIRONMENT 等 +``` + +多个环境或快照类型各自使用独立的 env 文件： + +```bash +cp snapshot.env.example snapshot-hoodi.env +cp snapshot.env.example snapshot-mainnet-mpt.env +``` + +所有可配置变量及其说明见 [`snapshot.env.example`](./snapshot.env.example)。这些文件**不可提交到 git**（在 `.gitignore` 中添加 `*.env`）。 + +同时建议在 repo Settings → General 中开启 **"Automatically delete head branches"**，PR merge 后分支自动删除，无需手动维护。 + +### 3. 配置 cron job + +每个环境 / 快照类型各添加一条 cron 记录： + +```bash +crontab -e +``` + +```cron +REPO=/data/run-morph-node/ops/snapshot + +# mainnet 标准 snapshot（使用默认的 snapshot.env） +0 2 1,15 * * python3 $REPO/snapshot_make.py >> /var/log/snapshot-mainnet.log 2>&1 + +# mainnet mpt-snapshot（env 文件中设置 SNAPSHOT_PREFIX=mpt-snapshot） +0 3 1,15 * * ENV_FILE=$REPO/snapshot-mainnet-mpt.env python3 $REPO/snapshot_make.py >> /var/log/snapshot-mainnet-mpt.log 2>&1 + +# hoodi +0 2 1,15 * * ENV_FILE=$REPO/snapshot-hoodi.env python3 $REPO/snapshot_make.py >> /var/log/snapshot-hoodi.log 2>&1 +``` + +### 4. 启动 metrics server + +在节点服务器上用 pm2 托管 `metrics_server.py`，使其随机器重启自动恢复： + +```bash +pm2 startup # 将 pm2 自身注册为系统开机服务（仅需执行一次） +pm2 start python3 --name morph-snapshot-metrics -- /data/run-morph-node/ops/snapshot/metrics_server.py +pm2 save +``` + +启动后采集侧即可通过 `http://:6060/metrics` 拉取指标。 + +暴露的 metrics： + +| Metric | 类型 | 说明 | +|---|---|---| +| `morph_snapshot_readme_update_status` | gauge | 1 = 成功，0 = 失败 | +| `morph_snapshot_readme_update_timestamp_seconds` | gauge | 最后一次执行的 Unix 时间戳 | + +Labels：`environment`（mainnet / hoodi / holesky）、`snapshot`（快照名称） + +> 默认 metrics 文件路径：`/tmp/morph_snapshot_metrics.prom` +> 如需修改，通过环境变量 `METRICS_FILE` 统一传入（对 `update_readme.py` 和 `metrics_server.py` 同时生效）。 + +## 关键设计决策 + +- **base_height 在 geth 重启后采集**：snapshot 制作完成、geth 单独启动后再查询 RPC，读取的是 snapshot 实际对应的区块状态，比停止前采集更准确。morph-node 在确认高度后再启动。 +- **失败时兜底恢复**：`snapshot_make.py` 在异常时尝试拉起两个进程，避免服务持续中断。 +- **不依赖 GitHub Actions 和 git CLI**：`update_metadata.py` 直接调用 GitHub REST API，服务器只需要 Python 3，`GH_TOKEN` 是唯一需要的凭证。 +- **新记录插入表格顶部**：最新 snapshot 始终出现在表格第一行，便于用户快速找到。 +- **通过 PR 而非直接 push 合并变更**：创建新分支并开启 PR，保留 review 机会，避免自动化脚本直接写入 main 分支。 +- **每个环境 / 类型独立 env 文件**：通过 `ENV_FILE` 环境变量指定，各配置互不干扰，同一台机器可以跑多种 snapshot 类型。 diff --git a/ops/snapshot/metrics_server.py b/ops/snapshot/metrics_server.py new file mode 100644 index 0000000..5e6ef27 --- /dev/null +++ b/ops/snapshot/metrics_server.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +""" +Lightweight Prometheus metrics HTTP server for morph snapshot automation. + +Reads a .prom file written by update_readme.py and serves it on :6060/metrics. +Intended to run as a persistent process (e.g. managed by pm2). + +Environment variables: + METRICS_FILE - path to the .prom file (default: /tmp/morph_snapshot_metrics.prom) + METRICS_PORT - port to listen on (default: 6060) +""" + +import http.server +import os +import socket + +METRICS_FILE = os.environ.get("METRICS_FILE", "/tmp/morph_snapshot_metrics.prom") +PORT = int(os.environ.get("METRICS_PORT", "6060")) + +EMPTY_METRICS = ( + "# HELP morph_snapshot_readme_update_status 1 if last README update succeeded, 0 if failed\n" + "# TYPE morph_snapshot_readme_update_status gauge\n" + "# (no data yet — update_readme.sh has not run)\n" +) + + +class MetricsHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path != "/metrics": + self.send_response(404) + self.end_headers() + return + + try: + with open(METRICS_FILE, "r") as f: + content = f.read() + self.send_response(200) + except OSError: + content = EMPTY_METRICS + self.send_response(200) + + body = content.encode("utf-8") + self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + # Suppress per-request access logs to keep output clean + pass + + +if __name__ == "__main__": + server = http.server.HTTPServer(("0.0.0.0", PORT), MetricsHandler) + host = socket.gethostname() + print(f"morph-snapshot metrics server listening on http://{host}:{PORT}/metrics") + print(f"Reading metrics from: {METRICS_FILE}") + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nShutting down.") + server.server_close() + diff --git a/ops/snapshot/snapshot.env.example b/ops/snapshot/snapshot.env.example new file mode 100644 index 0000000..e416509 --- /dev/null +++ b/ops/snapshot/snapshot.env.example @@ -0,0 +1,54 @@ +# ───────────────────────────────────────────────────────────────────────────── +# Morph Snapshot Environment Configuration +# +# Copy this file and fill in the values (keep it in the same directory as the scripts): +# cp ops/snapshot/snapshot.env.example ops/snapshot/snapshot.env +# +# For multiple environments / snapshot types, use separate files: +# cp ops/snapshot/snapshot.env.example ops/snapshot/snapshot-hoodi.env +# ENV_FILE=ops/snapshot/snapshot-hoodi.env python3 ops/snapshot/snapshot_make.py +# +# snapshot.env (the default) is loaded automatically without ENV_FILE. +# ───────────────────────────────────────────────────────────────────────────── + +# ── Required ────────────────────────────────────────────────────────────────── + +# Target environment: mainnet | hoodi | holesky +ENVIRONMENT=mainnet + +# S3 bucket to upload snapshots to +S3_BUCKET=my-morph-snapshots + +# GitHub Fine-grained PAT with Contents:write and Pull requests:write +GH_TOKEN=ghp_xxxxxxxxxxxx + +# GitHub repository in owner/repo format +GITHUB_REPOSITORY=morph-l2/run-morph-node + +# ── Snapshot type ───────────────────────────────────────────────────────────── + +# Prefix for the snapshot name: snapshot | mpt-snapshot | full-snapshot +# Affects snapshot name (e.g. snapshot-20260309-1), S3 key, and branch name. +# Each type running on the same day will get a unique name and branch. +SNAPSHOT_PREFIX=snapshot + +# ── Paths ───────────────────────────────────────────────────────────────────── + +# Root directory of chain data for this environment +MORPH_HOME=/data/mainnet + +# Temporary work directory used during snapshot compression (cleared after use) +SNAPSHOT_WORK_DIR=/data/snapshot_work + +# Output path of the compressed snapshot file +SNAPSHOT_FILE=/data/snapshot.tar.gz + +# ── Service ─────────────────────────────────────────────────────────────────── + +# Geth JSON-RPC endpoint used to collect base_height after restart +GETH_RPC=http://127.0.0.1:8545 + +# ── README ──────────────────────────────────────────────────────────────────── + +# Path to README.md within the GitHub repository (relative to repo root) +README_PATH=README.md diff --git a/ops/snapshot/snapshot_make.py b/ops/snapshot/snapshot_make.py new file mode 100644 index 0000000..7071a75 --- /dev/null +++ b/ops/snapshot/snapshot_make.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +""" +ops/snapshot/snapshot_make.py + +Runs on the node server via cron (1st and 15th of each month). + +Responsibilities: + 1. Stop morph-geth and morph-node + 2. Create and compress a snapshot of chain data + 3. Upload the snapshot to S3 + 4. Restart morph-geth, wait for RPC, collect base_height + 5. Restart morph-node + 6. Call update_metadata.py to open a PR updating the README snapshot table + +Setup: + 1. Clone the repo to /data/run-morph-node on the node server + 2. Copy ops/snapshot/snapshot.env.example to ops/snapshot/snapshot.env and fill in values + For multiple environments/types, use separate files and pass via ENV_FILE: + cp ops/snapshot/snapshot.env.example ops/snapshot/snapshot-mainnet.env + cp ops/snapshot/snapshot.env.example ops/snapshot/snapshot-hoodi.env + + 3. Add to crontab (one entry per environment / snapshot type): + + REPO=/data/run-morph-node/ops/snapshot + + # mainnet standard snapshot (uses default snapshot.env) + 0 2 1,15 * * python3 $REPO/snapshot_make.py >> /var/log/snapshot-mainnet.log 2>&1 + + # mainnet mpt-snapshot + 0 3 1,15 * * ENV_FILE=$REPO/snapshot-mainnet-mpt.env \ + python3 $REPO/snapshot_make.py >> /var/log/snapshot-mainnet-mpt.log 2>&1 + + # hoodi + 0 2 1,15 * * ENV_FILE=$REPO/snapshot-hoodi.env \ + python3 $REPO/snapshot_make.py >> /var/log/snapshot-hoodi.log 2>&1 +""" + +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.resolve())) + +SCRIPT_DIR = Path(__file__).parent.resolve() +REPO_DIR = SCRIPT_DIR.parent.parent + +# ── Env file loader ──────────────────────────────────────────────────────────── + +def load_env_file(path: str) -> None: + """Parse KEY=value lines (with or without 'export' prefix) into os.environ.""" + try: + with open(path) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[len("export "):] + if "=" in line: + key, _, value = line.partition("=") + value = value.strip().strip('"').strip("'") + os.environ.setdefault(key.strip(), value) + except FileNotFoundError: + print(f"WARNING: {path} not found, relying on existing environment variables") + +# ── Shell helpers ────────────────────────────────────────────────────────────── + +def run(args: list, check: bool = True) -> None: + print(f" $ {' '.join(str(a) for a in args)}") + subprocess.run(args, check=check) + +# ── Geth RPC ─────────────────────────────────────────────────────────────────── + +def get_block_height(rpc_url: str = "http://localhost:8545", + retries: int = 30, interval: int = 5) -> int: + payload = json.dumps({ + "jsonrpc": "2.0", "method": "eth_blockNumber", "params": [], "id": 1 + }).encode() + for i in range(1, retries + 1): + try: + req = urllib.request.Request( + rpc_url, data=payload, + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=5) as resp: + result = json.loads(resp.read())["result"] + if result: + return int(result, 16) + except Exception: + pass + print(f" attempt {i}: geth not ready yet, retrying in {interval}s...") + time.sleep(interval) + raise RuntimeError("geth RPC did not become available in time") + +# ── Main ─────────────────────────────────────────────────────────────────────── + +def main() -> None: + env_file = os.environ.get("ENV_FILE", str(SCRIPT_DIR / "snapshot.env")) + load_env_file(env_file) + + environment = os.environ.get("ENVIRONMENT", "mainnet") + morph_home = os.environ.get("MORPH_HOME", f"/data/{environment}") + s3_bucket = os.environ.get("S3_BUCKET", "") + if not s3_bucket: + print("ERROR: S3_BUCKET is required", file=sys.stderr) + sys.exit(1) + + geth_data_dir = os.path.join(morph_home, "geth-data") + node_data_dir = os.path.join(morph_home, "node-data") + work_dir = os.environ.get("SNAPSHOT_WORK_DIR", "/data/snapshot_work") + snapshot_file = os.environ.get("SNAPSHOT_FILE", "/data/snapshot.tar.gz") + + # SNAPSHOT_PREFIX allows different snapshot types to coexist: + # e.g. "snapshot", "mpt-snapshot", "full-snapshot" + snapshot_prefix = os.environ.get("SNAPSHOT_PREFIX", "snapshot") + date = datetime.now(timezone.utc).strftime("%Y%m%d") + snapshot_name = f"{snapshot_prefix}-{date}-1" + + os.environ["SNAPSHOT_NAME"] = snapshot_name + os.environ["ENVIRONMENT"] = environment + + print(f"=== Morph Snapshot: {snapshot_name} ({environment}) ===") + print(f"Started at: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}") + + gh_token = os.environ.get("GH_TOKEN", "") + gh_repo = os.environ.get("GITHUB_REPOSITORY", "") + + services_stopped = False + try: + # ── Step 0: Resolve snapshot_name before any destructive operation ──── + # Check GitHub now so that snapshot_name, S3 key, and branch all match. + if gh_token and gh_repo: + from update_metadata import resolve_snapshot_name + snapshot_name = resolve_snapshot_name(gh_repo, environment, snapshot_name, gh_token) + os.environ["SNAPSHOT_NAME"] = snapshot_name + print(f"Resolved snapshot name: {snapshot_name}") + + # ── Step 1: Stop services ───────────────────────────────────────────── + print("\n[1/6] Stopping services...") + run(["pm2", "stop", "morph-node"]) + run(["pm2", "stop", "morph-geth"]) + services_stopped = True + time.sleep(10) + print("✅ Services stopped") + + # ── Step 2: Create snapshot ─────────────────────────────────────────── + print("\n[2/6] Creating snapshot...") + if os.path.exists(work_dir): + shutil.rmtree(work_dir) + os.makedirs(work_dir) + shutil.copytree(os.path.join(geth_data_dir, "geth"), os.path.join(work_dir, "geth")) + shutil.copytree(os.path.join(node_data_dir, "data"), os.path.join(work_dir, "data")) + + print(f"Compressing to {snapshot_file}...") + run(["tar", "-czf", snapshot_file, "-C", work_dir, "."]) + shutil.rmtree(work_dir) + size = subprocess.check_output(["du", "-sh", snapshot_file]).decode().split()[0] + print(f"✅ Snapshot created: {size}") + + # ── Step 3: Upload to S3 ────────────────────────────────────────────── + print("\n[3/6] Uploading to S3...") + s3_key = f"{environment}/{snapshot_name}.tar.gz" + run(["aws", "s3", "cp", snapshot_file, f"s3://{s3_bucket}/{s3_key}", "--no-progress"]) + print(f"✅ Uploaded: s3://{s3_bucket}/{s3_key}") + + # ── Step 4: Start geth, collect base_height ─────────────────────────── + print("\n[4/6] Starting morph-geth and collecting base_height...") + run(["pm2", "start", "morph-geth"]) + print("Waiting for geth RPC to be ready...") + base_height = get_block_height() + os.environ["BASE_HEIGHT"] = str(base_height) + print(f"✅ Geth base height: {base_height}") + + # ── Step 5: Start morph-node ────────────────────────────────────────── + print("\n[5/6] Starting morph-node...") + run(["pm2", "start", "morph-node"]) + print("✅ morph-node started") + + # ── Step 6: Update README via GitHub API ────────────────────────────── + print("\n[6/6] Updating README snapshot table...") + run([sys.executable, str(REPO_DIR / "ops" / "snapshot" / "update_metadata.py")]) + + except Exception as e: + print(f"\nERROR: {e}", file=sys.stderr) + if services_stopped: + print("Recovering services...") + run(["pm2", "start", "morph-geth"], check=False) + run(["pm2", "start", "morph-node"], check=False) + print("Services recovered.") + sys.exit(1) + + print(f"\n=== Done at {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')} ===") + + +if __name__ == "__main__": + main() diff --git a/ops/snapshot/update_metadata.py b/ops/snapshot/update_metadata.py new file mode 100644 index 0000000..49d4584 --- /dev/null +++ b/ops/snapshot/update_metadata.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +Fetch snapshot metadata from the indexer API and update README.md via GitHub API. + +Given BASE_HEIGHT and SNAPSHOT_NAME, this script: + 1. Queries the internal explorer-indexer API for l1_msg_start_height + and derivation_start_height. + 2. Fetches README.md content from GitHub, applies the table update in memory. + 3. Creates a new branch, pushes the updated file, and opens a PR — + all via GitHub REST API (no git or gh CLI required). + +Environment variables: + ENVIRONMENT - mainnet | hoodi | holesky + SNAPSHOT_NAME - e.g. snapshot-20260225-1 + BASE_HEIGHT - L2 geth block height + GH_TOKEN - GitHub personal access token (repo scope) + GITHUB_REPOSITORY - owner/repo, e.g. morphl2/run-morph-node + README_PATH - path to README.md inside the repo (default: README.md) + L1_MSG_HEIGHT - (optional) skip indexer API, use this value directly + DERIV_HEIGHT - (optional) skip indexer API, use this value directly + DRY_RUN - set to "1" to skip README update and PR creation + +Usage: + # Full run (on Self-hosted Runner, hits internal indexer API): + ENVIRONMENT=mainnet SNAPSHOT_NAME=snapshot-20260225-1 BASE_HEIGHT=20169165 \\ + GH_TOKEN=ghp_xxx GITHUB_REPOSITORY=morphl2/run-morph-node \\ + python3 ops/snapshot/update_metadata.py + + # Local test with mock values — no git/gh CLI needed: + ENVIRONMENT=mainnet SNAPSHOT_NAME=snapshot-test-1 BASE_HEIGHT=20169165 \\ + L1_MSG_HEIGHT=24280251 DERIV_HEIGHT=24294756 \\ + GH_TOKEN=ghp_xxx GITHUB_REPOSITORY=morphl2/run-morph-node \\ + python3 ops/snapshot/update_metadata.py + + # Dry run — only fetches/prints metadata, touches nothing: + ENVIRONMENT=mainnet SNAPSHOT_NAME=snapshot-test-1 BASE_HEIGHT=20169165 \\ + L1_MSG_HEIGHT=24280251 DERIV_HEIGHT=24294756 DRY_RUN=1 \\ + python3 ops/snapshot/update_metadata.py +""" + +import base64 +import json +import os +import re +import sys +import urllib.error +import urllib.request + +# ── Constants ───────────────────────────────────────────────────────────────── + +INDEXER_HOSTS = { + "mainnet": "explorer-indexer.morphl2.io", + "hoodi": "explorer-indexer-hoodi.morphl2.io", + "holesky": "explorer-indexer-holesky.morphl2.io", +} + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, SCRIPT_DIR) + +# ── HTTP helpers ────────────────────────────────────────────────────────────── + +def _http_request(req: urllib.request.Request, url: str) -> dict: + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + body = e.read().decode(errors="replace") + raise RuntimeError(f"HTTP {e.code} {e.reason} — URL: {url}\nResponse: {body}") from None + except urllib.error.URLError as e: + raise RuntimeError(f"Network error — URL: {url}\n{e.reason}") from None + + +def http_get(url: str, token: str = "") -> dict: + headers = {"Accept": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + return _http_request(urllib.request.Request(url, headers=headers), url) + + +def http_get_or_none(url: str, token: str = "") -> dict | None: + """Like http_get but returns None on 404 instead of raising.""" + headers = {"Accept": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + if e.code == 404: + return None + body = e.read().decode(errors="replace") + raise RuntimeError(f"HTTP {e.code} {e.reason} — URL: {url}\nResponse: {body}") from None + except urllib.error.URLError as e: + raise RuntimeError(f"Network error — URL: {url}\n{e.reason}") from None + + +def http_post(url: str, payload: dict, token: str) -> dict: + data = json.dumps(payload).encode() + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {token}"} + return _http_request(urllib.request.Request(url, data=data, headers=headers, method="POST"), url) + + +def http_put(url: str, payload: dict, token: str) -> dict: + data = json.dumps(payload).encode() + headers = {"Content-Type": "application/json", "Authorization": f"Bearer {token}"} + return _http_request(urllib.request.Request(url, data=data, headers=headers, method="PUT"), url) + +# ── Indexer API ─────────────────────────────────────────────────────────────── + +def fetch_metadata(environment: str, base_height: str) -> tuple[str, str]: + """Return (l1_msg_start_height, derivation_start_height) as strings.""" + host = INDEXER_HOSTS[environment] + + def get(path): + url = f"https://{host}{path}" + print(f" GET {url}") + return http_get(url) + + l1_data = get(f"/v1/batch/l1_msg_start_height/{base_height}") + deriv_data = get(f"/v1/batch/derivation_start_height/{base_height}") + + if "l1_msg_start_height" not in l1_data: + raise RuntimeError(f"Unexpected indexer response for l1_msg_start_height: {l1_data}") + if "derivation_start_height" not in deriv_data: + raise RuntimeError(f"Unexpected indexer response for derivation_start_height: {deriv_data}") + + return str(l1_data["l1_msg_start_height"]), str(deriv_data["derivation_start_height"]) + +# ── GitHub API ──────────────────────────────────────────────────────────────── + +GITHUB_API = "https://api.github.com" + + +def gh_get_file(repo: str, path: str, token: str, ref: str = "main") -> tuple[str, str]: + """Fetch file content. Returns (decoded_content, blob_sha).""" + url = f"{GITHUB_API}/repos/{repo}/contents/{path}?ref={ref}" + data = http_get(url, token) + content = base64.b64decode(data["content"]).decode("utf-8") + return content, data["sha"] + + +def gh_get_main_sha(repo: str, token: str) -> str: + """Return the current commit SHA of the main branch.""" + url = f"{GITHUB_API}/repos/{repo}/git/ref/heads/main" + data = http_get(url, token) + return data["object"]["sha"] + + +def gh_branch_exists(repo: str, branch: str, token: str) -> bool: + url = f"{GITHUB_API}/repos/{repo}/git/ref/heads/{branch}" + return http_get_or_none(url, token) is not None + + +def resolve_snapshot_name(repo: str, environment: str, + snapshot_name: str, token: str) -> str: + """Return a snapshot_name whose branch does not yet exist on GitHub. + + Increments the trailing -N suffix until a free branch is found, so that + snapshot_name, S3 key, README row, and branch name all stay in sync. + + e.g. snapshot-20260309-1 → snapshot-20260309-2 if the -1 branch exists. + """ + base_name = re.sub(r"-\d+$", "", snapshot_name) + counter = 1 + candidate = f"{base_name}-{counter}" + while gh_branch_exists(repo, f"snapshot/{environment}-{candidate}", token): + counter += 1 + candidate = f"{base_name}-{counter}" + if candidate != snapshot_name: + print(f" Branch for {snapshot_name} already exists → using {candidate}") + return candidate + + +def gh_create_branch(repo: str, branch: str, sha: str, token: str) -> None: + """Create branch. snapshot_name must already be resolved via resolve_snapshot_name.""" + url = f"{GITHUB_API}/repos/{repo}/git/refs" + http_post(url, {"ref": f"refs/heads/{branch}", "sha": sha}, token) + print(f" Created branch: {branch}") + + +def gh_update_file(repo: str, path: str, content: str, + blob_sha: str, branch: str, message: str, token: str) -> None: + url = f"{GITHUB_API}/repos/{repo}/contents/{path}" + http_put(url, { + "message": message, + "content": base64.b64encode(content.encode("utf-8")).decode(), + "sha": blob_sha, + "branch": branch, + }, token) + print(f" Pushed {path} to branch: {branch}") + + +def gh_create_pr(repo: str, branch: str, title: str, body: str, token: str) -> str: + url = f"{GITHUB_API}/repos/{repo}/pulls" + data = http_post(url, { + "title": title, + "body": body, + "head": branch, + "base": "main", + }, token) + return data["html_url"] + +# ── README update (in-memory) ───────────────────────────────────────────────── + +def build_new_row(environment: str, snapshot_name: str, + deriv_height: str, l1_msg_height: str, base_height: str) -> str: + cdn_base = "https://snapshot.morphl2.io" + url = f"{cdn_base}/{environment}/{snapshot_name}.tar.gz" + return f"| [{snapshot_name}]({url}) | {deriv_height} | {l1_msg_height} | {base_height} |" + + +def apply_readme_update(content: str, environment: str, snapshot_name: str, + deriv_height: str, l1_msg_height: str, base_height: str) -> str: + """Import insert_row_content from update_readme.py and apply it.""" + from update_readme import insert_row_content, SECTION_MARKERS # noqa: E402 + + section_marker = SECTION_MARKERS[environment] + new_row = build_new_row(environment, snapshot_name, deriv_height, l1_msg_height, base_height) + return insert_row_content(content, section_marker, new_row) + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main() -> None: + dry_run = os.environ.get("DRY_RUN", "0") == "1" + + # Validate required env vars + required = ["ENVIRONMENT", "SNAPSHOT_NAME", "BASE_HEIGHT"] + if not dry_run: + required += ["GH_TOKEN", "GITHUB_REPOSITORY"] + + missing = [v for v in required if not os.environ.get(v)] + if missing: + print(f"ERROR: Missing required env vars: {', '.join(missing)}", file=sys.stderr) + sys.exit(1) + + environment = os.environ["ENVIRONMENT"] + snapshot_name = os.environ["SNAPSHOT_NAME"] + base_height = os.environ["BASE_HEIGHT"] + token = os.environ.get("GH_TOKEN", "") + repo = os.environ.get("GITHUB_REPOSITORY", "") + readme_path = os.environ.get("README_PATH", "README.md") + + if environment not in INDEXER_HOSTS: + print(f"ERROR: Unknown environment: {environment!r}. Must be: {' | '.join(INDEXER_HOSTS)}", + file=sys.stderr) + sys.exit(1) + + # ── Step 1: metadata ────────────────────────────────────────────────────── + l1_msg_height = os.environ.get("L1_MSG_HEIGHT", "") + deriv_height = os.environ.get("DERIV_HEIGHT", "") + + if l1_msg_height and deriv_height: + print(f"\n[1/3] Using provided metadata (API call skipped):") + else: + print(f"\n[1/3] Fetching metadata from indexer (base_height={base_height}) ...") + l1_msg_height, deriv_height = fetch_metadata(environment, base_height) + + print(f" l1_msg_start_height = {l1_msg_height}") + print(f" derivation_start_height = {deriv_height}") + + if dry_run: + print("\n[DRY RUN] Skipping README update and PR creation.") + print(f" Would insert: env={environment} snapshot={snapshot_name}") + print(f" base={base_height} l1_msg={l1_msg_height} deriv={deriv_height}") + return + + # ── Step 2: update README in memory, push via GitHub API ───────────────── + print(f"\n[2/3] Updating README via GitHub API ...") + current_content, blob_sha = gh_get_file(repo, readme_path, token) + updated_content = apply_readme_update( + current_content, environment, snapshot_name, deriv_height, l1_msg_height, base_height + ) + + branch = f"snapshot/{environment}-{snapshot_name}" + commit_msg = f"snapshot: add {snapshot_name} ({environment})" + main_sha = gh_get_main_sha(repo, token) + + gh_create_branch(repo, branch, main_sha, token) + gh_update_file(repo, readme_path, updated_content, blob_sha, branch, commit_msg, token) + + # ── Step 3: open PR ─────────────────────────────────────────────────────── + print(f"\n[3/3] Creating PR ...") + pr_body = ( + f"Auto-generated by snapshot workflow.\n\n" + f"- Environment: `{environment}`\n" + f"- Snapshot: `{snapshot_name}`\n" + f"- L2 Base Height: `{base_height}`\n" + f"- L1 Msg Start Height: `{l1_msg_height}`\n" + f"- Derivation Start Height: `{deriv_height}`" + ) + pr_url = gh_create_pr(repo, branch, commit_msg, pr_body, token) + + print(f"\n✅ Done. PR opened: {pr_url}") + + +if __name__ == "__main__": + main() diff --git a/ops/snapshot/update_readme.py b/ops/snapshot/update_readme.py new file mode 100644 index 0000000..048705a --- /dev/null +++ b/ops/snapshot/update_readme.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +""" +Update the snapshot table in README.md. + +Inserts a new row at the TOP of the target environment's snapshot table. + +Environment variables: + ENVIRONMENT - mainnet | hoodi | holesky + SNAPSHOT_NAME - e.g. snapshot-20260225-1 + BASE_HEIGHT - L2 geth block height (L2 Base Height) + L1_MSG_HEIGHT - l1_msg_start_height from indexer API + DERIV_HEIGHT - derivation_start_height from indexer API + METRICS_FILE - (optional) path to write Prometheus metrics + default: /tmp/morph_snapshot_metrics.prom + metrics_server.py reads this file and serves it on :6060/metrics + +Usage: + python3 ops/snapshot/update_readme.py +""" + +import os +import re +import sys +import time + +# ── Constants ───────────────────────────────────────────────────────────────── + +CDN_BASE = "https://snapshot.morphl2.io" + +SECTION_MARKERS = { + "mainnet": "**For mainnet**", + "hoodi": "**For hoodi testnet**", + "holesky": "**For holesky testnet(legacy)**", +} + +METRICS_FILE = os.environ.get("METRICS_FILE", "/tmp/morph_snapshot_metrics.prom") + +# ── Metrics ─────────────────────────────────────────────────────────────────── + +def write_metric(status: int, environment: str, snapshot_name: str) -> None: + """Write Prometheus metrics to METRICS_FILE. status: 1=success, 0=failure.""" + ts = int(time.time()) + labels = f'environment="{environment}",snapshot="{snapshot_name}"' + content = ( + "# HELP morph_snapshot_readme_update_status 1 if last README update succeeded, 0 if failed\n" + "# TYPE morph_snapshot_readme_update_status gauge\n" + f"morph_snapshot_readme_update_status{{{labels}}} {status}\n" + "# HELP morph_snapshot_readme_update_timestamp_seconds Unix timestamp of last run\n" + "# TYPE morph_snapshot_readme_update_timestamp_seconds gauge\n" + f"morph_snapshot_readme_update_timestamp_seconds{{{labels}}} {ts}\n" + ) + os.makedirs(os.path.dirname(os.path.abspath(METRICS_FILE)), exist_ok=True) + with open(METRICS_FILE, "w") as f: + f.write(content) + +# ── README update ───────────────────────────────────────────────────────────── + +def insert_row_content(content: str, section_marker: str, new_row: str) -> str: + """ + In-memory version: takes the README content as a string, inserts new_row + after the table separator in the target section, returns updated content. + """ + lines = content.splitlines(keepends=True) + in_section = False + inserted = False + result = [] + + for line in lines: + result.append(line) + + if section_marker in line: + in_section = True + + if in_section and not inserted and re.match(r"^\|[\s:|-]+\|", line): + result.append(new_row + "\n") + inserted = True + in_section = False + + if not inserted: + raise RuntimeError( + f"Could not find table separator for section: {section_marker!r}" + ) + + return "".join(result) + + +def insert_row(readme_path: str, section_marker: str, new_row: str) -> None: + """File-based wrapper around insert_row_content.""" + with open(readme_path, "r") as f: + content = f.read() + + updated = insert_row_content(content, section_marker, new_row) + + with open(readme_path, "w") as f: + f.write(updated) + +# ── Main ────────────────────────────────────────────────────────────────────── + +def main() -> None: + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + readme_path = sys.argv[1] + + # Validate required env vars + missing = [v for v in ("ENVIRONMENT", "SNAPSHOT_NAME", "BASE_HEIGHT", "L1_MSG_HEIGHT", "DERIV_HEIGHT") + if not os.environ.get(v)] + if missing: + print(f"ERROR: Missing required env vars: {', '.join(missing)}", file=sys.stderr) + write_metric(0, os.environ.get("ENVIRONMENT", "unknown"), + os.environ.get("SNAPSHOT_NAME", "unknown")) + sys.exit(1) + + environment = os.environ["ENVIRONMENT"] + snapshot_name = os.environ["SNAPSHOT_NAME"] + base_height = os.environ["BASE_HEIGHT"] + l1_msg_height = os.environ["L1_MSG_HEIGHT"] + deriv_height = os.environ["DERIV_HEIGHT"] + + # Validate environment + if environment not in SECTION_MARKERS: + print(f"ERROR: Unknown environment: {environment!r}. Must be: {' | '.join(SECTION_MARKERS)}", + file=sys.stderr) + write_metric(0, environment, snapshot_name) + sys.exit(1) + + section_marker = SECTION_MARKERS[environment] + url = f"{CDN_BASE}/{environment}/{snapshot_name}.tar.gz" + new_row = f"| [{snapshot_name}]({url}) | {deriv_height} | {l1_msg_height} | {base_height} |" + + try: + insert_row(readme_path, section_marker, new_row) + except Exception as e: + print(f"ERROR: {e}", file=sys.stderr) + write_metric(0, environment, snapshot_name) + sys.exit(1) + + print(f"✅ Inserted new row into [{environment}] table:") + print(f" {new_row}") + + write_metric(1, environment, snapshot_name) + print(f"📊 Metrics written to: {METRICS_FILE}") + + +if __name__ == "__main__": + main() +