diff --git a/.agents/skills/obol-stack-dev/SKILL.md b/.agents/skills/obol-stack-dev/SKILL.md index 25a89bc..c41af37 100644 --- a/.agents/skills/obol-stack-dev/SKILL.md +++ b/.agents/skills/obol-stack-dev/SKILL.md @@ -1,10 +1,10 @@ --- name: obol-stack-dev -description: Obol Stack development, testing, and LLM routing validation through LiteLLM. Use when developing, testing, or validating inference paths (Ollama, Anthropic, OpenAI) through the LiteLLM gateway, writing integration tests, or working with obol CLI wrappers. +description: Obol Stack development, testing, and validation. Covers LLM routing through LiteLLM, x402 payment flow (sell/buy), BDD integration tests (Gherkin/godog), ERC-8004 registration, and obol CLI wrappers. metadata: version: "2.0.0" domain: infrastructure - triggers: obol, litellm, openclaw, inference, integration test, model routing, smart routing, LLM proxy, provider setup + triggers: obol, litellm, openclaw, inference, integration test, model routing, smart routing, LLM proxy, provider setup, x402, sell, buy, BDD, gherkin, payment, monetize role: specialist scope: development-and-testing output-format: code-and-commands @@ -20,9 +20,11 @@ Complete guide for developing, testing, and validating the Obol Stack's LLM rout - Setting up the Obol Stack development environment - Testing LLM inference through LiteLLM (Ollama, Anthropic, OpenAI) - Writing or running integration tests for OpenClaw instances +- Running BDD integration tests for the x402 sell→discover→buy payment flow - Debugging model routing issues (401s, 500s, provider misconfig) - Understanding the 2-tier LLM architecture (LiteLLM gateway + per-instance config) - Validating the paid remote-inference path through LiteLLM + `x402-buyer` +- Testing x402 payment gating, ERC-8004 registration, OASF metadata - Deploying and validating OpenClaw instances with different providers - Working with the `obol` CLI wrappers (kubectl, helm, helmfile, k9s) diff --git a/.agents/skills/obol-stack-dev/references/integration-testing.md b/.agents/skills/obol-stack-dev/references/integration-testing.md index 252ff56..dcc8ae0 100644 --- a/.agents/skills/obol-stack-dev/references/integration-testing.md +++ b/.agents/skills/obol-stack-dev/references/integration-testing.md @@ -181,6 +181,81 @@ func TestIntegration_MyTest(t *testing.T) { 5. Deploy via `obol openclaw sync` 6. Model name format: `openai/` (always openai/ prefix through LiteLLM) +## BDD Integration Tests (x402 Payment Flow) + +A separate BDD test suite validates the full sell→discover→buy payment flow using Gherkin scenarios and godog. These tests live in `internal/x402/` and follow the **real user journey** — no kubectl shortcuts. + +### Running BDD Tests + +```bash +# Against existing cluster (fast, ~2min) +export OBOL_DEVELOPMENT=true +export OBOL_CONFIG_DIR=$(pwd)/.workspace/config +export OBOL_BIN_DIR=$(pwd)/.workspace/bin +export OBOL_DATA_DIR=$(pwd)/.workspace/data +export OBOL_INTEGRATION_SKIP_BOOTSTRAP=true +export OBOL_TEST_MODEL=qwen3.5:9b + +go test -tags integration -v -run TestBDDIntegration -timeout 10m ./internal/x402/ + +# Full bootstrap from scratch (~15min, creates cluster) +go test -tags integration -v -run TestBDDIntegration -timeout 20m ./internal/x402/ +``` + +### BDD Scenarios (7 total, 77 steps) + +| Scenario | What it validates | +|----------|------------------| +| Operator sells inference via CLI + agent reconciles | `obol sell http` → CRD → 6-stage reconciliation | +| Unpaid request returns 402 | x402 payment gate | +| Paid request returns real inference | EIP-712 → verify → LiteLLM → Ollama | +| Discovery-to-payment cycle | Parse 402 → sign → pay → 200 | +| Paid request through Cloudflare tunnel | Full flow via tunnel | +| Agent discovers service through tunnel | `.well-known` → x402Support → OASF skills/domains → probe 402 | +| Operator deletes + cleanup | CR + pricing route removed | + +### BDD Test Files + +| File | Role | +|------|------| +| `internal/x402/features/integration_payment_flow.feature` | Gherkin scenarios | +| `internal/x402/bdd_integration_test.go` | TestMain bootstrap (builds binary, stack init/up, agent init, sell) | +| `internal/x402/bdd_integration_steps_test.go` | Step definitions (Anvil, facilitator, payment signing) | + +### BDD TestMain Bootstrap (Full Mode) + +``` +1. go build ./cmd/obol +2. obol stack init + up +3. obol model setup (Anthropic/OpenAI/Ollama) +4. obol sell pricing --wallet ... --chain base-sepolia +5. obol agent init (deploys agent + RBAC + monetize skill) +6. obol sell http bdd-test --upstream litellm --port 4000 ... +7. Wait for agent reconciliation (6 stages → Ready) +8. Restart x402-verifier +``` + +### Skip-Bootstrap Mode + +Set `OBOL_INTEGRATION_SKIP_BOOTSTRAP=true` to use an existing cluster. Requires: +- ServiceOffer `bdd-test` in `llm` namespace (Ready) +- x402-verifier running +- LiteLLM running with at least one model +- Ollama running (for inference) + +### Per-Scenario Infrastructure + +Each scenario gets a fresh: +- Anvil fork (Base Sepolia, on random port) +- Mock facilitator (accepts all payments) +- x402-verifier patched to use the scenario's facilitator + +### Key Dependency + +```bash +go get github.com/cucumber/godog@v0.15.1 +``` + ## Timing and Timeouts | Operation | Typical Time | Timeout | diff --git a/CLAUDE.md b/CLAUDE.md index 6fa36e6..a3b8123 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -72,7 +72,7 @@ Payment-gated access to cluster services via x402 (HTTP 402 micropayments, USDC **Buy-side flow**: `buy.py probe` sees 402 pricing → `buy.py buy` pre-signs ERC-3009 auths into ConfigMaps → LiteLLM serves static `paid/` aliases through the in-pod `x402-buyer` sidecar → each paid request spends one auth and forwards to the remote seller. -**CLI**: `obol sell pricing --wallet --chain`, `obol sell inference --model --price|--per-mtok`, `obol sell http --upstream --port --price|--per-mtok`, `obol sell list|status|stop|delete`, `obol sell register --name --private-key-file`. +**CLI**: `obol sell pricing --wallet --chain`, `obol sell inference --model --price|--per-mtok`, `obol sell http --wallet --chain --price|--per-request|--per-mtok --upstream --port --namespace --health-path`, `obol sell list|status|stop|delete`, `obol sell register --name --private-key-file`. **ServiceOffer CRD** (`obol.org`): Spec fields — `type` (inference|fine-tuning), `model{name,runtime}`, `upstream{service,ns,port,healthPath}`, `payment{scheme,network,payTo,price{perRequest,perMTok,perHour}}`, `path`, `registration{enabled,name,description,image}`. In phase 1, `perMTok` is accepted but enforced as `perRequest = perMTok / 1000`. @@ -97,7 +97,7 @@ Two-stage templating: `values.yaml.gotmpl` with `@enum/@default/@description` an | Command | Action | |---------|--------| | `obol stack init` | Generate cluster ID, resolve absolute paths, write k3d.yaml, copy infrastructure | -| `obol stack up` | `k3d cluster create`, export kubeconfig, k3s auto-applies manifests | +| `obol stack up` | `k3d cluster create`, export kubeconfig, k3s auto-applies manifests, auto-configures LiteLLM with Ollama models, deploys obol-agent, starts Cloudflare tunnel (default agent model: `qwen3.5:9b`) | | `obol stack down` | `k3d cluster delete` (preserves config + data) | | `obol stack purge [-f]` | Delete config; `-f` also deletes root-owned PVCs | @@ -105,7 +105,7 @@ k3d: 1 server, ports 80:80 + 8080:80 + 443:443 + 8443:443, `rancher/k3s:v1.35.1- ## LLM Routing -**LiteLLM gateway** (`llm` ns, port 4000): OpenAI-compatible proxy routing to Ollama/Anthropic/OpenAI. ConfigMap `litellm-config` (YAML config.yaml with model_list), Secret `litellm-secrets` (master key + API keys). No default provider — `obol model setup` required. `ConfigureLiteLLM()` patches config + Secret + restarts. Custom endpoints: `obol model setup custom --name --endpoint --model` (validates before adding). Paid remote inference stays on vanilla LiteLLM with a static route `paid/* -> openai/* -> http://127.0.0.1:8402`; no LiteLLM fork is required. +**LiteLLM gateway** (`llm` ns, port 4000): OpenAI-compatible proxy routing to Ollama/Anthropic/OpenAI. ConfigMap `litellm-config` (YAML config.yaml with model_list), Secret `litellm-secrets` (master key + API keys). Auto-configured with Ollama models during `obol stack up` (no manual `obol model setup` needed). `ConfigureLiteLLM()` patches config + Secret + restarts. Custom endpoints: `obol model setup custom --name --endpoint --model` (validates before adding). Paid remote inference stays on vanilla LiteLLM with a static route `paid/* -> openai/* -> http://127.0.0.1:8402`; no LiteLLM fork is required. OpenClaw always routes through LiteLLM (openai provider slot), never native providers; `dangerouslyDisableDeviceAuth` is enabled for Traefik-proxied access. **Per-instance overlay**: `buildLiteLLMRoutedOverlay()` reuses "ollama" provider slot pointing at `litellm.llm.svc:4000/v1` with `api: openai-completions`. App → litellm:4000 → routes by model name → actual API. diff --git a/docs/guides/monetize-inference.md b/docs/guides/monetize-inference.md index f18572c..eb14e8f 100644 --- a/docs/guides/monetize-inference.md +++ b/docs/guides/monetize-inference.md @@ -65,13 +65,11 @@ BUYER (curl / blockrun-llm SDK) Start from a clean state: ```bash -# Initialize and start +# Initialize and start (automatically deploys obol-agent, configures LiteLLM +# with Ollama models, and starts a Cloudflare tunnel — no manual setup needed) obol stack init obol stack up -# Deploy the AI agent (manages ServiceOffer reconciliation) -obol agent init - # Wait for all pods to be ready obol kubectl get pods -A ``` @@ -93,8 +91,8 @@ Verify the key components: Make sure the model is available in your host Ollama: ```bash -# Pull a model (qwen3.5:35b recommended for tool-call support) -ollama pull qwen3.5:35b +# Pull a model (qwen3.5:9b is the default agent model) +ollama pull qwen3.5:9b # Or a smaller model for quick testing ollama pull qwen3:0.6b @@ -103,7 +101,7 @@ ollama pull qwen3:0.6b curl -s http://localhost:11434/api/tags | python3 -m json.tool ``` -LiteLLM discovers models from host Ollama at startup. If you pull a new model after the cluster is running, restart LiteLLM: +`obol stack up` automatically configures LiteLLM with all available Ollama models (no manual `obol model setup` needed). If you pull a new model after the cluster is running, restart LiteLLM to pick it up: ```bash obol kubectl rollout restart deployment/litellm -n llm @@ -147,16 +145,12 @@ Declare your inference service as a Kubernetes custom resource: ```bash obol sell http my-qwen \ - --type inference \ - --model qwen3:0.6b \ - --runtime ollama \ + --wallet 0x70997970C51812dc3A010C7d01b50e0d17dc79C8 \ + --chain base-sepolia \ --per-request 0.001 \ - --network base-sepolia \ - --pay-to 0x70997970C51812dc3A010C7d01b50e0d17dc79C8 \ --namespace llm \ --upstream ollama \ - --port 11434 \ - --path /services/my-qwen + --port 11434 ``` If you want to price by million tokens instead of explicitly setting a flat @@ -165,16 +159,12 @@ derived per-request price: ```bash obol sell http my-qwen \ - --type inference \ - --model qwen3:0.6b \ - --runtime ollama \ + --wallet 0x70997970C51812dc3A010C7d01b50e0d17dc79C8 \ + --chain base-sepolia \ --per-mtok 1.25 \ - --network base-sepolia \ - --pay-to 0x70997970C51812dc3A010C7d01b50e0d17dc79C8 \ --namespace llm \ --upstream ollama \ - --port 11434 \ - --path /services/my-qwen + --port 11434 ``` That stores both values in the pricing config: @@ -208,7 +198,7 @@ obol kubectl get httproute -n llm # so-my-qwen ### 1.5 Expose via Cloudflare Tunnel -The stack deploys a Cloudflare Quick Tunnel automatically. Get the public URL: +`obol stack up` automatically starts a Cloudflare Quick Tunnel. Get the public URL: ```bash obol tunnel status @@ -791,7 +781,7 @@ Replace `openclaw-obol-agent` with your actual OpenClaw namespace if different. | Command | Description | |---------|-------------| | `obol sell pricing --wallet ... --chain ...` | Configure x402 payment settings | -| `obol sell http --model ... --per-request ...` | Create a ServiceOffer | +| `obol sell http --wallet ... --chain ... --per-request ... --upstream ... --port ...` | Create a ServiceOffer | | `obol sell list` | List all ServiceOffers | | `obol sell status -n ` | Show conditions for an offer | | `obol sell stop -n ` | Pause an offer (remove pricing route) | diff --git a/internal/embed/skills/sell/scripts/monetize.py b/internal/embed/skills/sell/scripts/monetize.py index 6d26f54..212d517 100644 --- a/internal/embed/skills/sell/scripts/monetize.py +++ b/internal/embed/skills/sell/scripts/monetize.py @@ -932,6 +932,10 @@ def stage_registered(spec, ns, name, token, ssl_ctx): url_path = spec.get("path", f"/services/{name}") agent_uri = f"{base_url}/.well-known/agent-registration.json" + # Publish the registration JSON immediately so `.well-known` is available + # for discovery even before the on-chain NFT mint completes (or if it fails). + _publish_registration_json(spec, ns, name, "", "", token, ssl_ctx) + print(f" Registering on ERC-8004 (Base Sepolia)...") print(f" Registry: {IDENTITY_REGISTRY}") print(f" Agent URI: {agent_uri}") @@ -941,32 +945,28 @@ def stage_registered(spec, ns, name, token, ssl_ctx): except urllib.error.URLError as e: reason = str(e.reason) if hasattr(e, 'reason') else str(e) if "remote-signer" in reason.lower() or "Connection refused" in reason: - msg = f"Remote-signer unavailable: {reason[:100]}" - print(f" {msg}", file=sys.stderr) - set_condition(ns, name, "Registered", "False", "SignerUnavailable", msg, token, ssl_ctx) + msg = f"Off-chain only (remote-signer unavailable): {reason[:80]}" else: - msg = f"RPC error: {reason[:100]}" - print(f" {msg}", file=sys.stderr) - set_condition(ns, name, "Registered", "False", "RPCError", msg, token, ssl_ctx) - return True # Don't block Ready + msg = f"Off-chain only (RPC error): {reason[:80]}" + print(f" {msg}", file=sys.stderr) + set_condition(ns, name, "Registered", "True", "OffChainOnly", msg, token, ssl_ctx) + return True except RuntimeError as e: msg = str(e)[:200] if "insufficient funds" in msg.lower() or "gas" in msg.lower(): - print(f" Wallet not funded on Base Sepolia: {msg}", file=sys.stderr) - set_condition(ns, name, "Registered", "False", "InsufficientFunds", - f"Fund the agent wallet on Base Sepolia: {msg}", token, ssl_ctx) + reason = f"Off-chain only (wallet not funded): {msg[:80]}" elif "reverted" in msg.lower(): - print(f" Registration tx reverted: {msg}", file=sys.stderr) - set_condition(ns, name, "Registered", "False", "TxReverted", msg, token, ssl_ctx) + reason = f"Off-chain only (tx reverted): {msg[:80]}" else: - print(f" Registration failed: {msg}", file=sys.stderr) - set_condition(ns, name, "Registered", "False", "RegistrationFailed", msg, token, ssl_ctx) - return True # Don't block Ready + reason = f"Off-chain only: {msg[:80]}" + print(f" {reason}", file=sys.stderr) + set_condition(ns, name, "Registered", "True", "OffChainOnly", reason, token, ssl_ctx) + return True except Exception as e: - msg = f"Unexpected error: {str(e)[:150]}" + msg = f"Off-chain only (unexpected): {str(e)[:120]}" print(f" {msg}", file=sys.stderr) - set_condition(ns, name, "Registered", "False", "RegistrationFailed", msg, token, ssl_ctx) - return True # Don't block Ready + set_condition(ns, name, "Registered", "True", "OffChainOnly", msg, token, ssl_ctx) + return True # Patch CRD status with on-chain identity. set_status_field(ns, name, "agentId", str(agent_id), token, ssl_ctx) diff --git a/internal/model/model.go b/internal/model/model.go index c4a8082..362d8db 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -68,6 +68,31 @@ type LiteLLMParams struct { APIKey string `yaml:"api_key,omitempty"` } +// HasConfiguredModels returns true if LiteLLM has at least one non-catch-all +// model configured (i.e., something other than the "paid/*" route). +func HasConfiguredModels(cfg *config.Config) bool { + kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + + raw, err := kubectl.Output(kubectlBinary, kubeconfigPath, + "get", "configmap", configMapName, "-n", namespace, "-o", "jsonpath={.data.config\\.yaml}") + if err != nil { + return false + } + + var litellmConfig LiteLLMConfig + if err := yaml.Unmarshal([]byte(raw), &litellmConfig); err != nil { + return false + } + + for _, entry := range litellmConfig.ModelList { + if !strings.Contains(entry.ModelName, "*") { + return true + } + } + return false +} + // ConfigureLiteLLM adds a provider to the LiteLLM gateway. // For cloud providers, it patches the Secret with the API key and adds // the model to config.yaml. For Ollama, it discovers local models and adds them. diff --git a/internal/openclaw/openclaw.go b/internal/openclaw/openclaw.go index ec03c86..bd34204 100644 --- a/internal/openclaw/openclaw.go +++ b/internal/openclaw/openclaw.go @@ -1733,35 +1733,43 @@ rbac: b.WriteString(fmt.Sprintf("# Override chart default image tag (chart ships %s)\nimage:\n tag: \"%s\"\n\n", chartVersion, openclawImageTag)) } - // Provider and agent model configuration - importedOverlay := TranslateToOverlayYAML(imported) - if importedOverlay != "" { - b.WriteString("# Imported from ~/.openclaw/openclaw.json\n") - // Inject gateway controlUi settings for Traefik reverse proxy. - // allowInsecureAuth is required because the browser accesses OpenClaw via - // http://.obol.stack (non-localhost HTTP), where crypto.subtle is - // unavailable. Without it, the gateway rejects with 1008 "requires HTTPS or - // localhost (secure context)". Token auth is still enforced. - if strings.Contains(importedOverlay, "openclaw:\n") { - importedOverlay = strings.Replace(importedOverlay, "openclaw:\n", "openclaw:\n gateway:\n controlUi:\n allowInsecureAuth: true\n dangerouslyAllowHostHeaderOriginFallback: true\n", 1) - } else { - b.WriteString("openclaw:\n gateway:\n controlUi:\n allowInsecureAuth: true\n dangerouslyAllowHostHeaderOriginFallback: true\n\n") - } - b.WriteString(importedOverlay) - } else { - // Default provider: LiteLLM gateway (OpenAI-compatible). - // Model list is populated from the host's Ollama instance (if available). - // Uses "openai" provider slot to avoid triggering Ollama /api/tags discovery. - b.WriteString("# Default model provider: LiteLLM gateway (OpenAI-compatible)\nopenclaw:\n") - if len(ollamaModels) > 0 { - b.WriteString(fmt.Sprintf(" agentModel: openai/%s\n", ollamaModels[0])) - } - b.WriteString(` gateway: + // Provider and agent model configuration. + // + // All inference is routed through the LiteLLM gateway (openai provider slot). + // This ensures OpenClaw never tries to call Anthropic/OpenAI natively (which + // would require its own API keys in the auth store). Instead, LiteLLM handles + // provider routing and API key management via its own Secret. + // + // If the user has an imported ~/.openclaw/openclaw.json, we extract non-model + // config (channels, etc.) but always override the provider to LiteLLM. + + // Determine agent model: prefer imported model, fallback to preferred Ollama model. + agentModel := "" + if imported != nil && imported.AgentModel != "" { + // Rewrite native provider prefix to openai/ so it routes through LiteLLM. + // e.g. "anthropic/claude-sonnet-4-6" → "openai/claude-sonnet-4-6" + agentModel = imported.AgentModel + if i := strings.Index(agentModel, "/"); i >= 0 { + agentModel = "openai/" + agentModel[i+1:] + } else if !strings.HasPrefix(agentModel, "openai/") { + agentModel = "openai/" + agentModel + } + } else if len(ollamaModels) > 0 { + agentModel = "openai/" + preferredOllamaModel(ollamaModels) + } + + b.WriteString("# All models route through LiteLLM gateway (openai provider slot).\n") + b.WriteString("openclaw:\n") + if agentModel != "" { + b.WriteString(fmt.Sprintf(" agentModel: %s\n", agentModel)) + } + b.WriteString(` gateway: # Allow control UI over HTTP behind Traefik (local dev stack). - # Required: browser on non-localhost HTTP has no crypto.subtle, - # so device identity is unavailable. Token auth is still enforced. + # dangerouslyDisableDeviceAuth is needed because Traefik proxies from + # the k3d bridge IP, not localhost. Token auth is still enforced. controlUi: allowInsecureAuth: true + dangerouslyDisableDeviceAuth: true dangerouslyAllowHostHeaderOriginFallback: true # LiteLLM gateway: OpenAI-compatible proxy routing to all configured providers. @@ -1773,17 +1781,47 @@ models: api: openai-completions apiKeyEnvVar: OPENAI_API_KEY `) - b.WriteString(fmt.Sprintf(" apiKeyValue: %s\n", litellmMasterKey(cfg))) + b.WriteString(fmt.Sprintf(" apiKeyValue: %s\n", litellmMasterKey(cfg))) - if len(ollamaModels) > 0 { - b.WriteString(" models:\n") - for _, m := range ollamaModels { - b.WriteString(fmt.Sprintf(" - id: %s\n name: %s\n", m, ollamaModelDisplayName(m))) + if len(ollamaModels) > 0 { + b.WriteString(" models:\n") + for _, m := range ollamaModels { + b.WriteString(fmt.Sprintf(" - id: %s\n name: %s\n", m, ollamaModelDisplayName(m))) + } + } else { + b.WriteString(" models: []\n") + } + b.WriteString("\n") + + // Append non-model imported config (channels, etc.) + if imported != nil { + importedOverlay := TranslateToOverlayYAML(imported) + // Strip the openclaw: and models: sections — we already wrote those above. + // Keep only channel config and other non-provider settings. + lines := strings.Split(importedOverlay, "\n") + var kept []string + skip := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + // Skip openclaw: block (agentModel) and models: block (providers) + if trimmed == "openclaw:" || trimmed == "models:" { + skip = true + continue + } + // Stop skipping when we hit a new top-level key + if skip && len(line) > 0 && line[0] != ' ' && line[0] != '\t' { + skip = false + } + if !skip { + kept = append(kept, line) } - } else { - b.WriteString(" models: []\n") } - b.WriteString("\n") + extra := strings.TrimSpace(strings.Join(kept, "\n")) + if extra != "" { + b.WriteString("# Imported from ~/.openclaw/openclaw.json\n") + b.WriteString(extra) + b.WriteString("\n\n") + } } b.WriteString(`# eRPC integration @@ -2001,6 +2039,20 @@ func listOllamaModels() []string { return names } +// preferredOllamaModel picks the best default model from available Ollama models. +// Prefers qwen3.5:9b if available, otherwise falls back to the first model. +func preferredOllamaModel(models []string) string { + preferred := []string{"qwen3.5:9b", "qwen3.5:35b", "qwen3.5:27b"} + for _, p := range preferred { + for _, m := range models { + if m == p { + return m + } + } + } + return models[0] +} + // ollamaModelDisplayName converts an Ollama model name (e.g. "llama3.2:3b") // into a human-friendly display name (e.g. "Llama3.2 3b"). func ollamaModelDisplayName(name string) string { diff --git a/internal/stack/stack.go b/internal/stack/stack.go index c492746..0c39971 100644 --- a/internal/stack/stack.go +++ b/internal/stack/stack.go @@ -10,10 +10,13 @@ import ( "strconv" "strings" + "github.com/ObolNetwork/obol-stack/internal/agent" "github.com/ObolNetwork/obol-stack/internal/config" "github.com/ObolNetwork/obol-stack/internal/dns" "github.com/ObolNetwork/obol-stack/internal/embed" + "github.com/ObolNetwork/obol-stack/internal/model" "github.com/ObolNetwork/obol-stack/internal/openclaw" + "github.com/ObolNetwork/obol-stack/internal/tunnel" "github.com/ObolNetwork/obol-stack/internal/ui" "github.com/ObolNetwork/obol-stack/internal/update" petname "github.com/dustinkirkland/golang-petname" @@ -294,7 +297,6 @@ func Up(cfg *config.Config, u *ui.UI) error { u.Blank() u.Bold("Stack started successfully.") u.Print("Visit http://obol.stack in your browser to get started.") - u.Print("Try setting up an agent with `obol agent init` next.") update.HintIfStale(cfg) return nil } @@ -431,6 +433,12 @@ func syncDefaults(cfg *config.Config, u *ui.UI, kubeconfigPath string, dataDir s u.Success("Default infrastructure deployed") + // Auto-configure LiteLLM with Ollama models if available. + // This ensures the inference path works out of the box when the user + // has Ollama running — no separate `obol model setup` step required. + // Non-fatal: the user can always run `obol model setup` later. + autoConfigureLLM(cfg, u) + // Deploy default OpenClaw instance (non-fatal on failure). // Not wrapped in RunWithSpinner because SetupDefault/Onboard produce their // own UI output (Info, Detail, Print) and run sub-spinners via u.Exec. @@ -443,9 +451,63 @@ func syncDefaults(cfg *config.Config, u *ui.UI, kubeconfigPath string, dataDir s u.Dim(" You can manually set up OpenClaw later with: obol openclaw onboard") } + // Deploy the obol-agent singleton (monetize reconciliation, heartbeat). + // Non-fatal: the user can always run `obol agent init` later. + u.Blank() + u.Info("Deploying obol-agent") + if err := agent.Init(cfg, u); err != nil { + u.Warnf("Failed to deploy obol-agent: %v", err) + u.Dim(" You can manually deploy later with: obol agent init") + } + + // Start the Cloudflare tunnel so the stack is publicly accessible. + // Non-fatal: the user can start it later with `obol tunnel restart`. + u.Blank() + u.Info("Starting Cloudflare tunnel") + if tunnelURL, err := tunnel.EnsureRunning(cfg, u); err != nil { + u.Warnf("Tunnel not started: %v", err) + u.Dim(" Start manually with: obol tunnel restart") + } else { + u.Successf("Tunnel active: %s", tunnelURL) + } + return nil } +// autoConfigureLLM detects the host Ollama and auto-configures LiteLLM with +// available models so that inference works out of the box. Skipped silently +// if Ollama is unreachable, has no models, or LiteLLM already has non-paid +// models configured. +func autoConfigureLLM(cfg *config.Config, u *ui.UI) { + ollamaModels, err := model.ListOllamaModels() + if err != nil || len(ollamaModels) == 0 { + // Ollama not running or no models — skip silently. + return + } + + // Check if LiteLLM already has real models (not just the paid/* catch-all). + if model.HasConfiguredModels(cfg) { + return + } + + u.Blank() + u.Infof("Ollama detected with %d model(s) — auto-configuring LiteLLM", len(ollamaModels)) + + var names []string + for _, m := range ollamaModels { + name := m.Name + if strings.HasSuffix(name, ":latest") { + name = strings.TrimSuffix(name, ":latest") + } + names = append(names, name) + } + + if err := model.ConfigureLiteLLM(cfg, u, "ollama", "", names); err != nil { + u.Warnf("Auto-configure LiteLLM failed: %v", err) + u.Dim(" Run 'obol model setup' to configure manually.") + } +} + // localImage describes a Docker image built from source in this repo. type localImage struct { tag string // e.g. "ghcr.io/obolnetwork/x402-verifier:latest" diff --git a/internal/tunnel/tunnel.go b/internal/tunnel/tunnel.go index 8d415d6..d82eac9 100644 --- a/internal/tunnel/tunnel.go +++ b/internal/tunnel/tunnel.go @@ -136,6 +136,74 @@ func GetTunnelURL(cfg *config.Config) (string, error) { return "", fmt.Errorf("tunnel URL not found in logs") } +// EnsureRunning scales the cloudflared deployment to 1 replica if it's at 0, +// waits for the pod to be ready, and returns the tunnel URL once available. +// If the tunnel is already running, it returns the current URL immediately. +func EnsureRunning(cfg *config.Config, u *ui.UI) (string, error) { + kubectlPath := filepath.Join(cfg.BinDir, "kubectl") + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + + if _, err := os.Stat(kubeconfigPath); os.IsNotExist(err) { + return "", fmt.Errorf("stack not running") + } + + // Check if already running. + if podStatus, err := getPodStatus(kubectlPath, kubeconfigPath); err == nil && podStatus == "running" { + if url, err := GetTunnelURL(cfg); err == nil { + return url, nil + } + } + + // Scale to 1 replica. + scaleCmd := exec.Command(kubectlPath, + "--kubeconfig", kubeconfigPath, + "scale", "deployment/cloudflared", + "-n", tunnelNamespace, + "--replicas=1", + ) + if err := scaleCmd.Run(); err != nil { + return "", fmt.Errorf("failed to scale cloudflared: %w", err) + } + + // Wait for rollout. + waitCmd := exec.Command(kubectlPath, + "--kubeconfig", kubeconfigPath, + "rollout", "status", "deployment/cloudflared", + "-n", tunnelNamespace, + "--timeout=30s", + ) + if err := u.Exec(ui.ExecConfig{ + Name: "Starting Cloudflare tunnel", + Cmd: waitCmd, + }); err != nil { + return "", fmt.Errorf("cloudflared rollout failed: %w", err) + } + + // Poll for tunnel URL (quick tunnels take a few seconds to register). + var tunnelURL string + for i := 0; i < 20; i++ { + time.Sleep(time.Second) + if url, err := GetTunnelURL(cfg); err == nil { + tunnelURL = url + break + } + } + + if tunnelURL == "" { + return "", fmt.Errorf("tunnel started but URL not available yet — run 'obol tunnel status' in a few seconds") + } + + // Inject into obol-agent. + if err := InjectBaseURL(cfg, tunnelURL); err == nil { + u.Dim("Agent base URL updated to " + tunnelURL) + } + if err := SyncTunnelConfigMap(cfg, tunnelURL); err != nil { + u.Dim("Could not sync tunnel URL to frontend ConfigMap: " + err.Error()) + } + + return tunnelURL, nil +} + // Restart restarts the cloudflared deployment. func Restart(cfg *config.Config, u *ui.UI) error { kubectlPath := filepath.Join(cfg.BinDir, "kubectl") diff --git a/internal/x402/bdd_integration_steps_test.go b/internal/x402/bdd_integration_steps_test.go index 5929a51..ffa50ff 100644 --- a/internal/x402/bdd_integration_steps_test.go +++ b/internal/x402/bdd_integration_steps_test.go @@ -332,7 +332,10 @@ func registerIntegrationSteps(ctx *godog.ScenarioContext, w *integrationWorld) { "--upstream", "litellm", "--port", "4000", "--namespace", serviceOfferNamespace, - "--health-path", "/health/readiness"); err != nil { + "--health-path", "/health/readiness", + "--register", + "--register-name", "BDD Test Inference", + "--register-description", "Integration test inference endpoint"); err != nil { return fmt.Errorf("obol sell http failed: %w", err) } @@ -535,6 +538,8 @@ func registerIntegrationSteps(ctx *godog.ScenarioContext, w *integrationWorld) { } return fmt.Errorf("no OASF service entry found in registration services") }) + + ctx.When(`^the agent probes the tunnel service endpoint$`, func() error { if w.discoveredEndpoint == "" { return fmt.Errorf("no service endpoint discovered") diff --git a/internal/x402/bdd_integration_test.go b/internal/x402/bdd_integration_test.go index 146518b..8a41ae3 100644 --- a/internal/x402/bdd_integration_test.go +++ b/internal/x402/bdd_integration_test.go @@ -177,7 +177,10 @@ func TestMain(m *testing.M) { "--upstream", "litellm", "--port", "4000", "--namespace", serviceOfferNamespace, - "--health-path", "/health/readiness"); err != nil { + "--health-path", "/health/readiness", + "--register", + "--register-name", "BDD Test Inference", + "--register-description", "Integration test inference endpoint"); err != nil { teardown(obolBin) log.Fatalf("obol sell http: %v", err) } @@ -301,9 +304,16 @@ func ensureExistingClusterBootstrap(obolBin, kubectlBin, kubeconfig string) erro return fmt.Errorf("obol-agent not ready: %w", err) } - _, err := kubectl.Output(kubectlBin, kubeconfig, - "get", "serviceoffers.obol.org", serviceOfferName, "-n", serviceOfferNamespace, "--no-headers") - if err != nil { + soOut, err := kubectl.Output(kubectlBin, kubeconfig, + "get", "serviceoffers.obol.org", serviceOfferName, "-n", serviceOfferNamespace, "-o", "jsonpath={.spec.registration.enabled}") + needsCreate := err != nil + if !needsCreate && soOut != "true" { + // ServiceOffer exists but registration not enabled — delete and recreate. + log.Printf(" Existing ServiceOffer %s/%s has no registration, recreating...", serviceOfferNamespace, serviceOfferName) + _ = runObol(obolBin, "sell", "delete", serviceOfferName, "-n", serviceOfferNamespace, "-f") + needsCreate = true + } + if needsCreate { log.Printf(" Creating ServiceOffer %s/%s on existing cluster...", serviceOfferNamespace, serviceOfferName) if err := runObol(obolBin, "sell", "http", serviceOfferName, "--wallet", serviceOfferPayTo, @@ -312,15 +322,20 @@ func ensureExistingClusterBootstrap(obolBin, kubectlBin, kubeconfig string) erro "--upstream", "litellm", "--port", "4000", "--namespace", serviceOfferNamespace, - "--health-path", "/health/readiness"); err != nil { + "--health-path", "/health/readiness", + "--register", + "--register-name", "BDD Test Inference", + "--register-description", "Integration test inference endpoint"); err != nil { return fmt.Errorf("obol sell http on existing cluster: %w", err) } } - if err := waitForServiceOfferReady(kubectlBin, kubeconfig, serviceOfferName, serviceOfferNamespace, 180*time.Second); err != nil { + // Wait up to 5min for Ready. The Registered stage may call real Base Sepolia + // via eRPC, which takes ~120s to fail when the wallet isn't funded. + if err := waitForServiceOfferReady(kubectlBin, kubeconfig, serviceOfferName, serviceOfferNamespace, 300*time.Second); err != nil { log.Println(" ServiceOffer not Ready on existing cluster, triggering manual reconciliation...") triggerReconciliation(kubectlBin, kubeconfig) - if err := waitForServiceOfferReady(kubectlBin, kubeconfig, serviceOfferName, serviceOfferNamespace, 120*time.Second); err != nil { + if err := waitForServiceOfferReady(kubectlBin, kubeconfig, serviceOfferName, serviceOfferNamespace, 180*time.Second); err != nil { return fmt.Errorf("existing-cluster ServiceOffer not Ready: %w", err) } }