From 7b088136b9280f06e4525e6393c68549b3befe0c Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 26 Feb 2026 01:17:22 +0000 Subject: [PATCH 001/135] wip: Raw structure of improvements to gun that run in production --- src/hb_http.erl | 2 +- src/hb_http_client.erl | 349 ++++++++++++++++++++++++++++------------- src/hb_http_server.erl | 1 + 3 files changed, 240 insertions(+), 112 deletions(-) diff --git a/src/hb_http.erl b/src/hb_http.erl index 799554835..18e2a890c 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -7,7 +7,7 @@ -export([start/0]). -export([get/2, get/3, post/3, post/4, request/2, request/4, request/5]). -export([message_to_request/2, reply/4, accept_to_codec/2]). --export([req_to_tabm_singleton/3]). +-export([req_to_tabm_singleton/3, response_status_to_atom/1]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index d04de1f1c..c511ca07d 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -5,10 +5,9 @@ -include("include/hb.hrl"). -export([start_link/1, init_prometheus/0, response_status_to_atom/1, request/2]). -export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]). +-export([setup_conn/1]). -record(state, { - pid_by_peer = #{}, - status_by_pid = #{}, opts = #{} }). @@ -17,10 +16,25 @@ -define(DEFAULT_KEEPALIVE_TIMEOUT, 60_000). -define(DEFAULT_CONNECT_TIMEOUT, 60_000). +%% Connection Pool +-define(CONNECTIONS_ETS, hb_http_client_connections). +-define(CONN_STATUS_ETS, hb_http_client_conn_status). +-define(CONN_COUNTER_ETS, hb_http_client_conn_counter). +-define(CONN_TERM, connection_pool_size). +-define(DEFAULT_CONN_POOL_READ_SIZE, 3). +-define(DEFAULT_CONN_POOL_WRITE_SIZE, 3). + + %%% ================================================================== %%% Public interface. %%% ================================================================== +%% @doc Use Opts to configure connection pool size. +setup_conn(Opts) -> + ConnPoolReadSize = hb_maps:get(conn_pool_read_size,Opts, ?DEFAULT_CONN_POOL_READ_SIZE), + ConnPoolWriteSize = hb_maps:get(conn_pool_write_size,Opts, ?DEFAULT_CONN_POOL_WRITE_SIZE), + persistent_term:put(?CONN_TERM, {ConnPoolReadSize, ConnPoolWriteSize}). + start_link(Opts) -> gen_server:start_link({local, ?MODULE}, ?MODULE, Opts, []). @@ -133,11 +147,11 @@ httpc_req(Args, Opts) -> end, ?event({http_client_outbound, Method, URL, Request}), HTTPCOpts = [{full_result, true}, {body_format, binary}], - StartTime = os:system_time(millisecond), + StartTime = os:system_time(native), case httpc:request(Method, Request, [], HTTPCOpts) of {ok, {{_, Status, _}, RawRespHeaders, RespBody}} -> download_metric(RespBody), - EndTime = os:system_time(millisecond), + EndTime = os:system_time(native), RespHeaders = [ {list_to_binary(Key), list_to_binary(Value)} @@ -164,11 +178,12 @@ gun_req(Args, Opts) -> gun_req(Args, ReestablishedConnection, Opts) -> StartTime = os:system_time(native), #{ peer := Peer, path := Path, method := Method } = Args, + ConnType = get_connection_type(Method), Response = - case catch gen_server:call(?MODULE, {get_connection, Args, Opts}, infinity) of + case get_connection(Peer, ConnType, Args, Opts) of {ok, PID} -> ar_rate_limiter:throttle(Peer, Path, Opts), - case do_gun_request(PID, Args, Opts) of + case do_gun_request(PID, Args, hb_opts:mimic_default_types(Opts, existing, Opts)) of {error, Error} when Error == {shutdown, normal}; Error == noproc -> case ReestablishedConnection of @@ -181,6 +196,7 @@ gun_req(Args, ReestablishedConnection, Opts) -> {'EXIT', _} -> {error, client_error}; Error -> + ?event(http_client, {gun_error, Error}), Error end, EndTime = os:system_time(native), @@ -201,6 +217,74 @@ gun_req(Args, ReestablishedConnection, Opts) -> end, Response. +%% @doc Determine the connection type based on the HTTP method. +%% Read operations (GET, HEAD) use the 'read' connection. +%% Write operations (POST, PUT, DELETE, etc.) use the 'write' connection. +get_connection_type(<<"GET">>) -> read; +get_connection_type(<<"get">>) -> read; +get_connection_type(<<"HEAD">>) -> read; +get_connection_type(<<"head">>) -> read; +get_connection_type(get) -> read; +get_connection_type(head) -> read; +get_connection_type(_) -> write. + +%% @doc Get the pool size for a connection type. +get_pool_size(read) -> + {ReadSize, _} = persistent_term:get(?CONN_TERM, {?DEFAULT_CONN_POOL_READ_SIZE, ?DEFAULT_CONN_POOL_WRITE_SIZE}), + ReadSize; +get_pool_size(write) -> + {_, WriteSize} = persistent_term:get(?CONN_TERM, {?DEFAULT_CONN_POOL_READ_SIZE, ?DEFAULT_CONN_POOL_WRITE_SIZE}), + WriteSize. + + +%% @doc Get the next connection index using round-robin selection. +%% Uses ets:update_counter for atomic increment. +get_next_conn_index(Peer, ConnType) -> + PoolSize = get_pool_size(ConnType), + CounterKey = {Peer, ConnType}, + %% Atomically increment and wrap around using update_counter + %% If key doesn't exist, it will be created with default 0 + try + Index = ets:update_counter(?CONN_COUNTER_ETS, CounterKey, {2, 1, PoolSize, 1}), + Index + catch + error:badarg -> + %% Key doesn't exist, initialize it + ets:insert_new(?CONN_COUNTER_ETS, {CounterKey, 1}), + 1 + end. + +%% @doc Get a connection for a peer+type, using ETS for fast lookup. +%% If no connection exists, it will be created via the gen_server. +%% Uses round-robin to distribute requests across the connection pool. +get_connection(Peer, ConnType, Args, Opts) -> + PoolSize = get_pool_size(ConnType), + ConnIndex = get_next_conn_index(Peer, ConnType), + ConnKey = {Peer, ConnType, ConnIndex}, + get_connection_by_key(ConnKey, PoolSize, Args, Opts, 0). + +%% @doc Try to get a connection by key, with fallback to other pool connections. +get_connection_by_key(ConnKey, PoolSize, Args, Opts, Attempts) when Attempts < PoolSize -> + case ets:lookup(?CONNECTIONS_ETS, ConnKey) of + [{ConnKey, PID}] -> + %% Found a connection, check if it's still alive and connected + case ets:lookup(?CONN_STATUS_ETS, PID) of + [{PID, connected, _MonitorRef, _ConnKey}] -> + {ok, PID}; + [{PID, {connecting, _}, _MonitorRef, _ConnKey}] -> + %% Connection is being established, wait for it via gen_server + catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, infinity); + [] -> + %% Status not found, connection might be dead, create new one + catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, infinity) + end; + [] -> + %% No connection, create one via gen_server + catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, infinity) + end; +get_connection_by_key(_ConnKey, _PoolSize, _Args, _Opts, _Attempts) -> + {error, no_available_connection}. + %% @doc Record the duration of the request in an async process. We write the %% data to prometheus if the application is enabled, as well as invoking the %% `http_monitor' if appropriate. @@ -210,13 +294,13 @@ record_duration(Details, Opts) -> % First, write to prometheus if it is enabled. Prometheus works % only with strings as lists, so we encode the data before granting % it. - GetFormat = - fun - (<<"request-category">>) -> - path_to_category(maps:get(<<"request-path">>, Details)); - (Key) -> - hb_util:list(maps:get(Key, Details)) - end, + GetFormat = fun + (<<"request-category">>) -> + path_to_category(maps:get(<<"request-path">>, Details)); + + (Key) -> + hb_util:list(maps:get(Key, Details)) + end, case application:get_application(prometheus) of undefined -> ok; _ -> @@ -284,6 +368,7 @@ maybe_invoke_monitor(Details, Opts) -> %%% ================================================================== init(Opts) -> + init_ets_tables(), case hb_opts:get(prometheus, not hb_features:test(), Opts) of true -> ?event({starting_prometheus_application, @@ -308,7 +393,40 @@ init(Opts) -> false -> {ok, #state{ opts = Opts }} end. +init_ets_tables() -> + init_ets_table(?CONNECTIONS_ETS), + init_ets_table(?CONN_STATUS_ETS), + init_counter_ets_table(?CONN_COUNTER_ETS). + +init_ets_table(Table) -> + case ets:whereis(Table) of + undefined -> + ets:new(Table, [ + named_table, + public, + set, + {read_concurrency, true}, + {write_concurrency, true} + ]); + _ -> + ok + end. + +init_counter_ets_table(Table) -> + case ets:whereis(Table) of + undefined -> + ets:new(Table, [ + named_table, + public, + set, + {write_concurrency, true} + ]); + _ -> + ok + end. + init_prometheus() -> + application:ensure_all_started([prometheus, prometheus_cowboy]), hb_prometheus:declare(counter, [ {name, gun_requests_total}, {labels, [http_method, status_class, category]}, @@ -349,47 +467,34 @@ init_prometheus() -> ]), ok. -handle_call({get_connection, Args, Opts}, From, - #state{ pid_by_peer = PIDPeer, status_by_pid = StatusByPID } = State) -> - Peer = hb_maps:get(peer, Args, undefined, Opts), - case hb_maps:get(Peer, PIDPeer, not_found, Opts) of - not_found -> - {ok, PID} = open_connection(Args, hb_maps:merge(State#state.opts, Opts, Opts)), - MonitorRef = monitor(process, PID), - PIDPeer2 = hb_maps:put(Peer, PID, PIDPeer, Opts), - StatusByPID2 = - hb_maps:put( - PID, - {{connecting, [{From, Args}]}, MonitorRef, Peer}, - StatusByPID, - Opts - ), - { - reply, - {ok, PID}, - State#state{ - pid_by_peer = PIDPeer2, - status_by_pid = StatusByPID2 - } - }; - PID -> - case hb_maps:get(PID, StatusByPID, undefined, Opts) of - {{connecting, PendingRequests}, MonitorRef, Peer} -> - StatusByPID2 = - hb_maps:put(PID, - { - {connecting, [{From, Args} | PendingRequests]}, - MonitorRef, - Peer - }, - StatusByPID, - Opts - ), - {noreply, State#state{ status_by_pid = StatusByPID2 }}; - {connected, _MonitorRef, Peer} -> - {reply, {ok, PID}, State} - end - end; +%% TODO: Do we need a genserver? Do we need to handle this in a call? +handle_call({get_connection, ConnKey, Args, Opts}, From, State) -> + ArgsOpts = maps:get(opts, Args, #{}), + HttpOpts = maps:get(opts, hb_opts:mimic_default_types(Opts, existing, #{deep => true}), #{}), + MergedHttpOpts = maps:merge(ArgsOpts, HttpOpts), + MergedArgs = Args#{opts => MergedHttpOpts}, + %% ConnKey = {Peer, ConnType, Index} where ConnType is 'read' or 'write' + %% and Index is 1..PoolSize for round-robin distribution + %% Double-check ETS to handle race conditions + case ets:lookup(?CONNECTIONS_ETS, ConnKey) of + [{ConnKey, PID}] -> + %% Connection exists, check status + case ets:lookup(?CONN_STATUS_ETS, PID) of + [{PID, connected, _MonitorRef, _ConnKey}] -> + {reply, {ok, PID}, State}; + [{PID, {connecting, PendingRequests}, MonitorRef, ConnKey}] -> + %% Add to pending requests list + ets:insert(?CONN_STATUS_ETS, {PID, {connecting, [{From, MergedArgs} | PendingRequests]}, MonitorRef, ConnKey}), + {noreply, State}; + [] -> + %% Status not found, PID is stale - remove and create new + ets:delete(?CONNECTIONS_ETS, ConnKey), + create_new_connection(ConnKey, MergedArgs, From, State) + end; + [] -> + %% No connection exists, create one + create_new_connection(ConnKey, MergedArgs, From, State) + end; handle_call(Request, _From, State) -> ?event(warning, {unhandled_call, {module, ?MODULE}, {request, Request}}), @@ -399,31 +504,33 @@ handle_cast(Cast, State) -> ?event(warning, {unhandled_cast, {module, ?MODULE}, {cast, Cast}}), {noreply, State}. -handle_info({gun_up, PID, _Protocol}, #state{ status_by_pid = StatusByPID } = State) -> - case hb_maps:get(PID, StatusByPID, not_found) of - not_found -> +handle_info({gun_up, PID, Protocol}, State) -> + case ets:lookup(?CONN_STATUS_ETS, PID) of + [] -> %% A connection timeout should have occurred. {noreply, State}; - {{connecting, PendingRequests}, MonitorRef, Peer} -> - [gen_server:reply(ReplyTo, {ok, PID}) || {ReplyTo, _} <- PendingRequests], - StatusByPID2 = hb_maps:put(PID, {connected, MonitorRef, Peer}, StatusByPID), - inc_prometheus_gauge(outbound_connections), - {noreply, State#state{ status_by_pid = StatusByPID2 }}; - {connected, _MonitorRef, Peer} -> + [{PID, {connecting, PendingRequests}, MonitorRef, ConnKey}] -> + ?event(http_client, {gun_up, {protocol, Protocol}, {conn_key, ConnKey}}), + [gen_server:reply(ReplyTo, {ok, PID}) || {ReplyTo, _} <- PendingRequests], + ets:insert(?CONN_STATUS_ETS, {PID, connected, MonitorRef, ConnKey}), + inc_prometheus_gauge(outbound_connections), + {noreply, State}; + [{PID, connected, _MonitorRef, ConnKey}] -> ?event(warning, - {gun_up_pid_already_exists, {peer, Peer}}), + {gun_up_pid_already_exists, {conn_key, ConnKey}}), {noreply, State} end; -handle_info({gun_error, PID, Reason}, - #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> - case hb_maps:get(PID, StatusByPID, not_found) of - not_found -> +%% TODO: gun_error and gun_down logic can be merged +handle_info({gun_error, PID, Reason}, State) -> + case ets:lookup(?CONN_STATUS_ETS, PID) of + [] -> ?event(warning, {gun_connection_error_with_unknown_pid}), {noreply, State}; - {Status, _MonitorRef, Peer} -> - PIDByPeer2 = hb_maps:remove(Peer, PIDByPeer), - StatusByPID2 = hb_maps:remove(PID, StatusByPID), + [{PID, Status, MonitorRef, ConnKey}] -> + ets:delete(?CONNECTIONS_ETS, ConnKey), + ets:delete(?CONN_STATUS_ETS, PID), + demonitor(MonitorRef, [flush]), Reason2 = case Reason of timeout -> @@ -441,8 +548,8 @@ handle_info({gun_error, PID, Reason}, ok end, gun:shutdown(PID), - ?event({connection_error, {reason, Reason}}), - {noreply, State#state{ status_by_pid = StatusByPID2, pid_by_peer = PIDByPeer2 }} + ?event(http_client, {connection_error, {conn_key, ConnKey}, {reason, Reason}}), + {noreply, State} end; handle_info({gun_down, PID, Protocol, Reason, _KilledStreams}, @@ -451,17 +558,18 @@ handle_info({gun_down, PID, Protocol, Reason, _KilledStreams}, not_found -> ?event(warning, {gun_connection_down_with_unknown_pid, {protocol, Protocol}}), - {noreply, State}; - {Status, _MonitorRef, Peer} -> - PIDByPeer2 = hb_maps:remove(Peer, PIDByPeer), - StatusByPID2 = hb_maps:remove(PID, StatusByPID), - Reason2 = - case Reason of - {Type, _} -> - Type; - _ -> - Reason - end, + {noreply, State}; + [{PID, Status, MonitorRef, ConnKey}] -> + ets:delete(?CONNECTIONS_ETS, ConnKey), + ets:delete(?CONN_STATUS_ETS, PID), + demonitor(MonitorRef, [flush]), + Reason2 = + case Reason of + {Type, _} -> + Type; + _ -> + Reason + end, case Status of {connecting, PendingRequests} -> reply_error(PendingRequests, Reason2); @@ -469,22 +577,18 @@ handle_info({gun_down, PID, Protocol, Reason, _KilledStreams}, dec_prometheus_gauge(outbound_connections), ok end, - {noreply, - State#state{ - status_by_pid = StatusByPID2, - pid_by_peer = PIDByPeer2 - } - } + gun:shutdown(PID), + ?event(http_outbound, {gun_shutdown_after_down, {conn_key, ConnKey}}), + {noreply, State} end; -handle_info({'DOWN', _Ref, process, PID, Reason}, - #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> - case hb_maps:get(PID, StatusByPID, not_found) of - not_found -> - {noreply, State}; - {Status, _MonitorRef, Peer} -> - PIDByPeer2 = hb_maps:remove(Peer, PIDByPeer), - StatusByPID2 = hb_maps:remove(PID, StatusByPID), +handle_info({'DOWN', _Ref, process, PID, Reason}, State) -> + case ets:lookup(?CONN_STATUS_ETS, PID) of + [] -> + {noreply, State}; + [{PID, Status, _MonitorRef, ConnKey}] -> + ets:delete(?CONNECTIONS_ETS, ConnKey), + ets:delete(?CONN_STATUS_ETS, PID), case Status of {connecting, PendingRequests} -> reply_error(PendingRequests, Reason); @@ -492,27 +596,40 @@ handle_info({'DOWN', _Ref, process, PID, Reason}, dec_prometheus_gauge(outbound_connections), ok end, - {noreply, - State#state{ - status_by_pid = StatusByPID2, - pid_by_peer = PIDByPeer2 - } - } - end; + {noreply, State} + end; handle_info(Message, State) -> ?event(warning, {unhandled_info, {module, ?MODULE}, {message, Message}}), {noreply, State}. -terminate(Reason, #state{ status_by_pid = StatusByPID }) -> +terminate(Reason, _State) -> ?event(info,{http_client_terminating, {reason, Reason}}), - hb_maps:map(fun(PID, _Status) -> gun:shutdown(PID) end, StatusByPID), + ets:foldl( + fun({PID, _Status, _MonitorRef, _ConnKey}, Acc) -> + gun:shutdown(PID), + Acc + end, + ok, + ?CONN_STATUS_ETS + ), ok. %%% ================================================================== %%% Private functions. %%% ================================================================== +%% @doc Create a new connection and store it in ETS. +create_new_connection(ConnKey, Args, From, State) -> + MergedOpts = hb_maps:merge(State#state.opts, hb_maps:get(opts, Args, #{}), #{}), + {ok, PID} = open_connection(Args, MergedOpts), + MonitorRef = monitor(process, PID), + %% Store connection in ETS + ets:insert(?CONNECTIONS_ETS, {ConnKey, PID}), + %% Store status with monitor ref and conn key + ets:insert(?CONN_STATUS_ETS, {PID, {connecting, [{From, Args}]}, MonitorRef, ConnKey}), + {reply, {ok, PID}, State}. + %% @doc Safe wrapper for prometheus_gauge:inc/2. inc_prometheus_gauge(Name) -> case application:get_application(prometheus) of @@ -574,6 +691,7 @@ open_connection(#{ peer := Peer }, Opts) -> GunOpts = case Proto = hb_opts:get(protocol, DefaultProto, Opts) of http3 -> BaseGunOpts#{protocols => [http3], transport => quic}; + http1 -> BaseGunOpts#{protocols => [http]}; _ -> BaseGunOpts end, ?event(http_outbound, @@ -682,9 +800,13 @@ do_gun_request(PID, Args, Opts) -> Ref = gun:request(PID, Method, Path, Headers, Body), ResponseArgs = #{ - pid => PID, stream_ref => Ref, - timer => Timer, limit => hb_maps:get(limit, Args, infinity, Opts), - counter => 0, acc => [], start => os:system_time(microsecond), + pid => PID, + stream_ref => Ref, + timer => Timer, + limit => hb_maps:get(limit, Args, infinity, Opts), + counter => 0, + acc => [], + start => os:system_time(microsecond), is_peer_request => hb_maps:get(is_peer_request, Args, true, Opts) }, Response = await_response(hb_maps:merge(Args, ResponseArgs, Opts), Opts), @@ -734,9 +856,14 @@ await_response(Args, Opts) -> }; {error, timeout} = Response -> record_response_status(Method, Response, Path), + ?event(http_outbound, {gun_cancel, {path, Path}}), gun:cancel(PID, Ref), log(warn, gun_await_process_down, Args, Response, Opts), Response; + {error,{connection_error,{stream_closed, Message}}} = Response -> + ?event(http_outbound, {gun_cancel, {path, Path}, {message, Message}}), + gun:cancel(PID, Ref), + Response; {error, Reason} = Response when is_tuple(Reason) -> record_response_status(Method, Response, Path), log(warn, gun_await_process_down, Args, Reason, Opts), @@ -789,7 +916,7 @@ upload_metric(_) -> % gun_requests_total metrics. get_status_class({ok, {{Status, _}, _, _, _, _}}) -> get_status_class(Status); -get_status_class({ok, Status, _RespHeaders, _Body}) -> +get_status_class({ok, Status, _RespondeHeaders, _Body}) -> get_status_class(Status); get_status_class({error, connection_closed}) -> <<"connection-closed">>; diff --git a/src/hb_http_server.erl b/src/hb_http_server.erl index ea949ebc0..592af7c86 100644 --- a/src/hb_http_server.erl +++ b/src/hb_http_server.erl @@ -36,6 +36,7 @@ start() -> hb_opts:default_message_with_env(), Loaded ), + hb_http_client:setup_conn(MergedConfig), %% Apply store defaults before starting store StoreOpts = hb_opts:get(store, no_store, MergedConfig), StoreDefaults = hb_opts:get(store_defaults, #{}, MergedConfig), From fd2f53c71cbc1aff764a70c67dbeee855de503d4 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 27 Feb 2026 00:13:06 +0000 Subject: [PATCH 002/135] impr: Code structure --- src/hb_http.erl | 2 +- src/hb_http_client.erl | 390 +++++++++++++++---------------- src/hb_http_conn_monitor.erl | 109 +++++++++ src/hb_opts.erl | 26 ++- src/include/hb_arweave_nodes.hrl | 63 ++--- src/include/hb_http_client.hrl | 17 ++ src/include/hb_opts.hrl | 2 + 7 files changed, 358 insertions(+), 251 deletions(-) create mode 100644 src/hb_http_conn_monitor.erl create mode 100644 src/include/hb_http_client.hrl create mode 100644 src/include/hb_opts.hrl diff --git a/src/hb_http.erl b/src/hb_http.erl index 18e2a890c..799554835 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -7,7 +7,7 @@ -export([start/0]). -export([get/2, get/3, post/3, post/4, request/2, request/4, request/5]). -export([message_to_request/2, reply/4, accept_to_codec/2]). --export([req_to_tabm_singleton/3, response_status_to_atom/1]). +-export([req_to_tabm_singleton/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index c511ca07d..791e71c26 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -3,28 +3,17 @@ -module(hb_http_client). -behaviour(gen_server). -include("include/hb.hrl"). --export([start_link/1, init_prometheus/0, response_status_to_atom/1, request/2]). --export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]). --export([setup_conn/1]). +-include("include/hb_opts.hrl"). +-include("include/hb_http_client.hrl"). +%% Public API +-export([request/2, response_status_to_atom/1, setup_conn/1]). +%% GenServer +-export([start_link/1, init/1, response_status_to_atom/1, request/2]). -record(state, { opts = #{} }). --define(DEFAULT_RETRIES, 0). --define(DEFAULT_RETRY_TIME, 1000). --define(DEFAULT_KEEPALIVE_TIMEOUT, 60_000). --define(DEFAULT_CONNECT_TIMEOUT, 60_000). - -%% Connection Pool --define(CONNECTIONS_ETS, hb_http_client_connections). --define(CONN_STATUS_ETS, hb_http_client_conn_status). --define(CONN_COUNTER_ETS, hb_http_client_conn_counter). --define(CONN_TERM, connection_pool_size). --define(DEFAULT_CONN_POOL_READ_SIZE, 3). --define(DEFAULT_CONN_POOL_WRITE_SIZE, 3). - - %%% ================================================================== %%% Public interface. %%% ================================================================== @@ -63,7 +52,7 @@ request(Args, RemainingRetries, Opts) -> end. do_request(Args, Opts) -> - case hb_opts:get(http_client, gun, Opts) of + case hb_opts:get(http_client, ?DEFAULT_HTTP_CLIENT, Opts) of gun -> gun_req(Args, Opts); httpc -> httpc_req(Args, Opts) end. @@ -217,9 +206,41 @@ gun_req(Args, ReestablishedConnection, Opts) -> end, Response. -%% @doc Determine the connection type based on the HTTP method. -%% Read operations (GET, HEAD) use the 'read' connection. -%% Write operations (POST, PUT, DELETE, etc.) use the 'write' connection. +%% Connection Pool Logic + +init_ets_tables() -> + init_ets_table(?CONNECTIONS_ETS), + init_ets_table(?CONN_STATUS_ETS), + init_counter_ets_table(?CONN_COUNTER_ETS). + +init_ets_table(Table) -> + case ets:whereis(Table) of + undefined -> + ets:new(Table, [ + named_table, + public, + set, + {read_concurrency, true}, + {write_concurrency, true} + ]); + _ -> + ok + end. + +init_counter_ets_table(Table) -> + case ets:whereis(Table) of + undefined -> + ets:new(Table, [ + named_table, + public, + set, + {write_concurrency, true} + ]); + _ -> + ok + end. + + get_connection_type(<<"GET">>) -> read; get_connection_type(<<"get">>) -> read; get_connection_type(<<"HEAD">>) -> read; @@ -236,7 +257,6 @@ get_pool_size(write) -> {_, WriteSize} = persistent_term:get(?CONN_TERM, {?DEFAULT_CONN_POOL_READ_SIZE, ?DEFAULT_CONN_POOL_WRITE_SIZE}), WriteSize. - %% @doc Get the next connection index using round-robin selection. %% Uses ets:update_counter for atomic increment. get_next_conn_index(Peer, ConnType) -> @@ -285,45 +305,6 @@ get_connection_by_key(ConnKey, PoolSize, Args, Opts, Attempts) when Attempts < P get_connection_by_key(_ConnKey, _PoolSize, _Args, _Opts, _Attempts) -> {error, no_available_connection}. -%% @doc Record the duration of the request in an async process. We write the -%% data to prometheus if the application is enabled, as well as invoking the -%% `http_monitor' if appropriate. -record_duration(Details, Opts) -> - spawn( - fun() -> - % First, write to prometheus if it is enabled. Prometheus works - % only with strings as lists, so we encode the data before granting - % it. - GetFormat = fun - (<<"request-category">>) -> - path_to_category(maps:get(<<"request-path">>, Details)); - - (Key) -> - hb_util:list(maps:get(Key, Details)) - end, - case application:get_application(prometheus) of - undefined -> ok; - _ -> - prometheus_histogram:observe( - http_request_duration_seconds, - lists:map( - GetFormat, - [ - <<"request-method">>, - <<"status-class">>, - <<"request-category">> - ] - ), - maps:get(<<"duration">>, Details) - ) - end, - maybe_invoke_monitor( - Details#{ <<"path">> => <<"duration">> }, - Opts - ) - end - ). - %% @doc Invoke the HTTP monitor message with AO-Core, if it is set in the %% node message key. We invoke the given message with the `body' set to a signed %% version of the details. This allows node operators to configure their machine @@ -393,81 +374,6 @@ init(Opts) -> false -> {ok, #state{ opts = Opts }} end. -init_ets_tables() -> - init_ets_table(?CONNECTIONS_ETS), - init_ets_table(?CONN_STATUS_ETS), - init_counter_ets_table(?CONN_COUNTER_ETS). - -init_ets_table(Table) -> - case ets:whereis(Table) of - undefined -> - ets:new(Table, [ - named_table, - public, - set, - {read_concurrency, true}, - {write_concurrency, true} - ]); - _ -> - ok - end. - -init_counter_ets_table(Table) -> - case ets:whereis(Table) of - undefined -> - ets:new(Table, [ - named_table, - public, - set, - {write_concurrency, true} - ]); - _ -> - ok - end. - -init_prometheus() -> - application:ensure_all_started([prometheus, prometheus_cowboy]), - hb_prometheus:declare(counter, [ - {name, gun_requests_total}, - {labels, [http_method, status_class, category]}, - { - help, - "The total number of GUN requests." - } - ]), - hb_prometheus:declare(gauge, [{name, outbound_connections}, - {help, "The current number of the open outbound network connections"}]), - hb_prometheus:declare(histogram, [ - {name, http_request_duration_seconds}, - {buckets, [0.01, 0.1, 0.5, 1, 5, 10, 30, 60]}, - {labels, [http_method, status_class, category]}, - { - help, - "The total duration of an hb_http_client:req call. This includes more than" - " just the GUN request itself (e.g. establishing a connection, " - "throttling, etc...)" - } - ]), - hb_prometheus:declare(histogram, [ - {name, http_client_get_chunk_duration_seconds}, - {buckets, [0.1, 1, 10, 60]}, - {labels, [status_class, peer]}, - { - help, - "The total duration of an HTTP GET chunk request made to a peer." - } - ]), - hb_prometheus:declare(counter, [ - {name, http_client_downloaded_bytes_total}, - {help, "The total amount of bytes requested via HTTP, per remote endpoint"} - ]), - hb_prometheus:declare(counter, [ - {name, http_client_uploaded_bytes_total}, - {help, "The total amount of bytes posted via HTTP, per remote endpoint"} - ]), - ok. - -%% TODO: Do we need a genserver? Do we need to handle this in a call? handle_call({get_connection, ConnKey, Args, Opts}, From, State) -> ArgsOpts = maps:get(opts, Args, #{}), HttpOpts = maps:get(opts, hb_opts:mimic_default_types(Opts, existing, #{deep => true}), #{}), @@ -521,7 +427,6 @@ handle_info({gun_up, PID, Protocol}, State) -> {noreply, State} end; -%% TODO: gun_error and gun_down logic can be merged handle_info({gun_error, PID, Reason}, State) -> case ets:lookup(?CONN_STATUS_ETS, PID) of [] -> @@ -586,9 +491,10 @@ handle_info({'DOWN', _Ref, process, PID, Reason}, State) -> case ets:lookup(?CONN_STATUS_ETS, PID) of [] -> {noreply, State}; - [{PID, Status, _MonitorRef, ConnKey}] -> + [{PID, Status, MonitorRef, ConnKey}] -> ets:delete(?CONNECTIONS_ETS, ConnKey), ets:delete(?CONN_STATUS_ETS, PID), + demonitor(MonitorRef, [flush]), case Status of {connecting, PendingRequests} -> reply_error(PendingRequests, Reason); @@ -606,8 +512,9 @@ handle_info(Message, State) -> terminate(Reason, _State) -> ?event(info,{http_client_terminating, {reason, Reason}}), ets:foldl( - fun({PID, _Status, _MonitorRef, _ConnKey}, Acc) -> + fun({PID, _Status, MonitorRef, _ConnKey}, Acc) -> gun:shutdown(PID), + demonitor(MonitorRef, [flush]), Acc end, ok, @@ -630,31 +537,6 @@ create_new_connection(ConnKey, Args, From, State) -> ets:insert(?CONN_STATUS_ETS, {PID, {connecting, [{From, Args}]}, MonitorRef, ConnKey}), {reply, {ok, PID}, State}. -%% @doc Safe wrapper for prometheus_gauge:inc/2. -inc_prometheus_gauge(Name) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - try prometheus_gauge:inc(Name) - catch _:_ -> - init_prometheus(), - prometheus_gauge:inc(Name) - end - end. - -%% @doc Safe wrapper for prometheus_gauge:dec/2. -dec_prometheus_gauge(Name) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> prometheus_gauge:dec(Name) - end. - -inc_prometheus_counter(Name, Labels, Value) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> prometheus_counter:inc(Name, Labels, Value) - end. - open_connection(#{ peer := Peer }, Opts) -> {Host, Port} = parse_peer(Peer, Opts), ?event(http_outbound, {parsed_peer, {peer, Peer}, {host, Host}, {port, Port}}), @@ -729,41 +611,6 @@ reply_error([PendingRequest | PendingRequests], Reason) -> gen_server:reply(ReplyTo, {error, Reason}), reply_error(PendingRequests, Reason). -record_response_status(Method, Response) -> - record_response_status(Method, Response, undefined). -record_response_status(Method, Response, Path) -> - inc_prometheus_counter(gun_requests_total, - [ - hb_util:list(method_to_bin(Method)), - hb_util:list(get_status_class(Response)), - hb_util:list(path_to_category(Path)) - ], - 1 - ). - -method_to_bin(get) -> - <<"GET">>; -method_to_bin(post) -> - <<"POST">>; -method_to_bin(put) -> - <<"PUT">>; -method_to_bin(head) -> - <<"HEAD">>; -method_to_bin(delete) -> - <<"DELETE">>; -method_to_bin(connect) -> - <<"CONNECT">>; -method_to_bin(options) -> - <<"OPTIONS">>; -method_to_bin(trace) -> - <<"TRACE">>; -method_to_bin(patch) -> - <<"PATCH">>; -method_to_bin(Method) when is_binary(Method) -> - Method; -method_to_bin(_) -> - <<"unknown">>. - do_gun_request(PID, Args, Opts) -> Timer = inet:start_timer( @@ -890,7 +737,127 @@ log(Type, Event, #{method := Method, peer := Peer, path := Path}, Reason, Opts) ), ok. -%% @doc Record instances of downloaded bytes from the remote server. +%% Metrics + +init_prometheus() -> + application:ensure_all_started([prometheus, prometheus_cowboy]), + prometheus_counter:new([ + {name, gun_requests_total}, + {labels, [http_method, status_class, category]}, + { + help, + "The total number of GUN requests." + } + ]), + prometheus_gauge:new([{name, outbound_connections}, + {help, "The current number of the open outbound network connections"}]), + prometheus_histogram:new([ + {name, http_request_duration_seconds}, + {buckets, [0.01, 0.1, 0.5, 1, 5, 10, 30, 60]}, + {labels, [http_method, status_class, category]}, + { + help, + "The total duration of an hb_http_client:req call. This includes more than" + " just the GUN request itself (e.g. establishing a connection, " + "throttling, etc...)" + } + ]), + prometheus_histogram:new([ + {name, http_client_get_chunk_duration_seconds}, + {buckets, [0.1, 1, 10, 60]}, + {labels, [status_class, peer]}, + { + help, + "The total duration of an HTTP GET chunk request made to a peer." + } + ]), + prometheus_counter:new([ + {name, http_client_downloaded_bytes_total}, + {help, "The total amount of bytes requested via HTTP, per remote endpoint"} + ]), + prometheus_counter:new([ + {name, http_client_uploaded_bytes_total}, + {help, "The total amount of bytes posted via HTTP, per remote endpoint"} + ]), + ?event(started), + ok. + +%% @doc Record the duration of the request in an async process. We write the +%% data to prometheus if the application is enabled, as well as invoking the +%% `http_monitor' if appropriate. +record_duration(Details, Opts) -> + spawn( + fun() -> + % First, write to prometheus if it is enabled. Prometheus works + % only with strings as lists, so we encode the data before granting + % it. + GetFormat = fun + (<<"request-category">>) -> + path_to_category(maps:get(<<"request-path">>, Details)); + + (Key) -> + hb_util:list(maps:get(Key, Details)) + end, + case application:get_application(prometheus) of + undefined -> ok; + _ -> + prometheus_histogram:observe( + http_request_duration_seconds, + lists:map( + GetFormat, + [ + <<"request-method">>, + <<"status-class">>, + <<"request-category">> + ] + ), + maps:get(<<"duration">>, Details) + ) + end, + maybe_invoke_monitor( + Details#{ <<"path">> => <<"duration">> }, + Opts + ) + end + ). + +record_response_status(Method, Response) -> + record_response_status(Method, Response, undefined). +record_response_status(Method, Response, Path) -> + inc_prometheus_counter(gun_requests_total, + [ + hb_util:list(method_to_bin(Method)), + hb_util:list(get_status_class(Response)), + hb_util:list(path_to_category(Path)) + ], + 1 + ). + +%% @doc Safe wrapper for prometheus_gauge:inc/2. +inc_prometheus_gauge(Name) -> + case application:get_application(prometheus) of + undefined -> ok; + _ -> + try prometheus_gauge:inc(Name) + catch _:_ -> + init_prometheus(), + prometheus_gauge:inc(Name) + end + end. + +%% @doc Safe wrapper for prometheus_gauge:dec/2. +dec_prometheus_gauge(Name) -> + case application:get_application(prometheus) of + undefined -> ok; + _ -> prometheus_gauge:dec(Name) + end. + +inc_prometheus_counter(Name, Labels, Value) -> + case application:get_application(prometheus) of + undefined -> ok; + _ -> prometheus_counter:inc(Name, Labels, Value) + end. + download_metric(Data) -> inc_prometheus_counter( http_client_downloaded_bytes_total, @@ -912,6 +879,29 @@ upload_metric(Body) when is_binary(Body) -> upload_metric(_) -> ok. +method_to_bin(get) -> + <<"GET">>; +method_to_bin(post) -> + <<"POST">>; +method_to_bin(put) -> + <<"PUT">>; +method_to_bin(head) -> + <<"HEAD">>; +method_to_bin(delete) -> + <<"DELETE">>; +method_to_bin(connect) -> + <<"CONNECT">>; +method_to_bin(options) -> + <<"OPTIONS">>; +method_to_bin(trace) -> + <<"TRACE">>; +method_to_bin(patch) -> + <<"PATCH">>; +method_to_bin(Method) when is_binary(Method) -> + Method; +method_to_bin(_) -> + <<"unknown">>. + % @doc Return the HTTP status class label for cowboy_requests_total and % gun_requests_total metrics. get_status_class({ok, {{Status, _}, _, _, _, _}}) -> diff --git a/src/hb_http_conn_monitor.erl b/src/hb_http_conn_monitor.erl new file mode 100644 index 000000000..86c05113a --- /dev/null +++ b/src/hb_http_conn_monitor.erl @@ -0,0 +1,109 @@ +%%% @doc A gen_server that monitors gun connection mailbox sizes and reports +%%% metrics to Prometheus. This provides visibility into connection health +%%% and potential backpressure issues. +-module(hb_http_conn_monitor). +-behaviour(gen_server). +-include("include/hb.hrl"). +-include("include/hb_http_client.hrl"). +-export([start_link/1]). +-export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]). + +-define(MONITORING_INTERVAL_MS, 5000). + +%%% ================================================================== +%%% Public interface. +%%% ================================================================== + +start_link(Opts) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, Opts, []). + +%%% ================================================================== +%%% gen_server callbacks. +%%% ================================================================== + +init(Opts) -> + case hb_opts:get(prometheus, not hb_features:test(), Opts) of + true -> + init_prometheus(), + erlang:send_after(?MONITORING_INTERVAL_MS, self(), conn_mailbox_monitoring); + false -> + no_op + end, + {ok, #{}}. + +handle_call(Request, _From, State) -> + ?event(warning, {unhandled_call, {module, ?MODULE}, {request, Request}}), + {reply, ok, State}. + +handle_cast(Cast, State) -> + ?event(warning, {unhandled_cast, {module, ?MODULE}, {cast, Cast}}), + {noreply, State}. + +handle_info(conn_mailbox_monitoring, State) -> + spawn(fun() -> + case ets:whereis(?CONNECTIONS_ETS) of + undefined -> + ok; + _ -> + ets:foldl( + fun({ConnKey, ConnPID}, Acc) -> + sample_conn_pid(ConnKey, ConnPID), + Acc + end, + ok, + ?CONNECTIONS_ETS + ) + end + end), + erlang:send_after(?MONITORING_INTERVAL_MS, self(), conn_mailbox_monitoring), + {noreply, State}; + +handle_info(Message, State) -> + ?event(warning, {unhandled_info, {module, ?MODULE}, {message, Message}}), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +%%% ================================================================== +%%% Private functions. +%%% ================================================================== + +init_prometheus() -> + application:ensure_all_started([prometheus]), + case application:get_application(prometheus) of + undefined -> + ok; + _ -> + try + prometheus_gauge:new([ + {name, gun_mailbox_size}, + {labels, [conn_id]}, + {help, "Gun connection mailbox size"} + ]), + ok + catch + error:{mf_already_exists, _, _} -> + %% Metric already registered, this is fine + ok + end + end. + +sample_conn_pid(ConnKey, ConnPID) -> + case process_info(ConnPID, message_queue_len) of + {message_queue_len, Len} -> + report(ConnKey, Len); + undefined -> + ok + end. + +report(ConnKey, Value) -> + ConnKeyString = conn_key_string(ConnKey), + prometheus_gauge:set( + gun_mailbox_size, + [ConnKeyString], + Value). + +conn_key_string({Peer, ConnType, Index}) -> + iolist_to_binary([hb_util:bin(Peer), "_", atom_to_binary(ConnType), "_", integer_to_binary(Index)]). + diff --git a/src/hb_opts.erl b/src/hb_opts.erl index ce8f64ad3..6a87516cd 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -18,6 +18,7 @@ -export([ensure_node_history/2]). -export([check_required_opts/2]). -include("include/hb.hrl"). +-include("include/hb_opts.hrl"). -include("include/hb_arweave_nodes.hrl"). %%% Environment variables that can be used to override the default message. @@ -63,6 +64,7 @@ <<"store-module">> => hb_store_lmdb }). -define(DEFAULT_GATEWAY, <<"https://arweave.net">>). +-define(DEFAULT_HTTP_OPTS, #{http_client => ?DEFAULT_HTTP_CLIENT, protocol => http2}). -define(ENV_KEYS, #{ priv_key_location => {"HB_KEY", "hyperbeam-key.json"}, @@ -136,7 +138,7 @@ default_message() -> initialized => true, %% What HTTP client should the node use? %% Options: gun, httpc - http_client => gun, + http_client => ?DEFAULT_HTTP_CLIENT, %% Scheduling mode: Determines when the SU should inform the recipient %% that an assignment has been scheduled for a message. %% Options: aggressive(!), local_confirmation, remote_confirmation, @@ -319,8 +321,7 @@ default_message() -> <<"path">> => <<"^/arweave/chunk">>, <<"method">> => <<"GET">> }, - <<"nodes">> => - ?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES, + <<"nodes">> => add_opts(?DATA_NODES ++ ?TIP_NODES), <<"strategy">> => <<"Shuffled-Range">>, <<"choose">> => length( @@ -338,8 +339,7 @@ default_message() -> <<"path">> => <<"^/arweave/chunk">>, <<"method">> => <<"POST">> }, - <<"nodes">> => - ?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES, + <<"nodes">> => add_opts(?DATA_NODES ++ ?TIP_NODES), <<"strategy">> => <<"Shuffled-Range">>, <<"choose">> => length( @@ -357,8 +357,7 @@ default_message() -> <<"path">> => <<"^/arweave/tx">>, <<"method">> => <<"POST">> }, - <<"nodes">> => - ?ARWEAVE_BOOTSTRAP_CHAIN_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES, + <<"nodes">> => add_opts(?CHAIN_NODES ++ ?TIP_NODES), <<"parallel">> => true, <<"responses">> => 3, <<"stop-after">> => false, @@ -378,7 +377,7 @@ default_message() -> %% the first 200. #{ <<"template">> => <<"^/arweave">>, - <<"nodes">> => ?ARWEAVE_BOOTSTRAP_CHAIN_NODES, + <<"nodes">> => add_opts(?CHAIN_NODES), <<"parallel">> => true, <<"stop-after">> => 1, <<"admissible-status">> => 200 @@ -862,6 +861,17 @@ ensure_node_history(Opts, RequiredOpts) -> {error, validation_failed} end. +%% @doc Util to add opts to nodes. +add_opts(Items) -> + add_opts(Items, ?DEFAULT_HTTP_OPTS). +add_opts(Items, Opts) -> + lists:map( + fun (Item) when is_map(Item) -> + Item#{<<"opts">> => Opts} + end, + Items + ). + %%% Tests -ifdef(TEST). diff --git a/src/include/hb_arweave_nodes.hrl b/src/include/hb_arweave_nodes.hrl index d7ea8e3bf..92bd000d0 100644 --- a/src/include/hb_arweave_nodes.hrl +++ b/src/include/hb_arweave_nodes.hrl @@ -6,16 +6,14 @@ <<"min">> => 0, <<"max">> => 57_600_000_000_000, <<"center">> => 28_800_000_000_000, - <<"with">> => <<"http://data-1.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-1.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 0, <<"max">> => 57_600_000_000_000, <<"center">> => 28_800_000_000_000, - <<"with">> => <<"http://data-13.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-13.arweave.xyz:1984">> }, %% Partitions 0-3 #{ @@ -23,8 +21,7 @@ <<"min">> => 0, <<"max">> => 14_400_000_000_000, <<"center">> => 7_200_000_000_000, - <<"with">> => <<"http://data-2.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-2.arweave.xyz:1984">> }, %% Partitions 4-7 #{ @@ -32,8 +29,7 @@ <<"min">> => 14_400_000_000_000, <<"max">> => 28_800_000_000_000, <<"center">> => 21_600_000_000_000, - <<"with">> => <<"http://data-3.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-3.arweave.xyz:1984">> }, %% Partitions 8-11 #{ @@ -41,8 +37,7 @@ <<"min">> => 28_800_000_000_000, <<"max">> => 43_200_000_000_000, <<"center">> => 36_000_000_000_000, - <<"with">> => <<"http://data-4.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-4.arweave.xyz:1984">> }, %% Partitions 12-15 #{ @@ -50,8 +45,7 @@ <<"min">> => 43_200_000_000_000, <<"max">> => 57_600_000_000_000, <<"center">> => 50_400_000_000_000, - <<"with">> => <<"http://data-5.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-5.arweave.xyz:1984">> }, %% Partitions 16-31 #{ @@ -59,32 +53,28 @@ <<"min">> => 57_600_000_000_000, <<"max">> => 115_200_000_000_000, <<"center">> => 86_400_000_000_000, - <<"with">> => <<"http://data-2.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-2.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 57_600_000_000_000, <<"max">> => 115_200_000_000_000, <<"center">> => 86_400_000_000_000, - <<"with">> => <<"http://data-3.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-3.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 57_600_000_000_000, <<"max">> => 115_200_000_000_000, <<"center">> => 86_400_000_000_000, - <<"with">> => <<"http://data-14.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-14.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 57_600_000_000_000, <<"max">> => 115_200_000_000_000, <<"center">> => 86_400_000_000_000, - <<"with">> => <<"http://data-15.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-15.arweave.xyz:1984">> }, %% Partitions 32-47 #{ @@ -92,32 +82,28 @@ <<"min">> => 115_200_000_000_000, <<"max">> => 172_800_000_000_000, <<"center">> => 144_000_000_000_000, - <<"with">> => <<"http://data-4.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-4.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 115_200_000_000_000, <<"max">> => 172_800_000_000_000, <<"center">> => 144_000_000_000_000, - <<"with">> => <<"http://data-5.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-5.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 115_200_000_000_000, <<"max">> => 172_800_000_000_000, <<"center">> => 144_000_000_000_000, - <<"with">> => <<"http://data-16.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-16.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 115_200_000_000_000, <<"max">> => 172_800_000_000_000, <<"center">> => 144_000_000_000_000, - <<"with">> => <<"http://data-17.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-17.arweave.xyz:1984">> } % Exclude these data nodes for now since their partitions are covered % by the tip nodes (and the tip nodes are faster to read from). @@ -190,40 +176,35 @@ <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-1.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-1.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-2.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-2.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-3.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-3.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-4.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-4.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-5.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-5.arweave.xyz:1984">> } ]). @@ -231,12 +212,10 @@ [ #{ <<"match">> => <<"^/arweave">>, - <<"with">> => <<"http://chain-1.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://chain-1.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, - <<"with">> => <<"http://chain-2.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://chain-2.arweave.xyz:1984">> } ]). diff --git a/src/include/hb_http_client.hrl b/src/include/hb_http_client.hrl new file mode 100644 index 000000000..0b2c0e9bf --- /dev/null +++ b/src/include/hb_http_client.hrl @@ -0,0 +1,17 @@ +-define(DEFAULT_RETRIES, 0). +-define(DEFAULT_RETRY_TIME, 1000). +-define(DEFAULT_KEEPALIVE_TIMEOUT, 60_000). +-define(DEFAULT_CONNECT_TIMEOUT, 60_000). + +%% Connection Pool +-define(DEFAULT_CONN_POOL_READ_SIZE, 3). +-define(DEFAULT_CONN_POOL_WRITE_SIZE, 3). +%% Keep track of available connections +-define(CONNECTIONS_ETS, hb_http_client_connections). +%% Used to keep status of the connection +-define(CONN_STATUS_ETS, hb_http_client_conn_status). +%% Used for Round-robin connection +-define(CONN_COUNTER_ETS, hb_http_client_conn_counter). +%% Used to load connection pool configuration +-define(CONN_TERM, connection_pool_size). + diff --git a/src/include/hb_opts.hrl b/src/include/hb_opts.hrl new file mode 100644 index 000000000..55460293f --- /dev/null +++ b/src/include/hb_opts.hrl @@ -0,0 +1,2 @@ +-define(DEFAULT_HTTP_CLIENT, httpc). + From 957eaa2da3ab894c0b1fdff3dc278ab5b5ab7938 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 1 Mar 2026 14:24:19 -0500 Subject: [PATCH 003/135] chore: update opts --- src/hb_opts.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 6a87516cd..cac3b072c 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -321,7 +321,7 @@ default_message() -> <<"path">> => <<"^/arweave/chunk">>, <<"method">> => <<"GET">> }, - <<"nodes">> => add_opts(?DATA_NODES ++ ?TIP_NODES), + <<"nodes">> => add_opts(?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES), <<"strategy">> => <<"Shuffled-Range">>, <<"choose">> => length( @@ -339,7 +339,7 @@ default_message() -> <<"path">> => <<"^/arweave/chunk">>, <<"method">> => <<"POST">> }, - <<"nodes">> => add_opts(?DATA_NODES ++ ?TIP_NODES), + <<"nodes">> => add_opts(?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES), <<"strategy">> => <<"Shuffled-Range">>, <<"choose">> => length( @@ -357,7 +357,7 @@ default_message() -> <<"path">> => <<"^/arweave/tx">>, <<"method">> => <<"POST">> }, - <<"nodes">> => add_opts(?CHAIN_NODES ++ ?TIP_NODES), + <<"nodes">> => add_opts(?ARWEAVE_BOOTSTRAP_CHAIN_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES), <<"parallel">> => true, <<"responses">> => 3, <<"stop-after">> => false, @@ -377,7 +377,7 @@ default_message() -> %% the first 200. #{ <<"template">> => <<"^/arweave">>, - <<"nodes">> => add_opts(?CHAIN_NODES), + <<"nodes">> => add_opts(?ARWEAVE_BOOTSTRAP_CHAIN_NODES), <<"parallel">> => true, <<"stop-after">> => 1, <<"admissible-status">> => 200 From d5c61353170b6241b31504e1b4284803e8cb0dac Mon Sep 17 00:00:00 2001 From: speeddragon Date: Sun, 1 Mar 2026 20:10:32 +0000 Subject: [PATCH 004/135] fix: compile --- src/hb_http_client.erl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index 791e71c26..8b5ada823 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -8,7 +8,8 @@ %% Public API -export([request/2, response_status_to_atom/1, setup_conn/1]). %% GenServer --export([start_link/1, init/1, response_status_to_atom/1, request/2]). +-export([start_link/1, init/1]). +-export([handle_cast/2, handle_call/3, handle_info/2, terminate/2]). -record(state, { opts = #{} @@ -22,6 +23,7 @@ setup_conn(Opts) -> ConnPoolReadSize = hb_maps:get(conn_pool_read_size,Opts, ?DEFAULT_CONN_POOL_READ_SIZE), ConnPoolWriteSize = hb_maps:get(conn_pool_write_size,Opts, ?DEFAULT_CONN_POOL_WRITE_SIZE), + ?event({conn, {pool_read_num, ConnPoolReadSize}, {pool_write_num, ConnPoolWriteSize}}), persistent_term:put(?CONN_TERM, {ConnPoolReadSize, ConnPoolWriteSize}). start_link(Opts) -> @@ -46,7 +48,7 @@ request(Args, RemainingRetries, Opts) -> StatusAtom = response_status_to_atom(Status), RetryResponses = hb_opts:get(http_retry_response, [], Opts), case lists:member(StatusAtom, RetryResponses) of - true -> maybe_retry(RemainingRetries, Args, Response, Opts); + true -> maybe_retry(RemainingRetries, Args, Response, Opts); false -> Response end end. @@ -457,10 +459,9 @@ handle_info({gun_error, PID, Reason}, State) -> {noreply, State} end; -handle_info({gun_down, PID, Protocol, Reason, _KilledStreams}, - #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> - case hb_maps:get(PID, StatusByPID, not_found) of - not_found -> +handle_info({gun_down, PID, Protocol, Reason, _KilledStreams}, State) -> + case ets:lookup(?CONN_STATUS_ETS, PID) of + [] -> ?event(warning, {gun_connection_down_with_unknown_pid, {protocol, Protocol}}), {noreply, State}; From e2146677a2d3ebe15f69d8cc9cf8225fe9c0a687 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Sun, 1 Mar 2026 23:03:54 +0000 Subject: [PATCH 005/135] impr: Change default to gun, add conn monitoring --- src/hb_sup.erl | 6 +++++- src/include/hb_opts.hrl | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/hb_sup.erl b/src/hb_sup.erl index 28b0176eb..aaea1c38b 100644 --- a/src/hb_sup.erl +++ b/src/hb_sup.erl @@ -32,7 +32,11 @@ init(Opts) -> type => worker, modules => [hb_http_client] }, - {ok, {SupFlags, [GunChild | StoreChildren]}}. + HttpMailBoxMonitoringChild = #{ + id => hb_http_conn_monitor, + start => {hb_http_conn_monitor, start_link, [Opts]} + }, + {ok, {SupFlags, [HttpMailBoxMonitoringChild | [GunChild | StoreChildren]]}}. %% @doc Generate a child spec for stores in the given Opts. store_children(Store) when not is_list(Store) -> diff --git a/src/include/hb_opts.hrl b/src/include/hb_opts.hrl index 55460293f..a0eb30283 100644 --- a/src/include/hb_opts.hrl +++ b/src/include/hb_opts.hrl @@ -1,2 +1,2 @@ --define(DEFAULT_HTTP_CLIENT, httpc). +-define(DEFAULT_HTTP_CLIENT, gun). From b3cc164d7d2bdf48933ca0d1a413e11f0d52187f Mon Sep 17 00:00:00 2001 From: speeddragon Date: Mon, 2 Mar 2026 14:48:25 +0000 Subject: [PATCH 006/135] impr: Force HTTP2 --- src/hb_http_client.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index 8b5ada823..ae023d5e6 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -574,8 +574,8 @@ open_connection(#{ peer := Peer }, Opts) -> GunOpts = case Proto = hb_opts:get(protocol, DefaultProto, Opts) of http3 -> BaseGunOpts#{protocols => [http3], transport => quic}; - http1 -> BaseGunOpts#{protocols => [http]}; - _ -> BaseGunOpts + http2 -> BaseGunOpts#{protocols => [http2]}; + http1 -> BaseGunOpts#{protocols => [http]} end, ?event(http_outbound, {gun_open, From 50e78f5830db29aac5f8a051afa59ff5e1f57f6a Mon Sep 17 00:00:00 2001 From: speeddragon Date: Mon, 2 Mar 2026 15:24:04 +0000 Subject: [PATCH 007/135] impr: Move some event topics in critical path to prefix debug_ --- src/dev_message.erl | 4 +- src/hb_ao.erl | 94 +++++++++++++++++++------------------- src/hb_ao_device.erl | 4 +- src/hb_ao_test_vectors.erl | 4 +- src/hb_cache.erl | 2 +- src/hb_event.erl | 2 +- src/hb_http.erl | 42 ++++++++--------- src/hb_http_client.erl | 12 ++--- src/hb_http_multi.erl | 6 +-- src/hb_message.erl | 4 +- 10 files changed, 87 insertions(+), 87 deletions(-) diff --git a/src/dev_message.erl b/src/dev_message.erl index 4cd72ee8b..857845e8e 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -635,7 +635,7 @@ set(Base, NewValuesMsg, Opts) -> ), % Base message with keys-to-unset removed BaseValues = hb_maps:without(UnsetKeys, Base, Opts), - ?event(message_set, + ?event(debug_message_set, {performing_set, {conflicting_keys, ConflictingKeys}, {keys_to_unset, UnsetKeys}, @@ -979,4 +979,4 @@ test_verify(KeyType) -> #{ <<"path">> => <<"verify">>, <<"body">> => Signed }, #{ hashpath => ignore } ) - ). + ). \ No newline at end of file diff --git a/src/hb_ao.erl b/src/hb_ao.erl index f26e29f28..8ab031b45 100644 --- a/src/hb_ao.erl +++ b/src/hb_ao.erl @@ -142,7 +142,7 @@ resolve(Base, Req, Opts) -> {stage, 1, prepare_multimessage_resolution, {path_parts, PathParts}} ), MessagesToExec = [ Req#{ <<"path">> => Path } || Path <- PathParts ], - ?event(ao_core, + ?event(debug_ao_core, {stage, 1, prepare_multimessage_resolution, @@ -164,7 +164,7 @@ resolve_many([ID], Opts) when ?IS_ID(ID) -> % 2. The main AO-Core logic looks for linkages between message input % pairs and outputs. With only a single ID, there is not a valid pairing % to use in looking up a cached result. - ?event(ao_core, {stage, na, resolve_directly_to_id, ID, {opts, Opts}}, Opts), + ?event(debug_ao_core, {stage, na, resolve_directly_to_id, ID, {opts, Opts}}, Opts), try {ok, ensure_message_loaded(ID, Opts)} catch _:_:_ -> {error, not_found} end; @@ -190,20 +190,20 @@ resolve_many({as, DevID, Msg}, Opts) -> resolve_many([{resolve, Subres}], Opts) -> resolve_many(Subres, Opts); resolve_many(MsgList, Opts) -> - ?event(ao_core, {resolve_many, MsgList}, Opts), + ?event(debug_ao_core, {resolve_many, MsgList}, Opts), Res = do_resolve_many(MsgList, Opts), - ?event(ao_core, {resolve_many_complete, {res, Res}, {reqs, MsgList}}, Opts), + ?event(debug_ao_core, {resolve_many_complete, {res, Res}, {reqs, MsgList}}, Opts), Res. do_resolve_many([], _Opts) -> {failure, <<"Attempted to resolve an empty message sequence.">>}; do_resolve_many([Res], Opts) -> - ?event(ao_core, {stage, 11, resolve_complete, Res}), + ?event(debug_ao_core, {stage, 11, resolve_complete, Res}), hb_cache:ensure_loaded(maybe_force_message(Res, Opts), Opts); do_resolve_many([Base, Req | MsgList], Opts) -> - ?event(ao_core, {stage, 0, resolve_many, {base, Base}, {req, Req}}), + ?event(debug_ao_core, {stage, 0, resolve_many, {base, Base}, {req, Req}}), case resolve_stage(1, Base, Req, Opts) of {ok, Res} -> - ?event(ao_core, + ?event(debug_ao_core, { stage, 13, @@ -216,19 +216,19 @@ do_resolve_many([Base, Req | MsgList], Opts) -> do_resolve_many([Res | MsgList], Opts); Res -> % The result is not a resolvable message. Return it. - ?event(ao_core, {stage, 13, resolve_many_terminating_early, Res}), + ?event(debug_ao_core, {stage, 13, resolve_many_terminating_early, Res}), maybe_force_message(Res, Opts) end. resolve_stage(1, Link, Req, Opts) when ?IS_LINK(Link) -> % If the first message is a link, we should load the message and % continue with the resolution. - ?event(ao_core, {stage, 1, resolve_base_link, {link, Link}}, Opts), + ?event(debug_ao_core, {stage, 1, resolve_base_link, {link, Link}}, Opts), resolve_stage(1, hb_cache:ensure_loaded(Link, Opts), Req, Opts); resolve_stage(1, Base, Link, Opts) when ?IS_LINK(Link) -> % If the second message is a link, we should load the message and % continue with the resolution. - ?event(ao_core, {stage, 1, resolve_req_link, {link, Link}}, Opts), + ?event(debug_ao_core, {stage, 1, resolve_req_link, {link, Link}}, Opts), resolve_stage(1, Base, hb_cache:ensure_loaded(Link, Opts), Opts); resolve_stage(1, {as, DevID, Ref}, Req, Opts) when ?IS_ID(Ref) orelse ?IS_LINK(Ref) -> % Normalize `as' requests with a raw ID or link as the path. Links will be @@ -237,27 +237,27 @@ resolve_stage(1, {as, DevID, Ref}, Req, Opts) when ?IS_ID(Ref) orelse ?IS_LINK(R resolve_stage(1, {as, DevID, Link}, Req, Opts) when ?IS_LINK(Link) -> % If the first message is an `as' with a link, we should load the message and % continue with the resolution. - ?event(ao_core, {stage, 1, resolve_base_as_link, {link, Link}}, Opts), + ?event(debug_ao_core, {stage, 1, resolve_base_as_link, {link, Link}}, Opts), resolve_stage(1, {as, DevID, hb_cache:ensure_loaded(Link, Opts)}, Req, Opts); resolve_stage(1, {as, DevID, Raw = #{ <<"path">> := ID }}, Req, Opts) when ?IS_ID(ID) -> % If the first message is an `as' with an ID, we should load the message and % apply the non-path elements of the sub-request to it. - ?event(ao_core, {stage, 1, subresolving_with_load, {dev, DevID}, {id, ID}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolving_with_load, {dev, DevID}, {id, ID}}, Opts), RemBase = hb_maps:without([<<"path">>], Raw, Opts), - ?event(subresolution, {loading_message, {id, ID}, {params, RemBase}}, Opts), + ?event(debug_subresolution, {loading_message, {id, ID}, {params, RemBase}}, Opts), Baseb = ensure_message_loaded(ID, Opts), - ?event(subresolution, {loaded_message, {msg, Baseb}}, Opts), + ?event(debug_subresolution, {loaded_message, {msg, Baseb}}, Opts), Basec = hb_maps:merge(Baseb, RemBase, Opts), - ?event(subresolution, {merged_message, {msg, Basec}}, Opts), + ?event(debug_subresolution, {merged_message, {msg, Basec}}, Opts), Based = set(Basec, <<"device">>, DevID, Opts), - ?event(subresolution, {loaded_parameterized_message, {msg, Based}}, Opts), + ?event(debug_subresolution, {loaded_parameterized_message, {msg, Based}}, Opts), resolve_stage(1, Based, Req, Opts); resolve_stage(1, Raw = {as, DevID, SubReq}, Req, Opts) -> % Set the device of the message to the specified one and resolve the sub-path. % As this is the first message, we will then continue to execute the request % on the result. - ?event(ao_core, {stage, 1, subresolving_base, {dev, DevID}, {subreq, SubReq}}, Opts), - ?event(subresolution, {as, {dev, DevID}, {subreq, SubReq}, {req, Req}}), + ?event(debug_ao_core, {stage, 1, subresolving_base, {dev, DevID}, {subreq, SubReq}}, Opts), + ?event(debug_subresolution, {as, {dev, DevID}, {subreq, SubReq}, {req, Req}}), case subresolve(SubReq, DevID, SubReq, Opts) of {ok, SubRes} -> % The subresolution has returned a new message. Continue with it. @@ -276,7 +276,7 @@ resolve_stage(1, RawBase, ReqOuter = #{ <<"path">> := {as, DevID, ReqInner} }, O % Set the device to the specified `DevID' and resolve the message. Merging % the `ReqInner' into the `ReqOuter' message first. We return the result % of the sub-resolution directly. - ?event(ao_core, {stage, 1, subresolving_from_request, {dev, DevID}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolving_from_request, {dev, DevID}}, Opts), LoadedInner = ensure_message_loaded(ReqInner, Opts), Req = hb_maps:merge( @@ -296,17 +296,17 @@ resolve_stage(1, RawBase, ReqOuter = #{ <<"path">> := {as, DevID, ReqInner} }, O resolve_stage(1, {resolve, Subres}, Req, Opts) -> % If the first message is a `{resolve, Subres}' tuple, we should execute it % directly, then apply the request to the result. - ?event(ao_core, {stage, 1, subresolving_base_message, {subres, Subres}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolving_base_message, {subres, Subres}}, Opts), % Unlike the `request' case for pre-subresolutions, we do not need to unset % the `force_message' option, because the result should be a message, anyway. % If it is not, it is more helpful to have the message placed into the `body' % of a result, which can then be executed upon. case resolve_many(Subres, Opts) of {ok, Base} -> - ?event(ao_core, {stage, 1, subresolve_success, {new_base, Base}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolve_success, {new_base, Base}}, Opts), resolve_stage(1, Base, Req, Opts); OtherRes -> - ?event(ao_core, + ?event(debug_ao_core, {stage, 1, subresolve_failed, @@ -321,7 +321,7 @@ resolve_stage(1, Base, {resolve, Subres}, Opts) -> % execute the subresolution directly to gain the underlying `Req' for % our execution. We assume that the subresolution is already in a normalized, % executable form, so we pass it to `resolve_many' for execution. - ?event(ao_core, {stage, 1, subresolving_request_message, {subres, Subres}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolving_request_message, {subres, Subres}}, Opts), % We make sure to unset the `force_message' option so that if the subresolution % returns a literal, the rest of `resolve' will normalize it to a path. case resolve_many(Subres, maps:without([force_message], Opts)) of @@ -348,31 +348,31 @@ resolve_stage(1, Base, {resolve, Subres}, Opts) -> end; resolve_stage(1, Base, Req, Opts) when is_list(Base) -> % Normalize lists to numbered maps (base=1) if necessary. - ?event(ao_core, {stage, 1, list_normalize}, Opts), + ?event(debug_ao_core, {stage, 1, list_normalize}, Opts), resolve_stage(1, normalize_keys(Base, Opts), Req, Opts ); resolve_stage(1, Base, NonMapReq, Opts) when not is_map(NonMapReq) -> - ?event(ao_core, {stage, 1, path_normalize}), + ?event(debug_ao_core, {stage, 1, path_normalize}), resolve_stage(1, Base, #{ <<"path">> => NonMapReq }, Opts); resolve_stage(1, RawBase, RawReq, Opts) -> % Normalize the path to a private key containing the list of remaining % keys to resolve. - ?event(ao_core, {stage, 1, normalize}, Opts), + ?event(debug_ao_core, {stage, 1, normalize}, Opts), Base = normalize_keys(RawBase, Opts), Req = normalize_keys(RawReq, Opts), resolve_stage(2, Base, Req, Opts); resolve_stage(2, Base, Req, Opts) -> - ?event(ao_core, {stage, 2, cache_lookup}, Opts), + ?event(debug_ao_core, {stage, 2, cache_lookup}, Opts), % Lookup request in the cache. If we find a result, return it. % If we do not find a result, we continue to the next stage, % unless the cache lookup returns `halt' (the user has requested that we % only return a result if it is already in the cache). case hb_cache_control:maybe_lookup(Base, Req, Opts) of {ok, Res} -> - ?event(ao_core, {stage, 2, cache_hit, {res, Res}, {opts, Opts}}, Opts), + ?event(debug_ao_core, {stage, 2, cache_hit, {res, Res}, {opts, Opts}}, Opts), {ok, Res}; {continue, NewBase, NewReq} -> resolve_stage(3, NewBase, NewReq, Opts); @@ -381,10 +381,10 @@ resolve_stage(2, Base, Req, Opts) -> resolve_stage(3, Base, Req, Opts) when not is_map(Base) or not is_map(Req) -> % Validation check: If the messages are not maps, we cannot find a key % in them, so return not_found. - ?event(ao_core, {stage, 3, validation_check_type_error}, Opts), + ?event(debug_ao_core, {stage, 3, validation_check_type_error}, Opts), {error, not_found}; resolve_stage(3, Base, Req, Opts) -> - ?event(ao_core, {stage, 3, validation_check}, Opts), + ?event(debug_ao_core, {stage, 3, validation_check}, Opts), % Validation checks: If `paranoid_message_verification' is enabled, we should % verify the base and request messages prior to execution. hb_message:paranoid_verify( @@ -398,7 +398,7 @@ resolve_stage(3, Base, Req, Opts) -> ), resolve_stage(4, Base, Req, Opts); resolve_stage(4, Base, Req, Opts) -> - ?event(ao_core, {stage, 4, persistent_resolver_lookup}, Opts), + ?event(debug_ao_core, {stage, 4, persistent_resolver_lookup}, Opts), % Persistent-resolver lookup: Search for local (or Distributed % Erlang cluster) processes that are already performing the execution. % Before we search for a live executor, we check if the device specifies @@ -461,7 +461,7 @@ resolve_stage(4, Base, Req, Opts) -> end end. resolve_stage(5, Base, Req, ExecName, Opts) -> - ?event(ao_core, {stage, 5, device_lookup}, Opts), + ?event(debug_ao_core, {stage, 5, device_lookup}, Opts), % Device lookup: Find the Erlang function that should be utilized to % execute Req on Base. {ResolvedFunc, NewOpts} = @@ -528,7 +528,7 @@ resolve_stage(5, Base, Req, ExecName, Opts) -> end, resolve_stage(6, ResolvedFunc, Base, Req, ExecName, NewOpts). resolve_stage(6, Func, Base, Req, ExecName, Opts) -> - ?event(ao_core, {stage, 6, ExecName, execution}, Opts), + ?event(debug_ao_core, {stage, 6, ExecName, execution}, Opts), % Execution. ExecOpts = execution_opts(Opts), Args = @@ -542,7 +542,7 @@ resolve_stage(6, Func, Base, Req, ExecName, Opts) -> TruncatedArgs = hb_ao_device:truncate_args(Func, Args), MsgRes = maybe_profiled_apply(Func, TruncatedArgs, Base, Req, Opts), ?event( - ao_result, + debug_ao_result, { ao_result, {exec_name, ExecName}, @@ -597,7 +597,7 @@ resolve_stage(6, Func, Base, Req, ExecName, Opts) -> ), resolve_stage(7, Base, Req, Res, ExecName, Opts); resolve_stage(7, Base, Req, {St, Res}, ExecName, Opts = #{ on := On = #{ <<"step">> := _ }}) -> - ?event(ao_core, {stage, 7, ExecName, executing_step_hook, {on, On}}, Opts), + ?event(debug_ao_core, {stage, 7, ExecName, executing_step_hook, {on, On}}, Opts), % If the `step' hook is defined, we execute it. Note: This function clause % matches directly on the `on' key of the `Opts' map. This is in order to % remove the expensive lookup check that would otherwise be performed on every @@ -623,18 +623,18 @@ resolve_stage(7, Base, Req, {St, Res}, ExecName, Opts = #{ on := On = #{ <<"step Error end; resolve_stage(7, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 7, ExecName, no_step_hook}, Opts), + ?event(debug_ao_core, {stage, 7, ExecName, no_step_hook}, Opts), resolve_stage(8, Base, Req, Res, ExecName, Opts); resolve_stage(8, Base, Req, {ok, {resolve, Sublist}}, ExecName, Opts) -> - ?event(ao_core, {stage, 8, ExecName, subresolve_result}, Opts), + ?event(debug_ao_core, {stage, 8, ExecName, subresolve_result}, Opts), % If the result is a `{resolve, Sublist}' tuple, we need to execute it % as a sub-resolution. resolve_stage(9, Base, Req, resolve_many(Sublist, Opts), ExecName, Opts); resolve_stage(8, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), + ?event(debug_ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), resolve_stage(9, Base, Req, Res, ExecName, Opts); resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> - ?event(ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), + ?event(debug_ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), % Cryptographic linking. Now that we have generated the result, we % need to cryptographically link the output to its input via a hashpath. resolve_stage(10, Base, Req, @@ -663,7 +663,7 @@ resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> Opts ); resolve_stage(9, Base, Req, {Status, Res}, ExecName, Opts) when is_map(Res) -> - ?event(ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), + ?event(debug_ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), ?event(hashpath, {resetting_hashpath_res, {base, Base}, {req, Req}, {opts, Opts}}), % Skip cryptographic linking and reset the hashpath if the result is abnormal. Priv = hb_private:from_message(Res), @@ -672,27 +672,27 @@ resolve_stage(9, Base, Req, {Status, Res}, ExecName, Opts) when is_map(Res) -> {Status, Res#{ <<"priv">> => maps:without([<<"hashpath">>], Priv) }}, ExecName, Opts); resolve_stage(9, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), + ?event(debug_ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), % Skip cryptographic linking and continue if we don't have a map that can have % a hashpath at all. resolve_stage(10, Base, Req, Res, ExecName, Opts); resolve_stage(10, Base, Req, {ok, Res}, ExecName, Opts) -> - ?event(ao_core, {stage, 10, ExecName, result_caching}, Opts), + ?event(debug_ao_core, {stage, 10, ExecName, result_caching}, Opts), % Result caching: Optionally, cache the result of the computation locally. hb_cache_control:maybe_store(Base, Req, Res, Opts), resolve_stage(11, Base, Req, {ok, Res}, ExecName, Opts); resolve_stage(10, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), + ?event(debug_ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), % Skip result caching if the result is abnormal. resolve_stage(11, Base, Req, Res, ExecName, Opts); resolve_stage(11, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 11, ExecName}, Opts), + ?event(debug_ao_core, {stage, 11, ExecName}, Opts), % Notify processes that requested the resolution while we were executing and % unregister ourselves from the group. hb_persistent:unregister_notify(ExecName, Req, Res, Opts), resolve_stage(12, Base, Req, Res, ExecName, Opts); resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> - ?event(ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), + ?event(debug_ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), % Check if we should fork out a new worker process for the current execution case {is_map(Res), hb_opts:get(spawn_worker, false, Opts#{ prefer => local })} of {A, B} when (A == false) or (B == false) -> @@ -704,7 +704,7 @@ resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> Res end; resolve_stage(12, _Base, _Req, OtherRes, ExecName, Opts) -> - ?event(ao_core, {stage, 12, ExecName, abnormal_status_skip_spawning}, Opts), + ?event(debug_ao_core, {stage, 12, ExecName, abnormal_status_skip_spawning}, Opts), OtherRes. %% @doc Execute a sub-resolution. @@ -880,7 +880,7 @@ error_infinite(Base, Req, Opts) -> error_execution(ExecGroup, Req, Whence, {Class, Exception, Stacktrace}, Opts) -> Error = {error, Whence, {Class, Exception, Stacktrace}}, hb_persistent:unregister_notify(ExecGroup, Req, Error, Opts), - ?event(ao_core, {handle_error, Error, {opts, Opts}}, Opts), + ?event(debug_ao_core, {handle_error, Error, {opts, Opts}}, Opts), case hb_opts:get(error_strategy, throw, Opts) of throw -> erlang:raise(Class, Exception, Stacktrace); _ -> Error diff --git a/src/hb_ao_device.erl b/src/hb_ao_device.erl index 22ed65f5c..ffbf5b027 100644 --- a/src/hb_ao_device.erl +++ b/src/hb_ao_device.erl @@ -439,11 +439,11 @@ do_is_direct_key_access(error, Key, Opts) -> do_is_direct_key_access(<<"message@1.0">>, Key, _Opts) -> not lists:member(Key, ?MESSAGE_KEYS); do_is_direct_key_access(Dev, NormKey, Opts) -> - ?event(read_cached, {calculating_info, {device, Dev}}), + ?event(debug_read_cached, {calculating_info, {device, Dev}}), case info(#{ <<"device">> => Dev}, Opts) of Info = #{ exports := Exports } when not is_map_key(handler, Info) andalso not is_map_key(default, Info) -> - ?event(read_cached, + ?event(debug_read_cached, {exports, {device, Dev}, {key, NormKey}, diff --git a/src/hb_ao_test_vectors.erl b/src/hb_ao_test_vectors.erl index a9677790a..886f1ceee 100644 --- a/src/hb_ao_test_vectors.erl +++ b/src/hb_ao_test_vectors.erl @@ -969,7 +969,7 @@ step_hook_test(InitOpts) -> #{ <<"step">> => fun(_, Req, _) -> - ?event(ao_core, {step_hook, {self(), Ref}}), + ?event(debug_ao_core, {step_hook, {self(), Ref}}), Self ! {step, Ref}, {ok, Req} end @@ -1149,4 +1149,4 @@ benchmark_set_multiple_deep_test(Opts) -> <<"Set two keys operations:">>, ?BENCHMARK_ITERATIONS, Time - ). \ No newline at end of file + ). diff --git a/src/hb_cache.erl b/src/hb_cache.erl index 2968e7792..6980bf4f2 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -123,7 +123,7 @@ ensure_loaded(Ref, Link = {link, ID, LinkOpts = #{ <<"lazy">> := true }}, RawOpt end, case CacheReadResult of {ok, LoadedMsg} -> - ?event(caching, + ?event(debug_caching, {lazy_loaded, {link, ID}, {msg, LoadedMsg}, diff --git a/src/hb_event.erl b/src/hb_event.erl index 2507ceb75..93a18ad16 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -74,7 +74,7 @@ increment(debug_linkify, _Message, _Opts, _Count) -> ignored; increment(debug_id, _Message, _Opts, _Count) -> ignored; increment(debug_enc, _Message, _Opts, _Count) -> ignored; increment(debug_commitments, _Message, _Opts, _Count) -> ignored; -increment(ao_core, _Message, _Opts, _Count) -> ignored; +increment(debug_ao_core, _Message, _Opts, _Count) -> ignored; increment(ao_internal, _Message, _Opts, _Count) -> ignored; increment(ao_devices, _Message, _Opts, _Count) -> ignored; increment(ao_subresolution, _Message, _Opts, _Count) -> ignored; diff --git a/src/hb_http.erl b/src/hb_http.erl index 799554835..38fed4e84 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -166,7 +166,7 @@ request_response(Method, Peer, Path, Response, Duration, Opts) -> % constructed from the header key-value pair list. HeaderMap = hb_maps:merge(hb_maps:from_list(Headers), MaybeSetCookie, Opts), NormHeaderMap = hb_ao:normalize_keys(HeaderMap, Opts), - ?event(http_outbound, + ?event(debug_http_outbound, {normalized_response_headers, {norm_header_map, NormHeaderMap}}, Opts ), @@ -186,7 +186,7 @@ request_response(Method, Peer, Path, Response, Duration, Opts) -> Key when is_binary(Key) -> Msg = http_response_to_httpsig(Status, NormHeaderMap, Body, Opts), ?event( - http_outbound, + debug_http_outbound, {result_is_single_key, {key, Key}, {msg, Msg}}, Opts ), @@ -230,7 +230,7 @@ request_response(Method, Peer, Path, Response, Duration, Opts) -> %% @doc Convert an HTTP response to a message. outbound_result_to_message(<<"ans104@1.0">>, Status, Headers, Body, Opts) -> - ?event(http_outbound, + ?event(debug_http_outbound, {result_is_ans104, {headers, Headers}, {body, Body}}, Opts ), @@ -261,7 +261,7 @@ outbound_result_to_message(<<"ans104@1.0">>, Status, Headers, Body, Opts) -> outbound_result_to_message(<<"httpsig@1.0">>, Status, Headers, Body, Opts) end; outbound_result_to_message(<<"httpsig@1.0">>, Status, Headers, Body, Opts) -> - ?event(http_outbound, {result_is_httpsig, {body, Body}}, Opts), + ?event(debug_http_outbound, {result_is_httpsig, {body, Body}}, Opts), { hb_http_client:response_status_to_atom(Status), http_response_to_httpsig(Status, Headers, Body, Opts) @@ -302,7 +302,7 @@ route_to_request(M, {ok, #{ <<"uri">> := XPath, <<"opts">> := ReqOpts}}, Opts) - % The request is a direct HTTP URL, so we need to split the path into a % host and path. URI = uri_string:parse(XPath), - ?event(http_outbound, {parsed_uri, {uri, {explicit, URI}}}), + ?event(debug_http_outbound, {parsed_uri, {uri, {explicit, URI}}}), Method = hb_ao:get(<<"method">>, M, <<"GET">>, Opts), % We must remove the path and host from the message, because they are not % valid for outbound requests. The path is retrieved from the route, and @@ -328,10 +328,10 @@ route_to_request(M, {ok, #{ <<"uri">> := XPath, <<"opts">> := ReqOpts}}, Opts) - Query -> [<<"?", Query/binary>>] end, Path = iolist_to_binary(PathParts), - ?event(http_outbound, {parsed_req, {node, Node}, {method, Method}, {path, Path}}), + ?event(debug_http_outbound, {parsed_req, {node, Node}, {method, Method}, {path, Path}}), {ok, Method, Node, Path, MsgWithoutMeta, hb_util:deep_merge(Opts, ReqOpts, Opts)}; route_to_request(M, {ok, Routes}, Opts) -> - ?event(http_outbound, {found_routes, {req, M}, {routes, Routes}}), + ?event(debug_http_outbound, {found_routes, {req, M}, {routes, Routes}}), % The result is a route, so we leave it to `request' to handle it. Path = hb_ao:get(<<"path">>, M, <<"/">>, Opts), Method = hb_ao:get(<<"method">>, M, <<"GET">>, Opts), @@ -362,7 +362,7 @@ prepare_request(Format, Method, Peer, Path, RawMessage, Opts) -> Opts ), {ok, CookieReset} = dev_codec_cookie:reset(Message, Opts), - ?event(http, {cookie_lines, CookieLines}), + ?event(debug_http, {cookie_lines, CookieLines}), { #{ <<"cookie">> => CookieLines }, CookieReset @@ -411,8 +411,8 @@ prepare_request(Format, Method, Peer, Path, RawMessage, Opts) -> ), Body = hb_maps:get(<<"body">>, FullEncoding, <<>>, Opts), Headers = hb_maps:without([<<"body">>], FullEncoding, Opts), - ?event(http, {request_headers, {explicit, {headers, Headers}}}), - ?event(http, {request_body, {explicit, {body, Body}}}), + ?event(debug_http, {request_headers, {explicit, {headers, Headers}}}), + ?event(debug_http, {request_body, {explicit, {body, Body}}}), hb_maps:merge( ReqBase, #{ headers => maps:merge(MaybeCookie, Headers), body => Body }, @@ -491,7 +491,7 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> ReqHdr = cowboy_req:header(<<"access-control-request-headers">>, Req, <<"">>), HeadersWithCors = add_cors_headers(HeadersBeforeCors, ReqHdr, Opts), EncodedHeaders = hb_private:reset(HeadersWithCors), - ?event(http, + ?event(debug_http, {http_replying, {status, {explicit, Status}}, {path, hb_maps:get(<<"path">>, Req, undefined_path, Opts)}, @@ -511,7 +511,7 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> ReplyDuration * 1000000, Status ), - ?event(http, {reply_headers, {explicit, PostStreamReq}}), + ?event(debug_http, {reply_headers, {explicit, PostStreamReq}}), ?event(http_server_short, {sent, {status, Status}, @@ -597,7 +597,7 @@ add_cors_headers(Msg, ReqHdr, Opts) -> %% @doc Generate the headers and body for a HTTP response message. encode_reply(Status, TABMReq, Message, Opts) -> Codec = accept_to_codec(TABMReq, Message, Opts), - ?event(http, {encoding_reply, {codec, Codec}, {message, Message}}), + ?event(debug_http, {encoding_reply, {codec, Codec}, {message, Message}}), BaseHdrs = hb_maps:merge( #{ @@ -613,7 +613,7 @@ encode_reply(Status, TABMReq, Message, Opts) -> hb_util:atom( hb_maps:get(<<"accept-bundle">>, TABMReq, false, Opts) ), - ?event(http, + ?event(debug_http, {encoding_reply, {status, Status}, {codec, Codec}, @@ -847,7 +847,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> error -> default_codec(Opts) end end, - ?event(http, + ?event(debug_http, {parsing_req, {path, FullPath}, {query, QueryKeys}, @@ -858,7 +858,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> ?event({req_to_tabm_singleton, {codec, Codec}}), case Codec of <<"httpsig@1.0">> -> - ?event( + ?event(debug_http, {req_to_tabm_singleton, {request, {explicit, Req}, {body, {string, Body}} @@ -897,7 +897,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> ), case ar_tx:verify(TX) of true -> - ?event(tx, {valid_tx_signature, TX}), + ?event(debug_tx, {valid_tx_signature, TX}), StructuredTX = hb_message:convert( TX, @@ -911,7 +911,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> end; Codec -> % Assume that the codec stores the encoded message in the `body' field. - ?event(http, {decoding_body, {codec, Codec}, {body, {string, Body}}}), + ?event(debug_http, {decoding_body, {codec, Codec}, {body, {string, Body}}}), Decoded = hb_message:convert( Body, @@ -920,7 +920,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> Opts ), ReqMessage = hb_maps:merge(PrimitiveMsg, Decoded, Opts), - ?event( + ?event(debug_http, {verifying_encoded_message, {codec, Codec}, {body, {string, Body}}, @@ -951,7 +951,7 @@ httpsig_to_tabm_singleton(PrimMsg, Req, Body, Opts) -> ), Opts ), - ?event(http, {decoded, Decoded}, Opts), + ?event(debug_http, {decoded, Decoded}, Opts), ForceSignedRequests = hb_opts:get(force_signed_requests, false, Opts), case (not ForceSignedRequests) orelse hb_message:verify(Decoded, all, Opts) of true -> @@ -991,7 +991,7 @@ httpsig_to_tabm_singleton(PrimMsg, Req, Body, Opts) -> %% 1. The path in the message %% 2. The path in the request URI normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> - ?event({adding_method_and_path_from_request, {explicit, Req}}), + ?event(debug_http, {adding_method_and_path_from_request, {explicit, Req}}), Method = cowboy_req:method(Req), MsgPath = hb_maps:get( diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index ae023d5e6..4cdf348c4 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -98,7 +98,7 @@ httpc_req(Args, Opts) -> 443 -> "https"; _ -> "http" end, - ?event(http_client, {httpc_req, {explicit, Args}}), + ?event(debug_http_client, {httpc_req, {explicit, Args}}), URL = binary_to_list(iolist_to_binary([Scheme, "://", Host, ":", integer_to_binary(Port), Path])), FilteredHeaders = hb_maps:without([<<"content-type">>, <<"cookie">>], Headers, Opts), HeaderKV = @@ -149,7 +149,7 @@ httpc_req(Args, Opts) -> || {Key, Value} <- RawRespHeaders ], - ?event(http_client, {httpc_resp, Status, RespHeaders, RespBody}), + ?event(debug_http_client, {httpc_resp, Status, RespHeaders, RespBody}), record_duration(#{ <<"request-method">> => method_to_bin(Method), <<"request-path">> => hb_util:bin(Path), @@ -343,7 +343,7 @@ maybe_invoke_monitor(Details, Opts) -> % execute. ReqMsgs = hb_singleton:from(Req, Opts), Res = hb_ao:resolve_many(ReqMsgs, Opts), - ?event(http_monitor, {resolved_monitor, Res}) + ?event(debug_http_monitor, {resolved_monitor, Res}) end. %%% ================================================================== @@ -484,7 +484,7 @@ handle_info({gun_down, PID, Protocol, Reason, _KilledStreams}, State) -> ok end, gun:shutdown(PID), - ?event(http_outbound, {gun_shutdown_after_down, {conn_key, ConnKey}}), + ?event(http_outbound, {gun_shutdown_after_down, {conn_key, ConnKey}, {protocol, Protocol}}), {noreply, State} end; @@ -725,7 +725,7 @@ await_response(Args, Opts) -> %% @doc Debug `http` state logging. log(Type, Event, #{method := Method, peer := Peer, path := Path}, Reason, Opts) -> ?event( - http, + debug_http, {gun_log, {type, Type}, {event, Event}, @@ -989,4 +989,4 @@ path_to_category(Path) -> <<"/~cache@1.0/read", _/binary>> -> <<"Remote Read">>; undefined -> <<"unknown">>; _ -> <<"unknown">> - end. + end. \ No newline at end of file diff --git a/src/hb_http_multi.erl b/src/hb_http_multi.erl index 95129d8ae..c4dae4299 100644 --- a/src/hb_http_multi.erl +++ b/src/hb_http_multi.erl @@ -88,7 +88,7 @@ request(Config, Method, Path, Message, Opts) -> Opts ) end, - ?event(http, {multirequest_results, {admissible_results, AdmissibleResults}, {all_responses, AllResponses}}), + ?event(debug_http, {multirequest_results, {admissible_results, AdmissibleResults}, {all_responses, AllResponses}}), case AdmissibleResults of [] -> {error, {no_viable_responses, AllResponses}}; Results -> if Responses == 1 -> hd(Results); true -> Results end @@ -155,7 +155,7 @@ serial_multirequest([Node|Nodes], Remaining, Method, Path, Message, Admissible, {ErlStatus, Res} = hb_http:request(Method, Node, Path, Message, Opts), case is_admissible(ErlStatus, Res, Admissible, Statuses, Opts) of true -> - ?event(http, {admissible_status, {response, Res}}), + ?event(debug_http, {admissible_status, {response, Res}}), {AdmissibleAcc, AllAcc} = serial_multirequest( Nodes, Remaining - 1, @@ -168,7 +168,7 @@ serial_multirequest([Node|Nodes], Remaining, Method, Path, Message, Admissible, ), {[{ErlStatus, Res} | AdmissibleAcc], [{ErlStatus, Res} | AllAcc]}; false -> - ?event(http, {inadmissible_status, {response, Res}}), + ?event(debug_http, {inadmissible_status, {response, Res}}), {AdmissibleAcc, AllAcc} = serial_multirequest( Nodes, Remaining, diff --git a/src/hb_message.erl b/src/hb_message.erl index b34267811..13fb10e37 100644 --- a/src/hb_message.erl +++ b/src/hb_message.erl @@ -694,7 +694,7 @@ unsafe_match(RawMap1, RawMap2, Mode, Path, Opts) -> fun(Key) -> lists:member(Key, Keys1) end, Keys1 ), - ?event(match, + ?event(debug_match, {match, {keys1, Keys1}, {keys2, Keys2}, @@ -709,7 +709,7 @@ unsafe_match(RawMap1, RawMap2, Mode, Path, Opts) -> lists:all( fun(<<"commitments">>) -> true; (Key) -> - ?event(match, {matching_key, Key}), + ?event(debug_match, {matching_key, Key}), Val1 = hb_ao:normalize_keys( hb_maps:get(Key, NormMap1, not_found, Opts), From 033d1a85b9cf836d677021b653d72249e13319cc Mon Sep 17 00:00:00 2001 From: speeddragon Date: Mon, 2 Mar 2026 19:10:53 +0000 Subject: [PATCH 008/135] parallel set to 1 --- src/hb_opts.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hb_opts.erl b/src/hb_opts.erl index cac3b072c..fc395f498 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -328,7 +328,7 @@ default_message() -> ?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES ), - <<"parallel">> => 4, + <<"parallel">> => 1, <<"responses">> => 1, <<"stop-after">> => true, <<"admissible-status">> => 200 From 0a6684056007d22f3d084a462341f47f9322f42f Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 3 Mar 2026 00:05:23 +0000 Subject: [PATCH 009/135] fix: While observer_cli doesn't implement the fix, this contains a fix to access top via shell as well via release --- rebar.config | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index bd8a10487..6d50fc2ee 100644 --- a/rebar.config +++ b/rebar.config @@ -12,7 +12,12 @@ ] }, {no_events, [{erl_opts, [{d, 'NO_EVENTS', true}]}]}, - {top, [{deps, [observer_cli]}, {erl_opts, [{d, 'AO_TOP', true}]}]}, + {top, [ + {deps, [{observer_cli, {git, "https://github.com/permaweb/observer_cli.git", + {ref, "7f90812dcd43d7954d0ba066cb94d5e934e729b5"}}}]}, + {erl_opts, [{d, 'AO_TOP', true}]}, + {relx, [{release, {'hb', "0.0.1"}, [hb, b64fast, cowboy, gun, luerl, prometheus, prometheus_cowboy, prometheus_ranch, elmdb, observer_cli]}]} + ]}, {store_events, [{erl_opts, [{d, 'STORE_EVENTS', true}]}]}, {ao_profiling, [{erl_opts, [{d, 'AO_PROFILING', true}]}]}, {eflame, From 12147c9eafe06eb77a360b38ec69a253dce45610 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 3 Mar 2026 17:13:54 +0000 Subject: [PATCH 010/135] Remove ranch --- src/hb_http_server.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hb_http_server.erl b/src/hb_http_server.erl index 592af7c86..520985198 100644 --- a/src/hb_http_server.erl +++ b/src/hb_http_server.erl @@ -197,7 +197,7 @@ new_server(RawNodeMsg) -> % Attempt to start the prometheus application, if possible. try application:ensure_all_started([prometheus, prometheus_cowboy, prometheus_ranch]), - prometheus_registry:register_collectors([hb_metrics_collector, prometheus_ranch_collector]), + prometheus_registry:register_collectors([hb_metrics_collector]), ProtoOpts#{ metrics_callback => fun prometheus_cowboy2_instrumenter:observe/1, From 4912a28e80836d54401c8ca3633b5c01aef143bf Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 3 Mar 2026 16:59:03 -0500 Subject: [PATCH 011/135] chore: connection pool start event --- src/hb_http_client.erl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index 4cdf348c4..1f8255309 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -23,7 +23,13 @@ setup_conn(Opts) -> ConnPoolReadSize = hb_maps:get(conn_pool_read_size,Opts, ?DEFAULT_CONN_POOL_READ_SIZE), ConnPoolWriteSize = hb_maps:get(conn_pool_write_size,Opts, ?DEFAULT_CONN_POOL_WRITE_SIZE), - ?event({conn, {pool_read_num, ConnPoolReadSize}, {pool_write_num, ConnPoolWriteSize}}), + ?event( + connection_pool, + {conn, + {pool_read_num, ConnPoolReadSize}, + {pool_write_num, ConnPoolWriteSize} + } + ), persistent_term:put(?CONN_TERM, {ConnPoolReadSize, ConnPoolWriteSize}). start_link(Opts) -> From 68d9f8c2a975637d54c873d61d59735d50c86dd2 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 3 Mar 2026 17:13:42 -0500 Subject: [PATCH 012/135] fix: return `not_found` quickly on remote store read non-ID --- src/hb_store_arweave.erl | 5 +++-- src/hb_store_remote_node.erl | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/hb_store_arweave.erl b/src/hb_store_arweave.erl index fec49492e..35b982d44 100644 --- a/src/hb_store_arweave.erl +++ b/src/hb_store_arweave.erl @@ -90,11 +90,12 @@ read_offset(_, _) -> not_found. %% @doc Read the data at the given key, reading the `local-store' first if %% available. -read(StoreOpts, ID) -> +read(StoreOpts, ID) when ?IS_ID(ID) -> case hb_store_remote_node:read_local_cache(StoreOpts, ID) of {ok, Message} -> {ok, Message}; not_found -> do_read(StoreOpts, ID) - end. + end; +read(_, _) -> not_found. %% @doc Read the data at the given key, reading the provided Arweave index store %% as a source of offsets. After offsets have been found, the data is loaded diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index c769c553e..074d1a46c 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -52,7 +52,7 @@ type(Opts = #{ <<"node">> := Node }, Key) -> %% @param Opts A map of options (including node configuration). %% @param Key The key to read. %% @returns {ok, Msg} on success or not_found if the key is missing. -read(Opts = #{ <<"node">> := Node }, Key) -> +read(Opts = #{ <<"node">> := Node }, Key) when ?IS_ID(Key) -> ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), HTTPRes = hb_http:get( @@ -70,7 +70,10 @@ read(Opts = #{ <<"node">> := Node }, Key) -> {error, _Err} -> ?event(store_remote_node, {read_not_found, {key, Key}}), not_found - end. + end; +read(_, Key) -> + ?event(store_remote_node, {ignoring_non_id, {key, Key}}), + not_found. %% @doc Cache the data if the cache is enabled. The `local-store' option may %% either be `false' or a store definition to use as the local cache. Additional From a7ff10182d20c0761cb265ef6c316ab4a50f3fa4 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 3 Mar 2026 20:05:43 -0500 Subject: [PATCH 013/135] impr: add IP to `http_server_short` print --- src/hb_http.erl | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/hb_http.erl b/src/hb_http.erl index 38fed4e84..dfb17312c 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -517,6 +517,7 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> {status, Status}, {duration, ReqDuration}, {method, cowboy_req:method(Req)}, + {ip, {string, real_ip(Req, Opts)}}, {path, {string, uri_string:percent_decode( @@ -1064,20 +1065,7 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> WithPeer = case hb_maps:get(<<"ao-peer-port">>, NormalBody, undefined, Opts) of undefined -> NormalBody; P2PPort -> - % Calculate the peer address from the request. We honor the - % `x-real-ip' header if it is present. - RealIP = - case hb_maps:get(<<"x-real-ip">>, RawHeaders, undefined, Opts) of - undefined -> - {{A, B, C, D}, _} = cowboy_req:peer(Req), - hb_util:bin( - io_lib:format( - "~b.~b.~b.~b", - [A, B, C, D] - ) - ); - IP -> IP - end, + RealIP = real_ip(Req, Opts), Peer = <>, (hb_message:without_unless_signed(<<"ao-peer-port">>, NormalBody, Opts))#{ <<"ao-peer">> => Peer @@ -1089,6 +1077,20 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> Device -> WithPeer#{<<"device">> => Device} end. +%% @doc Determine the caller, honoring the `x-real-ip' header if present. +real_ip(Req = #{ headers := RawHeaders }, Opts) -> + case hb_maps:get(<<"x-real-ip">>, RawHeaders, undefined, Opts) of + undefined -> + {{A, B, C, D}, _} = cowboy_req:peer(Req), + hb_util:bin( + io_lib:format( + "~b.~b.~b.~b", + [A, B, C, D] + ) + ); + IP -> IP + end. + %%% Metrics init_prometheus() -> From 0718f41a77e13763d60064e1bd80c53f240cdc9d Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 3 Mar 2026 21:44:07 -0500 Subject: [PATCH 014/135] Merge pull request #730 from permaweb/feat/rate-limiter feat: Add rate limiter device --- src/dev_rate_limit.erl | 177 +++++++++++++++++++++++++++++++++++++++++ src/hb_http.erl | 3 +- src/hb_opts.erl | 4 + 3 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 src/dev_rate_limit.erl diff --git a/src/dev_rate_limit.erl b/src/dev_rate_limit.erl new file mode 100644 index 000000000..2d9c14bfc --- /dev/null +++ b/src/dev_rate_limit.erl @@ -0,0 +1,177 @@ +%%% @doc A basic rate limiter device. It is intended for use as a `~hook@1.0` +%%% `on/request` handler. It limits the number of requests per minute from a +%%% given IP address, returning a 429 status code and response if the limit is +%%% exceeded. +%%% +%%% The device can be configured with the following node message options: +%%% +%%% ``` +%%% rate_limit: The maximum number of requests per minute from a given IP +%%% address. Default: 1,000. +%%% rate_limit_exempt: A list of peer IDs that are exempt from the rate +%%% limit. Default: []. +%%% ``` +-module(dev_rate_limit). +-export([request/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(LOOKUP_TIMEOUT, 1000). +-define(DEFAULT_RATE_LIMIT, 1000). +-define(DEFAULT_BUCKET_TIME, 60). + +request(_, Msg, Opts) -> + ?event(rate_limit, {request, {msg, Msg}}), + Reference = request_reference(hb_maps:get(<<"request">>, Msg, #{}, Opts), Opts), + case check_limit(Reference, Opts) of + true -> + ?event(rate_limit, {rate_limit_exceeded, {caller, Reference}}), + % Transform the given request into a request to return a 429 status + % code and response. + {ok, + #{ + <<"body">> => + [ + #{ + <<"status">> => 429, + <<"reason">> => <<"rate-limited">>, + <<"body">> => <<"Rate limit exceeded.">> + } + ] + } + }; + false -> + ?event(rate_limit, {rate_limit_allowed, {caller, Reference}}), + {ok, Msg} + end. + +server_id(Opts) -> + {?MODULE, hb_util:human_id(hb_opts:get(priv_wallet, undefined, Opts))}. + +%% @doc Determine the reference of the caller. +request_reference(Msg, Opts) -> + hb_maps:get(<<"ao-peer">>, Msg, undefined, Opts). + +check_limit(IP, Opts) -> + PID = ensure_rate_limiter_started(Opts), + PID ! {request, Self = self(), IP}, + receive + {rate_limit_result, Result} -> Result + after ?LOOKUP_TIMEOUT -> + ?event(warning, {rate_limit_timeout, restarting}), + hb_name:unregister(server_id(Opts)), + check_limit(IP, Opts) + end. + +ensure_rate_limiter_started(Opts) -> + ServerID = server_id(Opts), + case hb_name:lookup({?MODULE, ServerID}) of + PID when is_pid(PID) -> PID; + undefined -> + spawn( + fun() -> + hb_name:register({?MODULE, ServerID}, self()), + Limit = + hb_opts:get( + rate_limit, + ?DEFAULT_RATE_LIMIT, + Opts + ), + BucketTime = + hb_opts:get( + rate_limit_bucket_time, + ?DEFAULT_BUCKET_TIME, + Opts + ), + ExemptPeers = hb_opts:get(rate_limit_exempt, [], Opts), + ?event( + rate_limit, + {started_rate_limiter, + {server_id, ServerID}, + {limit, Limit}, + {exempt_peers, ExemptPeers} + } + ), + server_loop( + #{ + limit => Limit, + peers => #{ IP => infinity || IP <- ExemptPeers }, + bucket_time => BucketTime + } + ) + end + ) + end. + +server_loop(State) -> + receive + {request, Self, IP} -> + NewState = increment(IP, State), + Self ! {rate_limit_result, is_limited(IP, NewState)}, + server_loop(NewState) + end. + +increment(IP, #{ bucket_time := BucketTime } = State) -> + increment(IP, erlang:system_time(second) div BucketTime, State). +increment(IP, Bucket, S = #{ peers := Peers }) -> + case maps:get(IP, Peers, #{}) of + infinity -> S; + #{ since := Bucket, count := Count } -> + S#{ peers => Peers#{ IP => #{ since => Bucket, count => Count + 1 }}}; + _ -> + S#{ peers => Peers#{ IP => #{ since => Bucket, count => 1 }}} + end. + +%% @doc Check if the IP is limited. Assumes the IP is in the state (added by +%% increment/2). +is_limited(IP, #{ peers := Peers }) when map_get(IP, Peers) =:= infinity -> false; +is_limited(IP, #{ limit := Limit, peers := Peers }) -> + maps:get(count, maps:get(IP, Peers, #{}), 0) > Limit. + +%%% Tests + +rate_limit_test() -> + ServerOpts = #{ + rate_limit => 2, + rate_limit_exempt => [], + rate_limit_bucket_time => 10_000, + on => + #{ + <<"request">> => + #{ + <<"device">> => <<"rate-limit@1.0">> + } + } + }, + ServerNode = hb_http_server:start_node(ServerOpts), + ?assertMatch( + {ok, _}, + hb_http:get(ServerNode, <<"id">>, #{}) + ), + ?assertMatch( + {ok, _}, + hb_http:get(ServerNode, <<"id">>, #{}) + ), + ?assertMatch( + {error, #{ <<"status">> := 429 }}, + hb_http:get(ServerNode, <<"id">>, #{}) + ). + +rate_limit_reset_test() -> + ServerOpts = #{ + rate_limit => 2, + rate_limit_exempt => [], + rate_limit_bucket_time => 2, + on => + #{ + <<"request">> => + #{ + <<"device">> => <<"rate-limit@1.0">> + } + } + }, + ServerNode = hb_http_server:start_node(ServerOpts), + ?assertMatch({ok, _}, hb_http:get(ServerNode, <<"id">>, #{})), + ?assertMatch({ok, _}, hb_http:get(ServerNode, <<"id">>, #{})), + timer:sleep(2_000), + ?assertMatch({ok, _}, hb_http:get(ServerNode, <<"id">>, #{})). \ No newline at end of file diff --git a/src/hb_http.erl b/src/hb_http.erl index dfb17312c..bc24658be 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -515,7 +515,8 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> ?event(http_server_short, {sent, {status, Status}, - {duration, ReqDuration}, + {ip, {string, real_ip(Req, Opts)}}, + {duration, EndTime - hb_maps:get(start_time, Req, undefined, Opts)}, {method, cowboy_req:method(Req)}, {ip, {string, real_ip(Req, Opts)}}, {path, diff --git a/src/hb_opts.erl b/src/hb_opts.erl index fc395f498..3027d1f39 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -203,6 +203,7 @@ default_message() -> #{<<"name">> => <<"profile@1.0">>, <<"module">> => dev_profile}, #{<<"name">> => <<"push@1.0">>, <<"module">> => dev_push}, #{<<"name">> => <<"query@1.0">>, <<"module">> => dev_query}, + #{<<"name">> => <<"rate-limit@1.0">>, <<"module">> => dev_rate_limit}, #{<<"name">> => <<"relay@1.0">>, <<"module">> => dev_relay}, #{<<"name">> => <<"router@1.0">>, <<"module">> => dev_router}, #{<<"name">> => <<"scheduler@1.0">>, <<"module">> => dev_scheduler}, @@ -453,6 +454,9 @@ default_message() -> on => #{ <<"request">> => [ + #{ + <<"device">> => <<"rate-limit@1.0">> + }, #{ <<"device">> => <<"auth-hook@1.0">>, <<"path">> => <<"request">>, From 7eba0b29b08cbcc9c914a61641e9867ef8b35842 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 3 Mar 2026 22:33:17 -0500 Subject: [PATCH 015/135] impr: add IP to `priv/ip` of requests --- src/dev_rate_limit.erl | 2 +- src/hb_http.erl | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/dev_rate_limit.erl b/src/dev_rate_limit.erl index 2d9c14bfc..474d5cc37 100644 --- a/src/dev_rate_limit.erl +++ b/src/dev_rate_limit.erl @@ -50,7 +50,7 @@ server_id(Opts) -> %% @doc Determine the reference of the caller. request_reference(Msg, Opts) -> - hb_maps:get(<<"ao-peer">>, Msg, undefined, Opts). + hb_private:get(<<"ip">>, Msg, Opts). check_limit(IP, Opts) -> PID = ensure_rate_limiter_started(Opts), diff --git a/src/hb_http.erl b/src/hb_http.erl index bc24658be..22f6d59f1 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -129,7 +129,6 @@ request(Method, Peer, Path, RawMessage, Opts) -> Error -> Error end. - request_response(Method, Peer, Path, Response, Duration, Opts) -> {_ErlStatus, Status, Headers, Body} = Response, @@ -1063,6 +1062,7 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> <<"">> -> hb_message:without_unless_signed(<<"body">>, WithCookie, Opts); _ -> WithCookie end, + RealIP = real_ip(Req, Opts), WithPeer = case hb_maps:get(<<"ao-peer-port">>, NormalBody, undefined, Opts) of undefined -> NormalBody; P2PPort -> @@ -1072,10 +1072,11 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> <<"ao-peer">> => Peer } end, + WithPrivIP = hb_private:set(NormalBody, <<"ip">>, RealIP, Opts), % Add device from PrimMsg if present case maps:get(<<"device">>, PrimMsg, not_found) of - not_found -> WithPeer; - Device -> WithPeer#{<<"device">> => Device} + not_found -> WithPrivIP; + Device -> WithPrivIP#{<<"device">> => Device} end. %% @doc Determine the caller, honoring the `x-real-ip' header if present. From 4752b18dd0a0cdd77d0da65353e47ef5de458ef6 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 4 Mar 2026 19:15:00 -0500 Subject: [PATCH 016/135] chore: ignore noisy `ao-core` events --- src/hb_event.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 93a18ad16..2507ceb75 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -74,7 +74,7 @@ increment(debug_linkify, _Message, _Opts, _Count) -> ignored; increment(debug_id, _Message, _Opts, _Count) -> ignored; increment(debug_enc, _Message, _Opts, _Count) -> ignored; increment(debug_commitments, _Message, _Opts, _Count) -> ignored; -increment(debug_ao_core, _Message, _Opts, _Count) -> ignored; +increment(ao_core, _Message, _Opts, _Count) -> ignored; increment(ao_internal, _Message, _Opts, _Count) -> ignored; increment(ao_devices, _Message, _Opts, _Count) -> ignored; increment(ao_subresolution, _Message, _Opts, _Count) -> ignored; From 712f59ee453d7c3765380c09601c89f1b80dcf5e Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 5 Mar 2026 16:54:26 +0000 Subject: [PATCH 017/135] fix: patch Manifest PR --- src/dev_manifest.erl | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index b30cc6c0d..e6a55f3d4 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -101,9 +101,9 @@ request(Base, Req, Opts) -> _ -> Rest end, {ok, Req#{ <<"body">> => [CastedMsg|Rest2] }}; - Error -> - ?event({manifest_not_cast, {error, Error}}), - Error + {error, not_found} -> + ?event({manifest_not_cast, {error, not_found}}), + {ok, Req} end; _ -> {ok, Req} @@ -115,7 +115,7 @@ maybe_cast_manifest(ID, Opts) when ?IS_ID(ID) -> case hb_cache:read(ID, Opts) of {ok, Msg} -> maybe_cast_manifest(Msg, Opts); _ -> - ?event(maybe_cast_manifest, {message_not_found, {id, ID}}), + ?event(debug_maybe_cast_manifest, {message_not_found, {id, ID}}), {error, not_found} end; maybe_cast_manifest(Msg, Opts) when is_map(Msg) orelse ?IS_LINK(Msg) -> @@ -125,15 +125,16 @@ maybe_cast_manifest(Msg, Opts) when is_map(Msg) orelse ?IS_LINK(Msg) -> _ -> case hb_maps:find(<<"content-type">>, Msg, Opts) of {ok, <<"application/x.arweave-manifest+json">>} -> - ?event(maybe_cast_manifest, {manifest_casting, {msg, Msg}}), + ?event(debug_maybe_cast_manifest, {manifest_casting, {msg, Msg}}), {ok, {as, <<"manifest@1.0">>, Msg}}; - _ -> - {ok, Msg} + Value -> + ?event(debug_maybe_cast_manifest, {manifest_casting_not_expected, Value}), + {error, not_found} end end; maybe_cast_manifest(Msg, _Opts) -> - ?event(maybe_cast_manifest, {message_is_not_manifest, {msg, Msg}}), - {ok, Msg}. + ?event(debug_maybe_cast_manifest, {message_is_not_manifest, {msg, Msg}}), + {error, not_found}. %% @doc Find and deserialize a manifest from the given base, returning a %% message with the `~manifest@1.0' device. From 423e226aad2e53a789a3809d5ef3bf55d5e049c0 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 5 Mar 2026 13:08:28 -0500 Subject: [PATCH 018/135] chore: remove spammy events --- src/hb_event.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hb_event.erl b/src/hb_event.erl index 2507ceb75..60dbf2a02 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -74,6 +74,8 @@ increment(debug_linkify, _Message, _Opts, _Count) -> ignored; increment(debug_id, _Message, _Opts, _Count) -> ignored; increment(debug_enc, _Message, _Opts, _Count) -> ignored; increment(debug_commitments, _Message, _Opts, _Count) -> ignored; +increment(message_set, _Message, _Opts, _Count) -> ignored; +increment(read_cached, _Message, _Opts, _Count) -> ignored; increment(ao_core, _Message, _Opts, _Count) -> ignored; increment(ao_internal, _Message, _Opts, _Count) -> ignored; increment(ao_devices, _Message, _Opts, _Count) -> ignored; From 7da475db9955ad30d4d6b2853eb3ee54091c6b87 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 5 Mar 2026 13:56:33 -0500 Subject: [PATCH 019/135] fix: manifest load deduplication and `index` handling --- src/dev_manifest.erl | 82 +++++++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 28 deletions(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index e6a55f3d4..460771b80 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -91,34 +91,65 @@ route(Key, M1, M2, Opts) -> %% `manifest@1.0' before execution. Allowing `/ID/path` style access for old data. request(Base, Req, Opts) -> ?event({on_req_manifest_detector, {base, Base}, {req, Req}}), - case hb_maps:find(<<"body">>, Req, Opts) of - {ok, [PrimaryMsg|Rest]} -> - case maybe_cast_manifest(PrimaryMsg, Opts) of - {ok, CastedMsg} -> - %% For to go to index if no key provided. - Rest2 = case Rest of - [] -> [#{<<"path">> => <<"index">>}]; - _ -> Rest - end, - {ok, Req#{ <<"body">> => [CastedMsg|Rest2] }}; - {error, not_found} -> - ?event({manifest_not_cast, {error, not_found}}), - {ok, Req} - end; - _ -> + maybe + {ok, [PrimaryMsg|Rest]} ?= hb_maps:find(<<"body">>, Req, Opts), + {ok, Loaded} ?= load(PrimaryMsg, Opts), + % Must handle three cases: + % 1. The maybe_cast is not a manifest, so we return the *loaded* request, + % such that the work to load it is not wasted. + % 2. The maybe_cast is a manifest, and there are no other elements of + % the path, so we add the `index' path and return. + % 3. The maybe_cast is a manifest, and there are other elements of + % the path, so we return the original request sequence with the first + % message replaced with the casted manifest. + case {Rest, maybe_cast_manifest(Loaded, Opts)} of + {_, ignored} -> + {ok, Req#{ <<"body">> => [Loaded|Rest] }}; + {[], {ok, Casted}} -> + {ok, Req#{ <<"body">> => [Casted, #{<<"path">> => <<"index">>}] }}; + {_, {ok, Casted}} -> + {ok, Req#{ <<"body">> => [Casted|Rest] }} + end + else + {error, not_found} -> + { + ok, + Req#{ + <<"body">> => + [ + #{ + <<"status">> => 404, + <<"body">> => <<"Not Found">> + } + ] + } + }; + Error -> + ?event(debug_manifest, {request_ignored, {unexpected, Error}}), + % On other errors, we return the original request. {ok, Req} end. %% @doc Cast a message to `manifest@1.0` if it has the correct content-type but %% no other device is specified. -maybe_cast_manifest(ID, Opts) when ?IS_ID(ID) -> +load({as, _, _}, _Opts) -> skip; +load(Msg, _Opts) when is_map(Msg) -> {ok, Msg}; +load(ID, Opts) when ?IS_ID(ID) -> case hb_cache:read(ID, Opts) of - {ok, Msg} -> maybe_cast_manifest(Msg, Opts); + {ok, Msg} -> {ok, Msg}; _ -> - ?event(debug_maybe_cast_manifest, {message_not_found, {id, ID}}), + ?event(debug_maybe_cast_manifest, {message_load_failed, {id, ID}}), {error, not_found} end; -maybe_cast_manifest(Msg, Opts) when is_map(Msg) orelse ?IS_LINK(Msg) -> +load(Msg, Opts) when ?IS_LINK(Msg) -> + try {ok, hb_cache:ensure_loaded(Msg, Opts)} + catch + _ -> + ?event(debug_maybe_cast_manifest, {message_load_failed, {link, Msg}}), + {error, not_found} + end. + +maybe_cast_manifest(Msg, Opts) -> case hb_maps:find(<<"device">>, Msg, Opts) of {ok, X} when X == <<"manifest@1.0">> orelse X == <<"message@1.0">> -> {ok, Msg}; @@ -127,14 +158,10 @@ maybe_cast_manifest(Msg, Opts) when is_map(Msg) orelse ?IS_LINK(Msg) -> {ok, <<"application/x.arweave-manifest+json">>} -> ?event(debug_maybe_cast_manifest, {manifest_casting, {msg, Msg}}), {ok, {as, <<"manifest@1.0">>, Msg}}; - Value -> - ?event(debug_maybe_cast_manifest, {manifest_casting_not_expected, Value}), - {error, not_found} + _IgnoredContentType -> + {ok, Msg} end - end; -maybe_cast_manifest(Msg, _Opts) -> - ?event(debug_maybe_cast_manifest, {message_is_not_manifest, {msg, Msg}}), - {error, not_found}. + end. %% @doc Find and deserialize a manifest from the given base, returning a %% message with the `~manifest@1.0' device. @@ -224,7 +251,6 @@ resolve_test() -> hb_http:get(Node, << ManifestID/binary, "/nested/page2" >>, Opts)), % Making the same requests to a node with the `request' hook enabled should % yield the same results. - ?hr(), ?event({legacy_manifest_id, LegacyManifestID}), ?assertMatch( {ok, #{ <<"body">> := <<"Page 1">> }}, @@ -282,7 +308,7 @@ create_generic_manifest(Opts) -> %% @doc Download the manifest raw data. %% NOTE: This test requests data to arweave node -manifest_download_via_raw_endpoint_test() -> +manifest_download_via_raw_endpoint_test_ignore() -> Opts = #{ arweave_index_ids => true, store => [ From 2f091a54eb11dfe1ecc23655a2af0e8435fbbe3c Mon Sep 17 00:00:00 2001 From: James Piechota Date: Thu, 5 Mar 2026 08:18:04 -0500 Subject: [PATCH 020/135] chore: add chain-3, ping 6ms vs. 90 ms for chain-1/2 --- src/hb_opts.erl | 2 +- src/include/hb_arweave_nodes.hrl | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 3027d1f39..7fe697eb4 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -374,7 +374,7 @@ default_message() -> <<"opts">> => #{ http_client => httpc, protocol => http2 } } }, - %% General Arweave requests: race both chain nodes, take + %% General Arweave requests: race all chain nodes, take %% the first 200. #{ <<"template">> => <<"^/arweave">>, diff --git a/src/include/hb_arweave_nodes.hrl b/src/include/hb_arweave_nodes.hrl index 92bd000d0..73cba8b53 100644 --- a/src/include/hb_arweave_nodes.hrl +++ b/src/include/hb_arweave_nodes.hrl @@ -210,6 +210,10 @@ -define(ARWEAVE_BOOTSTRAP_CHAIN_NODES, [ + #{ + <<"match">> => <<"^/arweave">>, + <<"with">> => <<"http://chain-3.arweave.xyz:1984">> + }, #{ <<"match">> => <<"^/arweave">>, <<"with">> => <<"http://chain-1.arweave.xyz:1984">> From 778024808574f579d7b08b6ad1d0e84f2a4a7c0a Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 5 Mar 2026 14:32:10 -0500 Subject: [PATCH 021/135] wip: improvements to manifest routing logic --- src/dev_manifest.erl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index 460771b80..2d84863aa 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -94,6 +94,7 @@ request(Base, Req, Opts) -> maybe {ok, [PrimaryMsg|Rest]} ?= hb_maps:find(<<"body">>, Req, Opts), {ok, Loaded} ?= load(PrimaryMsg, Opts), + ?event(debug_manifest, {loaded, Loaded}), % Must handle three cases: % 1. The maybe_cast is not a manifest, so we return the *loaded* request, % such that the work to load it is not wasted. @@ -104,14 +105,20 @@ request(Base, Req, Opts) -> % message replaced with the casted manifest. case {Rest, maybe_cast_manifest(Loaded, Opts)} of {_, ignored} -> + ?event( + debug_manifest, + {non_manifest_returning_loaded, {loaded, Loaded}, {rest, Rest}}), {ok, Req#{ <<"body">> => [Loaded|Rest] }}; {[], {ok, Casted}} -> + ?event(debug_manifest, {manifest_returning_index, {req, Req}}), {ok, Req#{ <<"body">> => [Casted, #{<<"path">> => <<"index">>}] }}; {_, {ok, Casted}} -> + ?event(debug_manifest, {manifest_returning_subpath, {req, Req}}), {ok, Req#{ <<"body">> => [Casted|Rest] }} end else {error, not_found} -> + ?event(debug_manifest, {not_found_on_load, {req, Req}}), { ok, Req#{ @@ -151,15 +158,14 @@ load(Msg, Opts) when ?IS_LINK(Msg) -> maybe_cast_manifest(Msg, Opts) -> case hb_maps:find(<<"device">>, Msg, Opts) of - {ok, X} when X == <<"manifest@1.0">> orelse X == <<"message@1.0">> -> - {ok, Msg}; + {ok, X} when X == <<"manifest@1.0">> -> {ok, Msg}; _ -> case hb_maps:find(<<"content-type">>, Msg, Opts) of - {ok, <<"application/x.arweave-manifest+json">>} -> + {ok, <<"application/x.arweave-manifest json">>} -> ?event(debug_maybe_cast_manifest, {manifest_casting, {msg, Msg}}), {ok, {as, <<"manifest@1.0">>, Msg}}; _IgnoredContentType -> - {ok, Msg} + ignored end end. From 70e5a916edc9f1bce530364a7f58cd1f5572cb66 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 5 Mar 2026 14:43:03 -0500 Subject: [PATCH 022/135] wip: testing setup --- src/dev_manifest.erl | 7 ++++--- src/hb_opts.erl | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index 2d84863aa..c0983af3a 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -139,17 +139,18 @@ request(Base, Req, Opts) -> %% @doc Cast a message to `manifest@1.0` if it has the correct content-type but %% no other device is specified. -load({as, _, _}, _Opts) -> skip; load(Msg, _Opts) when is_map(Msg) -> {ok, Msg}; +load(List, _Opts) when is_list(List) -> skip; +load({as, _, _}, _Opts) -> skip; load(ID, Opts) when ?IS_ID(ID) -> case hb_cache:read(ID, Opts) of - {ok, Msg} -> {ok, Msg}; + {ok, Msg} -> load(Msg, Opts); _ -> ?event(debug_maybe_cast_manifest, {message_load_failed, {id, ID}}), {error, not_found} end; load(Msg, Opts) when ?IS_LINK(Msg) -> - try {ok, hb_cache:ensure_loaded(Msg, Opts)} + try load(hb_cache:ensure_loaded(Msg, Opts), Opts) catch _ -> ?event(debug_maybe_cast_manifest, {message_load_failed, {link, Msg}}), diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 3027d1f39..56a4337ce 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -31,6 +31,7 @@ -else. -define(DEFAULT_PRINT_OPTS, [ + dev_manifest, http, hb_gateway_client, error, http_error, cron_error, hook_error, warning, http_server_short, compute_short, push_short, copycat_short, bundler_short From e461f31a54a5eff0d7682785def22c4cd066efd3 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 5 Mar 2026 15:21:32 -0500 Subject: [PATCH 023/135] fix: return in-spec for `raw` -- data is in `body` not `data` --- src/dev_arweave.erl | 4 ++-- src/hb_gateway_client.erl | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index 87c87ac35..426d3636d 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -295,7 +295,7 @@ get_raw(Base, Request, Opts) -> "/", (hb_util:bin(FullContentLength))/binary >>, - <<"data">> => Data + <<"body">> => Data } }; false -> @@ -303,7 +303,7 @@ get_raw(Base, Request, Opts) -> {ok, Data} -> {ok, Header#{ <<"content-type">> => ContentType, - <<"data">> => Data + <<"body">> => Data }}; Error -> ?event( diff --git a/src/hb_gateway_client.erl b/src/hb_gateway_client.erl index e29479144..8826bfc39 100644 --- a/src/hb_gateway_client.erl +++ b/src/hb_gateway_client.erl @@ -110,14 +110,19 @@ data(ID, Opts) -> }, case hb_http:request(Req, Opts) of {ok, Res} -> + Data = + case hb_maps:find(<<"data">>, Res, Opts) of + {ok, D} -> D; + _ -> hb_ao:get(<<"body">>, Res, <<>>, Opts) + end, ?event(gateway, {data, {id, ID}, {response, Res}, - {body, hb_ao:get(<<"body">>, Res, <<>>, Opts)} + {data, Data} } ), - {ok, hb_ao:get(<<"body">>, Res, <<>>, Opts)}; + {ok, Data}; Res -> ?event(gateway, {request_error, {id, ID}, {response, Res}}), {error, no_viable_gateway} From 41146fb83cdff9063fc120a6f43feb2ffb24a4b5 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 5 Mar 2026 15:23:59 -0500 Subject: [PATCH 024/135] chore: settings and prints --- src/hb_http.erl | 5 ++--- src/hb_opts.erl | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/hb_http.erl b/src/hb_http.erl index 22f6d59f1..dc582dc83 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -514,8 +514,8 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> ?event(http_server_short, {sent, {status, Status}, - {ip, {string, real_ip(Req, Opts)}}, {duration, EndTime - hb_maps:get(start_time, Req, undefined, Opts)}, + {body_size, byte_size(EncodedBody)}, {method, cowboy_req:method(Req)}, {ip, {string, real_ip(Req, Opts)}}, {path, @@ -524,8 +524,7 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> hb_maps:get(<<"path">>, TABMReq, <<"[NO PATH]">>, Opts) ) } - }, - {body_size, byte_size(EncodedBody)} + } } ), {ok, PostStreamReq, no_state}. diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 56a4337ce..7e76537b3 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -269,7 +269,7 @@ default_message() -> debug_trace_type => ?DEFAULT_TRACE_TYPE, short_trace_len => 20, debug_show_priv => if_present, - debug_resolve_links => true, + debug_resolve_links => false, debug_print_fail_mode => long, trusted => #{}, snp_enforced_keys => [ From dc01a2d24dd4a4e8198580487fe17c81c5e06b48 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 5 Mar 2026 15:45:00 -0500 Subject: [PATCH 025/135] fix: tolerate `content-type` values that suffer from the `+` encoding ambiguity --- src/dev_manifest.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index c0983af3a..aa0c33573 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -162,7 +162,7 @@ maybe_cast_manifest(Msg, Opts) -> {ok, X} when X == <<"manifest@1.0">> -> {ok, Msg}; _ -> case hb_maps:find(<<"content-type">>, Msg, Opts) of - {ok, <<"application/x.arweave-manifest json">>} -> + {ok, <<"application/x.arweave-manifest", _/binary>>} -> ?event(debug_maybe_cast_manifest, {manifest_casting, {msg, Msg}}), {ok, {as, <<"manifest@1.0">>, Msg}}; _IgnoredContentType -> From f3bbd173104355dbc31d661fcd13d86ba03ea12f Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 5 Mar 2026 20:39:10 -0500 Subject: [PATCH 026/135] chore: fix erroneous opts additions --- src/hb_opts.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 783807fa5..d7fa7df99 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -31,7 +31,6 @@ -else. -define(DEFAULT_PRINT_OPTS, [ - dev_manifest, http, hb_gateway_client, error, http_error, cron_error, hook_error, warning, http_server_short, compute_short, push_short, copycat_short, bundler_short From 742b07abaee3f6ba70b533f0d9d47f78a7f0e050 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 6 Mar 2026 08:25:39 -0500 Subject: [PATCH 027/135] fix: `raw` returns the data of a transaction in its `body` field --- src/dev_arweave.erl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index 426d3636d..629901f9a 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -1307,7 +1307,8 @@ get_tx_basic_data_exclude_data_test() -> }, Opts ), - Data = hb_ao:get(<<"data">>, RawData, Opts), + ?event(debug_test, {raw_data, RawData}), + Data = hb_ao:get(<<"body">>, RawData, Opts), StructuredWithData = Structured#{ <<"data">> => Data }, ?assert(hb_message:verify(StructuredWithData, all, Opts)), DataHash = hb_util:encode(crypto:hash(sha256, Data)), @@ -1343,7 +1344,7 @@ get_tx_data_tag_exclude_data_test() -> }, Opts ), - Data = hb_ao:get(<<"data">>, RawData, Opts), + Data = hb_ao:get(<<"body">>, RawData, Opts), StructuredWithData = Structured#{ <<"data">> => Data }, ?assert(hb_message:verify(StructuredWithData, all, Opts)), DataHash = hb_util:encode(crypto:hash(sha256, Data)), @@ -1421,7 +1422,7 @@ get_raw_range_tx_test() -> ?event(debug_test, {result, Result}), ?assertEqual( {ok, <<"{\"d">>}, - hb_maps:find(<<"data">>, Result, Opts) + hb_maps:find(<<"body">>, Result, Opts) ), {ok, Result2} = hb_ao:resolve( @@ -1441,7 +1442,7 @@ get_raw_range_tx_test() -> ), ?assertEqual( {ok, <<"ame Cr">>}, - hb_maps:find(<<"data">>, Result2, Opts) + hb_maps:find(<<"body">>, Result2, Opts) ). get_raw_range_ans104_test() -> @@ -1466,7 +1467,7 @@ get_raw_range_ans104_test() -> ?event(debug_test, {result, Result}), ?assertEqual( {ok, <<"{\n">>}, - hb_maps:find(<<"data">>, Result, Opts) + hb_maps:find(<<"body">>, Result, Opts) ), {ok, Result2} = hb_ao:resolve( @@ -1486,7 +1487,7 @@ get_raw_range_ans104_test() -> ), ?assertEqual( {ok, <<"t #972">>}, - hb_maps:find(<<"data">>, Result2, Opts) + hb_maps:find(<<"body">>, Result2, Opts) ). get_tx_rsa_nested_bundle_test() -> From 6259f12a9695a7039e7c081fde181b91e31f7997 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 6 Mar 2026 15:53:43 -0500 Subject: [PATCH 028/135] feat: sliding window and granularly recharging rate-limiter with `debt`. --- src/dev_rate_limit.erl | 215 +++++++++++++++++++++++++++++------------ 1 file changed, 152 insertions(+), 63 deletions(-) diff --git a/src/dev_rate_limit.erl b/src/dev_rate_limit.erl index 474d5cc37..bcced5538 100644 --- a/src/dev_rate_limit.erl +++ b/src/dev_rate_limit.erl @@ -1,31 +1,75 @@ %%% @doc A basic rate limiter device. It is intended for use as a `~hook@1.0` -%%% `on/request` handler. It limits the number of requests per minute from a +%%% `on/request` handler. It limits the number of requests per time period from a %%% given IP address, returning a 429 status code and response if the limit is %%% exceeded. %%% %%% The device can be configured with the following node message options: %%% %%% ``` -%%% rate_limit: The maximum number of requests per minute from a given IP -%%% address. Default: 1,000. -%%% rate_limit_exempt: A list of peer IDs that are exempt from the rate -%%% limit. Default: []. +%%% rate_limit_requests: The maximum number of requests per period from a +%%% given user. +%%% Default: 1000. +%%% rate_limit_period: The rate at which peer's fully recharge balances. +%%% Default: 60 (unit: seconds). +%%% rate_limit_max: The maximum `balance' that a peer may hold. +%%% Default: 1000. +%%% rate_limit_min: The minimum `balance' that a peer may hold. +%%% Default: -1000. +%%% rate_limit_exempt: A list of peer IDs that are exempt from the limit. +%%% Default: []. %%% ``` +%%% +%%% Notably, the `balance` of a user -- in terms of their available limit -- may +%%% become _negative_ if they continue to make calls even after exceeding their +%%% limit. The effect of this is that users that make too many requests to the +%%% server repeatedly simply receive no further service. The `rate_limit_min` +%%% option can be used to specify the minimum balance that users will hit. Any +%%% further requests are rejected but do not diminish their balance further. -module(dev_rate_limit). -export([request/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(LOOKUP_TIMEOUT, 1000). --define(DEFAULT_RATE_LIMIT, 1000). --define(DEFAULT_BUCKET_TIME, 60). +-define(DEFAULT_MAX, 1_000). +-define(DEFAULT_MIN, -1_000). +-define(DEFAULT_REQS, 1000). +-define(DEFAULT_PERIOD, 60). +%% @doc `on/request' handler that triggers rate limit counting and returns a +%% 429 status code and response if the limit is exceeded. The response includes +%% a `retry-after' header that indicates the number of seconds the client should +%% wait before making the next request. request(_, Msg, Opts) -> ?event(rate_limit, {request, {msg, Msg}}), Reference = request_reference(hb_maps:get(<<"request">>, Msg, #{}, Opts), Opts), - case check_limit(Reference, Opts) of - true -> - ?event(rate_limit, {rate_limit_exceeded, {caller, Reference}}), + case is_limited(Reference, Opts) of + {true, Balance} -> + ?event( + rate_limit, + {rate_limit_exceeded, {caller, Reference}, {balance, Balance}} + ), + RechargeRate = + hb_opts:get(rate_limit_requests, ?DEFAULT_REQS, Opts) / + hb_opts:get(rate_limit_period, ?DEFAULT_PERIOD, Opts), + RawRetryAfter = ceil(abs(Balance) / RechargeRate), % ...seconds + % If the node config specifies a `min` balance of `0`, callers may + % have a non-negative balance but still be rate-limited. In this case, + % we bump the `retry-after` to 1 second so as not to confuse the + % caller. + RetryAfter = + if RawRetryAfter =< 0.0 -> 1; + true -> RawRetryAfter + end, + RetryAfterBin = hb_util:bin(RetryAfter), + ?event( + rate_limit, + {rate_limit_exceeded, + {caller, Reference}, + {balance, Balance}, + {retry_after, RetryAfterBin} + } + ), % Transform the given request into a request to return a 429 status % code and response. {ok, @@ -35,7 +79,8 @@ request(_, Msg, Opts) -> #{ <<"status">> => 429, <<"reason">> => <<"rate-limited">>, - <<"body">> => <<"Rate limit exceeded.">> + <<"body">> => <<"Rate limit exceeded.">>, + <<"retry-after">> => RetryAfterBin } ] } @@ -45,96 +90,132 @@ request(_, Msg, Opts) -> {ok, Msg} end. +%% @doc The singleton ID of the rate limiter server. This allows us to run +%% multiple rate limiters on the same node if needed, each with its own +%% configuration, but with all of the callers sharing the same rate limiter +%% server. server_id(Opts) -> {?MODULE, hb_util:human_id(hb_opts:get(priv_wallet, undefined, Opts))}. -%% @doc Determine the reference of the caller. -request_reference(Msg, Opts) -> - hb_private:get(<<"ip">>, Msg, Opts). +%% @doc Determine the reference of the caller. Presently only the `ip` form +%% may be used to identify the caller. +request_reference(Msg, Opts) -> hb_private:get(<<"ip">>, Msg, Opts). -check_limit(IP, Opts) -> +%% @doc Check if the caller is limited according to the current state of the +%% rate limiter server. +is_limited(Reference, Opts) -> PID = ensure_rate_limiter_started(Opts), - PID ! {request, Self = self(), IP}, + PID ! {request, self(), Reference}, receive - {rate_limit_result, Result} -> Result + {incremented, Balance} when Balance > 0 -> false; + {incremented, Balance} when Balance =< 0 -> {true, Balance} after ?LOOKUP_TIMEOUT -> ?event(warning, {rate_limit_timeout, restarting}), hb_name:unregister(server_id(Opts)), - check_limit(IP, Opts) + is_limited(Reference, Opts) end. +%% @doc Ensure that the rate limiter server is started and return the PID of +%% the server. In the event of two instanteous spawns, one of the new processes +%% will fail with an error and the other will succeed. The effect to the caller +%% is the same: A rate limiter is available to query. ensure_rate_limiter_started(Opts) -> - ServerID = server_id(Opts), - case hb_name:lookup({?MODULE, ServerID}) of + case hb_name:lookup(ServerID = server_id(Opts)) of PID when is_pid(PID) -> PID; undefined -> spawn( fun() -> - hb_name:register({?MODULE, ServerID}, self()), - Limit = - hb_opts:get( - rate_limit, - ?DEFAULT_RATE_LIMIT, - Opts - ), - BucketTime = - hb_opts:get( - rate_limit_bucket_time, - ?DEFAULT_BUCKET_TIME, - Opts - ), - ExemptPeers = hb_opts:get(rate_limit_exempt, [], Opts), + % Exit the process if we cannot register the server ID. + ok = hb_name:register(ServerID, self()), + Reqs = hb_opts:get(rate_limit_requests, ?DEFAULT_REQS, Opts), + Period = hb_opts:get(rate_limit_period, ?DEFAULT_PERIOD, Opts), + Max = hb_opts:get(rate_limit_max, ?DEFAULT_MAX, Opts), + Min = hb_opts:get(rate_limit_min, ?DEFAULT_MIN, Opts), + Exempt = hb_opts:get(rate_limit_exempt, [], Opts), ?event( rate_limit, {started_rate_limiter, {server_id, ServerID}, - {limit, Limit}, - {exempt_peers, ExemptPeers} + {reqs, Reqs}, + {period, Period}, + {max, Max}, + {min, Min}, + {exempt, Exempt} } ), server_loop( #{ - limit => Limit, - peers => #{ IP => infinity || IP <- ExemptPeers }, - bucket_time => BucketTime + reqs => Reqs, + period => Period, + max => Max, + min => Min, + peers => #{ Ref => infinity || Ref <- Exempt } } ) end ) end. +%% @doc The main loop of the rate limiter server. Only responds to two messages: +%% - `{request, Self, Reference}': Debit the account of the given reference by 1. +%% - `{balance, PID, Reference}': Return the current balance of the given reference. +%% The `balance` call is not presently used, but seems sensible to have. server_loop(State) -> + ?event({server_loop, {state, State}}), receive - {request, Self, IP} -> - NewState = increment(IP, State), - Self ! {rate_limit_result, is_limited(IP, NewState)}, - server_loop(NewState) + {request, PID, Reference} -> + NewState = debit(Reference, 1, State, Now = erlang:system_time(millisecond)), + ?event({state_after_debit, NewState}), + PID ! {incremented, account_balance(Reference, NewState, Now)}, + server_loop(NewState); + {balance, PID, Reference} -> + PID ! {balance, account_balance(Reference, State)}, + server_loop(State) end. -increment(IP, #{ bucket_time := BucketTime } = State) -> - increment(IP, erlang:system_time(second) div BucketTime, State). -increment(IP, Bucket, S = #{ peers := Peers }) -> - case maps:get(IP, Peers, #{}) of - infinity -> S; - #{ since := Bucket, count := Count } -> - S#{ peers => Peers#{ IP => #{ since => Bucket, count => Count + 1 }}}; - _ -> - S#{ peers => Peers#{ IP => #{ since => Bucket, count => 1 }}} +%% @doc Debit the account of the given reference by the given quantity. +debit(Ref, Amount, State = #{ peers := Peers, min := Min }, Now) -> + case account_balance(Ref, State, Now) of + infinity -> State; + Balance -> + State#{ + peers => + Peers#{ + Ref => + #{ + balance => max(Min, Balance - Amount), + last => Now + } + } + } end. -%% @doc Check if the IP is limited. Assumes the IP is in the state (added by -%% increment/2). -is_limited(IP, #{ peers := Peers }) when map_get(IP, Peers) =:= infinity -> false; -is_limited(IP, #{ limit := Limit, peers := Peers }) -> - maps:get(count, maps:get(IP, Peers, #{}), 0) > Limit. +%% @doc Calculate the current balance for a user, including unused capacity +%% accrued since the last interaction. +account_balance(Reference, State) -> + account_balance(Reference, State, erlang:system_time(millisecond)). +account_balance( + Reference, + #{ max := Max, reqs := Reqs, period := Period, peers := Peers }, + Time + ) -> + ?event({account_balance, {target, Reference}, {peers, Peers}, {time, Time}}), + case maps:get(Reference, Peers, not_found) of + infinity -> infinity; + not_found -> Max; + #{ balance := Balance, last := LastInteraction } -> + RechargeRate = Reqs / (Period * 1000), + RechargedSinceLast = (Time - LastInteraction) * RechargeRate, + min(Max, Balance + RechargedSinceLast) + end. %%% Tests rate_limit_test() -> ServerOpts = #{ - rate_limit => 2, - rate_limit_exempt => [], - rate_limit_bucket_time => 10_000, + rate_limit_requests => 2, + rate_limit_period => 1, + rate_limit_max => 2, on => #{ <<"request">> => @@ -148,10 +229,12 @@ rate_limit_test() -> {ok, _}, hb_http:get(ServerNode, <<"id">>, #{}) ), + ?debug_wait(100), ?assertMatch( {ok, _}, hb_http:get(ServerNode, <<"id">>, #{}) ), + ?debug_wait(100), ?assertMatch( {error, #{ <<"status">> := 429 }}, hb_http:get(ServerNode, <<"id">>, #{}) @@ -159,9 +242,11 @@ rate_limit_test() -> rate_limit_reset_test() -> ServerOpts = #{ - rate_limit => 2, + rate_limit_requests => 2, + rate_limit_period => 1, + rate_limit_max => 2, + rate_limit_min => 0, rate_limit_exempt => [], - rate_limit_bucket_time => 2, on => #{ <<"request">> => @@ -173,5 +258,9 @@ rate_limit_reset_test() -> ServerNode = hb_http_server:start_node(ServerOpts), ?assertMatch({ok, _}, hb_http:get(ServerNode, <<"id">>, #{})), ?assertMatch({ok, _}, hb_http:get(ServerNode, <<"id">>, #{})), - timer:sleep(2_000), + ?assertMatch( + {error, #{ <<"status">> := 429 }}, + hb_http:get(ServerNode, <<"id">>, #{}) + ), + timer:sleep(1_000), ?assertMatch({ok, _}, hb_http:get(ServerNode, <<"id">>, #{})). \ No newline at end of file From 5b5ecc6a8a9dbe1494f26351d5e9e47519266b1c Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 6 Mar 2026 17:10:04 -0500 Subject: [PATCH 029/135] impr: terminate streams with 429'd peers --- src/hb_http.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/hb_http.erl b/src/hb_http.erl index dc582dc83..5a31f2f1a 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -501,7 +501,12 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> ), ReqBeforeStream = Req#{ resp_headers => EncodedHeaders }, PostStreamReq = cowboy_req:stream_reply(Status, #{}, ReqBeforeStream), - cowboy_req:stream_body(EncodedBody, nofin, PostStreamReq), + Fin = + case should_finalize_stream(Status, EncodedBody) of + true -> fin; + false -> nofin + end, + cowboy_req:stream_body(EncodedBody, Fin, PostStreamReq), EndTime = os:system_time(millisecond), ReqDuration = EndTime - hb_maps:get(start_time, Req, undefined, Opts), ReplyDuration = EndTime - ReplyStartTime, @@ -529,6 +534,10 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> ), {ok, PostStreamReq, no_state}. +%% @doc Determine if the stream should be finalized. +should_finalize_stream(429, _EncodedBody) -> true; +should_finalize_stream(_, _EncodedBody) -> false. + %% @doc Handle replying with cookies if the message contains them. Returns the %% new Cowboy `Req` object, and the message with the cookies removed. Both %% `set-cookie' and `cookie' fields are treated as viable sources of cookies. From ebbdade33a3c257df3f29c1cc9d20c8791794495 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 6 Mar 2026 17:10:49 -0500 Subject: [PATCH 030/135] impr: return 429 error directly, rather than applying any subsequent hooks --- src/dev_rate_limit.erl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/dev_rate_limit.erl b/src/dev_rate_limit.erl index bcced5538..87c8d9c87 100644 --- a/src/dev_rate_limit.erl +++ b/src/dev_rate_limit.erl @@ -72,17 +72,12 @@ request(_, Msg, Opts) -> ), % Transform the given request into a request to return a 429 status % code and response. - {ok, + {error, #{ - <<"body">> => - [ - #{ - <<"status">> => 429, - <<"reason">> => <<"rate-limited">>, - <<"body">> => <<"Rate limit exceeded.">>, - <<"retry-after">> => RetryAfterBin - } - ] + <<"status">> => 429, + <<"reason">> => <<"rate-limited">>, + <<"body">> => <<"Rate limit exceeded.">>, + <<"retry-after">> => RetryAfterBin } }; false -> @@ -166,7 +161,12 @@ server_loop(State) -> {request, PID, Reference} -> NewState = debit(Reference, 1, State, Now = erlang:system_time(millisecond)), ?event({state_after_debit, NewState}), - PID ! {incremented, account_balance(Reference, NewState, Now)}, + Balance = account_balance(Reference, NewState, Now), + ?event( + rate_limit_short, + {rate_limit_debited, {target, Reference}, {balance, Balance}} + ), + PID ! {incremented, Balance}, server_loop(NewState); {balance, PID, Reference} -> PID ! {balance, account_balance(Reference, State)}, From a31684c965009979ed2d7d2f551ea4778f3378c7 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Fri, 6 Mar 2026 14:04:16 -0500 Subject: [PATCH 031/135] impr: refactor dev_bundler to remove redundant dispatch server Now the dev_bundler server manages the dispatch workers directly --- src/dev_bundler.erl | 898 +++++++++++++++++++++++++++-- src/dev_bundler_cache.erl | 6 +- src/dev_bundler_dispatch.erl | 928 +----------------------------- src/include/dev_bundler.hrl | 40 ++ test/arbundles.js/upload-items.js | 24 +- 5 files changed, 899 insertions(+), 997 deletions(-) create mode 100644 src/include/dev_bundler.hrl diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index 439311959..e7e636c3a 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -15,8 +15,9 @@ %%% available for reading instantly (`optimistically'), even before the %%% transaction is dispatched. -module(dev_bundler). --export([tx/3, item/3]). +-export([tx/3, item/3, ensure_server/1, stop_server/0, get_state/0]). -include("include/hb.hrl"). +-include("include/dev_bundler.hrl"). -include_lib("eunit/include/eunit.hrl"). %%% Default options. @@ -114,59 +115,84 @@ stop_server() -> hb_name:unregister(?SERVER_NAME) end. +%% @doc Return the current bundler server state for tests. +get_state() -> + case hb_name:lookup(?SERVER_NAME) of + undefined -> undefined; + PID -> + PID ! {get_state, self(), Ref = make_ref()}, + receive + {state, Ref, State} -> State + after 1000 -> timeout + end + end. + %% @doc Initialize the bundler server. init(Opts) -> - % Start the dispatcher to recover any in-progress bundles - dev_bundler_dispatch:ensure_dispatcher(Opts), - % Recover any unbundled items from cache + NumWorkers = hb_opts:get(bundler_workers, ?DEFAULT_NUM_WORKERS, Opts), + Workers = lists:map( + fun(_) -> + WorkerPID = spawn_link(fun dev_bundler_dispatch:worker_loop/0), + {WorkerPID, idle} + end, + lists:seq(1, NumWorkers) + ), {UnbundledItems, RecoveredBytes} = recover_unbundled_items(Opts), - InitialState = #{ - max_size => hb_opts:get( - bundler_max_size, ?DEFAULT_MAX_SIZE, Opts), - max_idle_time => hb_opts:get( + InitialState = #state{ + max_size = hb_opts:get(bundler_max_size, ?DEFAULT_MAX_SIZE, Opts), + max_idle_time = hb_opts:get( bundler_max_idle_time, ?DEFAULT_MAX_IDLE_TIME, Opts), - max_items => hb_opts:get( - bundler_max_items, ?DEFAULT_MAX_ITEMS, Opts), - queue => UnbundledItems, - bytes => RecoveredBytes + max_items = hb_opts:get(bundler_max_items, ?DEFAULT_MAX_ITEMS, Opts), + queue = UnbundledItems, + bytes = RecoveredBytes, + workers = maps:from_list(Workers), + task_queue = queue:new(), + bundles = #{}, + opts = Opts }, - % If recovered items are ready to dispatch, do so immediately - State = maybe_dispatch(InitialState, Opts), - server(State, Opts). + State = maybe_dispatch(recover_bundles(InitialState)), + server(assign_tasks(State), Opts). %% @doc Recover unbundled items from cache and calculate their total size. %% Returns {Items, TotalBytes}. recover_unbundled_items(Opts) -> UnbundledItems = dev_bundler_cache:load_unbundled_items(Opts), ?event(bundler_short, {recovered_unbundled_items, length(UnbundledItems)}), - % Calculate total bytes for recovered items - RecoveredBytes = lists:foldl( - fun(Item, Acc) -> - Acc + erlang:external_size(Item) - end, - 0, - UnbundledItems - ), + RecoveredBytes = queue_bytes(UnbundledItems), {UnbundledItems, RecoveredBytes}. -%% @doc The main loop of the bundler server. Simply waits for messages to be -%% added to the queue, and then dispatches them when the queue is large enough. -server(State = #{ max_idle_time := MaxIdleTime }, Opts) -> +%% @doc The main loop of the bundler server. +server(State = #state{max_idle_time = MaxIdleTime}, Opts) -> receive {item, Item} -> - server(maybe_dispatch(add_to_queue(Item, State, Opts), Opts), Opts); + State1 = add_to_queue(Item, State, Opts), + server(assign_tasks(maybe_dispatch(State1)), Opts); + {task_complete, WorkerPID, Task, Result} -> + State1 = handle_task_complete(WorkerPID, Task, Result, State), + server(assign_tasks(State1), Opts); + {task_failed, WorkerPID, Task, Reason} -> + State1 = handle_task_failed(WorkerPID, Task, Reason, State), + server(assign_tasks(State1), Opts); + {retry_task, Task} -> + State1 = enqueue_task(Task, State), + server(assign_tasks(State1), Opts); + {get_state, From, Ref} -> + From ! {state, Ref, State}, + server(State, Opts); stop -> + maps:foreach( + fun(WorkerPID, _) -> WorkerPID ! stop end, + State#state.workers + ), exit(normal) after MaxIdleTime -> - Q = maps:get(queue, State), - dev_bundler_dispatch:dispatch(Q, Opts), - server(State#{ queue => [] }, Opts) + server(assign_tasks(dispatch_queue(State)), Opts) end. %% @doc Add an item to the queue. Update the state with the new queue %% and approximate total byte size of the queue. %% Note: Item has already been verified and cached before reaching here. -add_to_queue(Item, State = #{ queue := Queue, bytes := Bytes }, Opts) -> +add_to_queue(Item, State = #state{queue = Queue, bytes = Bytes}, Opts) -> ItemSize = erlang:external_size(Item), NewQueue = [Item | Queue], NewBytes = Bytes + ItemSize, @@ -176,28 +202,20 @@ add_to_queue(Item, State = #{ queue := Queue, bytes := Bytes }, Opts) -> {queue_size, length(NewQueue)}, {queue_bytes, NewBytes} }), - State#{ - queue => NewQueue, - bytes => NewBytes - }. + State#state{queue = NewQueue, bytes = NewBytes}. %% @doc Dispatch the queue if it is ready. %% Only dispatches up to max_items at a time to respect the limit. -maybe_dispatch(State = #{queue := Q, max_items := MaxItems}, Opts) -> - case dispatchable(State, Opts) of +maybe_dispatch(State = #state{queue = Q, max_items = MaxItems}) -> + case dispatchable(State) of true -> - % Only dispatch up to max_items, keep the rest in queue {ToDispatch, Remaining} = split_queue(Q, MaxItems), - dev_bundler_dispatch:dispatch(ToDispatch, Opts), - % Recalculate bytes for remaining items - RemainingBytes = lists:foldl( - fun(Item, Acc) -> Acc + erlang:external_size(Item) end, - 0, - Remaining - ), - NewState = State#{queue => Remaining, bytes => RemainingBytes}, - % Check if we should dispatch again (in case we have more than max_items) - maybe_dispatch(NewState, Opts); + State1 = dispatch_items(ToDispatch, State), + NewState = State1#state{ + queue = Remaining, + bytes = queue_bytes(Remaining) + }, + maybe_dispatch(NewState); false -> State end. @@ -209,15 +227,280 @@ split_queue(Queue, MaxItems) -> {ToDispatch, Remaining}. %% @doc Returns whether the queue is dispatchable. -dispatchable(#{ queue := Q, max_items := MaxLen }, _Opts) - when length(Q) >= MaxLen -> +dispatchable(#state{queue = Q, max_items = MaxLen}) when length(Q) >= MaxLen -> true; -dispatchable(#{ bytes := Bytes, max_size := MaxSize }, _Opts) - when Bytes >= MaxSize -> +dispatchable(#state{bytes = Bytes, max_size = MaxSize}) when Bytes >= MaxSize -> true; -dispatchable(_State, _Opts) -> +dispatchable(_State) -> false. +%% @doc Return the total size of a queue of items. +queue_bytes(Items) -> + lists:foldl( + fun(Item, Acc) -> Acc + erlang:external_size(Item) end, + 0, + Items + ). + +%% @doc Dispatch all currently queued items immediately. +dispatch_queue(State = #state{queue = []}) -> + State; +dispatch_queue(State = #state{queue = Queue}) -> + dispatch_items(Queue, State#state{queue = [], bytes = 0}). + +%% @doc Create a bundle and enqueue its initial post task. +dispatch_items([], State) -> + State; +dispatch_items(Items, State = #state{bundles = Bundles, opts = Opts}) -> + BundleID = make_ref(), + Bundle = #bundle{ + id = BundleID, + items = Items, + status = initializing, + tx = undefined, + proofs = #{}, + start_time = erlang:timestamp() + }, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle, Bundles) + }, + ?event( + bundler_short, + {dispatching_bundle, + {timestamp, dev_bundler_dispatch:format_timestamp()}, + {bundle_id, BundleID}, + {num_items, length(Items)} + } + ), + Task = #task{ + bundle_id = BundleID, + type = post_tx, + data = Items, + opts = Opts + }, + enqueue_task(Task, State1). + +%% @doc Enqueue a task for worker execution. +enqueue_task(Task, State = #state{task_queue = Queue}) -> + State#state{task_queue = queue:in(Task, Queue)}. + +%% @doc Assign pending tasks to all idle workers. +assign_tasks(State = #state{workers = Workers}) -> + IdleWorkers = maps:filter( + fun(_, Status) -> Status =:= idle end, + Workers + ), + assign_tasks(maps:keys(IdleWorkers), State). + +assign_tasks([], State) -> + State; +assign_tasks([WorkerPID | Rest], State = #state{workers = Workers, task_queue = Queue}) -> + case queue:out(Queue) of + {{value, Task}, Queue1} -> + WorkerPID ! {execute_task, self(), Task}, + State1 = State#state{ + task_queue = Queue1, + workers = maps:put(WorkerPID, {busy, Task}, Workers) + }, + assign_tasks(Rest, State1); + {empty, _} -> + State + end. + +%% @doc Handle successful task completion. +handle_task_complete(WorkerPID, Task, Result, State = #state{ + workers = Workers, + bundles = Bundles + }) -> + #task{bundle_id = BundleID} = Task, + ?event(bundler_debug, {task_complete, dev_bundler_dispatch:format_task(Task)}), + State1 = State#state{ + workers = maps:put(WorkerPID, idle, Workers) + }, + case maps:get(BundleID, Bundles, undefined) of + undefined -> + ?event(bundler_short, {bundle_not_found, BundleID}), + State1; + Bundle -> + task_completed(Task, Bundle, Result, State1) + end. + +%% @doc Handle task failure and schedule a retry. +handle_task_failed(WorkerPID, Task, Reason, State = #state{ + workers = Workers, + opts = Opts + }) -> + RetryCount = Task#task.retry_count, + BaseDelay = hb_opts:get( + retry_base_delay_ms, ?DEFAULT_RETRY_BASE_DELAY_MS, Opts), + MaxDelay = hb_opts:get( + retry_max_delay_ms, ?DEFAULT_RETRY_MAX_DELAY_MS, Opts), + Jitter = hb_opts:get(retry_jitter, ?DEFAULT_RETRY_JITTER, Opts), + BaseDelayWithBackoff = min(BaseDelay * (1 bsl RetryCount), MaxDelay), + JitterFactor = (rand:uniform() * 2 - 1) * Jitter, + Delay = round(BaseDelayWithBackoff * (1 + JitterFactor)), + ?event( + bundler_short, + {task_failed_retrying, + dev_bundler_dispatch:format_task(Task), + {reason, {explicit, Reason}}, + {retry_count, RetryCount}, + {delay_ms, Delay} + } + ), + Task1 = Task#task{retry_count = RetryCount + 1}, + erlang:send_after(Delay, self(), {retry_task, Task1}), + State#state{ + workers = maps:put(WorkerPID, idle, Workers) + }. + +%% @doc Apply task completion effects to server state. +task_completed(#task{bundle_id = BundleID, type = post_tx}, Bundle, CommittedTX, State) -> + Bundles = State#state.bundles, + Opts = State#state.opts, + Bundle1 = Bundle#bundle{status = tx_posted, tx = CommittedTX}, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle1, Bundles) + }, + BuildProofsTask = #task{ + bundle_id = BundleID, + type = build_proofs, + data = CommittedTX, + opts = Opts + }, + enqueue_task(BuildProofsTask, State1); +task_completed(#task{bundle_id = BundleID, type = build_proofs}, Bundle, Proofs, State) -> + Bundles = State#state.bundles, + Opts = State#state.opts, + case Proofs of + [] -> + bundle_complete(Bundle, State); + _ -> + ProofsMap = maps:from_list([ + {maps:get(offset, Proof), #proof{proof = Proof, status = pending}} + || Proof <- Proofs + ]), + Bundle1 = Bundle#bundle{ + proofs = ProofsMap, + status = proofs_built + }, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle1, Bundles) + }, + lists:foldl( + fun(ProofData, StateAcc) -> + ProofTask = #task{ + bundle_id = BundleID, + type = post_proof, + data = ProofData, + opts = Opts + }, + enqueue_task(ProofTask, StateAcc) + end, + State1, + Proofs + ) + end; +task_completed( + #task{bundle_id = BundleID, type = post_proof, data = ProofData}, + Bundle, + _Result, + State + ) -> + Bundles = State#state.bundles, + Offset = maps:get(offset, ProofData), + Proofs = Bundle#bundle.proofs, + Proofs1 = maps:update_with( + Offset, + fun(Proof) -> Proof#proof{status = seeded} end, + Proofs + ), + Bundle1 = Bundle#bundle{proofs = Proofs1}, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle1, Bundles) + }, + AllSeeded = lists:all( + fun(#proof{status = Status}) -> Status =:= seeded end, + maps:values(Proofs1) + ), + case AllSeeded of + true -> + bundle_complete(Bundle1, State1); + false -> + State1 + end. + +%% @doc Mark a bundle as complete and remove it from state. +bundle_complete(Bundle, State = #state{opts = Opts}) -> + ok = dev_bundler_cache:complete_tx(Bundle#bundle.tx, Opts), + ElapsedTime = + timer:now_diff(erlang:timestamp(), Bundle#bundle.start_time) / 1000000, + ?event( + bundler_short, + {bundle_complete, + {bundle_id, Bundle#bundle.id}, + {timestamp, dev_bundler_dispatch:format_timestamp()}, + {tx, {explicit, hb_message:id(Bundle#bundle.tx, signed, Opts)}}, + {elapsed_time_s, ElapsedTime} + } + ), + State#state{bundles = maps:remove(Bundle#bundle.id, State#state.bundles)}. + +%% @doc Recover in-progress bundles from cache after a crash. +recover_bundles(State = #state{opts = Opts}) -> + lists:foldl( + fun({TXID, Status}, StateAcc) -> + recover_bundle(TXID, Status, StateAcc) + end, + State, + dev_bundler_cache:load_bundle_states(Opts) + ). + +%% @doc Recover a single bundle and enqueue any follow-up work. +recover_bundle(TXID, Status, State = #state{opts = Opts}) -> + ?event( + bundler_short, + {recovering_bundle, + {tx_id, {explicit, TXID}}, + {status, Status} + } + ), + try + CommittedTX = dev_bundler_cache:load_tx(TXID, Opts), + Items = dev_bundler_cache:load_bundled_items(TXID, Opts), + BundleID = make_ref(), + Bundle = #bundle{ + id = BundleID, + items = Items, + status = tx_posted, + tx = CommittedTX, + proofs = #{}, + start_time = erlang:timestamp() + }, + Bundles = State#state.bundles, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle, Bundles) + }, + Task = #task{ + bundle_id = BundleID, + type = build_proofs, + data = CommittedTX, + opts = Opts + }, + enqueue_task(Task, State1) + catch + _:Error:Stack -> + ?event( + bundler_short, + {failed_to_recover_bundle, + {tx_id, {explicit, TXID}}, + {error, Error}, + {stack, Stack} + } + ), + State + end. + %%%=================================================================== %%% Tests %%%=================================================================== @@ -516,6 +799,458 @@ recover_respects_max_items_test() -> stop_test_servers(ServerHandle) end. +complete_task_sequence_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 2, + retry_base_delay_ms => 100, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [ + new_structured_data_item(1, 10, Opts), + new_structured_data_item(2, 10, Opts) + ], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + ?assertEqual(1, length(TXs)), + Proofs = hb_mock_server:get_requests(chunk, 1, ServerHandle), + ?assertEqual(1, length(Proofs)), + State = get_state(), + ?assertNotEqual(undefined, State), + ?assertNotEqual(timeout, State), + Workers = State#state.workers, + IdleWorkers = [ + PID + || {PID, Status} <- maps:to_list(Workers), Status =:= idle + ], + ?assertEqual(maps:size(Workers), length(IdleWorkers)), + Queue = State#state.task_queue, + ?assert(queue:is_empty(Queue)), + Bundles = State#state.bundles, + ?assertEqual(0, maps:size(Bundles)), + ok + after + stop_test_servers(ServerHandle) + end. + +recover_bundles_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store() + }, + hb_http_server:start_node(Opts), + Item1 = new_structured_data_item(1, 10, Opts), + Item2 = new_structured_data_item(2, 10, Opts), + Item3 = new_structured_data_item(3, 10, Opts), + ok = dev_bundler_cache:write_item(Item1, Opts), + ok = dev_bundler_cache:write_item(Item2, Opts), + ok = dev_bundler_cache:write_item(Item3, Opts), + {ok, TX} = dev_codec_tx:to( + lists:reverse([Item1, Item2, Item3]), #{}, #{}), + CommittedTX = hb_message:convert( + TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), + ok = dev_bundler_cache:write_tx(CommittedTX, [Item1, Item2, Item3], Opts), + Item4 = new_structured_data_item(4, 10, Opts), + ok = dev_bundler_cache:write_item(Item4, Opts), + {ok, TX2} = dev_codec_tx:to(lists:reverse([Item4]), #{}, #{}), + CommittedTX2 = hb_message:convert( + TX2, <<"structured@1.0">>, <<"tx@1.0">>, Opts), + ok = dev_bundler_cache:write_tx(CommittedTX2, [Item4], Opts), + ok = dev_bundler_cache:complete_tx(CommittedTX2, Opts), + ensure_server(Opts), + State = get_state(), + ?assertEqual(1, maps:size(State#state.bundles)), + [Bundle] = maps:values(State#state.bundles), + ?assertNotEqual(undefined, Bundle#bundle.start_time), + ?assertEqual(#{}, Bundle#bundle.proofs), + RecoveredItems = [ + hb_message:with_commitments( + #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) + || Item <- Bundle#bundle.items + ], + ?assertEqual( + lists:sort([Item1, Item2, Item3]), + lists:sort(RecoveredItems) + ), + ?assertEqual(tx_posted, Bundle#bundle.status), + ?assert(hb_message:verify(Bundle#bundle.tx)), + ?assertEqual( + hb_message:id(CommittedTX, signed, Opts), + hb_message:id(Bundle#bundle.tx, signed, Opts) + ), + ok + after + stop_test_servers(ServerHandle) + end. + +post_tx_price_failure_retry_test() -> + Anchor = rand:bytes(32), + FailCount = 3, + setup_test_counter(price_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => fun(_Req) -> + Count = increment_test_counter(price_attempts_counter) - 1, + case Count < FailCount of + true -> {500, <<"error">>}; + false -> {200, <<"12345">>} + end + end, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 50, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + ?assertEqual(1, length(TXs)), + FinalCount = get_test_counter(price_attempts_counter), + ?assertEqual(FailCount + 1, FinalCount), + ok + after + cleanup_test_counter(price_attempts_counter), + stop_test_servers(ServerHandle) + end. + +post_tx_anchor_failure_retry_test() -> + Price = 12345, + FailCount = 3, + setup_test_counter(anchor_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => fun(_Req) -> + Count = increment_test_counter(anchor_attempts_counter) - 1, + case Count < FailCount of + true -> {500, <<"error">>}; + false -> {200, hb_util:encode(rand:bytes(32))} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 50, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + ?assertEqual(1, length(TXs)), + FinalCount = get_test_counter(anchor_attempts_counter), + ?assertEqual(FailCount + 1, FinalCount), + ok + after + cleanup_test_counter(anchor_attempts_counter), + stop_test_servers(ServerHandle) + end. + +post_tx_post_failure_retry_test() -> + Anchor = rand:bytes(32), + Price = 12345, + FailCount = 4, + setup_test_counter(tx_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + Count = increment_test_counter(tx_attempts_counter) - 1, + case Count < FailCount of + true -> {400, <<"Transaction verification failed">>}; + false -> {200, <<"OK">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 50, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, FailCount + 1, ServerHandle), + ?assertEqual(FailCount + 1, length(TXs)), + FinalCount = get_test_counter(tx_attempts_counter), + ?assertEqual(FailCount + 1, FinalCount), + ok + after + cleanup_test_counter(tx_attempts_counter), + stop_test_servers(ServerHandle) + end. + +post_proof_failure_retry_test() -> + Anchor = rand:bytes(32), + Price = 12345, + FailCount = 2, + setup_test_counter(chunk_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + chunk => fun(_Req) -> + Count = increment_test_counter(chunk_attempts_counter) - 1, + case Count < FailCount of + true -> {500, <<"error">>}; + false -> {200, <<"OK">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 50, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, floor(4.5 * ?DATA_CHUNK_SIZE), Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + ?assertEqual(1, length(TXs)), + Chunks = hb_mock_server:get_requests(chunk, FailCount + 5, ServerHandle), + ?assertEqual(FailCount + 5, length(Chunks)), + FinalCount = get_test_counter(chunk_attempts_counter), + ?assertEqual(FailCount + 5, FinalCount), + ok + after + cleanup_test_counter(chunk_attempts_counter), + stop_test_servers(ServerHandle) + end. + +rapid_dispatch_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + timer:sleep(100), + {200, <<"OK">>} + end + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + bundler_workers => 3 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + lists:foreach( + fun(I) -> + Items = [new_structured_data_item(I, 10, Opts)], + submit_test_items(Items, Opts) + end, + lists:seq(1, 10) + ), + TXs = hb_mock_server:get_requests(tx, 10, ServerHandle), + ?assertEqual(10, length(TXs)), + ok + after + stop_test_servers(ServerHandle) + end. + +one_bundle_fails_others_continue_test() -> + Anchor = rand:bytes(32), + Price = 12345, + setup_test_counter(mixed_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + Count = increment_test_counter(mixed_attempts_counter) - 1, + case Count of + 0 -> {200, <<"OK">>}; + _ -> {400, <<"fail">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 100, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items1 = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items1, Opts), + Items2 = [new_structured_data_item(2, 10, Opts)], + submit_test_items(Items2, Opts), + TXs = hb_mock_server:get_requests(tx, 5, ServerHandle), + ?assert(length(TXs) >= 5, length(TXs)), + ok + after + cleanup_test_counter(mixed_attempts_counter), + stop_test_servers(ServerHandle) + end. + +parallel_task_execution_test() -> + Anchor = rand:bytes(32), + Price = 12345, + SleepTime = 120, + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + chunk => fun(_Req) -> + timer:sleep(SleepTime), + {200, <<"OK">>} + end + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + bundler_workers => 5 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + lists:foreach( + fun(I) -> + Items = [new_structured_data_item(I, 10, Opts)], + submit_test_items(Items, Opts) + end, + lists:seq(1, 10) + ), + StartTime = erlang:system_time(millisecond), + Chunks = hb_mock_server:get_requests(chunk, 10, ServerHandle), + ElapsedTime = erlang:system_time(millisecond) - StartTime, + ?assertEqual(10, length(Chunks)), + ?assert(ElapsedTime < 2000, "ElapsedTime: " ++ integer_to_list(ElapsedTime)), + ok + after + stop_test_servers(ServerHandle) + end. + +exponential_backoff_timing_test() -> + Anchor = rand:bytes(32), + Price = 12345, + FailCount = 5, + setup_test_counter(backoff_cap_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + Timestamp = erlang:system_time(millisecond), + Attempt = increment_test_counter(backoff_cap_counter), + Count = Attempt - 1, + add_test_attempt_timestamp(backoff_cap_counter, Attempt, Timestamp), + case Count < FailCount of + true -> {400, <<"fail">>}; + false -> {200, <<"OK">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 100, + retry_max_delay_ms => 500, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, FailCount + 1, ServerHandle, 5000), + ?assertEqual(FailCount + 1, length(TXs)), + Timestamps = test_attempt_timestamps(backoff_cap_counter), + ?assertEqual(6, length(Timestamps)), + [T1, T2, T3, T4, T5, T6] = Timestamps, + Delay1 = T2 - T1, + Delay2 = T3 - T2, + Delay3 = T4 - T3, + Delay4 = T5 - T4, + Delay5 = T6 - T5, + ?assert(Delay1 >= 70 andalso Delay1 =< 200, Delay1), + ?assert(Delay2 >= 150 andalso Delay2 =< 300, Delay2), + ?assert(Delay3 >= 300 andalso Delay3 =< 500, Delay3), + ?assert(Delay4 >= 400 andalso Delay4 =< 700, Delay4), + ?assert(Delay5 >= 400 andalso Delay5 =< 700, Delay5), + ok + after + cleanup_test_counter(backoff_cap_counter), + stop_test_servers(ServerHandle) + end. + +independent_task_retry_counts_test() -> + Anchor = rand:bytes(32), + Price = 12345, + setup_test_counter(independent_retry_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + Count = increment_test_counter(independent_retry_counter) - 1, + case Count < 2 of + true -> {400, <<"fail">>}; + false -> {200, <<"OK">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 100, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items1 = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items1, Opts), + hb_mock_server:get_requests(tx, 3, ServerHandle), + Items2 = [new_structured_data_item(2, 10, Opts)], + submit_test_items(Items2, Opts), + TotalAttempts = 4, + TXs = hb_mock_server:get_requests(tx, TotalAttempts, ServerHandle), + ?assertEqual(TotalAttempts, length(TXs)), + ok + after + cleanup_test_counter(independent_retry_counter), + stop_test_servers(ServerHandle) + end. + invalid_item_test() -> Anchor = rand:bytes(32), Price = 12345, @@ -584,14 +1319,12 @@ cache_write_failure_test() -> <<"error">> := <<"cache_write_failed">>}}, Result), ok after - stop_server(), - dev_bundler_dispatch:stop_dispatcher() + stop_server() end. stop_test_servers(ServerHandle) -> hb_mock_server:stop(ServerHandle), - stop_server(), - dev_bundler_dispatch:stop_dispatcher(). + stop_server(). test_bundle(Opts) -> Anchor = rand:bytes(32), @@ -660,6 +1393,24 @@ test_api_error(Responses) -> new_data_item(Index, Size) -> new_data_item(Index, Size, hb:wallet()). +new_structured_data_item(Index, Size, Opts) -> + hb_message:convert( + new_data_item(Index, Size), + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ). + +submit_test_items([], _Opts) -> + ok; +submit_test_items(Items, Opts) -> + lists:foreach( + fun(Item) -> + ?assertMatch({ok, _}, item(#{}, Item, Opts)) + end, + Items + ). + new_data_item(Index, Size, Wallet) -> Data = rand:bytes(Size), Tag = <<"tag", (integer_to_binary(Index))/binary>>, @@ -780,4 +1531,35 @@ start_mock_gateway(Responses) -> } ] }, - {ServerHandle, NodeOpts}. \ No newline at end of file + {ServerHandle, NodeOpts}. + +setup_test_counter(Table) -> + cleanup_test_counter(Table), + ets:new(Table, [named_table, public, set]), + ok. + +cleanup_test_counter(Table) -> + case ets:info(Table) of + undefined -> ok; + _ -> ets:delete(Table), ok + end. + +increment_test_counter(Table) -> + ets:update_counter(Table, Table, {2, 1}, {Table, 0}). + +get_test_counter(Table) -> + case ets:lookup(Table, Table) of + [{_, Value}] -> Value; + [] -> 0 + end. + +add_test_attempt_timestamp(Table, Attempt, Timestamp) -> + ets:insert(Table, {{Table, Attempt}, Timestamp}). + +test_attempt_timestamps(Table) -> + TimestampEntries = [ + {Attempt, Timestamp} + || {{Prefix1, Attempt}, Timestamp} <- ets:tab2list(Table), + Prefix1 =:= Table + ], + [Timestamp || {_, Timestamp} <- lists:sort(TimestampEntries)]. \ No newline at end of file diff --git a/src/dev_bundler_cache.erl b/src/dev_bundler_cache.erl index e0df60d6a..c38de8864 100644 --- a/src/dev_bundler_cache.erl +++ b/src/dev_bundler_cache.erl @@ -458,11 +458,7 @@ bundler_optimistic_cache_test() -> ), ok after - case hb_name:lookup(bundler_server) of - undefined -> ok; - PID -> PID ! stop, hb_name:unregister(bundler_server) - end, - dev_bundler_dispatch:stop_dispatcher() + dev_bundler:stop_server() end. new_data_item(Index, SizeOrData, Opts) -> diff --git a/src/dev_bundler_dispatch.erl b/src/dev_bundler_dispatch.erl index 2c286091c..174fa47c3 100644 --- a/src/dev_bundler_dispatch.erl +++ b/src/dev_bundler_dispatch.erl @@ -1,166 +1,13 @@ %%% @doc A dispatcher for the bundler device (dev_bundler). This module %%% manages a worker pool to handle bundle building, TX posting, proof -%%% generation, and chunk seeding. Failed tasks are automatically re-queued -%%% for immediate retry until successful. +%%% generation, and chunk seeding. Server-side dispatch state lives in +%%% `dev_bundler'; this module only owns worker execution. -module(dev_bundler_dispatch). --export([dispatch/2, ensure_dispatcher/1, stop_dispatcher/0]). +-export([worker_loop/0, format_task/1, format_timestamp/0]). -include("include/hb.hrl"). +-include("include/dev_bundler.hrl"). -include_lib("eunit/include/eunit.hrl"). -%%% State record for the dispatcher process. --record(state, { - workers, % Map of WorkerPID => idle | {busy, Task} - task_queue, % Queue of pending tasks - bundles, % Map of BundleID => #bundle{} - opts % Configuration options -}). - -%%% Task record representing work to be done by a worker. --record(task, { - bundle_id, % ID of the bundle this task belongs to - type, % Task type: post_tx | build_proofs | post_proof - data, % Task-specific data (map) - opts, % Configuration options - retry_count = 0 % Number of times this task has been retried -}). - -%%% Proof record to track individual proof seeding status. --record(proof, { - proof, % The proof data (chunk, merkle path, etc) - status % pending | seeded -}). - -%%% Bundle record to track bundle progress through the dispatch pipeline. --record(bundle, { - id, % Unique bundle identifier - items, % List of dataitems to bundle - status, % Current state (initializing, tx_built, tx_posted, proofs_built) - tx, % The built/signed transaction - proofs, % Map of offset => #proof{} records - start_time % The time the bundle was started -}). - -%%% Default options. --define(DISPATCHER_NAME, bundler_dispatcher). --define(DEFAULT_NUM_WORKERS, 20). --define(DEFAULT_RETRY_BASE_DELAY_MS, 1000). --define(DEFAULT_RETRY_MAX_DELAY_MS, 600000). % 10 minutes --define(DEFAULT_RETRY_JITTER, 0.25). % ±25% jitter - -%% @doc Dispatch the queue. -dispatch([], _Opts) -> - ok; -dispatch(Items, Opts) -> - PID = ensure_dispatcher(Opts), - PID ! {dispatch, Items}. - -%% @doc Return the PID of the dispatch server. If the server is not running, -%% it is started and registered with the name `?SERVER_NAME'. -ensure_dispatcher(Opts) -> - case hb_name:lookup(?DISPATCHER_NAME) of - undefined -> - PID = spawn(fun() -> init(Opts) end), - ?event(bundler_short, {starting_dispatcher, {pid, PID}}), - hb_name:register(?DISPATCHER_NAME, PID), - hb_name:lookup(?DISPATCHER_NAME); - PID -> PID - end. - -stop_dispatcher() -> - case hb_name:lookup(?DISPATCHER_NAME) of - undefined -> ok; - PID -> - PID ! stop, - hb_name:unregister(?DISPATCHER_NAME) - end. - -get_state() -> - case hb_name:lookup(?DISPATCHER_NAME) of - undefined -> undefined; - PID -> - PID ! {get_state, self(), Ref = make_ref()}, - receive - {state, Ref, State} -> State - after 1000 -> timeout - end - end. - -%% @doc Initialize the dispatcher with worker pool. -init(Opts) -> - NumWorkers = hb_opts:get(bundler_workers, ?DEFAULT_NUM_WORKERS, Opts), - Workers = lists:map( - fun(_) -> - WorkerPID = spawn_link(fun() -> worker_loop() end), - {WorkerPID, idle} - end, - lists:seq(1, NumWorkers) - ), - State = #state{ - workers = maps:from_list(Workers), - task_queue = queue:new(), - bundles = #{}, - opts = Opts - }, - % Recover any in-progress bundles from cache - State1 = recover_bundles(State), - dispatcher(assign_tasks(State1)). - -%% @doc The main loop of the dispatcher. Manages task queue and worker pool. -dispatcher(State) -> - receive - {dispatch, Items} -> - % Create a new bundle and queue the post_tx task - Opts = State#state.opts, - BundleID = make_ref(), - Bundle = #bundle{ - id = BundleID, - items = Items, - status = initializing, - tx = undefined, - proofs = #{}, - start_time = erlang:timestamp() - }, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle, State#state.bundles) - }, - ?event(bundler_short, - {dispatching_bundle, - {timestamp, format_timestamp()}, - {bundle_id, BundleID}, - {num_items, length(Items)} - } - ), - Task = #task{bundle_id = BundleID, type = post_tx, data = Items, opts = Opts}, - State2 = enqueue_task(Task, State1), - % Assign tasks to idle workers - dispatcher(assign_tasks(State2)); - {task_complete, WorkerPID, Task, Result} -> - State1 = handle_task_complete(WorkerPID, Task, Result, State), - dispatcher(assign_tasks(State1)); - {task_failed, WorkerPID, Task, Reason} -> - State1 = handle_task_failed(WorkerPID, Task, Reason, State), - dispatcher(assign_tasks(State1)); - {retry_task, Task} -> - % Re-enqueue the task after backoff delay - State1 = enqueue_task(Task, State), - dispatcher(assign_tasks(State1)); - {get_state, From, Ref} -> - From ! {state, Ref, State}, - dispatcher(State); - stop -> - % Stop all workers - maps:foreach( - fun(WorkerPID, _) -> WorkerPID ! stop end, - State#state.workers - ), - exit(normal) - end. - -%% @doc Enqueue a task to the task queue. -enqueue_task(Task, State) -> - Queue = State#state.task_queue, - State#state{task_queue = queue:in(Task, Queue)}. - %% @doc Format a task for logging. format_task(#task{bundle_id = BundleID, type = post_tx, data = DataItems}) -> {post_tx, {timestamp, format_timestamp()}, {bundle, BundleID}, @@ -179,223 +26,6 @@ format_timestamp() -> Millisecs = (MegaSecs * 1000000 + Secs) * 1000 + (MicroSecs div 1000), calendar:system_time_to_rfc3339(Millisecs, [{unit, millisecond}, {offset, "Z"}]). -%% @doc Assign tasks to all idle workers until no idle workers -%% or no tasks remain. -assign_tasks(State) -> - IdleWorkers = maps:filter( - fun(_, Status) -> Status =:= idle end, - State#state.workers), - assign_tasks(maps:keys(IdleWorkers), State). - -assign_tasks([], State) -> - % No more idle workers - State; -assign_tasks([WorkerPID | Rest], State) -> - Workers = State#state.workers, - Queue = State#state.task_queue, - case queue:out(Queue) of - {{value, Task}, Queue1} -> - % Assign task to this worker - WorkerPID ! {execute_task, self(), Task}, - State1 = State#state{ - task_queue = Queue1, - workers = maps:put(WorkerPID, {busy, Task}, Workers) - }, - % Continue with remaining idle workers - assign_tasks(Rest, State1); - {empty, _} -> - % No more tasks, stop - State - end. - -handle_task_complete(WorkerPID, Task, Result, State) -> - Workers = State#state.workers, - Bundles = State#state.bundles, - #task{bundle_id = BundleID} = Task, - ?event(bundler_debug, {task_complete, format_task(Task)}), - % Update worker to idle - State1 = State#state{ - workers = maps:put(WorkerPID, idle, Workers) - }, - case maps:get(BundleID, Bundles, undefined) of - undefined -> - ?event(bundler_short, {bundle_not_found, BundleID}), - State1; - Bundle -> - task_completed(Task, Bundle, Result, State1) - end. - -handle_task_failed(WorkerPID, Task, Reason, State) -> - Workers = State#state.workers, - Opts = State#state.opts, - RetryCount = Task#task.retry_count, - % Calculate exponential backoff delay - BaseDelay = hb_opts:get(retry_base_delay_ms, ?DEFAULT_RETRY_BASE_DELAY_MS, Opts), - MaxDelay = hb_opts:get(retry_max_delay_ms, ?DEFAULT_RETRY_MAX_DELAY_MS, Opts), - Jitter = hb_opts:get(retry_jitter, ?DEFAULT_RETRY_JITTER, Opts), - % Compute base delay with exponential backoff: min(base * 2^retry_count, max_delay) - BaseDelayWithBackoff = min(BaseDelay * (1 bsl RetryCount), MaxDelay), - % Apply jitter: delay * (1 + random(-jitter, +jitter)) - % This distributes the delay across [delay * (1-jitter), delay * (1+jitter)] - JitterFactor = (rand:uniform() * 2 - 1) * Jitter, % Random value in [-jitter, +jitter] - Delay = round(BaseDelayWithBackoff * (1 + JitterFactor)), - ?event( - bundler_short, - {task_failed_retrying, format_task(Task), - {reason, {explicit, Reason}}, - {retry_count, RetryCount}, {delay_ms, Delay} - } - ), - % Update worker to idle - State1 = State#state{ - workers = maps:put(WorkerPID, idle, Workers) - }, - % Increment retry count and schedule delayed retry - Task1 = Task#task{retry_count = RetryCount + 1}, - erlang:send_after(Delay, self(), {retry_task, Task1}), - State1. - -task_completed(#task{bundle_id = BundleID, type = post_tx}, Bundle, CommittedTX, State) -> - Bundles = State#state.bundles, - Opts = State#state.opts, - Bundle1 = Bundle#bundle{status = tx_posted, tx = CommittedTX}, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle1, Bundles) - }, - BuildProofsTask = #task{ - bundle_id = BundleID, type = build_proofs, - data = CommittedTX, opts = Opts}, - enqueue_task(BuildProofsTask, State1); - -task_completed(#task{bundle_id = BundleID, type = build_proofs}, Bundle, Proofs, State) -> - Bundles = State#state.bundles, - Opts = State#state.opts, - case Proofs of - [] -> - % No proofs, bundle complete - bundle_complete(Bundle, State); - _ -> - % Proofs built, wrap each in a proof record with offset as key - ProofsMap = maps:from_list([ - {maps:get(offset, P), #proof{proof = P, status = pending}} || P <- Proofs - ]), - Bundle1 = Bundle#bundle{ - proofs = ProofsMap, - status = proofs_built - }, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle1, Bundles) - }, - % Enqueue all post_proof tasks - lists:foldl( - fun(ProofData, S) -> - ProofTask = #task{ - bundle_id = BundleID, - type = post_proof, - data = ProofData, - opts = Opts - }, - enqueue_task(ProofTask, S) - end, - State1, - Proofs - ) - end; - -task_completed(#task{bundle_id = BundleID, type = post_proof, data = ProofData}, Bundle, _Result, State) -> - Bundles = State#state.bundles, - Offset = maps:get(offset, ProofData), - Proofs = Bundle#bundle.proofs, - Proofs1 = maps:update_with( - Offset, - fun(P) -> P#proof{status = seeded} end, - Proofs - ), - Bundle1 = Bundle#bundle{proofs = Proofs1}, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle1, Bundles) - }, - % Check if all proofs are seeded - AllSeeded = lists:all( - fun(#proof{status = Status}) -> Status =:= seeded end, - maps:values(Proofs1) - ), - case AllSeeded of - true -> - bundle_complete(Bundle, State1); - false -> - State1 - end. - -%% @doc Mark a bundle as complete and remove it from state. -bundle_complete(Bundle, State) -> - Opts = State#state.opts, - ok = dev_bundler_cache:complete_tx(Bundle#bundle.tx, Opts), - ElapsedTime = - timer:now_diff(erlang:timestamp(), Bundle#bundle.start_time) / 1000000, - ?event(bundler_short, {bundle_complete, {bundle_id, Bundle#bundle.id}, - {timestamp, format_timestamp()}, - {tx, {explicit, hb_message:id(Bundle#bundle.tx, signed, Opts)}}, - {elapsed_time_s, ElapsedTime}}), - State#state{bundles = maps:remove(Bundle#bundle.id, State#state.bundles)}. - -%%% Recovery - -%% @doc Recover in-progress bundles from cache after a crash. -recover_bundles(State) -> - Opts = State#state.opts, - % Reconstruct bundles and enqueue appropriate tasks - lists:foldl( - fun({TXID, Status}, StateAcc) -> - recover_bundle(TXID, Status, StateAcc) - end, - State, - dev_bundler_cache:load_bundle_states(Opts) - ). - -%% @doc Recover a single bundle based on its cached state. -recover_bundle(TXID, Status, State) -> - Opts = State#state.opts, - ?event(bundler_short, {recovering_bundle, - {tx_id, {explicit, TXID}}, - {status, Status} - }), - try - % Load the TX and its items - CommittedTX = dev_bundler_cache:load_tx(TXID, Opts), - Items = dev_bundler_cache:load_bundled_items(TXID, Opts), - % Create a new bundle record - BundleID = make_ref(), - Bundle = #bundle{ - id = BundleID, - items = Items, - status = tx_posted, - tx = CommittedTX, - proofs = #{}, - start_time = erlang:timestamp() - }, - % Add bundle to state - Bundles = State#state.bundles, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle, Bundles) - }, - - % Enqueue appropriate task based on status - Task = #task{ - bundle_id = BundleID, type = build_proofs, - data = CommittedTX, opts = Opts}, - enqueue_task(Task, State1) - catch - _:Error:Stack -> - ?event(bundler_short, {failed_to_recover_bundle, - {tx_id, {explicit, TXID}}, - {error, Error}, - {stack, Stack} - }), - % Skip this bundle and continue - State - end. - %%% Worker implementation %% @doc Worker loop - executes tasks and reports back to dispatcher. @@ -566,553 +196,3 @@ get_anchor(Opts) -> Opts ). -%%%=================================================================== -%%% Tests -%%%=================================================================== - -complete_task_sequence_test() -> - Anchor = rand:bytes(32), - Price = 12345, - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)} - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 100, - retry_jitter => 0 - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts), new_data_item(2, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to be posted - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), - ?assertEqual(1, length(TXs)), - % Wait for chunk to be posted - Proofs = hb_mock_server:get_requests(chunk, 1, ServerHandle), - ?assertEqual(1, length(Proofs)), - % Verify dispatcher state - State = get_state(), - ?assertNotEqual(undefined, State), - ?assertNotEqual(timeout, State), - % All workers should be idle - Workers = State#state.workers, - IdleWorkers = [PID || {PID, Status} <- maps:to_list(Workers), Status =:= idle], - ?assertEqual(maps:size(Workers), length(IdleWorkers)), - % Task queue should be empty - Queue = State#state.task_queue, - ?assert(queue:is_empty(Queue)), - % Bundle should be completed and removed - Bundles = State#state.bundles, - ?assertEqual(0, maps:size(Bundles)), - ok - after - cleanup_dispatcher(ServerHandle) - end. - -post_tx_price_failure_retry_test() -> - Anchor = rand:bytes(32), - FailCount = 3, - setup_test_counter(price_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => fun(_Req) -> - Count = increment_test_counter(price_attempts_counter) - 1, - case Count < FailCount of - true -> {500, <<"error">>}; - false -> {200, <<"12345">>} - end - end, - tx_anchor => {200, hb_util:encode(Anchor)} - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 50, - retry_jitter => 0 - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to eventually be posted - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), - ?assertEqual(1, length(TXs)), - % Verify it retried multiple times - FinalCount = get_test_counter(price_attempts_counter), - ?assertEqual(FailCount+1, FinalCount), - ok - after - cleanup_test_counter(price_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -post_tx_anchor_failure_retry_test() -> - Price = 12345, - FailCount = 3, - setup_test_counter(anchor_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => fun(_Req) -> - Count = increment_test_counter(anchor_attempts_counter) - 1, - case Count < FailCount of - true -> {500, <<"error">>}; - false -> {200, hb_util:encode(rand:bytes(32))} - end - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 50, - retry_jitter => 0 - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to eventually be posted - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), - ?assertEqual(1, length(TXs)), - % Verify it retried multiple times - FinalCount = get_test_counter(anchor_attempts_counter), - ?assertEqual(FailCount+1, FinalCount), - ok - after - cleanup_test_counter(anchor_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -post_tx_post_failure_retry_test() -> - Anchor = rand:bytes(32), - Price = 12345, - FailCount = 4, - setup_test_counter(tx_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - Count = increment_test_counter(tx_attempts_counter) - 1, - case Count < FailCount of - true -> {400, <<"Transaction verification failed">>}; - false -> {200, <<"OK">>} - end - end - }), - try - % Use short retry delays for testing. - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 50, - retry_jitter => 0 % Disable jitter for deterministic tests - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to eventually succeed - TXs = hb_mock_server:get_requests(tx, FailCount+1, ServerHandle), - ?assertEqual(FailCount+1, length(TXs)), - % Verify final attempt succeeded - FinalCount = get_test_counter(tx_attempts_counter), - ?assertEqual(FailCount+1, FinalCount), - ok - after - cleanup_test_counter(tx_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -post_proof_failure_retry_test() -> - Anchor = rand:bytes(32), - Price = 12345, - FailCount = 2, - setup_test_counter(chunk_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - chunk => fun(_Req) -> - Count = increment_test_counter(chunk_attempts_counter) - 1, - case Count < FailCount of - true -> {500, <<"error">>}; - false -> {200, <<"OK">>} - end - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 50, - retry_jitter => 0 - }, - hb_http_server:start_node(Opts), - % Large enough for multiple chunks - Items = [new_data_item(1, floor(4.5 * ?DATA_CHUNK_SIZE), Opts)], - dispatch(Items, Opts), - % Wait for TX - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), - ?assertEqual(1, length(TXs)), - % Wait for chunks to eventually succeed - Chunks = hb_mock_server:get_requests(chunk, FailCount+5, ServerHandle), - ?assertEqual( FailCount+5, length(Chunks)), - % Verify retries happened - FinalCount = get_test_counter(chunk_attempts_counter), - ?assertEqual(FailCount+5, FinalCount), - ok - after - cleanup_test_counter(chunk_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -empty_dispatch_test() -> - Opts = #{}, - dispatch([], Opts), - % Should not crash - ok. - -rapid_dispatch_test() -> - Anchor = rand:bytes(32), - Price = 12345, - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - timer:sleep(100), - {200, <<"OK">>} - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - bundler_workers => 3 - }, - hb_http_server:start_node(Opts), - % Dispatch 10 bundles rapidly - lists:foreach( - fun(I) -> - Items = [new_data_item(I, 10, Opts)], - dispatch(Items, Opts) - end, - lists:seq(1, 10) - ), - - % Wait for all 10 TXs - TXs = hb_mock_server:get_requests(tx, 10, ServerHandle), - ?assertEqual(10, length(TXs)), - ok - after - cleanup_dispatcher(ServerHandle) - end. - -one_bundle_fails_others_continue_test() -> - Anchor = rand:bytes(32), - Price = 12345, - setup_test_counter(mixed_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - % First TX succeeds, all following attempts fail. - Count = increment_test_counter(mixed_attempts_counter) - 1, - case Count of - 0 -> {200, <<"OK">>}; - _ -> {400, <<"fail">>} - end - end - }), - try - % Use short retry delays for testing (100ms base, with exponential backoff) - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 100, - retry_jitter => 0 % Disable jitter for deterministic tests - }, - hb_http_server:start_node(Opts), - % Dispatch first bundle (will keep failing) - Items1 = [new_data_item(1, 10, Opts)], - dispatch(Items1, Opts), - % Dispatch second bundle (will succeed) - Items2 = [new_data_item(2, 10, Opts)], - dispatch(Items2, Opts), - % Wait for at least 5 TX attempts (1 success + multiple retries) - TXs = hb_mock_server:get_requests(tx, 5, ServerHandle), - ?assert(length(TXs) >= 5, length(TXs)), - ok - after - cleanup_test_counter(mixed_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -parallel_task_execution_test() -> - Anchor = rand:bytes(32), - Price = 12345, - SleepTime = 120, - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - chunk => fun(_Req) -> - timer:sleep(SleepTime), - {200, <<"OK">>} - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - bundler_workers => 5 - }, - hb_http_server:start_node(Opts), - % Dispatch 3 bundles, each with 2 chunks - lists:foreach( - fun(I) -> - Items = [new_data_item(I, 10, Opts)], - dispatch(Items, Opts) - end, - lists:seq(1, 10) - ), - % With 3 workers and 1s delay, 10 chunks should complete in ~2s not 9s - StartTime = erlang:system_time(millisecond), - Chunks = hb_mock_server:get_requests(chunk, 10, ServerHandle), - ElapsedTime = erlang:system_time(millisecond) - StartTime, - ?assertEqual(10, length(Chunks)), - % Should take ~2-3 seconds with parallelism, not 9+ - ?assert(ElapsedTime < 2000, "ElapsedTime: " ++ integer_to_list(ElapsedTime)), - ok - after - cleanup_dispatcher(ServerHandle) - end. - -exponential_backoff_timing_test() -> - Anchor = rand:bytes(32), - Price = 12345, - FailCount = 5, - setup_test_counter(backoff_cap_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - Timestamp = erlang:system_time(millisecond), - Attempt = increment_test_counter(backoff_cap_counter), - Count = Attempt - 1, - % Store timestamp by attempt number. - add_test_attempt_timestamp(backoff_cap_counter, Attempt, Timestamp), - case Count < FailCount of - true -> {400, <<"fail">>}; - false -> {200, <<"OK">>} - end - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 100, - retry_max_delay_ms => 500, % Cap at 500ms - retry_jitter => 0 % Disable jitter for deterministic tests - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to eventually succeed - TXs = hb_mock_server:get_requests(tx, FailCount+1, ServerHandle, 5000), - ?assertEqual(FailCount+1, length(TXs)), - % Verify backoff respects cap - Timestamps = test_attempt_timestamps(backoff_cap_counter), - ?assertEqual(6, length(Timestamps)), - [T1, T2, T3, T4, T5, T6] = Timestamps, - % Calculate actual delays - Delay1 = T2 - T1, - Delay2 = T3 - T2, - Delay3 = T4 - T3, - Delay4 = T5 - T4, - Delay5 = T6 - T5, - % Expected: ~100ms, ~200ms, ~400ms, ~500ms (capped), ~500ms (capped) - ?assert(Delay1 >= 70 andalso Delay1 =< 200, Delay1), - ?assert(Delay2 >= 150 andalso Delay2 =< 300, Delay2), - ?assert(Delay3 >= 300 andalso Delay3 =< 500, Delay3), - ?assert(Delay4 >= 400 andalso Delay4 =< 700, Delay4), - ?assert(Delay5 >= 400 andalso Delay5 =< 700, Delay5), - ok - after - cleanup_test_counter(backoff_cap_counter), - cleanup_dispatcher(ServerHandle) - end. - -independent_task_retry_counts_test() -> - Anchor = rand:bytes(32), - Price = 12345, - setup_test_counter(independent_retry_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - % Use request ordering to distinguish bundles - % First 3 requests are bundle1 (fail, fail, succeed) - % 4th request is bundle2 (succeed) - Count = increment_test_counter(independent_retry_counter) - 1, - case Count < 2 of - true -> {400, <<"fail">>}; % First 2 attempts fail - false -> {200, <<"OK">>} % Rest succeed - end - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 100, - retry_jitter => 0 % Disable jitter for deterministic tests - }, - hb_http_server:start_node(Opts), - % Dispatch first bundle (will fail twice and retry) - Items1 = [new_data_item(1, 10, Opts)], - dispatch(Items1, Opts), - % Wait a bit for first bundle to start failing - hb_mock_server:get_requests(tx, 3, ServerHandle), - % Dispatch second bundle (will succeed on first try since we're past the 2 failures) - Items2 = [new_data_item(2, 10, Opts)], - dispatch(Items2, Opts), - % Verify we got all TX requests logged - TotalAttempts = 4, - TXs = hb_mock_server:get_requests(tx, TotalAttempts, ServerHandle), - ?assertEqual(TotalAttempts, length(TXs)), - ok - after - cleanup_test_counter(independent_retry_counter), - cleanup_dispatcher(ServerHandle) - end. - -recover_bundles_test() -> - Anchor = rand:bytes(32), - Price = 12345, - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)} - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store() - }, - hb_http_server:start_node(Opts), - % Create some test items - Item1 = new_data_item(1, 10, Opts), - Item2 = new_data_item(2, 10, Opts), - Item3 = new_data_item(3, 10, Opts), - % Write items to cache as unbundled - ok = dev_bundler_cache:write_item(Item1, Opts), - ok = dev_bundler_cache:write_item(Item2, Opts), - ok = dev_bundler_cache:write_item(Item3, Opts), - % Create a bundle TX and cache it with posted status - {ok, TX} = dev_codec_tx:to(lists:reverse([Item1, Item2, Item3]), #{}, #{}), - CommittedTX = hb_message:convert(TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), - ok = dev_bundler_cache:write_tx(CommittedTX, [Item1, Item2, Item3], Opts), - % Create a second bundle that is already complete (should not be recovered) - Item4 = new_data_item(4, 10, Opts), - ok = dev_bundler_cache:write_item(Item4, Opts), - {ok, TX2} = dev_codec_tx:to(lists:reverse([Item4]), #{}, #{}), - CommittedTX2 = hb_message:convert(TX2, <<"structured@1.0">>, <<"tx@1.0">>, Opts), - ok = dev_bundler_cache:write_tx(CommittedTX2, [Item4], Opts), - ok = dev_bundler_cache:complete_tx(CommittedTX2, Opts), - % Now initialize dispatcher which should recover only the posted bundle - ensure_dispatcher(Opts), - State = get_state(), - % Get the recovered bundle (should only be 1, not the completed one) - ?assertEqual(1, maps:size(State#state.bundles)), - [Bundle] = maps:values(State#state.bundles), - ?assertNotEqual(undefined, Bundle#bundle.start_time), - ?assertEqual(#{}, Bundle#bundle.proofs), - RecoveredItems = [ - hb_message:with_commitments( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) - || Item <- Bundle#bundle.items], - ?assertEqual( - lists:sort([Item1, Item2, Item3]), - lists:sort(RecoveredItems)), - ?assertEqual(tx_posted, Bundle#bundle.status), - ?assert(hb_message:verify(Bundle#bundle.tx)), - ?assertEqual( - hb_message:id(CommittedTX, signed, Opts), - hb_message:id(Bundle#bundle.tx, signed, Opts)), - ok - after - cleanup_dispatcher(ServerHandle) - end. - -%%% Test Helper Functions - -new_data_item(Index, Size, Opts) -> - Data = rand:bytes(Size), - Tag = <<"tag", (integer_to_binary(Index))/binary>>, - Value = <<"value", (integer_to_binary(Index))/binary>>, - Item = ar_bundles:sign_item( - #tx{ - data = Data, - tags = [{Tag, Value}] - }, - hb:wallet() - ), - hb_message:convert(Item, <<"structured@1.0">>, <<"ans104@1.0">>, Opts). - -start_mock_gateway(Responses) -> - DefaultResponse = {200, <<>>}, - Endpoints = [ - {"/chunk", chunk, maps:get(chunk, Responses, DefaultResponse)}, - {"/tx", tx, maps:get(tx, Responses, DefaultResponse)}, - {"/price/:size", price, maps:get(price, Responses, DefaultResponse)}, - {"/tx_anchor", tx_anchor, maps:get(tx_anchor, Responses, DefaultResponse)} - ], - {ok, MockServer, ServerHandle} = hb_mock_server:start(Endpoints), - NodeOpts = #{ - gateway => MockServer, - routes => [ - #{ - <<"template">> => <<"/arweave">>, - <<"node">> => #{ - <<"match">> => <<"^/arweave">>, - <<"with">> => MockServer, - <<"opts">> => #{http_client => httpc, protocol => http2} - } - } - ] - }, - {ServerHandle, NodeOpts}. - -cleanup_dispatcher(ServerHandle) -> - stop_dispatcher(), - timer:sleep(10), % Ensure dispatcher fully stops - hb_mock_server:stop(ServerHandle). - -setup_test_counter(Table) -> - cleanup_test_counter(Table), - ets:new(Table, [named_table, public, set]), - ok. - -cleanup_test_counter(Table) -> - case ets:info(Table) of - undefined -> ok; - _ -> ets:delete(Table), ok - end. - -increment_test_counter(Table) -> - ets:update_counter(Table, Table, {2, 1}, {Table, 0}). - -get_test_counter(Table) -> - case ets:lookup(Table, Table) of - [{_, Value}] -> Value; - [] -> 0 - end. - -add_test_attempt_timestamp(Table, Attempt, Timestamp) -> - ets:insert(Table, {{Table, Attempt}, Timestamp}). - -test_attempt_timestamps(Table) -> - TimestampEntries = [ - {Attempt, Timestamp} - || {{Prefix1, Attempt}, Timestamp} <- ets:tab2list(Table), - Prefix1 =:= Table - ], - [Timestamp || {_, Timestamp} <- lists:sort(TimestampEntries)]. diff --git a/src/include/dev_bundler.hrl b/src/include/dev_bundler.hrl new file mode 100644 index 000000000..c601785e0 --- /dev/null +++ b/src/include/dev_bundler.hrl @@ -0,0 +1,40 @@ +%%% Shared state and task records for the bundler server and workers. + +-record(state, { + max_size, + max_idle_time, + max_items, + queue, + bytes, + workers, + task_queue, + bundles, + opts +}). + +-record(task, { + bundle_id, + type, + data, + opts, + retry_count = 0 +}). + +-record(proof, { + proof, + status +}). + +-record(bundle, { + id, + items, + status, + tx, + proofs, + start_time +}). + +-define(DEFAULT_NUM_WORKERS, 20). +-define(DEFAULT_RETRY_BASE_DELAY_MS, 1000). +-define(DEFAULT_RETRY_MAX_DELAY_MS, 600000). +-define(DEFAULT_RETRY_JITTER, 0.25). diff --git a/test/arbundles.js/upload-items.js b/test/arbundles.js/upload-items.js index f5e629f51..a075d6735 100644 --- a/test/arbundles.js/upload-items.js +++ b/test/arbundles.js/upload-items.js @@ -4,11 +4,11 @@ const { ArweaveSigner, createData } = require("@dha-team/arbundles"); // Configuration const BUNDLER_URL = "http://localhost:8734"; -const WALLET_PATH = "../../hyperbeam-key.json"; +const DEFAULT_WALLET = "../../hyperbeam-key.json"; const CONCURRENT_UPLOADS = 100; // Number of parallel uploads -async function performanceTest(itemCount, bytesPerItem = 0) { - const wallet = require(WALLET_PATH); +async function performanceTest(walletPath, itemCount, bytesPerItem = 0) { + const wallet = require(path.resolve(walletPath)); const signer = new ArweaveSigner(wallet); const endpoint = `${BUNDLER_URL}/~bundler@1.0/item?codec-device=ans104@1.0`; @@ -132,24 +132,28 @@ async function performanceTest(itemCount, bytesPerItem = 0) { // Main execution if (require.main === module) { - const itemCount = parseInt(process.argv[2], 10); - const bytesPerItem = parseInt(process.argv[3], 10) || 0; - + // If the first arg looks like a number, treat it as itemCount and use the default wallet + const firstIsNumber = !isNaN(parseInt(process.argv[2], 10)); + const walletPath = firstIsNumber ? DEFAULT_WALLET : (process.argv[2] || DEFAULT_WALLET); + const itemCount = parseInt(firstIsNumber ? process.argv[2] : process.argv[3], 10); + const bytesPerItem = parseInt(firstIsNumber ? process.argv[3] : process.argv[4], 10) || 0; + if (!itemCount || itemCount < 1 || isNaN(itemCount)) { - console.error("Usage: node upload-items.js [bytes_per_item]"); + console.error("Usage: node upload-items.js [wallet_path] [bytes_per_item]"); console.error(""); console.error("Arguments:"); + console.error(" wallet_path - Path to Arweave wallet JSON (default: ../../hyperbeam-key.json)"); console.error(" number_of_items - Number of data items to create and upload"); console.error(" bytes_per_item - Minimum size of each item in bytes (optional)"); console.error(""); console.error("Examples:"); console.error(" node upload-items.js 100"); - console.error(" node upload-items.js 100 1024 # 100 items, ~1KB each"); - console.error(" node upload-items.js 50 10485760 # 50 items, ~10MB each"); + console.error(" node upload-items.js 100 1024"); + console.error(" node upload-items.js /path/to/wallet.json 100 1024"); process.exit(1); } - performanceTest(itemCount, bytesPerItem) + performanceTest(walletPath, itemCount, bytesPerItem) .then(() => { process.exit(0); }) From 1b1655b1dc525828f2b9decb075e273f7ed8f989 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Fri, 6 Mar 2026 16:09:26 -0500 Subject: [PATCH 032/135] impr: refactor dev_bundler and let recovery run asynchronously Continues the refactoring of the dev_bundler modules and implements functionality to allow unbundled items and unfinished bundles to be recovered incrementally rather than requiring that all data be loaded first before any of it is processed --- src/dev_bundler.erl | 180 ++++-------- src/dev_bundler_cache.erl | 165 ++++------- src/dev_bundler_recovery.erl | 272 ++++++++++++++++++ ...dler_dispatch.erl => dev_bundler_task.erl} | 100 ++++--- 4 files changed, 439 insertions(+), 278 deletions(-) create mode 100644 src/dev_bundler_recovery.erl rename src/{dev_bundler_dispatch.erl => dev_bundler_task.erl} (78%) diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index e7e636c3a..ceffa327c 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -43,7 +43,7 @@ item(_Base, Req, Opts) -> ok -> % Queue the item for bundling % (fire-and-forget, ignore errors) - ServerPID ! {item, Item}, + ServerPID ! {enqueue_item, Item}, {ok, #{ <<"id">> => ItemID, <<"timestamp">> => erlang:system_time(millisecond) @@ -132,41 +132,36 @@ init(Opts) -> NumWorkers = hb_opts:get(bundler_workers, ?DEFAULT_NUM_WORKERS, Opts), Workers = lists:map( fun(_) -> - WorkerPID = spawn_link(fun dev_bundler_dispatch:worker_loop/0), + WorkerPID = spawn_link(fun dev_bundler_task:worker_loop/0), {WorkerPID, idle} end, lists:seq(1, NumWorkers) ), - {UnbundledItems, RecoveredBytes} = recover_unbundled_items(Opts), InitialState = #state{ max_size = hb_opts:get(bundler_max_size, ?DEFAULT_MAX_SIZE, Opts), max_idle_time = hb_opts:get( bundler_max_idle_time, ?DEFAULT_MAX_IDLE_TIME, Opts), max_items = hb_opts:get(bundler_max_items, ?DEFAULT_MAX_ITEMS, Opts), - queue = UnbundledItems, - bytes = RecoveredBytes, + queue = [], + bytes = 0, workers = maps:from_list(Workers), task_queue = queue:new(), bundles = #{}, opts = Opts }, - State = maybe_dispatch(recover_bundles(InitialState)), - server(assign_tasks(State), Opts). - -%% @doc Recover unbundled items from cache and calculate their total size. -%% Returns {Items, TotalBytes}. -recover_unbundled_items(Opts) -> - UnbundledItems = dev_bundler_cache:load_unbundled_items(Opts), - ?event(bundler_short, {recovered_unbundled_items, length(UnbundledItems)}), - RecoveredBytes = queue_bytes(UnbundledItems), - {UnbundledItems, RecoveredBytes}. + dev_bundler_recovery:recover_unbundled_items(self(), Opts), + dev_bundler_recovery:recover_bundles(self(), Opts), + server(assign_tasks(InitialState), Opts). %% @doc The main loop of the bundler server. server(State = #state{max_idle_time = MaxIdleTime}, Opts) -> receive - {item, Item} -> + {enqueue_item, Item} -> State1 = add_to_queue(Item, State, Opts), server(assign_tasks(maybe_dispatch(State1)), Opts); + {recover_bundle, CommittedTX, Items} -> + State1 = recover_bundle(CommittedTX, Items, State), + server(assign_tasks(State1), Opts); {task_complete, WorkerPID, Task, Result} -> State1 = handle_task_complete(WorkerPID, Task, Result, State), server(assign_tasks(State1), Opts); @@ -189,7 +184,7 @@ server(State = #state{max_idle_time = MaxIdleTime}, Opts) -> server(assign_tasks(dispatch_queue(State)), Opts) end. -%% @doc Add an item to the queue. Update the state with the new queue +%% @doc Add an enqueue_item to the queue. Update the state with the new queue %% and approximate total byte size of the queue. %% Note: Item has already been verified and cached before reaching here. add_to_queue(Item, State = #state{queue = Queue, bytes = Bytes}, Opts) -> @@ -210,7 +205,7 @@ maybe_dispatch(State = #state{queue = Q, max_items = MaxItems}) -> case dispatchable(State) of true -> {ToDispatch, Remaining} = split_queue(Q, MaxItems), - State1 = dispatch_items(ToDispatch, State), + State1 = create_bundle(ToDispatch, State), NewState = State1#state{ queue = Remaining, bytes = queue_bytes(Remaining) @@ -246,12 +241,12 @@ queue_bytes(Items) -> dispatch_queue(State = #state{queue = []}) -> State; dispatch_queue(State = #state{queue = Queue}) -> - dispatch_items(Queue, State#state{queue = [], bytes = 0}). + create_bundle(Queue, State#state{queue = [], bytes = 0}). %% @doc Create a bundle and enqueue its initial post task. -dispatch_items([], State) -> +create_bundle([], State) -> State; -dispatch_items(Items, State = #state{bundles = Bundles, opts = Opts}) -> +create_bundle(Items, State = #state{bundles = Bundles, opts = Opts}) -> BundleID = make_ref(), Bundle = #bundle{ id = BundleID, @@ -267,7 +262,7 @@ dispatch_items(Items, State = #state{bundles = Bundles, opts = Opts}) -> ?event( bundler_short, {dispatching_bundle, - {timestamp, dev_bundler_dispatch:format_timestamp()}, + {timestamp, dev_bundler_task:format_timestamp()}, {bundle_id, BundleID}, {num_items, length(Items)} } @@ -313,7 +308,7 @@ handle_task_complete(WorkerPID, Task, Result, State = #state{ bundles = Bundles }) -> #task{bundle_id = BundleID} = Task, - ?event(bundler_debug, {task_complete, dev_bundler_dispatch:format_task(Task)}), + ?event(bundler_debug, dev_bundler_task:log_task(task_complete, Task, [])), State1 = State#state{ workers = maps:put(WorkerPID, idle, Workers) }, @@ -341,12 +336,11 @@ handle_task_failed(WorkerPID, Task, Reason, State = #state{ Delay = round(BaseDelayWithBackoff * (1 + JitterFactor)), ?event( bundler_short, - {task_failed_retrying, - dev_bundler_dispatch:format_task(Task), + dev_bundler_task:log_task(task_failed_retrying, Task, [ {reason, {explicit, Reason}}, {retry_count, RetryCount}, {delay_ms, Delay} - } + ]) ), Task1 = Task#task{retry_count = RetryCount + 1}, erlang:send_after(Delay, self(), {retry_task, Task1}), @@ -439,67 +433,35 @@ bundle_complete(Bundle, State = #state{opts = Opts}) -> bundler_short, {bundle_complete, {bundle_id, Bundle#bundle.id}, - {timestamp, dev_bundler_dispatch:format_timestamp()}, + {timestamp, dev_bundler_task:format_timestamp()}, {tx, {explicit, hb_message:id(Bundle#bundle.tx, signed, Opts)}}, {elapsed_time_s, ElapsedTime} } ), State#state{bundles = maps:remove(Bundle#bundle.id, State#state.bundles)}. -%% @doc Recover in-progress bundles from cache after a crash. -recover_bundles(State = #state{opts = Opts}) -> - lists:foldl( - fun({TXID, Status}, StateAcc) -> - recover_bundle(TXID, Status, StateAcc) - end, - State, - dev_bundler_cache:load_bundle_states(Opts) - ). - %% @doc Recover a single bundle and enqueue any follow-up work. -recover_bundle(TXID, Status, State = #state{opts = Opts}) -> - ?event( - bundler_short, - {recovering_bundle, - {tx_id, {explicit, TXID}}, - {status, Status} - } - ), - try - CommittedTX = dev_bundler_cache:load_tx(TXID, Opts), - Items = dev_bundler_cache:load_bundled_items(TXID, Opts), - BundleID = make_ref(), - Bundle = #bundle{ - id = BundleID, - items = Items, - status = tx_posted, - tx = CommittedTX, - proofs = #{}, - start_time = erlang:timestamp() - }, - Bundles = State#state.bundles, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle, Bundles) - }, - Task = #task{ - bundle_id = BundleID, - type = build_proofs, - data = CommittedTX, - opts = Opts - }, - enqueue_task(Task, State1) - catch - _:Error:Stack -> - ?event( - bundler_short, - {failed_to_recover_bundle, - {tx_id, {explicit, TXID}}, - {error, Error}, - {stack, Stack} - } - ), - State - end. +recover_bundle(CommittedTX, Items, State = #state{opts = Opts}) -> + BundleID = make_ref(), + Bundle = #bundle{ + id = BundleID, + items = Items, + status = tx_posted, + tx = CommittedTX, + proofs = #{}, + start_time = erlang:timestamp() + }, + Bundles = State#state.bundles, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle, Bundles) + }, + Task = #task{ + bundle_id = BundleID, + type = build_proofs, + data = CommittedTX, + opts = Opts + }, + enqueue_task(Task, State1). %%%=================================================================== %%% Tests @@ -734,29 +696,6 @@ dispatch_blocking_test() -> stop_test_servers(ServerHandle) end. -recover_unbundled_items_test() -> - Opts = #{store => hb_test_utils:test_store()}, - % Create and cache some items - Item1 = hb_message:convert(new_data_item(1, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - Item2 = hb_message:convert(new_data_item(2, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - Item3 = hb_message:convert(new_data_item(3, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - ok = dev_bundler_cache:write_item(Item1, Opts), - ok = dev_bundler_cache:write_item(Item2, Opts), - ok = dev_bundler_cache:write_item(Item3, Opts), - % Bundle Item2 with a fake TX - FakeTX = ar_tx:sign(#tx{format = 2, tags = [{<<"test">>, <<"tx">>}]}, hb:wallet()), - StructuredTX = hb_message:convert(FakeTX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), - ok = dev_bundler_cache:write_tx(StructuredTX, [Item2], Opts), - % Now recover unbundled items - {RecoveredItems, RecoveredBytes} = recover_unbundled_items(Opts), - ?assertEqual(3924, RecoveredBytes), - RecoveredItems2 = [ - hb_message:with_commitments( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) - || Item <- RecoveredItems], - ?assertEqual(lists:sort([Item1, Item3]), lists:sort(RecoveredItems2)), - ok. - %% @doc Test that items are recovered and posted while respecting the %% max_items limit. recover_respects_max_items_test() -> @@ -847,6 +786,10 @@ recover_bundles_test() -> Anchor = rand:bytes(32), Price = 12345, {ServerHandle, NodeOpts} = start_mock_gateway(#{ + chunk => fun(_Req) -> + timer:sleep(250), + {200, <<"OK">>} + end, price => {200, integer_to_binary(Price)}, tx_anchor => {200, hb_util:encode(Anchor)} }), @@ -876,25 +819,20 @@ recover_bundles_test() -> ok = dev_bundler_cache:complete_tx(CommittedTX2, Opts), ensure_server(Opts), State = get_state(), - ?assertEqual(1, maps:size(State#state.bundles)), - [Bundle] = maps:values(State#state.bundles), - ?assertNotEqual(undefined, Bundle#bundle.start_time), - ?assertEqual(#{}, Bundle#bundle.proofs), - RecoveredItems = [ - hb_message:with_commitments( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) - || Item <- Bundle#bundle.items - ], - ?assertEqual( - lists:sort([Item1, Item2, Item3]), - lists:sort(RecoveredItems) - ), - ?assertEqual(tx_posted, Bundle#bundle.status), - ?assert(hb_message:verify(Bundle#bundle.tx)), - ?assertEqual( - hb_message:id(CommittedTX, signed, Opts), - hb_message:id(Bundle#bundle.tx, signed, Opts) + ?assertNotEqual(undefined, State), + ?assertNotEqual(timeout, State), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle, 200), + ?assertEqual([], TXs), + ?assert( + hb_util:wait_until( + fun() -> + dev_bundler_cache:load_bundle_states(Opts) =:= [] + end, + 2000 + ) ), + FinalState = get_state(), + ?assertEqual(0, maps:size(FinalState#state.bundles)), ok after stop_test_servers(ServerHandle) diff --git a/src/dev_bundler_cache.erl b/src/dev_bundler_cache.erl index c38de8864..979a96d2b 100644 --- a/src/dev_bundler_cache.erl +++ b/src/dev_bundler_cache.erl @@ -7,17 +7,18 @@ %%% %%% Recovery flow: %%% 1. Load unbundled items (where bundle = <<>>) back into dev_bundler queue -%%% 2. Load TX states and reconstruct dev_bundler_dispatch bundles +%%% 2. Load TX states and reconstruct in-progress bundler bundles %%% 3. Enqueue appropriate tasks based on status -module(dev_bundler_cache). -export([ write_item/2, write_tx/3, complete_tx/2, - load_unbundled_items/1, load_bundle_states/1, - load_bundled_items/2, - load_tx/2 + load_tx/2, + load_items/2, + load_items/4, + list_item_ids/1 ]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -55,11 +56,13 @@ get_item_bundle(Item, Opts) when is_map(Item) -> %% @doc Construct the pseudopath for an item's bundle reference. %% Item should be a structured message. item_path(Item, Opts) when is_map(Item) -> + item_path(item_id(Item, Opts), Opts); +item_path(ItemID, Opts) when is_binary(ItemID) -> Store = hb_opts:get(store, no_viable_store, Opts), hb_store:path(Store, [ ?BUNDLER_PREFIX, <<"item">>, - item_id(Item, Opts), + ItemID, <<"bundle">> ]). @@ -110,57 +113,6 @@ tx_path(TX, Opts) -> %%% Recovery operations -%% @doc Load all unbundled items (where bundle = <<>>) from cache. -%% Returns list of actual Item messages for re-queuing. -load_unbundled_items(Opts) -> - Store = hb_opts:get(store, no_viable_store, Opts), - ItemsPath = hb_store:path(Store, [?BUNDLER_PREFIX, <<"item">>]), - % List all item IDs - ItemIDs = case hb_cache:list(ItemsPath, Opts) of - [] -> []; - List -> List - end, - ?event(bundler_short, - {recovering_all_unbundled_items, length(ItemIDs)} - ), - % Filter for unbundled items and load them - lists:filtermap( - fun(ItemIDStr) -> - % Read the bundle pseudopath directly - BundlePath = hb_store:path(Store, [ - ?BUNDLER_PREFIX, - <<"item">>, - ItemIDStr, - <<"bundle">> - ]), - case read_pseudopath(BundlePath, Opts) of - {ok, <<>>} -> - % Unbundled item - load it fully (resolve all links) - case hb_cache:read(ItemIDStr, Opts) of - {ok, Item} -> - FullyLoadedItem = hb_cache:ensure_all_loaded(Item, Opts), - ?event(bundler_short, - {recovered_unbundled_item, - {id, {string, ItemIDStr}} - } - ), - {true, FullyLoadedItem}; - _ -> - ?event(bundler_short, - {failed_to_recover_unbundled_item, - {id, {string, ItemIDStr}} - } - ), - false - end; - _ -> - % Already bundled or not found - false - end - end, - ItemIDs - ). - %% @doc Load all bundle TX states from cache. %% Returns list of {TXID, Status} tuples. load_bundle_states(Opts) -> @@ -193,60 +145,6 @@ load_bundle_states(Opts) -> TXIDs ). -%% @doc Load all data items associated with a bundle TX. -%% Uses the item pseudopaths to find items with matching tx-id. -load_bundled_items(TXID, Opts) -> - Store = hb_opts:get(store, no_viable_store, Opts), - ItemsPath = hb_store:path(Store, [?BUNDLER_PREFIX, <<"item">>]), - % List all item IDs - ItemIDs = case hb_cache:list(ItemsPath, Opts) of - [] -> []; - List -> List - end, - ?event(bundler_short, {recovering_bundled_items, - {count, length(ItemIDs)}}), - % Filter for items belonging to this TX and load them - lists:filtermap( - fun(ItemIDStr) -> - % Read the bundle pseudopath directly - BundlePath = hb_store:path(Store, [ - ?BUNDLER_PREFIX, - <<"item">>, - ItemIDStr, - <<"bundle">> - ]), - case read_pseudopath(BundlePath, Opts) of - {ok, BundleTXID} when BundleTXID =:= TXID -> - % This item belongs to our bundle - load it fully (resolve all links) - case hb_cache:read(ItemIDStr, Opts) of - {ok, Item} -> - FullyLoadedItem = hb_cache:ensure_all_loaded(Item, Opts), - ?event( - bundler_debug, - {loaded_tx_item, - {tx_id, {explicit, TXID}}, - {item_id, {explicit, ItemIDStr}} - } - ), - {true, FullyLoadedItem}; - _ -> - ?event( - error, - {failed_to_load_tx_item, - {tx_id, {explicit, TXID}}, - {item_id, {explicit, ItemIDStr}} - } - ), - false - end; - _ -> - % Doesn't belong to this bundle or not found - false - end - end, - ItemIDs - ). - %% @doc Load a TX from cache by its ID. load_tx(TXID, Opts) -> ?event(bundler_debug, {load_tx, {tx_id, {explicit, TXID}}}), @@ -275,6 +173,47 @@ read_pseudopath(Path, Opts) -> _ -> not_found end. +%% @doc List all cached bundler item IDs. +list_item_ids(Opts) -> + Store = hb_opts:get(store, no_viable_store, Opts), + ItemsPath = hb_store:path(Store, [?BUNDLER_PREFIX, <<"item">>]), + case hb_cache:list(ItemsPath, Opts) of + [] -> []; + List -> List + end. + +%% @doc Load all items whose bundle pseudopath matches BundleID. +load_items(BundleID, Opts) -> + load_items( + BundleID, + Opts, + fun(_ItemID, _Item) -> ok end, + fun(_ItemID) -> ok end + ). + +%% @doc Load all items whose bundle pseudopath matches BundleID and invoke callbacks. +load_items(BundleID, Opts, OnLoaded, OnFailed) -> + lists:filtermap( + fun(ItemID) -> + BundlePath = item_path(ItemID, Opts), + case read_pseudopath(BundlePath, Opts) of + {ok, BundleID} -> + case hb_cache:read(ItemID, Opts) of + {ok, Item} -> + FullyLoadedItem = hb_cache:ensure_all_loaded(Item, Opts), + OnLoaded(ItemID, FullyLoadedItem), + {true, FullyLoadedItem}; + _ -> + OnFailed(ItemID), + false + end; + _ -> + false + end + end, + list_item_ids(Opts) + ). + %%% Tests basic_cache_test() -> @@ -306,7 +245,7 @@ load_unbundled_items_test() -> % Link item2 to a bundle, leave others unbundled ok = write_tx(TX, [Item2], Opts), % Load unbundled items - UnbundledItems1 = load_unbundled_items(Opts), + UnbundledItems1 = load_items(<<>>, Opts), UnbundledItems2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, @@ -348,7 +287,7 @@ load_bundled_items_test() -> ok = write_tx(TX1, [Item1, Item2], Opts), ok = write_tx(TX2, [Item3], Opts), % Load items for bundle 1 - Bundle1Items1 = load_bundled_items(tx_id(TX1, Opts), Opts), + Bundle1Items1 = load_items(tx_id(TX1, Opts), Opts), Bundle1Items2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, @@ -357,7 +296,7 @@ load_bundled_items_test() -> Bundle1Items3 = lists:sort(Bundle1Items2), ?assertEqual(lists:sort([Item1, Item2]), Bundle1Items3), % Load items for bundle 2 - Bundle2Items1 = load_bundled_items(tx_id(TX2, Opts), Opts), + Bundle2Items1 = load_items(tx_id(TX2, Opts), Opts), Bundle2Items2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, diff --git a/src/dev_bundler_recovery.erl b/src/dev_bundler_recovery.erl new file mode 100644 index 000000000..59f989f25 --- /dev/null +++ b/src/dev_bundler_recovery.erl @@ -0,0 +1,272 @@ +%%% @doc Logic for handling bundler recocery on node restart. +%%% +%%% When a bundler is running it will cache the state of each uploaded item +%%% or bundle as it move through the bundling and upload process. If the node +%%% is restarted before it can finish including all uploaded items in a bundle, +%%% or finish seeding all bundles in process, the recovery process will ensure +%%% that the data in process is recovered and resumed. +-module(dev_bundler_recovery). +-export([ + recover_unbundled_items/2, + recover_bundles/2 +]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Spawn a process to recover unbundled items. +recover_unbundled_items(ServerPID, Opts) -> + spawn(fun() -> do_recover_unbundled_items(ServerPID, Opts) end). + +%% @doc Spawn a process to recover in-progress bundles. +recover_bundles(ServerPID, Opts) -> + spawn(fun() -> do_recover_bundles(ServerPID, Opts) end). + +do_recover_unbundled_items(ServerPID, Opts) -> + try + ItemIDs = dev_bundler_cache:list_item_ids(Opts), + ?event(bundler_short, {recover_unbundled_items_start, + {count, length(ItemIDs)}}), + UnbundledItems = dev_bundler_cache:load_items( + <<>>, + Opts, + fun(ItemID, Item) -> + ?event( + bundler_short, + {recovered_unbundled_item, + {id, {string, ItemID}} + } + ), + ServerPID ! {enqueue_item, Item} + end, + fun(ItemID) -> + ?event( + bundler_short, + {failed_to_recover_unbundled_item, + {id, {string, ItemID}} + } + ) + end + ), + ?event(bundler_short, {recover_unbundled_items_complete, + {count, length(UnbundledItems)}}), + ok + catch + _:Error:Stack -> + ?event( + error, + {recover_unbundled_items_failed, + {error, Error}, + {stack, Stack} + } + ) + end. + +do_recover_bundles(ServerPID, Opts) -> + try + ?event(bundler_short, {recover_bundles_start}), + BundleStates = dev_bundler_cache:load_bundle_states(Opts), + lists:foreach( + fun({TXID, Status}) -> + recover_bundle(ServerPID, TXID, Status, Opts) + end, + BundleStates + ), + ?event(bundler_short, {recover_bundles_complete, + {count, length(BundleStates)}}), + ok + catch + _:Error:Stack -> + ?event( + error, + {recover_bundles_failed, + {error, Error}, + {stack, Stack} + } + ) + end. + +recover_bundle(ServerPID, TXID, Status, Opts) -> + ?event( + bundler_short, + {recovering_bundle, + {tx_id, {explicit, TXID}}, + {status, Status} + } + ), + try + CommittedTX = dev_bundler_cache:load_tx(TXID, Opts), + case CommittedTX of + not_found -> + throw(tx_not_found); + _ -> + Items = dev_bundler_cache:load_items( + TXID, + Opts, + fun(ItemID, _Item) -> + ?event( + bundler_debug, + {loaded_bundle_item, + {tx_id, {explicit, TXID}}, + {item_id, {explicit, ItemID}} + } + ) + end, + fun(ItemID) -> + ?event( + error, + {failed_to_load_bundle_item, + {tx_id, {explicit, TXID}}, + {item_id, {explicit, ItemID}} + } + ), + throw({failed_to_load_bundle_item, ItemID}) + end + ), + ServerPID ! {recover_bundle, CommittedTX, Items} + end + catch + _:Error:Stack -> + ?event( + error, + {failed_to_recover_bundle, + {tx_id, {explicit, TXID}}, + {error, Error}, + {stack, Stack} + } + ) + end. + +%%%=================================================================== +%%% Tests +%%%=================================================================== + +recover_unbundled_items_test() -> + Opts = #{store => hb_test_utils:test_store()}, + Item1 = new_data_item(1, 10, Opts), + Item2 = new_data_item(2, 10, Opts), + Item3 = new_data_item(3, 10, Opts), + ok = dev_bundler_cache:write_item(Item1, Opts), + ok = dev_bundler_cache:write_item(Item2, Opts), + ok = dev_bundler_cache:write_item(Item3, Opts), + FakeTX = new_bundle_tx([Item2], Opts), + ok = dev_bundler_cache:write_tx(FakeTX, [Item2], Opts), + recover_unbundled_items(self(), Opts), + RecoveredItems = receive_enqueue_items(2), + RecoveredItems1 = normalize_items(RecoveredItems, Opts), + ?assertEqual( + lists:sort([Item1, Item3]), + lists:sort(RecoveredItems1) + ). + +recover_bundles_skips_complete_test() -> + Opts = #{store => hb_test_utils:test_store()}, + Item1 = new_data_item(1, 10, Opts), + Item2 = new_data_item(2, 10, Opts), + Item3 = new_data_item(3, 10, Opts), + ok = dev_bundler_cache:write_item(Item1, Opts), + ok = dev_bundler_cache:write_item(Item2, Opts), + ok = dev_bundler_cache:write_item(Item3, Opts), + PostedTX = new_bundle_tx([Item1, Item2], Opts), + CompletedTX = new_bundle_tx([Item3], Opts), + ok = dev_bundler_cache:write_tx(PostedTX, [Item1, Item2], Opts), + ok = dev_bundler_cache:write_tx(CompletedTX, [Item3], Opts), + ok = dev_bundler_cache:complete_tx(CompletedTX, Opts), + recover_bundles(self(), Opts), + {RecoveredTX, RecoveredItems} = receive_recovered_bundle(), + RecoveredItems1 = normalize_items(RecoveredItems, Opts), + ?assertEqual( + hb_message:id(PostedTX, signed, Opts), + hb_message:id(RecoveredTX, signed, Opts) + ), + ?assertEqual( + lists:sort([Item1, Item2]), + lists:sort(RecoveredItems1) + ), + receive + {recover_bundle, _, _} -> + erlang:error(unexpected_second_recovered_bundle) + after 200 -> + ok + end. + +recover_bundles_failed_bundle_items_continue_test() -> + Opts = #{store => hb_test_utils:test_store()}, + ValidItem = new_data_item(1, 10, Opts), + ok = dev_bundler_cache:write_item(ValidItem, Opts), + ValidTX = new_bundle_tx([ValidItem], Opts), + ok = dev_bundler_cache:write_tx(ValidTX, [ValidItem], Opts), + BrokenTX = new_bundle_tx([], Opts), + ok = dev_bundler_cache:write_tx(BrokenTX, [], Opts), + MissingItemID = <<"missing-item">>, + ok = write_missing_item_bundle(MissingItemID, BrokenTX, Opts), + recover_bundles(self(), Opts), + {RecoveredTX, RecoveredItems} = receive_recovered_bundle(), + RecoveredItems1 = normalize_items(RecoveredItems, Opts), + ?assertEqual( + hb_message:id(ValidTX, signed, Opts), + hb_message:id(RecoveredTX, signed, Opts) + ), + ?assertEqual([ValidItem], RecoveredItems1), + receive + {recover_bundle, _, _} -> + erlang:error(unexpected_broken_bundle_recovered) + after 200 -> + ok + end. + +receive_enqueue_items(Count) -> + receive_enqueue_items(Count, []). + +receive_enqueue_items(0, Items) -> + lists:reverse(Items); +receive_enqueue_items(Count, Items) -> + receive + {enqueue_item, Item} -> + receive_enqueue_items(Count - 1, [Item | Items]) + after 1000 -> + erlang:error({missing_enqueue_items, Count}) + end. + +receive_recovered_bundle() -> + receive + {recover_bundle, CommittedTX, Items} -> + {CommittedTX, Items} + after 1000 -> + erlang:error(missing_recovered_bundle) + end. + +normalize_items(Items, Opts) -> + [ + hb_message:with_commitments( + #{ <<"commitment-device">> => <<"ans104@1.0">> }, + Item, + Opts + ) + || Item <- Items + ]. + +write_missing_item_bundle(ItemID, TX, Opts) -> + Store = hb_opts:get(store, no_viable_store, Opts), + Path = hb_store:path(Store, [ + <<"~bundler@1.0">>, + <<"item">>, + ItemID, + <<"bundle">> + ]), + hb_store:write(Store, Path, hb_message:id(TX, signed, Opts)). + +new_data_item(Index, Size, Opts) -> + Tag = <<"tag", (integer_to_binary(Index))/binary>>, + Value = <<"value", (integer_to_binary(Index))/binary>>, + Item = ar_bundles:sign_item( + #tx{ + data = rand:bytes(Size), + tags = [{Tag, Value}] + }, + hb:wallet() + ), + hb_message:convert(Item, <<"structured@1.0">>, <<"ans104@1.0">>, Opts). + +new_bundle_tx(Items, Opts) -> + {ok, TX} = dev_codec_tx:to(lists:reverse(Items), #{}, #{}), + hb_message:convert(TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts). diff --git a/src/dev_bundler_dispatch.erl b/src/dev_bundler_task.erl similarity index 78% rename from src/dev_bundler_dispatch.erl rename to src/dev_bundler_task.erl index 174fa47c3..22b3fc804 100644 --- a/src/dev_bundler_dispatch.erl +++ b/src/dev_bundler_task.erl @@ -1,33 +1,13 @@ -%%% @doc A dispatcher for the bundler device (dev_bundler). This module -%%% manages a worker pool to handle bundle building, TX posting, proof -%%% generation, and chunk seeding. Server-side dispatch state lives in -%%% `dev_bundler'; this module only owns worker execution. --module(dev_bundler_dispatch). --export([worker_loop/0, format_task/1, format_timestamp/0]). +%%% @doc Implements the different bundling primitives: +%%% - post_tx: Building and posting an L1 transaction +%%% - build_proofs:Chunking up the bundle data and building the chunk proofs +%%% - post_proof: Seeding teh chunks to the Arweave network +-module(dev_bundler_task). +-export([worker_loop/0, log_task/3, format_timestamp/0]). -include("include/hb.hrl"). -include("include/dev_bundler.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% @doc Format a task for logging. -format_task(#task{bundle_id = BundleID, type = post_tx, data = DataItems}) -> - {post_tx, {timestamp, format_timestamp()}, {bundle, BundleID}, - {num_items, length(DataItems)}}; -format_task(#task{bundle_id = BundleID, type = build_proofs, data = CommittedTX}) -> - {build_proofs, {timestamp, format_timestamp()}, {bundle, BundleID}, - {tx, {explicit, hb_message:id(CommittedTX, signed, #{})}}}; -format_task(#task{bundle_id = BundleID, type = post_proof, data = Proof}) -> - Offset = maps:get(offset, Proof), - {post_proof, {timestamp, format_timestamp()}, {bundle, BundleID}, - {offset, Offset}}. - -%% @doc Format erlang:timestamp() as a user-friendly RFC3339 string with milliseconds. -format_timestamp() -> - {MegaSecs, Secs, MicroSecs} = erlang:timestamp(), - Millisecs = (MegaSecs * 1000000 + Secs) * 1000 + (MicroSecs div 1000), - calendar:system_time_to_rfc3339(Millisecs, [{unit, millisecond}, {offset, "Z"}]). - -%%% Worker implementation - %% @doc Worker loop - executes tasks and reports back to dispatcher. worker_loop() -> receive @@ -47,7 +27,7 @@ worker_loop() -> %% @doc Execute a specific task. execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> try - ?event(bundler_debug, {execute_task, format_task(Task)}), + ?event(bundler_debug, log_task(executing_task, Task, [])), % Get price and anchor {ok, TX} = dev_codec_tx:to(lists:reverse(Items), #{}, #{}), DataSize = TX#tx.data_size, @@ -85,23 +65,22 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> {_, ErrorReason} -> {error, ErrorReason} end; {PriceErr, AnchorErr} -> - ?event(bundle_short, {post_tx_failed, - format_task(Task), - {price, PriceErr}, - {anchor, AnchorErr}}), + ?event(bundle_short, + log_task(task_failed, Task, [ + {price, PriceErr}, + {anchor, AnchorErr} + ])), {error, {PriceErr, AnchorErr}} end catch - _:Err:_Stack -> - ?event(bundle_short, {post_tx_failed, - format_task(Task), - {error, Err}}), + _:Err:_Stack -> + ?event(bundle_short, log_task(task_failed, Task, [{error, Err}])), {error, Err} end; execute_task(#task{type = build_proofs, data = CommittedTX, opts = Opts} = Task) -> try - ?event(bundler_debug, {execute_task, format_task(Task)}), + ?event(bundler_debug, log_task(executing_task, Task, [])), % Calculate chunks and proofs TX = hb_message:convert( CommittedTX, <<"tx@1.0">>, <<"structured@1.0">>, Opts), @@ -141,7 +120,7 @@ execute_task(#task{type = build_proofs, data = CommittedTX, opts = Opts} = Task) hb_event:increment(bundler_short, built_proofs, length(Proofs) - 1), ?event( bundler_short, - {built_proofs, + {built_proofs, {bundle, Task#task.bundle_id}, {num_proofs, length(Proofs)} }, @@ -150,16 +129,14 @@ execute_task(#task{type = build_proofs, data = CommittedTX, opts = Opts} = Task) {ok, Proofs} catch _:Err:_Stack -> - ?event(bundler_short, {build_proofs_failed, - format_task(Task), - {error, Err}}), + ?event(bundler_short, log_task(task_failed, Task, [{error, Err}])), {error, Err} end; execute_task(#task{type = post_proof, data = Proof, opts = Opts} = Task) -> #{chunk := Chunk, data_path := DataPath, offset := Offset, data_size := DataSize, data_root := DataRoot} = Proof, - ?event(bundler_debug, {execute_task, format_task(Task)}), + ?event(bundler_debug, log_task(executing_task, Task, [])), Request = #{ <<"chunk">> => hb_util:encode(Chunk), <<"data_path">> => hb_util:encode(DataPath), @@ -176,9 +153,7 @@ execute_task(#task{type = post_proof, data = Proof, opts = Opts} = Task) -> end catch _:Err:_Stack -> - ?event(bundler_short, {post_proof_failed, - format_task(Task), - {error, Err}}), + ?event(bundler_short, log_task(task_failed, Task, [{error, Err}])), {error, Err} end. @@ -196,3 +171,40 @@ get_anchor(Opts) -> Opts ). +%%%=================================================================== +%%% Logging +%%%=================================================================== + +%% @doc Return a complete task event tuple for logging. +log_task(Event, Task, ExtraLogs) -> + erlang:list_to_tuple([Event | format_task(Task) ++ ExtraLogs]). + +%% @doc Format a task for logging. +format_task(#task{bundle_id = BundleID, type = post_tx, data = DataItems}) -> + [ + {task_type, post_tx}, + {timestamp, format_timestamp()}, + {bundle, BundleID}, + {num_items, length(DataItems)} + ]; +format_task(#task{bundle_id = BundleID, type = build_proofs, data = CommittedTX}) -> + [ + {task_type, build_proofs}, + {timestamp, format_timestamp()}, + {bundle, BundleID}, + {tx, {explicit, hb_message:id(CommittedTX, signed, #{})}} + ]; +format_task(#task{bundle_id = BundleID, type = post_proof, data = Proof}) -> + Offset = maps:get(offset, Proof), + [ + {task_type, post_proof}, + {timestamp, format_timestamp()}, + {bundle, BundleID}, + {offset, Offset} + ]. + +%% @doc Format erlang:timestamp() as a user-friendly RFC3339 string with milliseconds. +format_timestamp() -> + {MegaSecs, Secs, MicroSecs} = erlang:timestamp(), + Millisecs = (MegaSecs * 1000000 + Secs) * 1000 + (MicroSecs div 1000), + calendar:system_time_to_rfc3339(Millisecs, [{unit, millisecond}, {offset, "Z"}]). \ No newline at end of file From 6e8130b0560f69875bed07eea979817fcc953c5a Mon Sep 17 00:00:00 2001 From: James Piechota Date: Sat, 7 Mar 2026 07:17:47 -0500 Subject: [PATCH 033/135] impr: speed up bundler by removing tx data when only header is needed also force an lmdb flush to disk to ensure recovered items are correctly persisted --- src/dev_bundler_cache.erl | 19 +++++++++++++++++-- src/dev_bundler_recovery.erl | 7 +++---- src/dev_bundler_task.erl | 6 +++--- test/arbundles.js/upload-items.js | 1 + 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/dev_bundler_cache.erl b/src/dev_bundler_cache.erl index 979a96d2b..ff5dac963 100644 --- a/src/dev_bundler_cache.erl +++ b/src/dev_bundler_cache.erl @@ -81,7 +81,8 @@ write_tx(TX, Items, Opts) when is_map(TX) -> ok = link_item_to_tx(Item, TX, Opts) end, Items - ). + ), + ok. complete_tx(TX, Opts) -> set_tx_status(TX, <<"complete">>, Opts). @@ -163,7 +164,10 @@ load_tx(TXID, Opts) -> %% @doc Write a value to a pseudopath. write_pseudopath(Path, Value, Opts) -> Store = hb_opts:get(store, no_viable_store, Opts), - hb_store:write(Store, Path, Value). + Result = hb_store:write(Store, Path, Value), + % force a flush to disk + hb_store:read(Store, Path), + Result. %% @doc Read a value from a pseudopath. read_pseudopath(Path, Opts) -> @@ -256,6 +260,17 @@ load_unbundled_items_test() -> ?assertEqual(lists:sort([Item1, Item3]), UnbundledItems3), ok. +recovered_items_relink_to_original_bundle_path_test() -> + Opts = #{store => hb_test_utils:test_store()}, + Item = new_data_item(1, <<"data1">>, Opts), + ok = write_item(Item, Opts), + [RecoveredItem] = load_items(<<>>, Opts), + TX = new_tx(1, Opts), + ok = write_tx(TX, [RecoveredItem], Opts), + ?assertEqual(tx_id(TX, Opts), get_item_bundle(Item, Opts)), + ?assertEqual([], load_items(<<>>, Opts)), + ok. + load_bundle_states_test() -> Opts = #{store => hb_test_utils:test_store()}, TX1 = new_tx(1, Opts), diff --git a/src/dev_bundler_recovery.erl b/src/dev_bundler_recovery.erl index 59f989f25..986a59a3d 100644 --- a/src/dev_bundler_recovery.erl +++ b/src/dev_bundler_recovery.erl @@ -23,9 +23,7 @@ recover_bundles(ServerPID, Opts) -> do_recover_unbundled_items(ServerPID, Opts) -> try - ItemIDs = dev_bundler_cache:list_item_ids(Opts), - ?event(bundler_short, {recover_unbundled_items_start, - {count, length(ItemIDs)}}), + ?event(bundler_short, {recover_unbundled_items_start}), UnbundledItems = dev_bundler_cache:load_items( <<>>, Opts, @@ -63,8 +61,9 @@ do_recover_unbundled_items(ServerPID, Opts) -> do_recover_bundles(ServerPID, Opts) -> try - ?event(bundler_short, {recover_bundles_start}), BundleStates = dev_bundler_cache:load_bundle_states(Opts), + ?event(bundler_short, {recover_bundles_start, + {count, length(BundleStates)}}), lists:foreach( fun({TXID, Status}) -> recover_bundle(ServerPID, TXID, Status, Opts) diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl index 22b3fc804..e4f7839c9 100644 --- a/src/dev_bundler_task.erl +++ b/src/dev_bundler_task.erl @@ -40,7 +40,7 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> SignedTX = ar_tx:sign(TX#tx{ anchor = Anchor, reward = Price }, Wallet), % Convert and post Committed = hb_message:convert( - SignedTX, + SignedTX#tx{ data = <<>> }, #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, Opts), @@ -65,7 +65,7 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> {_, ErrorReason} -> {error, ErrorReason} end; {PriceErr, AnchorErr} -> - ?event(bundle_short, + ?event(bundler_short, log_task(task_failed, Task, [ {price, PriceErr}, {anchor, AnchorErr} @@ -74,7 +74,7 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> end catch _:Err:_Stack -> - ?event(bundle_short, log_task(task_failed, Task, [{error, Err}])), + ?event(bundler_short, log_task(task_failed, Task, [{error, Err}])), {error, Err} end; diff --git a/test/arbundles.js/upload-items.js b/test/arbundles.js/upload-items.js index a075d6735..9bc63d475 100644 --- a/test/arbundles.js/upload-items.js +++ b/test/arbundles.js/upload-items.js @@ -68,6 +68,7 @@ async function performanceTest(walletPath, itemCount, bytesPerItem = 0) { const uploadPromises = batch.map(async (item) => { try { + console.log(`Posting data item: ${item.id}`); const response = await fetch(endpoint, { method: "POST", headers: { From 87588e1daea085c7404473092e9073d8245d9d9f Mon Sep 17 00:00:00 2001 From: James Piechota Date: Sat, 7 Mar 2026 09:08:15 -0500 Subject: [PATCH 034/135] fix: fix bundler regression causing proofs not to be posted (unreleased regression within this PR) Now we do 2 converts before posting a TX: - one to build a Header-only TX that we use for posting. This makes the posting process much quicker - one to biuld a full-data TX tha we use for caching and recovery We shouldn't need to cache the full-data TX, so a future optmization can address that. We'll just have to be careful about rebuilding the data payload from the cache data items to ensure the reult is the same as when the original TX was posted --- src/dev_bundler_recovery.erl | 17 ++++++++++++----- src/dev_bundler_task.erl | 22 +++++++++++++++++++--- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/dev_bundler_recovery.erl b/src/dev_bundler_recovery.erl index 986a59a3d..5368fd70a 100644 --- a/src/dev_bundler_recovery.erl +++ b/src/dev_bundler_recovery.erl @@ -55,7 +55,8 @@ do_recover_unbundled_items(ServerPID, Opts) -> {recover_unbundled_items_failed, {error, Error}, {stack, Stack} - } + }, + Opts ) end. @@ -80,7 +81,8 @@ do_recover_bundles(ServerPID, Opts) -> {recover_bundles_failed, {error, Error}, {stack, Stack} - } + }, + Opts ) end. @@ -116,7 +118,8 @@ recover_bundle(ServerPID, TXID, Status, Opts) -> {failed_to_load_bundle_item, {tx_id, {explicit, TXID}}, {item_id, {explicit, ItemID}} - } + }, + Opts ), throw({failed_to_load_bundle_item, ItemID}) end @@ -131,7 +134,8 @@ recover_bundle(ServerPID, TXID, Status, Opts) -> {tx_id, {explicit, TXID}}, {error, Error}, {stack, Stack} - } + }, + Opts ) end. @@ -189,7 +193,10 @@ recover_bundles_skips_complete_test() -> end. recover_bundles_failed_bundle_items_continue_test() -> - Opts = #{store => hb_test_utils:test_store()}, + Opts = #{ + store => hb_test_utils:test_store(), + debug_print => false + }, ValidItem = new_data_item(1, 10, Opts), ok = dev_bundler_cache:write_item(ValidItem, Opts), ValidTX = new_bundle_tx([ValidItem], Opts), diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl index e4f7839c9..23df78de0 100644 --- a/src/dev_bundler_task.erl +++ b/src/dev_bundler_task.erl @@ -39,16 +39,32 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), SignedTX = ar_tx:sign(TX#tx{ anchor = Anchor, reward = Price }, Wallet), % Convert and post + % We build two Structured version of the TX: + % - Header only is used for posting. This greatly speeds up + % the posting process. + % - Full TX including data is used for recovery + % + % TODO: as a future improvement we should be able to recover + % from the TX header alone, but we have to be careful about + % how we rebuild the TX data to ensure it matche the already + % posted TX. Committed = hb_message:convert( + SignedTX, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, + Opts), + CommittedHeader = hb_message:convert( SignedTX#tx{ data = <<>> }, #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, Opts), - ?event(bundler_short, {posting_tx, - {tx, {explicit, hb_message:id(Committed, signed, Opts)}}}), + ?event(bundler_short, log_task(posting_tx, + Task, + [{tx, {explicit, hb_message:id(Committed, signed, Opts)}}] + )), PostTXResponse = hb_ao:resolve( #{ <<"device">> => <<"arweave@2.9">> }, - Committed#{ + CommittedHeader#{ <<"path">> => <<"/tx">>, <<"method">> => <<"POST">> }, From 4cfd75fbe6901b7154efedc4b0425876ad51323f Mon Sep 17 00:00:00 2001 From: James Piechota Date: Sat, 7 Mar 2026 09:18:38 -0500 Subject: [PATCH 035/135] chore: typos --- src/dev_bundler_task.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl index 23df78de0..e442fb311 100644 --- a/src/dev_bundler_task.erl +++ b/src/dev_bundler_task.erl @@ -39,14 +39,14 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), SignedTX = ar_tx:sign(TX#tx{ anchor = Anchor, reward = Price }, Wallet), % Convert and post - % We build two Structured version of the TX: - % - Header only is used for posting. This greatly speeds up + % We build two Structured versions of the TX: + % - Header-only is used for posting. This greatly speeds up % the posting process. % - Full TX including data is used for recovery % % TODO: as a future improvement we should be able to recover % from the TX header alone, but we have to be careful about - % how we rebuild the TX data to ensure it matche the already + % how we rebuild the TX data to ensure it matches the already % posted TX. Committed = hb_message:convert( SignedTX, From 066f89ffd570030fc5cd7cb05c044b4deb5e6e06 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Sat, 7 Mar 2026 10:02:09 -0500 Subject: [PATCH 036/135] impr: post the #tx record directly in bundler instead of using resolve This breaks the HB loose coupling but for large bundles the TX posts in seconds instead of minutes --- src/dev_arweave.erl | 32 ++++++++++++++++++-------------- src/dev_bundler_task.erl | 20 +++----------------- 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index 629901f9a..a1cec45d8 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -5,7 +5,7 @@ %%% `/arweave` route in the node's configuration message. -module(dev_arweave). -export([tx/3, raw/3, chunk/3, block/3, current/3, status/3, price/3, tx_anchor/3]). --export([post_tx/3, post_tx/4, post_binary_ans104/2, post_json_chunk/2]). +-export([post_tx_header/2, post_tx/3, post_tx/4, post_binary_ans104/2, post_json_chunk/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -65,19 +65,7 @@ extract_target(Base, Request, Opts) -> post_tx(_Base, Request, Opts, <<"tx@1.0">>) -> TX = hb_message:convert(Request, <<"tx@1.0">>, Opts), - JSON = ar_tx:tx_to_json_struct(TX#tx{ data = <<>> }), - Serialized = hb_json:encode(JSON), - LogExtra = [ - {codec, <<"tx@1.0">>}, - {id, {explicit, hb_util:human_id(TX#tx.id)}} - ], - Res = request( - <<"POST">>, - <<"/tx">>, - #{ <<"body">> => Serialized }, - LogExtra, - Opts - ), + Res = post_tx_header(TX, Opts), case Res of {ok, _} -> CacheRes = hb_cache:write(Request, Opts), @@ -101,6 +89,22 @@ post_tx(_Base, Request, Opts, <<"ans104@1.0">>) -> ], post_binary_ans104(Serialized, LogExtra, Opts). + +post_tx_header(TX, Opts) -> + JSON = ar_tx:tx_to_json_struct(TX#tx{ data = <<>> }), + Serialized = hb_json:encode(JSON), + LogExtra = [ + {codec, <<"tx@1.0">>}, + {id, {explicit, hb_util:human_id(TX#tx.id)}} + ], + request( + <<"POST">>, + <<"/tx">>, + #{ <<"body">> => Serialized }, + LogExtra, + Opts + ). + post_binary_ans104(SerializedTX, Opts) -> LogExtra = [ {codec, <<"ans104@1.0">>}, diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl index e442fb311..810b5dfa8 100644 --- a/src/dev_bundler_task.erl +++ b/src/dev_bundler_task.erl @@ -38,12 +38,6 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> % Sign the TX Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), SignedTX = ar_tx:sign(TX#tx{ anchor = Anchor, reward = Price }, Wallet), - % Convert and post - % We build two Structured versions of the TX: - % - Header-only is used for posting. This greatly speeds up - % the posting process. - % - Full TX including data is used for recovery - % % TODO: as a future improvement we should be able to recover % from the TX header alone, but we have to be careful about % how we rebuild the TX data to ensure it matches the already @@ -53,23 +47,15 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, Opts), - CommittedHeader = hb_message:convert( - SignedTX#tx{ data = <<>> }, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, - Opts), ?event(bundler_short, log_task(posting_tx, Task, [{tx, {explicit, hb_message:id(Committed, signed, Opts)}}] )), - PostTXResponse = hb_ao:resolve( - #{ <<"device">> => <<"arweave@2.9">> }, - CommittedHeader#{ - <<"path">> => <<"/tx">>, - <<"method">> => <<"POST">> - }, + PostTXResponse = dev_arweave:post_tx_header( + SignedTX, Opts ), + ?event(bundler_short, {post_tx_response, PostTXResponse}), case PostTXResponse of {ok, _Result} -> dev_bundler_cache:write_tx( From 79cf33518d774941f92cb2faba4961f9b528d4ce Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 7 Mar 2026 11:20:29 -0500 Subject: [PATCH 037/135] impr: `hb_maps` for `get` in 500 page rendering --- src/hb_format.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hb_format.erl b/src/hb_format.erl index b05f4b4c3..ee07ee0ed 100644 --- a/src/hb_format.erl +++ b/src/hb_format.erl @@ -401,9 +401,9 @@ escape_format(Else) -> Else. %% @doc Format an error message as a string. error(ErrorMsg, Opts) -> - Type = hb_ao:get(<<"type">>, ErrorMsg, <<"">>, Opts), - Details = hb_ao:get(<<"details">>, ErrorMsg, <<"">>, Opts), - Stacktrace = hb_ao:get(<<"stacktrace">>, ErrorMsg, <<"">>, Opts), + Type = hb_maps:get(<<"type">>, ErrorMsg, <<"[No type]">>, Opts), + Details = hb_maps:get(<<"details">>, ErrorMsg, <<"[No details]">>, Opts), + Stacktrace = hb_maps:get(<<"stacktrace">>, ErrorMsg, <<"[No trace]">>, Opts), hb_util:bin( [ <<"Termination type: '">>, Type, From d9a5f74f5cc921e7fc07c334f68db656b1a74fb6 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Sat, 7 Mar 2026 11:59:35 -0500 Subject: [PATCH 038/135] chore: remove debugging log that shouldn't have been committed: --- src/dev_bundler_task.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl index 810b5dfa8..c07f774f3 100644 --- a/src/dev_bundler_task.erl +++ b/src/dev_bundler_task.erl @@ -55,7 +55,6 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> SignedTX, Opts ), - ?event(bundler_short, {post_tx_response, PostTXResponse}), case PostTXResponse of {ok, _Result} -> dev_bundler_cache:write_tx( From 3b69346807c798e537f5796894a01b8a3977fcb5 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 7 Mar 2026 13:11:19 -0500 Subject: [PATCH 039/135] chore: params --- src/dev_arweave.erl | 2 +- src/hb_opts.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index a1cec45d8..f6983a296 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -492,7 +492,7 @@ fill_gaps(ChunkInfos, Offset, EndOffset, Opts) -> %% @doc Fetch chunks at the given offsets in parallel and parse the responses %% into {AbsoluteStartOffset, AbsoluteEndOffset, ChunkBinary} tuples. fetch_and_collect(Offsets, Opts) -> - Concurrency = hb_opts:get(chunk_fetch_concurrency, 10, Opts), + Concurrency = hb_opts:get(arweave_chunk_fetch_concurrency, 10, Opts), Results = hb_pmap:parallel_map( Offsets, fun(O) -> get_chunk(O, Opts) end, diff --git a/src/hb_opts.erl b/src/hb_opts.erl index d7fa7df99..67028055d 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -437,7 +437,7 @@ default_message() -> % responses are not verifiable. ans104_trust_gql => true, % Number of chunks to fetch in parallel when loading a TX or dataitem. - chunk_fetch_concurrency => 10, + arweave_chunk_fetch_concurrency => 5, http_extra_opts => #{ force_message => true, From 322e32601b68d6c7dfcf7e97c0ecbf0404b99632 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 7 Mar 2026 12:59:31 -0500 Subject: [PATCH 040/135] fix: manifests return 404 on load error now, rather than 5xx. --- src/dev_manifest.erl | 28 +++++++++++++--------------- src/hb_maps.erl | 23 ++++++++++++++++++++++- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index aa0c33573..0a84089dd 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -64,15 +64,10 @@ route(ID, _, _, Opts) when ?IS_ID(ID) -> route(Key, M1, M2, Opts) -> ?event(debug_manifest, {manifest_lookup, {key, Key}, {m1, M1}, {m2, {explicit, M2}}}), {ok, Manifest} = manifest(M1, M2, Opts), - Res = hb_ao:get( - <<"paths/", Key/binary>>, - {as, <<"message@1.0">>, Manifest}, - Opts - ), - ?event({manifest_lookup_result, {res, Res}}), - case Res of - not_found -> - %% Support materialized view in some JavaScript frameworks + {ok, Res} = maps:find(<<"paths">>, Manifest), + case maps:get(Key, Res, no_path_match) of + no_path_match -> + % Support materialized view in some JavaScript frameworks. case hb_opts:get(manifest_404, fallback, Opts) of error -> ?event({manifest_404_error, {key, Key}}), @@ -81,9 +76,11 @@ route(Key, M1, M2, Opts) -> ?event({manifest_fallback, {key, Key}}), route(<<"index">>, M1, M2, Opts) end; - _ -> - ?event({manifest_lookup_success, {key, Key}}), - {ok, Res} + Result -> + ?event({manifest_lookup_success, {key, Key}, {result, Result}}), + try {ok, hb_cache:ensure_loaded(Result, Opts)} + catch _:_:_ -> {error, not_found} + end end. %% @doc Implement the `on/request' hook for the `manifest@1.0' device, finding @@ -174,11 +171,12 @@ maybe_cast_manifest(Msg, Opts) -> %% message with the `~manifest@1.0' device. manifest(Base, _Req, Opts) -> JSON = - hb_ao:get_first( + hb_maps:get_first( [ - {{as, <<"message@1.0">>, Base}, [<<"data">>]}, - {{as, <<"message@1.0">>, Base}, [<<"body">>]} + {Base, <<"data">>}, + {Base, <<"body">>} ], + not_found, Opts ), FlatManifest = #{ <<"paths">> := FlatPaths } = hb_json:decode(JSON), diff --git a/src/hb_maps.erl b/src/hb_maps.erl index 391e018dc..bcb38d2d7 100644 --- a/src/hb_maps.erl +++ b/src/hb_maps.erl @@ -17,7 +17,7 @@ %%% yourself from the inevitable issues that will arise from using this %%% module without understanding the full implications. You have been warned. -module(hb_maps). --export([get/2, get/3, get/4, put/3, put/4, find/2, find/3]). +-export([get/2, get/3, get/4, get_first/2, get_first/3, put/3, put/4, find/2, find/3]). -export([is_key/2, is_key/3, keys/1, keys/2, values/1, values/2]). -export([map/2, map/3, filter/2, filter/3, filtermap/2, filtermap/3]). -export([fold/3, fold/4, take/2, take/3, size/1, size/2]). @@ -26,6 +26,27 @@ -export([from_list/1, to_list/1, to_list/2]). -include_lib("eunit/include/eunit.hrl"). +%%% HyperBEAM-specific functions + +-spec get_first( + Paths :: [{Base :: map() | binary(), Path :: binary()}], + Opts :: map() +) -> term(). +get_first(Paths, Opts) -> + get_first(Paths, not_found, Opts). + +-spec get_first( + Paths :: [{Base :: map() | binary(), Path :: binary()}], + Default :: term(), + Opts :: map() +) -> term(). +get_first([], Default, _Opts) -> Default; +get_first([{Base, Path}|Paths], Default, Opts) -> + case find(Path, Base, Opts) of + {ok, Value} -> Value; + error -> get_first(Paths, Default, Opts) + end. + -spec get(Key :: term(), Map :: map()) -> term(). get(Key, Map) -> get(Key, Map, undefined). From 94db201f914544ffe68522022e09fd523423d64d Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 7 Mar 2026 18:05:06 -0500 Subject: [PATCH 041/135] chore: event --- src/dev_manifest.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index 0a84089dd..c1f7e3922 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -15,7 +15,7 @@ info() -> %% @doc Return the fallback index page when the manifest itself is requested. index(M1, M2, Opts) -> - ?event(debug_manifest, {index_request, {m1, M1}, {m2, M2}}), + ?event(debug_manifest, {index_request, {base, M1}, {request, M2}}, Opts), case route(<<"index">>, M1, M2, Opts) of {ok, Index} -> ?event({manifest_index_returned, Index}), From 27995e22b92f84b3808f0b7472e5c0cf9361fd99 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 7 Mar 2026 20:04:25 -0500 Subject: [PATCH 042/135] chore: fix edge-case bug for manifests that contain an `id` subdirectory --- src/dev_manifest.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index c1f7e3922..a561c0255 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -187,7 +187,7 @@ manifest(Base, _Req, Opts) -> %% @doc Generate a nested message of links to content from a parsed (and %% structured) manifest. -linkify(#{ <<"id">> := ID }, Opts) -> +linkify(#{ <<"id">> := ID }, Opts) when is_binary(ID) -> LinkOptsBase = (maps:with([store], Opts))#{ scope => [local, remote]}, {link, ID, LinkOptsBase#{ <<"type">> => <<"link">>, <<"lazy">> => false }}; linkify(Manifest, Opts) when is_map(Manifest) -> From 91c3199526a0288431bb5bf6a763b76ca9921dd8 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Mon, 9 Mar 2026 15:02:23 -0400 Subject: [PATCH 043/135] impr: both from and to can be negative offsest from tip (copycat) --- src/dev_copycat_arweave.erl | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/dev_copycat_arweave.erl b/src/dev_copycat_arweave.erl index c48e51af1..13c9930a7 100644 --- a/src/dev_copycat_arweave.erl +++ b/src/dev_copycat_arweave.erl @@ -28,22 +28,24 @@ arweave(_Base, Request, Opts) -> parse_range(Request, Opts) -> From = case hb_maps:find(<<"from">>, Request, Opts) of - {ok, Height} -> - RequestedFrom = hb_util:int(Height), - case RequestedFrom < 0 of - true -> latest_height(Opts) + RequestedFrom; - false -> RequestedFrom - end; + {ok, FromHeight} -> normalize_height(FromHeight, Opts); error -> latest_height(Opts) end, To = case hb_maps:find(<<"to">>, Request, Opts) of - {ok, ToHeight} -> hb_util:int(ToHeight); + {ok, ToHeight} -> normalize_height(ToHeight, Opts); error -> undefined end, {From, To}. +normalize_height(Height, Opts) -> + RequestedHeight = hb_util:int(Height), + case RequestedHeight < 0 of + true -> latest_height(Opts) + RequestedHeight; + false -> RequestedHeight + end. + latest_height(Opts) -> case hb_ao:resolve( <>, @@ -302,7 +304,7 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> false -> #{items_count => 0, bundle_count => 0, skipped_count => 0}; true -> ?event(copycat_debug, {fetching_bundle_header, - {tx_id, {explicit, TXID}}, + {tx_id, {string, TXID}}, {tx_end_offset, TXEndOffset}, {tx_data_size, TX#tx.data_size} }), @@ -328,6 +330,11 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> BundleIndex ) end), + ?event(copycat_debug, + {bundle_items_indexed, + {tx_id, {string, TXID}}, + {items_count, ItemsCount} + }), % Single event increment for the batch record_event_metrics(<<"item_indexed">>, ItemsCount, TotalTime), #{items_count => ItemsCount, bundle_count => 1, skipped_count => 0}; @@ -940,16 +947,21 @@ auto_stop_partial_index_test() -> ?assertNot(has_any_indexed_tx(Block-1, Opts)), ok. -negative_from_parse_range_test() -> +negative_parse_range_test() -> {_TestStore, _StoreOpts, Opts} = setup_index_opts(), {ok, Tip} = hb_ao:resolve( <>, Opts ), - {From, To} = parse_range(#{ <<"from">> => <<"-3">> }, Opts), - ?assertEqual(hb_util:int(Tip) - 3, From), - ?assertEqual(undefined, To), + {NegativeFrom, UndefinedTo} = + parse_range(#{ <<"from">> => <<"-3">> }, Opts), + ?assertEqual(hb_util:int(Tip) - 3, NegativeFrom), + ?assertEqual(undefined, UndefinedTo), + {PositiveFrom, NegativeTo} = + parse_range(#{ <<"from">> => <<"10">>, <<"to">> => <<"-3">> }, Opts), + ?assertEqual(10, PositiveFrom), + ?assertEqual(hb_util:int(Tip) - 3, NegativeTo), ok. negative_from_index_test() -> From 58228632253d03aaf2cb15ba83d54403e4b4782f Mon Sep 17 00:00:00 2001 From: James Piechota Date: Wed, 11 Mar 2026 14:02:04 -0400 Subject: [PATCH 044/135] log: add bundler_upload_error log --- src/dev_bundler_task.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl index c07f774f3..2a0d90f1d 100644 --- a/src/dev_bundler_task.erl +++ b/src/dev_bundler_task.erl @@ -74,8 +74,10 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> {error, {PriceErr, AnchorErr}} end catch - _:Err:_Stack -> + _:Err:Stack -> ?event(bundler_short, log_task(task_failed, Task, [{error, Err}])), + ?event(bundler_upload_error, + log_task(task_failed, Task, [{error, Err}, {trace, Stack}])), {error, Err} end; From 299fe21ba35bdc9f5e86183cfd4204ca89dccdcb Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 11 Mar 2026 17:39:35 +0100 Subject: [PATCH 045/135] feat: support multiple blacklist providers - Add `blacklist_providers` config key accepting a list of providers - Keep backward compatibility with singular `blacklist_provider` key - Each provider is fetched independently with try/catch isolation - Merge all provider blacklists into a single ETS cache (union) - Add tests for multi-provider, backward compat, failure resilience, and numbered-message config format --- src/dev_blacklist.erl | 218 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 198 insertions(+), 20 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 6d9739ce9..1b939ab92 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -1,9 +1,12 @@ %%% @doc A request hook device for content moderation by blacklist. %%% -%%% The node operator configures a blacklist provider via the `blacklist-provider` -%%% key in the node message options. The provider can be a message or a path that -%%% returns a message or binary. If a binary is returned from the provider, it is -%%% parsed as a newline-delimited list of IDs. +%%% The node operator configures blacklist providers via the +%%% `blacklist-providers` key (a list) or the legacy `blacklist-provider` key +%%% (a single provider) in the node message options. Each provider can be a +%%% message or a path that returns a message or binary. If a binary is returned +%%% from a provider, it is parsed as a newline-delimited list of IDs. When +%%% multiple providers are configured, their blacklists are merged into a single +%%% cache (union of all IDs). %%% %%% The device is intended for use as a `~hook@1.0` `on/request` handler. It %%% blocks requests when any ID present in the hook payload matches the active @@ -94,24 +97,62 @@ maybe_refresh(Opts) -> skip_update end. -%% @doc Fetch the blacklist and insert the IDs into the cache table. +%% @doc Fetch blacklists from all configured providers and insert IDs into the +%% cache table. fetch_and_insert_ids(Opts) -> ensure_cache_table(Opts), - case hb_opts:get(blacklist_provider, no_provider, Opts) of - no_provider -> {ok, 0}; - Provider -> - case execute_provider(Provider, Opts) of - {ok, Blacklist} -> - {ok, IDs} = parse_blacklist(Blacklist, Opts), - ?event({parsed_blacklist, {ids, IDs}}), - BlacklistID = hb_message:id(Blacklist, all, Opts), - ?event({update_blacklist_cache, {ids, IDs}, {blacklist_id, BlacklistID}}), - Table = cache_table_name(Opts), - {ok, insert_ids(IDs, BlacklistID, Table, Opts)}; - {error, _} = Error -> - ?event({execute_provider_error, Error}), - Error + Providers = resolve_providers(Opts), + Total = lists:foldl( + fun(Provider, Acc) -> + case fetch_single_provider(Provider, Opts) of + {ok, Count} -> Acc + Count; + {error, _} -> Acc end + end, + 0, + Providers + ), + {ok, Total}. + +%% @doc Resolve the configured providers into a list. Checks the plural +%% `blacklist_providers` key first, then falls back to the singular +%% `blacklist_provider` key for backward compatibility. +resolve_providers(Opts) -> + case hb_opts:get(blacklist_providers, not_found, Opts) of + not_found -> + case hb_opts:get(blacklist_provider, no_provider, Opts) of + no_provider -> []; + SingleProvider -> [SingleProvider] + end; + Providers when is_list(Providers) -> + Providers; + Providers when is_map(Providers) -> + hb_util:message_to_ordered_list(Providers, Opts); + Other -> + [Other] + end. + +%% @doc Fetch a single provider's blacklist and insert its IDs into the cache. +fetch_single_provider(Provider, Opts) -> + try + case execute_provider(Provider, Opts) of + {ok, Blacklist} -> + {ok, IDs} = parse_blacklist(Blacklist, Opts), + ?event({parsed_blacklist, {ids, IDs}}), + BlacklistID = hb_message:id(Blacklist, all, Opts), + ?event({update_blacklist_cache, + {ids, IDs}, {blacklist_id, BlacklistID}}), + Table = cache_table_name(Opts), + {ok, insert_ids(IDs, BlacklistID, Table, Opts)}; + {error, _} = Error -> + ?event({execute_provider_error, Error}), + Error + end + catch + Type:Reason -> + ?event({provider_fetch_error, + {type, Type}, {reason, Reason}, {provider, Provider}}), + {error, {Type, Reason}} end. %% @doc Execute the blacklist provider, returning the result. @@ -331,4 +372,141 @@ blacklist_from_external_http_test() -> <<"reason">> := <<"content-policy">> }}, hb_http:get(Node, SignedID1, NodeOpts) - ). \ No newline at end of file + ). + +%% @doc Test that multiple providers merge their blacklists. +multiple_providers_test() -> + {ok, #{ + opts := Opts0, + signed1 := SignedID1, + unsigned2 := UnsignedID2, + unsigned3 := UnsignedID3 + }} = setup_test_env(), + Blacklist1 = #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => <> + }, + Blacklist2 = #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => <> + }, + BlacklistMsg1 = hb_message:commit(Blacklist1, Opts0), + BlacklistMsg2 = hb_message:commit(Blacklist2, Opts0), + {ok, BlacklistID1} = hb_cache:write(BlacklistMsg1, Opts0), + {ok, BlacklistID2} = hb_cache:write(BlacklistMsg2, Opts0), + Opts1 = Opts0#{ + blacklist_providers => [BlacklistID1, BlacklistID2], + on => #{ + <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } + } + }, + Node = hb_http_server:start_node(Opts1), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, SignedID1, Opts1) + ), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, <<"/", UnsignedID2/binary>>, Opts1) + ), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + ok. + +%% @doc Test that the singular `blacklist_provider` key still works. +backward_compat_test() -> + {ok, #{ + opts := Opts0, + signed1 := SignedID1, + unsigned3 := UnsignedID3, + blacklist := BlacklistID + }} = setup_test_env(), + Opts1 = Opts0#{ + blacklist_provider => BlacklistID, + on => #{ + <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } + } + }, + Node = hb_http_server:start_node(Opts1), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, SignedID1, Opts1) + ), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + ok. + +%% @doc Test that a failing provider does not prevent other providers from +%% contributing entries. +provider_failure_resilience_test() -> + {ok, #{ + opts := Opts0, + signed1 := SignedID1, + unsigned3 := UnsignedID3, + blacklist := BlacklistID + }} = setup_test_env(), + BadProvider = <<"aaaabbbbccccddddeeeeffffgggghhhhiiiijjjjkkkk">>, + Opts1 = Opts0#{ + blacklist_providers => [BadProvider, BlacklistID], + on => #{ + <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } + } + }, + Node = hb_http_server:start_node(Opts1), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, SignedID1, Opts1) + ), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + ok. + +%% @doc Test that `blacklist_providers` works when given as a numbered message +%% (map with integer keys), as would arrive from structured config formats. +plural_numbered_message_test() -> + {ok, #{ + opts := Opts0, + signed1 := SignedID1, + unsigned2 := UnsignedID2, + unsigned3 := UnsignedID3 + }} = setup_test_env(), + Blacklist1 = #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => <> + }, + Blacklist2 = #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => <> + }, + BlacklistMsg1 = hb_message:commit(Blacklist1, Opts0), + BlacklistMsg2 = hb_message:commit(Blacklist2, Opts0), + {ok, BlacklistID1} = hb_cache:write(BlacklistMsg1, Opts0), + {ok, BlacklistID2} = hb_cache:write(BlacklistMsg2, Opts0), + NumberedProviders = + hb_util:list_to_numbered_message([BlacklistID1, BlacklistID2]), + Opts1 = Opts0#{ + blacklist_providers => NumberedProviders, + on => #{ + <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } + } + }, + Node = hb_http_server:start_node(Opts1), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, SignedID1, Opts1) + ), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, <<"/", UnsignedID2/binary>>, Opts1) + ), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + ok. \ No newline at end of file From 87d5dde5acf2d4a70c5ba21c9d14d3ba6b8eb807 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Wed, 11 Mar 2026 19:08:06 +0100 Subject: [PATCH 046/135] Simplify: drop backward compat, use blacklist_providers only - Remove fallback to singular blacklist_provider key - Remove numbered-message map handling from resolve_providers - Remove ?DEFAULT_PROVIDER macro (stale under new plural API) - Remove backward_compat_test and plural_numbered_message_test - Update existing tests to use blacklist_providers --- src/dev_blacklist.erl | 114 +++++------------------------------------- 1 file changed, 13 insertions(+), 101 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 1b939ab92..2233cb0c1 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -1,12 +1,10 @@ %%% @doc A request hook device for content moderation by blacklist. %%% %%% The node operator configures blacklist providers via the -%%% `blacklist-providers` key (a list) or the legacy `blacklist-provider` key -%%% (a single provider) in the node message options. Each provider can be a -%%% message or a path that returns a message or binary. If a binary is returned -%%% from a provider, it is parsed as a newline-delimited list of IDs. When -%%% multiple providers are configured, their blacklists are merged into a single -%%% cache (union of all IDs). +%%% `blacklist-providers` key (a list) in the node message options. Each provider +%%% can be a message or a path that returns a message or binary. If a binary is +%%% returned from a provider, it is parsed as a newline-delimited list of IDs. +%%% Multiple providers are merged into a single cache (union of all IDs). %%% %%% The device is intended for use as a `~hook@1.0` `on/request` handler. It %%% blocks requests when any ID present in the hook payload matches the active @@ -24,12 +22,6 @@ -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(DEFAULT_PROVIDER, - [#{ - <<"data-protocol">> => <<"content-policy">>, - <<"body">> => #{ <<"body">> => <<>> } - }] -). -define(DEFAULT_MIN_WAIT, 60). %% @doc Hook handler: block requests that involve blacklisted IDs. @@ -114,22 +106,11 @@ fetch_and_insert_ids(Opts) -> ), {ok, Total}. -%% @doc Resolve the configured providers into a list. Checks the plural -%% `blacklist_providers` key first, then falls back to the singular -%% `blacklist_provider` key for backward compatibility. +%% @doc Resolve the configured providers into a list. resolve_providers(Opts) -> - case hb_opts:get(blacklist_providers, not_found, Opts) of - not_found -> - case hb_opts:get(blacklist_provider, no_provider, Opts) of - no_provider -> []; - SingleProvider -> [SingleProvider] - end; - Providers when is_list(Providers) -> - Providers; - Providers when is_map(Providers) -> - hb_util:message_to_ordered_list(Providers, Opts); - Other -> - [Other] + case hb_opts:get(blacklist_providers, [], Opts) of + Providers when is_list(Providers) -> Providers; + _ -> [] end. %% @doc Fetch a single provider's blacklist and insert its IDs into the cache. @@ -296,7 +277,7 @@ basic_test() -> }} = setup_test_env(), Opts1 = Opts0#{ - blacklist_provider => BlacklistID, + blacklist_providers => [BlacklistID], on => #{ <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } } @@ -323,7 +304,7 @@ default_provider_test() -> signed1 := SignedID1, unsigned3 := UnsignedID3 }} = setup_test_env(), - Opts1 = Opts0#{ blacklist_provider => ?DEFAULT_PROVIDER }, + Opts1 = Opts0#{ blacklist_providers => [] }, Node = hb_http_server:start_node(Opts1), ?assertMatch( {ok, <<"test-3">>}, @@ -351,11 +332,11 @@ blacklist_from_external_http_test() -> #{ store => RootStore, priv_wallet => ar_wallet:new(), - blacklist_provider => - << + blacklist_providers => + [<< "/~relay@1.0/call?relay-method=GET&relay-path=", BlacklistHostNode/binary, BlacklistID/binary - >>, + >>], on => #{ <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } } @@ -415,31 +396,6 @@ multiple_providers_test() -> ), ok. -%% @doc Test that the singular `blacklist_provider` key still works. -backward_compat_test() -> - {ok, #{ - opts := Opts0, - signed1 := SignedID1, - unsigned3 := UnsignedID3, - blacklist := BlacklistID - }} = setup_test_env(), - Opts1 = Opts0#{ - blacklist_provider => BlacklistID, - on => #{ - <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } - } - }, - Node = hb_http_server:start_node(Opts1), - ?assertMatch( - {error, #{ <<"status">> := 451 }}, - hb_http:get(Node, SignedID1, Opts1) - ), - ?assertMatch( - {ok, <<"test-3">>}, - hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) - ), - ok. - %% @doc Test that a failing provider does not prevent other providers from %% contributing entries. provider_failure_resilience_test() -> @@ -465,48 +421,4 @@ provider_failure_resilience_test() -> {ok, <<"test-3">>}, hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) ), - ok. - -%% @doc Test that `blacklist_providers` works when given as a numbered message -%% (map with integer keys), as would arrive from structured config formats. -plural_numbered_message_test() -> - {ok, #{ - opts := Opts0, - signed1 := SignedID1, - unsigned2 := UnsignedID2, - unsigned3 := UnsignedID3 - }} = setup_test_env(), - Blacklist1 = #{ - <<"data-protocol">> => <<"content-policy">>, - <<"body">> => <> - }, - Blacklist2 = #{ - <<"data-protocol">> => <<"content-policy">>, - <<"body">> => <> - }, - BlacklistMsg1 = hb_message:commit(Blacklist1, Opts0), - BlacklistMsg2 = hb_message:commit(Blacklist2, Opts0), - {ok, BlacklistID1} = hb_cache:write(BlacklistMsg1, Opts0), - {ok, BlacklistID2} = hb_cache:write(BlacklistMsg2, Opts0), - NumberedProviders = - hb_util:list_to_numbered_message([BlacklistID1, BlacklistID2]), - Opts1 = Opts0#{ - blacklist_providers => NumberedProviders, - on => #{ - <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } - } - }, - Node = hb_http_server:start_node(Opts1), - ?assertMatch( - {error, #{ <<"status">> := 451 }}, - hb_http:get(Node, SignedID1, Opts1) - ), - ?assertMatch( - {error, #{ <<"status">> := 451 }}, - hb_http:get(Node, <<"/", UnsignedID2/binary>>, Opts1) - ), - ?assertMatch( - {ok, <<"test-3">>}, - hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) - ), ok. \ No newline at end of file From a80056b4f38d8aa8d800bb0cda7efb4087defbac Mon Sep 17 00:00:00 2001 From: James Piechota Date: Wed, 11 Mar 2026 17:35:00 -0400 Subject: [PATCH 047/135] fix: cherrypick singleton code from feat/safe-harbour --- src/dev_bundler.erl | 12 +++----- src/dev_rate_limit.erl | 68 ++++++++++++++++++++---------------------- src/hb_name.erl | 37 +++++++++++++++++++++++ 3 files changed, 74 insertions(+), 43 deletions(-) diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index ceffa327c..36653396c 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -98,14 +98,10 @@ cache_item(Item, Opts) -> %% @doc Return the PID of the bundler server. If the server is not running, %% it is started and registered with the name `?SERVER_NAME'. ensure_server(Opts) -> - case hb_name:lookup(?SERVER_NAME) of - undefined -> - PID = spawn(fun() -> init(Opts) end), - ?event(bundler_short, {starting_bundler_server, {pid, PID}}), - hb_name:register(?SERVER_NAME, PID), - hb_name:lookup(?SERVER_NAME); - PID -> PID - end. + hb_name:singleton( + ?SERVER_NAME, + fun() -> init(Opts) end + ). stop_server() -> case hb_name:lookup(?SERVER_NAME) of diff --git a/src/dev_rate_limit.erl b/src/dev_rate_limit.erl index 87c8d9c87..7d4f90ba0 100644 --- a/src/dev_rate_limit.erl +++ b/src/dev_rate_limit.erl @@ -115,41 +115,39 @@ is_limited(Reference, Opts) -> %% will fail with an error and the other will succeed. The effect to the caller %% is the same: A rate limiter is available to query. ensure_rate_limiter_started(Opts) -> - case hb_name:lookup(ServerID = server_id(Opts)) of - PID when is_pid(PID) -> PID; - undefined -> - spawn( - fun() -> - % Exit the process if we cannot register the server ID. - ok = hb_name:register(ServerID, self()), - Reqs = hb_opts:get(rate_limit_requests, ?DEFAULT_REQS, Opts), - Period = hb_opts:get(rate_limit_period, ?DEFAULT_PERIOD, Opts), - Max = hb_opts:get(rate_limit_max, ?DEFAULT_MAX, Opts), - Min = hb_opts:get(rate_limit_min, ?DEFAULT_MIN, Opts), - Exempt = hb_opts:get(rate_limit_exempt, [], Opts), - ?event( - rate_limit, - {started_rate_limiter, - {server_id, ServerID}, - {reqs, Reqs}, - {period, Period}, - {max, Max}, - {min, Min}, - {exempt, Exempt} - } - ), - server_loop( - #{ - reqs => Reqs, - period => Period, - max => Max, - min => Min, - peers => #{ Ref => infinity || Ref <- Exempt } - } - ) - end - ) - end. + ServerID = server_id(Opts), + hb_name:singleton( + ServerID, + fun() -> start_server(ServerID, Opts) end + ). + +start_server(ServerID, Opts) -> + % Exit the process if we cannot register the server ID. + Reqs = hb_opts:get(rate_limit_requests, ?DEFAULT_REQS, Opts), + Period = hb_opts:get(rate_limit_period, ?DEFAULT_PERIOD, Opts), + Max = hb_opts:get(rate_limit_max, ?DEFAULT_MAX, Opts), + Min = hb_opts:get(rate_limit_min, ?DEFAULT_MIN, Opts), + Exempt = hb_opts:get(rate_limit_exempt, [], Opts), + ?event( + rate_limit, + {started_rate_limiter, + {server_id, ServerID}, + {reqs, Reqs}, + {period, Period}, + {max, Max}, + {min, Min}, + {exempt, Exempt} + } + ), + server_loop( + #{ + reqs => Reqs, + period => Period, + max => Max, + min => Min, + peers => #{ Ref => infinity || Ref <- Exempt } + } + ). %% @doc The main loop of the rate limiter server. Only responds to two messages: %% - `{request, Self, Reference}': Debit the account of the given reference by 1. diff --git a/src/hb_name.erl b/src/hb_name.erl index e85c05162..65f1c2eab 100644 --- a/src/hb_name.erl +++ b/src/hb_name.erl @@ -5,6 +5,7 @@ %%% There can only ever be one registrant for a given name at a time. -module(hb_name). -export([start/0, register/1, register/2, unregister/1, lookup/1, all/0]). +-export([singleton/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(NAME_TABLE, hb_name_registry). @@ -58,6 +59,42 @@ unregister(Name) -> ets:delete(?NAME_TABLE, Name), ok. +%% @doc Atomic singleton lookup/spawn+register operation. +%% +%% Multiple callers may simultanoeously invoke this function, but the PID of +%% only one surviving surivor will be registered and returned to all callers. +%% If the given function crashes on spawn, then the operation will retry the +%% operation until they successfully spawn and register a process -- which will +%% promptly fail. The result is that the intended semantics are still preserved: +%% Calling `singleton' will always return the PID of a process that owns that name +%% at the time of return. +singleton(Name, Fun) -> + case lookup(Name) of + Registered when is_pid(Registered) -> Registered; + undefined -> singleton_spawn(Name, Fun) + end. + +%% @doc Perform the actual atomic spawn+register operation. +singleton_spawn(Name, Fun) -> + start(), + Parent = self(), + ReadyRef = make_ref(), + {PID, MonitorRef} = + spawn_monitor( + fun() -> + ok = ?MODULE:register(Name, Spawned = self()), + Parent ! {spawned, ReadyRef, Spawned}, + Fun() + end + ), + receive + {spawned, ReadyRef, PID} -> + erlang:demonitor(MonitorRef, [flush]), + PID; + {'DOWN', MonitorRef, process, _, _} -> + singleton(Name, Fun) + end. + %%% @doc Lookup a name -> PID. lookup(Name) when is_atom(Name) -> case whereis(Name) of From af25f8cfceca1eef8847f2d1f12a978e02897101 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 11 Mar 2026 20:20:00 -0400 Subject: [PATCH 048/135] chore: add `blacklist@1.0` to `on/request` defaults --- src/hb_opts.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 67028055d..bc37efc88 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -475,6 +475,9 @@ default_message() -> }, #{ <<"device">> => <<"manifest@1.0">> + }, + #{ + <<"device">> => <<"blacklist@1.0">> } ] }, From e29989017cfd59462495e2524d1a821b5fa18919 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 11 Mar 2026 21:03:02 -0400 Subject: [PATCH 049/135] impr: silent singleton enforcement flow --- src/hb_name.erl | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/hb_name.erl b/src/hb_name.erl index 65f1c2eab..09f95edab 100644 --- a/src/hb_name.erl +++ b/src/hb_name.erl @@ -79,20 +79,23 @@ singleton_spawn(Name, Fun) -> start(), Parent = self(), ReadyRef = make_ref(), - {PID, MonitorRef} = - spawn_monitor( + PID = + spawn( fun() -> - ok = ?MODULE:register(Name, Spawned = self()), - Parent ! {spawned, ReadyRef, Spawned}, - Fun() + Spawned = self(), + case catch ?MODULE:register(Name, Spawned) of + ok -> + Parent ! {spawned, ReadyRef, Spawned}, + Fun(); + _ -> + Parent ! {spawn_failed, ReadyRef}, + ok + end end ), receive - {spawned, ReadyRef, PID} -> - erlang:demonitor(MonitorRef, [flush]), - PID; - {'DOWN', MonitorRef, process, _, _} -> - singleton(Name, Fun) + {spawned, ReadyRef, PID} -> PID; + {spawn_failed, ReadyRef} -> singleton_spawn(Name, Fun) end. %%% @doc Lookup a name -> PID. From 205c2ca687707d3c4ea7fb6c1cabd1cd3d524c52 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 11 Mar 2026 21:03:28 -0400 Subject: [PATCH 050/135] impr: use singleton pattern on table creation --- src/dev_blacklist.erl | 46 +++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 2233cb0c1..ea4c90e44 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -205,25 +205,33 @@ insert_ids([ID | IDs], Value, Table, Opts) when ?IS_ID(ID) -> %% @doc Ensure the cache table exists. ensure_cache_table(Opts) -> TableName = cache_table_name(Opts), - case ets:info(TableName) of - undefined -> - ?event({creating_table, TableName}), - ets:new( - TableName, - [ - named_table, - set, - public, - {read_concurrency, true}, - {write_concurrency, true} - ] - ), - fetch_and_insert_ids(Opts); - _ -> - ?event({table_exists, TableName}), - ok - end, - TableName. + hb_name:singleton( + TableName, + fun() -> + case ets:info(TableName) of + undefined -> + ?event({creating_table, TableName}), + ets:new( + TableName, + [ + named_table, + set, + public, + {read_concurrency, true}, + {write_concurrency, true} + ] + ), + hb_util:until( + fun() -> ets:info(TableName) =/= undefined end, + 100 + ), + fetch_and_insert_ids(Opts); + _ -> + ?event({table_exists, TableName}), + ok + end + end + ). %% @doc Calculate the name of the cache table given the `Opts`. cache_table_name(Opts) -> From 04dbb46a73a26464cba53265c58558093d7744d9 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 11 Mar 2026 21:22:24 -0400 Subject: [PATCH 051/135] fix: ets table generation --- src/dev_blacklist.erl | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index ea4c90e44..8ebed5c56 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -205,11 +205,11 @@ insert_ids([ID | IDs], Value, Table, Opts) when ?IS_ID(ID) -> %% @doc Ensure the cache table exists. ensure_cache_table(Opts) -> TableName = cache_table_name(Opts), - hb_name:singleton( - TableName, - fun() -> - case ets:info(TableName) of - undefined -> + case ets:info(TableName) of + undefined -> + hb_name:singleton( + TableName, + fun() -> ?event({creating_table, TableName}), ets:new( TableName, @@ -221,17 +221,18 @@ ensure_cache_table(Opts) -> {write_concurrency, true} ] ), - hb_util:until( - fun() -> ets:info(TableName) =/= undefined end, - 100 - ), - fetch_and_insert_ids(Opts); - _ -> - ?event({table_exists, TableName}), - ok - end - end - ). + fetch_and_insert_ids(Opts), + receive kill -> ok end + end + ), + hb_util:until( + fun() -> ets:info(TableName) =/= undefined end, + 100 + ), + TableName; + _ -> + TableName + end. %% @doc Calculate the name of the cache table given the `Opts`. cache_table_name(Opts) -> From 39d53327525f38cfd7aabdd610a1c4cb5b9ccc41 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 11 Mar 2026 21:48:39 -0400 Subject: [PATCH 052/135] fix: improve collection of blacklist IDs --- src/dev_blacklist.erl | 10 ++++++++-- src/dev_manifest.erl | 13 ++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 8ebed5c56..5ca11388b 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -104,6 +104,7 @@ fetch_and_insert_ids(Opts) -> 0, Providers ), + ?event(blacklist_short, {fetched_and_inserted_ids, Total}, Opts), {ok, Total}. %% @doc Resolve the configured providers into a list. @@ -175,9 +176,14 @@ parse_blacklist_line(Line) -> collect_ids(Msg, Opts) -> lists:usort(collect_ids(Msg, [], Opts)). collect_ids(Bin, Acc, _Opts) when ?IS_ID(Bin) -> [hb_util:human_id(Bin) | Acc]; collect_ids(Bin, Acc, _Opts) when is_binary(Bin) -> Acc; -collect_ids(Link, Acc, Opts) when ?IS_LINK(Link) -> - collect_ids(hb_cache:ensure_loaded(Link, Opts), Acc, Opts); +collect_ids({link, ID, _}, Acc, _Opts) when ?IS_ID(ID) -> + [hb_util:human_id(ID) | Acc]; collect_ids(Msg, Acc, Opts) when is_map(Msg) -> + case hb_maps:get(<<"path">>, Msg, undefined, Opts) of + Path when ?IS_ID(Path) -> [hb_util:human_id(Path)]; + _ -> [] + end ++ + hb_maps:keys(hb_maps:get(<<"commitments">>, Msg, #{}, Opts), Opts) ++ hb_maps:fold( fun(_Key, Value, AccIn) -> collect_ids(Value, AccIn, Opts) end, Acc, diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index a561c0255..0de5401f1 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -117,15 +117,10 @@ request(Base, Req, Opts) -> {error, not_found} -> ?event(debug_manifest, {not_found_on_load, {req, Req}}), { - ok, - Req#{ - <<"body">> => - [ - #{ - <<"status">> => 404, - <<"body">> => <<"Not Found">> - } - ] + error, + #{ + <<"status">> => 404, + <<"body">> => <<"Not Found">> } }; Error -> From 3110223c060275a3ba577538a95eb705f1399c1a Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Wed, 4 Mar 2026 19:36:17 -0500 Subject: [PATCH 053/135] Revert "Merge pull request #729 from permaweb/speeddragon/impr/manifest-hook" This reverts commit de6262bfe3dded79b97abd48324a2cd90cd16ad6, reversing changes made to 37c508f6049c897f8bdd37dad3b74af7eb5db078. --- src/dev_manifest.erl | 16 +++--------- src/hb_format.erl | 2 +- src/hb_http.erl | 24 ++++++++++++++++-- src/hb_structured_fields.erl | 4 +-- ...7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin | Bin 6988 -> 0 bytes ...IS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin | Bin 5528 -> 0 bytes ...-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin | Bin 66912 -> 0 bytes 7 files changed, 28 insertions(+), 18 deletions(-) delete mode 100644 test/arbundles.js/ans-104-manifest-42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin delete mode 100644 test/arbundles.js/ans-104-manifest-index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin delete mode 100644 test/arbundles.js/ans-104-manifest-item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index 0de5401f1..24cb4228c 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -1,7 +1,7 @@ %%% @doc An Arweave path manifest resolution device. Follows the v1 schema: %%% https://specs.ar.io/?tx=lXLd0OPwo-dJLB_Amz5jgIeDhiOkjXuM3-r0H_aiNj0 -module(dev_manifest). --export([index/3, info/0, request/3]). +-export([index/3, info/0]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -62,7 +62,7 @@ route(ID, _, _, Opts) when ?IS_ID(ID) -> ?event({manifest_reading_id, ID}), hb_cache:read(ID, Opts); route(Key, M1, M2, Opts) -> - ?event(debug_manifest, {manifest_lookup, {key, Key}, {m1, M1}, {m2, {explicit, M2}}}), + ?event(debug_manifest, {manifest_lookup, {key, Key}, {m1, M1}, {m2, M2}}), {ok, Manifest} = manifest(M1, M2, Opts), {ok, Res} = maps:find(<<"paths">>, Manifest), case maps:get(Key, Res, no_path_match) of @@ -70,7 +70,6 @@ route(Key, M1, M2, Opts) -> % Support materialized view in some JavaScript frameworks. case hb_opts:get(manifest_404, fallback, Opts) of error -> - ?event({manifest_404_error, {key, Key}}), {error, not_found}; fallback -> ?event({manifest_fallback, {key, Key}}), @@ -202,14 +201,7 @@ linkify(Manifest, _Opts) -> %%% Tests resolve_test() -> - Opts = #{ - store => hb_opts:get(store, no_viable_store, #{}), - on => #{ - <<"request">> => #{ - <<"device">> => <<"manifest@1.0">> - } - } - }, + Opts = #{ store => hb_opts:get(store, no_viable_store, #{}) }, IndexPage = #{ <<"content-type">> => <<"text/html">>, <<"body">> => <<"Page 1">> @@ -412,4 +404,4 @@ load_and_store(LmdbStore, File) -> <<"ans104@1.0">>, Opts ), - _ = hb_cache:write(Message, #{store => LmdbStore}). \ No newline at end of file + _ = hb_cache:write(Message, #{store => LmdbStore}). diff --git a/src/hb_format.erl b/src/hb_format.erl index ee07ee0ed..1603266dd 100644 --- a/src/hb_format.erl +++ b/src/hb_format.erl @@ -958,7 +958,7 @@ format_key(true, Committed, Key, ToPrint, Opts) -> case lists:member(NormKey = hb_ao:normalize_key(Key, Opts), Committed) of true when ToPrint == undefined -> <<"* ", NormKey/binary>>; true -> <<"* ", ToPrint/binary>>; - false -> format_key(false, Committed, Key, ToPrint, Opts) + false -> format_key(false, Committed, Key, undefined, Opts) end. %% @doc Return a formatted list of short IDs, given a raw list of IDs. diff --git a/src/hb_http.erl b/src/hb_http.erl index 5a31f2f1a..ccbff2049 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -463,7 +463,7 @@ prepare_request(Format, Method, Peer, Path, RawMessage, Opts) -> %% @doc Reply to the client's HTTP request with a message. reply(Req, TABMReq, Message, Opts) -> Status = - case hb_maps:get(<<"status">>, Message, not_found, Opts) of + case hb_ao:get(<<"status">>, Message, Opts) of not_found -> 200; S-> S end, @@ -719,6 +719,20 @@ encode_reply(Status, TABMReq, Message, Opts) -> ) ) }; + {_, <<"manifest@1.0">>, _} -> + MessageID = hb_message:id(Message, signed, Opts), + { + 307, + #{ + <<"location">> => + << + "/", + MessageID/binary, + "~manifest@1.0/index" + >> + }, + <<"Manifesting your data...">> + }; _ -> % Other codecs are already in binary format, so we can just convert % the message to the codec. We also include all of the top-level @@ -774,6 +788,12 @@ accept_to_codec(OriginalReq, Reply = #{ <<"content-type">> := Link }, Opts) when Reply#{ <<"content-type">> => hb_cache:ensure_loaded(Link, Opts) }, Opts ); +accept_to_codec( + _, + #{ <<"content-type">> := <<"application/x.arweave-manifest", _/binary>> }, + _Opts + ) -> + <<"manifest@1.0">>; accept_to_codec(_OriginalReq, #{ <<"content-type">> := CT }, _Opts) -> <<"httpsig@1.0">>; accept_to_codec(OriginalReq, _, Opts) -> @@ -1427,4 +1447,4 @@ request_error_handling_test() -> Opts ), % The result should be an error tuple, not crash with badmatch - ?assertMatch({error, _}, Result). \ No newline at end of file + ?assertMatch({error, _}, Result). diff --git a/src/hb_structured_fields.erl b/src/hb_structured_fields.erl index c4faef841..658202e15 100644 --- a/src/hb_structured_fields.erl +++ b/src/hb_structured_fields.erl @@ -311,9 +311,7 @@ parse_bare_item(<<"?0", R/bits>>) -> {false, R}; parse_bare_item(<<"?1", R/bits>>) -> % Parse a boolean true. - {true, R}; -parse_bare_item(<<>>) -> - {{binary, <<>>}, <<>>}. + {true, R}. %% @doc Parse an integer or decimal binary. parse_number(<>, L, Acc) when ?IS_DIGIT(C) -> diff --git a/test/arbundles.js/ans-104-manifest-42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin b/test/arbundles.js/ans-104-manifest-42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin deleted file mode 100644 index 898f953aefea08525dd0836a0bb35d50639a36af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6988 zcmb7|dEDc4oyS2r76F0b7C~PFx*lXm(=<&Hfn06Vv`v~OZCa3$q)D3lY@1eQQDK=u z6hvKMltXj@U6tWd(Pdb<1XS<^FAfnFP>$I}84zSpP-iu#d>vr9J%7phoz?tHc%@;Rp-u#F4 z_8-r>>%rSM-Tv^O9@z5Cw|9H!WBbJC-!iYIF#M@)H=ySkH?8-7GJCFfz}@Fh_tgIL zrym}!DSeN7cI(a0ezdsv^bvPe{o+T;>BohQ@mD@{!oB7>7e99MCC|S`oc>;Sf91YE zp1$GSzut1nmA`oEkRP79_L?)+pLO8As(a-tw_QNoy!OpoFFO7Q&kCF7`<(G#>;HVo zD;xKE{=}nx7aVoV-S(+oUNMgy7925?tSy!Pk!~EZawuEuU`4w`m@h|c6Qb2pWX7p z2M>PsiftF(`OEtsyYPgsUwwdo(B-F3H_x9tw@39_P*ku8z1I>er)>@<7a1WUH|e1<;dOE zFc19ZHkH3E6wlAG|?aAHd(ARkE;t{ha*u5v$zj$>2)Kk!x9^L=p-~Z+rqjSfJ7ky-}&702J zaL#v_t>=6+cY*z-lc?y-uhedY4>uk>W{>HG8y|jX^~d%*;<3-)eD?>p-uI6FEw?{- zj=t@?uju(3l*4|y0sivt#Po%i2KJ^19I z7p^~{E8gUe0s~%m)>~mzkOoe72iAh_vha6na57~#OUWYymQOZ z`~4cd`L<(z>kfi%es1@VjOW`vL*B_2w%A|)a%dm*OMcUJ|FQoOx4g;k`rG?oz5TPN zANR${As5|-&(^fhJK#lp->YxD#r@{ho38!%yaW(t zfmW7-z%f-hF@5j2bd4NMHF=@|uI!n*7AGIIV&B^z3j&}iyV_Bl7P<0NQvgy69DlaU zCs)7q;_Bm9uYC8oKu$*S>SI?=v?yLs7f--7&>C>@iRlfsboKF{T)q93#pgTzUNcHu zXZ6}+SL?FlD6(pAf599s9t+_J@#!{1)J2)><#IidYC4li)g?_g7v_+lF_SQ@&BW!# z_V4ffzl$%+ajYfraafd?s-xkKui5~aOf9tl3u~knK4 zq(-x%Q0DWZNsh{sRP{#$yq&*-CFDdi{K(W|fPngPJVv1`v;^f?zk*RYyhKK$sMd68 zR!PNKL+J5yq}ae*ors$_Kj3zVUV)OnE1O>3j1$g}64?O=q15YjW>yvr$Ql^9Zmx=9 zBHlwwGd+OEWX}amXVRRyiZB*i;*?Ysz{uz&qlLvHfansD4|qTp7&*ncM7sU79M&Ps z#woUr7He%Mr1Dl^0~AP2`eNUrh&Gi0)@9Ea%ZAq098FCCvW`qU2AE{Qt!^_eHUK`Z zhqCY4rprX3?zF~s*sK@AvK3Tlw^62AD(q*#4Q*)3byJDtXa?XNx?0GyW)>UdYpgrd z9kthEa!{!p`EbFlrBeiL;kZ-hSUfh1le`{G;|!KtXr+c`;zt1K7zkjqcotf7j3Ekx zPy?l58B0i_&<;A~@HUgdwH6 zIz@v~ldwu&vymqa3}+#&YbEng6Qjj*xR+JB!}e`083zDalf;5eaak&t#(aV32-Tvg z@@!|~HL#S1I2B2%kS5k;OxC*&hZH-YxI~SUmGcuJ9|3ibUxE%sQptuJV9k&ka;p+e zinYcNLeMsbrHUxh-x7*N`4c{F`h_l$1r{u}t&z2Ht&EbLNNms4xDt4zHv%O; zYS-94rzHhElnVx$(I|{GcYBn!NgNiCQWlt0S_NwEx-r1YA=hV6hOfXf-V^gKkS~UK zSm9)FE)EnSYahWk3`YY9GSn4$T+0G2rCcZ4kTsxCK34#AqNH^Py{0w|aDS=|nzeDR zPqg%CBrhrS15=Fw+;gdciYZy(pxPV&qgu_8*|s1Rw9pqjG}L3d6F~1($UZVp!y$=+ zblA!qS&X%4hgtAe4T%+S=@#caI3o;nW@}beL2Q*n>vh_7NnIUuQzOAK&rpf5kX8y= zvvf>tTGiy?_8iiTjd2RFS)B#H;#Qg#Fdfr*HHXz(NhD#6nU(~=L;`~<0$tq`gGh(7 zNmTX$)%Uy|`J=^vYL4q=p%SYJw&e7fIoxTX^l+M{({eQ}%S8wdf(c&6FnbVGSqjhI z=Aw%IEiakOsJ*gPj-BfeYaXcqRG3jYa%_{l~qXE63!tf9krzv6QDNONF$Ie1kGXU^vOv*r9 z$>Q8e$F8b*(c|(6j@74&3+KCq9GY0(q|Q{V5TjZ$U{6RWYxSYFNSEVsr9EtAVYJ_G z27rasX~G*}s7Q{QNe}Y!C@>?KSRvJlDGmm}Op3DUedrC>Xt4`)M7uh*OG`w(5K`q{ zp%0r4v~8o%*lmxhlSP^w>y;qR&p^b&aX(UKemtuYh^2 z@-$q~n*)I?<_9q@xw%5Cfo3mImnY`JxdDRa%8t&kYvPIVx1K``uv}iKNf&FI_FC;_>Fp@Lj@DN+Eipp0Cjjrol^4?|Ez5T(9n z3FweP0ur%dw9l}1ZX7IS#+^I_&nZNMnP%^=TdpWxRR*{;t>;Q%onYp+hY6(-g3YQS zK9{%(+g9lE2+rQ$P992+B#R*1Szd>u1YIu_I&E{9795lAj_Ou~mdg47Ldt!6-keQl z7-@B>Y<%qG;Xnt~(N;CfvrDktGt9o!=P@4~_gsI1>ww@a>N60Bbh$7?cn2Q#y+XFV zTH%5H84D0dmM5@e&F4jgGdW!r%}&*<22;AzuNw0_h))D2F?0v2lq~%3=wmdNCREpu!ibFr=zy)99*9BDyXqZG223{=+hc;fKi34J~?bdVS~qQal0$!Sz9T%R=;n< zip;@)EL2)ZR5d7Jmi(RjON=~)sKYF;&mndvpJGzqLC1~UWK=hkLe$Mo8Hfra9JM@t z=JajHQoy&x2hj~naJ|;d@+_;{6oXMca^Z2aq&bjJ7b%?Z(5l?$ir5lIpU|J5)B(~{}+5j6~-h8`#$p%uho5_;@K_u42i&)dIjCq@3 zn0}Hoa(#B;AEZA@c#Rt>9a5@OjQe(fCL5iIqeg!A{)T+MJDt@km4R1FrOMd#VVZ6* z!JuA5FwX^0TA&NCYJ2l+WnVe((2)SUxYWq?4M7=9LTOYA+)8a$DVY=2} zXB1Ga&T7-CILVN>BqJDnq`ssF+0FEi|BdQBsB4tZt@_LUq0zZ%O12$LjX20f6@n zrC(r{+M;p`uW}LSkllm_kz8eJV`-;M3@OYTO+9K%HE{_ErV&jqyG<5r`$NPi$xCV> zXVk@`POBt?X*qIIP>c$sNw=-3h!sTDk-U*kMI{o+TH7%_d(mVEn3TxsQr9lj26>07 zDl>4>Nw|_HVX~%!UMV-pQ$cE5_LQG0-kj~$-_9chnJG2krD&ln1`uGY#{#u#YcT{g zc&r6lgsi7po6no$W=)@1aI46$+1p&~P!=PqyL}~tJKDU2B};v4l}^#RZzg8wG8V0u zCY?%LpEKe_$3mH4#(5%%!g3K)!>mhYpyUAQD&@)2C^^x&pgjxcbB*iiLYo_y!K5=B z&K;k{5OwaUy*yH}6@n>c+xEA%JpdjS3VpPRE{wB%!?Uump}Ag8E(42&!=xP-c@x} zZM53Lq|Isy>k?u#De-kQTl{wNNTppfC-_oxQ?q-4-5d?_4Xc_W!v++%k}TDgYKdh^ zV@EZuSz-oUu?l6)6TG$ba!e%#N+D5oRCI|O6yS*kq20IzN;Zyr6;xMDuHR{hjai5aCbK#hORB}3kJlxA8CsLC*~GgI=(Y82e~e(P|aK;0PCZ%GHF2YG^}U4 z#6&h0Vc4P)DWv@su)U@bp{7tJP>)j zpt;#&dlsgM$)HVisJWRUQ5`la1+6*g3O>s_aeHZK9>*FM(JU@IvZpRazUt6UQ(Mb| z9gzo|L|6Hcgr|{*Hpj5rCfXLrRRk3oP{BOkHkDCn)XU!FI5C~=dy*0x3&~Qsrt*s@ z&WlN{AF)O*?&lFPO_ec}^h(9BrIjVzQ8i0UT1)NhMDvD=fzQc}?doEglogmVrD9MW zGeJtlfSH5BGBr|iV1=m$-Mo@ZOAR;IhKDn{vBR}@6t0~c;%PT$w7RWIt8c4_0OJNyOLpwta868J*<+ zzx({ZpSwap%j#W=t{>mmvt~v48(C|KSr@bdN3CELH$^mc?D7`6GQrJYm4X=;w;M&~{GS=$a_#mgTK-O$6_ zUEew*B6ZIzCw82mLz|mAs+V2dv~D>ur(vq?@P>C!Pn9Mw>pHV;=8^|wxB4!YHFda- zxI?#v*o>Gr%twxt2vIVw#)<(uDx5P>`^v0?%n_6 z=cgWd=(BNEHMwMV{r>BFWg zU6Kc;G)lNFP~u zE@M>goG;&=H*s8J`{u5l3spyQ8cN#E-bfqVuyjX8X~FsQ`Yy-rYcIWTcdzJ)m{fLm z?_1mc;#u)jPha$cyVBDlJ?9GVt!}#(RrYv!a^LCjlV4x#vCe%z?S&bV%k9e=>bK?f z*T<@#tK5*D%`AO!)}Hvu*6wMcu5~+4ek@H5i~MBmJBOqDKN?YXprgCqGPwO4OWf{? zNk<#v^^u2aUY3NJZ;qe#)91Ct9UI4l99%y*`)Y<^F4CFZ{aM7imAcY7d+C*lo$Tp+ z!^*s@nq>9(^S$Rc?>X2ye?{Y8qrP7C{?nh7&$k`heRpqE*oEYG4o$ma|D$8;ntNtA zYX+ar>MTs?t$g)QRh3iD6u0mCTX^Jwo5gosIsR?OGYeGft0%s2$ly_ytR8#KFs6CW z0sGeG_8t1OMo;0E$KQD2laRfoQJZ(z+D43iIIn2+>Z|k<5noO0U%R;a{(Qx)frgVQ z%@JcG&&GY=tY{`CKK9;ZMOS^*Kv(&XN_qP!cTJyb`S!Omu75b;&|M zBxFR0_|bU3e>W+UrZ_-xsL|s9<2c~vR5s2|hCl3Zpar-cOeq449e05WGz&OL+7qIi znpK!-e5@!N;qlT&>-aB(#3`#@3MgsXXo=1Sa5Mat=u^##P7xNsLZfdZ z_W@8tRYF2pazV0kAag!<2VPaRjDk{i>%U0(^eAT4uVyhX!|dq zXE~fBOd_#~p;?w@2rEGaC;~$)5w-t0(JB+mYSJyZogh8>M`;Gv5*$t{Tvi*GfhpoK zOsP!5uvGQX&uA}hp#xB{B{;|g0v!7#W<-`i7V(=9cy9;2ZwO5x#*>K z+Uz6&tj>UJVi{9F?Fi+H8n7}?A)1*8GhdpfQ;9r;uHjFAkrylilE%%!EN?3)oJPHc zf@aOm!%XU;#{|Z_dcTC9!`Bc=qJl+JsxluTP06lb#j+YuW|~1swX^b(q*?jDiWkd4 zj{`K*SPCrpvnqcTFDFNkESg!8Qpg%P+RI9(^7eOK1#R{SVKEcc2w~Ps8Jgz!h{1Kx zJ`N8PK2Ct)&j&s&s;!9IPExF1%C`>Kv|KKi(iNwq8LJ9b4yF>CCf}hAv|BI55Dd{I zB=Y}74icRMRtL{ZB7uVQljD_|B!xOjtxSp|(S!sgmV#<9WgMQQOwlNW7pA}zY6XU> zv3W;IUI}6`I&kHxuRup?^jB8yd1m zC>RIL!s(Qzw1|ap19Pgd${>+lN#Y!kqYztx@b4Ir zs0I~|L@km6Kf+e+Xr6bV27EJJBnTABnwQ~li{v&yz%eUu2acU~IM?x-(r~^))CriK zMNG~L!h}|U8UiqKOo8zKlo|!1jzJ(eKyF4Q@K_i)wzDF+C>NXVRX{>Ja1+6KvvZGuS6NiTL`CgXcps< zX%Sgej_{Ew1kql&4Pxi4mLA_6Oha#J{`A=>RJTP|$OM;0$04hmLizQET^o1Aud?ko8*ODWWba=4V+cEofL zRU+6dXugmRK6M})oZ;9~g0sm5ncg6K3E-Dc`#@sou&8VU|Hu%&z~Y6EEWmnkxXXp~ zP|}d%RZhToN&pG(s5B#&-PW^s(Y)*V^zzgD63oBjHC+fSj&8qvC? zx>DU%ZC&@4mabOpV6z1iuW>>`z)1|Tv5k!(4vxVX8*JiW-U~72VG|y~HpzPqU_*Z1 zu+95_-&yXtOO;AJ3{HOU&7!Kh_w48UzVq$pS@*Pezi;ovcmCg>y>$9bJFoudLx1~q zZ@>JXp7oQ~OZ{Jd?e~0t=iM*(h2u|txVU!TORkK3=|6w;-v9LJ=REffx2+%h+(&-o zzW?}|-~7*~K3n?C8{YD)|fjZi-&*Z(=T}R z{Ig$t^>2RL)l0wd7jJ#@kG!?~J8$`?r<{NDQUA@}FO=W=OV7RkRUdrwcYoJEy7~>D z{HDKn{|Eo%?C*W__ildVi^I!nAdmH~??L~i(efimJ=1<*A-~Cey zZ~xA}@!GdOG&S?C^56ZXzx$T|u>AhoZ#?)%|LsNpq5gl|dh$1~{fqDUz3>0t4}RCT zf9%!o{^U2;fA#)vDg5t0`m7JHyyfbvKJ;Gi6Q^GJ%|H2u>VJFDM}O^*6of zx%Zto{;6-g?@M3)!QoSLU-{B=e&|ap=hpvfeCpxZ{3lP9jxBtmy#B8Bdp`4e_h4bUiyb0``$0y z_Z`3XE% zc-4=*y!x8e?|aXW{@kPg`0suCWbQ`kGhWSo(Oae$etYIQU;V20r=Qq)#i#!2>{p)k z#@b)LXX@C`d*!dr-}9mmzwWco{nNEKRX+U6p>Ll5AHV;LZ+_vo{n=X=|H+45bN>TB z_U`(_?LT?+U;oG_Ui#~~p)Y*qbI<*)mDl{hcQ>wF`g?EsJOA~Mzx7`%{KT?=S@;B|g^Yafp`PAw|U;M*sV&##UA@ozl)$@l*9$6mep)t~y(=Ei;@e`sIK8x9<7HANcBH|NZ6D*D@cR``D-N|NhsWy?*uG&DWm!p#O=*>DOMIz5USG zw|?depLx@}e(~*VuX)SAdEk#;_badbzw>Ni?H_C5dG zgPq%7edX*2FQ5Lx_P_k0Tif6Me=fc6XWsuUFMjV=mY($oAN}$#|AX}tKd^J(4}V~K zzp(uJ=YM|sZ@ukZ@2&sX+urw%((kP7Uw-pbZ~fs<-t+G354`Ce)ptJo-g6hPy#Dq7 zz4?m!|DXH*_zib<{?4NPmw*1ok3amj`=9^f-#PiKo$}kf`=0ov7ug?r^{qeu(CVKR ztoQxa-t8~$e)muP+{^#)!!P*2o!UDezuNhk&wcUB)i?h8cl_b!Klr@Y{ODKy$@Tj? zAA8%+|MciD9^CrEbNGrR^@1A>}eUJEO zN`C&07tS;rf!hf3EBh_ic3Q2Px8Vd{vvK04v*Y+1ZLbx4%ehu7f61x4-+sky*PUIr zoS$`DwdVdk8_k9vWNzHp@dEe8jk?pqbH8MHcD)pPXje*By|8)b4*e~yW(y~F+(xC@ z&ez5|+vi{Q=wjifpSAJ0rMjqjWk3J$t(o(+jfZyCTc?>vpAB~-$j?4@adczs$`kZz zEtkucCcOeiV0op<%Br`Po4D7jx0>ys>tq~1^Q3*9J~r%|^by!C`tSn#7JV+c_7Z*W zxOPCF=iP(NcC(&7$O+l(6)IT|RLM%CQgHox{@T^;^Sj#*KSsk+MY)rTy-YPO*c_nC-*PsXU>@Gut;s()CCvaisvkN<+jt_ z&re^waQ^hk%br=D7i>nCUv_J}@zb44&C}OsxTJuF;Tu-RH)J+ZMfj` zT3%{a+!;Xt{BX!Qd$_RCtks;B?^?&ksF@Ni+?_kyg|=I7?zrb_HSPjKFkd>Ais*Sb zcfga>(*Qb(p|(oHdSA9n=tnH}iqpmr!xfxx3(dw%m8Y?d1>53k!-ZDc1>Fc{-A$)c z3-khTOEs@mZUO*ioxq_9)2EdU+`ZsLtLAu(Y;Hm?le_0`%rxtD?7Wrr8ot{OR!}f& z9~-l&zozVO+ubjfO4&Kk$aZ#QWHnoE)+(S=Uf_UG;!6j>-mZBK7vKG$?QOe3wcYG& zRkLdZkaNyP)!N34bc zCIIyBS#x%`?QRwuHWrtt_*?}8E7#oOu~EC(xX@^If?^Bbt~h?+7Ps*2qPsua+-($_ zw%=)CHGN<1zp8rOX*aya7Fs~hIeRWBRSFevXZ_xxisxe;_lud$n!7h~?+|_DD_$Gu z0jMfwhP!TQ(k%@0SoQ*1x2?8U$xdb4&E0G<3y|v68`%R@b+_%biW&NgCtLJ=Y_x@j z*X`x7)4JEl?|PM>+OPfbGpAd7=wVbhj<(Qf-Pu!>$3{nAf+sD2D9x_VV8_rFk0kt~ zn=Uuo*zSDWsdycq8o7YA!M6BV0KXs}14g|%C>rLuZcm3Zk+YBkL5VF7i#CdXPZp@l zQFC~saxI2hneM$qLs+_ITMs&@0u%^Cn{6LJ2CR2#!5FRU5dF&g-fcHlIM{DBx3JRH z{Q|u$?CmYrom%Z6b4;9O4j@Za-*vsMYS6m@@xDNqkXnD!se3hmiG&9+T4Mm>yN%qOvuG?D0HCqQ=y*t^xp zgKMc{8#jPCZkwtNw7*#FQ@q$bN>2bk786@GTKB0L4P|;8hy0c!MtbO8hTt=F^!=pX z1rQ9li*=mAojOlP)H~wtafg~Rlxxk6ZGN;{#p32M>kWWnqbad`Xo$BLCrEp<*4!;- z+#0C1@A0ac==#uU!q%aouw>P%RNMwVJi!0{&KB?cklJ{{TG6o}7W%hBG>z53I4(Fa zP7Q5(wOS0EQ<0AbHd3t8n8o1A&c^l@PJ>1zy}AHk#@<3$R%}KHF#RCLY=OrlD$_{9 zHM)xi`pmgsmx5XWkDkWF?`E_DZ$b1yNBw5at3-*9+$~XxmitV&Ok&fRpOTx#G#->$ z0AMGe(_Q_1IurZjfeG)YQ)mpjUM2%zhnv1 zk5%UFn90FV6g?)!XEZ|s0v?K4_dEK-Gj*J~r8D%GW|l<#*whb0*~rAl;}aj0BRI(v zX+RX6y#np4G zHw#ysw!=L3noU0+>ofybe7#IkgIihf{6IXo+TcURd^7`5T)bQ%vb6^B(W`+iwWgcR zn(H($7TtB#bHR!E8xu{4ALHhalje_8=8x0nk28HcCT`J*bbFx=2oMci^D1t$ zZ}p1ngYAygpY?W7t-VZ#wrEB^nRWcCD<3Tr8`-`FAsx!enkXiooo}|i+eEo)eLIG- zBxu$n9h06$w6xf4xPf<@wVu-~QlzxCX0OUE-zuo(-ztdp*eZyL0e(9ITxK8+5Zz&H zs~40O!QXINF4PDlyg}p;GP2iT^#MJFz5|M=nOf6#V?QlyKa@tYnjI-hx)suI8Ja3f z>k}aF&XLB1V9y10x3U-vG(wH6-68b?>%2nkgi!DFtpS8y?O>|$J|(TGmugTFRXQ84 zWtHtcpiKGrUP0hVq2@NWf~q~5n^*vPVM?{MhV(p1g%AMz%MK}D8U;XK&}1syb!xo4W=Ahrjo3!;C zT?;!W_b_eZJ5D=OmWr}^X%7Uh?LuGV2j?14DUhH!--a5V36WfGue68du0fAf5jwPq zrXsEdzlpM&rO8d9zMI-FukqI)7Y5-Kx*SIO)xr<9asr5(-L`n}f0 z(W35+jO-PXh*7R}+F2W5DUqU2yz?RPb9v4_MF``!9eb}pF%mH3f9EqTpL<6 zQr%_m+=)KC+A<&E^vLGA^@OI>t`|T9TUXl6mb1mMq_wBzQnbN%FVy77$T2if^)>@& z5U0k9`LT&%ix!iPU;rUhs~PWjscd^^0R=F^Y}3i@O_sL#7cd?g0Vvsz=~ZPLjpR2x zz>!JLtoI9&~kmM$(*q|u9OrWo_dbL+7lgjv902MAo z64%OBpgqnr*~$}_1m!Wn28#Jgg^7%{2NgHAc@A5to~00CUY>gC*qGeqGGjG*-YaYb z?HbmSU{RhAC$c#d9kurobDPkMlT*wa*D3_fh32l?o&g7E(L9Nh8_O1xEf~YJZWbi7 zkTcn$@je!LuE6d0}%_%?ZEDCfU;E}X&1|EfCg0*)|qT8YrNb$ z05?PjW~BZc>W5Yt-&WR>@YJ()^EGbX0v`*mio5WwWz&2Z2TKgg)$Lt~H9W zYxZ!{o<*6>*^mDdii&isZ!Nki*Zm!$m& zWMyHsqs}!XtUDw>)m+%@Smr|0LIZHHj!wl+t&kzT=G@+A9&=OJy@iR$N_U6I6`;>+ zSZw;+&)G4z4{LSQ9<#^6?21MpF~K<7Y1eWFsx7dpA~N_V3(FlJ9KMg0-IT%tyU?78qN@X3KDYM6ne>PoYq72~6cYmqHzuwjc-#VbpRIS8&B`o>2I} zrU~5(>$qMflv3;Vxv=?o3+XDdn-EaF&3$riAvKb=m5{9j91Kg%-?I zstNrVTN7FJd1Rf`fEFN74a4SmL{-(mhRWI&Db`5sI3KPl8HM$>y-Y2^o_;1RkC2Jy zy*;;bo{fW3Su(wn9ongFSopqE&Ezv<4@38sW4m$xosaAr?}i@$p)fdT&ZH4gK_OXw zXT#y@p{79?H6^UoY3+n+PdTSX^SYqgQ?Payvtthl+c7Cqc@inwPNRyU8qEToBAR#t zJ*@y9i|Rq|9h%I5YA`&RvxzA!Q#v28i)cfn3ZKk8$lb9bba0RykC~(iaxR2hA@_LG z3lyqh0|!G<#_)0WP%yUKDZ}0bfdS2wn=nViVlB(Gg%$lIn>#9?X^C>##wt62$pboc1<6QG^C*mVOlFj|JMNti3vHZ^xC) zs|)r@2Dvo2d8#K4U|FQ>EV=HUJb;Fg0Z{H$)6T7M?^_}pK zLHDX!qf;+K)bC#n=gqiL2Qu?6l<0?7A|0LTh&;4_B)3R45A14oplOFARTEs=5r-^W z!YH0reb%H6(gK~f#zZ#Yy9PetkYQ7$QKMGi9JV*HZcw364eGT4`Z2nxxHYd1_mB2~ zbx#|09i@ROG$6eU*uX=n4T!ZJVEEFOYBVBu_pa&^hdcPXnINrdN+xiS$?Ku79caX? zhltsgG|o89>D&)F0*#wHp1W(5VkI#nZBMqq*7URCUSb3$C96dHY}f#uChX#(D$I1+ zKD79Q6`pRxjN%68L8RfbRSk@4&~xeGOVR@HJ*~RwQxmGLl$s9>P!n=;O3l*;QmiPK zx;)C**+PepdcTg?2F#JL!>`cEWt9uup4}}&vscXSz(?L|RNb~0 zu+-D3xqflkg4`uYY_X2-kUx90MdJvY%G1w*+LLF_Y|4pRF7V{!I45++bUN`2W`R~- zOt!}h0 z4{as7OwtMw*4kkAXOP~3lv-)^NI{*(U8ph70t9E3_IXZOCu<`k9_)=u1Jh@$IQ@hv zaj+p8#c5B^*U7WL}Vyr0o?WGrMw#bI;7qMGT z<#Gc~+J;Ka1uYa$ZSu+@d-KMEvk&=#jv9m&5G)o?5u=QEmt{-=Ua1Ew(4A6IXlG!p zZWX(3gB|e5P;3*gPMV8iRxsO*U|zV1?~~3|`KAjwpSXo_G}{Qc%SZ|7y)N9>xy*Ip z`kg&9I-0e`)=j(n%|>MbK@SV$yNQ?%J|1V?jV2;~;5AQ=L{YdF6uWTPWzUjjIHAB7 zh35H?4gIWiEr~qVWf@3Nor{l+o8{xL9J1Yta-#&W%T)(-lSTm>qDCa^f{A<({~Ap? zmX`VK(+CQcCn>3mSz^^OVW^!i6++@~3m=jz6xmVqpvqR!3BXFBI4X{xwn}tJMn#uJJRlQnI0#i3USUGx1>b;Ho)sNgng^8E_^&?@N>1rgsFR7S zU;67M z?J9Un?Im6~MHd?Z?4H_Rx%Tk;p&%ULLd@N3rDb~m0Cb$FJ+;sgMQ}{oB*dEFgF183 z&A=}XE03Vl0>b24aL#K4!j9PPY=C$}!s+@Q*eyxy396nC?+~~@mAX~Op97(7Gg+l) z-bRMP03j{yxD|FMokvtJm$8)U%BVF;Qk?9NI53BNWYHNhq;SD3xYW3nCC9-G=qt@3 zBRb3?+7xD83I)^+XTqVuuTP9j@L<+HXn4q$q%t9KP9jT)ZJH9Z7aGuSz@LkT!^F49 z9k$@#;}?{H@*va-T#e{gNwM01I42dW`%3IuvsYX8R$QlPuQtKT3L%cyk)uL~V=CCB zY}6VV(Gi|-AZmh-4@JMAz29X)EThXLr7T zOn4j?aYDk#pF=yCndZi~OW`=clQ(nWV6?cS4nyz>kUjN4I)e=a0%>#@ozTNc&)4Wy zKkKsw19r_^SEBO+d*QET(pv;82T9lH;J;womO&T`|8&ecGzmryjWeVTAE~UI-+dM}Q zNUSF=#iLtTj9x{Ad~zO585R*pAyTNdBNmThhx|E|EM#Fnm9SLI3e~ISS1 zBjB!G*MSFO?q4U!Z(%9n{7uH2LIJCofK|P2kqwINPh=^hPSL{E=R9m3k;NtW2L*!> zMTj@jvj>Sax=pyJAh=%WPl}Cra|EC?hUIF{O&1z^>PM0E(&`^k=!-UCIzq&E zIDPX>@k%*HkIRQXe_so{D=9Uo~a`bX3ot44zUwQ!?U-(owN=nzqO z2Gt%SPljIE0k^|QtH9fSHyg5UKORmz4xxt-H{Pmyhdjyv!>&U{DwI>Bg7ndrKfn=e zd4w9e`LM9=jEJlRj+b-EjAb-L=}+Ln+T7gq3(yZE667Gqokj4HM+$MAedQ8KW3+Bg zW`Gee)(`3X&s@4-4kD zleX+lQ`Qy@%|{2z1i!$9V*m#=CWpj;g}o1=b$r@0e)^c0n6l=zKr#>uNeWAqYalO}M>#!R zO|}OwTFCSIsjO~iS?dbLg>FUqbHbYq3&LR<#qW(|+tCeC<}ad3BgLY#Ny2s)vr<(5lt5WEE|rx6+>3O)f9Xq6 zJzW|ZO3rzRCw)p}E8@|UHME%FuN0O0gq4Q{3CSoAPl`iBXv1bH+$EVzDq*MpS)?n9 z;zmuKscf7NJ7T6D3yDE*Xw(b<9kw)fgP!1*gf;`#)bd6ht5tRmJq#tu(DulrA4BBK zFQMe5uw5wbpT(kSr|m|K6b}aKMHI`jU7CG=@DK3aR)zaS-)*tZFf0CSfT?3Yi|o z+sAk$%}b0a!wZw>8Z~skqR&K4{XSe;ITjT%@)AUhoFuzBHVtq@Ya4Avl63M?MEH(a z1)gR)*BV*>RKzkCBT~5cKIvI>(O|q=V--@zf9Qz&)au)@Xjm2DR@UuDLs}H&*GW`I znZY<|FqIVtNG!6EGN{?4AVQT$2X-!sCXa9|03VeD2nj*)4{>Y8TJvHQb}ryoU@iVsdS8s?kP8G%66eLAQhOt z2E$5p*>|76De3Tus|FtRFdj) za<5qK!e|SkNJKp>t7%?2J-ILrJCS^LUC3!VNbqT5jns*^({|dB`m~G}m4?;olEZH3 z26R}+<5+>!^6|JwyAzZU)(xNi;>j~`Cq?6(dhyJuQJ7VcCA|3XnbCtV49(+1i{s^_ zD|Lq4o#lM?OP4w29C(?Bbslg;BCh95maGv^CSNOddRyqT=HX)L4i>{$`+-Uv1(Nm&Kk}@Po#`fC|Fb-rY}O*A`W$-=ZL%}g^PG8wI+#< zvFeEHGdg~RJEo|QhKrzm+r#w%xMRSD;a~6AmWcf<(?_f1)5jKP5NwvVl*5va6Irvn zh@rUDMD)D8xS}|BP}-+T-45=kaNtSr2gP9<85VZ0$_I^2k9?RZYx%ZKynTm6b@dj- z7hrousHAwp9#6%UgVHXw+9`EC%6V6mZn*pdQ8a)eIO6gzW&V$vgq!lE@DrPn`i81b z76PFJ4qoKXTZJ*(+1PMf0lQRZxzQ63!r7e#6V)^S&^-&OM>-zFdloNy^(-6e68i|x zclafbvR`q099)jZae%l1azb9SW!5TvwSv7Ia|**vAUaSNZtUR?%00Ii{kiJd$jrgT zdCjdZ5=9gVo3Tmkx-Ne3kxYzI@jP4v$+uS;)43??*3Kbp3h}5)8Hi9_BpZpeP5rN4 zw;xYFylM}-(Wl5axq8ihH2RS0yRO}dm8Hiw;h~XuoOFRAG^pA;_PpJ;oe39_RDlCG zxw3M`1E~=YDPab}I4rA*Of>SwgjkSMm6K{vGB;`kK})<|Txq9gr5#m{v%qmhy|!F+ zzGu~WgI(3VGPA=K+dV6`3tTGfG2Ab%>GZ7Wh;)m*wK+-vwALm6atO2Pwl!4gg7CU(%m z4mr$TX!xB?&~y)YKJCEqYr~zYB7Yo`H14pEBrcwyyDKy)zjH^lh2(9BzLGAVo^AQA zF`g)4^bv<&vDKg`_}xxM;;SjhIbOj~FEok}06UgI+Sr||; zitKCJjqM3!T|{yua!B=}!{G)IEGgo(WVS|6!$jhR2I6EL0k?@7=9Y&n)Du=U+z_xZ zV1S5xvM6fLm%uDk4PeRH?NU{M*VH=qL$(_E1jeF?j3FztiD8+?St&PJp+uR6NwB(;y9aHKg(Y->Qu|%Xf0)F zq}aZL?Bv-K{C;x1xPBIAZ|wDn$#v;Uw|;L1|D2`tC+v__imF7E5r-Pq%XWZ%oP$ZO znZX!m4)oWm+D{CyHA%H3&L5mT(ZXP|SyA@v3G_%UtRJwivc|*=0k}lQ4kN?_c`sv{ z;9?a*H4ryk#*HK{JlA*J74qb#+*P>zM<(mEtUZAaH5dLSCq*4(=!l_nBcZqThYP&8R>AW_(+fyyF@1v%~4np@WTB18xW zH;K?l30R$jb+DFLolPuIbw*r4BqnQZ2L&#kJ6N>lOOqC5j+(atGR5nzwF7Uvoc$VTE={jBn2-^q0^gT&SG3~b~S92+UF0Hc%6tQkg9 z6sOIF>$IZ4nahNxb5_;P%49FuoV{r6$T9BNM2ne0qA{+3RV8u>b_C%7Vum0}`n$Nk zs)URlG9c?p39M)}Le8*UPBu4nY_xdd6&BOBsp3;7o;s16vR0qkeX5XOJDz*0V68g& z+fNnNy5sg4P~BCO;CByFnG?l{(o<6-FFF3e+Jm{Nr%p@_pYRF+GTLG;_KZt?0s|7M zJZT*1xk`6Hdq8pNvMA2)0Qz(V+cGhX&5`*&&z44M%=5H5ZMiya-oYKL3~o~r-O_P3 zFNKbs682g*ObhAfqB}cJ(jpEgvq@fX(I%42&{WLrkRXFrkt!5B@;L9Nz3G-z4snW} z10htjwy3ySDeEbjf(xuPTRH)FR>)$Jnh-G~<(DYdHmo7krfHo;YE5$rqp%v=$m9K_ z4Q~O)`GYkZ1lW>U%Bx1T9ns1Bf$}DgcsHNa{?e<}wS@Prx!~ba(W*m6qHKIcFQmI5 z|Ee$Ul7@ATMFCuAW$z%c76Sy!jd3sq-JKW=_>iwpF__0jCCm`OY}AN7c^u@;vUW?$ zs!$Aeb_?Vpg&@TJnUYCJ!0-ze?S!6vqSiZv2AP_1$J z4vQaN!)hO2vd8cQ=?!`m$g2Vqi`FfP7q=w4dxJ4!Mnq%iF@c}8bv#UJ&f|I$x>LFl z4xp7Aq-+NeKLZ7~ZZ??#p+}oN3aSP*ncSn}QD|E-+tN!N*A+&!0fO4jDrajF*x3JC zQ0`)&e$No!S!1v5%T|Kei&jJ#+%y6-7qPFt)z|O-Vv66rN+H>arI3uI5belJkmT_# zJ<+e3Z3%bxM66h1+CP z9rZ~(rN0rzBd*@EdLwSPcrn07=01O-1gLRcdLkUG$(!?SO4F1kS zoy>=>hcPw*9~HoYy!Wz5X-EWp={+()M3llTGW+gnEf8xo z;jzqDL>N@VYRBfFS4j*6_v}WpE=bM;lNI?Fc9D=t9T_6^0tF7uvYe2bf&qSKf+rfo zw=fAqf_{+JdXc1;6{KkZZi{3MS9Jab!Pm(Y3={)L_rmhT6mIv~NbnS-F;%H09!+4Q z#kpTVvWYJ7AsAm!KrD17D)KQge4N=cLZfAb--XC2`B+Y8630&Q4(LhriLew|fW&6t zGGb&^p&M|DSzenAFHTlc*)Yxu>ksiqk4}QCi>)3YvdTF74gPs17OLng-y8T(dzl!ex;YJ)CRccJy}2V{j22ga{9W(969E4I?svG8EFXH6L z=sAsvB#|E4hWmkD8JoNU^*0sU)>qbn59G`6RT3_37bXvRnB3@F zTQ6pOxhX-H5^~bCh(c_Dt3N@Ga{v|OIu7^C@U^82xI<}29MHP;>fa-37l;g zH=)+uOn?kfcBTb$Hm=u!h_#==mC2MUgwmWLu|)w$h2`K=R+38{-F#g|Hzy8uf^8=% zD23-PYq})+e&(U472hn#?@g>SYn(em<4pM+DZdTz@<@^iAyP>1u^|xKK=T4TK{bAR zaJMkwFri)dg!feFG6ZM(%-AXRs87kV6G?P~u4Phn2@6J!RdTShpX@yM?Hz(!qMXt1W>8y7&ms-`&k=H|ksm5>`!$e?cAut}>VQYNL?XDE^uZe(n`sfFsC?+UhcNtgLq-JF(cO|FS%Qd{ z;)|~HJD%^AYc5A7$mQ(=Ht-2@{fMt3$D(l(L&ReES|roay!5jc#cKWvNDJqlqt0;oQVqbhI_jaNqx5gHgBnfXLVhKD$9vcCLetZ)h= zCIYiSrY2Gq%K0m;XZDK7zz>Hqer>W-*>klZGiJO2`y-@PelwmufYYS9W4Ok?mt;u8 zMF?_N_l3t^gFW_o-Q8dz?^6xw$D~muW-q}#SK#5)m8`Lsoybdd?SsCbw<}=Fhd%&E zG_oJg)KpjH!t8bZYQ44#&Shxqx8x$u7gu9R_v5e8GNH&7ZOVHV`i@Lf#4K~y za!h1&^46;SD7h@DV2P7ayq^hnU56C9!<_M zp{?mP)n#(AApuiPm0fPY^6Np&fVURv%*(>Xa>0dHSEFM2rAZ&|Q4lJJQGdaO0i4ov zonxb`EQ>OL7V_<~Rc_%0BtaROZifE%;> zOU>)-uP3c-G>%KnOiP693rx>Ik_7qL^tL*<_SuHenK4_)Y&eZZ6SuUxoC%P0~_1wrRo8v^|Q=Qd?l5*1@pBFlyiQE-gJ3mIX= z$^dmKeVSj$%!ve%NK6-0+s)2a^(d<@g=s$&!W0M3+#i$~D!Du5jbMJLr)-nXWyZ}+_t8~P~n^B2kIdzLiB3(5YYwf7l-s{ z_EyG6DI-rNjI>JTXp)zWDalIxRHDDg#l|c;7O)>5OZ+FymnuI?{{&5{9C4XJ$i|1k zVeyb5DH;ruQI>S!NcS2F`7+tAD0bJQHrCqt2iR*oW*sW@+HxKZ2!oCfIrM%&0*8@6Wj-> zZiA#Yj38D1N>M_OhY3AULhiCiuR&GRu5pvJA;=g2ag`H*Al$ep2P;Mwjuv1M8?3+( zJ+iwRtC4&p9BcZ>^uYC0D8Mj{n!F4K3?;?zs1ei2pvN%P4A=q8kW7aMZ=bg*rFmmDrXcgv1dI;0EhsW_ z-CEKk85dXsBjJ3G{o=?$*nzbEq69BAeK%zZfP;OOo}v5t`x7>MBy|jz^^&|P2lKjf z2Pt|bOko-C+-bslcQ7YO+!u3LL2{B@G3L2o)pF#1s%OBzVmo0hm2&_0p|(*|%j+2z z_Hrjc>Ja;4=Kf_%Ng*cEMnq_#X4dU`ID80pxQ`Xt(#T?j zE)v30%o-?dfO)YnMPfxzi?dhoSL81uR)n%8F&D`Pa=a6xyv!(M=DJ{2^JX~2n9)I7 zA;IjxQXmFkD7!}s4+9A>s6IM&C7YP5s%Aunhp5CBS3m6fFar>x? zJ1?kAD!6jmDGWv4-Igi9MZG3$IY$95BNOM*OnM^kmF_fsG3Z(j;+OLnDCSIS!#SOf zhDuT;p2k}WgQ1x|4&e_B-Ws2a8F5V<2_`yMAg@-QJSlx%vyk%@u!%*7+ajk~1pAti z8>En#8FhlWL8yROVU@s)7A!BwGj|+El(1hp5Xy`v51)**j}Fm2YWXs5YTHH@Nczar z69SbK?MMU@Lc(G!DuWUaEL;%1pOVJ($Sf0qUo}jU4O5n1ic&JOqbQsiMfwk)6Fb5` zMJ&1T>}XSh0K)jKP{Y{Y5QTi)2(-OV4NC<9k2Wz~rj81_U?agU2ysn5i{PqOPLW+x zGvs@07{iLg?_^(OGzcwJFsZP;OdRdP6o%t8882QVdgJS;OhkXUjjfE#iE|#t;`N{D={X3si9`lV^iALUP5r!`{IjgUuX&luMIT3a`tE)Ww(e zT;XODfmakli3e>26pKbEh8N5xN|&3JeN+I?Ec+2H7kv0ryVh6^vH|a~(E;^eHv8S< zek*vqN8MDC{l*5uo$BD_vz(Ya7P-vNvMT&qvMD&8YEBEc4AbGo-zG9ry+L}637K@( zOl$PO#>U1(7DEF&F5JAalJXm#Hc40z&uNL&3yyIf?sg7_yUV0wILkIE6)dgtDT;9i zRa+;`l}{=lo)nQy|0LNm9+G~qDLJ{5FjW=xxZ4x5UyR@cw>`NYin18HiXyXWFneTk z2$R5phJ8CRa$?KQjyUz!g!%L=Kdl9khm-tpiynq;&ykdgd#SoDYZ+Pt0gZKp4-zo& z-dkmR53sxkfw(p$z`;Ut6g*nQqs|nrcFYya3hXIFd0F6a9eRPVqhTS=w#b&p@s6G} z(k~-k(JLW7932AAVbaJq;U~dB_LBGmm10LPMIPB zryy({ACoG8Oc}Zc2KPN1xtx%d2ZQy+V9SxgO4(ZsR;}w{KaC^hct;YBH;W{FQMXN)J&& zIcA9E+@$3cjv=H7AV&na4%nhYWocph;BqsT~hw(`_3ZG;YdW9DG5L3Kv@b3s!3o@0U9#V=4+{zxdwQFC-%wwk?UdxL&U>3Q_%~)cV5Cg*`f?0cK9kdyt7Q zT*3WuQY~b!SM$VwtOG!M&Km$Dgoq8c@mp}KeDLejk@TVhcBgPM7dz@~j!wQO;+QJ{-#i?=)-sOr9+(0t z7ZpNO78p`78xi#B8N?7AXmgft*Td9YcaI^r&0+xxQYo}qGRB0FuQqI}NDWZg=>=7okt z;Uyr9cAdsyB?Ri{$FfCE&nI)lvQnrN3A4C_dK(p-^#B1dhpLG3!!Cf-FZ3l6PP=QkQ3ov#fuNtS@Du@^4 zbaP6%y1+#swu)jfT9Bni0DSwoutr?-7mksbi~${>^l(l2{_8Vg!>G2OhE{)_&>>Cm z3D72%7kN;zz|ipRD(iuv0o+cT$uok8eKBamBr5rFzV0S5nfy2-v6-iL=YPiGvujig zJ%kUL+KKAGzG$>R!cBv03t9&()5x28ufvUO>-I1p02$J9Kvty*hm8OYM;l_}fiy5G z`xS7kG)c=Hjz-drBbyr5?VAb`ad$?F`ylcph6Uu%xPKW=pzOucM@qUcjq@o*ep2*+ zebe-MNfQBmm?;~DxP(ZWHzo2VUK7+uNi;n$wf^5PqcEJ6fME@5=;yxQJ8I8JO=THr(*7&1gOaFNn#-EhKRUsGCPX|x$Nu)o=09D$vjYq z4pp~Ksi2rIqV|*@F~+Lqa##(+CiygP=OXV_V2oZYTmuJh=xuoq+|e4>6F!3Lv;Z#w zu@R6J@EwC$S(>3%gKZB?kYy$0#_2^dN=;;823{J5gx(N*ll9~9izW;pW;i1uB+m$y z=YaeS=oR?0nxrKX%9Eu$U(q7sNVut^K*Zp-Z7|dEz}zGs-ir%oHv>ipMF0_4;lNoe zv*!B*AM4gj--?gfBszousvL>+_>nkU5YjA2VUf2@}`D)@okM_YhA)T+N=k9hAKg z@F*;(yt_an*X`Tky32iw-r%N3({KAC0*eZw61YOd_(GQsQSit!d*I!X*}Qu{?Ftc~0mz zVgRnULufD~w@}Hg5Q#Hl^$r^@xTtM;g>S5;CI^QuDP+rPk|LNys~1K}lCUSTM8pOl z5Y32cGOyla$k3p1*F)?N1L+fE*<_?xCCui{i>ovfxm$RVB^bmeDarU!2o*#_)NY@~ zZtsw5t%QAb<#4DjF+vx^xFwRNL_1M zAO4}X=X8f&@iJ%Ck96==gqfzS=#-&*y$t=0xFKWHMM`=keYf{q zJ8|Kd&37+JkEs2UN4I-5D3lAA1}F=K)B_1K1$I6R zZ50;XL~hJ;Edmp*9j{z%L)SQ!I1JE4pwcN4{Fe99&NXgAd#>AUPs35sqMY z?t~5Dz+h92Z#e~|oQEOn669orfl1Q9b0Uhv3jJueGo>yua7E-UtijAKG=Igc_{`c` z$5_njuUTZ7CXIkpRu!|6pCC6F@}im~MYkPu3LZR$gkNYydC*S7uRu7o!z+L`f3>}a z%L*J?fF+vixd5&U2-QM59Ei@8o7xAM-zw>NHKZU!jKEwC3ZzCLGH~Z6ZWWwX3v^(H z+CC6u8=bV7i?S!R;sn zTh>enCY*lEt?&e6UZ*pZMh3aX!zrtga6}-JgBTigW0!jhJU|j+2@;fyTxr-TGkFva zC_4>Z)IR5IR4r?D+FpVq>|{3%($i}g)Y8bvRja${xxlk^0qcd~?ljW1i{#Ktlu<{z z)kv$LBcb&O!{I$?o0v8-(l)5ZWcKtkh#V7gowwb6qyY{WZ3)Xoz%;Ke z@4Qi(UR_!Pg6}4f{^B7>e=&mew-l;860f#`q@_qo1AVS_(Iz4}@6})kDJ-AAeEr7C z+~X@Y$Mo_gub>!M{j$r~yiNeKA1O3ox}jPi%W5rW-4cqBL+vb%d%TyViqd-H7Ay8; zl-{&NwsVAtUZdl37Z4Hp1(5#6DS8^HICTBm}WXgAQA zvvtX-ySZGKni&;`(E%F6yGUaPsCw)INDj?Nl{;{-)}>=x=)$t*-O@#K@G<0?M`z|l za+8Q_7xA_ZtM$WWX3{4Km$-&g94ts9T@ZNHx2!yNsIm z+$bm*Ua7C7K}T##;>tm$&o;(vFWEQbk%1eAc1ui2Qv!u%n0>&^W4pe~nY=E51n$v| zKa8sv?MEb?!lTGdgSUtP1Hvuqk&<;$Pc_!&MUYI~F_2EfAE732J~<_8sbmp*kTP1_ zY$T4dSj$T|BpD5z=(qrOa$Gmm;Sj3{tg7U4z!6-0Dw-x%3D+ev&EcM@qMFsa@;%EjUw>fPP|pV$XVH^5ThcB&=bd zHt=BV=$Sk4`R)e|R|sd`+N!yH?#P33%d3~Jd3rohun0%p%3?~0=wwyRJvkS%`esjE?Mo<1g=+= zyRZi5bxH*?x4B6m!E6>@!QBU_NPL3$(TnHTP&YE7zjD*?#{!cdhj4@EpR$InWP$K9 zMM@x+wN4FZU-*V5LR2wMayQLYh29%t0Vps79M9|J< zanjG=h>Ry9nMvj=k&A0y2}@j@`A1{<}!5VMkGEFH%)9fDb+Scn(;fH;Z@S8tvF+? zr*Xi>NnyBuph|j_`My&a&gY8Q}NnH3W6*Wo1 zdiR}c98D5Urx($51kubD73)FF(@wZteKvI`x~U}GJ(+wH2IHSyeZ}P3gB)kC zwKd!tb|Bt+sC>zut$CAB8njVJ&uYrjUrFbEcYraSezb?GLJ5JXrh-&qL4#T3ZlJfE(OgrVwo7v z)R-hvZN71#3Rd|@g<4x%BRk-n30qxgECd&(^jp15HlTGDtO*Hxg1ttek2n+%$nI}LWBON*MDboQ=D!YgsRky4OrpT!;3&mQdL2BqRG)nkdhad7e+*b|- z0pI4!p?@S^vDdO(r${F-AX1)8PQ6ctT$)J+RtzWBso49OP#+(&+d5cc-YD$)3~L)o zE4ET%l(f2G2GCb4X{7_UNGlys8vNwB0Lptfak(%BBV80?UK`ta4@T|v;qEM0`~rLi zfglLnmj}@OIJ)4TDscZh(dcPG={Asm|C zol1L$*l2opI_(_-rRm+7$U6i~P|jzp_9#|cBtv)vcY^SlB!i*#$)q+wZkD+5M?EAY zLAsV1R^b+{w(UTJJK(OjMN>HWV7WG&V`Kn!kI{*@e|@8iIJC#s@Ty)z5ilN|m#V1( z$0HLAtIF5Wy6mysj>MNos<+hzmBL>P_sFQ5+OtMXdcVDcFVi9WMtfas$9GGPER1j; zbX>Z|GchWWwU04qOw3%kA#nE`pLL8o|LGf=5pNjPbnNg%TJPT&%ED)NWXnL+`^o>no z({M3w+}bVEX)V%bsjN#fT?@G{XN||_v|)q@*P6H#jo?!y)wyjpvdXHasNG9zZ~EvV z(o8#+H5YAYPGvj0b7xz9og7t}s~2b-VAFkawkT#dadmv&!&G7uOwDGL$dOGfNg{O# zG03(DP$*t1Z~^+hfW*oiCCnH0s%5dxX(~lgA6Z*S_3WP1-avhM3vxUO5bgggc-uf2 zk+mfQ+ES1kl~bbFl_}PW6zeaBG=^^nHsyB>yjC({xag08N)>`jtB5RsNF4;tdR}H; zYBcA``pWYW>4+ppJqMHPN_0zB%zZe<%wgzC2|qglmySInIk|S}I&?9QPwh5~)j-*t zWP+kTxx{^IhCmqrsPKk;!~`m-c=jSU4TKWIr={d?W}2cTR7g~4rvp#2OASthb$Yv( zF+9fWa4>eVF0rNpR$%&7)QBj4%zEuG6U!w-@Vz{BMchzkRY23gQWu-Hl!A&^q~24t znV@hmMa4)bbb&S`xx9KNqLC70 zYReEFAF{x`+69)_~%tJ=8#1TUxU|LcvoFF~s<2$&r zA%2XITE(1>%ylIoOx^G)D6v41I$*7WyjEMJ8@LcrBuAMS4MpIYB;B6FIuJR+8BSi# zOK?Ic6bkKvZjbI$@l)t>Ki1@mxwvACD|nM=q28{dz75Adj)X^e&(Egxk;Z6g3 Date: Mon, 2 Mar 2026 18:37:39 +0100 Subject: [PATCH 054/135] fix: Harden event server against crash and stale PID caching - Wrap prometheus_counter:inc/3 in try-catch in handle_events/0 to prevent server crash on Prometheus errors - Add is_process_alive/1 check in find_event_server/0 so long-lived processes recover from a dead cached PID --- src/hb_event.erl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 60dbf2a02..315603075 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -157,7 +157,13 @@ raw_counters() -> %% result in the process dictionary to avoid looking it up multiple times. find_event_server() -> case erlang:get({event_server, ?MODULE}) of - {cached, Pid} -> Pid; + {cached, Pid} -> + case is_process_alive(Pid) of + true -> Pid; + false -> + erlang:erase({event_server, ?MODULE}), + find_event_server() + end; undefined -> PID = case hb_name:lookup(?MODULE) of @@ -217,7 +223,11 @@ handle_events() -> end; _ -> ignored end, - prometheus_counter:inc(<<"event">>, [TopicBin, EventName], Count), + try + prometheus_counter:inc(<<"event">>, [TopicBin, EventName], Count) + catch _:_ -> + ok + end, handle_events() end. From e7af0d753d592bea7d7b77401839d6e99dab6703 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 2 Mar 2026 19:10:08 +0100 Subject: [PATCH 055/135] fix: Add timeout to cpu_sup:avg5 in metrics collector - Wrap cpu_sup:avg5/0 in spawn_monitor with 2s timeout - Kill worker on timeout to prevent process leaks --- src/hb_metrics_collector.erl | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/hb_metrics_collector.erl b/src/hb_metrics_collector.erl index 18804b04b..afb2be508 100644 --- a/src/hb_metrics_collector.erl +++ b/src/hb_metrics_collector.erl @@ -23,7 +23,7 @@ collect_mf(_Registry, Callback) -> ) ), - SystemLoad = cpu_sup:avg5(), + SystemLoad = safe_avg5(), Callback( create_gauge( @@ -58,5 +58,30 @@ collect_metrics(process_uptime_seconds, Uptime) -> %%==================================================================== %% Private Functions %%==================================================================== + +%% @doc Wrapper around cpu_sup:avg5/0 with a 2-second timeout. +%% cpu_sup:avg5/0 uses an infinity timeout to os_mon internally; +%% if the port program stalls, it blocks the Prometheus scrape indefinitely. +%% On timeout, the worker is killed to avoid leaking blocked processes. +safe_avg5() -> + Ref = make_ref(), + Self = self(), + {Pid, MonRef} = spawn_monitor(fun() -> Self ! {Ref, catch cpu_sup:avg5()} end), + receive + {Ref, Load} when is_integer(Load) -> + erlang:demonitor(MonRef, [flush]), + Load; + {Ref, _} -> + erlang:demonitor(MonRef, [flush]), + 0; + {'DOWN', MonRef, process, Pid, _} -> + 0 + after 2000 -> + exit(Pid, kill), + erlang:demonitor(MonRef, [flush]), + receive {Ref, _} -> ok after 0 -> ok end, + 0 + end. + create_gauge(Name, Help, Data) -> prometheus_model_helpers:create_mf(Name, Help, gauge, ?MODULE, Data). \ No newline at end of file From 72b4c2491cfafb875bc48619b773416c105d471d Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 2 Mar 2026 19:10:36 +0100 Subject: [PATCH 056/135] fix: Use selective ETS query for event counters - Replace ets:tab2list with ets:match_object filtering for event rows - Avoids materializing the entire prometheus_counter_table --- src/hb_event.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 315603075..7201c385e 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -150,7 +150,10 @@ raw_counters() -> []. -else. raw_counters() -> - ets:tab2list(prometheus_counter_table). + ets:match_object( + prometheus_counter_table, + {{default, <<"event">>, '_', '_'}, '_', '_'} + ). -endif. %% @doc Find the event server, creating it if it doesn't exist. We cache the From 3d9248788a004dd498eb5a0fbeb33eb412c40683 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 2 Mar 2026 19:11:39 +0100 Subject: [PATCH 057/135] fix: Remove unnecessary spawn in metric recording paths - Inline Prometheus ETS writes in hb_http and hb_store_arweave - Guard observe in hb_http_client record_duration spawn - Eliminates per-request process overhead under load --- src/hb_http.erl | 37 ++++++++++++++++++------------------- src/hb_http_client.erl | 18 ++++++++++-------- src/hb_store_arweave.erl | 26 +++++++++++++------------- 3 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/hb_http.erl b/src/hb_http.erl index ccbff2049..cc35e51bb 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -1154,25 +1154,24 @@ init_prometheus() -> ]). record_request_metric(TotalDuration, ReplyDuration, StatusCode) -> - spawn( - fun() -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - prometheus_histogram:observe( - http_request_server_duration_seconds, - [StatusCode], - TotalDuration - ), - prometheus_histogram:observe( - http_request_server_reply_duration_seconds, - [StatusCode], - ReplyDuration - ) - end, - ok - end - ). + case application:get_application(prometheus) of + undefined -> ok; + _ -> + try + prometheus_histogram:observe( + http_request_server_duration_seconds, + [StatusCode], + TotalDuration + ), + prometheus_histogram:observe( + http_request_server_reply_duration_seconds, + [StatusCode], + ReplyDuration + ) + catch _:_ -> + ok + end + end. %%% Tests diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index 1f8255309..6b32fc527 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -798,17 +798,17 @@ record_duration(Details, Opts) -> % First, write to prometheus if it is enabled. Prometheus works % only with strings as lists, so we encode the data before granting % it. - GetFormat = fun - (<<"request-category">>) -> - path_to_category(maps:get(<<"request-path">>, Details)); - - (Key) -> - hb_util:list(maps:get(Key, Details)) - end, + GetFormat = + fun + (<<"request-category">>) -> + path_to_category(maps:get(<<"request-path">>, Details)); + (Key) -> + hb_util:list(maps:get(Key, Details)) + end, case application:get_application(prometheus) of undefined -> ok; _ -> - prometheus_histogram:observe( + try prometheus_histogram:observe( http_request_duration_seconds, lists:map( GetFormat, @@ -820,6 +820,8 @@ record_duration(Details, Opts) -> ), maps:get(<<"duration">>, Details) ) + catch _:_ -> ok + end end, maybe_invoke_monitor( Details#{ <<"path">> => <<"duration">> }, diff --git a/src/hb_store_arweave.erl b/src/hb_store_arweave.erl index 35b982d44..032709447 100644 --- a/src/hb_store_arweave.erl +++ b/src/hb_store_arweave.erl @@ -254,20 +254,20 @@ write_offset( %% @doc Record the partition that data is found in when it is requested. record_partition_metric(Offset) when is_integer(Offset) -> - spawn( - fun () -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - Partition = Offset div ?PARTITION_SIZE, - prometheus_counter:inc( - hb_store_arweave_requests_partition, - [Partition], - 1 - ) + case application:get_application(prometheus) of + undefined -> ok; + _ -> + try + Partition = Offset div ?PARTITION_SIZE, + prometheus_counter:inc( + hb_store_arweave_requests_partition, + [Partition], + 1 + ) + catch _:_ -> + ok end - end - ). + end. %% @doc Initialize the Prometheus metrics for the Arweave store. Executed on %% `start/1' of the store. From dbe938ecda182bca7bf008f176e53756fa7e08c0 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 2 Mar 2026 19:12:58 +0100 Subject: [PATCH 058/135] fix: Guard unprotected Prometheus writes in hot paths - Add try-catch to dec_prometheus_gauge and inc_prometheus_counter - Guard retry path in inc_prometheus_gauge - Wrap hb_store_lmdb:name_hit_metrics to prevent LMDB read crashes --- src/hb_http_client.erl | 30 ++++++++++++++++++++++++++---- src/hb_store_lmdb.erl | 4 +++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index 6b32fc527..4597ead8b 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -849,8 +849,12 @@ inc_prometheus_gauge(Name) -> _ -> try prometheus_gauge:inc(Name) catch _:_ -> - init_prometheus(), - prometheus_gauge:inc(Name) + try + init_prometheus(), + prometheus_gauge:inc(Name) + catch _:_ -> + ok + end end end. @@ -858,13 +862,31 @@ inc_prometheus_gauge(Name) -> dec_prometheus_gauge(Name) -> case application:get_application(prometheus) of undefined -> ok; - _ -> prometheus_gauge:dec(Name) + _ -> + try prometheus_gauge:dec(Name) + catch _:_ -> + try + init_prometheus(), + prometheus_gauge:dec(Name) + catch _:_ -> + ok + end + end end. inc_prometheus_counter(Name, Labels, Value) -> case application:get_application(prometheus) of undefined -> ok; - _ -> prometheus_counter:inc(Name, Labels, Value) + _ -> + try prometheus_counter:inc(Name, Labels, Value) + catch _:_ -> + try + init_prometheus(), + prometheus_counter:inc(Name, Labels, Value) + catch _:_ -> + ok + end + end end. download_metric(Data) -> diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index acbe5980b..59bff2f5a 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -613,7 +613,9 @@ reset(Opts) -> %% @doc Increment the hit metrics for the current store's name. name_hit_metrics(Name) -> - prometheus_counter:inc(hb_store_lmdb_hit, [Name], 1). + try prometheus_counter:inc(hb_store_lmdb_hit, [Name], 1) + catch _:_ -> ok + end. init_prometheus() -> hb_prometheus:declare(histogram, [ From e7c544be75c853f273e748b73026db0e3be65c2e Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 2 Mar 2026 19:13:38 +0100 Subject: [PATCH 059/135] fix: Cache Prometheus startup failure with retry backoff - Check ensure_all_started result instead of ignoring it - Cache failure timestamp in persistent_term, retry after 60s - Callers degrade gracefully when Prometheus is unavailable --- src/hb_prometheus.erl | 67 +++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/src/hb_prometheus.erl b/src/hb_prometheus.erl index f4aa717ae..d69579b2e 100644 --- a/src/hb_prometheus.erl +++ b/src/hb_prometheus.erl @@ -2,22 +2,49 @@ -module(hb_prometheus). -export([ensure_started/0, declare/2, measure_and_report/2, measure_and_report/3]). -%% @doc Ensure the Prometheus application has been started. +-define(STARTUP_RETRY_INTERVAL, 60). % seconds + +%% @doc Ensure the Prometheus application has been started. Caches startup +%% failure with a timestamp to avoid repeated blocking ensure_all_started +%% calls on hot paths, but retries after a cooldown period. ensure_started() -> case application:get_application(prometheus) of undefined -> - application:ensure_all_started( - [prometheus, prometheus_cowboy, prometheus_ranch] - ), - ok; + case persistent_term:get(hb_prometheus_start_failed, undefined) of + FailedAt when is_integer(FailedAt) -> + case erlang:monotonic_time(second) - FailedAt >= ?STARTUP_RETRY_INTERVAL of + true -> attempt_start(); + false -> {error, not_started} + end; + undefined -> + attempt_start() + end; _ -> ok end. +attempt_start() -> + case application:ensure_all_started( + [prometheus, prometheus_cowboy, prometheus_ranch] + ) of + {ok, _} -> + persistent_term:erase(hb_prometheus_start_failed), + ok; + {error, _} = Err -> + persistent_term:put( + hb_prometheus_start_failed, + erlang:monotonic_time(second) + ), + Err + end. + %% @doc Declare a new Prometheus metric in a replay-safe manner. declare(Type, Metric) -> - ok = ensure_started(), - try do_declare(Type, Metric) - catch error:mfa_already_exists -> ok + case ensure_started() of + ok -> + try do_declare(Type, Metric) + catch error:mfa_already_exists -> ok + end; + _ -> ok end. do_declare(histogram, Metric) -> prometheus_histogram:declare(Metric); @@ -26,16 +53,20 @@ do_declare(gauge, Metric) -> prometheus_gauge:declare(Metric); do_declare(Type, _Metric) -> throw({unsupported_metric_type, Type}). %% @doc Measure function duration and report metric, ensuring that the Prometheus -%% application has been started first. -measure_and_report(Fun, Metric) -> +%% application has been started first. If Prometheus is unavailable, the function +%% is executed without measurement. +measure_and_report(Fun, Metric) -> measure_and_report(Fun, Metric, []). measure_and_report(Fun, Metric, Labels) -> - ok = ensure_started(), - Start = erlang:monotonic_time(), - try Fun() - after - DurationNative = erlang:monotonic_time() - Start, - try prometheus_histogram:observe(Metric, Labels, DurationNative) - catch _:_ -> ok - end + case ensure_started() of + ok -> + Start = erlang:monotonic_time(), + try Fun() + after + DurationNative = erlang:monotonic_time() - Start, + try prometheus_histogram:observe(Metric, Labels, DurationNative) + catch _:_ -> ok + end + end; + _ -> Fun() end. From cf6f5f5985681ade81873cc40082c1516770da74 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 2 Mar 2026 19:14:37 +0100 Subject: [PATCH 060/135] fix: Bound event server Prometheus wait and handle late startup - Cap await_prometheus_started to ~30s to prevent unbounded mailbox growth - Messages stay in mailbox during wait instead of being consumed and lost - Wrap declare in try-catch, redeclare lazily on inc failure via ensure_event_counter --- src/hb_event.erl | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 7201c385e..df4b1eb62 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -9,6 +9,7 @@ -define(OVERLOAD_QUEUE_LENGTH, 10000). -define(MAX_MEMORY, 1_000_000_000). % 1GB -define(MAX_EVENT_NAME_LENGTH, 100). +-define(MAX_PROMETHEUS_WAIT, 300). % ~30s at 100ms intervals -ifdef(NO_EVENTS). log(_X) -> ok. @@ -182,19 +183,25 @@ find_event_server() -> server() -> await_prometheus_started(), - prometheus_counter:declare( + ensure_event_counter(), + handle_events(). + +ensure_event_counter() -> + try prometheus_counter:declare( [ {name, <<"event">>}, {help, <<"AO-Core execution events">>}, {labels, [topic, event]} - ]), - handle_events(). + ]) + catch _:_ -> ok + end. + handle_events() -> receive {increment, TopicBin, EventName, Count} -> case erlang:process_info(self(), message_queue_len) of {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH -> - % Print a warning, but do so less frequently the more + % Print a warning, but do so less frequently the more % overloaded the system is. {memory, MemorySize} = erlang:process_info(self(), memory), case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of @@ -229,19 +236,24 @@ handle_events() -> try prometheus_counter:inc(<<"event">>, [TopicBin, EventName], Count) catch _:_ -> - ok + ensure_event_counter() end, handle_events() end. -%% @doc Delay the event server until prometheus is started. +%% @doc Delay the event server until prometheus is started. Messages +%% accumulate in the mailbox and are processed once handle_events/0 starts. +%% Gives up after ?MAX_PROMETHEUS_WAIT attempts to avoid unbounded mailbox +%% growth if Prometheus never comes up. await_prometheus_started() -> - receive - Msg -> - case application:get_application(prometheus) of - undefined -> await_prometheus_started(); - _ -> self() ! Msg, ok - end + await_prometheus_started(?MAX_PROMETHEUS_WAIT). +await_prometheus_started(0) -> ok; +await_prometheus_started(Remaining) -> + case application:get_application(prometheus) of + undefined -> + receive after 100 -> ok end, + await_prometheus_started(Remaining - 1); + _ -> ok end. parse_name(Name) when is_tuple(Name) -> From 18915f51c195e66a853ff2a857bd521a9be059fc Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Mon, 2 Mar 2026 20:39:33 +0100 Subject: [PATCH 061/135] fix: Restore spawn in record_request_metric for latency isolation - Keep metric writes off the HTTP reply path to isolate request tail latency - Retain try-catch inside the spawn to guard against metric-path hiccups --- src/hb_http.erl | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/hb_http.erl b/src/hb_http.erl index cc35e51bb..67cc4f48f 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -1154,24 +1154,28 @@ init_prometheus() -> ]). record_request_metric(TotalDuration, ReplyDuration, StatusCode) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - try - prometheus_histogram:observe( - http_request_server_duration_seconds, - [StatusCode], - TotalDuration - ), - prometheus_histogram:observe( - http_request_server_reply_duration_seconds, - [StatusCode], - ReplyDuration - ) - catch _:_ -> - ok + spawn( + fun() -> + case application:get_application(prometheus) of + undefined -> ok; + _ -> + try + prometheus_histogram:observe( + http_request_server_duration_seconds, + [StatusCode], + TotalDuration + ), + prometheus_histogram:observe( + http_request_server_reply_duration_seconds, + [StatusCode], + ReplyDuration + ) + catch _:_ -> + ok + end end - end. + end + ). %%% Tests From b4ca0bdd9835042cbb5a89ee42a537d254e5e924 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 5 Mar 2026 15:05:57 +0000 Subject: [PATCH 062/135] fix: dev_copycat_arweave test --- src/dev_copycat_arweave.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/dev_copycat_arweave.erl b/src/dev_copycat_arweave.erl index 13c9930a7..45e1e8476 100644 --- a/src/dev_copycat_arweave.erl +++ b/src/dev_copycat_arweave.erl @@ -1000,6 +1000,7 @@ negative_from_index_test() -> ok. setup_index_opts() -> + application:ensure_all_started([hb]), TestStore = hb_test_utils:test_store(), StoreOpts = #{ <<"index-store">> => [TestStore] }, Store = [ From b651a0a06fdf56c86041385ae4d4992210f26ea9 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 5 Mar 2026 15:59:46 +0000 Subject: [PATCH 063/135] disable failure test, until we find a unsupported tx --- src/hb_store_gateway.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hb_store_gateway.erl b/src/hb_store_gateway.erl index 9799b3a14..ea992a30b 100644 --- a/src/hb_store_gateway.erl +++ b/src/hb_store_gateway.erl @@ -475,7 +475,8 @@ verifiability_test() -> ?assert(hb_message:verify(Structured)). %% @doc Reading an unsupported signature type transaction should fail -failure_to_process_message_test() -> +%% TODO: Enable when we find a TX that we don't support +failure_to_process_message_test_disabled() -> hb_http_server:start_node(#{}), ?assertEqual(failure, hb_cache:read( From 9ba746f94d844ced08976a6a92ddd40f4188e61b Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 6 Mar 2026 20:50:40 +0000 Subject: [PATCH 064/135] fix: Test in dev_codec_tx, hb_ao_test_vectors, hb_http_client and hb_store_arweave --- src/dev_codec_tx.erl | 1 + src/hb_ao_test_vectors.erl | 3 +++ src/hb_http_client.erl | 1 + src/hb_store_arweave.erl | 2 ++ 4 files changed, 7 insertions(+) diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index 46f91bc93..ad31ace25 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -1204,6 +1204,7 @@ real_no_data_tx_test() -> ). do_real_tx_verify(TXID, ExpectedIDs) -> + application:ensure_all_started([hb]), Opts = #{}, {ok, #{ <<"body">> := TXJSON }} = hb_http:request( #{ diff --git a/src/hb_ao_test_vectors.erl b/src/hb_ao_test_vectors.erl index 886f1ceee..919750c53 100644 --- a/src/hb_ao_test_vectors.erl +++ b/src/hb_ao_test_vectors.erl @@ -841,6 +841,9 @@ load_as_test(Opts) -> <<"device">> => <<"test-device@1.0">>, <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } }, + % There is a race condition where we write to the store and a + % reset happens making not read the written value. + timer:sleep(10), {ok, ID} = hb_cache:write(Msg, Opts), {ok, ReadMsg} = hb_cache:read(ID, Opts), ?assert(hb_message:match(Msg, ReadMsg, primary, Opts)), diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index 4597ead8b..c9508ba5e 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -10,6 +10,7 @@ %% GenServer -export([start_link/1, init/1]). -export([handle_cast/2, handle_call/3, handle_info/2, terminate/2]). +-export([init_prometheus/0]). -record(state, { opts = #{} diff --git a/src/hb_store_arweave.erl b/src/hb_store_arweave.erl index 032709447..6a9bf2bc4 100644 --- a/src/hb_store_arweave.erl +++ b/src/hb_store_arweave.erl @@ -304,6 +304,7 @@ init_prometheus() -> %%% Tests write_read_tx_test() -> + application:ensure_all_started(hb), Store = [hb_test_utils:test_store()], Opts = #{ <<"index-store">> => Store @@ -346,6 +347,7 @@ write_read_tx_test() -> %% @doc The L1 TX has bundle tags, but data is not a valid bundle. write_read_fake_bundle_tx_test() -> + application:ensure_all_started(hb), Store = [hb_test_utils:test_store()], Opts = #{ <<"index-store">> => Store From e401c54d2944468d88e454566ef28c05e294f6b3 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 6 Mar 2026 22:47:10 +0000 Subject: [PATCH 065/135] fix: Add host to httpsig requests --- src/hb_http.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/hb_http.erl b/src/hb_http.erl index 67cc4f48f..521a76fa4 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -1102,10 +1102,13 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> end, WithPrivIP = hb_private:set(NormalBody, <<"ip">>, RealIP, Opts), % Add device from PrimMsg if present - case maps:get(<<"device">>, PrimMsg, not_found) of + WithDevice = case maps:get(<<"device">>, PrimMsg, not_found) of not_found -> WithPrivIP; Device -> WithPrivIP#{<<"device">> => Device} - end. + end, + % Add host + Host = cowboy_req:host(Req), + WithDevice#{<<"host">> => Host}. %% @doc Determine the caller, honoring the `x-real-ip' header if present. real_ip(Req = #{ headers := RawHeaders }, Opts) -> From f4d24ae56dd09854424597840e8c0d5bd8645423 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 6 Mar 2026 23:47:46 +0000 Subject: [PATCH 066/135] fix: prometheus declare instead of new --- src/hb_http_client.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index c9508ba5e..a029eeb5a 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -749,7 +749,7 @@ log(Type, Event, #{method := Method, peer := Peer, path := Path}, Reason, Opts) init_prometheus() -> application:ensure_all_started([prometheus, prometheus_cowboy]), - prometheus_counter:new([ + prometheus_counter:declare([ {name, gun_requests_total}, {labels, [http_method, status_class, category]}, { @@ -757,9 +757,9 @@ init_prometheus() -> "The total number of GUN requests." } ]), - prometheus_gauge:new([{name, outbound_connections}, + prometheus_gauge:declare([{name, outbound_connections}, {help, "The current number of the open outbound network connections"}]), - prometheus_histogram:new([ + prometheus_histogram:declare([ {name, http_request_duration_seconds}, {buckets, [0.01, 0.1, 0.5, 1, 5, 10, 30, 60]}, {labels, [http_method, status_class, category]}, @@ -770,7 +770,7 @@ init_prometheus() -> "throttling, etc...)" } ]), - prometheus_histogram:new([ + prometheus_histogram:declare([ {name, http_client_get_chunk_duration_seconds}, {buckets, [0.1, 1, 10, 60]}, {labels, [status_class, peer]}, @@ -779,11 +779,11 @@ init_prometheus() -> "The total duration of an HTTP GET chunk request made to a peer." } ]), - prometheus_counter:new([ + prometheus_counter:declare([ {name, http_client_downloaded_bytes_total}, {help, "The total amount of bytes requested via HTTP, per remote endpoint"} ]), - prometheus_counter:new([ + prometheus_counter:declare([ {name, http_client_uploaded_bytes_total}, {help, "The total amount of bytes posted via HTTP, per remote endpoint"} ]), From d33a75805e979ff36ddee52ba9c432c039f8fc36 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Sat, 7 Mar 2026 00:19:29 +0000 Subject: [PATCH 067/135] fix: init hb on hb_client --- src/hb_client.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/hb_client.erl b/src/hb_client.erl index e1b37e62b..3298ee2f7 100644 --- a/src/hb_client.erl +++ b/src/hb_client.erl @@ -128,6 +128,7 @@ upload(Msg, Opts, <<"tx@1.0">>) when is_map(Msg) -> %%% Tests upload_empty_raw_ans104_test() -> + application:ensure_all_started(hb), Serialized = ar_bundles:serialize( ar_bundles:sign_item(#tx{ data = <<"TEST">> @@ -139,6 +140,7 @@ upload_empty_raw_ans104_test() -> ?assertMatch({ok, _}, Result). upload_raw_ans104_test() -> + application:ensure_all_started(hb), Serialized = ar_bundles:serialize( ar_bundles:sign_item(#tx{ data = <<"TEST">>, @@ -151,6 +153,7 @@ upload_raw_ans104_test() -> ?assertMatch({ok, _}, Result). upload_raw_ans104_with_anchor_test() -> + application:ensure_all_started(hb), Serialized = ar_bundles:serialize( ar_bundles:sign_item(#tx{ data = <<"TEST">>, @@ -164,6 +167,7 @@ upload_raw_ans104_with_anchor_test() -> ?assertMatch({ok, _}, Result). upload_empty_message_test() -> + application:ensure_all_started(hb), Msg = #{ <<"data">> => <<"TEST">> }, Committed = hb_message:commit( @@ -176,6 +180,7 @@ upload_empty_message_test() -> ?assertMatch({ok, _}, Result). upload_single_layer_message_test() -> + application:ensure_all_started(hb), Msg = #{ <<"data">> => <<"TEST">>, <<"basic">> => <<"value">>, From 5bb8bef982bd8806217ac80d3bda12db3d5a6801 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Mon, 9 Mar 2026 18:52:24 +0000 Subject: [PATCH 068/135] Reapply "Merge pull request #729 from permaweb/speeddragon/impr/manifest-hook" This reverts commit 9f0d114cde692df64827414b74bbb70d74567f26. --- src/dev_manifest.erl | 16 +++++++++--- src/hb_format.erl | 2 +- src/hb_http.erl | 24 ++---------------- src/hb_structured_fields.erl | 4 ++- ...7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin | Bin 0 -> 6988 bytes ...IS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin | Bin 0 -> 5528 bytes ...-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin | Bin 0 -> 66912 bytes 7 files changed, 18 insertions(+), 28 deletions(-) create mode 100644 test/arbundles.js/ans-104-manifest-42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin create mode 100644 test/arbundles.js/ans-104-manifest-index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin create mode 100644 test/arbundles.js/ans-104-manifest-item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index 24cb4228c..0de5401f1 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -1,7 +1,7 @@ %%% @doc An Arweave path manifest resolution device. Follows the v1 schema: %%% https://specs.ar.io/?tx=lXLd0OPwo-dJLB_Amz5jgIeDhiOkjXuM3-r0H_aiNj0 -module(dev_manifest). --export([index/3, info/0]). +-export([index/3, info/0, request/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -62,7 +62,7 @@ route(ID, _, _, Opts) when ?IS_ID(ID) -> ?event({manifest_reading_id, ID}), hb_cache:read(ID, Opts); route(Key, M1, M2, Opts) -> - ?event(debug_manifest, {manifest_lookup, {key, Key}, {m1, M1}, {m2, M2}}), + ?event(debug_manifest, {manifest_lookup, {key, Key}, {m1, M1}, {m2, {explicit, M2}}}), {ok, Manifest} = manifest(M1, M2, Opts), {ok, Res} = maps:find(<<"paths">>, Manifest), case maps:get(Key, Res, no_path_match) of @@ -70,6 +70,7 @@ route(Key, M1, M2, Opts) -> % Support materialized view in some JavaScript frameworks. case hb_opts:get(manifest_404, fallback, Opts) of error -> + ?event({manifest_404_error, {key, Key}}), {error, not_found}; fallback -> ?event({manifest_fallback, {key, Key}}), @@ -201,7 +202,14 @@ linkify(Manifest, _Opts) -> %%% Tests resolve_test() -> - Opts = #{ store => hb_opts:get(store, no_viable_store, #{}) }, + Opts = #{ + store => hb_opts:get(store, no_viable_store, #{}), + on => #{ + <<"request">> => #{ + <<"device">> => <<"manifest@1.0">> + } + } + }, IndexPage = #{ <<"content-type">> => <<"text/html">>, <<"body">> => <<"Page 1">> @@ -404,4 +412,4 @@ load_and_store(LmdbStore, File) -> <<"ans104@1.0">>, Opts ), - _ = hb_cache:write(Message, #{store => LmdbStore}). + _ = hb_cache:write(Message, #{store => LmdbStore}). \ No newline at end of file diff --git a/src/hb_format.erl b/src/hb_format.erl index 1603266dd..ee07ee0ed 100644 --- a/src/hb_format.erl +++ b/src/hb_format.erl @@ -958,7 +958,7 @@ format_key(true, Committed, Key, ToPrint, Opts) -> case lists:member(NormKey = hb_ao:normalize_key(Key, Opts), Committed) of true when ToPrint == undefined -> <<"* ", NormKey/binary>>; true -> <<"* ", ToPrint/binary>>; - false -> format_key(false, Committed, Key, undefined, Opts) + false -> format_key(false, Committed, Key, ToPrint, Opts) end. %% @doc Return a formatted list of short IDs, given a raw list of IDs. diff --git a/src/hb_http.erl b/src/hb_http.erl index 521a76fa4..a6a497ad3 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -463,7 +463,7 @@ prepare_request(Format, Method, Peer, Path, RawMessage, Opts) -> %% @doc Reply to the client's HTTP request with a message. reply(Req, TABMReq, Message, Opts) -> Status = - case hb_ao:get(<<"status">>, Message, Opts) of + case hb_maps:get(<<"status">>, Message, not_found, Opts) of not_found -> 200; S-> S end, @@ -719,20 +719,6 @@ encode_reply(Status, TABMReq, Message, Opts) -> ) ) }; - {_, <<"manifest@1.0">>, _} -> - MessageID = hb_message:id(Message, signed, Opts), - { - 307, - #{ - <<"location">> => - << - "/", - MessageID/binary, - "~manifest@1.0/index" - >> - }, - <<"Manifesting your data...">> - }; _ -> % Other codecs are already in binary format, so we can just convert % the message to the codec. We also include all of the top-level @@ -788,12 +774,6 @@ accept_to_codec(OriginalReq, Reply = #{ <<"content-type">> := Link }, Opts) when Reply#{ <<"content-type">> => hb_cache:ensure_loaded(Link, Opts) }, Opts ); -accept_to_codec( - _, - #{ <<"content-type">> := <<"application/x.arweave-manifest", _/binary>> }, - _Opts - ) -> - <<"manifest@1.0">>; accept_to_codec(_OriginalReq, #{ <<"content-type">> := CT }, _Opts) -> <<"httpsig@1.0">>; accept_to_codec(OriginalReq, _, Opts) -> @@ -1453,4 +1433,4 @@ request_error_handling_test() -> Opts ), % The result should be an error tuple, not crash with badmatch - ?assertMatch({error, _}, Result). + ?assertMatch({error, _}, Result). \ No newline at end of file diff --git a/src/hb_structured_fields.erl b/src/hb_structured_fields.erl index 658202e15..c4faef841 100644 --- a/src/hb_structured_fields.erl +++ b/src/hb_structured_fields.erl @@ -311,7 +311,9 @@ parse_bare_item(<<"?0", R/bits>>) -> {false, R}; parse_bare_item(<<"?1", R/bits>>) -> % Parse a boolean true. - {true, R}. + {true, R}; +parse_bare_item(<<>>) -> + {{binary, <<>>}, <<>>}. %% @doc Parse an integer or decimal binary. parse_number(<>, L, Acc) when ?IS_DIGIT(C) -> diff --git a/test/arbundles.js/ans-104-manifest-42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin b/test/arbundles.js/ans-104-manifest-42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin new file mode 100644 index 0000000000000000000000000000000000000000..898f953aefea08525dd0836a0bb35d50639a36af GIT binary patch literal 6988 zcmb7|dEDc4oyS2r76F0b7C~PFx*lXm(=<&Hfn06Vv`v~OZCa3$q)D3lY@1eQQDK=u z6hvKMltXj@U6tWd(Pdb<1XS<^FAfnFP>$I}84zSpP-iu#d>vr9J%7phoz?tHc%@;Rp-u#F4 z_8-r>>%rSM-Tv^O9@z5Cw|9H!WBbJC-!iYIF#M@)H=ySkH?8-7GJCFfz}@Fh_tgIL zrym}!DSeN7cI(a0ezdsv^bvPe{o+T;>BohQ@mD@{!oB7>7e99MCC|S`oc>;Sf91YE zp1$GSzut1nmA`oEkRP79_L?)+pLO8As(a-tw_QNoy!OpoFFO7Q&kCF7`<(G#>;HVo zD;xKE{=}nx7aVoV-S(+oUNMgy7925?tSy!Pk!~EZawuEuU`4w`m@h|c6Qb2pWX7p z2M>PsiftF(`OEtsyYPgsUwwdo(B-F3H_x9tw@39_P*ku8z1I>er)>@<7a1WUH|e1<;dOE zFc19ZHkH3E6wlAG|?aAHd(ARkE;t{ha*u5v$zj$>2)Kk!x9^L=p-~Z+rqjSfJ7ky-}&702J zaL#v_t>=6+cY*z-lc?y-uhedY4>uk>W{>HG8y|jX^~d%*;<3-)eD?>p-uI6FEw?{- zj=t@?uju(3l*4|y0sivt#Po%i2KJ^19I z7p^~{E8gUe0s~%m)>~mzkOoe72iAh_vha6na57~#OUWYymQOZ z`~4cd`L<(z>kfi%es1@VjOW`vL*B_2w%A|)a%dm*OMcUJ|FQoOx4g;k`rG?oz5TPN zANR${As5|-&(^fhJK#lp->YxD#r@{ho38!%yaW(t zfmW7-z%f-hF@5j2bd4NMHF=@|uI!n*7AGIIV&B^z3j&}iyV_Bl7P<0NQvgy69DlaU zCs)7q;_Bm9uYC8oKu$*S>SI?=v?yLs7f--7&>C>@iRlfsboKF{T)q93#pgTzUNcHu zXZ6}+SL?FlD6(pAf599s9t+_J@#!{1)J2)><#IidYC4li)g?_g7v_+lF_SQ@&BW!# z_V4ffzl$%+ajYfraafd?s-xkKui5~aOf9tl3u~knK4 zq(-x%Q0DWZNsh{sRP{#$yq&*-CFDdi{K(W|fPngPJVv1`v;^f?zk*RYyhKK$sMd68 zR!PNKL+J5yq}ae*ors$_Kj3zVUV)OnE1O>3j1$g}64?O=q15YjW>yvr$Ql^9Zmx=9 zBHlwwGd+OEWX}amXVRRyiZB*i;*?Ysz{uz&qlLvHfansD4|qTp7&*ncM7sU79M&Ps z#woUr7He%Mr1Dl^0~AP2`eNUrh&Gi0)@9Ea%ZAq098FCCvW`qU2AE{Qt!^_eHUK`Z zhqCY4rprX3?zF~s*sK@AvK3Tlw^62AD(q*#4Q*)3byJDtXa?XNx?0GyW)>UdYpgrd z9kthEa!{!p`EbFlrBeiL;kZ-hSUfh1le`{G;|!KtXr+c`;zt1K7zkjqcotf7j3Ekx zPy?l58B0i_&<;A~@HUgdwH6 zIz@v~ldwu&vymqa3}+#&YbEng6Qjj*xR+JB!}e`083zDalf;5eaak&t#(aV32-Tvg z@@!|~HL#S1I2B2%kS5k;OxC*&hZH-YxI~SUmGcuJ9|3ibUxE%sQptuJV9k&ka;p+e zinYcNLeMsbrHUxh-x7*N`4c{F`h_l$1r{u}t&z2Ht&EbLNNms4xDt4zHv%O; zYS-94rzHhElnVx$(I|{GcYBn!NgNiCQWlt0S_NwEx-r1YA=hV6hOfXf-V^gKkS~UK zSm9)FE)EnSYahWk3`YY9GSn4$T+0G2rCcZ4kTsxCK34#AqNH^Py{0w|aDS=|nzeDR zPqg%CBrhrS15=Fw+;gdciYZy(pxPV&qgu_8*|s1Rw9pqjG}L3d6F~1($UZVp!y$=+ zblA!qS&X%4hgtAe4T%+S=@#caI3o;nW@}beL2Q*n>vh_7NnIUuQzOAK&rpf5kX8y= zvvf>tTGiy?_8iiTjd2RFS)B#H;#Qg#Fdfr*HHXz(NhD#6nU(~=L;`~<0$tq`gGh(7 zNmTX$)%Uy|`J=^vYL4q=p%SYJw&e7fIoxTX^l+M{({eQ}%S8wdf(c&6FnbVGSqjhI z=Aw%IEiakOsJ*gPj-BfeYaXcqRG3jYa%_{l~qXE63!tf9krzv6QDNONF$Ie1kGXU^vOv*r9 z$>Q8e$F8b*(c|(6j@74&3+KCq9GY0(q|Q{V5TjZ$U{6RWYxSYFNSEVsr9EtAVYJ_G z27rasX~G*}s7Q{QNe}Y!C@>?KSRvJlDGmm}Op3DUedrC>Xt4`)M7uh*OG`w(5K`q{ zp%0r4v~8o%*lmxhlSP^w>y;qR&p^b&aX(UKemtuYh^2 z@-$q~n*)I?<_9q@xw%5Cfo3mImnY`JxdDRa%8t&kYvPIVx1K``uv}iKNf&FI_FC;_>Fp@Lj@DN+Eipp0Cjjrol^4?|Ez5T(9n z3FweP0ur%dw9l}1ZX7IS#+^I_&nZNMnP%^=TdpWxRR*{;t>;Q%onYp+hY6(-g3YQS zK9{%(+g9lE2+rQ$P992+B#R*1Szd>u1YIu_I&E{9795lAj_Ou~mdg47Ldt!6-keQl z7-@B>Y<%qG;Xnt~(N;CfvrDktGt9o!=P@4~_gsI1>ww@a>N60Bbh$7?cn2Q#y+XFV zTH%5H84D0dmM5@e&F4jgGdW!r%}&*<22;AzuNw0_h))D2F?0v2lq~%3=wmdNCREpu!ibFr=zy)99*9BDyXqZG223{=+hc;fKi34J~?bdVS~qQal0$!Sz9T%R=;n< zip;@)EL2)ZR5d7Jmi(RjON=~)sKYF;&mndvpJGzqLC1~UWK=hkLe$Mo8Hfra9JM@t z=JajHQoy&x2hj~naJ|;d@+_;{6oXMca^Z2aq&bjJ7b%?Z(5l?$ir5lIpU|J5)B(~{}+5j6~-h8`#$p%uho5_;@K_u42i&)dIjCq@3 zn0}Hoa(#B;AEZA@c#Rt>9a5@OjQe(fCL5iIqeg!A{)T+MJDt@km4R1FrOMd#VVZ6* z!JuA5FwX^0TA&NCYJ2l+WnVe((2)SUxYWq?4M7=9LTOYA+)8a$DVY=2} zXB1Ga&T7-CILVN>BqJDnq`ssF+0FEi|BdQBsB4tZt@_LUq0zZ%O12$LjX20f6@n zrC(r{+M;p`uW}LSkllm_kz8eJV`-;M3@OYTO+9K%HE{_ErV&jqyG<5r`$NPi$xCV> zXVk@`POBt?X*qIIP>c$sNw=-3h!sTDk-U*kMI{o+TH7%_d(mVEn3TxsQr9lj26>07 zDl>4>Nw|_HVX~%!UMV-pQ$cE5_LQG0-kj~$-_9chnJG2krD&ln1`uGY#{#u#YcT{g zc&r6lgsi7po6no$W=)@1aI46$+1p&~P!=PqyL}~tJKDU2B};v4l}^#RZzg8wG8V0u zCY?%LpEKe_$3mH4#(5%%!g3K)!>mhYpyUAQD&@)2C^^x&pgjxcbB*iiLYo_y!K5=B z&K;k{5OwaUy*yH}6@n>c+xEA%JpdjS3VpPRE{wB%!?Uump}Ag8E(42&!=xP-c@x} zZM53Lq|Isy>k?u#De-kQTl{wNNTppfC-_oxQ?q-4-5d?_4Xc_W!v++%k}TDgYKdh^ zV@EZuSz-oUu?l6)6TG$ba!e%#N+D5oRCI|O6yS*kq20IzN;Zyr6;xMDuHR{hjai5aCbK#hORB}3kJlxA8CsLC*~GgI=(Y82e~e(P|aK;0PCZ%GHF2YG^}U4 z#6&h0Vc4P)DWv@su)U@bp{7tJP>)j zpt;#&dlsgM$)HVisJWRUQ5`la1+6*g3O>s_aeHZK9>*FM(JU@IvZpRazUt6UQ(Mb| z9gzo|L|6Hcgr|{*Hpj5rCfXLrRRk3oP{BOkHkDCn)XU!FI5C~=dy*0x3&~Qsrt*s@ z&WlN{AF)O*?&lFPO_ec}^h(9BrIjVzQ8i0UT1)NhMDvD=fzQc}?doEglogmVrD9MW zGeJtlfSH5BGBr|iV1=m$-Mo@ZOAR;IhKDn{vBR}@6t0~c;%PT$w7RWIt8c4_0OJNyOLpwta868J*<+ zzx({ZpSwap%j#W=t{>mmvt~v48(C|KSr@bdN3CELH$^mc?D7`6GQrJYm4X=;w;M&~{GS=$a_#mgTK-O$6_ zUEew*B6ZIzCw82mLz|mAs+V2dv~D>ur(vq?@P>C!Pn9Mw>pHV;=8^|wxB4!YHFda- zxI?#v*o>Gr%twxt2vIVw#)<(uDx5P>`^v0?%n_6 z=cgWd=(BNEHMwMV{r>BFWg zU6Kc;G)lNFP~u zE@M>goG;&=H*s8J`{u5l3spyQ8cN#E-bfqVuyjX8X~FsQ`Yy-rYcIWTcdzJ)m{fLm z?_1mc;#u)jPha$cyVBDlJ?9GVt!}#(RrYv!a^LCjlV4x#vCe%z?S&bV%k9e=>bK?f z*T<@#tK5*D%`AO!)}Hvu*6wMcu5~+4ek@H5i~MBmJBOqDKN?YXprgCqGPwO4OWf{? zNk<#v^^u2aUY3NJZ;qe#)91Ct9UI4l99%y*`)Y<^F4CFZ{aM7imAcY7d+C*lo$Tp+ z!^*s@nq>9(^S$Rc?>X2ye?{Y8qrP7C{?nh7&$k`heRpqE*oEYG4o$ma|D$8;ntNtA zYX+ar>MTs?t$g)QRh3iD6u0mCTX^Jwo5gosIsR?OGYeGft0%s2$ly_ytR8#KFs6CW z0sGeG_8t1OMo;0E$KQD2laRfoQJZ(z+D43iIIn2+>Z|k<5noO0U%R;a{(Qx)frgVQ z%@JcG&&GY=tY{`CKK9;ZMOS^*Kv(&XN_qP!cTJyb`S!Omu75b;&|M zBxFR0_|bU3e>W+UrZ_-xsL|s9<2c~vR5s2|hCl3Zpar-cOeq449e05WGz&OL+7qIi znpK!-e5@!N;qlT&>-aB(#3`#@3MgsXXo=1Sa5Mat=u^##P7xNsLZfdZ z_W@8tRYF2pazV0kAag!<2VPaRjDk{i>%U0(^eAT4uVyhX!|dq zXE~fBOd_#~p;?w@2rEGaC;~$)5w-t0(JB+mYSJyZogh8>M`;Gv5*$t{Tvi*GfhpoK zOsP!5uvGQX&uA}hp#xB{B{;|g0v!7#W<-`i7V(=9cy9;2ZwO5x#*>K z+Uz6&tj>UJVi{9F?Fi+H8n7}?A)1*8GhdpfQ;9r;uHjFAkrylilE%%!EN?3)oJPHc zf@aOm!%XU;#{|Z_dcTC9!`Bc=qJl+JsxluTP06lb#j+YuW|~1swX^b(q*?jDiWkd4 zj{`K*SPCrpvnqcTFDFNkESg!8Qpg%P+RI9(^7eOK1#R{SVKEcc2w~Ps8Jgz!h{1Kx zJ`N8PK2Ct)&j&s&s;!9IPExF1%C`>Kv|KKi(iNwq8LJ9b4yF>CCf}hAv|BI55Dd{I zB=Y}74icRMRtL{ZB7uVQljD_|B!xOjtxSp|(S!sgmV#<9WgMQQOwlNW7pA}zY6XU> zv3W;IUI}6`I&kHxuRup?^jB8yd1m zC>RIL!s(Qzw1|ap19Pgd${>+lN#Y!kqYztx@b4Ir zs0I~|L@km6Kf+e+Xr6bV27EJJBnTABnwQ~li{v&yz%eUu2acU~IM?x-(r~^))CriK zMNG~L!h}|U8UiqKOo8zKlo|!1jzJ(eKyF4Q@K_i)wzDF+C>NXVRX{>Ja1+6KvvZGuS6NiTL`CgXcps< zX%Sgej_{Ew1kql&4Pxi4mLA_6Oha#J{`A=>RJTP|$OM;0$04hmLizQET^o1Aud?ko8*ODWWba=4V+cEofL zRU+6dXugmRK6M})oZ;9~g0sm5ncg6K3E-Dc`#@sou&8VU|Hu%&z~Y6EEWmnkxXXp~ zP|}d%RZhToN&pG(s5B#&-PW^s(Y)*V^zzgD63oBjHC+fSj&8qvC? zx>DU%ZC&@4mabOpV6z1iuW>>`z)1|Tv5k!(4vxVX8*JiW-U~72VG|y~HpzPqU_*Z1 zu+95_-&yXtOO;AJ3{HOU&7!Kh_w48UzVq$pS@*Pezi;ovcmCg>y>$9bJFoudLx1~q zZ@>JXp7oQ~OZ{Jd?e~0t=iM*(h2u|txVU!TORkK3=|6w;-v9LJ=REffx2+%h+(&-o zzW?}|-~7*~K3n?C8{YD)|fjZi-&*Z(=T}R z{Ig$t^>2RL)l0wd7jJ#@kG!?~J8$`?r<{NDQUA@}FO=W=OV7RkRUdrwcYoJEy7~>D z{HDKn{|Eo%?C*W__ildVi^I!nAdmH~??L~i(efimJ=1<*A-~Cey zZ~xA}@!GdOG&S?C^56ZXzx$T|u>AhoZ#?)%|LsNpq5gl|dh$1~{fqDUz3>0t4}RCT zf9%!o{^U2;fA#)vDg5t0`m7JHyyfbvKJ;Gi6Q^GJ%|H2u>VJFDM}O^*6of zx%Zto{;6-g?@M3)!QoSLU-{B=e&|ap=hpvfeCpxZ{3lP9jxBtmy#B8Bdp`4e_h4bUiyb0``$0y z_Z`3XE% zc-4=*y!x8e?|aXW{@kPg`0suCWbQ`kGhWSo(Oae$etYIQU;V20r=Qq)#i#!2>{p)k z#@b)LXX@C`d*!dr-}9mmzwWco{nNEKRX+U6p>Ll5AHV;LZ+_vo{n=X=|H+45bN>TB z_U`(_?LT?+U;oG_Ui#~~p)Y*qbI<*)mDl{hcQ>wF`g?EsJOA~Mzx7`%{KT?=S@;B|g^Yafp`PAw|U;M*sV&##UA@ozl)$@l*9$6mep)t~y(=Ei;@e`sIK8x9<7HANcBH|NZ6D*D@cR``D-N|NhsWy?*uG&DWm!p#O=*>DOMIz5USG zw|?depLx@}e(~*VuX)SAdEk#;_badbzw>Ni?H_C5dG zgPq%7edX*2FQ5Lx_P_k0Tif6Me=fc6XWsuUFMjV=mY($oAN}$#|AX}tKd^J(4}V~K zzp(uJ=YM|sZ@ukZ@2&sX+urw%((kP7Uw-pbZ~fs<-t+G354`Ce)ptJo-g6hPy#Dq7 zz4?m!|DXH*_zib<{?4NPmw*1ok3amj`=9^f-#PiKo$}kf`=0ov7ug?r^{qeu(CVKR ztoQxa-t8~$e)muP+{^#)!!P*2o!UDezuNhk&wcUB)i?h8cl_b!Klr@Y{ODKy$@Tj? zAA8%+|MciD9^CrEbNGrR^@1A>}eUJEO zN`C&07tS;rf!hf3EBh_ic3Q2Px8Vd{vvK04v*Y+1ZLbx4%ehu7f61x4-+sky*PUIr zoS$`DwdVdk8_k9vWNzHp@dEe8jk?pqbH8MHcD)pPXje*By|8)b4*e~yW(y~F+(xC@ z&ez5|+vi{Q=wjifpSAJ0rMjqjWk3J$t(o(+jfZyCTc?>vpAB~-$j?4@adczs$`kZz zEtkucCcOeiV0op<%Br`Po4D7jx0>ys>tq~1^Q3*9J~r%|^by!C`tSn#7JV+c_7Z*W zxOPCF=iP(NcC(&7$O+l(6)IT|RLM%CQgHox{@T^;^Sj#*KSsk+MY)rTy-YPO*c_nC-*PsXU>@Gut;s()CCvaisvkN<+jt_ z&re^waQ^hk%br=D7i>nCUv_J}@zb44&C}OsxTJuF;Tu-RH)J+ZMfj` zT3%{a+!;Xt{BX!Qd$_RCtks;B?^?&ksF@Ni+?_kyg|=I7?zrb_HSPjKFkd>Ais*Sb zcfga>(*Qb(p|(oHdSA9n=tnH}iqpmr!xfxx3(dw%m8Y?d1>53k!-ZDc1>Fc{-A$)c z3-khTOEs@mZUO*ioxq_9)2EdU+`ZsLtLAu(Y;Hm?le_0`%rxtD?7Wrr8ot{OR!}f& z9~-l&zozVO+ubjfO4&Kk$aZ#QWHnoE)+(S=Uf_UG;!6j>-mZBK7vKG$?QOe3wcYG& zRkLdZkaNyP)!N34bc zCIIyBS#x%`?QRwuHWrtt_*?}8E7#oOu~EC(xX@^If?^Bbt~h?+7Ps*2qPsua+-($_ zw%=)CHGN<1zp8rOX*aya7Fs~hIeRWBRSFevXZ_xxisxe;_lud$n!7h~?+|_DD_$Gu z0jMfwhP!TQ(k%@0SoQ*1x2?8U$xdb4&E0G<3y|v68`%R@b+_%biW&NgCtLJ=Y_x@j z*X`x7)4JEl?|PM>+OPfbGpAd7=wVbhj<(Qf-Pu!>$3{nAf+sD2D9x_VV8_rFk0kt~ zn=Uuo*zSDWsdycq8o7YA!M6BV0KXs}14g|%C>rLuZcm3Zk+YBkL5VF7i#CdXPZp@l zQFC~saxI2hneM$qLs+_ITMs&@0u%^Cn{6LJ2CR2#!5FRU5dF&g-fcHlIM{DBx3JRH z{Q|u$?CmYrom%Z6b4;9O4j@Za-*vsMYS6m@@xDNqkXnD!se3hmiG&9+T4Mm>yN%qOvuG?D0HCqQ=y*t^xp zgKMc{8#jPCZkwtNw7*#FQ@q$bN>2bk786@GTKB0L4P|;8hy0c!MtbO8hTt=F^!=pX z1rQ9li*=mAojOlP)H~wtafg~Rlxxk6ZGN;{#p32M>kWWnqbad`Xo$BLCrEp<*4!;- z+#0C1@A0ac==#uU!q%aouw>P%RNMwVJi!0{&KB?cklJ{{TG6o}7W%hBG>z53I4(Fa zP7Q5(wOS0EQ<0AbHd3t8n8o1A&c^l@PJ>1zy}AHk#@<3$R%}KHF#RCLY=OrlD$_{9 zHM)xi`pmgsmx5XWkDkWF?`E_DZ$b1yNBw5at3-*9+$~XxmitV&Ok&fRpOTx#G#->$ z0AMGe(_Q_1IurZjfeG)YQ)mpjUM2%zhnv1 zk5%UFn90FV6g?)!XEZ|s0v?K4_dEK-Gj*J~r8D%GW|l<#*whb0*~rAl;}aj0BRI(v zX+RX6y#np4G zHw#ysw!=L3noU0+>ofybe7#IkgIihf{6IXo+TcURd^7`5T)bQ%vb6^B(W`+iwWgcR zn(H($7TtB#bHR!E8xu{4ALHhalje_8=8x0nk28HcCT`J*bbFx=2oMci^D1t$ zZ}p1ngYAygpY?W7t-VZ#wrEB^nRWcCD<3Tr8`-`FAsx!enkXiooo}|i+eEo)eLIG- zBxu$n9h06$w6xf4xPf<@wVu-~QlzxCX0OUE-zuo(-ztdp*eZyL0e(9ITxK8+5Zz&H zs~40O!QXINF4PDlyg}p;GP2iT^#MJFz5|M=nOf6#V?QlyKa@tYnjI-hx)suI8Ja3f z>k}aF&XLB1V9y10x3U-vG(wH6-68b?>%2nkgi!DFtpS8y?O>|$J|(TGmugTFRXQ84 zWtHtcpiKGrUP0hVq2@NWf~q~5n^*vPVM?{MhV(p1g%AMz%MK}D8U;XK&}1syb!xo4W=Ahrjo3!;C zT?;!W_b_eZJ5D=OmWr}^X%7Uh?LuGV2j?14DUhH!--a5V36WfGue68du0fAf5jwPq zrXsEdzlpM&rO8d9zMI-FukqI)7Y5-Kx*SIO)xr<9asr5(-L`n}f0 z(W35+jO-PXh*7R}+F2W5DUqU2yz?RPb9v4_MF``!9eb}pF%mH3f9EqTpL<6 zQr%_m+=)KC+A<&E^vLGA^@OI>t`|T9TUXl6mb1mMq_wBzQnbN%FVy77$T2if^)>@& z5U0k9`LT&%ix!iPU;rUhs~PWjscd^^0R=F^Y}3i@O_sL#7cd?g0Vvsz=~ZPLjpR2x zz>!JLtoI9&~kmM$(*q|u9OrWo_dbL+7lgjv902MAo z64%OBpgqnr*~$}_1m!Wn28#Jgg^7%{2NgHAc@A5to~00CUY>gC*qGeqGGjG*-YaYb z?HbmSU{RhAC$c#d9kurobDPkMlT*wa*D3_fh32l?o&g7E(L9Nh8_O1xEf~YJZWbi7 zkTcn$@je!LuE6d0}%_%?ZEDCfU;E}X&1|EfCg0*)|qT8YrNb$ z05?PjW~BZc>W5Yt-&WR>@YJ()^EGbX0v`*mio5WwWz&2Z2TKgg)$Lt~H9W zYxZ!{o<*6>*^mDdii&isZ!Nki*Zm!$m& zWMyHsqs}!XtUDw>)m+%@Smr|0LIZHHj!wl+t&kzT=G@+A9&=OJy@iR$N_U6I6`;>+ zSZw;+&)G4z4{LSQ9<#^6?21MpF~K<7Y1eWFsx7dpA~N_V3(FlJ9KMg0-IT%tyU?78qN@X3KDYM6ne>PoYq72~6cYmqHzuwjc-#VbpRIS8&B`o>2I} zrU~5(>$qMflv3;Vxv=?o3+XDdn-EaF&3$riAvKb=m5{9j91Kg%-?I zstNrVTN7FJd1Rf`fEFN74a4SmL{-(mhRWI&Db`5sI3KPl8HM$>y-Y2^o_;1RkC2Jy zy*;;bo{fW3Su(wn9ongFSopqE&Ezv<4@38sW4m$xosaAr?}i@$p)fdT&ZH4gK_OXw zXT#y@p{79?H6^UoY3+n+PdTSX^SYqgQ?Payvtthl+c7Cqc@inwPNRyU8qEToBAR#t zJ*@y9i|Rq|9h%I5YA`&RvxzA!Q#v28i)cfn3ZKk8$lb9bba0RykC~(iaxR2hA@_LG z3lyqh0|!G<#_)0WP%yUKDZ}0bfdS2wn=nViVlB(Gg%$lIn>#9?X^C>##wt62$pboc1<6QG^C*mVOlFj|JMNti3vHZ^xC) zs|)r@2Dvo2d8#K4U|FQ>EV=HUJb;Fg0Z{H$)6T7M?^_}pK zLHDX!qf;+K)bC#n=gqiL2Qu?6l<0?7A|0LTh&;4_B)3R45A14oplOFARTEs=5r-^W z!YH0reb%H6(gK~f#zZ#Yy9PetkYQ7$QKMGi9JV*HZcw364eGT4`Z2nxxHYd1_mB2~ zbx#|09i@ROG$6eU*uX=n4T!ZJVEEFOYBVBu_pa&^hdcPXnINrdN+xiS$?Ku79caX? zhltsgG|o89>D&)F0*#wHp1W(5VkI#nZBMqq*7URCUSb3$C96dHY}f#uChX#(D$I1+ zKD79Q6`pRxjN%68L8RfbRSk@4&~xeGOVR@HJ*~RwQxmGLl$s9>P!n=;O3l*;QmiPK zx;)C**+PepdcTg?2F#JL!>`cEWt9uup4}}&vscXSz(?L|RNb~0 zu+-D3xqflkg4`uYY_X2-kUx90MdJvY%G1w*+LLF_Y|4pRF7V{!I45++bUN`2W`R~- zOt!}h0 z4{as7OwtMw*4kkAXOP~3lv-)^NI{*(U8ph70t9E3_IXZOCu<`k9_)=u1Jh@$IQ@hv zaj+p8#c5B^*U7WL}Vyr0o?WGrMw#bI;7qMGT z<#Gc~+J;Ka1uYa$ZSu+@d-KMEvk&=#jv9m&5G)o?5u=QEmt{-=Ua1Ew(4A6IXlG!p zZWX(3gB|e5P;3*gPMV8iRxsO*U|zV1?~~3|`KAjwpSXo_G}{Qc%SZ|7y)N9>xy*Ip z`kg&9I-0e`)=j(n%|>MbK@SV$yNQ?%J|1V?jV2;~;5AQ=L{YdF6uWTPWzUjjIHAB7 zh35H?4gIWiEr~qVWf@3Nor{l+o8{xL9J1Yta-#&W%T)(-lSTm>qDCa^f{A<({~Ap? zmX`VK(+CQcCn>3mSz^^OVW^!i6++@~3m=jz6xmVqpvqR!3BXFBI4X{xwn}tJMn#uJJRlQnI0#i3USUGx1>b;Ho)sNgng^8E_^&?@N>1rgsFR7S zU;67M z?J9Un?Im6~MHd?Z?4H_Rx%Tk;p&%ULLd@N3rDb~m0Cb$FJ+;sgMQ}{oB*dEFgF183 z&A=}XE03Vl0>b24aL#K4!j9PPY=C$}!s+@Q*eyxy396nC?+~~@mAX~Op97(7Gg+l) z-bRMP03j{yxD|FMokvtJm$8)U%BVF;Qk?9NI53BNWYHNhq;SD3xYW3nCC9-G=qt@3 zBRb3?+7xD83I)^+XTqVuuTP9j@L<+HXn4q$q%t9KP9jT)ZJH9Z7aGuSz@LkT!^F49 z9k$@#;}?{H@*va-T#e{gNwM01I42dW`%3IuvsYX8R$QlPuQtKT3L%cyk)uL~V=CCB zY}6VV(Gi|-AZmh-4@JMAz29X)EThXLr7T zOn4j?aYDk#pF=yCndZi~OW`=clQ(nWV6?cS4nyz>kUjN4I)e=a0%>#@ozTNc&)4Wy zKkKsw19r_^SEBO+d*QET(pv;82T9lH;J;womO&T`|8&ecGzmryjWeVTAE~UI-+dM}Q zNUSF=#iLtTj9x{Ad~zO585R*pAyTNdBNmThhx|E|EM#Fnm9SLI3e~ISS1 zBjB!G*MSFO?q4U!Z(%9n{7uH2LIJCofK|P2kqwINPh=^hPSL{E=R9m3k;NtW2L*!> zMTj@jvj>Sax=pyJAh=%WPl}Cra|EC?hUIF{O&1z^>PM0E(&`^k=!-UCIzq&E zIDPX>@k%*HkIRQXe_so{D=9Uo~a`bX3ot44zUwQ!?U-(owN=nzqO z2Gt%SPljIE0k^|QtH9fSHyg5UKORmz4xxt-H{Pmyhdjyv!>&U{DwI>Bg7ndrKfn=e zd4w9e`LM9=jEJlRj+b-EjAb-L=}+Ln+T7gq3(yZE667Gqokj4HM+$MAedQ8KW3+Bg zW`Gee)(`3X&s@4-4kD zleX+lQ`Qy@%|{2z1i!$9V*m#=CWpj;g}o1=b$r@0e)^c0n6l=zKr#>uNeWAqYalO}M>#!R zO|}OwTFCSIsjO~iS?dbLg>FUqbHbYq3&LR<#qW(|+tCeC<}ad3BgLY#Ny2s)vr<(5lt5WEE|rx6+>3O)f9Xq6 zJzW|ZO3rzRCw)p}E8@|UHME%FuN0O0gq4Q{3CSoAPl`iBXv1bH+$EVzDq*MpS)?n9 z;zmuKscf7NJ7T6D3yDE*Xw(b<9kw)fgP!1*gf;`#)bd6ht5tRmJq#tu(DulrA4BBK zFQMe5uw5wbpT(kSr|m|K6b}aKMHI`jU7CG=@DK3aR)zaS-)*tZFf0CSfT?3Yi|o z+sAk$%}b0a!wZw>8Z~skqR&K4{XSe;ITjT%@)AUhoFuzBHVtq@Ya4Avl63M?MEH(a z1)gR)*BV*>RKzkCBT~5cKIvI>(O|q=V--@zf9Qz&)au)@Xjm2DR@UuDLs}H&*GW`I znZY<|FqIVtNG!6EGN{?4AVQT$2X-!sCXa9|03VeD2nj*)4{>Y8TJvHQb}ryoU@iVsdS8s?kP8G%66eLAQhOt z2E$5p*>|76De3Tus|FtRFdj) za<5qK!e|SkNJKp>t7%?2J-ILrJCS^LUC3!VNbqT5jns*^({|dB`m~G}m4?;olEZH3 z26R}+<5+>!^6|JwyAzZU)(xNi;>j~`Cq?6(dhyJuQJ7VcCA|3XnbCtV49(+1i{s^_ zD|Lq4o#lM?OP4w29C(?Bbslg;BCh95maGv^CSNOddRyqT=HX)L4i>{$`+-Uv1(Nm&Kk}@Po#`fC|Fb-rY}O*A`W$-=ZL%}g^PG8wI+#< zvFeEHGdg~RJEo|QhKrzm+r#w%xMRSD;a~6AmWcf<(?_f1)5jKP5NwvVl*5va6Irvn zh@rUDMD)D8xS}|BP}-+T-45=kaNtSr2gP9<85VZ0$_I^2k9?RZYx%ZKynTm6b@dj- z7hrousHAwp9#6%UgVHXw+9`EC%6V6mZn*pdQ8a)eIO6gzW&V$vgq!lE@DrPn`i81b z76PFJ4qoKXTZJ*(+1PMf0lQRZxzQ63!r7e#6V)^S&^-&OM>-zFdloNy^(-6e68i|x zclafbvR`q099)jZae%l1azb9SW!5TvwSv7Ia|**vAUaSNZtUR?%00Ii{kiJd$jrgT zdCjdZ5=9gVo3Tmkx-Ne3kxYzI@jP4v$+uS;)43??*3Kbp3h}5)8Hi9_BpZpeP5rN4 zw;xYFylM}-(Wl5axq8ihH2RS0yRO}dm8Hiw;h~XuoOFRAG^pA;_PpJ;oe39_RDlCG zxw3M`1E~=YDPab}I4rA*Of>SwgjkSMm6K{vGB;`kK})<|Txq9gr5#m{v%qmhy|!F+ zzGu~WgI(3VGPA=K+dV6`3tTGfG2Ab%>GZ7Wh;)m*wK+-vwALm6atO2Pwl!4gg7CU(%m z4mr$TX!xB?&~y)YKJCEqYr~zYB7Yo`H14pEBrcwyyDKy)zjH^lh2(9BzLGAVo^AQA zF`g)4^bv<&vDKg`_}xxM;;SjhIbOj~FEok}06UgI+Sr||; zitKCJjqM3!T|{yua!B=}!{G)IEGgo(WVS|6!$jhR2I6EL0k?@7=9Y&n)Du=U+z_xZ zV1S5xvM6fLm%uDk4PeRH?NU{M*VH=qL$(_E1jeF?j3FztiD8+?St&PJp+uR6NwB(;y9aHKg(Y->Qu|%Xf0)F zq}aZL?Bv-K{C;x1xPBIAZ|wDn$#v;Uw|;L1|D2`tC+v__imF7E5r-Pq%XWZ%oP$ZO znZX!m4)oWm+D{CyHA%H3&L5mT(ZXP|SyA@v3G_%UtRJwivc|*=0k}lQ4kN?_c`sv{ z;9?a*H4ryk#*HK{JlA*J74qb#+*P>zM<(mEtUZAaH5dLSCq*4(=!l_nBcZqThYP&8R>AW_(+fyyF@1v%~4np@WTB18xW zH;K?l30R$jb+DFLolPuIbw*r4BqnQZ2L&#kJ6N>lOOqC5j+(atGR5nzwF7Uvoc$VTE={jBn2-^q0^gT&SG3~b~S92+UF0Hc%6tQkg9 z6sOIF>$IZ4nahNxb5_;P%49FuoV{r6$T9BNM2ne0qA{+3RV8u>b_C%7Vum0}`n$Nk zs)URlG9c?p39M)}Le8*UPBu4nY_xdd6&BOBsp3;7o;s16vR0qkeX5XOJDz*0V68g& z+fNnNy5sg4P~BCO;CByFnG?l{(o<6-FFF3e+Jm{Nr%p@_pYRF+GTLG;_KZt?0s|7M zJZT*1xk`6Hdq8pNvMA2)0Qz(V+cGhX&5`*&&z44M%=5H5ZMiya-oYKL3~o~r-O_P3 zFNKbs682g*ObhAfqB}cJ(jpEgvq@fX(I%42&{WLrkRXFrkt!5B@;L9Nz3G-z4snW} z10htjwy3ySDeEbjf(xuPTRH)FR>)$Jnh-G~<(DYdHmo7krfHo;YE5$rqp%v=$m9K_ z4Q~O)`GYkZ1lW>U%Bx1T9ns1Bf$}DgcsHNa{?e<}wS@Prx!~ba(W*m6qHKIcFQmI5 z|Ee$Ul7@ATMFCuAW$z%c76Sy!jd3sq-JKW=_>iwpF__0jCCm`OY}AN7c^u@;vUW?$ zs!$Aeb_?Vpg&@TJnUYCJ!0-ze?S!6vqSiZv2AP_1$J z4vQaN!)hO2vd8cQ=?!`m$g2Vqi`FfP7q=w4dxJ4!Mnq%iF@c}8bv#UJ&f|I$x>LFl z4xp7Aq-+NeKLZ7~ZZ??#p+}oN3aSP*ncSn}QD|E-+tN!N*A+&!0fO4jDrajF*x3JC zQ0`)&e$No!S!1v5%T|Kei&jJ#+%y6-7qPFt)z|O-Vv66rN+H>arI3uI5belJkmT_# zJ<+e3Z3%bxM66h1+CP z9rZ~(rN0rzBd*@EdLwSPcrn07=01O-1gLRcdLkUG$(!?SO4F1kS zoy>=>hcPw*9~HoYy!Wz5X-EWp={+()M3llTGW+gnEf8xo z;jzqDL>N@VYRBfFS4j*6_v}WpE=bM;lNI?Fc9D=t9T_6^0tF7uvYe2bf&qSKf+rfo zw=fAqf_{+JdXc1;6{KkZZi{3MS9Jab!Pm(Y3={)L_rmhT6mIv~NbnS-F;%H09!+4Q z#kpTVvWYJ7AsAm!KrD17D)KQge4N=cLZfAb--XC2`B+Y8630&Q4(LhriLew|fW&6t zGGb&^p&M|DSzenAFHTlc*)Yxu>ksiqk4}QCi>)3YvdTF74gPs17OLng-y8T(dzl!ex;YJ)CRccJy}2V{j22ga{9W(969E4I?svG8EFXH6L z=sAsvB#|E4hWmkD8JoNU^*0sU)>qbn59G`6RT3_37bXvRnB3@F zTQ6pOxhX-H5^~bCh(c_Dt3N@Ga{v|OIu7^C@U^82xI<}29MHP;>fa-37l;g zH=)+uOn?kfcBTb$Hm=u!h_#==mC2MUgwmWLu|)w$h2`K=R+38{-F#g|Hzy8uf^8=% zD23-PYq})+e&(U472hn#?@g>SYn(em<4pM+DZdTz@<@^iAyP>1u^|xKK=T4TK{bAR zaJMkwFri)dg!feFG6ZM(%-AXRs87kV6G?P~u4Phn2@6J!RdTShpX@yM?Hz(!qMXt1W>8y7&ms-`&k=H|ksm5>`!$e?cAut}>VQYNL?XDE^uZe(n`sfFsC?+UhcNtgLq-JF(cO|FS%Qd{ z;)|~HJD%^AYc5A7$mQ(=Ht-2@{fMt3$D(l(L&ReES|roay!5jc#cKWvNDJqlqt0;oQVqbhI_jaNqx5gHgBnfXLVhKD$9vcCLetZ)h= zCIYiSrY2Gq%K0m;XZDK7zz>Hqer>W-*>klZGiJO2`y-@PelwmufYYS9W4Ok?mt;u8 zMF?_N_l3t^gFW_o-Q8dz?^6xw$D~muW-q}#SK#5)m8`Lsoybdd?SsCbw<}=Fhd%&E zG_oJg)KpjH!t8bZYQ44#&Shxqx8x$u7gu9R_v5e8GNH&7ZOVHV`i@Lf#4K~y za!h1&^46;SD7h@DV2P7ayq^hnU56C9!<_M zp{?mP)n#(AApuiPm0fPY^6Np&fVURv%*(>Xa>0dHSEFM2rAZ&|Q4lJJQGdaO0i4ov zonxb`EQ>OL7V_<~Rc_%0BtaROZifE%;> zOU>)-uP3c-G>%KnOiP693rx>Ik_7qL^tL*<_SuHenK4_)Y&eZZ6SuUxoC%P0~_1wrRo8v^|Q=Qd?l5*1@pBFlyiQE-gJ3mIX= z$^dmKeVSj$%!ve%NK6-0+s)2a^(d<@g=s$&!W0M3+#i$~D!Du5jbMJLr)-nXWyZ}+_t8~P~n^B2kIdzLiB3(5YYwf7l-s{ z_EyG6DI-rNjI>JTXp)zWDalIxRHDDg#l|c;7O)>5OZ+FymnuI?{{&5{9C4XJ$i|1k zVeyb5DH;ruQI>S!NcS2F`7+tAD0bJQHrCqt2iR*oW*sW@+HxKZ2!oCfIrM%&0*8@6Wj-> zZiA#Yj38D1N>M_OhY3AULhiCiuR&GRu5pvJA;=g2ag`H*Al$ep2P;Mwjuv1M8?3+( zJ+iwRtC4&p9BcZ>^uYC0D8Mj{n!F4K3?;?zs1ei2pvN%P4A=q8kW7aMZ=bg*rFmmDrXcgv1dI;0EhsW_ z-CEKk85dXsBjJ3G{o=?$*nzbEq69BAeK%zZfP;OOo}v5t`x7>MBy|jz^^&|P2lKjf z2Pt|bOko-C+-bslcQ7YO+!u3LL2{B@G3L2o)pF#1s%OBzVmo0hm2&_0p|(*|%j+2z z_Hrjc>Ja;4=Kf_%Ng*cEMnq_#X4dU`ID80pxQ`Xt(#T?j zE)v30%o-?dfO)YnMPfxzi?dhoSL81uR)n%8F&D`Pa=a6xyv!(M=DJ{2^JX~2n9)I7 zA;IjxQXmFkD7!}s4+9A>s6IM&C7YP5s%Aunhp5CBS3m6fFar>x? zJ1?kAD!6jmDGWv4-Igi9MZG3$IY$95BNOM*OnM^kmF_fsG3Z(j;+OLnDCSIS!#SOf zhDuT;p2k}WgQ1x|4&e_B-Ws2a8F5V<2_`yMAg@-QJSlx%vyk%@u!%*7+ajk~1pAti z8>En#8FhlWL8yROVU@s)7A!BwGj|+El(1hp5Xy`v51)**j}Fm2YWXs5YTHH@Nczar z69SbK?MMU@Lc(G!DuWUaEL;%1pOVJ($Sf0qUo}jU4O5n1ic&JOqbQsiMfwk)6Fb5` zMJ&1T>}XSh0K)jKP{Y{Y5QTi)2(-OV4NC<9k2Wz~rj81_U?agU2ysn5i{PqOPLW+x zGvs@07{iLg?_^(OGzcwJFsZP;OdRdP6o%t8882QVdgJS;OhkXUjjfE#iE|#t;`N{D={X3si9`lV^iALUP5r!`{IjgUuX&luMIT3a`tE)Ww(e zT;XODfmakli3e>26pKbEh8N5xN|&3JeN+I?Ec+2H7kv0ryVh6^vH|a~(E;^eHv8S< zek*vqN8MDC{l*5uo$BD_vz(Ya7P-vNvMT&qvMD&8YEBEc4AbGo-zG9ry+L}637K@( zOl$PO#>U1(7DEF&F5JAalJXm#Hc40z&uNL&3yyIf?sg7_yUV0wILkIE6)dgtDT;9i zRa+;`l}{=lo)nQy|0LNm9+G~qDLJ{5FjW=xxZ4x5UyR@cw>`NYin18HiXyXWFneTk z2$R5phJ8CRa$?KQjyUz!g!%L=Kdl9khm-tpiynq;&ykdgd#SoDYZ+Pt0gZKp4-zo& z-dkmR53sxkfw(p$z`;Ut6g*nQqs|nrcFYya3hXIFd0F6a9eRPVqhTS=w#b&p@s6G} z(k~-k(JLW7932AAVbaJq;U~dB_LBGmm10LPMIPB zryy({ACoG8Oc}Zc2KPN1xtx%d2ZQy+V9SxgO4(ZsR;}w{KaC^hct;YBH;W{FQMXN)J&& zIcA9E+@$3cjv=H7AV&na4%nhYWocph;BqsT~hw(`_3ZG;YdW9DG5L3Kv@b3s!3o@0U9#V=4+{zxdwQFC-%wwk?UdxL&U>3Q_%~)cV5Cg*`f?0cK9kdyt7Q zT*3WuQY~b!SM$VwtOG!M&Km$Dgoq8c@mp}KeDLejk@TVhcBgPM7dz@~j!wQO;+QJ{-#i?=)-sOr9+(0t z7ZpNO78p`78xi#B8N?7AXmgft*Td9YcaI^r&0+xxQYo}qGRB0FuQqI}NDWZg=>=7okt z;Uyr9cAdsyB?Ri{$FfCE&nI)lvQnrN3A4C_dK(p-^#B1dhpLG3!!Cf-FZ3l6PP=QkQ3ov#fuNtS@Du@^4 zbaP6%y1+#swu)jfT9Bni0DSwoutr?-7mksbi~${>^l(l2{_8Vg!>G2OhE{)_&>>Cm z3D72%7kN;zz|ipRD(iuv0o+cT$uok8eKBamBr5rFzV0S5nfy2-v6-iL=YPiGvujig zJ%kUL+KKAGzG$>R!cBv03t9&()5x28ufvUO>-I1p02$J9Kvty*hm8OYM;l_}fiy5G z`xS7kG)c=Hjz-drBbyr5?VAb`ad$?F`ylcph6Uu%xPKW=pzOucM@qUcjq@o*ep2*+ zebe-MNfQBmm?;~DxP(ZWHzo2VUK7+uNi;n$wf^5PqcEJ6fME@5=;yxQJ8I8JO=THr(*7&1gOaFNn#-EhKRUsGCPX|x$Nu)o=09D$vjYq z4pp~Ksi2rIqV|*@F~+Lqa##(+CiygP=OXV_V2oZYTmuJh=xuoq+|e4>6F!3Lv;Z#w zu@R6J@EwC$S(>3%gKZB?kYy$0#_2^dN=;;823{J5gx(N*ll9~9izW;pW;i1uB+m$y z=YaeS=oR?0nxrKX%9Eu$U(q7sNVut^K*Zp-Z7|dEz}zGs-ir%oHv>ipMF0_4;lNoe zv*!B*AM4gj--?gfBszousvL>+_>nkU5YjA2VUf2@}`D)@okM_YhA)T+N=k9hAKg z@F*;(yt_an*X`Tky32iw-r%N3({KAC0*eZw61YOd_(GQsQSit!d*I!X*}Qu{?Ftc~0mz zVgRnULufD~w@}Hg5Q#Hl^$r^@xTtM;g>S5;CI^QuDP+rPk|LNys~1K}lCUSTM8pOl z5Y32cGOyla$k3p1*F)?N1L+fE*<_?xCCui{i>ovfxm$RVB^bmeDarU!2o*#_)NY@~ zZtsw5t%QAb<#4DjF+vx^xFwRNL_1M zAO4}X=X8f&@iJ%Ck96==gqfzS=#-&*y$t=0xFKWHMM`=keYf{q zJ8|Kd&37+JkEs2UN4I-5D3lAA1}F=K)B_1K1$I6R zZ50;XL~hJ;Edmp*9j{z%L)SQ!I1JE4pwcN4{Fe99&NXgAd#>AUPs35sqMY z?t~5Dz+h92Z#e~|oQEOn669orfl1Q9b0Uhv3jJueGo>yua7E-UtijAKG=Igc_{`c` z$5_njuUTZ7CXIkpRu!|6pCC6F@}im~MYkPu3LZR$gkNYydC*S7uRu7o!z+L`f3>}a z%L*J?fF+vixd5&U2-QM59Ei@8o7xAM-zw>NHKZU!jKEwC3ZzCLGH~Z6ZWWwX3v^(H z+CC6u8=bV7i?S!R;sn zTh>enCY*lEt?&e6UZ*pZMh3aX!zrtga6}-JgBTigW0!jhJU|j+2@;fyTxr-TGkFva zC_4>Z)IR5IR4r?D+FpVq>|{3%($i}g)Y8bvRja${xxlk^0qcd~?ljW1i{#Ktlu<{z z)kv$LBcb&O!{I$?o0v8-(l)5ZWcKtkh#V7gowwb6qyY{WZ3)Xoz%;Ke z@4Qi(UR_!Pg6}4f{^B7>e=&mew-l;860f#`q@_qo1AVS_(Iz4}@6})kDJ-AAeEr7C z+~X@Y$Mo_gub>!M{j$r~yiNeKA1O3ox}jPi%W5rW-4cqBL+vb%d%TyViqd-H7Ay8; zl-{&NwsVAtUZdl37Z4Hp1(5#6DS8^HICTBm}WXgAQA zvvtX-ySZGKni&;`(E%F6yGUaPsCw)INDj?Nl{;{-)}>=x=)$t*-O@#K@G<0?M`z|l za+8Q_7xA_ZtM$WWX3{4Km$-&g94ts9T@ZNHx2!yNsIm z+$bm*Ua7C7K}T##;>tm$&o;(vFWEQbk%1eAc1ui2Qv!u%n0>&^W4pe~nY=E51n$v| zKa8sv?MEb?!lTGdgSUtP1Hvuqk&<;$Pc_!&MUYI~F_2EfAE732J~<_8sbmp*kTP1_ zY$T4dSj$T|BpD5z=(qrOa$Gmm;Sj3{tg7U4z!6-0Dw-x%3D+ev&EcM@qMFsa@;%EjUw>fPP|pV$XVH^5ThcB&=bd zHt=BV=$Sk4`R)e|R|sd`+N!yH?#P33%d3~Jd3rohun0%p%3?~0=wwyRJvkS%`esjE?Mo<1g=+= zyRZi5bxH*?x4B6m!E6>@!QBU_NPL3$(TnHTP&YE7zjD*?#{!cdhj4@EpR$InWP$K9 zMM@x+wN4FZU-*V5LR2wMayQLYh29%t0Vps79M9|J< zanjG=h>Ry9nMvj=k&A0y2}@j@`A1{<}!5VMkGEFH%)9fDb+Scn(;fH;Z@S8tvF+? zr*Xi>NnyBuph|j_`My&a&gY8Q}NnH3W6*Wo1 zdiR}c98D5Urx($51kubD73)FF(@wZteKvI`x~U}GJ(+wH2IHSyeZ}P3gB)kC zwKd!tb|Bt+sC>zut$CAB8njVJ&uYrjUrFbEcYraSezb?GLJ5JXrh-&qL4#T3ZlJfE(OgrVwo7v z)R-hvZN71#3Rd|@g<4x%BRk-n30qxgECd&(^jp15HlTGDtO*Hxg1ttek2n+%$nI}LWBON*MDboQ=D!YgsRky4OrpT!;3&mQdL2BqRG)nkdhad7e+*b|- z0pI4!p?@S^vDdO(r${F-AX1)8PQ6ctT$)J+RtzWBso49OP#+(&+d5cc-YD$)3~L)o zE4ET%l(f2G2GCb4X{7_UNGlys8vNwB0Lptfak(%BBV80?UK`ta4@T|v;qEM0`~rLi zfglLnmj}@OIJ)4TDscZh(dcPG={Asm|C zol1L$*l2opI_(_-rRm+7$U6i~P|jzp_9#|cBtv)vcY^SlB!i*#$)q+wZkD+5M?EAY zLAsV1R^b+{w(UTJJK(OjMN>HWV7WG&V`Kn!kI{*@e|@8iIJC#s@Ty)z5ilN|m#V1( z$0HLAtIF5Wy6mysj>MNos<+hzmBL>P_sFQ5+OtMXdcVDcFVi9WMtfas$9GGPER1j; zbX>Z|GchWWwU04qOw3%kA#nE`pLL8o|LGf=5pNjPbnNg%TJPT&%ED)NWXnL+`^o>no z({M3w+}bVEX)V%bsjN#fT?@G{XN||_v|)q@*P6H#jo?!y)wyjpvdXHasNG9zZ~EvV z(o8#+H5YAYPGvj0b7xz9og7t}s~2b-VAFkawkT#dadmv&!&G7uOwDGL$dOGfNg{O# zG03(DP$*t1Z~^+hfW*oiCCnH0s%5dxX(~lgA6Z*S_3WP1-avhM3vxUO5bgggc-uf2 zk+mfQ+ES1kl~bbFl_}PW6zeaBG=^^nHsyB>yjC({xag08N)>`jtB5RsNF4;tdR}H; zYBcA``pWYW>4+ppJqMHPN_0zB%zZe<%wgzC2|qglmySInIk|S}I&?9QPwh5~)j-*t zWP+kTxx{^IhCmqrsPKk;!~`m-c=jSU4TKWIr={d?W}2cTR7g~4rvp#2OASthb$Yv( zF+9fWa4>eVF0rNpR$%&7)QBj4%zEuG6U!w-@Vz{BMchzkRY23gQWu-Hl!A&^q~24t znV@hmMa4)bbb&S`xx9KNqLC70 zYReEFAF{x`+69)_~%tJ=8#1TUxU|LcvoFF~s<2$&r zA%2XITE(1>%ylIoOx^G)D6v41I$*7WyjEMJ8@LcrBuAMS4MpIYB;B6FIuJR+8BSi# zOK?Ic6bkKvZjbI$@l)t>Ki1@mxwvACD|nM=q28{dz75Adj)X^e&(Egxk;Z6g3 Date: Tue, 10 Mar 2026 14:40:08 +0000 Subject: [PATCH 069/135] impr: prometheus handling and fix POST tx endpoint --- rebar.config | 7 ++- src/dev_arweave.erl | 9 +-- src/dev_codec_tx.erl | 1 - src/dev_copycat_arweave.erl | 1 - src/hb_client.erl | 5 -- src/hb_event.erl | 54 ++++------------- src/hb_http.erl | 43 +++++++------- src/hb_http_client.erl | 112 ++++++++++-------------------------- src/hb_prometheus.erl | 60 +++++++++++++++---- src/hb_store_arweave.erl | 25 +++----- src/hb_store_lmdb.erl | 8 ++- 11 files changed, 132 insertions(+), 193 deletions(-) diff --git a/rebar.config b/rebar.config index 6d50fc2ee..4c583836a 100644 --- a/rebar.config +++ b/rebar.config @@ -1,8 +1,8 @@ {erl_opts, [debug_info, {d, 'COWBOY_QUICER', 1}, {d, 'GUN_QUICER', 1}]}. -{plugins, [pc, rebar3_rustler, rebar_edown_plugin]}. +{plugins, [pc, rebar3_rustler, rebar_edown_plugin, {rebar3_eunit_start, {git, "https://github.com/permaweb/rebar3_eunit_start.git", {ref, "04ec53fea187039770db0d4459b7aeb01a9021af"}}}]}. % Increase `scale_timeouts` when running on a slower machine. -{eunit_opts, [verbose, {scale_timeouts, 10}]}. +{eunit_opts, [verbose, {scale_timeouts, 10}, {start_applications, [prometheus, hb]}]}. {profiles, [ {quiet, @@ -113,7 +113,8 @@ {provider_hooks, [ {pre, [ - {compile, {cargo, build}} + {compile, {cargo, build}}, + {eunit, {default, rebar3_eunit_start}} ]}, {post, [ {compile, {pc, compile}}, diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index f6983a296..893ad14ec 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -116,14 +116,11 @@ post_binary_ans104(SerializedTX, LogExtra, Opts) -> Res = hb_http:post( hb_opts:get(bundler_ans104, not_found, Opts), #{ - <<"path">> => <<"/tx">>, + <<"path">> => <<"/~bundler@1.0/tx&codec-device=ans104@1.0">>, <<"content-type">> => <<"application/octet-stream">>, <<"body">> => SerializedTX }, - Opts#{ - http_client => - hb_opts:get(bundler_ans104_http_client, httpc, Opts) - } + Opts ), to_message(<<"/tx">>, <<"POST">>, Res, LogExtra, Opts). @@ -1168,7 +1165,6 @@ setup_arweave_index_opts(TXIDs) -> arweave_index_ids => true, arweave_index_store => IndexStore }, - application:ensure_all_started(hb), % Either: Index the blocks containing the TXs... % lists:foreach( % fun(Block) -> ok = index_test_block(Block, Opts) end, @@ -1849,7 +1845,6 @@ reassemble_bundle2_test() -> %% reassembles the bundle and nested items. This is also useful tool %% debugging tool to check that a bundle is present in the weave. assert_bundle_tx(TXID) -> - application:ensure_all_started(hb), Opts = #{}, {ok, #{ <<"body">> := OffsetBody }} = hb_http:request( diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index ad31ace25..46f91bc93 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -1204,7 +1204,6 @@ real_no_data_tx_test() -> ). do_real_tx_verify(TXID, ExpectedIDs) -> - application:ensure_all_started([hb]), Opts = #{}, {ok, #{ <<"body">> := TXJSON }} = hb_http:request( #{ diff --git a/src/dev_copycat_arweave.erl b/src/dev_copycat_arweave.erl index 45e1e8476..13c9930a7 100644 --- a/src/dev_copycat_arweave.erl +++ b/src/dev_copycat_arweave.erl @@ -1000,7 +1000,6 @@ negative_from_index_test() -> ok. setup_index_opts() -> - application:ensure_all_started([hb]), TestStore = hb_test_utils:test_store(), StoreOpts = #{ <<"index-store">> => [TestStore] }, Store = [ diff --git a/src/hb_client.erl b/src/hb_client.erl index 3298ee2f7..e1b37e62b 100644 --- a/src/hb_client.erl +++ b/src/hb_client.erl @@ -128,7 +128,6 @@ upload(Msg, Opts, <<"tx@1.0">>) when is_map(Msg) -> %%% Tests upload_empty_raw_ans104_test() -> - application:ensure_all_started(hb), Serialized = ar_bundles:serialize( ar_bundles:sign_item(#tx{ data = <<"TEST">> @@ -140,7 +139,6 @@ upload_empty_raw_ans104_test() -> ?assertMatch({ok, _}, Result). upload_raw_ans104_test() -> - application:ensure_all_started(hb), Serialized = ar_bundles:serialize( ar_bundles:sign_item(#tx{ data = <<"TEST">>, @@ -153,7 +151,6 @@ upload_raw_ans104_test() -> ?assertMatch({ok, _}, Result). upload_raw_ans104_with_anchor_test() -> - application:ensure_all_started(hb), Serialized = ar_bundles:serialize( ar_bundles:sign_item(#tx{ data = <<"TEST">>, @@ -167,7 +164,6 @@ upload_raw_ans104_with_anchor_test() -> ?assertMatch({ok, _}, Result). upload_empty_message_test() -> - application:ensure_all_started(hb), Msg = #{ <<"data">> => <<"TEST">> }, Committed = hb_message:commit( @@ -180,7 +176,6 @@ upload_empty_message_test() -> ?assertMatch({ok, _}, Result). upload_single_layer_message_test() -> - application:ensure_all_started(hb), Msg = #{ <<"data">> => <<"TEST">>, <<"basic">> => <<"value">>, diff --git a/src/hb_event.erl b/src/hb_event.erl index df4b1eb62..17ad2c753 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -9,7 +9,6 @@ -define(OVERLOAD_QUEUE_LENGTH, 10000). -define(MAX_MEMORY, 1_000_000_000). % 1GB -define(MAX_EVENT_NAME_LENGTH, 100). --define(MAX_PROMETHEUS_WAIT, 300). % ~30s at 100ms intervals -ifdef(NO_EVENTS). log(_X) -> ok. @@ -160,41 +159,27 @@ raw_counters() -> %% @doc Find the event server, creating it if it doesn't exist. We cache the %% result in the process dictionary to avoid looking it up multiple times. find_event_server() -> - case erlang:get({event_server, ?MODULE}) of - {cached, Pid} -> - case is_process_alive(Pid) of - true -> Pid; - false -> - erlang:erase({event_server, ?MODULE}), - find_event_server() - end; + case hb_name:lookup(?MODULE) of undefined -> - PID = - case hb_name:lookup(?MODULE) of - Pid when is_pid(Pid) -> Pid; - undefined -> - NewServer = spawn(fun() -> server() end), - hb_name:register(?MODULE, NewServer), - NewServer - end, - erlang:put({event_server, ?MODULE}, {cached, PID}), - PID + NewServer = spawn(fun() -> server() end), + hb_name:register(?MODULE, NewServer), + NewServer; + Pid -> Pid end. server() -> - await_prometheus_started(), + hb_prometheus:ensure_started(), ensure_event_counter(), handle_events(). ensure_event_counter() -> - try prometheus_counter:declare( + hb_prometheus:declare( + counter, [ {name, <<"event">>}, {help, <<"AO-Core execution events">>}, {labels, [topic, event]} - ]) - catch _:_ -> ok - end. + ]). handle_events() -> receive @@ -233,29 +218,10 @@ handle_events() -> end; _ -> ignored end, - try - prometheus_counter:inc(<<"event">>, [TopicBin, EventName], Count) - catch _:_ -> - ensure_event_counter() - end, + hb_prometheus:inc(counter, <<"event">>, [TopicBin, EventName], Count), handle_events() end. -%% @doc Delay the event server until prometheus is started. Messages -%% accumulate in the mailbox and are processed once handle_events/0 starts. -%% Gives up after ?MAX_PROMETHEUS_WAIT attempts to avoid unbounded mailbox -%% growth if Prometheus never comes up. -await_prometheus_started() -> - await_prometheus_started(?MAX_PROMETHEUS_WAIT). -await_prometheus_started(0) -> ok; -await_prometheus_started(Remaining) -> - case application:get_application(prometheus) of - undefined -> - receive after 100 -> ok end, - await_prometheus_started(Remaining - 1); - _ -> ok - end. - parse_name(Name) when is_tuple(Name) -> parse_name(element(1, Name)); parse_name(Name) when is_atom(Name) -> diff --git a/src/hb_http.erl b/src/hb_http.erl index a6a497ad3..ea9ccdce8 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -226,7 +226,6 @@ request_response(Method, Peer, Path, Response, Duration, Opts) -> ) end. - %% @doc Convert an HTTP response to a message. outbound_result_to_message(<<"ans104@1.0">>, Status, Headers, Body, Opts) -> ?event(debug_http_outbound, @@ -264,6 +263,17 @@ outbound_result_to_message(<<"httpsig@1.0">>, Status, Headers, Body, Opts) -> { hb_http_client:response_status_to_atom(Status), http_response_to_httpsig(Status, Headers, Body, Opts) + }; +outbound_result_to_message(<<"json@1.0">>, Status, Headers, Body, Opts) -> + ?event(debug, {headers, Headers}), + { + hb_http_client:response_status_to_atom(Status), + hb_message:convert( + Body, + <<"structured@1.0">>, + <<"json@1.0">>, + Opts + ) }. %% @doc Convert a HTTP response to a httpsig message. @@ -1074,13 +1084,12 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> WithPeer = case hb_maps:get(<<"ao-peer-port">>, NormalBody, undefined, Opts) of undefined -> NormalBody; P2PPort -> - RealIP = real_ip(Req, Opts), Peer = <>, (hb_message:without_unless_signed(<<"ao-peer-port">>, NormalBody, Opts))#{ <<"ao-peer">> => Peer } end, - WithPrivIP = hb_private:set(NormalBody, <<"ip">>, RealIP, Opts), + WithPrivIP = hb_private:set(WithPeer, <<"ip">>, RealIP, Opts), % Add device from PrimMsg if present WithDevice = case maps:get(<<"device">>, PrimMsg, not_found) of not_found -> WithPrivIP; @@ -1139,24 +1148,16 @@ init_prometheus() -> record_request_metric(TotalDuration, ReplyDuration, StatusCode) -> spawn( fun() -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - try - prometheus_histogram:observe( - http_request_server_duration_seconds, - [StatusCode], - TotalDuration - ), - prometheus_histogram:observe( - http_request_server_reply_duration_seconds, - [StatusCode], - ReplyDuration - ) - catch _:_ -> - ok - end - end + hb_prometheus:observe( + TotalDuration, + http_request_server_duration_seconds, + [StatusCode] + ), + hb_prometheus:observe( + ReplyDuration, + http_request_server_reply_duration_seconds, + [StatusCode] + ) end ). diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index a029eeb5a..fe21c716a 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -428,7 +428,7 @@ handle_info({gun_up, PID, Protocol}, State) -> ?event(http_client, {gun_up, {protocol, Protocol}, {conn_key, ConnKey}}), [gen_server:reply(ReplyTo, {ok, PID}) || {ReplyTo, _} <- PendingRequests], ets:insert(?CONN_STATUS_ETS, {PID, connected, MonitorRef, ConnKey}), - inc_prometheus_gauge(outbound_connections), + hb_prometheus:inc(gauge, outbound_connections), {noreply, State}; [{PID, connected, _MonitorRef, ConnKey}] -> ?event(warning, @@ -458,7 +458,7 @@ handle_info({gun_error, PID, Reason}, State) -> {connecting, PendingRequests} -> reply_error(PendingRequests, Reason2); connected -> - dec_prometheus_gauge(outbound_connections), + hb_prometheus:dec(gauge, outbound_connections), ok end, gun:shutdown(PID), @@ -487,7 +487,7 @@ handle_info({gun_down, PID, Protocol, Reason, _KilledStreams}, State) -> {connecting, PendingRequests} -> reply_error(PendingRequests, Reason2); _ -> - dec_prometheus_gauge(outbound_connections), + hb_prometheus:dec(gauge,outbound_connections), ok end, gun:shutdown(PID), @@ -507,7 +507,7 @@ handle_info({'DOWN', _Ref, process, PID, Reason}, State) -> {connecting, PendingRequests} -> reply_error(PendingRequests, Reason); _ -> - dec_prometheus_gauge(outbound_connections), + hb_prometheus:dec(gauge, outbound_connections), ok end, {noreply, State} @@ -748,8 +748,7 @@ log(Type, Event, #{method := Method, peer := Peer, path := Path}, Reason, Opts) %% Metrics init_prometheus() -> - application:ensure_all_started([prometheus, prometheus_cowboy]), - prometheus_counter:declare([ + hb_prometheus:declare(counter, [ {name, gun_requests_total}, {labels, [http_method, status_class, category]}, { @@ -757,9 +756,9 @@ init_prometheus() -> "The total number of GUN requests." } ]), - prometheus_gauge:declare([{name, outbound_connections}, + hb_prometheus:declare(gauge, [{name, outbound_connections}, {help, "The current number of the open outbound network connections"}]), - prometheus_histogram:declare([ + hb_prometheus:declare(histogram, [ {name, http_request_duration_seconds}, {buckets, [0.01, 0.1, 0.5, 1, 5, 10, 30, 60]}, {labels, [http_method, status_class, category]}, @@ -770,7 +769,7 @@ init_prometheus() -> "throttling, etc...)" } ]), - prometheus_histogram:declare([ + hb_prometheus:declare(histogram, [ {name, http_client_get_chunk_duration_seconds}, {buckets, [0.1, 1, 10, 60]}, {labels, [status_class, peer]}, @@ -779,11 +778,11 @@ init_prometheus() -> "The total duration of an HTTP GET chunk request made to a peer." } ]), - prometheus_counter:declare([ + hb_prometheus:declare(counter, [ {name, http_client_downloaded_bytes_total}, {help, "The total amount of bytes requested via HTTP, per remote endpoint"} ]), - prometheus_counter:declare([ + hb_prometheus:declare(counter, [ {name, http_client_uploaded_bytes_total}, {help, "The total amount of bytes posted via HTTP, per remote endpoint"} ]), @@ -796,9 +795,8 @@ init_prometheus() -> record_duration(Details, Opts) -> spawn( fun() -> - % First, write to prometheus if it is enabled. Prometheus works - % only with strings as lists, so we encode the data before granting - % it. + % Prometheus works only with strings as lists, so we encode the + % data before granting it. GetFormat = fun (<<"request-category">>) -> @@ -806,24 +804,18 @@ record_duration(Details, Opts) -> (Key) -> hb_util:list(maps:get(Key, Details)) end, - case application:get_application(prometheus) of - undefined -> ok; - _ -> - try prometheus_histogram:observe( - http_request_duration_seconds, - lists:map( - GetFormat, - [ - <<"request-method">>, - <<"status-class">>, - <<"request-category">> - ] - ), - maps:get(<<"duration">>, Details) - ) - catch _:_ -> ok - end - end, + Labels = lists:map( + GetFormat, + [ + <<"request-method">>, + <<"status-class">>, + <<"request-category">> + ]), + hb_prometheus:observe( + maps:get(<<"duration">>, Details), + http_request_duration_seconds, + Labels + ), maybe_invoke_monitor( Details#{ <<"path">> => <<"duration">> }, Opts @@ -834,7 +826,9 @@ record_duration(Details, Opts) -> record_response_status(Method, Response) -> record_response_status(Method, Response, undefined). record_response_status(Method, Response, Path) -> - inc_prometheus_counter(gun_requests_total, + hb_prometheus:inc( + counter, + gun_requests_total, [ hb_util:list(method_to_bin(Method)), hb_util:list(get_status_class(Response)), @@ -843,55 +837,9 @@ record_response_status(Method, Response, Path) -> 1 ). -%% @doc Safe wrapper for prometheus_gauge:inc/2. -inc_prometheus_gauge(Name) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - try prometheus_gauge:inc(Name) - catch _:_ -> - try - init_prometheus(), - prometheus_gauge:inc(Name) - catch _:_ -> - ok - end - end - end. - -%% @doc Safe wrapper for prometheus_gauge:dec/2. -dec_prometheus_gauge(Name) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - try prometheus_gauge:dec(Name) - catch _:_ -> - try - init_prometheus(), - prometheus_gauge:dec(Name) - catch _:_ -> - ok - end - end - end. - -inc_prometheus_counter(Name, Labels, Value) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - try prometheus_counter:inc(Name, Labels, Value) - catch _:_ -> - try - init_prometheus(), - prometheus_counter:inc(Name, Labels, Value) - catch _:_ -> - ok - end - end - end. - download_metric(Data) -> - inc_prometheus_counter( + hb_prometheus:inc( + counter, http_client_downloaded_bytes_total, [], byte_size(Data) @@ -903,7 +851,7 @@ upload_metric(#{method := Method, body := Body}) when is_atom(Method) -> upload_metric(#{ method := <<"POST">>, body := Body}) -> upload_metric(Body); upload_metric(#{ method := <<"PUT">>, body := Body}) -> upload_metric(Body); upload_metric(Body) when is_binary(Body) -> - inc_prometheus_counter( + hb_prometheus:inc(counter, http_client_uploaded_bytes_total, [], byte_size(Body) diff --git a/src/hb_prometheus.erl b/src/hb_prometheus.erl index d69579b2e..11680c919 100644 --- a/src/hb_prometheus.erl +++ b/src/hb_prometheus.erl @@ -1,6 +1,7 @@ %%% @doc HyperBEAM wrapper for Prometheus metrics. -module(hb_prometheus). -export([ensure_started/0, declare/2, measure_and_report/2, measure_and_report/3]). +-export([observe/2, observe/3, inc/2, inc/3, inc/4, dec/2, dec/3, dec/4]). -define(STARTUP_RETRY_INTERVAL, 60). % seconds @@ -55,18 +56,57 @@ do_declare(Type, _Metric) -> throw({unsupported_metric_type, Type}). %% @doc Measure function duration and report metric, ensuring that the Prometheus %% application has been started first. If Prometheus is unavailable, the function %% is executed without measurement. -measure_and_report(Fun, Metric) -> +measure_and_report(Fun, Metric) when is_function(Fun) -> measure_and_report(Fun, Metric, []). -measure_and_report(Fun, Metric, Labels) -> +measure_and_report(Fun, Metric, Labels) when is_function(Fun) -> + Start = erlang:monotonic_time(), + try Fun() + after + DurationNative = erlang:monotonic_time() - Start, + observe(DurationNative, Metric, Labels) + end. + +observe(Duration, Metric) when is_integer(Duration) -> + observe(Duration, Metric, []). +observe(Duration, Metric, Labels) when is_integer(Duration) -> + case ensure_started() of + ok -> + try prometheus_histogram:observe(Metric, Labels, Duration) + catch _:_ -> ok + end; + _ -> + ok + end. + +inc(Type, Metrics) -> + inc(Type, Metrics, []). +inc(Type, Metrics, Labels) -> + inc(Type, Metrics, Labels, 1). +inc(Type, Metrics, Labels, Value) -> case ensure_started() of ok -> - Start = erlang:monotonic_time(), - try Fun() - after - DurationNative = erlang:monotonic_time() - Start, - try prometheus_histogram:observe(Metric, Labels, DurationNative) - catch _:_ -> ok - end + try do_inc(Type, Metrics, Labels, Value) + catch error:mfa_already_exists -> ok + end; + _ -> ok + end. +do_inc(counter, Name, Labels, Value) -> + prometheus_counter:inc(Name, Labels, Value); +do_inc(gauge, Name, Labels, Value) -> + prometheus_gauge:inc(Name, Labels, Value). + +dec(Type, Metrics) -> + dec(Type, Metrics, []). +dec(Type, Metrics, Labels) -> + dec(Type, Metrics, Labels, 1). +dec(Type, Metrics, Labels, Value) -> + case ensure_started() of + ok -> + try do_dec(Type, Metrics, Labels, Value) + catch error:mfa_already_exists -> ok end; - _ -> Fun() + _ -> ok end. + +do_dec(gauge, Name, Labels, Value) -> + prometheus_gauge:dec(Name, Labels, Value). diff --git a/src/hb_store_arweave.erl b/src/hb_store_arweave.erl index 6a9bf2bc4..2b1a5a023 100644 --- a/src/hb_store_arweave.erl +++ b/src/hb_store_arweave.erl @@ -254,20 +254,15 @@ write_offset( %% @doc Record the partition that data is found in when it is requested. record_partition_metric(Offset) when is_integer(Offset) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - try - Partition = Offset div ?PARTITION_SIZE, - prometheus_counter:inc( - hb_store_arweave_requests_partition, - [Partition], - 1 - ) - catch _:_ -> - ok - end - end. + spawn(fun() -> + Partition = Offset div ?PARTITION_SIZE, + hb_prometheus:inc( + counter, + hb_store_arweave_requests_partition, + [Partition], + 1 + ) + end). %% @doc Initialize the Prometheus metrics for the Arweave store. Executed on %% `start/1' of the store. @@ -304,7 +299,6 @@ init_prometheus() -> %%% Tests write_read_tx_test() -> - application:ensure_all_started(hb), Store = [hb_test_utils:test_store()], Opts = #{ <<"index-store">> => Store @@ -347,7 +341,6 @@ write_read_tx_test() -> %% @doc The L1 TX has bundle tags, but data is not a valid bundle. write_read_fake_bundle_tx_test() -> - application:ensure_all_started(hb), Store = [hb_test_utils:test_store()], Opts = #{ <<"index-store">> => Store diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index 59bff2f5a..e1f6c7311 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -613,9 +613,11 @@ reset(Opts) -> %% @doc Increment the hit metrics for the current store's name. name_hit_metrics(Name) -> - try prometheus_counter:inc(hb_store_lmdb_hit, [Name], 1) - catch _:_ -> ok - end. + hb_prometheus:inc( + counter, + hb_store_lmdb_hit, + [Name], + 1). init_prometheus() -> hb_prometheus:declare(histogram, [ From f748353e2cef4368d293631efdcbc1ce8ddeca84 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 10 Mar 2026 14:48:31 +0000 Subject: [PATCH 070/135] impr: use singleton pattern --- src/hb_event.erl | 8 +------- src/hb_name.erl | 8 ++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 17ad2c753..afda46f11 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -159,13 +159,7 @@ raw_counters() -> %% @doc Find the event server, creating it if it doesn't exist. We cache the %% result in the process dictionary to avoid looking it up multiple times. find_event_server() -> - case hb_name:lookup(?MODULE) of - undefined -> - NewServer = spawn(fun() -> server() end), - hb_name:register(?MODULE, NewServer), - NewServer; - Pid -> Pid - end. + hb_name:singleton(?MODULE, fun() -> server() end). server() -> hb_prometheus:ensure_started(), diff --git a/src/hb_name.erl b/src/hb_name.erl index 09f95edab..d3c14db26 100644 --- a/src/hb_name.erl +++ b/src/hb_name.erl @@ -158,6 +158,14 @@ atom_test() -> term_test() -> basic_test({term, os:timestamp()}). +singleton_returns_spawned_pid_test() -> + Name = {singleton, os:timestamp()}, + Pid = singleton(Name, fun() -> receive stop -> ok end end), + ?assertEqual(Pid, lookup(Name)), + ?assertNotEqual(self(), Pid), + Pid ! stop, + hb_name:unregister(Name). + concurrency_test() -> Name = {concurrent_test, os:timestamp()}, SuccessCount = length([R || R <- spawn_test_workers(Name), R =:= ok]), From 5da22e99ccc81444eaf8c9ca6eb200a71100c1e5 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 10 Mar 2026 15:10:01 +0000 Subject: [PATCH 071/135] fix: increase timer on as_load test to temporarly fix test --- src/hb_ao_test_vectors.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hb_ao_test_vectors.erl b/src/hb_ao_test_vectors.erl index 919750c53..dae90fff7 100644 --- a/src/hb_ao_test_vectors.erl +++ b/src/hb_ao_test_vectors.erl @@ -843,7 +843,8 @@ load_as_test(Opts) -> }, % There is a race condition where we write to the store and a % reset happens making not read the written value. - timer:sleep(10), + % Lower this number can still produce flaky test + timer:sleep(100), {ok, ID} = hb_cache:write(Msg, Opts), {ok, ReadMsg} = hb_cache:read(ID, Opts), ?assert(hb_message:match(Msg, ReadMsg, primary, Opts)), From 23b706f35fa7a9b440619c085b1440fe8d201207 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 10 Mar 2026 16:07:11 +0000 Subject: [PATCH 072/135] fix: dev_name test that was broken by providing host header now --- src/dev_name.erl | 4 ++-- src/hb_http.erl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dev_name.erl b/src/dev_name.erl index 9e4dab29e..05ea41c88 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -102,7 +102,7 @@ request(HookMsg, HookReq, Opts) -> %% @doc Takes a request-given host and the host value in the node message and %% returns only the name component of the host, if it is present. If no name is %% present, an empty binary is returned. -name_from_host(Host, no_host) -> +name_from_host(Host, RawNodeHost) when RawNodeHost =:= no_host; RawNodeHost =:= <<"localhost">> -> case hd(binary:split(Host, <<".">>)) of <<>> -> {error, <<"No name found in `Host`.">>}; Name -> {ok, Name} @@ -113,7 +113,7 @@ name_from_host(ReqHost, RawNodeHost) -> WithoutNodeHost = binary:replace( ReqHost, - maps:get(host, uri_string:parse(RawNodeHost)), + maps:get(host, NodeHost), <<>> ), name_from_host(WithoutNodeHost, no_host). diff --git a/src/hb_http.erl b/src/hb_http.erl index ea9ccdce8..d1fa35e3e 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -1095,7 +1095,7 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> not_found -> WithPrivIP; Device -> WithPrivIP#{<<"device">> => Device} end, - % Add host + % Add host (for ARNS requests) Host = cowboy_req:host(Req), WithDevice#{<<"host">> => Host}. From a333ce99e0208b619b2f7f4e1b11b4b50cf74f71 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 10 Mar 2026 16:42:46 +0000 Subject: [PATCH 073/135] impr: Add option to remote store to only read IDs --- src/hb_store_remote_node.erl | 71 +++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index 074d1a46c..3c58b3000 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -52,28 +52,34 @@ type(Opts = #{ <<"node">> := Node }, Key) -> %% @param Opts A map of options (including node configuration). %% @param Key The key to read. %% @returns {ok, Msg} on success or not_found if the key is missing. -read(Opts = #{ <<"node">> := Node }, Key) when ?IS_ID(Key) -> - ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), - HTTPRes = - hb_http:get( - Node, - #{ <<"path">> => <<"/~cache@1.0/read">>, <<"target">> => Key }, - Opts - ), - case HTTPRes of - {ok, Res} -> - % returning the whole response to get the test-key - {ok, Msg} = hb_message:with_only_committed(Res, Opts), - ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), - maybe_cache(Opts, Msg, [Key]), - {ok, Msg}; - {error, _Err} -> - ?event(store_remote_node, {read_not_found, {key, Key}}), +read(Opts = #{ <<"node">> := Node }, Key) -> + ReadOnlyIDs = maps:get(<<"only-ids">>, Opts, false), + %% Limit read to ID keys if `<<"only-ids">>` is set to true. + ShouldRead = (ReadOnlyIDs andalso ?IS_ID(Key)) orelse not ReadOnlyIDs, + case ShouldRead of + true -> + ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), + HTTPRes = + hb_http:get( + Node, + #{ <<"path">> => <<"/~cache@1.0/read">>, <<"target">> => Key }, + Opts + ), + case HTTPRes of + {ok, Res} -> + % returning the whole response to get the test-key + {ok, Msg} = hb_message:with_only_committed(Res, Opts), + ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), + maybe_cache(Opts, Msg, [Key]), + {ok, Msg}; + {error, _Err} -> + ?event(store_remote_node, {read_not_found, {key, Key}}), + not_found + end; + false -> + ?event(store_remote_node, {ignoring_non_id, {key, Key}}), not_found - end; -read(_, Key) -> - ?event(store_remote_node, {ignoring_non_id, {key, Key}}), - not_found. + end. %% @doc Cache the data if the cache is enabled. The `local-store' option may %% either be `false' or a store definition to use as the local cache. Additional @@ -230,4 +236,25 @@ read_test() -> #{ <<"store-module">> => hb_store_remote_node, <<"node">> => Node } ], {ok, RetrievedMsg} = hb_cache:read(ID, #{ store => RemoteStore }), - ?assertMatch(#{ <<"test-key">> := Rand }, hb_cache:ensure_all_loaded(RetrievedMsg)). \ No newline at end of file + ?assertMatch(#{ <<"test-key">> := Rand }, hb_cache:ensure_all_loaded(RetrievedMsg)). + +read_only_ids_test() -> + LocalStore = hb_test_utils:test_store(), + hb_store:reset(LocalStore), + {ok, ID} = + hb_cache:write( + <<"message">>, + #{ store => LocalStore } + ), + Node = + hb_http_server:start_node( + #{ + store => LocalStore + } + ), + RemoteStore = [ + #{ <<"store-module">> => hb_store_remote_node, + <<"node">> => Node, + <<"only-ids">> => true } + ], + ?assertEqual(not_found, hb_cache:read(ID, #{ store => RemoteStore })). From 11ba07e87bc381fca6c63ee5b5ea7d39e13a56cd Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 10 Mar 2026 17:50:19 +0000 Subject: [PATCH 074/135] fix: dev_blacklist test when starting a new node with existing priv_wallet --- src/dev_blacklist.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 5ca11388b..7c38c03bc 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -249,7 +249,9 @@ cache_table_name(Opts) -> %%% Tests setup_test_env() -> - Opts0 = #{ store => hb_test_utils:test_store(), priv_wallet => hb:wallet() }, + %% We need to create a new priv_wallet to avoid conflift when starting a + %% new node from an existing priv_wallet address. + Opts0 = #{ store => hb_test_utils:test_store(), priv_wallet => ar_wallet:new() }, Msg1 = hb_message:commit(#{ <<"body">> => <<"test-1">> }, Opts0), Msg2 = hb_message:commit(#{ <<"body">> => <<"test-2">> }, Opts0), Msg3 = hb_message:commit(#{ <<"body">> => <<"test-3">> }, Opts0), From a9b4e2a00ad2c93194d373ca50e30d67f4374133 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 10 Mar 2026 18:39:31 +0000 Subject: [PATCH 075/135] fix: dev_bundler node start, when we already started one --- src/dev_bundler.erl | 46 ++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index 36653396c..47be72d4c 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -483,7 +483,7 @@ nested_bundle_test() -> ClientOpts = #{}, NodeOpts2 = maps:merge(NodeOpts, #{ bundler_max_items => 3 }), Node = hb_http_server:start_node(NodeOpts2#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store() }), %% Upload 3 data items across 4 chunks. @@ -530,7 +530,7 @@ tx_error_test() -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1 }), @@ -561,7 +561,7 @@ unsigned_dataitem_test() -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), debug_print => false }), @@ -593,7 +593,7 @@ idle_test() -> ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ bundler_max_idle_time => 400, - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store() }), % Test posting each of the supported signature types @@ -651,7 +651,7 @@ dispatch_blocking_test() -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 3 }), @@ -705,7 +705,7 @@ recover_respects_max_items_test() -> % Use max_items of 3, so 10 items should dispatch as 3+3+3+1 MaxItems = 3, Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => MaxItems }, @@ -743,7 +743,7 @@ complete_task_sequence_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 2, retry_base_delay_ms => 100, @@ -791,7 +791,7 @@ recover_bundles_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store() }, hb_http_server:start_node(Opts), @@ -850,7 +850,7 @@ post_tx_price_failure_retry_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1, retry_base_delay_ms => 50, @@ -886,7 +886,7 @@ post_tx_anchor_failure_retry_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1, retry_base_delay_ms => 50, @@ -924,7 +924,7 @@ post_tx_post_failure_retry_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1, retry_base_delay_ms => 50, @@ -962,7 +962,7 @@ post_proof_failure_retry_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1, retry_base_delay_ms => 50, @@ -997,7 +997,7 @@ rapid_dispatch_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1, bundler_workers => 3 @@ -1035,7 +1035,7 @@ one_bundle_fails_others_continue_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1, retry_base_delay_ms => 100, @@ -1069,7 +1069,7 @@ parallel_task_execution_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1, bundler_workers => 5 @@ -1114,7 +1114,7 @@ exponential_backoff_timing_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1, retry_base_delay_ms => 100, @@ -1163,7 +1163,7 @@ independent_task_retry_counts_test() -> }), try Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1, retry_base_delay_ms => 100, @@ -1195,7 +1195,7 @@ invalid_item_test() -> try ClientOpts = #{}, TestOpts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store() }, Node = hb_http_server:start_node(TestOpts#{ @@ -1207,7 +1207,7 @@ invalid_item_test() -> data = <<"testdata">>, tags = [{<<"tag1">>, <<"value1">>}] }, - hb:wallet() + ar_wallet:new() ), % Tamper with the data after signing (this invalidates the signature) TamperedItem = Item#tx{data = <<"tampereddata">>}, @@ -1241,7 +1241,7 @@ cache_write_failure_test() -> data = <<"testdata">>, tags = [{<<"tag1">>, <<"value1">>}] }, - hb:wallet() + ar_wallet:new() ), StructuredItem = hb_message:convert( Item, <<"structured@1.0">>, <<"ans104@1.0">>, GoodOpts), @@ -1274,7 +1274,7 @@ test_bundle(Opts) -> ClientOpts = #{}, NodeOpts2 = maps:merge(NodeOpts, Opts), Node = hb_http_server:start_node(NodeOpts2#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store() }), %% Upload 3 data items across 4 chunks. @@ -1303,7 +1303,7 @@ test_api_error(Responses) -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1 }), @@ -1325,7 +1325,7 @@ test_api_error(Responses) -> end. new_data_item(Index, Size) -> - new_data_item(Index, Size, hb:wallet()). + new_data_item(Index, Size, ar_wallet:new()). new_structured_data_item(Index, Size, Opts) -> hb_message:convert( From 59a1fe9b82d8fee697fe5c66ffbffd3e7eb8752f Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 10 Mar 2026 18:44:32 +0000 Subject: [PATCH 076/135] impr: Testing documentation --- docs/misc/hacking-on-hyperbeam.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/misc/hacking-on-hyperbeam.md b/docs/misc/hacking-on-hyperbeam.md index 6df0f5a9d..307efe1a5 100644 --- a/docs/misc/hacking-on-hyperbeam.md +++ b/docs/misc/hacking-on-hyperbeam.md @@ -103,4 +103,15 @@ since the last invocation. 3. Open the svg file in browser. -Happy hacking! \ No newline at end of file +Happy hacking! + +## Adding testing + +Here is a helpful list of common mistakes when writing tests. + +If you need to start a new node, make sure a new private key is defined using +`#{priv_wallet => ar_wallet:new()}`. If `hb:wallet()` is used, it will conflict +with the default HyperBEAM server that is started before eunit run. + +Avoid pattern match a list of commitments, since we cannot guarantee the order. +This will case tests to be flaky. From 584d93b8af5f8b30022b2c0a052cd9708145aec8 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 10 Mar 2026 15:01:49 -0400 Subject: [PATCH 077/135] Update hacking-on-hyperbeam.md --- docs/misc/hacking-on-hyperbeam.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/misc/hacking-on-hyperbeam.md b/docs/misc/hacking-on-hyperbeam.md index 307efe1a5..ebb69cbcd 100644 --- a/docs/misc/hacking-on-hyperbeam.md +++ b/docs/misc/hacking-on-hyperbeam.md @@ -103,15 +103,29 @@ since the last invocation. 3. Open the svg file in browser. -Happy hacking! - -## Adding testing +## Common testing pitfalls + +Here is a helpful list of common mistakes when writing tests: + +- If you need to start a new node, be sure to use a new private key unless you + have a specific reason to use an existing one. HyperBEAM HTTP servers are + registered using their wallet ID as their 'name', so re-use can cause issues. + You can get a new private key is defined using `#{ priv_wallet => ar_wallet:new() }`. +- Similarly, always be careful of your stores in your tests! Avoid using the + default stores, as this can lead to 'context leakage', where one part of your + test is unintentionally able to access data created/stored by a supposedly + different node in the environment. `hb_http_server:start_node/1` will generate + a new unique store for you by default, but avoid creating a named store unless + you need to (and know what you are doing). +- Always try to test your devices through the HTTP AO-Core API as well as through + the local `hb_ao:resolve/[2-3]` interfaces. Avoid direct `dev_name:key` calls + unless strictly necessary. The HTTP API is how users will interact with your + almost always system, and there can be subtle differences in how the interfaces + react. For example, the Erlang function call interface has no regard for how + keys are matched by AO-Core, so will mask any issues with the choice of which + device function to call to satisfy requests. -Here is a helpful list of common mistakes when writing tests. - -If you need to start a new node, make sure a new private key is defined using -`#{priv_wallet => ar_wallet:new()}`. If `hb:wallet()` is used, it will conflict -with the default HyperBEAM server that is started before eunit run. +Happy hacking! Avoid pattern match a list of commitments, since we cannot guarantee the order. This will case tests to be flaky. From c721f11be5c3adc296db59a08198ad225e13265f Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 11 Mar 2026 17:37:47 +0000 Subject: [PATCH 078/135] fix: tests with wallet --- src/dev_bundler_cache.erl | 13 +++++++------ src/dev_copycat_graphql.erl | 4 ++-- src/dev_query.erl | 4 ++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/dev_bundler_cache.erl b/src/dev_bundler_cache.erl index ff5dac963..5545fff0f 100644 --- a/src/dev_bundler_cache.erl +++ b/src/dev_bundler_cache.erl @@ -329,17 +329,18 @@ load_bundled_items_test() -> %% L4IItem1 (leaf) %% L4IItem2 (leaf) bundler_optimistic_cache_test() -> + Wallet = ar_wallet:new(), L3Item = ar_bundles:sign_item( #tx{ data = <<"l3item">>, tags = [{<<"idx">>, <<"1">>}] }, - hb:wallet() + Wallet ), L4Item1 = ar_bundles:sign_item( #tx{ data = <<"l4item1">>, tags = [{<<"idx">>, <<"2.1">>}] }, - hb:wallet() + Wallet ), L4Item2 = ar_bundles:sign_item( #tx{ data = <<"l4item2">>, tags = [{<<"idx">>, <<"2.2">>}] }, - hb:wallet() + Wallet ), % L3Bundle is itself a bundle wrapping the two L4 leaves. {undefined, L3BundlePayload} = ar_bundles:serialize_bundle( @@ -353,7 +354,7 @@ bundler_optimistic_cache_test() -> {<<"idx">>, <<"2">>} ] }, - hb:wallet() + Wallet ), {undefined, L2BundlePayload} = ar_bundles:serialize_bundle( list, [L3Item, L3Bundle], false), @@ -365,7 +366,7 @@ bundler_optimistic_cache_test() -> {<<"Bundle-Version">>, <<"2.0.0">>} ] }, - hb:wallet() + Wallet ), % Compute signed IDs for all items before posting. L2BundleID = hb_util:encode(ar_bundles:id(L2Bundle, signed)), @@ -375,7 +376,7 @@ bundler_optimistic_cache_test() -> L4Item2ID = hb_util:encode(ar_bundles:id(L4Item2, signed)), % Start a real node with LMDB and POST the serialized bundle wrapper over HTTP. Node = hb_http_server:start_node(#{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet, store => hb_test_utils:test_store(hb_store_lmdb) }), try diff --git a/src/dev_copycat_graphql.erl b/src/dev_copycat_graphql.erl index df8c9d47e..9f0752ec6 100644 --- a/src/dev_copycat_graphql.erl +++ b/src/dev_copycat_graphql.erl @@ -270,7 +270,7 @@ default_query(Parts) -> %% @doc Run node for testing run_test_node() -> Store = hb_test_utils:test_store(), - Opts = #{ store => Store, priv_wallet => hb:wallet() }, + Opts = #{ store => Store, priv_wallet => ar_wallet:new() }, Node = hb_http_server:start_node(Opts), {Node ,Opts}. %% @doc Basic test to test copycat device @@ -450,4 +450,4 @@ fetch_scheduler_location_test() -> ?assert(is_integer(Data)), ?assert(Data > 0), ?event({schedulers_indexed, Data}), - ok. + ok. \ No newline at end of file diff --git a/src/dev_query.erl b/src/dev_query.erl index 82c0bc77f..4dc43a09d 100644 --- a/src/dev_query.erl +++ b/src/dev_query.erl @@ -215,7 +215,7 @@ query_match_key(Path, Opts) -> %% @doc Return test options with a test store. test_setup() -> Store = hb_test_utils:test_store(), - Opts = #{ store => Store, priv_wallet => hb:wallet() }, + Opts = #{ store => Store, priv_wallet => ar_wallet:new() }, % Write a simple message. hb_cache:write( #{ @@ -365,4 +365,4 @@ http_test() -> Opts ), ?assertEqual(<<"binary-value">>, hb_maps:get(<<"basic">>, Msg, Opts)), - ok. + ok. \ No newline at end of file From 49344282f39a222523f8ba21f6f0d106960f2741 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 11 Mar 2026 19:15:12 +0000 Subject: [PATCH 079/135] fix: Revert changes, use httpc instead of gun in arns test --- src/dev_name.erl | 4 ++-- src/hb_http.erl | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/dev_name.erl b/src/dev_name.erl index 05ea41c88..c93e356e4 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -102,7 +102,7 @@ request(HookMsg, HookReq, Opts) -> %% @doc Takes a request-given host and the host value in the node message and %% returns only the name component of the host, if it is present. If no name is %% present, an empty binary is returned. -name_from_host(Host, RawNodeHost) when RawNodeHost =:= no_host; RawNodeHost =:= <<"localhost">> -> +name_from_host(Host, no_host) -> case hd(binary:split(Host, <<".">>)) of <<>> -> {error, <<"No name found in `Host`.">>}; Name -> {ok, Name} @@ -280,6 +280,6 @@ arns_host_resolution_test() -> <<"path">> => <<"content-type">>, <<"host">> => <<"draft-17_whitepaper">> }, - Opts + Opts#{http_client => httpc} ) ). \ No newline at end of file diff --git a/src/hb_http.erl b/src/hb_http.erl index d1fa35e3e..cef56fe7d 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -264,14 +264,14 @@ outbound_result_to_message(<<"httpsig@1.0">>, Status, Headers, Body, Opts) -> hb_http_client:response_status_to_atom(Status), http_response_to_httpsig(Status, Headers, Body, Opts) }; -outbound_result_to_message(<<"json@1.0">>, Status, Headers, Body, Opts) -> +outbound_result_to_message(Codec, Status, Headers, Body, Opts) -> ?event(debug, {headers, Headers}), { hb_http_client:response_status_to_atom(Status), hb_message:convert( Body, <<"structured@1.0">>, - <<"json@1.0">>, + Codec, Opts ) }. @@ -1091,13 +1091,10 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> end, WithPrivIP = hb_private:set(WithPeer, <<"ip">>, RealIP, Opts), % Add device from PrimMsg if present - WithDevice = case maps:get(<<"device">>, PrimMsg, not_found) of + case maps:get(<<"device">>, PrimMsg, not_found) of not_found -> WithPrivIP; Device -> WithPrivIP#{<<"device">> => Device} - end, - % Add host (for ARNS requests) - Host = cowboy_req:host(Req), - WithDevice#{<<"host">> => Host}. + end. %% @doc Determine the caller, honoring the `x-real-ip' header if present. real_ip(Req = #{ headers := RawHeaders }, Opts) -> From fe43eea3a57caed5f789f3306dedd4dcd8dd77ef Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 6 Mar 2026 11:58:21 -0500 Subject: [PATCH 080/135] impr: gun event logging --- src/hb_http_client.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index fe21c716a..d7a4f99ea 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -713,7 +713,7 @@ await_response(Args, Opts) -> record_response_status(Method, Response, Path), ?event(http_outbound, {gun_cancel, {path, Path}}), gun:cancel(PID, Ref), - log(warn, gun_await_process_down, Args, Response, Opts), + log(warning, gun_await_process_down, Args, timeout, Opts), Response; {error,{connection_error,{stream_closed, Message}}} = Response -> ?event(http_outbound, {gun_cancel, {path, Path}, {message, Message}}), @@ -721,18 +721,18 @@ await_response(Args, Opts) -> Response; {error, Reason} = Response when is_tuple(Reason) -> record_response_status(Method, Response, Path), - log(warn, gun_await_process_down, Args, Reason, Opts), + log(warning, gun_await_process_down, Args, Reason, Opts), Response; Response -> record_response_status(Method, Response, Path), - log(warn, gun_await_unknown, Args, Response, Opts), + log(warning, gun_await_unknown, Args, Response, Opts), Response end. %% @doc Debug `http` state logging. log(Type, Event, #{method := Method, peer := Peer, path := Path}, Reason, Opts) -> ?event( - debug_http, + Type, {gun_log, {type, Type}, {event, Event}, From dd383dd67917e9c63f2d2063a01ae8d23170db5e Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Fri, 6 Mar 2026 11:45:36 -0500 Subject: [PATCH 081/135] impr: prometheus management --- src/hb_event.erl | 2 +- src/hb_http_client_sup.erl | 1 + src/hb_prometheus.erl | 50 +++++++++++++++++--------------------- 3 files changed, 24 insertions(+), 29 deletions(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index afda46f11..93f44a9c6 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -268,4 +268,4 @@ benchmark_increment_test() -> ), hb_test_utils:benchmark_print(<<"Incremented">>, <<"events">>, Iterations), ?assert(Iterations >= 1000), - ok. + ok. \ No newline at end of file diff --git a/src/hb_http_client_sup.erl b/src/hb_http_client_sup.erl index 54c060610..39d08d006 100644 --- a/src/hb_http_client_sup.erl +++ b/src/hb_http_client_sup.erl @@ -13,6 +13,7 @@ -define(CHILD(I, Type, Opts), {I, {I, start_link, Opts}, permanent, ?SHUTDOWN_TIMEOUT, Type, [I]}). start_link(Opts) -> + hb_prometheus:ensure_started(), supervisor:start_link({local, ?MODULE}, ?MODULE, Opts). init(Opts) -> diff --git a/src/hb_prometheus.erl b/src/hb_prometheus.erl index 11680c919..9ff0be6bc 100644 --- a/src/hb_prometheus.erl +++ b/src/hb_prometheus.erl @@ -3,41 +3,35 @@ -export([ensure_started/0, declare/2, measure_and_report/2, measure_and_report/3]). -export([observe/2, observe/3, inc/2, inc/3, inc/4, dec/2, dec/3, dec/4]). --define(STARTUP_RETRY_INTERVAL, 60). % seconds - %% @doc Ensure the Prometheus application has been started. Caches startup %% failure with a timestamp to avoid repeated blocking ensure_all_started %% calls on hot paths, but retries after a cooldown period. ensure_started() -> - case application:get_application(prometheus) of - undefined -> - case persistent_term:get(hb_prometheus_start_failed, undefined) of - FailedAt when is_integer(FailedAt) -> - case erlang:monotonic_time(second) - FailedAt >= ?STARTUP_RETRY_INTERVAL of - true -> attempt_start(); - false -> {error, not_started} - end; - undefined -> - attempt_start() - end; - _ -> ok + case is_started() of + true -> ok; + false -> + application:ensure_all_started( + [prometheus, prometheus_cowboy, prometheus_ranch] + ), + wait_for_prometheus_started() end. -attempt_start() -> - case application:ensure_all_started( - [prometheus, prometheus_cowboy, prometheus_ranch] - ) of - {ok, _} -> - persistent_term:erase(hb_prometheus_start_failed), - ok; - {error, _} = Err -> - persistent_term:put( - hb_prometheus_start_failed, - erlang:monotonic_time(second) - ), - Err +%% @doc Lazy wait for prometheus to come up, after we have started the application. +wait_for_prometheus_started() -> + case is_started() of + true -> ok; + false -> + timer:sleep(1), + wait_for_prometheus_started() end. +%% @doc Check if prometheus has been started. +%% The application itself may return `ok` to Erlang before it is actually ready +%% for use, so we wait for the `ets` table to be created instead. +is_started() -> + Info = ets:info(prometheus_registry_table), + Info =/= undefined. + %% @doc Declare a new Prometheus metric in a replay-safe manner. declare(Type, Metric) -> case ensure_started() of @@ -109,4 +103,4 @@ dec(Type, Metrics, Labels, Value) -> end. do_dec(gauge, Name, Labels, Value) -> - prometheus_gauge:dec(Name, Labels, Value). + prometheus_gauge:dec(Name, Labels, Value). \ No newline at end of file From a13ac36cfbb96ebad7605981fb56d79019508faf Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 11 Mar 2026 21:09:28 +0000 Subject: [PATCH 082/135] fix: tests --- src/dev_query_test_vectors.erl | 32 ++++++++++++++++---------------- src/hb.erl | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/dev_query_test_vectors.erl b/src/dev_query_test_vectors.erl index 91f6d8dd9..0f6f37371 100644 --- a/src/dev_query_test_vectors.erl +++ b/src/dev_query_test_vectors.erl @@ -69,7 +69,7 @@ write_test_message_with_recipient(Recipient, Opts) -> simple_blocks_query_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -114,7 +114,7 @@ simple_blocks_query_test() -> block_by_height_query_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -165,7 +165,7 @@ block_by_height_query_test() -> simple_ans104_query_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -202,7 +202,7 @@ simple_ans104_query_test() -> Node, Query, #{ - <<"owners">> => [hb:address()] + <<"owners">> => [hb:address(Wallet)] }, Opts ), @@ -232,7 +232,7 @@ simple_ans104_query_test() -> transactions_query_tags_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -295,7 +295,7 @@ transactions_query_tags_test() -> transactions_query_owners_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -327,7 +327,7 @@ transactions_query_owners_test() -> Node, Query, #{ - <<"owners">> => [hb:address()] + <<"owners">> => [hb:address(Wallet)] }, Opts ), @@ -357,7 +357,7 @@ transactions_query_owners_test() -> transactions_query_recipients_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -422,7 +422,7 @@ transactions_query_recipients_test() -> transactions_query_ids_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -484,7 +484,7 @@ transactions_query_ids_test() -> transactions_query_combined_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -521,7 +521,7 @@ transactions_query_combined_test() -> Node, Query, #{ - <<"owners">> => [hb:address()], + <<"owners">> => [hb:address(Wallet)], <<"ids">> => [ExpectedID] }, Opts @@ -552,7 +552,7 @@ transactions_query_combined_test() -> transaction_query_by_id_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -602,7 +602,7 @@ transaction_query_by_id_test() -> transaction_query_full_test() -> Opts = #{ - priv_wallet => SenderKey = hb:wallet(), + priv_wallet => SenderKey = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -679,7 +679,7 @@ transaction_query_full_test() -> transaction_query_not_found_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Res = @@ -715,7 +715,7 @@ transaction_query_not_found_test() -> transaction_query_with_anchor_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -727,7 +727,7 @@ transaction_query_with_anchor_test() -> anchor = AnchorID = crypto:strong_rand_bytes(32), data = <<"test-data">> }, - hb:wallet() + Wallet ), <<"structured@1.0">>, <<"ans104@1.0">>, diff --git a/src/hb.erl b/src/hb.erl index c6cce5c75..a7b2cc041 100644 --- a/src/hb.erl +++ b/src/hb.erl @@ -91,7 +91,7 @@ -export([no_prod/3]). -export([read/1, read/2, debug_wait/4]). %%% Node wallet and address management: --export([address/0, wallet/0, wallet/1]). +-export([address/0, address/1, wallet/0, wallet/1]). -include("include/hb.hrl"). %% @doc Initialize system-wide settings for the hyperbeam node. From 0b9d76e5beac6097bb0ddea01ac3c64a2be3d15c Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 11 Mar 2026 22:11:33 +0000 Subject: [PATCH 083/135] fix: use gun with http1 to fix arns test --- src/dev_name.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/dev_name.erl b/src/dev_name.erl index c93e356e4..999f9bebb 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -280,6 +280,8 @@ arns_host_resolution_test() -> <<"path">> => <<"content-type">>, <<"host">> => <<"draft-17_whitepaper">> }, - Opts#{http_client => httpc} + %% Host header isn't added automatically if used HTTP2. + %% We need host header to resolve the ARNS name. + Opts#{http_client => gun, opts => #{protocol => http1} } ) ). \ No newline at end of file From 77870af45e8f97cde4c6eb95663e4962a58566a9 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 12 Mar 2026 18:31:30 +0000 Subject: [PATCH 084/135] fix: Add host header, fix tests --- src/dev_blacklist.erl | 46 +++++++++++++++++++++++------------------- src/dev_location.erl | 2 +- src/dev_name.erl | 6 ++---- src/dev_push.erl | 2 +- src/dev_whois.erl | 6 +++--- src/hb_http.erl | 6 ++++-- src/hb_http_server.erl | 2 +- src/hb_opts.erl | 6 +++--- test/config.flat | 2 +- test/config.json | 2 +- 10 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 7c38c03bc..33b17b529 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -27,28 +27,32 @@ %% @doc Hook handler: block requests that involve blacklisted IDs. request(_Base, HookReq, Opts) -> ?event({hook_req, HookReq}), - case is_match(HookReq, Opts) of - false -> - ?event(blacklist, {allowed, HookReq}, Opts), - {ok, HookReq}; - ID -> - ?event(blacklist, {blocked, ID}, Opts), - { - ok, - HookReq#{ - <<"body">> => - [#{ - <<"status">> => 451, - <<"reason">> => <<"content-policy">>, - <<"blocked-id">> => ID, + case hb_opts:get(blacklist_providers, false, Opts) of + false -> {ok, HookReq}; + _ -> + case is_match(HookReq, Opts) of + false -> + ?event(blacklist, {allowed, HookReq}, Opts), + {ok, HookReq}; + ID -> + ?event(blacklist, {blocked, ID}, Opts), + { + ok, + HookReq#{ <<"body">> => - << - "Requested message blocked by this node's ", - "content policy. Blocked ID: ", ID/binary - >> - }] - } - } + [#{ + <<"status">> => 451, + <<"reason">> => <<"content-policy">>, + <<"blocked-id">> => ID, + <<"body">> => + << + "Requested message blocked by this node's ", + "content policy. Blocked ID: ", ID/binary + >> + }] + } + } + end end. %% @doc Check if the message contains any blacklisted IDs. diff --git a/src/dev_location.erl b/src/dev_location.erl index 433a8afab..42ba3061b 100644 --- a/src/dev_location.erl +++ b/src/dev_location.erl @@ -199,7 +199,7 @@ default_url(Opts) -> case hb_opts:get(location_url, not_found, Opts) of not_found -> Port = hb_util:bin(hb_opts:get(port, 8734, Opts)), - Host = hb_opts:get(host, <<"localhost">>, Opts), + Host = hb_opts:get(node_host, <<"localhost">>, Opts), Protocol = hb_opts:get(protocol, http1, Opts), ProtoStr = case Protocol of diff --git a/src/dev_name.erl b/src/dev_name.erl index 999f9bebb..41cb8478b 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -75,7 +75,7 @@ request(HookMsg, HookReq, Opts) -> maybe {ok, Req} ?= hb_maps:find(<<"request">>, HookReq, Opts), {ok, Host} ?= hb_maps:find(<<"host">>, Req, Opts), - {ok, Name} ?= name_from_host(Host, hb_opts:get(host, no_host, Opts)), + {ok, Name} ?= name_from_host(Host, hb_opts:get(node_host, no_host, Opts)), {ok, ResolvedMsg} ?= resolve(Name, HookMsg, #{}, Opts), ModReq = case hb_maps:find(<<"body">>, HookReq, Opts) of @@ -280,8 +280,6 @@ arns_host_resolution_test() -> <<"path">> => <<"content-type">>, <<"host">> => <<"draft-17_whitepaper">> }, - %% Host header isn't added automatically if used HTTP2. - %% We need host header to resolve the ARNS name. - Opts#{http_client => gun, opts => #{protocol => http1} } + Opts ) ). \ No newline at end of file diff --git a/src/dev_push.erl b/src/dev_push.erl index 6ebfa1d63..bd19ce7fa 100644 --- a/src/dev_push.erl +++ b/src/dev_push.erl @@ -363,7 +363,7 @@ push_downstream_remote(TargetID, NextSlotOnProc, Origin, RawOpts) -> {ok, NewOpts} -> NewOpts; _ -> RawOpts end, - Self = hb_opts:get(host, host_not_specified, Opts), + Self = hb_opts:get(node_host, host_not_specified, Opts), ?event(remote_push, {push_downstream_remote, {target, TargetID}, diff --git a/src/dev_whois.erl b/src/dev_whois.erl index 61011c489..a30bf09fd 100644 --- a/src/dev_whois.erl +++ b/src/dev_whois.erl @@ -17,7 +17,7 @@ echo(_, Req, Opts) -> node(_, _, Opts) -> case ensure_host(Opts) of {ok, NewOpts} -> - {ok, hb_opts:get(host, <<"unknown">>, NewOpts)}; + {ok, hb_opts:get(node_host, <<"unknown">>, NewOpts)}; Error -> Error end. @@ -25,12 +25,12 @@ node(_, _, Opts) -> %% @doc Return the node message ensuring that the host is set. If it is not, we %% attempt to find the host information from the specified bootstrap node. ensure_host(Opts) -> - case hb_opts:get(host, <<"unknown">>, Opts) of + case hb_opts:get(node_host, <<"unknown">>, Opts) of <<"unknown">> -> case bootstrap_node_echo(Opts) of {ok, Host} -> % Set the host information in the persisted node message. - hb_http_server:set_opts(NewOpts = Opts#{ host => Host }), + hb_http_server:set_opts(NewOpts = Opts#{ node_host => Host }), {ok, NewOpts}; Error -> Error diff --git a/src/hb_http.erl b/src/hb_http.erl index cef56fe7d..8fc914e51 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -1091,10 +1091,12 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> end, WithPrivIP = hb_private:set(WithPeer, <<"ip">>, RealIP, Opts), % Add device from PrimMsg if present - case maps:get(<<"device">>, PrimMsg, not_found) of + WithDevice = case maps:get(<<"device">>, PrimMsg, not_found) of not_found -> WithPrivIP; Device -> WithPrivIP#{<<"device">> => Device} - end. + end, + Host = cowboy_req:host(Req), + WithDevice#{<<"host">> => Host}. %% @doc Determine the caller, honoring the `x-real-ip' header if present. real_ip(Req = #{ headers := RawHeaders }, Opts) -> diff --git a/src/hb_http_server.erl b/src/hb_http_server.erl index 520985198..428e18081 100644 --- a/src/hb_http_server.erl +++ b/src/hb_http_server.erl @@ -125,7 +125,7 @@ print_greeter(Config, PrivWallet) -> io_lib:format( "http://~s:~p", [ - hb_opts:get(host, <<"localhost">>, Config), + hb_opts:get(node_host, <<"localhost">>, Config), hb_opts:get(port, 8734, Config) ] ) diff --git a/src/hb_opts.erl b/src/hb_opts.erl index bc37efc88..5eadeb7a0 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -916,14 +916,14 @@ global_preference_test() -> load_flat_test() -> % File contents: % port: 1234 - % host: https://ao.computer + % node_host: https://ao.computer % await-inprogress: false {ok, Conf} = load("test/config.flat", #{}), ?event({loaded, {explicit, Conf}}), % Ensure we convert types as expected. ?assertEqual(1234, hb_maps:get(port, Conf)), % A binary - ?assertEqual(<<"https://ao.computer">>, hb_maps:get(host, Conf)), + ?assertEqual(<<"https://ao.computer">>, hb_maps:get(node_host, Conf)), % An atom, where the key contained a header-key `-' rather than a `_'. ?assertEqual(false, hb_maps:get(await_inprogress, Conf)). @@ -933,7 +933,7 @@ load_json_test() -> ?assertEqual(1234, hb_maps:get(port, Conf)), ?assertEqual(9001, hb_maps:get(example, Conf)), % A binary - ?assertEqual(<<"https://ao.computer">>, hb_maps:get(host, Conf)), + ?assertEqual(<<"https://ao.computer">>, hb_maps:get(node_host, Conf)), % An atom, where the key contained a header-key `-' rather than a `_'. ?assertEqual(false, hb_maps:get(await_inprogress, Conf)), % Ensure that a store with `ao-types' is loaded correctly. diff --git a/test/config.flat b/test/config.flat index be8c0d876..a28c640d9 100644 --- a/test/config.flat +++ b/test/config.flat @@ -1,3 +1,3 @@ port: 1234 -host: https://ao.computer +node_host: https://ao.computer await-inprogress: false \ No newline at end of file diff --git a/test/config.json b/test/config.json index 0be064746..37c2e2c96 100644 --- a/test/config.json +++ b/test/config.json @@ -1,7 +1,7 @@ { "port": 1234, "example": 9001, - "host": "https://ao.computer", + "node_host": "https://ao.computer", "await_inprogress": false, "store": [ { From 94c6efe8d4cda3205856facf3c90eceb8e4f5ebd Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 12 Mar 2026 16:43:36 -0400 Subject: [PATCH 085/135] fix: infinite loop in singleton registration race condition --- src/hb_name.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hb_name.erl b/src/hb_name.erl index d3c14db26..1aecccbf9 100644 --- a/src/hb_name.erl +++ b/src/hb_name.erl @@ -95,7 +95,7 @@ singleton_spawn(Name, Fun) -> ), receive {spawned, ReadyRef, PID} -> PID; - {spawn_failed, ReadyRef} -> singleton_spawn(Name, Fun) + {spawn_failed, ReadyRef} -> singleton(Name, Fun) end. %%% @doc Lookup a name -> PID. From 6f8eaaa6374d62b28f1a98e16a18dac7a7cde9e5 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Thu, 12 Mar 2026 16:51:24 -0400 Subject: [PATCH 086/135] chore: tidy form --- src/hb_store_remote_node.erl | 46 ++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index 3c58b3000..90890e9c3 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -52,34 +52,28 @@ type(Opts = #{ <<"node">> := Node }, Key) -> %% @param Opts A map of options (including node configuration). %% @param Key The key to read. %% @returns {ok, Msg} on success or not_found if the key is missing. +read(#{ <<"only-ids">> := true }, Key) when not ?IS_ID(Key) -> + not_found; read(Opts = #{ <<"node">> := Node }, Key) -> - ReadOnlyIDs = maps:get(<<"only-ids">>, Opts, false), - %% Limit read to ID keys if `<<"only-ids">>` is set to true. - ShouldRead = (ReadOnlyIDs andalso ?IS_ID(Key)) orelse not ReadOnlyIDs, - case ShouldRead of - true -> - ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), - HTTPRes = - hb_http:get( - Node, - #{ <<"path">> => <<"/~cache@1.0/read">>, <<"target">> => Key }, - Opts - ), - case HTTPRes of - {ok, Res} -> - % returning the whole response to get the test-key - {ok, Msg} = hb_message:with_only_committed(Res, Opts), - ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), - maybe_cache(Opts, Msg, [Key]), - {ok, Msg}; - {error, _Err} -> - ?event(store_remote_node, {read_not_found, {key, Key}}), - not_found - end; - false -> - ?event(store_remote_node, {ignoring_non_id, {key, Key}}), + ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), + HTTPRes = + hb_http:get( + Node, + #{ <<"path">> => <<"/~cache@1.0/read">>, <<"target">> => Key }, + Opts + ), + case HTTPRes of + {ok, Res} -> + % returning the whole response to get the test-key + {ok, Msg} = hb_message:with_only_committed(Res, Opts), + ?event(store_remote_node, {read_found, {result, Msg, response, Res}}), + maybe_cache(Opts, Msg, [Key]), + {ok, Msg}; + {error, _Err} -> + ?event(store_remote_node, {read_not_found, {key, Key}}), not_found - end. + end; +read(_, _) -> not_found. %% @doc Cache the data if the cache is enabled. The `local-store' option may %% either be `false' or a store definition to use as the local cache. Additional From 21f0ae251b4cc0829c2e866f49856760a936d005 Mon Sep 17 00:00:00 2001 From: Niko Storni Date: Fri, 13 Mar 2026 03:55:21 +0100 Subject: [PATCH 087/135] fix: handle as-tuples in blacklist ID collection - Add collect_ids clause for as-tuples so IDs inside manifest-cast messages are not silently skipped - Add test for manifest+blacklist with directly resolvable items - Add test for manifest+blacklist with legacy manifest content-type --- src/dev_blacklist.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 33b17b529..3e08e6f02 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -180,6 +180,7 @@ parse_blacklist_line(Line) -> collect_ids(Msg, Opts) -> lists:usort(collect_ids(Msg, [], Opts)). collect_ids(Bin, Acc, _Opts) when ?IS_ID(Bin) -> [hb_util:human_id(Bin) | Acc]; collect_ids(Bin, Acc, _Opts) when is_binary(Bin) -> Acc; +collect_ids({as, _, Msg}, Acc, Opts) -> collect_ids(Msg, Acc, Opts); collect_ids({link, ID, _}, Acc, _Opts) when ?IS_ID(ID) -> [hb_util:human_id(ID) | Acc]; collect_ids(Msg, Acc, Opts) when is_map(Msg) -> @@ -442,4 +443,4 @@ provider_failure_resilience_test() -> {ok, <<"test-3">>}, hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) ), - ok. \ No newline at end of file + ok. From ffee659ddb0807e6f81736288d457aae8ff48434 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 15 Mar 2026 02:21:25 -0400 Subject: [PATCH 088/135] fix: subdomain extraction from hostname Fixes a bug with hostname->subdomain resolution that led to `localhost` being looked up in ARNS. As it happened, there is actually a name for [localhost](https://localhost.arweave.net/) in ARNS, so this image would replace the HyperBuddy page when a user loaded went to their local node. Additionally, fixed two naming tests. --- src/dev_name.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/dev_name.erl b/src/dev_name.erl index 41cb8478b..2e5d1f189 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -103,9 +103,9 @@ request(HookMsg, HookReq, Opts) -> %% returns only the name component of the host, if it is present. If no name is %% present, an empty binary is returned. name_from_host(Host, no_host) -> - case hd(binary:split(Host, <<".">>)) of - <<>> -> {error, <<"No name found in `Host`.">>}; - Name -> {ok, Name} + case binary:split(Host, <<".">>, [global, trim_all]) of + [_Host] -> {error, <<"No subdomain found in `Host: ", Host/binary, "`.">>}; + [Name|_] -> {ok, Name} end; name_from_host(ReqHost, RawNodeHost) -> NodeHost = uri_string:parse(RawNodeHost), @@ -258,11 +258,11 @@ arns_opts() -> arns_json_snapshot_test() -> Opts = arns_opts(), ?assertMatch( - {ok, <<"application/pdf">>}, + {ok, <<"text/html">>}, hb_ao:resolve_many( [ #{ <<"device">> => <<"name@1.0">> }, - #{ <<"path">> => <<"draft-17_whitepaper">> }, + #{ <<"path">> => <<"001_permabytes">>, <<"load">> => true }, <<"content-type">> ], Opts @@ -273,12 +273,12 @@ arns_host_resolution_test() -> Opts = arns_opts(), Node = hb_http_server:start_node(Opts), ?assertMatch( - {ok, <<"application/pdf">>}, + {ok, <<"text/html">>}, hb_http:get( Node, #{ <<"path">> => <<"content-type">>, - <<"host">> => <<"draft-17_whitepaper">> + <<"host">> => <<"001_permabytes.localhost">> }, Opts ) From e0d2756523ad2cef1770149a647e1b37244c0c8f Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 13 Mar 2026 12:41:45 -0300 Subject: [PATCH 089/135] fix: gun orphans --- src/hb_http_client.erl | 2 +- src/hb_http_client_tests.erl | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 src/hb_http_client_tests.erl diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index d7a4f99ea..d3f739734 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -542,7 +542,7 @@ create_new_connection(ConnKey, Args, From, State) -> %% Store connection in ETS ets:insert(?CONNECTIONS_ETS, {ConnKey, PID}), %% Store status with monitor ref and conn key - ets:insert(?CONN_STATUS_ETS, {PID, {connecting, [{From, Args}]}, MonitorRef, ConnKey}), + ets:insert(?CONN_STATUS_ETS, {PID, {connecting, []}, MonitorRef, ConnKey}), {reply, {ok, PID}, State}. open_connection(#{ peer := Peer }, Opts) -> diff --git a/src/hb_http_client_tests.erl b/src/hb_http_client_tests.erl new file mode 100644 index 000000000..3df3cc028 --- /dev/null +++ b/src/hb_http_client_tests.erl @@ -0,0 +1,38 @@ +-module(hb_http_client_tests). +-include("include/hb.hrl"). +-include("include/hb_http_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% Regression test: create_new_connection used to reply to From immediately +%% AND store From in PendingRequests. gun_up would then reply again, +%% leaving orphan {Ref, {ok, PID}} messages in the caller's mailbox. +%% Fixed by storing an empty pending list when replying immediately. +orphan_message_leak_test_() -> + {timeout, 60, fun() -> + application:ensure_all_started(hb), + flush_mailbox(), + Peer = <<"https://arweave.net">>, + Args = #{ + peer => Peer, + path => <<"/info">>, + method => <<"GET">>, + headers => #{}, + body => <<>> + }, + Opts = #{http_client => gun, http_retry => 0}, + {ok, 200, _, _} = hb_http_client:request(Args, Opts), + timer:sleep(2000), + Orphans = flush_mailbox(), + ?debugFmt("Orphan messages after first request: ~p", [length(Orphans)]), + ?assertEqual(0, length(Orphans), + "No orphan messages should be left in caller mailbox") + end}. + +flush_mailbox() -> + flush_mailbox([]). +flush_mailbox(Acc) -> + receive + Msg -> flush_mailbox([Msg | Acc]) + after 0 -> + lists:reverse(Acc) + end. From 526c6447212605773a7b6d398f2c92bd17857d5e Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 13 Mar 2026 13:17:05 -0300 Subject: [PATCH 090/135] impr: avoid infinity on http client --- src/hb_http_client.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index d3f739734..426143199 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -302,14 +302,14 @@ get_connection_by_key(ConnKey, PoolSize, Args, Opts, Attempts) when Attempts < P {ok, PID}; [{PID, {connecting, _}, _MonitorRef, _ConnKey}] -> %% Connection is being established, wait for it via gen_server - catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, infinity); + catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, 10_000); [] -> %% Status not found, connection might be dead, create new one - catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, infinity) + catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, 10_000) end; [] -> %% No connection, create one via gen_server - catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, infinity) + catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, 10_000) end; get_connection_by_key(_ConnKey, _PoolSize, _Args, _Opts, _Attempts) -> {error, no_available_connection}. From 6f971f7bfe44b920ac9cf1a9f009cf5148128af5 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Fri, 13 Mar 2026 13:19:40 -0300 Subject: [PATCH 091/135] fix: gun edge case hang --- src/hb_http_client.erl | 36 ++++++++++++++++--------- src/hb_http_client_tests.erl | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index 426143199..df5612621 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -100,7 +100,7 @@ httpc_req(Args, Opts) -> body := Body } = Args, ?event({httpc_req, Args}), - {Host, Port} = parse_peer(Peer, Opts), + {ok, {Host, Port}} = parse_peer(Peer, Opts), Scheme = case Port of 443 -> "https"; _ -> "http" @@ -535,18 +535,26 @@ terminate(Reason, _State) -> %%% ================================================================== %% @doc Create a new connection and store it in ETS. -create_new_connection(ConnKey, Args, From, State) -> +create_new_connection(ConnKey, Args, _From, State) -> MergedOpts = hb_maps:merge(State#state.opts, hb_maps:get(opts, Args, #{}), #{}), - {ok, PID} = open_connection(Args, MergedOpts), - MonitorRef = monitor(process, PID), - %% Store connection in ETS - ets:insert(?CONNECTIONS_ETS, {ConnKey, PID}), - %% Store status with monitor ref and conn key - ets:insert(?CONN_STATUS_ETS, {PID, {connecting, []}, MonitorRef, ConnKey}), - {reply, {ok, PID}, State}. + case open_connection(Args, MergedOpts) of + {ok, PID} -> + MonitorRef = monitor(process, PID), + ets:insert(?CONNECTIONS_ETS, {ConnKey, PID}), + ets:insert(?CONN_STATUS_ETS, + {PID, {connecting, []}, MonitorRef, ConnKey}), + {reply, {ok, PID}, State}; + {error, _} = Err -> + {reply, Err, State} + end. open_connection(#{ peer := Peer }, Opts) -> - {Host, Port} = parse_peer(Peer, Opts), + case parse_peer(Peer, Opts) of + {error, _} = Err -> Err; + {ok, {Host, Port}} -> open_connection_gun(Host, Port, Peer, Opts) + end. + +open_connection_gun(Host, Port, Peer, Opts) -> ?event(http_outbound, {parsed_peer, {peer, Peer}, {host, Host}, {port, Port}}), BaseGunOpts = #{ @@ -598,15 +606,17 @@ parse_peer(Peer, Opts) -> Parsed = uri_string:parse(Peer), case Parsed of #{ host := Host, port := Port } -> - {hb_util:list(Host), Port}; + {ok, {hb_util:list(Host), Port}}; URI = #{ host := Host } -> - { + {ok, { hb_util:list(Host), case hb_maps:get(scheme, URI, undefined, Opts) of <<"https">> -> 443; _ -> hb_opts:get(port, 8734, Opts) end - } + }}; + _ -> + {error, {bad_peer, Peer}} end. reply_error([], _Reason) -> diff --git a/src/hb_http_client_tests.erl b/src/hb_http_client_tests.erl index 3df3cc028..041e12256 100644 --- a/src/hb_http_client_tests.erl +++ b/src/hb_http_client_tests.erl @@ -28,6 +28,51 @@ orphan_message_leak_test_() -> "No orphan messages should be left in caller mailbox") end}. +unreachable_peer_hang_test_() -> + {timeout, 30, fun() -> + application:ensure_all_started(hb), + Peer = <<"http://192.0.2.1:1984">>, + Args = #{ + peer => Peer, + path => <<"/info">>, + method => <<"GET">>, + headers => #{}, + body => <<>> + }, + Opts = #{http_client => gun, http_retry => 0}, + T0 = erlang:monotonic_time(millisecond), + Result = hb_http_client:request(Args, Opts), + Elapsed = erlang:monotonic_time(millisecond) - T0, + ?debugFmt("Unreachable peer: ~p in ~pms", [element(1, Result), Elapsed]), + ?assertMatch({error, _}, Result), + ?assert(Elapsed >= 4000 andalso Elapsed =< 15000, + "Should block for ~5s connect_timeout, not infinity") + end}. + +bad_peer_survives_test_() -> + {timeout, 30, fun() -> + application:ensure_all_started(hb), + ?assert(erlang:whereis(hb_http_client) =/= undefined), + ValidArgs = #{ + peer => <<"https://arweave.net">>, + path => <<"/info">>, + method => <<"GET">>, + headers => #{}, + body => <<>> + }, + Opts = #{http_client => gun, http_retry => 0}, + {ok, 200, _, _} = hb_http_client:request(ValidArgs, Opts), + BadArgs = ValidArgs#{peer => <<"not-a-valid-uri">>}, + BadResult = hb_http_client:request(BadArgs, Opts), + ?debugFmt("Bad peer result: ~p", [BadResult]), + ?assertMatch({error, _}, BadResult), + timer:sleep(500), + ?assert(erlang:whereis(hb_http_client) =/= undefined, + "gen_server must survive a bad peer URI"), + {ok, 200, _, _} = hb_http_client:request(ValidArgs, Opts), + ?debugFmt("Follow-up request to valid peer succeeded", []) + end}. + flush_mailbox() -> flush_mailbox([]). flush_mailbox(Acc) -> @@ -36,3 +81,10 @@ flush_mailbox(Acc) -> after 0 -> lists:reverse(Acc) end. + +summarize({caught, C, R}) when is_tuple(R) -> + {caught, C, element(1, R)}; +summarize({caught, C, R}) -> + {caught, C, R}; +summarize(Other) -> + Other. From 577ad3f2aa7af44f1ba5f2e4862a7093824c1621 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 11:56:17 -0400 Subject: [PATCH 092/135] chore: events > debugFmt --- src/hb_http_client_tests.erl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/hb_http_client_tests.erl b/src/hb_http_client_tests.erl index 041e12256..6bc1921ea 100644 --- a/src/hb_http_client_tests.erl +++ b/src/hb_http_client_tests.erl @@ -23,7 +23,7 @@ orphan_message_leak_test_() -> {ok, 200, _, _} = hb_http_client:request(Args, Opts), timer:sleep(2000), Orphans = flush_mailbox(), - ?debugFmt("Orphan messages after first request: ~p", [length(Orphans)]), + ?event(http_client_tests, {orphaned_messages, {length, length(Orphans)}}), ?assertEqual(0, length(Orphans), "No orphan messages should be left in caller mailbox") end}. @@ -43,7 +43,9 @@ unreachable_peer_hang_test_() -> T0 = erlang:monotonic_time(millisecond), Result = hb_http_client:request(Args, Opts), Elapsed = erlang:monotonic_time(millisecond) - T0, - ?debugFmt("Unreachable peer: ~p in ~pms", [element(1, Result), Elapsed]), + ?event(http_client_tests, + {unreachable_peer_result, {result, Result}, {elapsed, Elapsed}} + ), ?assertMatch({error, _}, Result), ?assert(Elapsed >= 4000 andalso Elapsed =< 15000, "Should block for ~5s connect_timeout, not infinity") @@ -64,13 +66,13 @@ bad_peer_survives_test_() -> {ok, 200, _, _} = hb_http_client:request(ValidArgs, Opts), BadArgs = ValidArgs#{peer => <<"not-a-valid-uri">>}, BadResult = hb_http_client:request(BadArgs, Opts), - ?debugFmt("Bad peer result: ~p", [BadResult]), + ?event(http_client_tests, {bad_peer_result, BadResult}), ?assertMatch({error, _}, BadResult), timer:sleep(500), ?assert(erlang:whereis(hb_http_client) =/= undefined, "gen_server must survive a bad peer URI"), {ok, 200, _, _} = hb_http_client:request(ValidArgs, Opts), - ?debugFmt("Follow-up request to valid peer succeeded", []) + ?event(http_client_tests, follow_up_request_to_valid_peer_succeeded) end}. flush_mailbox() -> From 770509192902c58608ea95d711f9e433e8334ad8 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 13:02:41 -0400 Subject: [PATCH 093/135] chore: events and nomenclature --- src/hb_http.erl | 14 +++++++------- src/hb_http_client.erl | 10 +++++----- src/hb_opts.erl | 8 ++++---- src/hb_store_arweave.erl | 13 +++++++------ 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/hb_http.erl b/src/hb_http.erl index 8fc914e51..fefaf4f26 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -521,9 +521,9 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> ReqDuration = EndTime - hb_maps:get(start_time, Req, undefined, Opts), ReplyDuration = EndTime - ReplyStartTime, record_request_metric( - ReqDuration * 1000000, - ReplyDuration * 1000000, - Status + ReqDuration * 1000000, + ReplyDuration * 1000000, + Status ), ?event(debug_http, {reply_headers, {explicit, PostStreamReq}}), ?event(http_server_short, @@ -1116,7 +1116,7 @@ real_ip(Req = #{ headers := RawHeaders }, Opts) -> init_prometheus() -> hb_prometheus:declare(histogram, [ - {name, http_request_server_reply_duration_seconds}, + {name, http_server_encoding_duration_seconds}, {labels, [status_code]}, {buckets, [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, @@ -1131,7 +1131,7 @@ init_prometheus() -> } ]), hb_prometheus:declare(histogram, [ - {name, http_request_server_duration_seconds}, + {name, http_server_duration_seconds}, {labels, [status_code]}, {buckets, [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, @@ -1149,12 +1149,12 @@ record_request_metric(TotalDuration, ReplyDuration, StatusCode) -> fun() -> hb_prometheus:observe( TotalDuration, - http_request_server_duration_seconds, + http_server_duration_seconds, [StatusCode] ), hb_prometheus:observe( ReplyDuration, - http_request_server_reply_duration_seconds, + http_server_encoding_duration_seconds, [StatusCode] ) end diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index df5612621..c17a6f6f9 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -562,7 +562,7 @@ open_connection_gun(Host, Port, Peer, Opts) -> #{ keepalive => hb_opts:get( - http_keepalive, + http_client_keepalive, ?DEFAULT_KEEPALIVE_TIMEOUT, Opts ) @@ -570,7 +570,7 @@ open_connection_gun(Host, Port, Peer, Opts) -> retry => 0, connect_timeout => hb_opts:get( - http_connect_timeout, + http_client_connect_timeout, ?DEFAULT_CONNECT_TIMEOUT, Opts ) @@ -632,7 +632,7 @@ reply_error([PendingRequest | PendingRequests], Reason) -> do_gun_request(PID, Args, Opts) -> Timer = inet:start_timer( - hb_opts:get(http_request_send_timeout, no_request_send_timeout, Opts) + hb_opts:get(http_client_send_timeout, no_request_send_timeout, Opts) ), Method = hb_maps:get(method, Args, undefined, Opts), Path = hb_maps:get(path, Args, undefined, Opts), @@ -769,7 +769,7 @@ init_prometheus() -> hb_prometheus:declare(gauge, [{name, outbound_connections}, {help, "The current number of the open outbound network connections"}]), hb_prometheus:declare(histogram, [ - {name, http_request_duration_seconds}, + {name, http_client_duration_seconds}, {buckets, [0.01, 0.1, 0.5, 1, 5, 10, 30, 60]}, {labels, [http_method, status_class, category]}, { @@ -823,7 +823,7 @@ record_duration(Details, Opts) -> ]), hb_prometheus:observe( maps:get(<<"duration">>, Details), - http_request_duration_seconds, + http_client_duration_seconds, Labels ), maybe_invoke_monitor( diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 5eadeb7a0..7fd2e54b3 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -235,10 +235,10 @@ default_message() -> trusted_device_signers => [], %% What should the node do if a client error occurs? client_error_strategy => throw, - %% HTTP request options - http_connect_timeout => 5000, - http_keepalive => 120000, - http_request_send_timeout => 300_000, + %% HTTP client request options + http_client_connect_timeout => 5000, + http_client_keepalive => 120000, + http_client_send_timeout => 300_000, port => 8734, wasm_allow_aot => false, %% Options for the relay device diff --git a/src/hb_store_arweave.erl b/src/hb_store_arweave.erl index 2b1a5a023..d1739edeb 100644 --- a/src/hb_store_arweave.erl +++ b/src/hb_store_arweave.erl @@ -77,14 +77,14 @@ read_offset(#{ <<"index-store">> := IndexStore }, ID) -> {ok, OffsetBinary} -> {Version, CodecName, StartOffset, Length} = hb_store_arweave_offset:decode(OffsetBinary), - record_partition_metric(StartOffset), {ok, #{ <<"version">> => Version, <<"codec-device">> => CodecName, <<"start-offset">> => StartOffset, <<"length">> => Length }}; - _ -> not_found + _ -> + not_found end; read_offset(_, _) -> not_found. @@ -129,6 +129,7 @@ do_read(StoreOpts, ID) -> {length, Length} } ), + record_partition_metric(StartOffset, ok), Loaded; {error, Reason} -> ?event( @@ -142,6 +143,7 @@ do_read(StoreOpts, ID) -> {reason, Reason} } ), + record_partition_metric(StartOffset, not_found), if Reason =:= not_found -> not_found; true -> {error, Reason} end @@ -253,13 +255,12 @@ write_offset( hb_store:write(IndexStore, hb_store_arweave_offset:path(ID), Value). %% @doc Record the partition that data is found in when it is requested. -record_partition_metric(Offset) when is_integer(Offset) -> +record_partition_metric(Offset, Result) when is_integer(Offset) -> spawn(fun() -> - Partition = Offset div ?PARTITION_SIZE, hb_prometheus:inc( counter, hb_store_arweave_requests_partition, - [Partition], + [Offset div ?PARTITION_SIZE, hb_util:bin(Result)], 1 ) end). @@ -288,7 +289,7 @@ init_prometheus() -> counter, [ {name, hb_store_arweave_requests_partition}, - {labels, [partition]}, + {labels, [partition, result]}, {help, "Partition where chunks are being requested"} ] ), From 36d67b0c89849caf25e4ecdaf6bcbcfa25da5792 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 14:50:56 -0400 Subject: [PATCH 094/135] impr: blacklist refresh handling --- src/dev_blacklist.erl | 132 ++++++++++++++++++++++++++++-------------- 1 file changed, 89 insertions(+), 43 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 3e08e6f02..80399f00d 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -17,7 +17,7 @@ %%% enforcing its own content policies based on its own free choice and %%% configuration. -module(dev_blacklist). --export([request/3, refresh/3]). +-export([request/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -57,7 +57,7 @@ request(_Base, HookReq, Opts) -> %% @doc Check if the message contains any blacklisted IDs. is_match(Msg, Opts) -> - maybe_refresh(Opts), + ensure_cache_table(Opts), IDs = collect_ids(Msg, Opts), MatchesFromIDs = fun(ID) -> ets:lookup(cache_table_name(Opts), ID) =/= [] end, case lists:filter(MatchesFromIDs, IDs) of @@ -65,49 +65,24 @@ is_match(Msg, Opts) -> [ID|_] -> ID end. -%% @doc Force a reload of the blacklist cache. Returns the number of newly -%% inserted IDs. -refresh(Base, Req, Opts) -> - ?event({refresh_called, {base, Base}, {req, Req}}), - maybe_refresh(Opts). - %%% Internal -%% @doc Fetch the blacklist and store the results in the cache table. -maybe_refresh(Opts) -> - ensure_cache_table(Opts), - MinWait = - hb_util:int( - hb_opts:get( - blacklist_refresh_frequency, - ?DEFAULT_MIN_WAIT, - Opts - ) - ), - Time = erlang:system_time(second), - case hb_opts:get(blacklist_last_refresh, 0, Opts) of - LastRefresh when (Time - LastRefresh) > MinWait -> - fetch_and_insert_ids(Opts), - hb_http_server:set_opts(Opts#{ blacklist_last_refresh => Time }); - _ -> - skip_update - end. - %% @doc Fetch blacklists from all configured providers and insert IDs into the %% cache table. fetch_and_insert_ids(Opts) -> ensure_cache_table(Opts), Providers = resolve_providers(Opts), - Total = lists:foldl( - fun(Provider, Acc) -> - case fetch_single_provider(Provider, Opts) of - {ok, Count} -> Acc + Count; - {error, _} -> Acc - end - end, - 0, - Providers - ), + Total = + lists:foldl( + fun(Provider, Acc) -> + case fetch_single_provider(Provider, Opts) of + {ok, Count} -> Acc + Count; + {error, _} -> Acc + end + end, + 0, + Providers + ), ?event(blacklist_short, {fetched_and_inserted_ids, Total}, Opts), {ok, Total}. @@ -145,10 +120,8 @@ fetch_single_provider(Provider, Opts) -> execute_provider(Provider, Opts) -> ?event({execute_provider, {provider, Provider}}), case hb_cache:ensure_loaded(Provider, Opts) of - Bin when is_binary(Bin) -> - hb_ao:resolve(#{ <<"path">> => Bin }, Opts); - Msgs when is_list(Msgs) -> - hb_ao:resolve_many(Msgs, Opts) + Bin when is_binary(Bin) -> hb_ao:resolve(#{ <<"path">> => Bin }, Opts); + Msgs when is_list(Msgs) -> hb_ao:resolve_many(Msgs, Opts) end. %% @doc Parse the blacklist body, returning a list of IDs. @@ -233,7 +206,7 @@ ensure_cache_table(Opts) -> ] ), fetch_and_insert_ids(Opts), - receive kill -> ok end + refresh_loop(Opts) end ), hb_util:until( @@ -245,6 +218,28 @@ ensure_cache_table(Opts) -> TableName end. +%% @doc Loop that periodically refreshes the blacklist cache. Runs on the +%% singleton process that is responsible for the cache ets table. +refresh_loop(Opts) -> + timer:send_after( + hb_util:int( + hb_opts:get( + blacklist_refresh_frequency, + ?DEFAULT_MIN_WAIT, + Opts + ) + ) * 1000, + self(), + refresh + ), + receive + refresh -> + fetch_and_insert_ids(Opts), + refresh_loop(Opts); + stop -> + ok + end. + %% @doc Calculate the name of the cache table given the `Opts`. cache_table_name(Opts) -> Wallet = hb_opts:get(priv_wallet, hb:wallet(), Opts), @@ -444,3 +439,54 @@ provider_failure_resilience_test() -> hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) ), ok. + +%% @doc Test that the blacklist cache is refreshed periodically. +refresh_periodically_test() -> + {ok, #{ + opts := Opts0 = #{ store := Store }, + signed1 := SignedID1, + unsigned3 := UnsignedID3 + }} = setup_test_env(), + InitialBlacklist = + #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => SignedID1 + }, + BlacklistMsg = hb_message:commit(InitialBlacklist, Opts0), + {ok, InitialBlacklistID} = hb_cache:write(BlacklistMsg, Opts0), + hb_store:make_link(Store, InitialBlacklistID, <<"mutable">>), + UpdatedBlacklist = + #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => <> + }, + UpdatedBlacklistMsg = hb_message:commit(UpdatedBlacklist, Opts0), + {ok, UpdatedBlacklistID} = hb_cache:write(UpdatedBlacklistMsg, Opts0), + hb_store:make_link(Store, InitialBlacklistID, <<"mutable">>), + Opts1 = Opts0#{ + blacklist_providers => [<<"/~cache@1.0/read?target=mutable">>], + on => #{ + <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } + }, + blacklist_refresh_frequency => 1 + }, + Node = hb_http_server:start_node(Opts1), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, SignedID1, Opts1) + ), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + hb_store:make_link(Store, UpdatedBlacklistID, <<"mutable">>), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + timer:sleep(1000), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + ok. \ No newline at end of file From 34acdcc25fb258fb916670fff17c415fcb8b25d7 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 12 Mar 2026 17:26:26 +0000 Subject: [PATCH 095/135] impr: Basic 52 char subdomain to TX - WIP --- src/dev_name.erl | 18 +++++++++++++++++ src/hb_util.erl | 52 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/dev_name.erl b/src/dev_name.erl index 2e5d1f189..ef87bedb5 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -125,6 +125,9 @@ overlay_loaded({as, DevID, Base}, Resolved, Opts) -> overlay_loaded(Base, Resolved, Opts) -> hb_maps:merge(Base, Resolved, Opts). +subdomain_to_tx_id(Subdomain) when byte_size(Subdomain) == 52 -> + b64fast:encode(hb_util:base32_decode(Subdomain)). + %%% Tests. no_resolvers_test() -> @@ -282,4 +285,19 @@ arns_host_resolution_test() -> }, Opts ) + ). + +subdomain_to_tx_id_test() -> + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + ?assertEqual(<<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, subdomain_to_tx_id(Subdomain)). + +resolve_52char_subdomain_if_txid_not_present_test() -> + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(#{}), + {ok, _} = hb_http:get( + Node, + #{ + <<"path">> => <<"/assets/index-C_KRlCcV.js">>, + <<"host">> => <> + } ). \ No newline at end of file diff --git a/src/hb_util.erl b/src/hb_util.erl index 7e4db770e..d9a2a0228 100644 --- a/src/hb_util.erl +++ b/src/hb_util.erl @@ -27,7 +27,7 @@ -export([check_size/2, check_value/2, check_type/2, ok_or_throw/3]). -export([all_atoms/0, binary_is_atom/1]). -export([lower_case_keys/2]). --export([base58_encode/1]). +-export([base58_encode/1, base32_encode/1, base32_decode/1]). -include("include/hb.hrl"). @@ -839,4 +839,52 @@ base58_encode_int(N) -> Rem = N rem 58, Char = binary:at(Alphabet, Rem), Rest = base58_encode_int(N div 58), - <>. \ No newline at end of file + <>. + +-define(ALPHABET, <<"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567">>). +base32_encode(Data) -> + Bits = << <> || <> <= Data >>, + encode_bits(Bits, <<>>). + +encode_bits(<<>>, Acc) -> + Acc; +encode_bits(Bits, Acc) when bit_size(Bits) < 5 -> + Acc; +encode_bits(<>, Acc) -> + <> = binary:part(?ALPHABET, Chunk, 1), + encode_bits(Rest, <>). + +-spec base32_decode(binary() | list()) -> binary(). +base32_decode(Input0) -> + Input = + case is_list(Input0) of + true -> list_to_binary(Input0); + false -> Input0 + end, + Upper = string:uppercase(Input), + decode(Upper, <<>>, 0, 0). + +decode(<<>>, Acc, Bits, Value) -> + case Bits of + 0 -> Acc; + _ -> + <> + end; +decode(<>, Acc, Bits, Value) -> + V = alphabet_val(C), + NewValue = (Value bsl 5) bor V, + NewBits = Bits + 5, + case NewBits >= 8 of + true -> + Byte = (NewValue bsr (NewBits - 8)) band 16#FF, + decode(Rest, <>, NewBits - 8, NewValue); + false -> + decode(Rest, Acc, NewBits, NewValue) + end. + +alphabet_val(C) when $A =< C, C =< $Z -> + C - $A; +alphabet_val(C) when $2 =< C, C =< $7 -> + 26 + (C - $2); +alphabet_val($=) -> + 0. \ No newline at end of file From a3b4f6603a6a53dfa207a7066660d85738410970 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Thu, 12 Mar 2026 23:11:23 +0000 Subject: [PATCH 096/135] impr: Working version of no TXID manifests, 52 chars manifests. Missing tests --- rebar.config | 1 + rebar.lock | 3 +++ src/dev_name.erl | 48 ++++++++++++++++++++++++++++++++++++++++------ src/hb_util.erl | 50 +----------------------------------------------- 4 files changed, 47 insertions(+), 55 deletions(-) diff --git a/rebar.config b/rebar.config index 4c583836a..41013af47 100644 --- a/rebar.config +++ b/rebar.config @@ -140,6 +140,7 @@ {deps, [ {elmdb, {git, "https://github.com/permaweb/elmdb-rs.git", {ref, "bfda2facebdb433eea753f82e7e8d45aefc6d87a"}}}, + {base32, "1.0.0"}, {b64fast, {git, "https://github.com/ArweaveTeam/b64fast.git", {ref, "58f0502e49bf73b29d95c6d02460d1fb8d2a5273"}}}, {cowlib, "2.16.0"}, {cowboy, "2.14.0"}, diff --git a/rebar.lock b/rebar.lock index 433624df1..9aaf8394a 100644 --- a/rebar.lock +++ b/rebar.lock @@ -4,6 +4,7 @@ {git,"https://github.com/ArweaveTeam/b64fast.git", {ref,"58f0502e49bf73b29d95c6d02460d1fb8d2a5273"}}, 0}, + {<<"base32">>,{pkg,<<"base32">>,<<"1.0.0">>},0}, {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.14.0">>},0}, {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.16.0">>},0}, {<<"ddskerl">>,{pkg,<<"ddskerl">>,<<"0.4.2">>},1}, @@ -30,6 +31,7 @@ [ {pkg_hash,[ {<<"accept">>, <<"CD6E34A2D7E28CA38B2D3CB233734CA0C221EFBC1F171F91FEC5F162CC2D18DA">>}, + {<<"base32">>, <<"1AB331F812FCC254C8F7D4348E1E5A6F2B9B32B7A260BF2BC3358E3BF14C841A">>}, {<<"cowboy">>, <<"565DCF221BA99B1255B0ADCEC24D2D8DBE79E46EC79F30F8373CCEADC6A41E2A">>}, {<<"cowlib">>, <<"54592074EBBBB92EE4746C8A8846E5605052F29309D3A873468D76CDF932076F">>}, {<<"ddskerl">>, <<"A51A90BE9AC9B36A94017670BED479C623B10CA9D4BDA1EDF3A0E48CAEEADA2A">>}, @@ -42,6 +44,7 @@ {<<"ranch">>, <<"25528F82BC8D7C6152C57666CA99EC716510FE0925CB188172F41CE93117B1B0">>}]}, {pkg_hash_ext,[ {<<"accept">>, <<"CA69388943F5DAD2E7232A5478F16086E3C872F48E32B88B378E1885A59F5649">>}, + {<<"base32">>, <<"0449285348ED0C4CD83C7198E76C5FD5A0451C4EF18695B9FD43792A503E551A">>}, {<<"cowboy">>, <<"EA99769574550FE8A83225C752E8A62780A586770EF408816B82B6FE6D46476B">>}, {<<"cowlib">>, <<"7F478D80D66B747344F0EA7708C187645CFCC08B11AA424632F78E25BF05DB51">>}, {<<"ddskerl">>, <<"63F907373D7E548151D584D4DA8A38928FD26EC9477B94C0FFAAD87D7CB69FE7">>}, diff --git a/src/dev_name.erl b/src/dev_name.erl index ef87bedb5..4967fe4c4 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -28,7 +28,7 @@ info(_) -> resolve(Key, _, Req, Opts) -> Resolvers = hb_opts:get(name_resolvers, [], Opts), ?event({resolvers, Resolvers}), - case match_resolver(Key, Resolvers, Opts) of + ArnsResolver = case match_resolver(Key, Resolvers, Opts) of {ok, Resolved} -> case hb_util:atom(hb_ao:get(<<"load">>, Req, true, Opts)) of false -> @@ -38,8 +38,27 @@ resolve(Key, _, Req, Opts) -> end; not_found -> not_found + end, + case ArnsResolver of + not_found -> + resolve_52char(Key, Req, Opts); + _ -> + ArnsResolver end. +%% @doc Try to resolve 52char subdomain back to its original TX ID when TX ID +%% isn't present. +resolve_52char(_, #{<<"body">> := [ID | _]}, _) when ?IS_ID(ID) -> + ?event({resolve_52char, {skip_becase_id_found, ID}}), + not_found; +resolve_52char(Key, _, Opts) when byte_size(Key) == 52 -> + TXID = subdomain_to_tx_id(Key), + ?event({resolve_52char, {key, Key}, {txid, TXID}}), + hb_cache:read(TXID, Opts); +resolve_52char(_, _, _) -> + ?event({resolve_52char, nothing_matched}), + not_found. + %% @doc Find the first resolver that matches the key and return its value. match_resolver(_Key, [], _Opts) -> not_found; @@ -76,7 +95,7 @@ request(HookMsg, HookReq, Opts) -> {ok, Req} ?= hb_maps:find(<<"request">>, HookReq, Opts), {ok, Host} ?= hb_maps:find(<<"host">>, Req, Opts), {ok, Name} ?= name_from_host(Host, hb_opts:get(node_host, no_host, Opts)), - {ok, ResolvedMsg} ?= resolve(Name, HookMsg, #{}, Opts), + {ok, ResolvedMsg} ?= resolve(Name, HookMsg, HookReq, Opts), ModReq = case hb_maps:find(<<"body">>, HookReq, Opts) of {ok, [OldBase|Rest]} -> @@ -126,7 +145,7 @@ overlay_loaded(Base, Resolved, Opts) -> hb_maps:merge(Base, Resolved, Opts). subdomain_to_tx_id(Subdomain) when byte_size(Subdomain) == 52 -> - b64fast:encode(hb_util:base32_decode(Subdomain)). + b64fast:encode(base32:decode(Subdomain)). %%% Tests. @@ -238,6 +257,7 @@ load_and_execute_test() -> arns_opts() -> JSONNames = <<"G_gb7SAgogHMtmqycwaHaC6uC-CZ3akACdFv5PUaEE8">>, Path = <>, + % TODO: Delete? hb_http_server:start_node(#{}), TempStore = hb_test_utils:test_store(), #{ @@ -291,13 +311,29 @@ subdomain_to_tx_id_test() -> Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, ?assertEqual(<<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, subdomain_to_tx_id(Subdomain)). -resolve_52char_subdomain_if_txid_not_present_test() -> +resolve_52char_subdomain_asset_if_txid_not_present_test() -> Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, Node = hb_http_server:start_node(#{}), - {ok, _} = hb_http:get( + {ok, R} = hb_http:get( Node, #{ <<"path">> => <<"/assets/index-C_KRlCcV.js">>, <<"host">> => <> } - ). \ No newline at end of file + ), + ?event(error, {r, R}). + +resolve_52char_subdomain_if_txid_not_present_test() -> + Opts = arns_opts(), + %% TX: 42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + {ok, R} = hb_http:get( + Node, + #{ + <<"path">> => <<"/">>, + <<"host">> => <> + }, + Opts + ), + ?event(error, {r, R}). diff --git a/src/hb_util.erl b/src/hb_util.erl index d9a2a0228..8a0030156 100644 --- a/src/hb_util.erl +++ b/src/hb_util.erl @@ -27,7 +27,7 @@ -export([check_size/2, check_value/2, check_type/2, ok_or_throw/3]). -export([all_atoms/0, binary_is_atom/1]). -export([lower_case_keys/2]). --export([base58_encode/1, base32_encode/1, base32_decode/1]). +-export([base58_encode/1]). -include("include/hb.hrl"). @@ -840,51 +840,3 @@ base58_encode_int(N) -> Char = binary:at(Alphabet, Rem), Rest = base58_encode_int(N div 58), <>. - --define(ALPHABET, <<"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567">>). -base32_encode(Data) -> - Bits = << <> || <> <= Data >>, - encode_bits(Bits, <<>>). - -encode_bits(<<>>, Acc) -> - Acc; -encode_bits(Bits, Acc) when bit_size(Bits) < 5 -> - Acc; -encode_bits(<>, Acc) -> - <> = binary:part(?ALPHABET, Chunk, 1), - encode_bits(Rest, <>). - --spec base32_decode(binary() | list()) -> binary(). -base32_decode(Input0) -> - Input = - case is_list(Input0) of - true -> list_to_binary(Input0); - false -> Input0 - end, - Upper = string:uppercase(Input), - decode(Upper, <<>>, 0, 0). - -decode(<<>>, Acc, Bits, Value) -> - case Bits of - 0 -> Acc; - _ -> - <> - end; -decode(<>, Acc, Bits, Value) -> - V = alphabet_val(C), - NewValue = (Value bsl 5) bor V, - NewBits = Bits + 5, - case NewBits >= 8 of - true -> - Byte = (NewValue bsr (NewBits - 8)) band 16#FF, - decode(Rest, <>, NewBits - 8, NewValue); - false -> - decode(Rest, Acc, NewBits, NewValue) - end. - -alphabet_val(C) when $A =< C, C =< $Z -> - C - $A; -alphabet_val(C) when $2 =< C, C =< $7 -> - 26 + (C - $2); -alphabet_val($=) -> - 0. \ No newline at end of file From 7a31717f5aae7263981fd6f4bc2ba6b0d26ea8b3 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 13 Mar 2026 00:14:15 +0000 Subject: [PATCH 097/135] impr: Added tests to dev_name 52char --- src/dev_manifest.erl | 31 ++++------------ src/dev_name.erl | 86 +++++++++++++++++++++++++++++++------------ src/hb_test_utils.erl | 19 ++++++++++ 3 files changed, 89 insertions(+), 47 deletions(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index 0de5401f1..38d8ca4d8 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -348,8 +348,8 @@ manifest_inner_redirect_test() -> %% Define the store LmdbStore = hb_test_utils:test_store(), %% Load transaction information to the store - load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), + hb_test_utils:load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), + hb_test_utils:load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), %% Start node Opts = #{store => LmdbStore}, Node = hb_http_server:start_node(Opts), @@ -366,9 +366,9 @@ manifest_inner_redirect_test() -> %% @doc Accessing `/TXID/assets/ArticleBlock-Dtwjc54T.js` should return valid message. access_key_path_in_manifest_test() -> LmdbStore = hb_test_utils:test_store(), - load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - load_and_store(LmdbStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), + hb_test_utils:load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), + hb_test_utils:load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), + hb_test_utils:load_and_store(LmdbStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), Opts = #{store => LmdbStore}, Node = hb_http_server:start_node(Opts), ?assertMatch( @@ -384,8 +384,8 @@ access_key_path_in_manifest_test() -> %% folder structure, like `assets/not_found.js . manifest_should_fallback_on_not_found_path_test() -> LmdbStore = hb_test_utils:test_store(), - load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), + hb_test_utils:load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), + hb_test_utils:load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), Opts = #{store => LmdbStore}, Node = hb_http_server:start_node(Opts), ?assertMatch( @@ -396,20 +396,3 @@ manifest_should_fallback_on_not_found_path_test() -> Opts ) ). - -%% @doc Load ans104 binary files to a store. -load_and_store(LmdbStore, File) -> - Opts = #{}, - {ok, SerializedItem} = - file:read_file( - hb_util:bin( - <<"test/arbundles.js/ans-104-manifest-", File/binary>> - ) - ), - Message = hb_message:convert( - ar_bundles:deserialize(SerializedItem), - <<"structured@1.0">>, - <<"ans104@1.0">>, - Opts - ), - _ = hb_cache:write(Message, #{store => LmdbStore}). \ No newline at end of file diff --git a/src/dev_name.erl b/src/dev_name.erl index 4967fe4c4..6e36aaba7 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -257,8 +257,6 @@ load_and_execute_test() -> arns_opts() -> JSONNames = <<"G_gb7SAgogHMtmqycwaHaC6uC-CZ3akACdFv5PUaEE8">>, Path = <>, - % TODO: Delete? - hb_http_server:start_node(#{}), TempStore = hb_test_utils:test_store(), #{ store => @@ -311,29 +309,71 @@ subdomain_to_tx_id_test() -> Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, ?assertEqual(<<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, subdomain_to_tx_id(Subdomain)). +resolve_52char_subdomain_if_txid_not_present_test() -> + Opts = load_manifest_opts(), + %% Test to load manifest with only subdomain + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _}}}, + hb_http:get( + Node, + #{ + <<"path">> => <<"/">>, + <<"host">> => <> + }, + Opts + ) + ). + resolve_52char_subdomain_asset_if_txid_not_present_test() -> + Opts = load_manifest_opts(), + %% Test to load asset with only subdomain (no TX ID present). Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, - Node = hb_http_server:start_node(#{}), - {ok, R} = hb_http:get( - Node, - #{ - <<"path">> => <<"/assets/index-C_KRlCcV.js">>, - <<"host">> => <> - } - ), - ?event(error, {r, R}). + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _}}}, + hb_http:get( + Node, + #{ + <<"path">> => <<"/assets/ArticleBlock-Dtwjc54T.js">>, + <<"host">> => <> + }, + Opts + ) + ). -resolve_52char_subdomain_if_txid_not_present_test() -> - Opts = arns_opts(), - %% TX: 42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA +ignore_52char_subdomain_if_txid_present_test() -> + Opts = load_manifest_opts(), Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, Node = hb_http_server:start_node(Opts), - {ok, R} = hb_http:get( - Node, - #{ - <<"path">> => <<"/">>, - <<"host">> => <> - }, - Opts - ), - ?event(error, {r, R}). + ?assertMatch( + {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _}}}, + hb_http:get( + Node, + #{ + <<"path">> => <<"/oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">>, + <<"host">> => <> + }, + Opts + ) + ). + +load_manifest_opts() -> + TempStore = hb_test_utils:test_store(), + %% Load TX data into the store + hb_test_utils:load_and_store(TempStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), + hb_test_utils:load_and_store(TempStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), + hb_test_utils:load_and_store(TempStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), + %% Opts + #{ + store => [TempStore], + on => #{ + <<"request">> => + [ + #{<<"device">> => <<"name@1.0">>}, + #{<<"device">> => <<"manifest@1.0">>} + ] + } + }. + diff --git a/src/hb_test_utils.erl b/src/hb_test_utils.erl index 02e36a1a8..b8ab409b4 100644 --- a/src/hb_test_utils.erl +++ b/src/hb_test_utils.erl @@ -7,6 +7,7 @@ -export([benchmark/1, benchmark/2, benchmark/3, benchmark_iterations/2]). -export([benchmark_print/2, benchmark_print/3, benchmark_print/4]). -export([compare_events/3, compare_events/4, compare_events/5]). +-export([load_and_store/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -261,3 +262,21 @@ format_time(Time) when is_integer(Time) -> hb_util:human_int(Time) ++ "s"; format_time(Time) -> hb_util:human_int(Time * 1000) ++ "ms". + +%% @doc Load ans104 binary files to a store. +load_and_store(Store, File) -> + Opts = #{}, + {ok, SerializedItem} = + file:read_file( + hb_util:bin( + <<"test/arbundles.js/ans-104-manifest-", File/binary>> + ) + ), + Message = hb_message:convert( + ar_bundles:deserialize(SerializedItem), + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ), + _ = hb_cache:write(Message, #{store => Store}). + From d44ca5421af686665c2ff8206cfe8a489f955677 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 13 Mar 2026 17:50:02 +0000 Subject: [PATCH 098/135] impr: Improve tests, add 404 on ARNS not in the list and not a 52 char subdomain --- src/dev_name.erl | 123 +++++++++++++++++++++++++++++++++++++++-------- src/hb_util.erl | 2 +- 2 files changed, 105 insertions(+), 20 deletions(-) diff --git a/src/dev_name.erl b/src/dev_name.erl index 6e36aaba7..4f7bde1b0 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -46,17 +46,27 @@ resolve(Key, _, Req, Opts) -> ArnsResolver end. -%% @doc Try to resolve 52char subdomain back to its original TX ID when TX ID -%% isn't present. -resolve_52char(_, #{<<"body">> := [ID | _]}, _) when ?IS_ID(ID) -> - ?event({resolve_52char, {skip_becase_id_found, ID}}), - not_found; -resolve_52char(Key, _, Opts) when byte_size(Key) == 52 -> +%% @doc Try to resolve 52char subdomain back to its original TX ID +resolve_52char(Key, HookMsg, Opts) when byte_size(Key) == 52 -> TXID = subdomain_to_tx_id(Key), - ?event({resolve_52char, {key, Key}, {txid, TXID}}), - hb_cache:read(TXID, Opts); -resolve_52char(_, _, _) -> - ?event({resolve_52char, nothing_matched}), + %% Clean up entries that doesn't have <<"path">> as key or aren't IDs. + Body = lists:filter( + fun (#{<<"path">> := _ }) -> true; + (ID) when ?IS_ID(ID) -> true; + (_) -> false + end, + maps:get(<<"body">>, HookMsg, []) + ), + case Body of + [TXID | _] -> + ?event({resolve_52char, same_txid_as_subdomain}), + %% To be resolved later, under normal flow + not_found; + _ -> + hb_cache:read(TXID, Opts) + end; +resolve_52char(Key, _, _) -> + ?event({resolve_52char, {key, Key}, {message, <<"Key isn't not a valid 52 char subdomain">>}}), not_found. %% @doc Find the first resolver that matches the key and return its value. @@ -98,6 +108,8 @@ request(HookMsg, HookReq, Opts) -> {ok, ResolvedMsg} ?= resolve(Name, HookMsg, HookReq, Opts), ModReq = case hb_maps:find(<<"body">>, HookReq, Opts) of + {ok, [ID|Rest]} when ?IS_ID(ID) -> + [ResolvedMsg|Rest]; {ok, [OldBase|Rest]} -> [overlay_loaded(OldBase, ResolvedMsg, Opts)|Rest]; {ok, []} -> @@ -114,8 +126,15 @@ request(HookMsg, HookReq, Opts) -> {ok, #{ <<"body">> => ModReq }} else Reason -> - ?event({request_hook_skip, {reason, Reason}, {hook_req, HookReq}}), - {ok, HookReq} + case maps:get(<<"body">>, HookReq, []) of + [] -> + ?event({request_hook_404, root_path}), + % No path provided should return 404 if not resolved (via ARNS or 52 char subdomain) + {error, #{<<"status">> => 404, <<"body">> => <<"Not Found">>}}; + _ -> + ?event({request_hook_skip, {reason, Reason}, {hook_req, HookReq}}), + {ok, HookReq} + end end. %% @doc Takes a request-given host and the host value in the node message and @@ -305,11 +324,31 @@ arns_host_resolution_test() -> ) ). +%% @doc If the sudbdomain provided isn't a valid ARNS or a 52 char subdomain +%% it should return 404 instead of HyperBEAM page. +invalid_arns_and_not_52char_host_resolution_gives_404_test() -> + Opts = arns_opts(), + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {error, #{<<"status">> := 404}}, + hb_http:get( + Node, + #{ + <<"path">> => <<"/">>, + <<"host">> => <<"non-existing-subdomain.localhost">> + }, + Opts + ) + ). + +%% @doc Unit test for 52 char subdomain to TX ID logic subdomain_to_tx_id_test() -> Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, ?assertEqual(<<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, subdomain_to_tx_id(Subdomain)). +%% @doc Resolving a 52 char subdomain without a TXID in the path should work. resolve_52char_subdomain_if_txid_not_present_test() -> + TestPath = <<"/">>, Opts = load_manifest_opts(), %% Test to load manifest with only subdomain Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, @@ -319,14 +358,17 @@ resolve_52char_subdomain_if_txid_not_present_test() -> hb_http:get( Node, #{ - <<"path">> => <<"/">>, + <<"path">> => TestPath, <<"host">> => <> }, Opts ) ). +%% @doc Loading assets from a manifest where only a 52 char subdomain is +%% provided should work. resolve_52char_subdomain_asset_if_txid_not_present_test() -> + TestPath = <<"/assets/ArticleBlock-Dtwjc54T.js">>, Opts = load_manifest_opts(), %% Test to load asset with only subdomain (no TX ID present). Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, @@ -336,15 +378,19 @@ resolve_52char_subdomain_asset_if_txid_not_present_test() -> hb_http:get( Node, #{ - <<"path">> => <<"/assets/ArticleBlock-Dtwjc54T.js">>, + <<"path">> => TestPath, <<"host">> => <> }, Opts ) ). -ignore_52char_subdomain_if_txid_present_test() -> - Opts = load_manifest_opts(), +%% @doc Loading assets from a manifest where a 52 char subdomain and TX ID +%% is provided should work. +resolve_52char_subdomain_asset_if_txid_present_test() -> + TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA/assets/ArticleBlock-Dtwjc54T.js">>, + Opts = load_manifest_opts(), + %% Test to load asset with only subdomain (no TX ID present). Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, Node = hb_http_server:start_node(Opts), ?assertMatch( @@ -352,7 +398,47 @@ ignore_52char_subdomain_if_txid_present_test() -> hb_http:get( Node, #{ - <<"path">> => <<"/oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">>, + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc When both 52 char subdomain and TX ID are provided and equal, ignore +%% the TXID from the assets path. +ignore_52char_subdomain_if_the_same_txid_is_provided_test() -> + TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, + Opts = load_manifest_opts(), + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _}}}, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc When a valid 52 char subdomain TXID doesn't match the TX ID provided, +%% the subdomain TXID is loaded, and tries to access the assets path defined. +%% In this case, sinse no assets exists with this TX ID, it should load the +%% index. +when_52char_subdomain_txid_it_doesnt_match_txid_provided_test() -> + TestPath = <<"/oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">>, + Opts = load_manifest_opts(), + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _}}}, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, <<"host">> => <> }, Opts @@ -375,5 +461,4 @@ load_manifest_opts() -> #{<<"device">> => <<"manifest@1.0">>} ] } - }. - + }. \ No newline at end of file diff --git a/src/hb_util.erl b/src/hb_util.erl index 8a0030156..7e4db770e 100644 --- a/src/hb_util.erl +++ b/src/hb_util.erl @@ -839,4 +839,4 @@ base58_encode_int(N) -> Rem = N rem 58, Char = binary:at(Alphabet, Rem), Rest = base58_encode_int(N div 58), - <>. + <>. \ No newline at end of file From c3617d33ca051e3ecc6c6fac51378c04ceee11d2 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Fri, 13 Mar 2026 18:45:41 +0000 Subject: [PATCH 099/135] impr: Fix PR comments --- src/dev_manifest.erl | 14 +++++++------- src/dev_name.erl | 31 ++++++++++++++----------------- src/hb_test_utils.erl | 4 ++-- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index 38d8ca4d8..707792ca2 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -348,8 +348,8 @@ manifest_inner_redirect_test() -> %% Define the store LmdbStore = hb_test_utils:test_store(), %% Load transaction information to the store - hb_test_utils:load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - hb_test_utils:load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), + hb_test_utils:preload(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), + hb_test_utils:preload(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), %% Start node Opts = #{store => LmdbStore}, Node = hb_http_server:start_node(Opts), @@ -366,9 +366,9 @@ manifest_inner_redirect_test() -> %% @doc Accessing `/TXID/assets/ArticleBlock-Dtwjc54T.js` should return valid message. access_key_path_in_manifest_test() -> LmdbStore = hb_test_utils:test_store(), - hb_test_utils:load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - hb_test_utils:load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - hb_test_utils:load_and_store(LmdbStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), + hb_test_utils:preload(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), + hb_test_utils:preload(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), + hb_test_utils:preload(LmdbStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), Opts = #{store => LmdbStore}, Node = hb_http_server:start_node(Opts), ?assertMatch( @@ -384,8 +384,8 @@ access_key_path_in_manifest_test() -> %% folder structure, like `assets/not_found.js . manifest_should_fallback_on_not_found_path_test() -> LmdbStore = hb_test_utils:test_store(), - hb_test_utils:load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - hb_test_utils:load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), + hb_test_utils:preload(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), + hb_test_utils:preload(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), Opts = #{store => LmdbStore}, Node = hb_http_server:start_node(Opts), ?assertMatch( diff --git a/src/dev_name.erl b/src/dev_name.erl index 4f7bde1b0..5c5109d97 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -28,7 +28,7 @@ info(_) -> resolve(Key, _, Req, Opts) -> Resolvers = hb_opts:get(name_resolvers, [], Opts), ?event({resolvers, Resolvers}), - ArnsResolver = case match_resolver(Key, Resolvers, Opts) of + NameResolver = case match_resolver(Key, Resolvers, Opts) of {ok, Resolved} -> case hb_util:atom(hb_ao:get(<<"load">>, Req, true, Opts)) of false -> @@ -39,22 +39,19 @@ resolve(Key, _, Req, Opts) -> not_found -> not_found end, - case ArnsResolver of + case NameResolver of not_found -> resolve_52char(Key, Req, Opts); _ -> - ArnsResolver + NameResolver end. %% @doc Try to resolve 52char subdomain back to its original TX ID resolve_52char(Key, HookMsg, Opts) when byte_size(Key) == 52 -> - TXID = subdomain_to_tx_id(Key), + TXID = subdomain_to_txid(Key), %% Clean up entries that doesn't have <<"path">> as key or aren't IDs. Body = lists:filter( - fun (#{<<"path">> := _ }) -> true; - (ID) when ?IS_ID(ID) -> true; - (_) -> false - end, + fun (Map) -> (is_map(map) andalso maps:is_key(<<"path">>, Map)) orelse ?IS_ID(Map) end, maps:get(<<"body">>, HookMsg, []) ), case Body of @@ -129,7 +126,7 @@ request(HookMsg, HookReq, Opts) -> case maps:get(<<"body">>, HookReq, []) of [] -> ?event({request_hook_404, root_path}), - % No path provided should return 404 if not resolved (via ARNS or 52 char subdomain) + % No path provided should return 404 if not resolved (via name resolvers or 52 char subdomain) {error, #{<<"status">> => 404, <<"body">> => <<"Not Found">>}}; _ -> ?event({request_hook_skip, {reason, Reason}, {hook_req, HookReq}}), @@ -163,8 +160,8 @@ overlay_loaded({as, DevID, Base}, Resolved, Opts) -> overlay_loaded(Base, Resolved, Opts) -> hb_maps:merge(Base, Resolved, Opts). -subdomain_to_tx_id(Subdomain) when byte_size(Subdomain) == 52 -> - b64fast:encode(base32:decode(Subdomain)). +subdomain_to_txid(Subdomain) when byte_size(Subdomain) == 52 -> + hb_util:human_id(b64fast:encode(base32:decode(Subdomain))). %%% Tests. @@ -342,9 +339,9 @@ invalid_arns_and_not_52char_host_resolution_gives_404_test() -> ). %% @doc Unit test for 52 char subdomain to TX ID logic -subdomain_to_tx_id_test() -> +subdomain_to_txid_test() -> Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, - ?assertEqual(<<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, subdomain_to_tx_id(Subdomain)). + ?assertEqual(<<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, subdomain_to_txid(Subdomain)). %% @doc Resolving a 52 char subdomain without a TXID in the path should work. resolve_52char_subdomain_if_txid_not_present_test() -> @@ -448,9 +445,9 @@ when_52char_subdomain_txid_it_doesnt_match_txid_provided_test() -> load_manifest_opts() -> TempStore = hb_test_utils:test_store(), %% Load TX data into the store - hb_test_utils:load_and_store(TempStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - hb_test_utils:load_and_store(TempStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - hb_test_utils:load_and_store(TempStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), + hb_test_utils:preload(TempStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), + hb_test_utils:preload(TempStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), + hb_test_utils:preload(TempStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), %% Opts #{ store => [TempStore], @@ -461,4 +458,4 @@ load_manifest_opts() -> #{<<"device">> => <<"manifest@1.0">>} ] } - }. \ No newline at end of file + }. diff --git a/src/hb_test_utils.erl b/src/hb_test_utils.erl index b8ab409b4..4774d4f3c 100644 --- a/src/hb_test_utils.erl +++ b/src/hb_test_utils.erl @@ -7,7 +7,7 @@ -export([benchmark/1, benchmark/2, benchmark/3, benchmark_iterations/2]). -export([benchmark_print/2, benchmark_print/3, benchmark_print/4]). -export([compare_events/3, compare_events/4, compare_events/5]). --export([load_and_store/2]). +-export([preload/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -264,7 +264,7 @@ format_time(Time) -> hb_util:human_int(Time * 1000) ++ "ms". %% @doc Load ans104 binary files to a store. -load_and_store(Store, File) -> +preload(Store, File) -> Opts = #{}, {ok, SerializedItem} = file:read_file( From e09efab30ab4d24907258c404b8e01c3a8449dc8 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 16:19:29 -0400 Subject: [PATCH 100/135] impr: separate `base32` name resolution into standalone device --- src/dev_b32_name.erl | 321 ++++++++++++++++++++++++++++++++++++++++++ src/dev_name.erl | 252 ++++++++------------------------- src/hb_opts.erl | 2 + src/hb_test_utils.erl | 23 ++- 4 files changed, 389 insertions(+), 209 deletions(-) create mode 100644 src/dev_b32_name.erl diff --git a/src/dev_b32_name.erl b/src/dev_b32_name.erl new file mode 100644 index 000000000..63b2c7139 --- /dev/null +++ b/src/dev_b32_name.erl @@ -0,0 +1,321 @@ +%%% @doc Allows Arweave message IDs to be used via their base32 encoding as +%%% subdomains on a HyperBEAM node. +-module(dev_b32_name). +-export([info/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +info(_Opts) -> + #{ + default => fun get/4, + excludes => [<<"keys">>, <<"set">>] + }. + +%% @doc Try to resolve 52char subdomain back to its original TX ID +get(Key, _, _HookMsg, _Opts) -> + ?event({resolve_52char, {key, Key}}), + case decode(Key) of + error -> + ?event({not_base32_id, {key, Key}}), + {error, not_found}; + ID -> + ?event({resolved_52char, {key, Key}, {id, ID}}), + {ok, ID} + end. + +%% @doc If the key is a 52-character binary, attempt to decode it as base32. +%% Else, return `error`. +decode(Key) when byte_size(Key) == 52 -> + try hb_util:human_id(base32:decode(Key)) + catch _:_ -> error + end; +decode(_Key) -> error. + +%% @doc Convert an ID into its base32 encoded string representation. +encode(ID) when ?IS_ID(ID) -> + hb_util:bin( + string:replace( + string:to_lower( + hb_util:list(base32:encode(hb_util:native_id(ID))) + ), + "=", + "", + all + ) + ). + +%%% Tests + +%% @doc If the sudbdomain provided isn't a valid ARNS or a 52 char subdomain +%% it should return 404 instead of HyperBEAM page. +invalid_arns_and_not_52char_host_resolution_gives_404_test() -> + Opts = dev_name:test_arns_opts(), + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {error, #{<<"status">> := 404}}, + hb_http:get( + Node, + #{ + <<"path">> => <<"/">>, + <<"host">> => <<"non-existing-subdomain.localhost">> + }, + Opts + ) + ). + +%% @doc Unit test for 52 char subdomain to TX ID logic +key_to_id_test() -> + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + ?assertEqual( + <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, + decode(Subdomain) + ). + +%% @doc Resolving a 52 char subdomain without a TXID in the path should work. +empty_path_manifest_test() -> + TestPath = <<"/">>, + Opts = load_manifest_opts(), + %% Test to load manifest with only subdomain + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"status">> := 200, + <<"commitments">> := + #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _} + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc Loading assets from a manifest where only a 52 char subdomain is +%% provided should work. +resolve_52char_subdomain_asset_if_txid_not_present_test() -> + TestPath = <<"/assets/ArticleBlock-Dtwjc54T.js">>, + Opts = load_manifest_opts(), + %% Test to load asset with only subdomain (no TX ID present). + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"status">> := 200, + <<"commitments">> := + #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _} + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc Loading assets from a manifest where a 52 char subdomain and TX ID +%% is provided should work. +subdomain_matches_path_id_and_loads_asset_test() -> + TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA/assets/ArticleBlock-Dtwjc54T.js">>, + Opts = load_manifest_opts(), + %% Test to load asset with only subdomain (no TX ID present). + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"status">> := 200, + <<"commitments">> := + #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _} + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc Validate the behavior when a subdomain and primary path ID match. The +%% duplicated ID in the request message stream should be ignored. +subdomain_matches_path_id_test() -> + #{ id1 := ID1, opts := Opts } = test_opts(), + ?assertMatch( + {ok, 1}, + hb_http:get( + hb_http_server:start_node(Opts), + #{ + <<"path">> => <>, + <<"host">> => subdomain(ID1, Opts) + }, + Opts + ) + ). + +%% @doc Validate the behavior when a subdomain and primary path ID match. Both +%% IDs should be executed, the subdomain first then the path ID. +subdomain_does_not_match_path_id_test() -> + #{ id1 := ID1, id2 := ID2, opts := Opts } + = test_opts(), + ?assertMatch( + {error, not_found}, + hb_http:get( + hb_http_server:start_node(Opts), + #{ + <<"path">> => <>, + <<"host">> => subdomain(ID2, Opts) + }, + Opts + ) + ). + +%% @doc When both 52 char subdomain and TX ID are provided and equal, ignore +%% the TXID from the assets path. +manifest_subdomain_matches_path_id_test() -> + TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, + Opts = load_manifest_opts(), + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"status">> := 200, + <<"commitments">> := + #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _} + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc When a valid 52 char subdomain TXID doesn't match the TX ID provided, +%% the subdomain TXID is loaded, and tries to access the assets path defined. +%% In this case, sinse no assets exists with this TX ID, it should load the +%% index. +manifest_subdomain_does_not_match_path_id_test() -> + TestPath = <<"/oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">>, + Opts = load_manifest_opts(), + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"commitments">> := + #{ + <<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _ + } + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +test_opts() -> + Store = [hb_test_utils:test_store()], + BaseOpts = #{ store => Store, priv_wallet => ar_wallet:new() }, + Msg1 = + #{ + <<"a">> => 1, + <<"b">> => 2, + <<"nested">> => #{ + <<"z">> => 26 + } + }, + Msg2 = + #{ + <<"a">> => 2, + <<"b">> => 4 + }, + MsgWithPath = + #{ + <<"a">> => 3, + <<"b">> => 6, + <<"c">> => 9, + <<"path">> => <<"nested">> + }, + SignedMsg3 = + hb_message:commit( + #{ <<"a">> => 3, <<"b">> => 6, <<"c">> => 9 }, + BaseOpts + ), + {ok, UnsignedID1} = hb_cache:write(Msg1, BaseOpts), + {ok, UnsignedID2} = hb_cache:write(Msg2, BaseOpts), + {ok, UnsignedIDWithPath} = hb_cache:write(MsgWithPath, BaseOpts), + {ok, _UnsignedID3} = hb_cache:write(SignedMsg3, BaseOpts), + #{ + opts => + BaseOpts#{ + store => Store, + name_resolvers => [#{ <<"device">> => <<"b32-name@1.0">> }], + on => + #{ + <<"request">> => [#{<<"device">> => <<"name@1.0">>}] + } + }, + id1 => UnsignedID1, + id2 => UnsignedID2, + id3 => SignedMsg3, + id_with_path => UnsignedIDWithPath, + messages => [Msg1, Msg2, SignedMsg3, MsgWithPath] + }. + +%% @doc Returns the subdomain for a given ID for testing purposes. +subdomain(ID, _Opts) when ?IS_ID(ID) -> + <<(encode(ID))/binary, ".localhost">>; +subdomain(ID, Opts) -> + subdomain(hb_message:id(ID, unsigned, Opts), Opts). + +%% @doc Returns `Opts' with a test environment preloaded with manifest related +%% IDs. +load_manifest_opts() -> + TempStore = hb_test_utils:test_store(), + BaseOpts = #{ store => [TempStore] }, + %% Load TX data into the store + lists:foreach( + fun(Ref) -> + hb_test_utils:preload( + BaseOpts, + <<"test/arbundles.js/ans-104-manifest-", Ref/binary>> + ) + end, + [ + <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>, + <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>, + <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">> + ] + ), + BaseOpts#{ + name_resolvers => [#{ <<"device">> => <<"b32-name@1.0">> }], + on => + #{ + <<"request">> => + [ + #{<<"device">> => <<"name@1.0">>}, + #{<<"device">> => <<"manifest@1.0">>} + ] + } + }. diff --git a/src/dev_name.erl b/src/dev_name.erl index 5c5109d97..8d8305239 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -5,6 +5,8 @@ %%% first resolver that matches. -module(dev_name). -export([info/1, request/3]). +%%% Public helpers. +-export([test_arns_opts/0]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -28,44 +30,15 @@ info(_) -> resolve(Key, _, Req, Opts) -> Resolvers = hb_opts:get(name_resolvers, [], Opts), ?event({resolvers, Resolvers}), - NameResolver = case match_resolver(Key, Resolvers, Opts) of + case match_resolver(Key, Resolvers, Opts) of {ok, Resolved} -> case hb_util:atom(hb_ao:get(<<"load">>, Req, true, Opts)) of - false -> - {ok, Resolved}; - true -> - hb_cache:read(Resolved, Opts) + false -> {ok, Resolved}; + true -> hb_cache:read(Resolved, Opts) end; - not_found -> - not_found - end, - case NameResolver of - not_found -> - resolve_52char(Key, Req, Opts); - _ -> - NameResolver + not_found -> not_found end. -%% @doc Try to resolve 52char subdomain back to its original TX ID -resolve_52char(Key, HookMsg, Opts) when byte_size(Key) == 52 -> - TXID = subdomain_to_txid(Key), - %% Clean up entries that doesn't have <<"path">> as key or aren't IDs. - Body = lists:filter( - fun (Map) -> (is_map(map) andalso maps:is_key(<<"path">>, Map)) orelse ?IS_ID(Map) end, - maps:get(<<"body">>, HookMsg, []) - ), - case Body of - [TXID | _] -> - ?event({resolve_52char, same_txid_as_subdomain}), - %% To be resolved later, under normal flow - not_found; - _ -> - hb_cache:read(TXID, Opts) - end; -resolve_52char(Key, _, _) -> - ?event({resolve_52char, {key, Key}, {message, <<"Key isn't not a valid 52 char subdomain">>}}), - not_found. - %% @doc Find the first resolver that matches the key and return its value. match_resolver(_Key, [], _Opts) -> not_found; @@ -104,14 +77,11 @@ request(HookMsg, HookReq, Opts) -> {ok, Name} ?= name_from_host(Host, hb_opts:get(node_host, no_host, Opts)), {ok, ResolvedMsg} ?= resolve(Name, HookMsg, HookReq, Opts), ModReq = - case hb_maps:find(<<"body">>, HookReq, Opts) of - {ok, [ID|Rest]} when ?IS_ID(ID) -> - [ResolvedMsg|Rest]; - {ok, [OldBase|Rest]} -> - [overlay_loaded(OldBase, ResolvedMsg, Opts)|Rest]; - {ok, []} -> - [ResolvedMsg] - end, + maybe_append_named_message( + ResolvedMsg, + hb_util:ok(hb_maps:find(<<"body">>, HookReq, Opts)), + Opts + ), ?event( {request_with_prepended_path, {name, Name}, @@ -126,7 +96,8 @@ request(HookMsg, HookReq, Opts) -> case maps:get(<<"body">>, HookReq, []) of [] -> ?event({request_hook_404, root_path}), - % No path provided should return 404 if not resolved (via name resolvers or 52 char subdomain) + % No path provided should return 404 if not resolved + % (via name resolvers or 52 char subdomain) {error, #{<<"status">> => 404, <<"body">> => <<"Not Found">>}}; _ -> ?event({request_hook_skip, {reason, Reason}, {hook_req, HookReq}}), @@ -134,6 +105,46 @@ request(HookMsg, HookReq, Opts) -> end end. +%% @doc After finding a hit for a named message, we should ensure that it is the +%% base message for the evaluation. If it is already present in the request, +%% however, we should not add it twice. Instead, we must add the version that +%% is loaded (if applicable). +%% +%% Eg: +%% base32IDA.hyperbeam/ -> [IDA] +%% base32IDA.hyperbeam/base64urlIDA/xyz -> [IDA, xyz] +%% base32IDA.hyperbeam/base64urlIDB/xyz -> [IDA, IDB, xyz] +maybe_append_named_message(ResolvedMsg, [], _Opts) -> [ResolvedMsg]; +maybe_append_named_message(ResolvedMsg, OldReq = [OldBase|ReqMsgsRest], Opts) -> + case permissive_id(OldBase, Opts) == permissive_id(ResolvedMsg, Opts) of + true when is_map(OldBase) or is_list(OldBase) -> OldReq; + true -> [ResolvedMsg|ReqMsgsRest]; + false -> + case is_map(OldBase) andalso hb_maps:get(<<"path">>, OldBase, not_found, Opts) of + not_found -> + ?event( + {skipping_old_base, + {old_base, OldBase}, + {resolved_msg, ResolvedMsg} + } + ), + [ResolvedMsg|ReqMsgsRest]; + _ -> [ResolvedMsg, as_message_or_link(OldBase)|ReqMsgsRest] + end + end. + +%% @doc Takes a message or resolution request (`as` or `resolve`) -- whether in +%% the form of an ID, link, or loaded map -- and returns its ID. +permissive_id(ID, _Opts) when ?IS_ID(ID) -> ID; +permissive_id({link, ID, _LinkOpts}, _Opts) -> ID; +permissive_id({as, _Device, Msg}, Opts) -> permissive_id(Msg, Opts); +permissive_id(Msg, Opts) when is_map(Msg) -> hb_message:id(Msg, signed, Opts). + +%% @doc Ensure that a message reference is converted to a message or link. +as_message_or_link(ID) when ?IS_ID(ID) -> {link, ID, #{}}; +as_message_or_link(Msg) when is_map(Msg) -> Msg; +as_message_or_link(Link) when ?IS_LINK(Link) -> Link. + %% @doc Takes a request-given host and the host value in the node message and %% returns only the name component of the host, if it is present. If no name is %% present, an empty binary is returned. @@ -153,16 +164,6 @@ name_from_host(ReqHost, RawNodeHost) -> ), name_from_host(WithoutNodeHost, no_host). -%% @doc Merge the base message with the resolved message, ensuring that `~` as -%% device specifiers are preserved. -overlay_loaded({as, DevID, Base}, Resolved, Opts) -> - {as, DevID, hb_maps:merge(Base, Resolved, Opts)}; -overlay_loaded(Base, Resolved, Opts) -> - hb_maps:merge(Base, Resolved, Opts). - -subdomain_to_txid(Subdomain) when byte_size(Subdomain) == 52 -> - hb_util:human_id(b64fast:encode(base32:decode(Subdomain))). - %%% Tests. no_resolvers_test() -> @@ -270,7 +271,7 @@ load_and_execute_test() -> %% @doc Return an `Opts` for an environment with the default ARNS name export %% and a temporary store for the test. -arns_opts() -> +test_arns_opts() -> JSONNames = <<"G_gb7SAgogHMtmqycwaHaC6uC-CZ3akACdFv5PUaEE8">>, Path = <>, TempStore = hb_test_utils:test_store(), @@ -293,7 +294,7 @@ arns_opts() -> %% @doc Names from JSON test. arns_json_snapshot_test() -> - Opts = arns_opts(), + Opts = test_arns_opts(), ?assertMatch( {ok, <<"text/html">>}, hb_ao:resolve_many( @@ -307,7 +308,7 @@ arns_json_snapshot_test() -> ). arns_host_resolution_test() -> - Opts = arns_opts(), + Opts = test_arns_opts(), Node = hb_http_server:start_node(Opts), ?assertMatch( {ok, <<"text/html">>}, @@ -319,143 +320,4 @@ arns_host_resolution_test() -> }, Opts ) - ). - -%% @doc If the sudbdomain provided isn't a valid ARNS or a 52 char subdomain -%% it should return 404 instead of HyperBEAM page. -invalid_arns_and_not_52char_host_resolution_gives_404_test() -> - Opts = arns_opts(), - Node = hb_http_server:start_node(Opts), - ?assertMatch( - {error, #{<<"status">> := 404}}, - hb_http:get( - Node, - #{ - <<"path">> => <<"/">>, - <<"host">> => <<"non-existing-subdomain.localhost">> - }, - Opts - ) - ). - -%% @doc Unit test for 52 char subdomain to TX ID logic -subdomain_to_txid_test() -> - Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, - ?assertEqual(<<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, subdomain_to_txid(Subdomain)). - -%% @doc Resolving a 52 char subdomain without a TXID in the path should work. -resolve_52char_subdomain_if_txid_not_present_test() -> - TestPath = <<"/">>, - Opts = load_manifest_opts(), - %% Test to load manifest with only subdomain - Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, - Node = hb_http_server:start_node(Opts), - ?assertMatch( - {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _}}}, - hb_http:get( - Node, - #{ - <<"path">> => TestPath, - <<"host">> => <> - }, - Opts - ) - ). - -%% @doc Loading assets from a manifest where only a 52 char subdomain is -%% provided should work. -resolve_52char_subdomain_asset_if_txid_not_present_test() -> - TestPath = <<"/assets/ArticleBlock-Dtwjc54T.js">>, - Opts = load_manifest_opts(), - %% Test to load asset with only subdomain (no TX ID present). - Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, - Node = hb_http_server:start_node(Opts), - ?assertMatch( - {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _}}}, - hb_http:get( - Node, - #{ - <<"path">> => TestPath, - <<"host">> => <> - }, - Opts - ) - ). - -%% @doc Loading assets from a manifest where a 52 char subdomain and TX ID -%% is provided should work. -resolve_52char_subdomain_asset_if_txid_present_test() -> - TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA/assets/ArticleBlock-Dtwjc54T.js">>, - Opts = load_manifest_opts(), - %% Test to load asset with only subdomain (no TX ID present). - Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, - Node = hb_http_server:start_node(Opts), - ?assertMatch( - {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _}}}, - hb_http:get( - Node, - #{ - <<"path">> => TestPath, - <<"host">> => <> - }, - Opts - ) - ). - -%% @doc When both 52 char subdomain and TX ID are provided and equal, ignore -%% the TXID from the assets path. -ignore_52char_subdomain_if_the_same_txid_is_provided_test() -> - TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, - Opts = load_manifest_opts(), - Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, - Node = hb_http_server:start_node(Opts), - ?assertMatch( - {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _}}}, - hb_http:get( - Node, - #{ - <<"path">> => TestPath, - <<"host">> => <> - }, - Opts - ) - ). - -%% @doc When a valid 52 char subdomain TXID doesn't match the TX ID provided, -%% the subdomain TXID is loaded, and tries to access the assets path defined. -%% In this case, sinse no assets exists with this TX ID, it should load the -%% index. -when_52char_subdomain_txid_it_doesnt_match_txid_provided_test() -> - TestPath = <<"/oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">>, - Opts = load_manifest_opts(), - Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, - Node = hb_http_server:start_node(Opts), - ?assertMatch( - {ok, #{<<"status">> := 200, <<"commitments">> := #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _}}}, - hb_http:get( - Node, - #{ - <<"path">> => TestPath, - <<"host">> => <> - }, - Opts - ) - ). - -load_manifest_opts() -> - TempStore = hb_test_utils:test_store(), - %% Load TX data into the store - hb_test_utils:preload(TempStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - hb_test_utils:preload(TempStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - hb_test_utils:preload(TempStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), - %% Opts - #{ - store => [TempStore], - on => #{ - <<"request">> => - [ - #{<<"device">> => <<"name@1.0">>}, - #{<<"device">> => <<"manifest@1.0">>} - ] - } - }. + ). \ No newline at end of file diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 7fd2e54b3..f32f690d7 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -43,6 +43,7 @@ -ifndef(TEST). -define(DEFAULT_NAME_RESOLVERS, [ + #{ <<"device">> => <<"~b32-name@1.0">> }, << "G_gb7SAgogHMtmqycwaHaC6uC-CZ3akACdFv5PUaEE8", "~json@1.0/deserialize&target=data" @@ -164,6 +165,7 @@ default_message() -> #{<<"name">> => <<"apply@1.0">>, <<"module">> => dev_apply}, #{<<"name">> => <<"auth-hook@1.0">>, <<"module">> => dev_auth_hook}, #{<<"name">> => <<"ans104@1.0">>, <<"module">> => dev_codec_ans104}, + #{<<"name">> => <<"b32-name@1.0">>, <<"module">> => dev_b32_name}, #{<<"name">> => <<"blacklist@1.0">>, <<"module">> => dev_blacklist}, #{<<"name">> => <<"bundler@1.0">>, <<"module">> => dev_bundler}, #{<<"name">> => <<"compute@1.0">>, <<"module">> => dev_cu}, diff --git a/src/hb_test_utils.erl b/src/hb_test_utils.erl index 4774d4f3c..2deb1ca44 100644 --- a/src/hb_test_utils.erl +++ b/src/hb_test_utils.erl @@ -264,19 +264,14 @@ format_time(Time) -> hb_util:human_int(Time * 1000) ++ "ms". %% @doc Load ans104 binary files to a store. -preload(Store, File) -> - Opts = #{}, - {ok, SerializedItem} = - file:read_file( - hb_util:bin( - <<"test/arbundles.js/ans-104-manifest-", File/binary>> - ) +preload(Opts, File) -> + {ok, SerializedItem} = file:read_file(hb_util:bin(File)), + Message = + hb_message:convert( + ar_bundles:deserialize(SerializedItem), + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts ), - Message = hb_message:convert( - ar_bundles:deserialize(SerializedItem), - <<"structured@1.0">>, - <<"ans104@1.0">>, - Opts - ), - _ = hb_cache:write(Message, #{store => Store}). + hb_cache:write(Message, Opts). From c2ede802d5f0b1236337d7cae045714cb5ccdaaa Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 16:56:36 -0400 Subject: [PATCH 101/135] fix: unify `~manifest@1.0` test environment generation --- src/dev_b32_name.erl | 34 ++++++++------------------------ src/dev_manifest.erl | 46 +++++++++++++++++++++++++++++--------------- 2 files changed, 38 insertions(+), 42 deletions(-) diff --git a/src/dev_b32_name.erl b/src/dev_b32_name.erl index 63b2c7139..868e53efb 100644 --- a/src/dev_b32_name.erl +++ b/src/dev_b32_name.erl @@ -26,9 +26,7 @@ get(Key, _, _HookMsg, _Opts) -> %% @doc If the key is a 52-character binary, attempt to decode it as base32. %% Else, return `error`. decode(Key) when byte_size(Key) == 52 -> - try hb_util:human_id(base32:decode(Key)) - catch _:_ -> error - end; + try hb_util:human_id(base32:decode(Key)) catch _:_ -> error end; decode(_Key) -> error. %% @doc Convert an ID into its base32 encoded string representation. @@ -74,7 +72,7 @@ key_to_id_test() -> %% @doc Resolving a 52 char subdomain without a TXID in the path should work. empty_path_manifest_test() -> TestPath = <<"/">>, - Opts = load_manifest_opts(), + Opts = manifest_opts(), %% Test to load manifest with only subdomain Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, Node = hb_http_server:start_node(Opts), @@ -100,7 +98,7 @@ empty_path_manifest_test() -> %% provided should work. resolve_52char_subdomain_asset_if_txid_not_present_test() -> TestPath = <<"/assets/ArticleBlock-Dtwjc54T.js">>, - Opts = load_manifest_opts(), + Opts = manifest_opts(), %% Test to load asset with only subdomain (no TX ID present). Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, Node = hb_http_server:start_node(Opts), @@ -126,7 +124,7 @@ resolve_52char_subdomain_asset_if_txid_not_present_test() -> %% is provided should work. subdomain_matches_path_id_and_loads_asset_test() -> TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA/assets/ArticleBlock-Dtwjc54T.js">>, - Opts = load_manifest_opts(), + Opts = manifest_opts(), %% Test to load asset with only subdomain (no TX ID present). Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, Node = hb_http_server:start_node(Opts), @@ -185,7 +183,7 @@ subdomain_does_not_match_path_id_test() -> %% the TXID from the assets path. manifest_subdomain_matches_path_id_test() -> TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, - Opts = load_manifest_opts(), + Opts = manifest_opts(), Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, Node = hb_http_server:start_node(Opts), ?assertMatch( @@ -212,7 +210,7 @@ manifest_subdomain_matches_path_id_test() -> %% index. manifest_subdomain_does_not_match_path_id_test() -> TestPath = <<"/oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">>, - Opts = load_manifest_opts(), + Opts = manifest_opts(), Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, Node = hb_http_server:start_node(Opts), ?assertMatch( @@ -291,24 +289,8 @@ subdomain(ID, Opts) -> %% @doc Returns `Opts' with a test environment preloaded with manifest related %% IDs. -load_manifest_opts() -> - TempStore = hb_test_utils:test_store(), - BaseOpts = #{ store => [TempStore] }, - %% Load TX data into the store - lists:foreach( - fun(Ref) -> - hb_test_utils:preload( - BaseOpts, - <<"test/arbundles.js/ans-104-manifest-", Ref/binary>> - ) - end, - [ - <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>, - <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>, - <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">> - ] - ), - BaseOpts#{ +manifest_opts() -> + (dev_manifest:test_env_opts())#{ name_resolvers => [#{ <<"device">> => <<"b32-name@1.0">> }], on => #{ diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index 707792ca2..17a87c250 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -2,6 +2,8 @@ %%% https://specs.ar.io/?tx=lXLd0OPwo-dJLB_Amz5jgIeDhiOkjXuM3-r0H_aiNj0 -module(dev_manifest). -export([index/3, info/0, request/3]). +%%% Public test exports +-export([test_env_opts/0]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -345,13 +347,7 @@ manifest_download_via_raw_endpoint_test_ignore() -> %% @doc Accessing `/TXID` of a manifest transaction should access the index key. manifest_inner_redirect_test() -> - %% Define the store - LmdbStore = hb_test_utils:test_store(), - %% Load transaction information to the store - hb_test_utils:preload(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - hb_test_utils:preload(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - %% Start node - Opts = #{store => LmdbStore}, + Opts = test_env_opts(), Node = hb_http_server:start_node(Opts), %% Request manifest to node. ?assertMatch( @@ -365,11 +361,7 @@ manifest_inner_redirect_test() -> %% @doc Accessing `/TXID/assets/ArticleBlock-Dtwjc54T.js` should return valid message. access_key_path_in_manifest_test() -> - LmdbStore = hb_test_utils:test_store(), - hb_test_utils:preload(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - hb_test_utils:preload(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - hb_test_utils:preload(LmdbStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), - Opts = #{store => LmdbStore}, + Opts = test_env_opts(), Node = hb_http_server:start_node(Opts), ?assertMatch( {ok, #{<<"commitments">> := #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _ }}}, @@ -383,10 +375,7 @@ access_key_path_in_manifest_test() -> %% This works with `not_found.js` but doesn't follow the logic if under a %% folder structure, like `assets/not_found.js . manifest_should_fallback_on_not_found_path_test() -> - LmdbStore = hb_test_utils:test_store(), - hb_test_utils:preload(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - hb_test_utils:preload(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - Opts = #{store => LmdbStore}, + Opts = test_env_opts(), Node = hb_http_server:start_node(Opts), ?assertMatch( {ok, #{<<"commitments">> := #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _ }}}, @@ -396,3 +385,28 @@ manifest_should_fallback_on_not_found_path_test() -> Opts ) ). + +%% @doc Returns `Opts' with the test manifest fixture flow used by `dev_b32_name'. +test_env_opts() -> + TempStore = hb_test_utils:test_store(), + BaseOpts = #{store => [TempStore]}, + lists:foreach( + fun(Ref) -> + hb_test_utils:preload( + BaseOpts, + <<"test/arbundles.js/ans-104-manifest-", Ref/binary>> + ) + end, + [ + <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>, + <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>, + <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">> + ] + ), + BaseOpts#{ + on => + #{ + <<"request">> => + [#{<<"device">> => <<"manifest@1.0">>}] + } + }. From 2b8563867d5276ab0768455962ec66d9534cd0fd Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 18:19:25 -0400 Subject: [PATCH 102/135] fix: improve readiness detection by awaiting the first `meta` key write --- src/dev_blacklist.erl | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 80399f00d..6ee859a00 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -22,7 +22,8 @@ -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(DEFAULT_MIN_WAIT, 60). +%%% The default frequency at which the blacklist cache is refreshed in seconds. +-define(DEFAULT_REFRESH_FREQUENCY, 60 * 5). %% @doc Hook handler: block requests that involve blacklisted IDs. request(_Base, HookReq, Opts) -> @@ -70,8 +71,6 @@ is_match(Msg, Opts) -> %% @doc Fetch blacklists from all configured providers and insert IDs into the %% cache table. fetch_and_insert_ids(Opts) -> - ensure_cache_table(Opts), - Providers = resolve_providers(Opts), Total = lists:foldl( fun(Provider, Acc) -> @@ -81,8 +80,16 @@ fetch_and_insert_ids(Opts) -> end end, 0, - Providers + resolve_providers(Opts) ), + Table = cache_table_name(Opts), + ets:insert(Table, {<<"meta/last-refresh">>, os:system_time(millisecond)}), + ?event( + {table_inserted, + {get_last_refresh, ets:lookup(Table, <<"meta/last-refresh">>)}, + {is_initialized, is_initialized(Table)} + } + ), ?event(blacklist_short, {fetched_and_inserted_ids, Total}, Opts), {ok, Total}. @@ -189,8 +196,9 @@ insert_ids([ID | IDs], Value, Table, Opts) when ?IS_ID(ID) -> %% @doc Ensure the cache table exists. ensure_cache_table(Opts) -> TableName = cache_table_name(Opts), - case ets:info(TableName) of - undefined -> + case is_initialized(TableName) of + true -> TableName; + false -> hb_name:singleton( TableName, fun() -> @@ -205,19 +213,24 @@ ensure_cache_table(Opts) -> {write_concurrency, true} ] ), + ?event({table_created, TableName}), fetch_and_insert_ids(Opts), refresh_loop(Opts) end ), hb_util:until( - fun() -> ets:info(TableName) =/= undefined end, - 100 + fun() -> is_initialized(TableName) end, + 10 ), - TableName; - _ -> TableName end. +%% @doc Check if the cache table is initialized. We do this by checking that the +%% `meta/last-refresh' key is present, although we do not care about its value. +is_initialized(TableName) -> + ets:info(TableName) =/= undefined + andalso ets:lookup(TableName, <<"meta/last-refresh">>) =/= []. + %% @doc Loop that periodically refreshes the blacklist cache. Runs on the %% singleton process that is responsible for the cache ets table. refresh_loop(Opts) -> @@ -225,7 +238,7 @@ refresh_loop(Opts) -> hb_util:int( hb_opts:get( blacklist_refresh_frequency, - ?DEFAULT_MIN_WAIT, + ?DEFAULT_REFRESH_FREQUENCY, Opts ) ) * 1000, @@ -236,8 +249,7 @@ refresh_loop(Opts) -> refresh -> fetch_and_insert_ids(Opts), refresh_loop(Opts); - stop -> - ok + stop -> ok end. %% @doc Calculate the name of the cache table given the `Opts`. @@ -251,7 +263,10 @@ cache_table_name(Opts) -> setup_test_env() -> %% We need to create a new priv_wallet to avoid conflift when starting a %% new node from an existing priv_wallet address. - Opts0 = #{ store => hb_test_utils:test_store(), priv_wallet => ar_wallet:new() }, + Opts0 = #{ + store => hb_test_utils:test_store(), + priv_wallet => ar_wallet:new() + }, Msg1 = hb_message:commit(#{ <<"body">> => <<"test-1">> }, Opts0), Msg2 = hb_message:commit(#{ <<"body">> => <<"test-2">> }, Opts0), Msg3 = hb_message:commit(#{ <<"body">> => <<"test-3">> }, Opts0), From e714420c241eddeeb6e80bef91491070357a2831 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sat, 14 Mar 2026 23:48:07 -0400 Subject: [PATCH 103/135] wip: Arweave global offsets as names --- src/dev_arweave.erl | 11 ++ src/dev_arweave_offset.erl | 215 +++++++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 src/dev_arweave_offset.erl diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index 893ad14ec..0c5559b77 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -4,13 +4,24 @@ %%% The node(s) that are used to query data may be configured by altering the %%% `/arweave` route in the node's configuration message. -module(dev_arweave). +-export([info/0]). -export([tx/3, raw/3, chunk/3, block/3, current/3, status/3, price/3, tx_anchor/3]). -export([post_tx_header/2, post_tx/3, post_tx/4, post_binary_ans104/2, post_json_chunk/2]). +%%% Helper functions +-export([get_chunk/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(IS_BLOCK_ID(X), (is_binary(X) andalso byte_size(X) == 64)). +%% @doc Route unknown keys through offset resolution first, then fall back to +%% the message device for direct key access. +info() -> + #{ + excludes => [<<"keys">>, <<"set">>, <<"set-path">>, <<"remove">>], + default => fun dev_arweave_offset:get/4 + }. + %% @doc Proxy the `/info' endpoint from the Arweave node. status(_Base, _Request, Opts) -> request(<<"GET">>, <<"/info">>, Opts). diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl new file mode 100644 index 000000000..58c623978 --- /dev/null +++ b/src/dev_arweave_offset.erl @@ -0,0 +1,215 @@ +%%% @doc A module for the Arweave device that implements the default key +%%% resolution logic. The default key returns slices of bytes inside Arweave as +%%% message representations. +-module(dev_arweave_offset). +-export([get/4]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Resolve either a message at an Arweave offset, or a direct key from the +%% base message if the key is not an integer. +get(Key, Base, Request, Opts) -> + case parse(Key) of + {ok, StartOffset, Length} -> + load_item_at_offset(StartOffset, Length, Opts); + error -> + dev_message:get(Key, Base, Request, Opts) + end. + +%% @doc Parse a path key as a global Arweave start offset. +parse(Key) -> + try + case binary:split(Key, <<"-">>) of + [Start, Length] -> + {ok, hb_util:int(Start), hb_util:int(Length)}; + [Start] -> + {ok, hb_util:int(Start), undefined} + end + catch + _:_ -> error + end. + +%% @doc Load an ANS-104 item whose header begins at the given global offset. +load_item_at_offset(StartOffset, Length, Opts) -> + maybe + {ok, ChunkJSON, FirstChunk} ?= item_chunk_from_offset(StartOffset, Opts), + {ok, HeaderSize, HeaderTX} ?= + try ar_bundles:deserialize_header(FirstChunk) + catch _:_ -> {error, invalid_ans104_header} + end, + {ok, ItemSize} ?= + if Length =:= undefined -> + item_size_from_offset(StartOffset, ChunkJSON, Opts); + true -> + {ok, Length} + end, + true ?= HeaderSize =< ItemSize, + DataSize = ItemSize - HeaderSize, + {HeaderData, RemainingLength} = + split_header_data(HeaderTX#tx.data, DataSize), + {ok, RemainingData} ?= + read_remaining_item_data( + StartOffset, + HeaderSize, + byte_size(HeaderData), + RemainingLength, + Opts + ), + FullTX = + HeaderTX#tx{ + data = <>, + data_size = DataSize + }, + {ok, + hb_message:convert( + FullTX, + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + )} + else + false -> {error, invalid_item_size}; + Error -> Error + end. + +%% @doc Read the chunk containing the given offset and trim it to begin at the +%% first byte of the requested item. +item_chunk_from_offset(StartOffset, Opts) -> + case dev_arweave:get_chunk(StartOffset + 1, Opts) of + {ok, ChunkJSON} -> + ChunkSize = hb_util:int(maps:get(<<"chunk_size">>, ChunkJSON)), + AbsEnd = hb_util:int(maps:get(<<"absolute_end_offset">>, ChunkJSON)), + Chunk = hb_util:decode(maps:get(<<"chunk">>, ChunkJSON)), + ChunkStart = AbsEnd - ChunkSize + 1, + Skip = (StartOffset + 1) - ChunkStart, + {ok, ChunkJSON, binary:part(Chunk, Skip, byte_size(Chunk) - Skip)}; + Error -> + Error + end. + +%% @doc Split the bytes already present after a decoded header from those that +%% still need to be read from Arweave. +split_header_data(HeaderData, DataSize) -> + PrefixSize = min(byte_size(HeaderData), DataSize), + { + binary:part(HeaderData, 0, PrefixSize), + DataSize - PrefixSize + }. + +%% @doc Read any bytes of the data segment that were not present in the first +%% header chunk. +read_remaining_item_data(_StartOffset, _HeaderSize, _PrefixSize, 0, _Opts) -> + {ok, <<>>}; +read_remaining_item_data(StartOffset, HeaderSize, PrefixSize, Length, Opts) -> + hb_store_arweave:read_chunks(StartOffset + HeaderSize + PrefixSize, Length, Opts). + +%% @doc Resolve the size of the item at the given offset by locating it in the +%% containing bundle header. We use the `note` attached to the Merkle leaf of +%% the `tx_path` for the chunk to find the size of the bundle that contains the +%% item. We then use the `note` attached to the Merkle leaf of the `data_path` +%% for the chunk to find the offset of the end of the chunk inside the bundle. +item_size_from_offset(StartOffset, ChunkJSON, Opts) -> + AbsEnd = hb_util:int(maps:get(<<"absolute_end_offset">>, ChunkJSON)), + BundleSize = + ar_merkle:extract_note( + hb_util:decode(maps:get(<<"tx_path">>, ChunkJSON)) + ), + ChunkEndInBundle = + ar_merkle:extract_note( + hb_util:decode(maps:get(<<"data_path">>, ChunkJSON)) + ), + BundleStartOffset = AbsEnd - ChunkEndInBundle, + case bundle_header(BundleStartOffset, BundleSize, Opts) of + {ok, HeaderSize, BundleIndex} -> + locate_bundle_item( + StartOffset, + BundleStartOffset + HeaderSize, + BundleIndex + ); + Error -> + Error + end. + +%% @doc Read and decode the containing bundle header for an item. +bundle_header(BundleStartOffset, _BundleSize, Opts) -> + case hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ + <<"path">> => <<"chunk">>, + <<"offset">> => BundleStartOffset + 1 + }, + Opts + ) of + {ok, FirstChunk} -> + case ar_bundles:bundle_header_size(FirstChunk) of + invalid_bundle_header -> + {error, invalid_bundle_header}; + HeaderSize -> + case read_bundle_header(BundleStartOffset, HeaderSize, FirstChunk, Opts) of + {ok, HeaderBin} -> + case ar_bundles:decode_bundle_header(HeaderBin) of + {_Items, BundleIndex} -> + {ok, HeaderSize, BundleIndex}; + invalid_bundle_header -> + {error, invalid_bundle_header} + end; + Error -> + Error + end + end; + Error -> + Error + end. + +%% @doc Read exactly the bytes needed to decode a bundle header. +read_bundle_header(_BundleStartOffset, HeaderSize, FirstChunk, _Opts) + when HeaderSize =< byte_size(FirstChunk) -> + {ok, binary:part(FirstChunk, 0, HeaderSize)}; +read_bundle_header(BundleStartOffset, HeaderSize, _FirstChunk, Opts) -> + hb_store_arweave:read_chunks(BundleStartOffset, HeaderSize, Opts). + +%% @doc Locate the item that starts at the given offset in a bundle header +%% index and return its serialized size. +locate_bundle_item(StartOffset, ItemStartOffset, [{_ID, Size} | _]) + when StartOffset =:= ItemStartOffset -> + {ok, Size}; +locate_bundle_item(StartOffset, ItemStartOffset, [{_ID, Size} | Rest]) + when StartOffset > ItemStartOffset -> + locate_bundle_item(StartOffset, ItemStartOffset + Size, Rest); +locate_bundle_item(_StartOffset, _ItemStartOffset, _BundleIndex) -> + {error, not_found}. + +%%% Tests + + +resolve_item_at_offset_test() -> + StartOffset = 384600234780716, + ExpectedID = <<"cTI07T1OrF0KZEqPmZji1VTdbeKJG7kMAVlLu7KQvyw">>, + {ok, Item} = + hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + hb_util:bin(StartOffset), + #{} + ), + ?assert(hb_message:verify(Item, all, #{})), + ?assertEqual(ExpectedID, hb_message:id(Item, signed, #{})). + +offset_as_name_resolver_lookup_test() -> + Opts = #{ + name_resolvers => [#{ <<"device">> => <<"arweave@2.9">> }], + on => + #{ + <<"request">> => [#{ <<"device">> => <<"name@2.9">> }] + } + }, + Node = hb_http_server:start_node(Opts), + {ok, Item} = + hb_http:get( + Node, + #{ + <<"path">> => <<"/">>, + <<"host">> => <<"384600234780716.localhost">> + }, + Opts + ), + ?assertEqual(<<"image/jpeg">>, hb_ao:get(<<"content-type">>, Item, Opts)). \ No newline at end of file From 74dc46aae24dd505dba4ff411e2169a6122bc756 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 15 Mar 2026 00:19:22 -0400 Subject: [PATCH 104/135] fix: only load message if the result is an ID --- src/dev_name.erl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/dev_name.erl b/src/dev_name.erl index 8d8305239..f5b32f758 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -33,12 +33,23 @@ resolve(Key, _, Req, Opts) -> case match_resolver(Key, Resolvers, Opts) of {ok, Resolved} -> case hb_util:atom(hb_ao:get(<<"load">>, Req, true, Opts)) of - false -> {ok, Resolved}; - true -> hb_cache:read(Resolved, Opts) + false -> + {ok, Resolved}; + true -> + maybe_load_resolved(Resolved, Opts) end; not_found -> not_found end. +%% @doc Load a resolved name target if it is a cache reference, otherwise +%% return the resolved value directly. +maybe_load_resolved(Resolved, Opts) when ?IS_ID(Resolved) -> + hb_cache:read(Resolved, Opts); +maybe_load_resolved(Resolved, Opts) when ?IS_LINK(Resolved) -> + {ok, hb_cache:ensure_loaded(Resolved, Opts)}; +maybe_load_resolved(Resolved, _Opts) -> + {ok, Resolved}. + %% @doc Find the first resolver that matches the key and return its value. match_resolver(_Key, [], _Opts) -> not_found; From 236336628444cc1d190ba12b8783fd3188ac5faa Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 15 Mar 2026 00:29:31 -0400 Subject: [PATCH 105/135] fix: tests; small bug fixes --- src/dev_arweave_offset.erl | 65 ++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl index 58c623978..9201a78ef 100644 --- a/src/dev_arweave_offset.erl +++ b/src/dev_arweave_offset.erl @@ -37,14 +37,16 @@ load_item_at_offset(StartOffset, Length, Opts) -> try ar_bundles:deserialize_header(FirstChunk) catch _:_ -> {error, invalid_ans104_header} end, - {ok, ItemSize} ?= - if Length =:= undefined -> - item_size_from_offset(StartOffset, ChunkJSON, Opts); + {ok, DataSize} ?= + if Length =/= undefined -> {ok, Length}; true -> - {ok, Length} + case item_size_from_offset(StartOffset, ChunkJSON, Opts) of + {ok, ItemSize} when HeaderSize =< ItemSize -> + {ok, ItemSize - HeaderSize}; + {ok, _ItemSize} -> false; + ItemSizeError -> ItemSizeError + end end, - true ?= HeaderSize =< ItemSize, - DataSize = ItemSize - HeaderSize, {HeaderData, RemainingLength} = split_header_data(HeaderTX#tx.data, DataSize), {ok, RemainingData} ?= @@ -181,25 +183,48 @@ locate_bundle_item(_StartOffset, _ItemStartOffset, _BundleIndex) -> %%% Tests +offset_item_cases_test() -> + Opts = #{}, + assert_offset_item( + <<"160399272861859">>, + 498852, + #{ <<"content-type">> => <<"image/png">> }, + Opts + ), + assert_offset_item( + <<"160399272861859-498852">>, + 498852, + #{ <<"content-type">> => <<"image/png">> }, + Opts + ), + assert_offset_item( + <<"384600234780716">>, + 856691, + #{ <<"content-type">> => <<"image/jpeg">> }, + Opts + ), + ok. -resolve_item_at_offset_test() -> - StartOffset = 384600234780716, - ExpectedID = <<"cTI07T1OrF0KZEqPmZji1VTdbeKJG7kMAVlLu7KQvyw">>, - {ok, Item} = - hb_ao:resolve( - #{ <<"device">> => <<"arweave@2.9">> }, - hb_util:bin(StartOffset), - #{} - ), - ?assert(hb_message:verify(Item, all, #{})), - ?assertEqual(ExpectedID, hb_message:id(Item, signed, #{})). +assert_offset_item(Path, DataSize, Tags, Opts) -> + {ok, Item} = hb_ao:resolve(#{ <<"device">> => <<"arweave@2.9">> }, Path, Opts), + TX = hb_message:convert(Item, <<"ans104@1.0">>, <<"structured@1.0">>, Opts), + ?assert(hb_message:verify(Item, all, Opts)), + ?assertEqual(DataSize, TX#tx.data_size), + ?assertEqual(DataSize, byte_size(TX#tx.data)), + maps:foreach( + fun(Key, Value) -> + ?assertEqual({ok, Value}, hb_maps:find(Key, Item, Opts)) + end, + Tags + ), + ok. offset_as_name_resolver_lookup_test() -> Opts = #{ name_resolvers => [#{ <<"device">> => <<"arweave@2.9">> }], on => #{ - <<"request">> => [#{ <<"device">> => <<"name@2.9">> }] + <<"request">> => [#{ <<"device">> => <<"name@1.0">> }] } }, Node = hb_http_server:start_node(Opts), @@ -208,8 +233,8 @@ offset_as_name_resolver_lookup_test() -> Node, #{ <<"path">> => <<"/">>, - <<"host">> => <<"384600234780716.localhost">> + <<"host">> => <<"152974576623958.localhost">> }, Opts ), - ?assertEqual(<<"image/jpeg">>, hb_ao:get(<<"content-type">>, Item, Opts)). \ No newline at end of file + ?assertEqual(<<"application/json">>, hb_ao:get(<<"content-type">>, Item, Opts)). From 83519afc7ed8cb0a3f0f78ca8e094ae333906b18 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 15 Mar 2026 00:29:58 -0400 Subject: [PATCH 106/135] impr: set Arweave offset lookup as initial name resolver --- src/hb_opts.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hb_opts.erl b/src/hb_opts.erl index f32f690d7..181b37185 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -43,7 +43,8 @@ -ifndef(TEST). -define(DEFAULT_NAME_RESOLVERS, [ - #{ <<"device">> => <<"~b32-name@1.0">> }, + #{ <<"device">> => <<"arweave@2.9">> }, + #{ <<"device">> => <<"b32-name@1.0">> }, << "G_gb7SAgogHMtmqycwaHaC6uC-CZ3akACdFv5PUaEE8", "~json@1.0/deserialize&target=data" From e8f9f1a280c91998eabd6d852d0e65ed02b9ce50 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 15 Mar 2026 00:42:19 -0400 Subject: [PATCH 107/135] impr: `/raw` API exposes a simple `offset` key whose response can be used as an offset name --- src/dev_arweave.erl | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index 0c5559b77..7c0a9b647 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -222,8 +222,9 @@ head_raw_tx(TXID, StartOffset, Length, Opts) -> ), {ok, #{ - <<"arweave-id">> => TXID, - <<"arweave-data-offset">> => StartOffset, + <<"raw-id">> => TXID, + <<"offset">> => StartOffset, + <<"data-offset">> => StartOffset, <<"content-type">> => ContentType, <<"header-length">> => 0, <<"content-length">> => Length, @@ -256,8 +257,9 @@ do_head_raw_ans104(TXID, ArweaveOffset, Length, Data, _Opts) -> ), {ok, #{ - <<"arweave-id">> => TXID, - <<"arweave-data-offset">> => ArweaveOffset + HeaderSize, + <<"raw-id">> => TXID, + <<"offset">> => ArweaveOffset, + <<"data-offset">> => ArweaveOffset + HeaderSize, <<"content-type">> => ContentType, <<"header-length">> => HeaderSize, <<"content-length">> => Length - HeaderSize, @@ -275,8 +277,8 @@ get_raw(Base, Request, Opts) -> Err = {error, _} -> Err; {ok, Header = #{ - <<"arweave-id">> := TXID, - <<"arweave-data-offset">> := ArweaveDataOffset, + <<"raw-id">> := TXID, + <<"data-offset">> := ArweaveDataOffset, <<"content-type">> := ContentType, <<"content-length">> := FullContentLength } From 70e5cbce23e7bf19c1f5d452414041b00b5cf6d2 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Sun, 15 Mar 2026 16:18:48 -0400 Subject: [PATCH 108/135] impr: remove `tx_path` parsing; abstract bundle header decoding --- src/dev_arweave.erl | 55 +++++++++++++++++++++++++++++++++- src/dev_arweave_offset.erl | 60 ++++++++----------------------------- src/dev_copycat_arweave.erl | 54 +++------------------------------ 3 files changed, 70 insertions(+), 99 deletions(-) diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index 7c0a9b647..a8b1f0a5f 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -8,7 +8,7 @@ -export([tx/3, raw/3, chunk/3, block/3, current/3, status/3, price/3, tx_anchor/3]). -export([post_tx_header/2, post_tx/3, post_tx/4, post_binary_ans104/2, post_json_chunk/2]). %%% Helper functions --export([get_chunk/2]). +-export([get_chunk/2, bundle_header/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -630,6 +630,59 @@ get_chunk(Offset, Opts) -> Path = <<"/chunk/", (hb_util:bin(Offset))/binary>>, request(<<"GET">>, Path, #{ <<"route-by">> => Offset }, Opts). +%% @doc Read and decode the bundle header index at the given global start +%% offset, returning the header size alongside the decoded index entries. +bundle_header(BundleStartOffset, Opts) -> + case hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ + <<"path">> => <<"chunk">>, + <<"offset">> => BundleStartOffset + 1 + }, + Opts + ) of + {ok, FirstChunk} -> + case ar_bundles:bundle_header_size(FirstChunk) of + invalid_bundle_header -> + {error, invalid_bundle_header}; + HeaderSize -> + case read_bundle_header(BundleStartOffset, HeaderSize, FirstChunk, Opts) of + {ok, HeaderBin} -> + case ar_bundles:decode_bundle_header(HeaderBin) of + {_Items, BundleIndex} -> + {ok, HeaderSize, BundleIndex}; + invalid_bundle_header -> + {error, invalid_bundle_header} + end; + Error -> + Error + end + end; + Error -> + Error + end. + +%% @doc Read exactly the bytes needed to decode a bundle header. +read_bundle_header(_BundleStartOffset, HeaderSize, FirstChunk, _Opts) + when HeaderSize =< byte_size(FirstChunk) -> + {ok, binary:part(FirstChunk, 0, HeaderSize)}; +read_bundle_header(BundleStartOffset, HeaderSize, FirstChunk, Opts) -> + RemainingSize = HeaderSize - byte_size(FirstChunk), + case hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ + <<"path">> => <<"chunk">>, + <<"offset">> => BundleStartOffset + byte_size(FirstChunk) + 1, + <<"length">> => RemainingSize + }, + Opts + ) of + {ok, RemainingChunk} -> + {ok, <>}; + Error -> + Error + end. + %% @doc Retrieve (and cache) block information from Arweave. If the `block' key %% is present, it is used to look up the associated block. If it is of Arweave %% block hash length (43 characters), it is used as an ID. If it is parsable as diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl index 9201a78ef..98f8cf39e 100644 --- a/src/dev_arweave_offset.erl +++ b/src/dev_arweave_offset.erl @@ -105,23 +105,25 @@ read_remaining_item_data(_StartOffset, _HeaderSize, _PrefixSize, 0, _Opts) -> read_remaining_item_data(StartOffset, HeaderSize, PrefixSize, Length, Opts) -> hb_store_arweave:read_chunks(StartOffset + HeaderSize + PrefixSize, Length, Opts). -%% @doc Resolve the size of the item at the given offset by locating it in the -%% containing bundle header. We use the `note` attached to the Merkle leaf of -%% the `tx_path` for the chunk to find the size of the bundle that contains the -%% item. We then use the `note` attached to the Merkle leaf of the `data_path` -%% for the chunk to find the offset of the end of the chunk inside the bundle. +%% @doc Determine the size of the item at an offset by locating it in the parent +%% Arweave transaction's bundle header. In order to do this we must: +%% 1. Find the global offset of the data root of the chunk. +%% 2. Jump to that location and read the header chunks until we find our item. +%% 3. Extract the item's size from the bundle header and return it. +%% We achieve objective (1) by extracting the `absolute_end_offset` from the +%% chunk JSON and subtracting the `data_path`'s note from it. The `data_path` +%% is the Merkle path of the chunk that contains the item, and its note is the +%% offset of the end of the chunk inside the bundle. The `absolute_end_offset` +%% is the global offset of the end of the chunk, so to calculate the bundle's +%% start offset we can simply perform `absolute_end_offset - data_path_note`. item_size_from_offset(StartOffset, ChunkJSON, Opts) -> AbsEnd = hb_util:int(maps:get(<<"absolute_end_offset">>, ChunkJSON)), - BundleSize = - ar_merkle:extract_note( - hb_util:decode(maps:get(<<"tx_path">>, ChunkJSON)) - ), ChunkEndInBundle = ar_merkle:extract_note( hb_util:decode(maps:get(<<"data_path">>, ChunkJSON)) ), BundleStartOffset = AbsEnd - ChunkEndInBundle, - case bundle_header(BundleStartOffset, BundleSize, Opts) of + case dev_arweave:bundle_header(BundleStartOffset, Opts) of {ok, HeaderSize, BundleIndex} -> locate_bundle_item( StartOffset, @@ -132,44 +134,6 @@ item_size_from_offset(StartOffset, ChunkJSON, Opts) -> Error end. -%% @doc Read and decode the containing bundle header for an item. -bundle_header(BundleStartOffset, _BundleSize, Opts) -> - case hb_ao:resolve( - #{ <<"device">> => <<"arweave@2.9">> }, - #{ - <<"path">> => <<"chunk">>, - <<"offset">> => BundleStartOffset + 1 - }, - Opts - ) of - {ok, FirstChunk} -> - case ar_bundles:bundle_header_size(FirstChunk) of - invalid_bundle_header -> - {error, invalid_bundle_header}; - HeaderSize -> - case read_bundle_header(BundleStartOffset, HeaderSize, FirstChunk, Opts) of - {ok, HeaderBin} -> - case ar_bundles:decode_bundle_header(HeaderBin) of - {_Items, BundleIndex} -> - {ok, HeaderSize, BundleIndex}; - invalid_bundle_header -> - {error, invalid_bundle_header} - end; - Error -> - Error - end - end; - Error -> - Error - end. - -%% @doc Read exactly the bytes needed to decode a bundle header. -read_bundle_header(_BundleStartOffset, HeaderSize, FirstChunk, _Opts) - when HeaderSize =< byte_size(FirstChunk) -> - {ok, binary:part(FirstChunk, 0, HeaderSize)}; -read_bundle_header(BundleStartOffset, HeaderSize, _FirstChunk, Opts) -> - hb_store_arweave:read_chunks(BundleStartOffset, HeaderSize, Opts). - %% @doc Locate the item that starts at the given offset in a bundle header %% index and return its serialized size. locate_bundle_item(StartOffset, ItemStartOffset, [{_ID, Size} | _]) diff --git a/src/dev_copycat_arweave.erl b/src/dev_copycat_arweave.erl index 13c9930a7..e55624213 100644 --- a/src/dev_copycat_arweave.erl +++ b/src/dev_copycat_arweave.erl @@ -312,7 +312,7 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> TXEndOffset, TX#tx.data_size, Opts ), case BundleRes of - {ok, {BundleIndex, HeaderSize}} -> + {ok, HeaderSize, BundleIndex} -> % Batch event tracking: measure total time and count for all write_offset calls {TotalTime, {_, ItemsCount}} = timer:tc(fun() -> lists:foldl( @@ -378,55 +378,9 @@ is_bundle_tx(TX, _Opts) -> download_bundle_header(EndOffset, Size, Opts) -> observe_event(<<"bundle_header">>, fun() -> - StartOffset = EndOffset - Size + 1, - case hb_ao:resolve( - << - ?ARWEAVE_DEVICE/binary, - "/chunk&offset=", - (hb_util:bin(StartOffset))/binary - >>, - Opts - ) of - {ok, FirstChunk} -> - % Most bundle headers can fit in a single chunk, but those with - % thousands of items might require multiple chunks to fully - % represent the item index. - HeaderSize = ar_bundles:bundle_header_size(FirstChunk), - case header_chunk(HeaderSize, FirstChunk, StartOffset, Opts) of - {ok, BundleHeader} -> - {_ItemsBin, BundleIndex} = - ar_bundles:decode_bundle_header(BundleHeader), - {ok, {BundleIndex, HeaderSize}}; - Error -> - Error - end; - Error -> - Error - end + dev_arweave:bundle_header(EndOffset - Size, Opts) end). -header_chunk(invalid_bundle_header, _FirstChunk, _StartOffset, _Opts) -> - {error, invalid_bundle_header}; -header_chunk(HeaderSize, FirstChunk, _StartOffset, _Opts) - when HeaderSize =< byte_size(FirstChunk) -> - {ok, FirstChunk}; -header_chunk(HeaderSize, FirstChunk, StartOffset, Opts) -> - Res = - hb_ao:resolve( - << - ?ARWEAVE_DEVICE/binary, - "/chunk&offset=", - (hb_util:bin(StartOffset + byte_size(FirstChunk)))/binary, - "&length=", - (hb_util:bin(HeaderSize - byte_size(FirstChunk)))/binary - >>, - Opts - ), - case Res of - {ok, OtherChunks} -> {ok, <>}; - Other -> Other - end. - resolve_tx_headers(TXIDs, Opts) -> Results = parallel_map( TXIDs, @@ -586,7 +540,7 @@ small_bundle_header_test() -> OffsetMsg = hb_json:decode(OffsetBody), EndOffset = hb_util:int(maps:get(<<"offset">>, OffsetMsg)), Size = hb_util:int(maps:get(<<"size">>, OffsetMsg)), - {ok, {BundleIndex, HeaderSize}} = + {ok, HeaderSize, BundleIndex} = download_bundle_header(EndOffset, Size, Opts), ?assertEqual(1704, length(BundleIndex)), ?assertEqual(109088, HeaderSize), @@ -607,7 +561,7 @@ large_bundle_header_test() -> OffsetMsg = hb_json:decode(OffsetBody), EndOffset = hb_util:int(maps:get(<<"offset">>, OffsetMsg)), Size = hb_util:int(maps:get(<<"size">>, OffsetMsg)), - {ok, {BundleIndex, HeaderSize}} = + {ok, HeaderSize, BundleIndex} = download_bundle_header(EndOffset, Size, Opts), ?assertEqual(15000, length(BundleIndex)), ?assertEqual(960032, HeaderSize), From ae0a39892b37b495f676cc70acf0d9ae3b3c2ec5 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 01:29:33 -0400 Subject: [PATCH 109/135] feat: add support for resolving nested bundled messages and resolution of mid-message offsets --- src/dev_arweave_offset.erl | 223 +++++++++++++++++++++++++++++-------- 1 file changed, 175 insertions(+), 48 deletions(-) diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl index 98f8cf39e..4eb9e9e5f 100644 --- a/src/dev_arweave_offset.erl +++ b/src/dev_arweave_offset.erl @@ -30,27 +30,32 @@ parse(Key) -> end. %% @doc Load an ANS-104 item whose header begins at the given global offset. -load_item_at_offset(StartOffset, Length, Opts) -> +load_item_at_offset(TargetOffset, Length, Opts) -> maybe - {ok, ChunkJSON, FirstChunk} ?= item_chunk_from_offset(StartOffset, Opts), - {ok, HeaderSize, HeaderTX} ?= - try ar_bundles:deserialize_header(FirstChunk) - catch _:_ -> {error, invalid_ans104_header} - end, + {ok, StartOffset, ItemSize} ?= message_from_offset(TargetOffset, Opts), + {ok, _ChunkJSON, FirstChunk} ?= chunk_from_offset(StartOffset, Opts), + {ok, HeaderSize, HeaderTX} ?= deserialize_header(FirstChunk), {ok, DataSize} ?= if Length =/= undefined -> {ok, Length}; - true -> - case item_size_from_offset(StartOffset, ChunkJSON, Opts) of - {ok, ItemSize} when HeaderSize =< ItemSize -> - {ok, ItemSize - HeaderSize}; - {ok, _ItemSize} -> false; - ItemSizeError -> ItemSizeError - end + HeaderSize =< ItemSize -> {ok, ItemSize - HeaderSize}; + true -> false end, {HeaderData, RemainingLength} = split_header_data(HeaderTX#tx.data, DataSize), + ?event( + arweave_offset_lookup, + {calculating_message_from_offset, + {target_offset, TargetOffset}, + {start_offset, StartOffset}, + {header_size, HeaderSize}, + {data_size, DataSize}, + {header_data, HeaderData}, + {remaining_length, RemainingLength} + }, + Opts + ), {ok, RemainingData} ?= - read_remaining_item_data( + read_remaining_data( StartOffset, HeaderSize, byte_size(HeaderData), @@ -59,7 +64,7 @@ load_item_at_offset(StartOffset, Length, Opts) -> ), FullTX = HeaderTX#tx{ - data = <>, + data = << HeaderData/binary, RemainingData/binary >>, data_size = DataSize }, {ok, @@ -76,7 +81,7 @@ load_item_at_offset(StartOffset, Length, Opts) -> %% @doc Read the chunk containing the given offset and trim it to begin at the %% first byte of the requested item. -item_chunk_from_offset(StartOffset, Opts) -> +chunk_from_offset(StartOffset, Opts) -> case dev_arweave:get_chunk(StartOffset + 1, Opts) of {ok, ChunkJSON} -> ChunkSize = hb_util:int(maps:get(<<"chunk_size">>, ChunkJSON)), @@ -89,6 +94,12 @@ item_chunk_from_offset(StartOffset, Opts) -> Error end. +%% @doc Safe wraper for ANS-104 header deserialization. +deserialize_header(Binary) -> + try ar_bundles:deserialize_header(Binary) + catch _:_ -> {error, <<"Invalid message header">>} + end. + %% @doc Split the bytes already present after a decoded header from those that %% still need to be read from Arweave. split_header_data(HeaderData, DataSize) -> @@ -100,67 +111,118 @@ split_header_data(HeaderData, DataSize) -> %% @doc Read any bytes of the data segment that were not present in the first %% header chunk. -read_remaining_item_data(_StartOffset, _HeaderSize, _PrefixSize, 0, _Opts) -> +read_remaining_data(_StartOffset, _HeaderSize, _PrefixSize, 0, _Opts) -> {ok, <<>>}; -read_remaining_item_data(StartOffset, HeaderSize, PrefixSize, Length, Opts) -> +read_remaining_data(StartOffset, HeaderSize, PrefixSize, Length, Opts) -> hb_store_arweave:read_chunks(StartOffset + HeaderSize + PrefixSize, Length, Opts). -%% @doc Determine the size of the item at an offset by locating it in the parent -%% Arweave transaction's bundle header. In order to do this we must: -%% 1. Find the global offset of the data root of the chunk. -%% 2. Jump to that location and read the header chunks until we find our item. -%% 3. Extract the item's size from the bundle header and return it. -%% We achieve objective (1) by extracting the `absolute_end_offset` from the -%% chunk JSON and subtracting the `data_path`'s note from it. The `data_path` -%% is the Merkle path of the chunk that contains the item, and its note is the -%% offset of the end of the chunk inside the bundle. The `absolute_end_offset` -%% is the global offset of the end of the chunk, so to calculate the bundle's -%% start offset we can simply perform `absolute_end_offset - data_path_note`. -item_size_from_offset(StartOffset, ChunkJSON, Opts) -> +%% @doc Locate the deepest bundled item that contains the given global offset. +message_from_offset(TargetOffset, Opts) -> + maybe + {ok, ChunkJSON, _Chunk} ?= chunk_from_offset(TargetOffset, Opts), + message_from_offset(TargetOffset, bundle_start_offset(ChunkJSON), Opts) + end. + +%% @doc Recover the global start offset of the containing bundle from the end +%% offset of the chunk in global space and its end offset inside the bundle. +bundle_start_offset(ChunkJSON) -> AbsEnd = hb_util:int(maps:get(<<"absolute_end_offset">>, ChunkJSON)), ChunkEndInBundle = ar_merkle:extract_note( hb_util:decode(maps:get(<<"data_path">>, ChunkJSON)) ), - BundleStartOffset = AbsEnd - ChunkEndInBundle, - case dev_arweave:bundle_header(BundleStartOffset, Opts) of - {ok, HeaderSize, BundleIndex} -> - locate_bundle_item( - StartOffset, + AbsEnd - ChunkEndInBundle. + +message_from_offset(TargetOffset, BundleStartOffset, Opts) -> + maybe + {ok, HeaderSize, BundleIndex} ?= + dev_arweave:bundle_header( + BundleStartOffset, + Opts + ), + {ok, ItemStartOffset, ItemSize} ?= + find_bundle_member( + TargetOffset, BundleStartOffset + HeaderSize, - BundleIndex - ); - Error -> - Error + BundleIndex, + Opts + ), + maybe_nested_item(TargetOffset, ItemStartOffset, ItemSize, Opts) + end. + +%% @doc If the containing item is itself a bundle and the offset lies in its +%% data payload, recurse into its bundle header. Otherwise return the item. +maybe_nested_item(TargetOffset, ItemStartOffset, ItemSize, Opts) -> + maybe + {ok, _ChunkJSON, FirstChunk} ?= chunk_from_offset(ItemStartOffset, Opts), + {ok, HeaderSize, HeaderTX} ?= deserialize_header(FirstChunk), + true ?= TargetOffset >= ItemStartOffset + HeaderSize, + true ?= dev_arweave_common:type(HeaderTX) =/= binary, + message_from_offset(TargetOffset, ItemStartOffset + HeaderSize, Opts) + else + false -> {ok, ItemStartOffset, ItemSize}; + {error, not_found} -> {ok, ItemStartOffset, ItemSize}; + Error -> Error end. -%% @doc Locate the item that starts at the given offset in a bundle header -%% index and return its serialized size. -locate_bundle_item(StartOffset, ItemStartOffset, [{_ID, Size} | _]) - when StartOffset =:= ItemStartOffset -> - {ok, Size}; -locate_bundle_item(StartOffset, ItemStartOffset, [{_ID, Size} | Rest]) - when StartOffset > ItemStartOffset -> - locate_bundle_item(StartOffset, ItemStartOffset + Size, Rest); -locate_bundle_item(_StartOffset, _ItemStartOffset, _BundleIndex) -> +%% @doc Locate the bundle member containing the given offset. +find_bundle_member(TargetOffset, ItemStartOffset, _BundleIndex, Opts) + when TargetOffset < ItemStartOffset -> + ?event( + arweave_offset_lookup, + {bundle_offset_search_exceeded_bounds, + {target_offset, TargetOffset}, + {item_start_offset, ItemStartOffset} + }, + Opts + ), + {error, not_found}; +find_bundle_member(TargetOffset, ItemStartOffset, [{ID, Size} | _], Opts) + when TargetOffset < ItemStartOffset + Size -> + % The target offset is within the current bundle member. + ?event( + arweave_offset_lookup, + {resolved_bundle_member, {id, ID}, {size, Size}}, + Opts + ), + {ok, ItemStartOffset, Size}; +find_bundle_member(TargetOffset, ItemStartOffset, [{_ID, Size} | Rest], Opts) -> + find_bundle_member(TargetOffset, ItemStartOffset + Size, Rest, Opts); +find_bundle_member(_TargetOffset, _ItemStartOffset, [], _Opts) -> {error, not_found}. %%% Tests offset_item_cases_test() -> Opts = #{}, + % A simple message. assert_offset_item( <<"160399272861859">>, 498852, #{ <<"content-type">> => <<"image/png">> }, Opts ), + % A reference with a given length. assert_offset_item( <<"160399272861859-498852">>, 498852, #{ <<"content-type">> => <<"image/png">> }, Opts ), + % A reference to a byte in the middle of the test message. + assert_offset_item( + <<"160399273000000">>, + 498852, + #{ <<"content-type">> => <<"image/png">> }, + Opts + ), + % % A megabyte reference to the item, occurring in the middle of the item. + % assert_offset_item( + % <<"160399273m">>, + % 498852, + % #{ <<"content-type">> => <<"image/jpeg">> }, + % Opts + % ), assert_offset_item( <<"384600234780716">>, 856691, @@ -169,6 +231,20 @@ offset_item_cases_test() -> ), ok. +offset_nested_item_test() -> + Opts = #{}, + TXID = <<"bndIwac23-s0K11TLC1N7z472sLGAkiOdhds87ZywoE">>, + Node = hb_http_server:start_node(), + {ok, Expected} = + hb_http:get( + Node, + <<"/~arweave@2.9/tx=", TXID/binary, "/1/2">>, + Opts + ), + {ItemStartOffset, _ItemSize} = + bundle_message_offset_from_tx(TXID, [1, 2], Opts), + assert_offset_matches(hb_util:bin(ItemStartOffset + 1), Expected, Opts). + assert_offset_item(Path, DataSize, Tags, Opts) -> {ok, Item} = hb_ao:resolve(#{ <<"device">> => <<"arweave@2.9">> }, Path, Opts), TX = hb_message:convert(Item, <<"ans104@1.0">>, <<"structured@1.0">>, Opts), @@ -183,6 +259,57 @@ assert_offset_item(Path, DataSize, Tags, Opts) -> ), ok. +assert_offset_matches(Path, Expected, Opts) -> + {ok, Item} = hb_ao:resolve(#{ <<"device">> => <<"arweave@2.9">> }, Path, Opts), + ExpectedTX = + hb_message:convert( + Expected, + <<"ans104@1.0">>, + <<"structured@1.0">>, + Opts + ), + TX = hb_message:convert(Item, <<"ans104@1.0">>, <<"structured@1.0">>, Opts), + ?assert(hb_message:verify(Item, all, Opts)), + ?assertEqual( + hb_message:id(Expected, signed, Opts), + hb_message:id(Item, signed, Opts) + ), + ?assertEqual(ExpectedTX#tx.data_size, TX#tx.data_size), + ok. + +bundle_message_offset_from_tx(TXID, Path, Opts) -> + {ok, #{ <<"body">> := OffsetBody }} = + hb_http:request( + #{ + <<"path">> => <<"/arweave/tx/", TXID/binary, "/offset">>, + <<"method">> => <<"GET">> + }, + Opts + ), + OffsetMsg = hb_json:decode(OffsetBody), + EndOffset = hb_util:int(maps:get(<<"offset">>, OffsetMsg)), + Size = hb_util:int(maps:get(<<"size">>, OffsetMsg)), + bundled_index_offset(EndOffset - Size, Path, Opts). + +bundled_index_offset(BundleStartOffset, [Index], Opts) -> + {ok, HeaderSize, BundleIndex} = + dev_arweave:bundle_header( + BundleStartOffset, + Opts + ), + nth_bundle_item(Index, BundleStartOffset + HeaderSize, BundleIndex); +bundled_index_offset(BundleStartOffset, [Index | Rest], Opts) -> + {ItemStartOffset, _ItemSize} = + bundled_index_offset(BundleStartOffset, [Index], Opts), + {ok, _ChunkJSON, FirstChunk} = chunk_from_offset(ItemStartOffset, Opts), + {ok, HeaderSize, _HeaderTX} = deserialize_header(FirstChunk), + bundled_index_offset(ItemStartOffset + HeaderSize, Rest, Opts). + +nth_bundle_item(1, ItemStartOffset, [{_ID, Size} | _]) -> + {ItemStartOffset, Size}; +nth_bundle_item(Index, ItemStartOffset, [{_ID, Size} | Rest]) when Index > 1 -> + nth_bundle_item(Index - 1, ItemStartOffset + Size, Rest). + offset_as_name_resolver_lookup_test() -> Opts = #{ name_resolvers => [#{ <<"device">> => <<"arweave@2.9">> }], From 2f573b0450d027e4a305c82c08902673110d6370 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 01:50:40 -0400 Subject: [PATCH 110/135] feat: add support for byte units (mb, gb, etc) in Arweave offset references --- src/dev_arweave_offset.erl | 97 ++++++++++++++++++++++++++++++++------ 1 file changed, 82 insertions(+), 15 deletions(-) diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl index 4eb9e9e5f..90eea1d3f 100644 --- a/src/dev_arweave_offset.erl +++ b/src/dev_arweave_offset.erl @@ -16,19 +16,73 @@ get(Key, Base, Request, Opts) -> dev_message:get(Key, Base, Request, Opts) end. -%% @doc Parse a path key as a global Arweave start offset. +%% @doc Parse a path key as a global Arweave start offset. The supported syntax +%% is as follows: +%% ``` +%% Reference :: Offset-Length +%% Offset :: [Unit] +%% Length :: +%% Unit :: +%% b : The global Arweave offset in absolute bytes (default). +%% k[i][b] : The global Arweave offset in absolute kilobytes or kibibytes. +%% m[i][b] : The global Arweave offset in absolute megabytes or mebibytes. +%% g[i][b] : The global Arweave offset in absolute gigabytes or gibibytes. +%% t[i][b] : The global Arweave offset in absolute terabytes or tebibytes. +%% p[i][b] : The global Arweave offset in absolute petabytes or pebibytes. +%% e[i][b] : The global Arweave offset in absolute exabytes or exbibytes. +%% z[i][b] : The global Arweave offset in absolute zettabytes or zebibytes. +%% y[i][b] : The global Arweave offset in absolute yottabytes or yobibytes. +%% ``` +%% In the scheme above, the `i` modifier in units indicates that the unit is in +%% binary multiples of the base unit. For example, `kib` is 1024 bytes, `mib` is +%% 1024 * 1024 bytes, etc. By contrast, the `kb` unit is decimal-oriented: `kb` +%% 1000 bytes, `mb` is 1000 * 1000 bytes, etc. To aid minimization of the bytes +%% required for the references, the `b` is always implied and need not be +%% specified. parse(Key) -> try - case binary:split(Key, <<"-">>) of - [Start, Length] -> - {ok, hb_util:int(Start), hb_util:int(Length)}; - [Start] -> - {ok, hb_util:int(Start), undefined} - end + {OffsetBin, Length} = + case binary:split(Key, <<"-">>) of + [Start, LengthBin] -> {Start, hb_util:int(LengthBin)}; + [Start] -> {Start, undefined} + end, + {ok, parse_unit(OffsetBin), Length} catch - _:_ -> error + Class:Error:StackTrace -> + ?event( + error, + {error, {invalid_offset_key, Key}, + {class, Class}, + {error, Error}, + {stack_trace, {trace, StackTrace}}} + ), + error end. +%% @doc Parses and applies a unit modifier to a base value, supporting both +%% the `kb` and `kib` unit formats. +parse_unit(Binary) -> parse_unit(0, Binary). +parse_unit(Complete, <<>>) -> Complete; +parse_unit(Base, <>) when Int >= $0 andalso Int =< $9 -> + parse_unit(Base * 10 + (Int - $0), Rest); +parse_unit(Base, <<"b">>) -> Base; +parse_unit(Base, <<"ki", _/binary>>) -> parse_unit(Base * 1024, <<"b">>); +parse_unit(Base, <<"mi", _/binary>>) -> parse_unit(Base * 1024, <<"ki">>); +parse_unit(Base, <<"gi", _/binary>>) -> parse_unit(Base * 1024, <<"mi">>); +parse_unit(Base, <<"ti", _/binary>>) -> parse_unit(Base * 1024, <<"gi">>); +parse_unit(Base, <<"pi", _/binary>>) -> parse_unit(Base * 1024, <<"ti">>); +parse_unit(Base, <<"ei", _/binary>>) -> parse_unit(Base * 1024, <<"pi">>); +parse_unit(Base, <<"zi", _/binary>>) -> parse_unit(Base * 1024, <<"zi">>); +parse_unit(Base, <<"yi", _/binary>>) -> parse_unit(Base * 1024, <<"zi">>); +parse_unit(Base, <<"k", _/binary>>) -> parse_unit(Base * 1000, <<"b">>); +parse_unit(Base, <<"m", _/binary>>) -> parse_unit(Base * 1000, <<"k">>); +parse_unit(Base, <<"g", _/binary>>) -> parse_unit(Base * 1000, <<"m">>); +parse_unit(Base, <<"t", _/binary>>) -> parse_unit(Base * 1000, <<"g">>); +parse_unit(Base, <<"p", _/binary>>) -> parse_unit(Base * 1000, <<"t">>); +parse_unit(Base, <<"e", _/binary>>) -> parse_unit(Base * 1000, <<"p">>); +parse_unit(Base, <<"z", _/binary>>) -> parse_unit(Base * 1000, <<"e">>); +parse_unit(Base, <<"y", _/binary>>) -> parse_unit(Base * 1000, <<"z">>). + %% @doc Load an ANS-104 item whose header begins at the given global offset. load_item_at_offset(TargetOffset, Length, Opts) -> maybe @@ -193,6 +247,19 @@ find_bundle_member(_TargetOffset, _ItemStartOffset, [], _Opts) -> %%% Tests +parse_offset_test() -> + ?assertEqual({ok, 160399272861859, undefined}, parse(<<"160399272861859">>)), + ?assertEqual({ok, 160399272861859, 498852}, parse(<<"160399272861859-498852">>)), + ?assertEqual({ok, 160399273000000, undefined}, parse(<<"160399273000000">>)), + ?assertEqual({ok, 160399273000000, 498852}, parse(<<"160399273000000-498852">>)), + ?assertEqual({ok, 160399273000000, undefined}, parse(<<"160399273m">>)), + ?assertEqual({ok, 160399273000000, 498852}, parse(<<"160399273m-498852">>)), + ?assertEqual( + {ok, 1337 * 1024 * 1024 * 1024 * 1024, undefined}, + parse(<<"1337tib">>) + ), + ok. + offset_item_cases_test() -> Opts = #{}, % A simple message. @@ -216,13 +283,13 @@ offset_item_cases_test() -> #{ <<"content-type">> => <<"image/png">> }, Opts ), - % % A megabyte reference to the item, occurring in the middle of the item. - % assert_offset_item( - % <<"160399273m">>, - % 498852, - % #{ <<"content-type">> => <<"image/jpeg">> }, - % Opts - % ), + % A megabyte reference to the item, occurring in the middle of the item. + assert_offset_item( + <<"160399273m">>, + 498852, + #{ <<"content-type">> => <<"image/png">> }, + Opts + ), assert_offset_item( <<"384600234780716">>, 856691, From 266c166a690c88a735026fd786f675923c15eb2e Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 02:29:05 -0400 Subject: [PATCH 111/135] impr: remove duplicated chunk fetches --- src/dev_arweave_offset.erl | 161 ++++++++++++++++++++++++++++++------- 1 file changed, 130 insertions(+), 31 deletions(-) diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl index 90eea1d3f..e377ccc9f 100644 --- a/src/dev_arweave_offset.erl +++ b/src/dev_arweave_offset.erl @@ -84,30 +84,73 @@ parse_unit(Base, <<"z", _/binary>>) -> parse_unit(Base * 1000, <<"e">>); parse_unit(Base, <<"y", _/binary>>) -> parse_unit(Base * 1000, <<"z">>). %% @doc Load an ANS-104 item whose header begins at the given global offset. -load_item_at_offset(TargetOffset, Length, Opts) -> +%% When a length is supplied it is treated as the exact ANS-104 data length, so +%% we can skip bundle index discovery and read only the remaining payload bytes. +load_item_at_offset(ExplicitOffset, Length, Opts) when is_integer(Length) -> maybe - {ok, StartOffset, ItemSize} ?= message_from_offset(TargetOffset, Opts), - {ok, _ChunkJSON, FirstChunk} ?= chunk_from_offset(StartOffset, Opts), - {ok, HeaderSize, HeaderTX} ?= deserialize_header(FirstChunk), - {ok, DataSize} ?= - if Length =/= undefined -> {ok, Length}; - HeaderSize =< ItemSize -> {ok, ItemSize - HeaderSize}; - true -> false - end, - {HeaderData, RemainingLength} = - split_header_data(HeaderTX#tx.data, DataSize), + {ok, _ChunkJSON, FirstChunk} ?= chunk_from_offset(ExplicitOffset, Opts), ?event( arweave_offset_lookup, - {calculating_message_from_offset, - {target_offset, TargetOffset}, - {start_offset, StartOffset}, - {header_size, HeaderSize}, - {data_size, DataSize}, - {header_data, HeaderData}, - {remaining_length, RemainingLength} + {loaded_explicit_offset, + {explicit_offset, ExplicitOffset}, + {length, Length} }, Opts ), + load_item_from_data_size(ExplicitOffset, Length, FirstChunk, Opts) + end; +load_item_at_offset(TargetOffset, undefined, Opts) -> + maybe + {ok, StartOffset, ItemSize, FirstChunk} ?= + message_from_offset(TargetOffset, Opts), + load_item_from_serialized_size(StartOffset, ItemSize, FirstChunk, Opts) + else + false -> {error, invalid_item_size}; + Error -> Error + end. + +%% @doc Load an item when the exact ANS-104 data length is already known. +load_item_from_data_size(StartOffset, DataSize, FirstChunk, Opts) -> + maybe + {ok, HeaderSize, HeaderTX} ?= deserialize_header(FirstChunk), + load_item_from_header(StartOffset, HeaderSize, HeaderTX, DataSize, Opts) + end. + +%% @doc Load an item when its serialized size is known from the containing +%% bundle index. +load_item_from_serialized_size(StartOffset, ItemSize, FirstChunk, Opts) -> + maybe + {ok, HeaderSize, HeaderTX} ?= deserialize_header(FirstChunk), + true ?= HeaderSize =< ItemSize, + load_item_from_header( + StartOffset, + HeaderSize, + HeaderTX, + ItemSize - HeaderSize, + Opts + ) + else + false -> {error, invalid_item_size}; + Error -> Error + end. + +%% @doc Complete an item load once the header has been decoded, using any data +%% bytes that were already present after the header before reading the tail. +load_item_from_header(StartOffset, HeaderSize, HeaderTX, DataSize, Opts) -> + {HeaderData, RemainingLength} = + split_header_data(HeaderTX#tx.data, DataSize), + ?event( + arweave_offset_lookup, + {calculating_message_from_offset, + {start_offset, StartOffset}, + {header_size, HeaderSize}, + {data_size, DataSize}, + {header_data, HeaderData}, + {remaining_length, RemainingLength} + }, + Opts + ), + maybe {ok, RemainingData} ?= read_remaining_data( StartOffset, @@ -127,10 +170,8 @@ load_item_at_offset(TargetOffset, Length, Opts) -> <<"structured@1.0">>, <<"ans104@1.0">>, Opts - )} - else - false -> {error, invalid_item_size}; - Error -> Error + ) + } end. %% @doc Read the chunk containing the given offset and trim it to begin at the @@ -173,8 +214,14 @@ read_remaining_data(StartOffset, HeaderSize, PrefixSize, Length, Opts) -> %% @doc Locate the deepest bundled item that contains the given global offset. message_from_offset(TargetOffset, Opts) -> maybe - {ok, ChunkJSON, _Chunk} ?= chunk_from_offset(TargetOffset, Opts), - message_from_offset(TargetOffset, bundle_start_offset(ChunkJSON), Opts) + {ok, ChunkJSON, FirstChunk} ?= chunk_from_offset(TargetOffset, Opts), + message_from_offset( + TargetOffset, + bundle_start_offset(ChunkJSON), + TargetOffset, + FirstChunk, + Opts + ) end. %% @doc Recover the global start offset of the containing bundle from the end @@ -187,7 +234,7 @@ bundle_start_offset(ChunkJSON) -> ), AbsEnd - ChunkEndInBundle. -message_from_offset(TargetOffset, BundleStartOffset, Opts) -> +message_from_offset(TargetOffset, BundleStartOffset, KnownOffset, KnownChunk, Opts) -> maybe {ok, HeaderSize, BundleIndex} ?= dev_arweave:bundle_header( @@ -201,21 +248,73 @@ message_from_offset(TargetOffset, BundleStartOffset, Opts) -> BundleIndex, Opts ), - maybe_nested_item(TargetOffset, ItemStartOffset, ItemSize, Opts) + maybe_nested_item( + TargetOffset, + ItemStartOffset, + ItemSize, + KnownOffset, + KnownChunk, + Opts + ) end. %% @doc If the containing item is itself a bundle and the offset lies in its %% data payload, recurse into its bundle header. Otherwise return the item. -maybe_nested_item(TargetOffset, ItemStartOffset, ItemSize, Opts) -> +maybe_nested_item( + TargetOffset, + ItemStartOffset, + ItemSize, + KnownOffset, + KnownChunk, + Opts + ) -> + maybe + {ok, FirstChunk} ?= + item_chunk(ItemStartOffset, KnownOffset, KnownChunk, Opts), + maybe_nested_item( + TargetOffset, + ItemStartOffset, + ItemSize, + FirstChunk, + KnownOffset, + KnownChunk, + Opts + ) + end. + +maybe_nested_item( + TargetOffset, + ItemStartOffset, + ItemSize, + FirstChunk, + KnownOffset, + KnownChunk, + Opts + ) -> maybe - {ok, _ChunkJSON, FirstChunk} ?= chunk_from_offset(ItemStartOffset, Opts), {ok, HeaderSize, HeaderTX} ?= deserialize_header(FirstChunk), true ?= TargetOffset >= ItemStartOffset + HeaderSize, true ?= dev_arweave_common:type(HeaderTX) =/= binary, - message_from_offset(TargetOffset, ItemStartOffset + HeaderSize, Opts) + message_from_offset( + TargetOffset, + ItemStartOffset + HeaderSize, + KnownOffset, + KnownChunk, + Opts + ) else - false -> {ok, ItemStartOffset, ItemSize}; - {error, not_found} -> {ok, ItemStartOffset, ItemSize}; + false -> {ok, ItemStartOffset, ItemSize, FirstChunk}; + {error, not_found} -> {ok, ItemStartOffset, ItemSize, FirstChunk}; + Error -> Error + end. + +%% @doc Reuse the first chunk we already have when the located item starts at the +%% same offset as the original request, otherwise fetch the item's first chunk. +item_chunk(ItemStartOffset, ItemStartOffset, FirstChunk, _Opts) -> + {ok, FirstChunk}; +item_chunk(ItemStartOffset, _KnownOffset, _KnownChunk, Opts) -> + case chunk_from_offset(ItemStartOffset, Opts) of + {ok, _ChunkJSON, FirstChunk} -> {ok, FirstChunk}; Error -> Error end. From 3818c14d4eac3a7afaf7fdb1f2dd37f6e7fc8e77 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 02:55:30 -0400 Subject: [PATCH 112/135] fix: reject lone unit symbols without associated ints --- src/dev_arweave_offset.erl | 54 ++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl index e377ccc9f..75abe6a64 100644 --- a/src/dev_arweave_offset.erl +++ b/src/dev_arweave_offset.erl @@ -46,42 +46,34 @@ parse(Key) -> [Start, LengthBin] -> {Start, hb_util:int(LengthBin)}; [Start] -> {Start, undefined} end, - {ok, parse_unit(OffsetBin), Length} + {ok, unit(OffsetBin), Length} catch - Class:Error:StackTrace -> - ?event( - error, - {error, {invalid_offset_key, Key}, - {class, Class}, - {error, Error}, - {stack_trace, {trace, StackTrace}}} - ), - error + _Class:_Error:_StackTrace -> error end. %% @doc Parses and applies a unit modifier to a base value, supporting both %% the `kb` and `kib` unit formats. -parse_unit(Binary) -> parse_unit(0, Binary). -parse_unit(Complete, <<>>) -> Complete; -parse_unit(Base, <>) when Int >= $0 andalso Int =< $9 -> - parse_unit(Base * 10 + (Int - $0), Rest); -parse_unit(Base, <<"b">>) -> Base; -parse_unit(Base, <<"ki", _/binary>>) -> parse_unit(Base * 1024, <<"b">>); -parse_unit(Base, <<"mi", _/binary>>) -> parse_unit(Base * 1024, <<"ki">>); -parse_unit(Base, <<"gi", _/binary>>) -> parse_unit(Base * 1024, <<"mi">>); -parse_unit(Base, <<"ti", _/binary>>) -> parse_unit(Base * 1024, <<"gi">>); -parse_unit(Base, <<"pi", _/binary>>) -> parse_unit(Base * 1024, <<"ti">>); -parse_unit(Base, <<"ei", _/binary>>) -> parse_unit(Base * 1024, <<"pi">>); -parse_unit(Base, <<"zi", _/binary>>) -> parse_unit(Base * 1024, <<"zi">>); -parse_unit(Base, <<"yi", _/binary>>) -> parse_unit(Base * 1024, <<"zi">>); -parse_unit(Base, <<"k", _/binary>>) -> parse_unit(Base * 1000, <<"b">>); -parse_unit(Base, <<"m", _/binary>>) -> parse_unit(Base * 1000, <<"k">>); -parse_unit(Base, <<"g", _/binary>>) -> parse_unit(Base * 1000, <<"m">>); -parse_unit(Base, <<"t", _/binary>>) -> parse_unit(Base * 1000, <<"g">>); -parse_unit(Base, <<"p", _/binary>>) -> parse_unit(Base * 1000, <<"t">>); -parse_unit(Base, <<"e", _/binary>>) -> parse_unit(Base * 1000, <<"p">>); -parse_unit(Base, <<"z", _/binary>>) -> parse_unit(Base * 1000, <<"e">>); -parse_unit(Base, <<"y", _/binary>>) -> parse_unit(Base * 1000, <<"z">>). +unit(Binary) -> unit(0, Binary). +unit(Complete, <<>>) -> Complete; +unit(Base, <>) when Int >= $0 andalso Int =< $9 -> + unit(Base * 10 + (Int - $0), Rest); +unit(Base, <<"b">>) -> Base; +unit(Base, <<"ki", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"b">>); +unit(Base, <<"mi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"ki">>); +unit(Base, <<"gi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"mi">>); +unit(Base, <<"ti", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"gi">>); +unit(Base, <<"pi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"ti">>); +unit(Base, <<"ei", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"pi">>); +unit(Base, <<"zi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"zi">>); +unit(Base, <<"yi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"zi">>); +unit(Base, <<"k", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"b">>); +unit(Base, <<"m", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"k">>); +unit(Base, <<"g", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"m">>); +unit(Base, <<"t", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"g">>); +unit(Base, <<"p", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"t">>); +unit(Base, <<"e", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"p">>); +unit(Base, <<"z", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"e">>); +unit(Base, <<"y", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"z">>). %% @doc Load an ANS-104 item whose header begins at the given global offset. %% When a length is supplied it is treated as the exact ANS-104 data length, so From 6d90c64b861e3d957d76d434da21d7c09df8f0a5 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Mon, 16 Mar 2026 10:02:54 -0400 Subject: [PATCH 113/135] fix: zebibyte infinite recursion Co-authored-by: Rani --- src/dev_arweave_offset.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl index 75abe6a64..ec21ac443 100644 --- a/src/dev_arweave_offset.erl +++ b/src/dev_arweave_offset.erl @@ -64,7 +64,7 @@ unit(Base, <<"gi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"mi">>); unit(Base, <<"ti", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"gi">>); unit(Base, <<"pi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"ti">>); unit(Base, <<"ei", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"pi">>); -unit(Base, <<"zi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"zi">>); +unit(Base, <<"zi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"ei">>); unit(Base, <<"yi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"zi">>); unit(Base, <<"k", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"b">>); unit(Base, <<"m", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"k">>); From 1a56755f3b48779ddc4540e63eb4a368e15a3841 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 16 Mar 2026 18:21:25 -0300 Subject: [PATCH 114/135] impr: ~9x speedup on events handling --- src/hb_event.erl | 120 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 85 insertions(+), 35 deletions(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 93f44a9c6..8d4ccaa6f 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -176,44 +176,51 @@ ensure_event_counter() -> ]). handle_events() -> + handle_events(0). +handle_events(N) -> receive {increment, TopicBin, EventName, Count} -> - case erlang:process_info(self(), message_queue_len) of - {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH -> - % Print a warning, but do so less frequently the more - % overloaded the system is. - {memory, MemorySize} = erlang:process_info(self(), memory), - case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of - 1 -> - ?debug_print( - {warning, - prometheus_event_queue_overloading, - {queue, Len}, - {current_message, EventName}, - {memory_bytes, MemorySize} - } - ); - _ -> ignored - end, - % If the size of this process is too large, exit such that - % we can be restarted by the next caller. - case MemorySize of - MemorySize when MemorySize > ?MAX_MEMORY -> - ?debug_print( - {error, - prometheus_event_queue_terminating_on_memory_overload, - {queue, Len}, - {memory_bytes, MemorySize}, - {current_message, EventName} - } - ), - exit(memory_overload); - _ -> no_action - end; + N2 = N + 1, + case N2 rem 1000 of + 0 -> + check_overload(EventName); + _ -> + ok + end, + prometheus_counter:inc(<<"event">>, [TopicBin, EventName], Count), + handle_events(N2) + end. + +check_overload(EventName) -> + case erlang:process_info(self(), message_queue_len) of + {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH -> + {memory, MemorySize} = erlang:process_info(self(), memory), + case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of + 1 -> + ?debug_print( + {warning, + prometheus_event_queue_overloading, + {queue, Len}, + {current_message, EventName}, + {memory_bytes, MemorySize} + } + ); _ -> ignored end, - hb_prometheus:inc(counter, <<"event">>, [TopicBin, EventName], Count), - handle_events() + case MemorySize of + MemorySize when MemorySize > ?MAX_MEMORY -> + ?debug_print( + {error, + prometheus_event_queue_terminating_on_memory_overload, + {queue, Len}, + {memory_bytes, MemorySize}, + {current_message, EventName} + } + ), + exit(memory_overload); + _ -> no_action + end; + _ -> ignored end. parse_name(Name) when is_tuple(Name) -> @@ -268,4 +275,47 @@ benchmark_increment_test() -> ), hb_test_utils:benchmark_print(<<"Incremented">>, <<"events">>, Iterations), ?assert(Iterations >= 1000), - ok. \ No newline at end of file + ok. + +-ifdef(NO_EVENTS). +benchmark_drain_rate_test() -> ok. +-else. +benchmark_drain_rate_test() -> + log(warmup, {warmup, 0}), + timer:sleep(100), + EventPid = hb_name:lookup(?MODULE), + wait_drain(EventPid, 5000), + N = 100000, + fill_mailbox(EventPid, N), + {DrainTime, _} = timer:tc(fun() -> + wait_drain(EventPid, 30000) + end), + DrainRate = round(N / (DrainTime / 1_000_000)), + hb_test_utils:benchmark_print( + <<"Drained">>, <<"events">>, DrainRate, 1), + ?assert(DrainRate >= 10000), + ok. + +fill_mailbox(_Pid, 0) -> ok; +fill_mailbox(Pid, N) -> + Pid ! {increment, <<"bench">>, <<"drain">>, 1}, + fill_mailbox(Pid, N - 1). + +wait_drain(Pid, Timeout) -> + Deadline = erlang:monotonic_time(millisecond) + Timeout, + wait_drain_loop(Pid, Deadline). + +wait_drain_loop(Pid, Deadline) -> + case erlang:process_info(Pid, message_queue_len) of + {message_queue_len, 0} -> ok; + {message_queue_len, _} -> + case erlang:monotonic_time(millisecond) >= Deadline of + true -> error(drain_timeout); + false -> + timer:sleep(10), + wait_drain_loop(Pid, Deadline) + end; + undefined -> + error(event_server_dead) + end. +-endif. From c26f53395b2d104b2e01976622f3c62655c850fd Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Mon, 16 Mar 2026 20:08:05 -0300 Subject: [PATCH 115/135] impr: event handling in batches --- src/hb_event.erl | 151 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 114 insertions(+), 37 deletions(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 8d4ccaa6f..215224ca9 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -9,6 +9,7 @@ -define(OVERLOAD_QUEUE_LENGTH, 10000). -define(MAX_MEMORY, 1_000_000_000). % 1GB -define(MAX_EVENT_NAME_LENGTH, 100). +-define(BATCH_MAX, 10000). -ifdef(NO_EVENTS). log(_X) -> ok. @@ -180,47 +181,87 @@ handle_events() -> handle_events(N) -> receive {increment, TopicBin, EventName, Count} -> - N2 = N + 1, - case N2 rem 1000 of - 0 -> - check_overload(EventName); - _ -> - ok - end, - prometheus_counter:inc(<<"event">>, [TopicBin, EventName], Count), + erlang:put(batch_keys, []), + N2 = drain_batch(N + 1, TopicBin, EventName, Count, ?BATCH_MAX - 1), + hb_prometheus:ensure_started(), + Keys = flush_batch(), + check_overload(Keys, N, N2), handle_events(N2) end. -check_overload(EventName) -> - case erlang:process_info(self(), message_queue_len) of - {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH -> - {memory, MemorySize} = erlang:process_info(self(), memory), - case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of - 1 -> - ?debug_print( - {warning, - prometheus_event_queue_overloading, - {queue, Len}, - {current_message, EventName}, - {memory_bytes, MemorySize} - } - ); +drain_batch(N, LastT, LastE, Acc, 0) -> + pd_inc({batch, LastT, LastE}, Acc), + N; +drain_batch(N, LastT, LastE, Acc, Remaining) -> + receive + {increment, TopicBin, EventName, Count} -> + case TopicBin =:= LastT andalso EventName =:= LastE of + true -> + drain_batch(N + 1, LastT, LastE, Acc + Count, Remaining - 1); + false -> + pd_inc({batch, LastT, LastE}, Acc), + drain_batch(N + 1, TopicBin, EventName, Count, Remaining - 1) + end + after 0 -> + pd_inc({batch, LastT, LastE}, Acc), + N + end. + +pd_inc(Key, Count) -> + case erlang:get(Key) of + undefined -> + erlang:put(Key, Count), + erlang:put(batch_keys, [Key | erlang:get(batch_keys)]); + Old when is_integer(Old) -> + erlang:put(Key, Old + Count) + end. + +flush_batch() -> + Keys = erlang:get(batch_keys), + lists:foreach( + fun(Key = {batch, Topic, Event}) -> + prometheus_counter:inc(<<"event">>, [Topic, Event], erlang:get(Key)), + erlang:erase(Key) + end, + Keys), + erlang:put(batch_keys, []), + Keys. + +check_overload(Keys, Prev, N) -> + case N div 1000 > Prev div 1000 of + true -> + case erlang:process_info(self(), message_queue_len) of + {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH -> + {memory, MemorySize} = erlang:process_info(self(), memory), + SampleKeys = lists:sublist(Keys, 5), + case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of + 1 -> + ?debug_print( + {warning, + prometheus_event_queue_overloading, + {queue, Len}, + {sample_keys, SampleKeys}, + {memory_bytes, MemorySize} + } + ); + _ -> ignored + end, + case MemorySize of + MemorySize when MemorySize > ?MAX_MEMORY -> + ?debug_print( + {error, + prometheus_event_queue_terminating_on_memory_overload, + {queue, Len}, + {memory_bytes, MemorySize}, + {sample_keys, SampleKeys} + } + ), + exit(memory_overload); + _ -> no_action + end; _ -> ignored - end, - case MemorySize of - MemorySize when MemorySize > ?MAX_MEMORY -> - ?debug_print( - {error, - prometheus_event_queue_terminating_on_memory_overload, - {queue, Len}, - {memory_bytes, MemorySize}, - {current_message, EventName} - } - ), - exit(memory_overload); - _ -> no_action end; - _ -> ignored + _ -> ok end. parse_name(Name) when is_tuple(Name) -> @@ -279,6 +320,7 @@ benchmark_increment_test() -> -ifdef(NO_EVENTS). benchmark_drain_rate_test() -> ok. +batch_correctness_test() -> ok. -else. benchmark_drain_rate_test() -> log(warmup, {warmup, 0}), @@ -286,16 +328,51 @@ benchmark_drain_rate_test() -> EventPid = hb_name:lookup(?MODULE), wait_drain(EventPid, 5000), N = 100000, + erlang:suspend_process(EventPid), fill_mailbox(EventPid, N), + erlang:resume_process(EventPid), {DrainTime, _} = timer:tc(fun() -> wait_drain(EventPid, 30000) end), - DrainRate = round(N / (DrainTime / 1_000_000)), + DrainRate = round(N / (max(1, DrainTime) / 1_000_000)), hb_test_utils:benchmark_print( <<"Drained">>, <<"events">>, DrainRate, 1), ?assert(DrainRate >= 10000), ok. +batch_correctness_test() -> + log(warmup, {warmup, 0}), + timer:sleep(100), + EventPid = hb_name:lookup(?MODULE), + wait_drain(EventPid, 5000), + NumKeys = 50, + N = 30000, + Keys = [{list_to_binary("corr_topic_" ++ integer_to_list(K)), + list_to_binary("corr_event_" ++ integer_to_list(K))} + || K <- lists:seq(1, NumKeys)], + Before = counters(), + BeforeCounts = [{T, E, deep_get([T, E], Before, 0)} || {T, E} <- Keys], + erlang:suspend_process(EventPid), + lists:foreach(fun(I) -> + {T, E} = lists:nth((I rem NumKeys) + 1, Keys), + EventPid ! {increment, T, E, 1} + end, lists:seq(1, N)), + erlang:resume_process(EventPid), + wait_drain(EventPid, 30000), + After = counters(), + PerKey = N div NumKeys, + lists:foreach(fun({T, E, BeforeVal}) -> + AfterVal = deep_get([T, E], After, 0), + ?assertEqual(PerKey, AfterVal - BeforeVal) + end, BeforeCounts), + ok. + +deep_get([Group, Name], Map, Default) -> + case maps:get(Group, Map, undefined) of + undefined -> Default; + Inner -> maps:get(Name, Inner, Default) + end. + fill_mailbox(_Pid, 0) -> ok; fill_mailbox(Pid, N) -> Pid ! {increment, <<"bench">>, <<"drain">>, 1}, From a34b7cd01ec0cb80816e7a4c5f1fe1732b7ac7ce Mon Sep 17 00:00:00 2001 From: James Piechota Date: Fri, 13 Mar 2026 15:23:46 -0400 Subject: [PATCH 116/135] fix: xxx_debug -> debug_xxx for ?event topics so they don't get added to prometheus --- src/dev_arweave.erl | 22 +++++++++++----------- src/dev_bundler.erl | 2 +- src/dev_bundler_cache.erl | 8 ++++---- src/dev_bundler_recovery.erl | 2 +- src/dev_bundler_task.erl | 6 +++--- src/dev_copycat_arweave.erl | 16 ++++++++++------ src/dev_process.erl | 2 +- src/dev_process_worker.erl | 6 +++--- src/dev_push.erl | 2 +- src/dev_scheduler.erl | 2 +- src/hb_http_server.erl | 2 +- src/hb_store_lmdb.erl | 22 +++++++++++----------- 12 files changed, 48 insertions(+), 44 deletions(-) diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index a8b1f0a5f..a539cf4c1 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -82,7 +82,7 @@ post_tx(_Base, Request, Opts, <<"tx@1.0">>) -> CacheRes = hb_cache:write(Request, Opts), case CacheRes of {ok, _} -> - ?event(arweave_debug, {tx_cached, {msg, Request}, {status, ok}}); + ?event(debug_arweave, {tx_cached, {msg, Request}, {status, ok}}); _ -> ?event(error, {tx_failed_to_cache, {msg, Request}, CacheRes}) end; @@ -396,7 +396,7 @@ get_chunk_range(_Base, Request, Opts) -> %% cannot span the strict data split threshold, so mixed ranges are rejected. fetch_chunk_range(Offset, Length, Opts) -> EndOffset = Offset + Length - 1, - ?event(arweave_debug, {fetch_chunk_range, + ?event(debug_arweave, {fetch_chunk_range, {offset, Offset}, {end_offset, EndOffset}, {size, Length}}), @@ -434,7 +434,7 @@ fetch_post_threshold(Offset, EndOffset, Opts) -> true -> ExtraOffset = min( lists:last(Offsets) + ?DATA_CHUNK_SIZE, EndOffset), - ?event(arweave_debug, {fetching_extra_chunk, + ?event(debug_arweave, {fetching_extra_chunk, {binary_size, BinarySize}, {expected_length, ExpectedLength}, {extra_offset, ExtraOffset}}), @@ -471,7 +471,7 @@ fill_gaps(ChunkInfos, Offset, EndOffset, Opts) -> % be needed. We have yet to find an L1 TX that is chunked in such % a way as to create gaps when using our naive 256KiB chunking. GapOffsets = [Start || {Start, _End} <- Gaps], - ?event(arweave_debug, + ?event(debug_arweave, {fill_gaps, {offset, Offset}, {end_offset, EndOffset}, @@ -490,7 +490,7 @@ fill_gaps(ChunkInfos, Offset, EndOffset, Opts) -> {gap_offsets, GapOffsets}}), case fetch_and_collect(GapOffsets, Opts) of {ok, NewInfos} -> - ?event(arweave_debug, {fill_gaps, NewInfos}), + ?event(debug_arweave, {fill_gaps, NewInfos}), fill_gaps( Sorted ++ NewInfos, Offset, EndOffset, Opts @@ -517,7 +517,7 @@ generate_offsets(Start, End, Step) -> generate_offsets(Current, End, _Step, Acc) when Current > End -> Offsets = lists:reverse(Acc), - ?event(arweave_debug, {fetch_chunk_offsets, {offsets, Offsets}}), + ?event(debug_arweave, {fetch_chunk_offsets, {offsets, Offsets}}), Offsets; generate_offsets(Current, End, Step, Acc) -> generate_offsets(Current + Step, End, Step, [Current | Acc]). @@ -533,7 +533,7 @@ collect_chunks([{ok, JSON} | Rest], Acc) -> Chunk = hb_util:decode(maps:get(<<"chunk">>, JSON)), AbsEnd = hb_util:int(maps:get(<<"absolute_end_offset">>, JSON)), AbsStart = AbsEnd - byte_size(Chunk) + 1, - ?event(arweave_debug, + ?event(debug_arweave, {collect_chunks, {abs_start, AbsStart}, {abs_end, AbsEnd}, @@ -598,7 +598,7 @@ assemble_chunks(ChunkInfos, Offset) -> % The first chunk may start before the requested offset; % trim the leading bytes to start exactly at Offset. Skip = Offset - ChunkStart, - ?event(arweave_debug, {assemble_chunks, + ?event(debug_arweave, {assemble_chunks, {skip, Skip}, {chunk_start, ChunkStart}, {offset, Offset}, @@ -607,7 +607,7 @@ assemble_chunks(ChunkInfos, Offset) -> }), binary:part(Data, Skip, byte_size(Data) - Skip); false -> - ?event(arweave_debug, {assemble_chunks, + ?event(debug_arweave, {assemble_chunks, {chunk_start, ChunkStart}, {offset, Offset}, {byte_size, byte_size(Data)} @@ -796,7 +796,7 @@ request(Method, Path, Opts) -> request(Method, Path, Extra, Opts) -> request(Method, Path, Extra, [], Opts). request(Method, Path, Extra, LogExtra, Opts) -> - ?event(arweave_debug, {request, + ?event(debug_arweave, {request, {method, Method}, {path, {explicit, Path}}, {log_extra, LogExtra}}), Res = hb_http:request( @@ -859,7 +859,7 @@ to_message(Path = <<"/tx">>, <<"POST">>, {ok, Response}, LogExtra, _Opts) -> to_message(Path = <<"/tx/", TXID/binary>>, <<"GET">>, {ok, #{ <<"body">> := Body }}, LogExtra, Opts) -> event_request(Path, <<"GET">>, 200, LogExtra), TXHeader = ar_tx:json_struct_to_tx(hb_json:decode(Body)), - ?event(arweave_debug, + ?event(debug_arweave, {arweave_tx_response, {path, {explicit, Path}}, {raw_body, {explicit, Body}}, diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index 47be72d4c..fb8f1c153 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -304,7 +304,7 @@ handle_task_complete(WorkerPID, Task, Result, State = #state{ bundles = Bundles }) -> #task{bundle_id = BundleID} = Task, - ?event(bundler_debug, dev_bundler_task:log_task(task_complete, Task, [])), + ?event(debug_bundler, dev_bundler_task:log_task(task_complete, Task, [])), State1 = State#state{ workers = maps:put(WorkerPID, idle, Workers) }, diff --git a/src/dev_bundler_cache.erl b/src/dev_bundler_cache.erl index 5545fff0f..ae538c933 100644 --- a/src/dev_bundler_cache.erl +++ b/src/dev_bundler_cache.erl @@ -90,7 +90,7 @@ complete_tx(TX, Opts) -> %% @doc Set the status of a bundle TX. set_tx_status(TX, Status, Opts) -> Path = tx_path(TX, Opts), - ?event(bundler_debug, {set_tx_status, {path, Path}, {status, Status}}), + ?event(debug_bundler, {set_tx_status, {path, Path}, {status, Status}}), write_pseudopath(Path, Status, Opts). %% @doc Get the status of a bundle TX. @@ -134,7 +134,7 @@ load_bundle_states(Opts) -> <<"complete">> -> false; % Skip completed bundles Status -> ?event( - bundler_debug, + debug_bundler, {loaded_tx_state, {id, {string, TXID}}, {status, Status} @@ -148,10 +148,10 @@ load_bundle_states(Opts) -> %% @doc Load a TX from cache by its ID. load_tx(TXID, Opts) -> - ?event(bundler_debug, {load_tx, {tx_id, {explicit, TXID}}}), + ?event(debug_bundler, {load_tx, {tx_id, {explicit, TXID}}}), case hb_cache:read(TXID, Opts) of {ok, TX} -> - ?event(bundler_debug, {loaded_tx, {tx_id, {explicit, TXID}}}), + ?event(debug_bundler, {loaded_tx, {tx_id, {explicit, TXID}}}), hb_cache:ensure_all_loaded(TX, Opts); _ -> ?event(error, {failed_to_load_tx, {tx_id, {explicit, TXID}}}), diff --git a/src/dev_bundler_recovery.erl b/src/dev_bundler_recovery.erl index 5368fd70a..1c619b79c 100644 --- a/src/dev_bundler_recovery.erl +++ b/src/dev_bundler_recovery.erl @@ -105,7 +105,7 @@ recover_bundle(ServerPID, TXID, Status, Opts) -> Opts, fun(ItemID, _Item) -> ?event( - bundler_debug, + debug_bundler, {loaded_bundle_item, {tx_id, {explicit, TXID}}, {item_id, {explicit, ItemID}} diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl index 2a0d90f1d..5978e9ba6 100644 --- a/src/dev_bundler_task.erl +++ b/src/dev_bundler_task.erl @@ -27,7 +27,7 @@ worker_loop() -> %% @doc Execute a specific task. execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> try - ?event(bundler_debug, log_task(executing_task, Task, [])), + ?event(debug_bundler, log_task(executing_task, Task, [])), % Get price and anchor {ok, TX} = dev_codec_tx:to(lists:reverse(Items), #{}, #{}), DataSize = TX#tx.data_size, @@ -83,7 +83,7 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> execute_task(#task{type = build_proofs, data = CommittedTX, opts = Opts} = Task) -> try - ?event(bundler_debug, log_task(executing_task, Task, [])), + ?event(debug_bundler, log_task(executing_task, Task, [])), % Calculate chunks and proofs TX = hb_message:convert( CommittedTX, <<"tx@1.0">>, <<"structured@1.0">>, Opts), @@ -139,7 +139,7 @@ execute_task(#task{type = build_proofs, data = CommittedTX, opts = Opts} = Task) execute_task(#task{type = post_proof, data = Proof, opts = Opts} = Task) -> #{chunk := Chunk, data_path := DataPath, offset := Offset, data_size := DataSize, data_root := DataRoot} = Proof, - ?event(bundler_debug, log_task(executing_task, Task, [])), + ?event(debug_bundler, log_task(executing_task, Task, [])), Request = #{ <<"chunk">> => hb_util:encode(Chunk), <<"data_path">> => hb_util:encode(DataPath), diff --git a/src/dev_copycat_arweave.erl b/src/dev_copycat_arweave.erl index e55624213..970b3d9b4 100644 --- a/src/dev_copycat_arweave.erl +++ b/src/dev_copycat_arweave.erl @@ -115,7 +115,7 @@ list_index_blocks(Current, To, Opts, Acc) -> end. fetch_block_header(Height, Opts) -> - ?event(copycat_debug, {fetching_block, Height}), + ?event(debug_copycat, {fetching_block, Height}), observe_event(<<"block_header">>, fun() -> hb_ao:resolve( << @@ -192,7 +192,7 @@ fetch_and_process_block(Current, To, Opts) -> process_block(BlockRes, Current, To, Opts) -> case BlockRes of {ok, Block} -> - ?event(copycat_debug, {{processing_block, Current}, + ?event(debug_copycat, {{processing_block, Current}, {indep_hash, hb_maps:get(<<"indep_hash">>, Block, <<>>)}}), case maybe_index_ids(Block, Opts) of {block_skipped, Results} -> @@ -286,7 +286,7 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> TXID = hb_util:encode(TX#tx.id), TXEndOffset = BlockStartOffset + EndOffset, TXStartOffset = TXEndOffset - TX#tx.data_size, - ?event(copycat_debug, {writing_index, + ?event(debug_copycat, {writing_index, {id, {explicit, TXID}}, {offset, TXStartOffset}, {size, TX#tx.data_size} @@ -303,7 +303,11 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> case is_bundle_tx(TX, Opts) of false -> #{items_count => 0, bundle_count => 0, skipped_count => 0}; true -> - ?event(copycat_debug, {fetching_bundle_header, + % Lightweight processing of block transactions to depth 2. We + % can avoid loading the full L1 TX data into memory, and instead + % only load the bundle header. But as a result we're unable to + % recurse any deeper than L2 dataitems. + ?event(debug_copycat, {fetching_bundle_header, {tx_id, {string, TXID}}, {tx_end_offset, TXEndOffset}, {tx_data_size, TX#tx.data_size} @@ -330,7 +334,7 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> BundleIndex ) end), - ?event(copycat_debug, + ?event(debug_copycat, {bundle_items_indexed, {tx_id, {string, TXID}}, {items_count, ItemsCount} @@ -400,7 +404,7 @@ resolve_tx_headers(TXIDs, Opts) -> resolve_tx_header(TXID, Opts) -> try - ?event(copycat_debug, {fetching_tx, {explicit, TXID}}), + ?event(debug_copycat, {fetching_tx, {explicit, TXID}}), ResolveRes = observe_event(<<"tx_header">>, fun() -> hb_ao:resolve( << diff --git a/src/dev_process.erl b/src/dev_process.erl index bec7cb058..12a124a14 100644 --- a/src/dev_process.erl +++ b/src/dev_process.erl @@ -461,7 +461,7 @@ store_result(ForceSnapshot, ProcID, Slot, Res, Req, Opts) -> false -> Res; true -> ?event( - compute_debug, + debug_compute, {snapshotting, {proc_id, ProcID}, {slot, Slot}}, Opts ), diff --git a/src/dev_process_worker.erl b/src/dev_process_worker.erl index e6bd9ab90..4789f1263 100644 --- a/src/dev_process_worker.erl +++ b/src/dev_process_worker.erl @@ -106,13 +106,13 @@ await(Worker, GroupName, Base, Req, Opts) -> receive {resolved, _, GroupName, {slot, RecvdSlot}, Res} when RecvdSlot == TargetSlot orelse TargetSlot == any -> - ?event(compute_debug, {notified_of_resolution, + ?event(debug_compute, {notified_of_resolution, {target, TargetSlot}, {group, GroupName} }), Res; {resolved, _, GroupName, {slot, RecvdSlot}, _Res} -> - ?event(compute_debug, {waiting_again, + ?event(debug_compute, {waiting_again, {target, TargetSlot}, {recvd, RecvdSlot}, {worker, Worker}, @@ -120,7 +120,7 @@ await(Worker, GroupName, Base, Req, Opts) -> }), await(Worker, GroupName, Base, Req, Opts); {'DOWN', _R, process, Worker, _Reason} -> - ?event(compute_debug, + ?event(debug_compute, {leader_died, {group, GroupName}, {leader, Worker}, diff --git a/src/dev_push.erl b/src/dev_push.erl index bd19ce7fa..9c260f017 100644 --- a/src/dev_push.erl +++ b/src/dev_push.erl @@ -1263,7 +1263,7 @@ nested_push_prompts_encoding_change() -> cache_control => <<"always">>, store => hb_opts:get(store) }, - ?event(push_debug, {opts, Opts}), + ?event(debug_push, {opts, Opts}), Base = dev_process_test_vectors:aos_process(Opts), hb_cache:write(Base, Opts), {ok, SchedInit} = diff --git a/src/dev_scheduler.erl b/src/dev_scheduler.erl index bb85c2cba..3d165342a 100644 --- a/src/dev_scheduler.erl +++ b/src/dev_scheduler.erl @@ -211,7 +211,7 @@ find_next_assignment(Base, Req, _Schedule, LastSlot, Opts) -> end, case LocalCacheRes of {ok, Worker, Assignment} -> - ?event(next_debug, + ?event(debug_next, {in_cache, {slot, LastSlot + 1}, {assignment, Assignment} diff --git a/src/hb_http_server.erl b/src/hb_http_server.erl index 428e18081..3b1f7cfc3 100644 --- a/src/hb_http_server.erl +++ b/src/hb_http_server.erl @@ -365,7 +365,7 @@ cors_reply(Req, _ServerID) -> <<"access-control-allow-methods">> => <<"GET, POST, PUT, DELETE, OPTIONS, PATCH">> }, Req), - ?event(http_debug, {cors_reply, {req, Req}, {req2, Req2}}), + ?event(debug_http, {cors_reply, {req, Req}, {req2, Req2}}), {ok, Req2, no_state}. %% @doc Handle all non-CORS preflight requests as AO-Core requests. Execution diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index e1f6c7311..ae813c2d1 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -1002,29 +1002,29 @@ isolated_type_debug_test() -> % 2. Create nested groups for "commitments" and "other-test-key" CommitmentsPath = <>, OtherKeyPath = <>, - ?event(isolated_debug, {creating_nested_groups, CommitmentsPath, OtherKeyPath}), + ?event(debug_isolated, {creating_nested_groups, CommitmentsPath, OtherKeyPath}), make_group(StoreOpts, CommitmentsPath), make_group(StoreOpts, OtherKeyPath), % 3. Add some actual data within those groups write(StoreOpts, <>, <<"signature_data_1">>), write(StoreOpts, <>, <<"nested_value">>), % 4. Test type detection on the nested paths - ?event(isolated_debug, {testing_main_message_type}), + ?event(debug_isolated, {testing_main_message_type}), MainType = type(StoreOpts, MessageID), - ?event(isolated_debug, {main_message_type, MainType}), - ?event(isolated_debug, {testing_commitments_type}), + ?event(debug_isolated, {main_message_type, MainType}), + ?event(debug_isolated, {testing_commitments_type}), CommitmentsType = type(StoreOpts, CommitmentsPath), - ?event(isolated_debug, {commitments_type, CommitmentsType}), - ?event(isolated_debug, {testing_other_key_type}), + ?event(debug_isolated, {commitments_type, CommitmentsType}), + ?event(debug_isolated, {testing_other_key_type}), OtherKeyType = type(StoreOpts, OtherKeyPath), - ?event(isolated_debug, {other_key_type, OtherKeyType}), + ?event(debug_isolated, {other_key_type, OtherKeyType}), % 5. Test what happens when reading these nested paths - ?event(isolated_debug, {reading_commitments_directly}), + ?event(debug_isolated, {reading_commitments_directly}), CommitmentsResult = read(StoreOpts, CommitmentsPath), - ?event(isolated_debug, {commitments_read_result, CommitmentsResult}), - ?event(isolated_debug, {reading_other_key_directly}), + ?event(debug_isolated, {commitments_read_result, CommitmentsResult}), + ?event(debug_isolated, {reading_other_key_directly}), OtherKeyResult = read(StoreOpts, OtherKeyPath), - ?event(isolated_debug, {other_key_read_result, OtherKeyResult}), + ?event(debug_isolated, {other_key_read_result, OtherKeyResult}), stop(StoreOpts). %% @doc Test that list function resolves links correctly From 2171c23b6144b475f68ac415b1e81857cd3430a5 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Fri, 13 Mar 2026 16:14:48 -0400 Subject: [PATCH 117/135] fix: allow binary event names --- src/hb_event.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hb_event.erl b/src/hb_event.erl index 93f44a9c6..cafe5a561 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -226,6 +226,8 @@ parse_name(Name) no_event_name; parse_name(Name) when is_list(Name) -> iolist_to_binary(Name); +parse_name(Name) when is_binary(Name) -> + Name; parse_name(_) -> no_event_name. %%% Benchmark tests From 867abb298600190c8150f4a4bbdc54af2cc58971 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Mon, 16 Mar 2026 21:06:20 -0400 Subject: [PATCH 118/135] fix: don't need full chunk to get dataitem header --- src/dev_arweave.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index a539cf4c1..f632372a5 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -236,11 +236,12 @@ head_raw_tx(TXID, StartOffset, Length, Opts) -> %% so to read the data associated with their IDs, we must first read the header %% chunk, deserialize it, and offset our data read from its starting offset. head_raw_ans104(TXID, ArweaveOffset, Length, Opts) -> + ?event(debug_raw, {head_raw_ans104, {txid, TXID}, {arweave_offset, ArweaveOffset}, {length, Length}}), HeaderReq = #{ <<"path">> => <<"chunk">>, <<"offset">> => ArweaveOffset + 1, - <<"length">> => ?DATA_CHUNK_SIZE + <<"length">> => min(Length, ?DATA_CHUNK_SIZE) }, case hb_ao:resolve(#{ <<"device">> => <<"arweave@2.9">> }, HeaderReq, Opts) of {ok, HeaderChunk} -> From 86b611c8ff0e347c186018e141dff67e0385fcce Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 17 Mar 2026 12:19:35 -0400 Subject: [PATCH 119/135] impr: remove process dictionary --- src/hb_event.erl | 70 ++++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 215224ca9..66194f882 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -181,50 +181,65 @@ handle_events() -> handle_events(N) -> receive {increment, TopicBin, EventName, Count} -> - erlang:put(batch_keys, []), - N2 = drain_batch(N + 1, TopicBin, EventName, Count, ?BATCH_MAX - 1), + {N2, Batch} = + drain_batch( + N + 1, + TopicBin, + EventName, + Count, + {#{}, []}, + ?BATCH_MAX - 1 + ), hb_prometheus:ensure_started(), - Keys = flush_batch(), + Keys = flush_batch(Batch), check_overload(Keys, N, N2), handle_events(N2) end. -drain_batch(N, LastT, LastE, Acc, 0) -> - pd_inc({batch, LastT, LastE}, Acc), - N; -drain_batch(N, LastT, LastE, Acc, Remaining) -> +drain_batch(N, LastT, LastE, Acc, Batch, 0) -> + {N, batch_inc({batch, LastT, LastE}, Acc, Batch)}; +drain_batch(N, LastT, LastE, Acc, Batch, Remaining) -> receive {increment, TopicBin, EventName, Count} -> case TopicBin =:= LastT andalso EventName =:= LastE of true -> - drain_batch(N + 1, LastT, LastE, Acc + Count, Remaining - 1); + drain_batch( + N + 1, + LastT, + LastE, + Acc + Count, + Batch, + Remaining - 1 + ); false -> - pd_inc({batch, LastT, LastE}, Acc), - drain_batch(N + 1, TopicBin, EventName, Count, Remaining - 1) + drain_batch( + N + 1, + TopicBin, + EventName, + Count, + batch_inc({batch, LastT, LastE}, Acc, Batch), + Remaining - 1 + ) end after 0 -> - pd_inc({batch, LastT, LastE}, Acc), - N + {N, batch_inc({batch, LastT, LastE}, Acc, Batch)} end. -pd_inc(Key, Count) -> - case erlang:get(Key) of +batch_inc(Key, Count, {Counts, Keys}) -> + case maps:get(Key, Counts, undefined) of undefined -> - erlang:put(Key, Count), - erlang:put(batch_keys, [Key | erlang:get(batch_keys)]); + {Counts#{ Key => Count }, [Key | Keys]}; Old when is_integer(Old) -> - erlang:put(Key, Old + Count) + {Counts#{ Key => Old + Count }, Keys} end. -flush_batch() -> - Keys = erlang:get(batch_keys), +flush_batch({Counts, Keys}) -> lists:foreach( fun(Key = {batch, Topic, Event}) -> - prometheus_counter:inc(<<"event">>, [Topic, Event], erlang:get(Key)), - erlang:erase(Key) + prometheus_counter:inc(<<"event">>, [Topic, Event], maps:get(Key, Counts)) end, - Keys), - erlang:put(batch_keys, []), + Keys + ), Keys. check_overload(Keys, Prev, N) -> @@ -331,9 +346,12 @@ benchmark_drain_rate_test() -> erlang:suspend_process(EventPid), fill_mailbox(EventPid, N), erlang:resume_process(EventPid), - {DrainTime, _} = timer:tc(fun() -> - wait_drain(EventPid, 30000) - end), + {DrainTime, _} = + timer:tc( + fun() -> + wait_drain(EventPid, 30000) + end + ), DrainRate = round(N / (max(1, DrainTime) / 1_000_000)), hb_test_utils:benchmark_print( <<"Drained">>, <<"events">>, DrainRate, 1), From 459294a00e9aee1248a8ba0f00a1f856ecbe23d7 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 17 Mar 2026 12:39:39 -0400 Subject: [PATCH 120/135] impr: test realism --- src/hb_event.erl | 37 +++++++++++++++++++++++++++---------- src/hb_util.erl | 2 +- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 66194f882..150b4aed9 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -7,7 +7,7 @@ -include_lib("eunit/include/eunit.hrl"). -define(OVERLOAD_QUEUE_LENGTH, 10000). --define(MAX_MEMORY, 1_000_000_000). % 1GB +-define(MAX_MEMORY, 50_000_000). % 50 MB -define(MAX_EVENT_NAME_LENGTH, 100). -define(BATCH_MAX, 10000). @@ -338,13 +338,23 @@ benchmark_drain_rate_test() -> ok. batch_correctness_test() -> ok. -else. benchmark_drain_rate_test() -> + NumKeys = 50, + NumEvents = 100000, log(warmup, {warmup, 0}), timer:sleep(100), EventPid = hb_name:lookup(?MODULE), wait_drain(EventPid, 5000), - N = 100000, erlang:suspend_process(EventPid), - fill_mailbox(EventPid, N), + Keys = + [ + { + hb_util:bin([<<"corr-topic-">>, hb_util:int(K)]), + hb_util:bin([<<"corr-event-">>, hb_util:int(K)]) + } + || + K <- lists:seq(1, NumKeys) + ], + fill_mailbox(EventPid, NumEvents, Keys), erlang:resume_process(EventPid), {DrainTime, _} = timer:tc( @@ -352,9 +362,13 @@ benchmark_drain_rate_test() -> wait_drain(EventPid, 30000) end ), - DrainRate = round(N / (max(1, DrainTime) / 1_000_000)), + DrainRate = round(NumEvents / (max(1, DrainTime) / 1_000_000)), hb_test_utils:benchmark_print( - <<"Drained">>, <<"events">>, DrainRate, 1), + <<"Drained">>, + <<"events">>, + DrainRate, + 1 + ), ?assert(DrainRate >= 10000), ok. @@ -363,7 +377,7 @@ batch_correctness_test() -> timer:sleep(100), EventPid = hb_name:lookup(?MODULE), wait_drain(EventPid, 5000), - NumKeys = 50, + NumKeys = 5, N = 30000, Keys = [{list_to_binary("corr_topic_" ++ integer_to_list(K)), list_to_binary("corr_event_" ++ integer_to_list(K))} @@ -391,10 +405,13 @@ deep_get([Group, Name], Map, Default) -> Inner -> maps:get(Name, Inner, Default) end. -fill_mailbox(_Pid, 0) -> ok; -fill_mailbox(Pid, N) -> - Pid ! {increment, <<"bench">>, <<"drain">>, 1}, - fill_mailbox(Pid, N - 1). +%% @doc Fill the event server mailbox with a list of keys. Rotate the keys to +%% ensure that we are testing the event server's ability to handle many different +%% types of event. +fill_mailbox(_Pid, 0, _Keys) -> ok; +fill_mailbox(Pid, N, Keys = [{Topic, Event}|_]) -> + Pid ! {increment, Topic, Event, 1}, + fill_mailbox(Pid, N - 1, hb_util:shuffle(Keys)). wait_drain(Pid, Timeout) -> Deadline = erlang:monotonic_time(millisecond) + Timeout, diff --git a/src/hb_util.erl b/src/hb_util.erl index 7e4db770e..653224ce2 100644 --- a/src/hb_util.erl +++ b/src/hb_util.erl @@ -20,7 +20,7 @@ -export([maybe_throw/2]). -export([is_hb_module/1, is_hb_module/2, all_hb_modules/0]). -export([ok/1, ok/2, until/1, until/2, until/3, wait_until/2]). --export([count/2, mean/1, stddev/1, variance/1, weighted_random/1]). +-export([count/2, mean/1, stddev/1, variance/1, weighted_random/1, shuffle/1]). -export([unique/1]). -export([split_depth_string_aware/2, split_depth_string_aware_single/2]). -export([unquote/1, split_escaped_single/2]). From 3aeea2d55c7a7c477a732500c73e636429fc5a30 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 17 Mar 2026 12:52:39 -0400 Subject: [PATCH 121/135] impr: simplify --- src/hb_event.erl | 81 +++++++----------------------------------------- 1 file changed, 12 insertions(+), 69 deletions(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index 150b4aed9..662f29c55 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -6,10 +6,9 @@ -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(OVERLOAD_QUEUE_LENGTH, 10000). +-define(OVERLOAD_QUEUE_LENGTH, 10_000). -define(MAX_MEMORY, 50_000_000). % 50 MB -define(MAX_EVENT_NAME_LENGTH, 100). --define(BATCH_MAX, 10000). -ifdef(NO_EVENTS). log(_X) -> ok. @@ -180,82 +179,26 @@ handle_events() -> handle_events(0). handle_events(N) -> receive - {increment, TopicBin, EventName, Count} -> - {N2, Batch} = - drain_batch( - N + 1, - TopicBin, - EventName, - Count, - {#{}, []}, - ?BATCH_MAX - 1 - ), - hb_prometheus:ensure_started(), - Keys = flush_batch(Batch), - check_overload(Keys, N, N2), - handle_events(N2) + {increment, Topic, Event, Count} -> + BatchCount = 0, + prometheus_counter:inc(<<"event">>, [Topic, Event], Count + BatchCount), + check_overload({Topic, Event}, N), + handle_events(N + 1) end. -drain_batch(N, LastT, LastE, Acc, Batch, 0) -> - {N, batch_inc({batch, LastT, LastE}, Acc, Batch)}; -drain_batch(N, LastT, LastE, Acc, Batch, Remaining) -> - receive - {increment, TopicBin, EventName, Count} -> - case TopicBin =:= LastT andalso EventName =:= LastE of - true -> - drain_batch( - N + 1, - LastT, - LastE, - Acc + Count, - Batch, - Remaining - 1 - ); - false -> - drain_batch( - N + 1, - TopicBin, - EventName, - Count, - batch_inc({batch, LastT, LastE}, Acc, Batch), - Remaining - 1 - ) - end - after 0 -> - {N, batch_inc({batch, LastT, LastE}, Acc, Batch)} - end. - -batch_inc(Key, Count, {Counts, Keys}) -> - case maps:get(Key, Counts, undefined) of - undefined -> - {Counts#{ Key => Count }, [Key | Keys]}; - Old when is_integer(Old) -> - {Counts#{ Key => Old + Count }, Keys} - end. - -flush_batch({Counts, Keys}) -> - lists:foreach( - fun(Key = {batch, Topic, Event}) -> - prometheus_counter:inc(<<"event">>, [Topic, Event], maps:get(Key, Counts)) - end, - Keys - ), - Keys. - -check_overload(Keys, Prev, N) -> - case N div 1000 > Prev div 1000 of - true -> +check_overload(Last, N) -> + case N div 1000 of + 0 -> case erlang:process_info(self(), message_queue_len) of {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH -> {memory, MemorySize} = erlang:process_info(self(), memory), - SampleKeys = lists:sublist(Keys, 5), case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of 1 -> ?debug_print( {warning, prometheus_event_queue_overloading, {queue, Len}, - {sample_keys, SampleKeys}, + {last_event, Last}, {memory_bytes, MemorySize} } ); @@ -268,7 +211,7 @@ check_overload(Keys, Prev, N) -> prometheus_event_queue_terminating_on_memory_overload, {queue, Len}, {memory_bytes, MemorySize}, - {sample_keys, SampleKeys} + {last_event, Last} } ), exit(memory_overload); @@ -378,7 +321,7 @@ batch_correctness_test() -> EventPid = hb_name:lookup(?MODULE), wait_drain(EventPid, 5000), NumKeys = 5, - N = 30000, + N = 30_000, Keys = [{list_to_binary("corr_topic_" ++ integer_to_list(K)), list_to_binary("corr_event_" ++ integer_to_list(K))} || K <- lists:seq(1, NumKeys)], From 3033681cd940317acb141859dc00ecb944d514bf Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 17 Mar 2026 17:22:35 +0000 Subject: [PATCH 122/135] fix: Make TX load, and doesn't try to parse OldMsg --- src/dev_b32_name.erl | 4 ++-- src/dev_manifest.erl | 9 ++++++++- src/dev_name.erl | 7 +------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/dev_b32_name.erl b/src/dev_b32_name.erl index 868e53efb..798358b9e 100644 --- a/src/dev_b32_name.erl +++ b/src/dev_b32_name.erl @@ -209,7 +209,7 @@ manifest_subdomain_matches_path_id_test() -> %% In this case, sinse no assets exists with this TX ID, it should load the %% index. manifest_subdomain_does_not_match_path_id_test() -> - TestPath = <<"/oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">>, + TestPath = <<"/1rTy7gQuK9lJydlKqCEhtGLp2WWG-GOrVo5JdiCmaxs">>, Opts = manifest_opts(), Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, Node = hb_http_server:start_node(Opts), @@ -218,7 +218,7 @@ manifest_subdomain_does_not_match_path_id_test() -> #{ <<"commitments">> := #{ - <<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _ + <<"1rTy7gQuK9lJydlKqCEhtGLp2WWG-GOrVo5JdiCmaxs">> := _ } } }, diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index 17a87c250..395cc2492 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -389,7 +389,14 @@ manifest_should_fallback_on_not_found_path_test() -> %% @doc Returns `Opts' with the test manifest fixture flow used by `dev_b32_name'. test_env_opts() -> TempStore = hb_test_utils:test_store(), - BaseOpts = #{store => [TempStore]}, + BaseOpts = + #{ + store => + [ + TempStore + #{<<"store-module">> => hb_store_gateway} + ] + }, lists:foreach( fun(Ref) -> hb_test_utils:preload( diff --git a/src/dev_name.erl b/src/dev_name.erl index f5b32f758..f7c275a7f 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -140,7 +140,7 @@ maybe_append_named_message(ResolvedMsg, OldReq = [OldBase|ReqMsgsRest], Opts) -> } ), [ResolvedMsg|ReqMsgsRest]; - _ -> [ResolvedMsg, as_message_or_link(OldBase)|ReqMsgsRest] + _ -> [ResolvedMsg, OldBase|ReqMsgsRest] end end. @@ -151,11 +151,6 @@ permissive_id({link, ID, _LinkOpts}, _Opts) -> ID; permissive_id({as, _Device, Msg}, Opts) -> permissive_id(Msg, Opts); permissive_id(Msg, Opts) when is_map(Msg) -> hb_message:id(Msg, signed, Opts). -%% @doc Ensure that a message reference is converted to a message or link. -as_message_or_link(ID) when ?IS_ID(ID) -> {link, ID, #{}}; -as_message_or_link(Msg) when is_map(Msg) -> Msg; -as_message_or_link(Link) when ?IS_LINK(Link) -> Link. - %% @doc Takes a request-given host and the host value in the node message and %% returns only the name component of the host, if it is present. If no name is %% present, an empty binary is returned. From 232284c24de80dd0a68854ef754c2c841b1f943b Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 17 Mar 2026 15:06:14 -0300 Subject: [PATCH 123/135] fix: div -> rem so the check runs --- src/hb_event.erl | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/hb_event.erl b/src/hb_event.erl index db4521096..426d73d50 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -187,7 +187,7 @@ handle_events(N) -> end. check_overload(Last, N) -> - case N div 1000 of + case N rem 1000 of 0 -> case erlang:process_info(self(), message_queue_len) of {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH -> @@ -281,6 +281,7 @@ benchmark_increment_test() -> -ifdef(NO_EVENTS). benchmark_drain_rate_test() -> ok. batch_correctness_test() -> ok. +overload_checks_past_first_thousand_test() -> ok. -else. benchmark_drain_rate_test() -> NumKeys = 50, @@ -344,6 +345,40 @@ batch_correctness_test() -> end, BeforeCounts), ok. +overload_checks_past_first_thousand_test() -> + {EventPid, Ref} = + spawn_monitor( + fun() -> + hb_prometheus:ensure_started(), + ensure_event_counter(), + handle_events(1000) + end + ), + erlang:suspend_process(EventPid), + Topic = lists:duplicate(256, $a), + Event = lists:duplicate(256, $b), + lists:foreach( + fun(_) -> + EventPid ! {increment, Topic, Event, 1} + end, + lists:seq(1, ?OVERLOAD_QUEUE_LENGTH + 100) + ), + {message_queue_len, QueueLen} = + erlang:process_info(EventPid, message_queue_len), + {memory, MemorySize} = erlang:process_info(EventPid, memory), + ?assert(QueueLen > ?OVERLOAD_QUEUE_LENGTH), + ?assert(MemorySize > ?MAX_MEMORY), + erlang:resume_process(EventPid), + receive + {'DOWN', Ref, process, EventPid, memory_overload} -> + ok; + {'DOWN', Ref, process, EventPid, Reason} -> + ?assertEqual(memory_overload, Reason) + after 5000 -> + exit(EventPid, kill), + error(memory_overload_not_triggered) + end. + deep_get([Group, Name], Map, Default) -> case maps:get(Group, Map, undefined) of undefined -> Default; From 86cd2a047c6ccaa9120d70839e87b87edf17498f Mon Sep 17 00:00:00 2001 From: James Piechota Date: Tue, 17 Mar 2026 15:02:02 -0400 Subject: [PATCH 124/135] fix: correctly handle inputs to dev_codec_tx when building a bundle Use hb_message:convert rather than dev_codec_ans104:to to correctly handle converting to/from TABM. Previous implementation could allow structure messages to make it to the codec which were not handled correctly --- src/dev_bundler.erl | 2 + src/dev_bundler_task.erl | 103 +++++++++++++++++++++++++++++++++------ src/dev_codec_tx.erl | 9 +++- 3 files changed, 97 insertions(+), 17 deletions(-) diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index fb8f1c153..a80784ab0 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -16,6 +16,8 @@ %%% transaction is dispatched. -module(dev_bundler). -export([tx/3, item/3, ensure_server/1, stop_server/0, get_state/0]). +%%% Test-only exports. +-export([start_mock_gateway/1]). -include("include/hb.hrl"). -include("include/dev_bundler.hrl"). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl index 5978e9ba6..6372dca1f 100644 --- a/src/dev_bundler_task.erl +++ b/src/dev_bundler_task.erl @@ -28,20 +28,8 @@ worker_loop() -> execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> try ?event(debug_bundler, log_task(executing_task, Task, [])), - % Get price and anchor - {ok, TX} = dev_codec_tx:to(lists:reverse(Items), #{}, #{}), - DataSize = TX#tx.data_size, - PriceResult = get_price(DataSize, Opts), - AnchorResult = get_anchor(Opts), - case {PriceResult, AnchorResult} of - {{ok, Price}, {ok, Anchor}} -> - % Sign the TX - Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), - SignedTX = ar_tx:sign(TX#tx{ anchor = Anchor, reward = Price }, Wallet), - % TODO: as a future improvement we should be able to recover - % from the TX header alone, but we have to be careful about - % how we rebuild the TX data to ensure it matches the already - % posted TX. + case build_signed_tx(Items, Opts) of + {ok, SignedTX} -> Committed = hb_message:convert( SignedTX, #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, @@ -65,7 +53,7 @@ execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> {ok, Committed}; {_, ErrorReason} -> {error, ErrorReason} end; - {PriceErr, AnchorErr} -> + {error, {PriceErr, AnchorErr}} -> ?event(bundler_short, log_task(task_failed, Task, [ {price, PriceErr}, @@ -160,6 +148,24 @@ execute_task(#task{type = post_proof, data = Proof, opts = Opts} = Task) -> {error, Err} end. +%% @doc Build and sign a bundle TX without posting it. +build_signed_tx(Items, Opts) -> + {ok, TX} = dev_codec_tx:to(lists:reverse(Items), #{}, #{}), + DataSize = TX#tx.data_size, + PriceResult = get_price(DataSize, Opts), + AnchorResult = get_anchor(Opts), + case {PriceResult, AnchorResult} of + {{ok, Price}, {ok, Anchor}} -> + Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), + SignedTX = ar_tx:sign( + TX#tx{anchor = Anchor, reward = Price}, + Wallet + ), + {ok, SignedTX}; + {PriceErr, AnchorErr} -> + {error, {PriceErr, AnchorErr}} + end. + get_price(DataSize, Opts) -> hb_ao:resolve( #{ <<"device">> => <<"arweave@2.9">> }, @@ -210,4 +216,69 @@ format_task(#task{bundle_id = BundleID, type = post_proof, data = Proof}) -> format_timestamp() -> {MegaSecs, Secs, MicroSecs} = erlang:timestamp(), Millisecs = (MegaSecs * 1000000 + Secs) * 1000 + (MicroSecs div 1000), - calendar:system_time_to_rfc3339(Millisecs, [{unit, millisecond}, {offset, "Z"}]). \ No newline at end of file + calendar:system_time_to_rfc3339(Millisecs, [{unit, millisecond}, {offset, "Z"}]). + +build_signed_tx_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = dev_bundler:start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + TestOpts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store() + }, + try + Timestamp = 12344567, + ListValue = [<<"a">>, <<"b">>, <<"c">>], + StructuredItems = [ + #{ + <<"body">> => <<"body1">>, + <<"tag1">> => <<"value1">>, + <<"timestamp">> => Timestamp + }, + #{ + <<"body">> => <<"body3">>, + <<"tag3">> => <<"value3">>, + <<"list">> => ListValue + }, + #{ + <<"body">> => <<"body2">>, + <<"tag2">> => <<"value2">> + } + ], + Items = [ + hb_message:commit( + Item, + TestOpts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true } + ) + || Item <- StructuredItems], + {ok, SignedTX} = build_signed_tx(Items, TestOpts), + ?assert(ar_tx:verify(SignedTX)), + ?assertEqual(Anchor, SignedTX#tx.anchor), + ?assertEqual(Price, SignedTX#tx.reward), + ?event(debug_test, {signed_tx, SignedTX}), + BundledTX = ar_bundles:deserialize(SignedTX), + ?event(debug_test, {bundled_tx, BundledTX}), + BundledItems = hb_util:numbered_keys_to_list(BundledTX#tx.data, #{}), + lists:foreach( + fun(Item) -> + ?assert(ar_bundles:verify_item(Item)) + end, + BundledItems + ), + BundledStructuredItems = [ + hb_message:convert( + Item, + <<"structured@1.0">>, + <<"ans104@1.0">>, + TestOpts + ) + || Item <- BundledItems], + ?assertEqual(lists:reverse(Items), BundledStructuredItems), + ok + after + hb_mock_server:stop(ServerHandle) + end. \ No newline at end of file diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index 46f91bc93..69d80f954 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -150,7 +150,14 @@ to(RawTABM, Req, Opts) when is_map(RawTABM) -> %% @doc List of ans104 items is bundled into a single L1 transaction. to(RawList, Req, Opts) when is_list(RawList) -> List = lists:map( - fun(Item) -> hb_util:ok(dev_codec_ans104:to(Item, Req, Opts)) end, + fun(Item) -> + hb_message:convert( + Item, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + Opts + ) + end, RawList), TX = #tx{ format = 2, From 38174cb2de64f81e18cb896ea5c92a72e9147978 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Tue, 17 Mar 2026 19:24:13 +0000 Subject: [PATCH 125/135] fix: load base32 as application --- rebar.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rebar.config b/rebar.config index 41013af47..6008157c9 100644 --- a/rebar.config +++ b/rebar.config @@ -16,7 +16,7 @@ {deps, [{observer_cli, {git, "https://github.com/permaweb/observer_cli.git", {ref, "7f90812dcd43d7954d0ba066cb94d5e934e729b5"}}}]}, {erl_opts, [{d, 'AO_TOP', true}]}, - {relx, [{release, {'hb', "0.0.1"}, [hb, b64fast, cowboy, gun, luerl, prometheus, prometheus_cowboy, prometheus_ranch, elmdb, observer_cli]}]} + {relx, [{release, {'hb', "0.0.1"}, [hb, b64fast, base32, cowboy, gun, luerl, prometheus, prometheus_cowboy, prometheus_ranch, elmdb, observer_cli]}]} ]}, {store_events, [{erl_opts, [{d, 'STORE_EVENTS', true}]}]}, {ao_profiling, [{erl_opts, [{d, 'AO_PROFILING', true}]}]}, @@ -169,7 +169,7 @@ ]}. {relx, [ - {release, {'hb', "0.0.1"}, [hb, b64fast, cowboy, gun, luerl, prometheus, prometheus_cowboy, prometheus_ranch, elmdb]}, + {release, {'hb', "0.0.1"}, [hb, b64fast, base32, cowboy, gun, luerl, prometheus, prometheus_cowboy, prometheus_ranch, elmdb]}, {sys_config, "config/app.config"}, {include_erts, true}, {extended_start_script, true}, @@ -211,4 +211,4 @@ {preprocess, true}, {private, true}, {hidden, true} -]}. +]}. \ No newline at end of file From 05e1f4a8a4998e6fa0ece06a7f85bd49641723ca Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 17 Mar 2026 15:34:00 -0400 Subject: [PATCH 126/135] Merge pull request #744 from permaweb/fix/hyperbuddy-path-resolution fix: resolve hyperbuddy paths from server root --- src/html/hyperbuddy@1.0/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/html/hyperbuddy@1.0/index.html b/src/html/hyperbuddy@1.0/index.html index 049ae14fa..aa2f6ef7f 100644 --- a/src/html/hyperbuddy@1.0/index.html +++ b/src/html/hyperbuddy@1.0/index.html @@ -5,10 +5,10 @@ HyperBEAM - +
- + From 8dda9ca90a26fb789050d8047de95c1c43b44b3c Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 17 Mar 2026 19:40:19 -0400 Subject: [PATCH 127/135] feat: add support for serving custom local files as part of the `hyperbuddy@1.0` device --- src/dev_hyperbuddy.erl | 47 +++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/src/dev_hyperbuddy.erl b/src/dev_hyperbuddy.erl index 18c70abf5..5ccc8a609 100644 --- a/src/dev_hyperbuddy.erl +++ b/src/dev_hyperbuddy.erl @@ -1,16 +1,17 @@ %%% @doc A device that renders a REPL-like interface for AO-Core via HTML. -module(dev_hyperbuddy). --export([info/0, format/3, return_file/2, return_error/2]). +-export([info/1, format/3, return_file/2, return_error/2]). -export([metrics/3, events/3]). -export([throw/3]). -include_lib("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). %% @doc Export an explicit list of files via http. -info() -> +info(Opts) -> + ServedRoutes = hb_maps:get(hyperbuddy_serve, Opts, #{}, Opts), #{ default => fun serve/4, - routes => #{ + serve => ServedRoutes#{ % Default message viewer page: <<"index">> => <<"index.html">>, <<"bundle.js">> => <<"bundle.js">>, @@ -141,20 +142,18 @@ throw(_Msg, _Req, Opts) -> end. %% @doc Serve a file from the priv directory. Only serves files that are explicitly -%% listed in the `routes' field of the `info/0' return value. +%% listed in the `routes' field of the `info/1' return value. serve(<<"keys">>, M1, _M2, Opts) -> dev_message:keys(M1, Opts); serve(<<"set">>, M1, M2, Opts) -> dev_message:set(M1, M2, Opts); serve(Key, _, _, Opts) -> ?event({hyperbuddy_serving, Key}), - Routes = hb_maps:get(routes, info(), no_routes, Opts), - case hb_maps:get(Key, Routes, undefined, Opts) of - undefined -> {error, not_found}; - Filename -> return_file(Filename) + ServeRoutes = hb_maps:get(serve, info(Opts), #{}, Opts), + case hb_maps:find(Key, ServeRoutes, Opts) of + {ok, Filename} -> return_file(<<"hyperbuddy@1.0">>, Filename, #{}); + error -> {error, not_found} end. %% @doc Read a file from disk and serve it as a static HTML page. -return_file(Name) -> - return_file(<<"hyperbuddy@1.0">>, Name, #{}). return_file(Device, Name) -> return_file(Device, Name, #{}). return_file(Device, Name, Template) -> @@ -173,7 +172,9 @@ return_file(Device, Name, Template) -> <<".css">> -> <<"text/css">>; <<".png">> -> <<"image/png">>; <<".ico">> -> <<"image/x-icon">>; - <<".ttf">> -> <<"font/ttf">> + <<".ttf">> -> <<"font/ttf">>; + <<".json">> -> <<"application/json">>; + _ -> <<"text/plain">> end } }; @@ -197,6 +198,7 @@ apply_template(Body, Template) when is_map(Template) -> apply_template(Body, []) -> Body; apply_template(Body, [{Key, Value} | Rest]) -> + ?event(debug_apply_template, {key, Key, value, Value}), apply_template( re:replace( Body, @@ -221,4 +223,25 @@ return_templated_file_test() -> ?assertNotEqual( binary:match(Body, <<"This is an error message.">>), nomatch - ). \ No newline at end of file + ). + +return_custom_json_test() -> + ?assertMatch( + {ok, + #{ + <<"body">> := JSONBin, + <<"content-type">> := <<"application/json">> + } + } when byte_size(JSONBin) > 0, + hb_ao:resolve( + #{ + <<"device">> => <<"hyperbuddy@1.0">> + }, + <<"custom.json">>, + #{ + hyperbuddy_serve => #{ + <<"custom.json">> => <<"test.json">> + } + } + ) + ). From 8151e1274ae29d399152dbcbc89349a4c42ccfe5 Mon Sep 17 00:00:00 2001 From: Sam Williams Date: Tue, 17 Mar 2026 20:02:12 -0400 Subject: [PATCH 128/135] chore: add small comment on usage --- src/dev_hyperbuddy.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dev_hyperbuddy.erl b/src/dev_hyperbuddy.erl index 5ccc8a609..0f6303412 100644 --- a/src/dev_hyperbuddy.erl +++ b/src/dev_hyperbuddy.erl @@ -6,7 +6,10 @@ -include_lib("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% @doc Export an explicit list of files via http. +%% @doc Export an explicit list of files via http. Filenames added to the +%% `hyperbuddy_serve' key of the node message will be served as static files. +%% Each filename must point to a path relative to the HyperBEAM instance's +%% build subdirectory as follows: `priv/html/hyperbuddy@1.0'. info(Opts) -> ServedRoutes = hb_maps:get(hyperbuddy_serve, Opts, #{}, Opts), #{ From 23805b26e6d6709cbf4f3b4606efec018f59d397 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Tue, 17 Mar 2026 22:02:08 -0400 Subject: [PATCH 129/135] chore: move `dev_codec_tx:to(List)` into `dev_bundler_task` The current logic to convert a list of items into a TX bundle is not completely HyperBEAM compliant. To avoid confusion we've moved the logic out of the core dev_codec_tx module. --- src/dev_bundler.erl | 7 ++- src/dev_bundler_recovery.erl | 2 +- src/dev_bundler_task.erl | 104 +++++++++++++++++++++++++++++++++-- src/dev_codec_tx.erl | 94 +++++++------------------------ 4 files changed, 125 insertions(+), 82 deletions(-) diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index a80784ab0..20b3aa7f0 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -803,14 +803,15 @@ recover_bundles_test() -> ok = dev_bundler_cache:write_item(Item1, Opts), ok = dev_bundler_cache:write_item(Item2, Opts), ok = dev_bundler_cache:write_item(Item3, Opts), - {ok, TX} = dev_codec_tx:to( - lists:reverse([Item1, Item2, Item3]), #{}, #{}), + TX = dev_bundler_task:data_items_to_tx( + lists:reverse([Item1, Item2, Item3]), Opts), CommittedTX = hb_message:convert( TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), ok = dev_bundler_cache:write_tx(CommittedTX, [Item1, Item2, Item3], Opts), Item4 = new_structured_data_item(4, 10, Opts), ok = dev_bundler_cache:write_item(Item4, Opts), - {ok, TX2} = dev_codec_tx:to(lists:reverse([Item4]), #{}, #{}), + TX2 = dev_bundler_task:data_items_to_tx( + lists:reverse([Item4]), Opts), CommittedTX2 = hb_message:convert( TX2, <<"structured@1.0">>, <<"tx@1.0">>, Opts), ok = dev_bundler_cache:write_tx(CommittedTX2, [Item4], Opts), diff --git a/src/dev_bundler_recovery.erl b/src/dev_bundler_recovery.erl index 1c619b79c..915b734bd 100644 --- a/src/dev_bundler_recovery.erl +++ b/src/dev_bundler_recovery.erl @@ -274,5 +274,5 @@ new_data_item(Index, Size, Opts) -> hb_message:convert(Item, <<"structured@1.0">>, <<"ans104@1.0">>, Opts). new_bundle_tx(Items, Opts) -> - {ok, TX} = dev_codec_tx:to(lists:reverse(Items), #{}, #{}), + TX = dev_bundler_task:data_items_to_tx(lists:reverse(Items), Opts), hb_message:convert(TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts). diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl index 6372dca1f..93ee6e39b 100644 --- a/src/dev_bundler_task.erl +++ b/src/dev_bundler_task.erl @@ -4,6 +4,8 @@ %%% - post_proof: Seeding teh chunks to the Arweave network -module(dev_bundler_task). -export([worker_loop/0, log_task/3, format_timestamp/0]). +%%% Test-only exports. +-export([data_items_to_tx/2]). -include("include/hb.hrl"). -include("include/dev_bundler.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -150,22 +152,41 @@ execute_task(#task{type = post_proof, data = Proof, opts = Opts} = Task) -> %% @doc Build and sign a bundle TX without posting it. build_signed_tx(Items, Opts) -> - {ok, TX} = dev_codec_tx:to(lists:reverse(Items), #{}, #{}), + TX = data_items_to_tx(Items, Opts), DataSize = TX#tx.data_size, PriceResult = get_price(DataSize, Opts), AnchorResult = get_anchor(Opts), case {PriceResult, AnchorResult} of {{ok, Price}, {ok, Anchor}} -> Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), - SignedTX = ar_tx:sign( - TX#tx{anchor = Anchor, reward = Price}, - Wallet - ), + SignedTX = + dev_arweave_common:normalize( + ar_tx:sign( + TX#tx{anchor = Anchor, reward = Price}, + Wallet + ) + ), {ok, SignedTX}; {PriceErr, AnchorErr} -> {error, {PriceErr, AnchorErr}} end. +data_items_to_tx(Items, Opts) -> + List = lists:map( + fun(Item) -> + hb_message:convert( + Item, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + <<"structured@1.0">>, + Opts + ) + end, + lists:reverse(Items)), + dev_arweave_common:normalize(#tx{ + format = 2, + data = List + }). + get_price(DataSize, Opts) -> hb_ao:resolve( #{ <<"device">> => <<"arweave@2.9">> }, @@ -281,4 +302,77 @@ build_signed_tx_test() -> ok after hb_mock_server:stop(ServerHandle) + end. + +build_signed_tx_on_arbundles_js_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = dev_bundler:start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + TestOpts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store() + }, + try + % Load an arweave.js-created dataitem + Item = ar_bundles:deserialize( + hb_util:ok( + file:read_file(<<"test/arbundles.js/ans104-item.bundle">>) + ) + ), + ?event(debug_test, {item, Item}), + ?assert(ar_bundles:verify_item(Item)), + % Load an arweave.js-created list bundle + {ok, Bin} = file:read_file(<<"test/arbundles.js/ans104-list-bundle.bundle">>), + BundledItem = ar_bundles:sign_item(#tx{ + format = ans104, + data = Bin, + data_size = byte_size(Bin), + tags = [ + {<<"Bundle-Format">>, <<"binary">>}, + {<<"Bundle-Version">>, <<"2.0.0">>} + ] + }, hb:wallet()), + ?event(debug_test, {bundled_item, BundledItem}), + ?assert(ar_bundles:verify_item(BundledItem)), + % Convert both dataitems to structured messages + ItemStructured = hb_message:convert(Item, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + TestOpts), + ?event(debug_test, {item_structured, ItemStructured}), + ?assert(hb_message:verify(ItemStructured, all, TestOpts)), + BundledItemStructured = hb_message:convert(BundledItem, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + TestOpts), + ?event(debug_test, {bundled_item_structured, BundledItemStructured}), + ?assert(hb_message:verify(BundledItemStructured, all, TestOpts)), + % Use build_signed_tx/2 to mimic the bundler worker logic. + {ok, SignedTX} = build_signed_tx( + [ItemStructured, BundledItemStructured], + TestOpts + ), + ?event(debug_test, {signed_tx, SignedTX}), + ?assert(ar_tx:verify(SignedTX)), + % Convert the signed TX to a structured message + StructuredTX = hb_message:convert(SignedTX, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, + TestOpts), + % ?event(debug_test, {structured_tx, StructuredTX}), + ?assert(hb_message:verify(StructuredTX, all, TestOpts)), + % Convert back to an L1 TX + SignedTXRoundtrip = hb_message:convert(StructuredTX, + #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + TestOpts), + ?event(debug_test, {signed_tx_roundtrip, SignedTXRoundtrip}), + ?assert(ar_tx:verify(SignedTXRoundtrip)), + ?assertEqual(SignedTX, SignedTXRoundtrip), + ok + after + hb_mock_server:stop(ServerHandle) end. \ No newline at end of file diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index 69d80f954..9ac8750a4 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -147,25 +147,6 @@ to(RawTABM, Req, Opts) when is_map(RawTABM) -> enforce_valid_tx(FinalTX), ?event({to_result, FinalTX}), {ok, FinalTX}; -%% @doc List of ans104 items is bundled into a single L1 transaction. -to(RawList, Req, Opts) when is_list(RawList) -> - List = lists:map( - fun(Item) -> - hb_message:convert( - Item, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - Opts - ) - end, - RawList), - TX = #tx{ - format = 2, - data = List - }, - Bundle = dev_arweave_common:normalize(TX), - ?event({to_result, Bundle}), - {ok, Bundle}; to(Other, _Req, _Opts) -> throw({invalid_tx, Other}). @@ -1528,61 +1509,28 @@ test_bundle_uncommitted(Encode, Decode) -> end, ok. -bundle_list_test() -> - % Load an arweave.js-created dataitem - Item = ar_bundles:deserialize( - hb_util:ok( - file:read_file(<<"test/arbundles.js/ans104-item.bundle">>) - ) - ), - ?event(debug_test, {item, Item}), - ?assert(ar_bundles:verify_item(Item)), - % Load an arweave.js-created list bundle - {ok, Bin} = file:read_file(<<"test/arbundles.js/ans104-list-bundle.bundle">>), - BundledItem = ar_bundles:sign_item(#tx{ - format = ans104, - data = Bin, - data_size = byte_size(Bin), - tags = [ - {<<"Bundle-Format">>, <<"binary">>}, - {<<"Bundle-Version">>, <<"2.0.0">>} - ] - }, hb:wallet()), - ?event(debug_test, {bundled_item, BundledItem}), - ?assert(ar_bundles:verify_item(BundledItem)), - % Convert both dataitems to structured messages - ItemStructured = hb_message:convert(Item, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, - #{}), - ?event(debug_test, {item_structured, ItemStructured}), - ?assert(hb_message:verify(ItemStructured, all, #{})), - BundledItemStructured = hb_message:convert(BundledItem, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, - #{}), - ?event(debug_test, {bundled_item_structured, BundledItemStructured}), - ?assert(hb_message:verify(BundledItemStructured, all, #{})), - % Use dev_codec_tx:to(List) to create a L1 TX bundle. We use this - % interface to mimic the logic used in dev_bundler - {ok, BundledTX} = dev_codec_tx:to( - [ItemStructured, BundledItemStructured], #{}, #{}), - SignedTX = ar_tx:sign(BundledTX, hb:wallet()), - ?event(debug_test, {signed_tx, SignedTX}), - ?assert(ar_tx:verify(SignedTX)), - % Convert the signed TX to a structured message - StructuredTX = hb_message:convert(SignedTX, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, +%% Disabled test that captures an issue we will face if we want to support +%% ao-types on tx's. +list_aotypes_test_disabled() -> + Items = [ + #{ <<"tag1">> => <<"value1">> }, + #{ <<"tag2">> => <<"value2">> }, + #{ <<"tag3">> => <<"value3">> } + ], + TX = hb_message:convert( + Items, #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, + <<"structured@1.0">>, #{}), - ?event(debug_test, {structured_tx, StructuredTX}), - ?assert(hb_message:verify(StructuredTX, all, #{})), - % Convert back to an L1 TX - SignedTXRoundtrip = hb_message:convert(StructuredTX, - #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + Anchor = crypto:strong_rand_bytes(32), + % SignedTX = ar_tx:sign( + % TX#tx{ anchor = Anchor, reward = 100 }, + % hb:wallet()), + % ?event(debug_test, {signed_tx, SignedTX}), + StructuredTX = hb_message:convert( + TX#tx{ anchor = Anchor, reward = 100 }, + <<"structured@1.0">>, + <<"tx@1.0">>, #{}), - ?event(debug_test, {signed_tx_roundtrip, SignedTXRoundtrip}), - ?assert(ar_tx:verify(SignedTXRoundtrip)), - ?assertEqual(SignedTX, SignedTXRoundtrip), + ?event(debug_test, {structured_tx, StructuredTX}), ok. \ No newline at end of file From 64676b55ce37d6c3dbea10f3d985534106106688 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 18 Mar 2026 01:29:22 +0000 Subject: [PATCH 130/135] impr: Blacklist handling of long blacklist file --- src/dev_blacklist.erl | 52 +++++++++++++++++++++++++++++++------------ src/dev_hook.erl | 2 ++ 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 6ee859a00..456239f0b 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -24,12 +24,20 @@ %%% The default frequency at which the blacklist cache is refreshed in seconds. -define(DEFAULT_REFRESH_FREQUENCY, 60 * 5). +-define(DEFAULT_WHITELIST, + [<<"/~hyperbuddy@1.0/metrics">>, + <<"/~hyperbuddy@1.0/styles.css">>, + <<"/~hyperbuddy@1.0/fonts.css">>, + <<"/~hyperbuddy@1.0/script.js">>, + <<"/~hyperbuddy@1.0/bundle.js">>]). %% @doc Hook handler: block requests that involve blacklisted IDs. request(_Base, HookReq, Opts) -> ?event({hook_req, HookReq}), case hb_opts:get(blacklist_providers, false, Opts) of - false -> {ok, HookReq}; + false -> + ?event(error, {no_providers}), + {ok, HookReq}; _ -> case is_match(HookReq, Opts) of false -> @@ -58,12 +66,21 @@ request(_Base, HookReq, Opts) -> %% @doc Check if the message contains any blacklisted IDs. is_match(Msg, Opts) -> - ensure_cache_table(Opts), - IDs = collect_ids(Msg, Opts), - MatchesFromIDs = fun(ID) -> ets:lookup(cache_table_name(Opts), ID) =/= [] end, - case lists:filter(MatchesFromIDs, IDs) of - [] -> false; - [ID|_] -> ID + WhitelistRoutes = hb_opts:get(blacklist_whitelist, ?DEFAULT_WHITELIST, Opts), + Path = hb_maps:get(<<"path">>, maps:get(<<"request">>, Msg, #{}), no_path), + ?event({is_match, {route, Path}}), + case lists:member(Path, WhitelistRoutes) of + false -> + ?event(error, {whitelist_route_no_match, {route, maps:get(<<"path">>, Msg, no_path)}}), + ensure_cache_table(Opts), + IDs = collect_ids(Msg, Opts), + MatchesFromIDs = fun(ID) -> ets:lookup(cache_table_name(Opts), ID) =/= [] end, + case lists:filter(MatchesFromIDs, IDs) of + [] -> false; + [ID|_] -> ID + end; + true -> + false end. %%% Internal @@ -106,10 +123,10 @@ fetch_single_provider(Provider, Opts) -> case execute_provider(Provider, Opts) of {ok, Blacklist} -> {ok, IDs} = parse_blacklist(Blacklist, Opts), - ?event({parsed_blacklist, {ids, IDs}}), + ?event({parsed_blacklist, {ids_lengh, length(IDs)}}), BlacklistID = hb_message:id(Blacklist, all, Opts), ?event({update_blacklist_cache, - {ids, IDs}, {blacklist_id, BlacklistID}}), + {ids_lengh, length(IDs)}, {blacklist_id, BlacklistID}}), Table = cache_table_name(Opts), {ok, insert_ids(IDs, BlacklistID, Table, Opts)}; {error, _} = Error -> @@ -195,6 +212,10 @@ insert_ids([ID | IDs], Value, Table, Opts) when ?IS_ID(ID) -> %% @doc Ensure the cache table exists. ensure_cache_table(Opts) -> + %% Options: + %% - continue: Don't wait for blacklist to be initialized + %% - halt: Close connection if not initilalized + FallbackMode = hb_opts:get(blacklist_fallback, halt, Opts), TableName = cache_table_name(Opts), case is_initialized(TableName) of true -> TableName; @@ -218,11 +239,14 @@ ensure_cache_table(Opts) -> refresh_loop(Opts) end ), - hb_util:until( - fun() -> is_initialized(TableName) end, - 10 - ), - TableName + case FallbackMode of + continue -> TableName; + halt -> + case is_initialized(TableName) of + true -> TableName; + false -> throw({error, #{<<"status">> => 503, <<"body">> => <<"Loading blacklist ...">>}}) + end + end end. %% @doc Check if the cache table is initialized. We do this by checking that the diff --git a/src/dev_hook.erl b/src/dev_hook.erl index 1696745ec..ef053d5d4 100644 --- a/src/dev_hook.erl +++ b/src/dev_hook.erl @@ -218,6 +218,8 @@ execute_handler(HookName, Handler, Req, Opts) -> _ -> {Status, Res} end catch + _:{error, #{<<"status">> := _} = Msg}:_ -> + {error, Msg}; Error:Reason:Stacktrace -> % If an exception occurs during execution, log it and return an error. ?event(hook_error, From 52bf0da944f5a3390e056e63b0aa40af32394e43 Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 18 Mar 2026 13:21:54 +0000 Subject: [PATCH 131/135] impr: Blacklist performance --- src/dev_blacklist.erl | 97 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 9 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 456239f0b..bc5e4d96c 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -16,6 +16,15 @@ %%% the Arweave network: No central enforcement, but each node is capable of %%% enforcing its own content policies based on its own free choice and %%% configuration. +%%% +%%% Configuration options: +%%% - blacklist_providers: List of providers to load in AO format. +%%% - blacklist_fallback: halt or continue. +%%% - Halt waits for X milliseconds before sending 503. +%%% - Continue allow the connection to fetch while blacklist is being loaded. +%%% - blacklist_timeout: How long should the request wait for the blacklist to be +%%% loaded. +%%% - blacklist_whitelist: List of endpoint path that are always whitelisted. -module(dev_blacklist). -export([request/3]). @@ -24,6 +33,9 @@ %%% The default frequency at which the blacklist cache is refreshed in seconds. -define(DEFAULT_REFRESH_FREQUENCY, 60 * 5). +-define(DEFAULT_REQUEST_TIMEOUT, 1000). +%% Fallback mode ptions: halt or continue +-define(DEFAULT_FALLBACK_MODE, halt). -define(DEFAULT_WHITELIST, [<<"/~hyperbuddy@1.0/metrics">>, <<"/~hyperbuddy@1.0/styles.css">>, @@ -42,6 +54,9 @@ request(_Base, HookReq, Opts) -> case is_match(HookReq, Opts) of false -> ?event(blacklist, {allowed, HookReq}, Opts), + %% Still trigger the download of the blacklist + try ensure_cache_table(Opts) + catch _:_ -> ok end, {ok, HookReq}; ID -> ?event(blacklist, {blocked, ID}, Opts), @@ -68,10 +83,9 @@ request(_Base, HookReq, Opts) -> is_match(Msg, Opts) -> WhitelistRoutes = hb_opts:get(blacklist_whitelist, ?DEFAULT_WHITELIST, Opts), Path = hb_maps:get(<<"path">>, maps:get(<<"request">>, Msg, #{}), no_path), - ?event({is_match, {route, Path}}), case lists:member(Path, WhitelistRoutes) of false -> - ?event(error, {whitelist_route_no_match, {route, maps:get(<<"path">>, Msg, no_path)}}), + ?event({path_do_not_match_whitelist, {path, Path}}), ensure_cache_table(Opts), IDs = collect_ids(Msg, Opts), MatchesFromIDs = fun(ID) -> ets:lookup(cache_table_name(Opts), ID) =/= [] end, @@ -80,6 +94,7 @@ is_match(Msg, Opts) -> [ID|_] -> ID end; true -> + ?event({path_match_whitelist, {path, Path}}), false end. @@ -165,14 +180,29 @@ parse_blacklist(Body, _Opts) when is_binary(Body) -> %% @doc Parse a single line of the blacklist body, returning the ID if it is valid, %% and `false' otherwise. parse_blacklist_line(Line) -> - Trimmed = string:trim(Line, both), - case Trimmed of + case trim_ascii(Line) of <<>> -> false; <<"#", _/binary>> -> false; ID when ?IS_ID(ID) -> {true, hb_util:human_id(ID)}; _ -> false end. +%% @doc Fast ASCII-only whitespace trim (strips \r, \n, \s, \t). +%% Avoids Unicode machinery of string:trim/2 for performance. +trim_ascii(<>) when C =:= $\s; C =:= $\t; C =:= $\r; C =:= $\n -> + trim_ascii(Rest); +trim_ascii(Bin) -> + trim_ascii_right(Bin, byte_size(Bin)). + +trim_ascii_right(_, 0) -> <<>>; +trim_ascii_right(Bin, Len) -> + case binary:at(Bin, Len - 1) of + C when C =:= $\s; C =:= $\t; C =:= $\r; C =:= $\n -> + trim_ascii_right(Bin, Len - 1); + _ -> + binary:part(Bin, 0, Len) + end. + %% @doc Collect all IDs found as elements of a given message. collect_ids(Msg, Opts) -> lists:usort(collect_ids(Msg, [], Opts)). collect_ids(Bin, Acc, _Opts) when ?IS_ID(Bin) -> [hb_util:human_id(Bin) | Acc]; @@ -214,8 +244,9 @@ insert_ids([ID | IDs], Value, Table, Opts) when ?IS_ID(ID) -> ensure_cache_table(Opts) -> %% Options: %% - continue: Don't wait for blacklist to be initialized - %% - halt: Close connection if not initilalized - FallbackMode = hb_opts:get(blacklist_fallback, halt, Opts), + %% - halt: Close connection with HTTP 503 if not initilalized + FallbackMode = hb_opts:get(blacklist_fallback, ?DEFAULT_FALLBACK_MODE, Opts), + RequestTimeout = hb_opts:get(blacklist_timeout, ?DEFAULT_REQUEST_TIMEOUT, Opts), TableName = cache_table_name(Opts), case is_initialized(TableName) of true -> TableName; @@ -242,9 +273,20 @@ ensure_cache_table(Opts) -> case FallbackMode of continue -> TableName; halt -> - case is_initialized(TableName) of + IsInitialized = + hb_util:wait_until( + fun() -> is_initialized(TableName) end, + RequestTimeout + ), + case IsInitialized of true -> TableName; - false -> throw({error, #{<<"status">> => 503, <<"body">> => <<"Loading blacklist ...">>}}) + false -> + throw({error, + #{ + <<"status">> => 503, + <<"body">> => <<"Loading blacklist ...">> + } + }) end end end. @@ -353,6 +395,19 @@ basic_test() -> ), ok. +%% @doc Ensure that the default provider does not block any requests. +first_request_always_return_503_test() -> + {ok, #{ + opts := Opts0, + unsigned3 := UnsignedID3 + }} = setup_test_env(), + Opts1 = Opts0#{ blacklist_providers => [] }, + Node = hb_http_server:start_node(Opts1#{blacklist_timeout => 0}), + ?assertMatch( + {failure, #{<<"status">> := 503, <<"body">> := <<"Loading blacklist ...">>}}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ). + %% @doc Ensure that the default provider does not block any requests. default_provider_test() -> {ok, #{ @@ -528,4 +583,28 @@ refresh_periodically_test() -> {error, #{ <<"status">> := 451 }}, hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) ), - ok. \ No newline at end of file + ok. + +%% @doc Test that parse_blacklist/2 can handle 1 million IDs within 2000ms. +parse_blacklist_performance_test() -> + GenID = fun() -> + B64 = base64:encode(crypto:strong_rand_bytes(32)), + %% base64:encode of 32 bytes = 44 chars (with 1 '=' padding). + %% Taking the first 43 chars gives a valid 43-byte binary ID. + binary:part(B64, 0, 43) + end, + IDs = [GenID() || _ <- lists:seq(1, 1000000)], + Body = iolist_to_binary(lists:join(<<"\n">>, IDs)), + Start = erlang:monotonic_time(millisecond), + {ok, Parsed} = parse_blacklist(Body, #{}), + Duration = erlang:monotonic_time(millisecond) - Start, + ?assert(length(Parsed) =:= 1000000), + ?assert(Duration =< 2000). + +wait_for_blacklist_to_load(Node) -> + hb_util:wait_until(fun() -> + case hb_http:get(Node, <<"/any_path">>, #{}) of + {error, #{ <<"status">> := 503 }} -> false; + _ -> true + end + end, 5000). From 6792fa1bc0228597b23f2eeb6b256cefd1bda2ba Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 18 Mar 2026 14:32:01 +0000 Subject: [PATCH 132/135] impr: Blacklist hook req handling --- src/dev_blacklist.erl | 58 ++++++++++++++++++------------------------- src/dev_hook.erl | 2 -- 2 files changed, 24 insertions(+), 36 deletions(-) diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index bc5e4d96c..b9a95bf56 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -48,17 +48,11 @@ request(_Base, HookReq, Opts) -> ?event({hook_req, HookReq}), case hb_opts:get(blacklist_providers, false, Opts) of false -> - ?event(error, {no_providers}), + ?event({no_providers}), {ok, HookReq}; _ -> case is_match(HookReq, Opts) of - false -> - ?event(blacklist, {allowed, HookReq}, Opts), - %% Still trigger the download of the blacklist - try ensure_cache_table(Opts) - catch _:_ -> ok end, - {ok, HookReq}; - ID -> + {blocked_txid, ID} -> ?event(blacklist, {blocked, ID}, Opts), { ok, @@ -75,7 +69,9 @@ request(_Base, HookReq, Opts) -> >> }] } - } + }; + Response -> + Response end end. @@ -86,16 +82,20 @@ is_match(Msg, Opts) -> case lists:member(Path, WhitelistRoutes) of false -> ?event({path_do_not_match_whitelist, {path, Path}}), - ensure_cache_table(Opts), - IDs = collect_ids(Msg, Opts), - MatchesFromIDs = fun(ID) -> ets:lookup(cache_table_name(Opts), ID) =/= [] end, - case lists:filter(MatchesFromIDs, IDs) of - [] -> false; - [ID|_] -> ID + case ensure_cache_table(Msg, Opts) of + {ok, Msg1} -> + IDs = collect_ids(Msg1, Opts), + MatchesFromIDs = fun(ID) -> ets:lookup(cache_table_name(Opts), ID) =/= [] end, + case lists:filter(MatchesFromIDs, IDs) of + [] -> {ok, Msg1}; + [ID|_] -> {blocked_txid, ID} + end; + {error, Msg1} -> + {error, Msg1} end; true -> ?event({path_match_whitelist, {path, Path}}), - false + {ok, Msg} end. %%% Internal @@ -241,7 +241,7 @@ insert_ids([ID | IDs], Value, Table, Opts) when ?IS_ID(ID) -> end. %% @doc Ensure the cache table exists. -ensure_cache_table(Opts) -> +ensure_cache_table(Msg, Opts) -> %% Options: %% - continue: Don't wait for blacklist to be initialized %% - halt: Close connection with HTTP 503 if not initilalized @@ -249,7 +249,7 @@ ensure_cache_table(Opts) -> RequestTimeout = hb_opts:get(blacklist_timeout, ?DEFAULT_REQUEST_TIMEOUT, Opts), TableName = cache_table_name(Opts), case is_initialized(TableName) of - true -> TableName; + true -> {ok, Msg}; false -> hb_name:singleton( TableName, @@ -271,7 +271,7 @@ ensure_cache_table(Opts) -> end ), case FallbackMode of - continue -> TableName; + continue -> {ok, Msg}; halt -> IsInitialized = hb_util:wait_until( @@ -279,14 +279,12 @@ ensure_cache_table(Opts) -> RequestTimeout ), case IsInitialized of - true -> TableName; + true -> {ok, Msg}; false -> - throw({error, - #{ - <<"status">> => 503, - <<"body">> => <<"Loading blacklist ...">> - } - }) + {error, Msg#{ + <<"status">> => 503, + <<"body">> => <<"Loading blacklist ...">> + }} end end end. @@ -600,11 +598,3 @@ parse_blacklist_performance_test() -> Duration = erlang:monotonic_time(millisecond) - Start, ?assert(length(Parsed) =:= 1000000), ?assert(Duration =< 2000). - -wait_for_blacklist_to_load(Node) -> - hb_util:wait_until(fun() -> - case hb_http:get(Node, <<"/any_path">>, #{}) of - {error, #{ <<"status">> := 503 }} -> false; - _ -> true - end - end, 5000). diff --git a/src/dev_hook.erl b/src/dev_hook.erl index ef053d5d4..1696745ec 100644 --- a/src/dev_hook.erl +++ b/src/dev_hook.erl @@ -218,8 +218,6 @@ execute_handler(HookName, Handler, Req, Opts) -> _ -> {Status, Res} end catch - _:{error, #{<<"status">> := _} = Msg}:_ -> - {error, Msg}; Error:Reason:Stacktrace -> % If an exception occurs during execution, log it and return an error. ?event(hook_error, From 329122eb5735e575e009cf073af1c0a046f158aa Mon Sep 17 00:00:00 2001 From: speeddragon Date: Wed, 18 Mar 2026 19:42:46 +0000 Subject: [PATCH 133/135] Revert "impr: Remove erlang_vm metrics" This reverts commit 0df0cd29173261bc32c649793a7f251428340081. --- config/app.config | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/config/app.config b/config/app.config index 4d5fec6c3..6ef0c7446 100644 --- a/config/app.config +++ b/config/app.config @@ -2,14 +2,6 @@ {prometheus, [ {cowboy_instrumenter, [ {duration_buckets, [0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 1, 2, 4, 10, 30, 60]} - ]}, - {collectors, [ - prometheus_boolean, - prometheus_counter, - prometheus_gauge, - prometheus_histogram, - prometheus_mnesia_collector, - prometheus_quantile_summary, - prometheus_summary]} + ]} ]} -]. \ No newline at end of file +]. From ed3c1b728ff2fd8c531e00c55b03331c0f456e44 Mon Sep 17 00:00:00 2001 From: James Piechota Date: Sat, 14 Mar 2026 14:44:43 -0400 Subject: [PATCH 134/135] impr: add hb_process_sampler to track attributes of all running processes Sampler runs periodically and gather message queue length, reductions, and memory use by process and adds to promethues. Arweave uses a similar system and it's been incredibly useful for debugging and we haven't noticed any negative performance impacts. controlled via the following opts: process_sampler: true|false process_sampler_interval: how often to sampe, default 15000ms --- src/hb_http_server.erl | 2 + src/hb_opts.erl | 2 + src/hb_process_sampler.erl | 284 +++++++++++++++++++++++++++++++++++++ src/hb_prometheus.erl | 6 +- 4 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 src/hb_process_sampler.erl diff --git a/src/hb_http_server.erl b/src/hb_http_server.erl index 3b1f7cfc3..8d2935eab 100644 --- a/src/hb_http_server.erl +++ b/src/hb_http_server.erl @@ -78,6 +78,7 @@ start(Opts) -> ]), hb:init(), BaseOpts = set_default_opts(Opts), + ok = hb_process_sampler:ensure_started(BaseOpts), {ok, Listener, _Port} = new_server(BaseOpts), {ok, Listener}. @@ -572,6 +573,7 @@ start_node(Opts) -> hb:init(), hb_sup:start_link(Opts), ServerOpts = set_default_opts(Opts), + ok = hb_process_sampler:ensure_started(ServerOpts), {ok, _Listener, Port} = new_server(ServerOpts), <<"http://localhost:", (hb_util:bin(Port))/binary, "/">>. diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 181b37185..33a469dc6 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -243,6 +243,8 @@ default_message() -> http_client_keepalive => 120000, http_client_send_timeout => 300_000, port => 8734, + process_sampler => true, + process_sampler_interval => 15000, wasm_allow_aot => false, %% Options for the relay device relay_http_client => httpc, diff --git a/src/hb_process_sampler.erl b/src/hb_process_sampler.erl new file mode 100644 index 000000000..42a5f92be --- /dev/null +++ b/src/hb_process_sampler.erl @@ -0,0 +1,284 @@ +%%% @doc Sample BEAM process state for diagnostics. +-module(hb_process_sampler). + +-export([ensure_started/1]). + +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(DEFAULT_SAMPLE_PROCESSES_INTERVAL, 15000). + +%% @doc Ensure the process sampler singleton is started if enabled. +ensure_started(Opts) -> + ProcessSamplerEnabled = + hb_opts:get(process_sampler, not hb_features:test(), Opts) + andalso hb_opts:get(prometheus, not hb_features:test(), Opts), + ?event(process_sampler, {process_sampler_enabled, ProcessSamplerEnabled}), + case ProcessSamplerEnabled of + true -> + _ = hb_name:singleton(?MODULE, fun() -> start(Opts) end), + ok; + false -> + ok + end. + +%% @doc Initialize the process sampler and enter its receive loop. +start(Opts) -> + ?event(process_sampler, {starting_process_sampler, + {interval, hb_opts:get(process_sampler_interval, ?DEFAULT_SAMPLE_PROCESSES_INTERVAL, Opts)}}), + hb_prometheus:ensure_started(), + schedule_process_sample(Opts), + loop( + #{ + opts => Opts + } + ). + +%% @doc Receive loop for the process sampler. +loop(State = #{ opts := Opts }) -> + receive + sample_processes -> + sample_processes(State), + schedule_process_sample(Opts), + loop(State); + Message -> + ?event(warning, {unhandled_info, {module, ?MODULE}, {message, Message}}), + loop(State) + end. + +%% @doc Schedule the next process sample if enabled. +schedule_process_sample(Opts) -> + case hb_opts:get( + process_sampler_interval, + ?DEFAULT_SAMPLE_PROCESSES_INTERVAL, + Opts + ) of + Interval when is_integer(Interval) andalso Interval > 0 -> + erlang:send_after(Interval, self(), sample_processes); + _ -> + ok + end. + +%% @doc Sample all BEAM processes and report aggregate metrics. +sample_processes(#{ opts := Opts }) -> + StartTime = erlang:monotonic_time(), + try + Processes = erlang:processes(), + ProcessData = + lists:filtermap( + fun(PID) -> process_function(PID, Opts) end, + Processes + ), + ProcessMetrics = accumulate_process_metrics(ProcessData), + report_process_metrics(ProcessMetrics), + EndTime = erlang:monotonic_time(), + ElapsedTime = + erlang:convert_time_unit( + EndTime - StartTime, + native, + microsecond + ), + ?event( + process_sampler, + {sample_processes, + {processes, length(Processes)}, + {elapsed_ms, ElapsedTime / 1000} + }, + Opts + ) + catch + Class:Reason:Stacktrace -> + ?event( + warning, + {process_sampler_failed, + {class, Class}, + {reason, Reason}, + {stacktrace, {trace, Stacktrace}} + }, + Opts + ) + end. + +%% @doc Sum process memory, reductions, and mailbox sizes by process name. +accumulate_process_metrics(ProcessData) -> + lists:foldl( + fun({_Status, ProcessName, Memory, Reductions, MsgQueueLen}, Acc) -> + {MemoryTotal, ReductionsTotal, MsgQueueLenTotal} = + maps:get(ProcessName, Acc, {0, 0, 0}), + maps:put( + ProcessName, + { + MemoryTotal + Memory, + ReductionsTotal + Reductions, + MsgQueueLenTotal + MsgQueueLen + }, + Acc + ) + end, + #{}, + ProcessData + ). + +%% @doc Report aggregate process metrics to Prometheus. +report_process_metrics(ProcessMetrics) -> + reset_process_info_metric(), + maps:foreach( + fun(ProcessName, {Memory, Reductions, MsgQueueLen}) -> + prometheus_gauge:set(process_info, [ProcessName, <<"memory">>], Memory), + prometheus_gauge:set( + process_info, + [ProcessName, <<"reductions">>], + Reductions + ), + prometheus_gauge:set( + process_info, + [ProcessName, <<"message_queue">>], + MsgQueueLen + ) + end, + ProcessMetrics + ), + report_memory_metrics(). + +%% @doc Recreate the per-process metric family to clear exited-process labels. +reset_process_info_metric() -> + _ = prometheus_gauge:deregister(process_info), + ok = + prometheus_gauge:new( + [ + {name, process_info}, + {labels, [process, type]}, + {help, + "Sampling info about active processes." + " Only set when process_sampler is enabled."} + ] + ). + +%% @doc Report BEAM memory totals through the process_info metric family. +report_memory_metrics() -> + prometheus_gauge:set( + process_info, + [<<"total">>, <<"memory">>], + erlang:memory(total) + ), + prometheus_gauge:set( + process_info, + [<<"processes">>, <<"memory">>], + erlang:memory(processes) + ), + prometheus_gauge:set( + process_info, + [<<"processes_used">>, <<"memory">>], + erlang:memory(processes_used) + ), + prometheus_gauge:set( + process_info, + [<<"system">>, <<"memory">>], + erlang:memory(system) + ), + prometheus_gauge:set( + process_info, + [<<"atom">>, <<"memory">>], + erlang:memory(atom) + ), + prometheus_gauge:set( + process_info, + [<<"atom_used">>, <<"memory">>], + erlang:memory(atom_used) + ), + prometheus_gauge:set( + process_info, + [<<"binary">>, <<"memory">>], + erlang:memory(binary) + ), + prometheus_gauge:set( + process_info, + [<<"code">>, <<"memory">>], + erlang:memory(code) + ), + prometheus_gauge:set( + process_info, + [<<"ets">>, <<"memory">>], + erlang:memory(ets) + ). + +%% @doc Sample a single process and return aggregate data for it. +process_function(PID, _Opts) -> + case process_info( + PID, + [ + current_stacktrace, + registered_name, + status, + memory, + reductions, + message_queue_len + ] + ) of + [{current_stacktrace, Stack}, + {registered_name, Name}, + {status, Status}, + {memory, Memory}, + {reductions, Reductions}, + {message_queue_len, MsgQueueLen}] -> + ProcessName = process_name(Name, Stack), + {true, {Status, ProcessName, Memory, Reductions, MsgQueueLen}}; + _ -> + false + end. + +%% @doc Resolve a readable process name from its registration or stack. +process_name([], []) -> + <<"unknown">>; +process_name([], Stack) -> + InitialCall = initial_call(lists:reverse(Stack)), + M = element(1, InitialCall), + F = element(2, InitialCall), + A = element(3, InitialCall), + << + (hb_util:bin(M))/binary, + ":", + (hb_util:bin(F))/binary, + "/", + (hb_util:bin(A))/binary + >>; +process_name(Name, _Stack) -> + hb_util:bin(Name). + +%% @doc Determine the initial stack frame for an anonymous process. +initial_call([]) -> + {unknown, unknown, 0}; +initial_call([{proc_lib, init_p_do_apply, _A, _Location} | Stack]) -> + initial_call(Stack); +initial_call([InitialCall | _Stack]) -> + InitialCall. + +%%% Tests + +%% @doc Ensure anonymous process names fall back to their initial call. +process_name_from_stack_test() -> + ?assertEqual( + <<"hb_http_server:init/2">>, + process_name( + [], + [ + {hb_http_server, init, 2, []}, + {proc_lib, init_p_do_apply, 3, []} + ] + ) + ). + +%% @doc Ensure registered names are returned directly. +process_name_registered_test() -> + ?assertEqual(<<"my_proc">>, process_name(my_proc, [])). + +%% @doc Ensure aggregate process metrics are summed by process name. +accumulate_process_metrics_test() -> + Metrics = + accumulate_process_metrics( + [ + {running, <<"worker">>, 10, 20, 1}, + {running, <<"worker">>, 5, 3, 2} + ] + ), + ?assertEqual({15, 23, 3}, maps:get(<<"worker">>, Metrics)). diff --git a/src/hb_prometheus.erl b/src/hb_prometheus.erl index 9ff0be6bc..8b4c3e7a8 100644 --- a/src/hb_prometheus.erl +++ b/src/hb_prometheus.erl @@ -37,7 +37,11 @@ declare(Type, Metric) -> case ensure_started() of ok -> try do_declare(Type, Metric) - catch error:mfa_already_exists -> ok + catch + error:mfa_already_exists -> + ok; + error:{mf_already_exists, _, _} -> + ok end; _ -> ok end. From 25b024a08762f6612803255d28facf30ded10880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Magalh=C3=A3es?= Date: Wed, 18 Mar 2026 21:19:59 +0000 Subject: [PATCH 135/135] Update observer_cli dependency reference in rebar.config Fix ~ issues while printing --- rebar.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 6008157c9..12c454091 100644 --- a/rebar.config +++ b/rebar.config @@ -14,7 +14,7 @@ {no_events, [{erl_opts, [{d, 'NO_EVENTS', true}]}]}, {top, [ {deps, [{observer_cli, {git, "https://github.com/permaweb/observer_cli.git", - {ref, "7f90812dcd43d7954d0ba066cb94d5e934e729b5"}}}]}, + {ref, "7e7a2613b262e08c43c539d50901e6f26a241b6f"}}}]}, {erl_opts, [{d, 'AO_TOP', true}]}, {relx, [{release, {'hb', "0.0.1"}, [hb, b64fast, base32, cowboy, gun, luerl, prometheus, prometheus_cowboy, prometheus_ranch, elmdb, observer_cli]}]} ]}, @@ -211,4 +211,4 @@ {preprocess, true}, {private, true}, {hidden, true} -]}. \ No newline at end of file +]}.