diff --git a/config/app.config b/config/app.config index 4d5fec6c3..6ef0c7446 100644 --- a/config/app.config +++ b/config/app.config @@ -2,14 +2,6 @@ {prometheus, [ {cowboy_instrumenter, [ {duration_buckets, [0.001, 0.01, 0.1, 0.25, 0.5, 0.75, 1, 2, 4, 10, 30, 60]} - ]}, - {collectors, [ - prometheus_boolean, - prometheus_counter, - prometheus_gauge, - prometheus_histogram, - prometheus_mnesia_collector, - prometheus_quantile_summary, - prometheus_summary]} + ]} ]} -]. \ No newline at end of file +]. diff --git a/docs/misc/hacking-on-hyperbeam.md b/docs/misc/hacking-on-hyperbeam.md index 6df0f5a9d..ebb69cbcd 100644 --- a/docs/misc/hacking-on-hyperbeam.md +++ b/docs/misc/hacking-on-hyperbeam.md @@ -103,4 +103,29 @@ since the last invocation. 3. Open the svg file in browser. -Happy hacking! \ No newline at end of file +## Common testing pitfalls + +Here is a helpful list of common mistakes when writing tests: + +- If you need to start a new node, be sure to use a new private key unless you + have a specific reason to use an existing one. HyperBEAM HTTP servers are + registered using their wallet ID as their 'name', so re-use can cause issues. + You can get a new private key is defined using `#{ priv_wallet => ar_wallet:new() }`. +- Similarly, always be careful of your stores in your tests! Avoid using the + default stores, as this can lead to 'context leakage', where one part of your + test is unintentionally able to access data created/stored by a supposedly + different node in the environment. `hb_http_server:start_node/1` will generate + a new unique store for you by default, but avoid creating a named store unless + you need to (and know what you are doing). +- Always try to test your devices through the HTTP AO-Core API as well as through + the local `hb_ao:resolve/[2-3]` interfaces. Avoid direct `dev_name:key` calls + unless strictly necessary. The HTTP API is how users will interact with your + almost always system, and there can be subtle differences in how the interfaces + react. For example, the Erlang function call interface has no regard for how + keys are matched by AO-Core, so will mask any issues with the choice of which + device function to call to satisfy requests. + +Happy hacking! + +Avoid pattern match a list of commitments, since we cannot guarantee the order. +This will case tests to be flaky. diff --git a/rebar.config b/rebar.config index bd8a10487..12c454091 100644 --- a/rebar.config +++ b/rebar.config @@ -1,8 +1,8 @@ {erl_opts, [debug_info, {d, 'COWBOY_QUICER', 1}, {d, 'GUN_QUICER', 1}]}. -{plugins, [pc, rebar3_rustler, rebar_edown_plugin]}. +{plugins, [pc, rebar3_rustler, rebar_edown_plugin, {rebar3_eunit_start, {git, "https://github.com/permaweb/rebar3_eunit_start.git", {ref, "04ec53fea187039770db0d4459b7aeb01a9021af"}}}]}. % Increase `scale_timeouts` when running on a slower machine. -{eunit_opts, [verbose, {scale_timeouts, 10}]}. +{eunit_opts, [verbose, {scale_timeouts, 10}, {start_applications, [prometheus, hb]}]}. {profiles, [ {quiet, @@ -12,7 +12,12 @@ ] }, {no_events, [{erl_opts, [{d, 'NO_EVENTS', true}]}]}, - {top, [{deps, [observer_cli]}, {erl_opts, [{d, 'AO_TOP', true}]}]}, + {top, [ + {deps, [{observer_cli, {git, "https://github.com/permaweb/observer_cli.git", + {ref, "7e7a2613b262e08c43c539d50901e6f26a241b6f"}}}]}, + {erl_opts, [{d, 'AO_TOP', true}]}, + {relx, [{release, {'hb', "0.0.1"}, [hb, b64fast, base32, cowboy, gun, luerl, prometheus, prometheus_cowboy, prometheus_ranch, elmdb, observer_cli]}]} + ]}, {store_events, [{erl_opts, [{d, 'STORE_EVENTS', true}]}]}, {ao_profiling, [{erl_opts, [{d, 'AO_PROFILING', true}]}]}, {eflame, @@ -108,7 +113,8 @@ {provider_hooks, [ {pre, [ - {compile, {cargo, build}} + {compile, {cargo, build}}, + {eunit, {default, rebar3_eunit_start}} ]}, {post, [ {compile, {pc, compile}}, @@ -134,6 +140,7 @@ {deps, [ {elmdb, {git, "https://github.com/permaweb/elmdb-rs.git", {ref, "bfda2facebdb433eea753f82e7e8d45aefc6d87a"}}}, + {base32, "1.0.0"}, {b64fast, {git, "https://github.com/ArweaveTeam/b64fast.git", {ref, "58f0502e49bf73b29d95c6d02460d1fb8d2a5273"}}}, {cowlib, "2.16.0"}, {cowboy, "2.14.0"}, @@ -162,7 +169,7 @@ ]}. {relx, [ - {release, {'hb', "0.0.1"}, [hb, b64fast, cowboy, gun, luerl, prometheus, prometheus_cowboy, prometheus_ranch, elmdb]}, + {release, {'hb', "0.0.1"}, [hb, b64fast, base32, cowboy, gun, luerl, prometheus, prometheus_cowboy, prometheus_ranch, elmdb]}, {sys_config, "config/app.config"}, {include_erts, true}, {extended_start_script, true}, diff --git a/rebar.lock b/rebar.lock index 433624df1..9aaf8394a 100644 --- a/rebar.lock +++ b/rebar.lock @@ -4,6 +4,7 @@ {git,"https://github.com/ArweaveTeam/b64fast.git", {ref,"58f0502e49bf73b29d95c6d02460d1fb8d2a5273"}}, 0}, + {<<"base32">>,{pkg,<<"base32">>,<<"1.0.0">>},0}, {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.14.0">>},0}, {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.16.0">>},0}, {<<"ddskerl">>,{pkg,<<"ddskerl">>,<<"0.4.2">>},1}, @@ -30,6 +31,7 @@ [ {pkg_hash,[ {<<"accept">>, <<"CD6E34A2D7E28CA38B2D3CB233734CA0C221EFBC1F171F91FEC5F162CC2D18DA">>}, + {<<"base32">>, <<"1AB331F812FCC254C8F7D4348E1E5A6F2B9B32B7A260BF2BC3358E3BF14C841A">>}, {<<"cowboy">>, <<"565DCF221BA99B1255B0ADCEC24D2D8DBE79E46EC79F30F8373CCEADC6A41E2A">>}, {<<"cowlib">>, <<"54592074EBBBB92EE4746C8A8846E5605052F29309D3A873468D76CDF932076F">>}, {<<"ddskerl">>, <<"A51A90BE9AC9B36A94017670BED479C623B10CA9D4BDA1EDF3A0E48CAEEADA2A">>}, @@ -42,6 +44,7 @@ {<<"ranch">>, <<"25528F82BC8D7C6152C57666CA99EC716510FE0925CB188172F41CE93117B1B0">>}]}, {pkg_hash_ext,[ {<<"accept">>, <<"CA69388943F5DAD2E7232A5478F16086E3C872F48E32B88B378E1885A59F5649">>}, + {<<"base32">>, <<"0449285348ED0C4CD83C7198E76C5FD5A0451C4EF18695B9FD43792A503E551A">>}, {<<"cowboy">>, <<"EA99769574550FE8A83225C752E8A62780A586770EF408816B82B6FE6D46476B">>}, {<<"cowlib">>, <<"7F478D80D66B747344F0EA7708C187645CFCC08B11AA424632F78E25BF05DB51">>}, {<<"ddskerl">>, <<"63F907373D7E548151D584D4DA8A38928FD26EC9477B94C0FFAAD87D7CB69FE7">>}, diff --git a/src/dev_arweave.erl b/src/dev_arweave.erl index 87c87ac35..f632372a5 100644 --- a/src/dev_arweave.erl +++ b/src/dev_arweave.erl @@ -4,13 +4,24 @@ %%% The node(s) that are used to query data may be configured by altering the %%% `/arweave` route in the node's configuration message. -module(dev_arweave). +-export([info/0]). -export([tx/3, raw/3, chunk/3, block/3, current/3, status/3, price/3, tx_anchor/3]). --export([post_tx/3, post_tx/4, post_binary_ans104/2, post_json_chunk/2]). +-export([post_tx_header/2, post_tx/3, post_tx/4, post_binary_ans104/2, post_json_chunk/2]). +%%% Helper functions +-export([get_chunk/2, bundle_header/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(IS_BLOCK_ID(X), (is_binary(X) andalso byte_size(X) == 64)). +%% @doc Route unknown keys through offset resolution first, then fall back to +%% the message device for direct key access. +info() -> + #{ + excludes => [<<"keys">>, <<"set">>, <<"set-path">>, <<"remove">>], + default => fun dev_arweave_offset:get/4 + }. + %% @doc Proxy the `/info' endpoint from the Arweave node. status(_Base, _Request, Opts) -> request(<<"GET">>, <<"/info">>, Opts). @@ -65,25 +76,13 @@ extract_target(Base, Request, Opts) -> post_tx(_Base, Request, Opts, <<"tx@1.0">>) -> TX = hb_message:convert(Request, <<"tx@1.0">>, Opts), - JSON = ar_tx:tx_to_json_struct(TX#tx{ data = <<>> }), - Serialized = hb_json:encode(JSON), - LogExtra = [ - {codec, <<"tx@1.0">>}, - {id, {explicit, hb_util:human_id(TX#tx.id)}} - ], - Res = request( - <<"POST">>, - <<"/tx">>, - #{ <<"body">> => Serialized }, - LogExtra, - Opts - ), + Res = post_tx_header(TX, Opts), case Res of {ok, _} -> CacheRes = hb_cache:write(Request, Opts), case CacheRes of {ok, _} -> - ?event(arweave_debug, {tx_cached, {msg, Request}, {status, ok}}); + ?event(debug_arweave, {tx_cached, {msg, Request}, {status, ok}}); _ -> ?event(error, {tx_failed_to_cache, {msg, Request}, CacheRes}) end; @@ -101,6 +100,22 @@ post_tx(_Base, Request, Opts, <<"ans104@1.0">>) -> ], post_binary_ans104(Serialized, LogExtra, Opts). + +post_tx_header(TX, Opts) -> + JSON = ar_tx:tx_to_json_struct(TX#tx{ data = <<>> }), + Serialized = hb_json:encode(JSON), + LogExtra = [ + {codec, <<"tx@1.0">>}, + {id, {explicit, hb_util:human_id(TX#tx.id)}} + ], + request( + <<"POST">>, + <<"/tx">>, + #{ <<"body">> => Serialized }, + LogExtra, + Opts + ). + post_binary_ans104(SerializedTX, Opts) -> LogExtra = [ {codec, <<"ans104@1.0">>}, @@ -112,14 +127,11 @@ post_binary_ans104(SerializedTX, LogExtra, Opts) -> Res = hb_http:post( hb_opts:get(bundler_ans104, not_found, Opts), #{ - <<"path">> => <<"/tx">>, + <<"path">> => <<"/~bundler@1.0/tx&codec-device=ans104@1.0">>, <<"content-type">> => <<"application/octet-stream">>, <<"body">> => SerializedTX }, - Opts#{ - http_client => - hb_opts:get(bundler_ans104_http_client, httpc, Opts) - } + Opts ), to_message(<<"/tx">>, <<"POST">>, Res, LogExtra, Opts). @@ -210,8 +222,9 @@ head_raw_tx(TXID, StartOffset, Length, Opts) -> ), {ok, #{ - <<"arweave-id">> => TXID, - <<"arweave-data-offset">> => StartOffset, + <<"raw-id">> => TXID, + <<"offset">> => StartOffset, + <<"data-offset">> => StartOffset, <<"content-type">> => ContentType, <<"header-length">> => 0, <<"content-length">> => Length, @@ -223,11 +236,12 @@ head_raw_tx(TXID, StartOffset, Length, Opts) -> %% so to read the data associated with their IDs, we must first read the header %% chunk, deserialize it, and offset our data read from its starting offset. head_raw_ans104(TXID, ArweaveOffset, Length, Opts) -> + ?event(debug_raw, {head_raw_ans104, {txid, TXID}, {arweave_offset, ArweaveOffset}, {length, Length}}), HeaderReq = #{ <<"path">> => <<"chunk">>, <<"offset">> => ArweaveOffset + 1, - <<"length">> => ?DATA_CHUNK_SIZE + <<"length">> => min(Length, ?DATA_CHUNK_SIZE) }, case hb_ao:resolve(#{ <<"device">> => <<"arweave@2.9">> }, HeaderReq, Opts) of {ok, HeaderChunk} -> @@ -244,8 +258,9 @@ do_head_raw_ans104(TXID, ArweaveOffset, Length, Data, _Opts) -> ), {ok, #{ - <<"arweave-id">> => TXID, - <<"arweave-data-offset">> => ArweaveOffset + HeaderSize, + <<"raw-id">> => TXID, + <<"offset">> => ArweaveOffset, + <<"data-offset">> => ArweaveOffset + HeaderSize, <<"content-type">> => ContentType, <<"header-length">> => HeaderSize, <<"content-length">> => Length - HeaderSize, @@ -263,8 +278,8 @@ get_raw(Base, Request, Opts) -> Err = {error, _} -> Err; {ok, Header = #{ - <<"arweave-id">> := TXID, - <<"arweave-data-offset">> := ArweaveDataOffset, + <<"raw-id">> := TXID, + <<"data-offset">> := ArweaveDataOffset, <<"content-type">> := ContentType, <<"content-length">> := FullContentLength } @@ -295,7 +310,7 @@ get_raw(Base, Request, Opts) -> "/", (hb_util:bin(FullContentLength))/binary >>, - <<"data">> => Data + <<"body">> => Data } }; false -> @@ -303,7 +318,7 @@ get_raw(Base, Request, Opts) -> {ok, Data} -> {ok, Header#{ <<"content-type">> => ContentType, - <<"data">> => Data + <<"body">> => Data }}; Error -> ?event( @@ -382,7 +397,7 @@ get_chunk_range(_Base, Request, Opts) -> %% cannot span the strict data split threshold, so mixed ranges are rejected. fetch_chunk_range(Offset, Length, Opts) -> EndOffset = Offset + Length - 1, - ?event(arweave_debug, {fetch_chunk_range, + ?event(debug_arweave, {fetch_chunk_range, {offset, Offset}, {end_offset, EndOffset}, {size, Length}}), @@ -420,7 +435,7 @@ fetch_post_threshold(Offset, EndOffset, Opts) -> true -> ExtraOffset = min( lists:last(Offsets) + ?DATA_CHUNK_SIZE, EndOffset), - ?event(arweave_debug, {fetching_extra_chunk, + ?event(debug_arweave, {fetching_extra_chunk, {binary_size, BinarySize}, {expected_length, ExpectedLength}, {extra_offset, ExtraOffset}}), @@ -457,7 +472,7 @@ fill_gaps(ChunkInfos, Offset, EndOffset, Opts) -> % be needed. We have yet to find an L1 TX that is chunked in such % a way as to create gaps when using our naive 256KiB chunking. GapOffsets = [Start || {Start, _End} <- Gaps], - ?event(arweave_debug, + ?event(debug_arweave, {fill_gaps, {offset, Offset}, {end_offset, EndOffset}, @@ -476,7 +491,7 @@ fill_gaps(ChunkInfos, Offset, EndOffset, Opts) -> {gap_offsets, GapOffsets}}), case fetch_and_collect(GapOffsets, Opts) of {ok, NewInfos} -> - ?event(arweave_debug, {fill_gaps, NewInfos}), + ?event(debug_arweave, {fill_gaps, NewInfos}), fill_gaps( Sorted ++ NewInfos, Offset, EndOffset, Opts @@ -488,7 +503,7 @@ fill_gaps(ChunkInfos, Offset, EndOffset, Opts) -> %% @doc Fetch chunks at the given offsets in parallel and parse the responses %% into {AbsoluteStartOffset, AbsoluteEndOffset, ChunkBinary} tuples. fetch_and_collect(Offsets, Opts) -> - Concurrency = hb_opts:get(chunk_fetch_concurrency, 10, Opts), + Concurrency = hb_opts:get(arweave_chunk_fetch_concurrency, 10, Opts), Results = hb_pmap:parallel_map( Offsets, fun(O) -> get_chunk(O, Opts) end, @@ -503,7 +518,7 @@ generate_offsets(Start, End, Step) -> generate_offsets(Current, End, _Step, Acc) when Current > End -> Offsets = lists:reverse(Acc), - ?event(arweave_debug, {fetch_chunk_offsets, {offsets, Offsets}}), + ?event(debug_arweave, {fetch_chunk_offsets, {offsets, Offsets}}), Offsets; generate_offsets(Current, End, Step, Acc) -> generate_offsets(Current + Step, End, Step, [Current | Acc]). @@ -519,7 +534,7 @@ collect_chunks([{ok, JSON} | Rest], Acc) -> Chunk = hb_util:decode(maps:get(<<"chunk">>, JSON)), AbsEnd = hb_util:int(maps:get(<<"absolute_end_offset">>, JSON)), AbsStart = AbsEnd - byte_size(Chunk) + 1, - ?event(arweave_debug, + ?event(debug_arweave, {collect_chunks, {abs_start, AbsStart}, {abs_end, AbsEnd}, @@ -584,7 +599,7 @@ assemble_chunks(ChunkInfos, Offset) -> % The first chunk may start before the requested offset; % trim the leading bytes to start exactly at Offset. Skip = Offset - ChunkStart, - ?event(arweave_debug, {assemble_chunks, + ?event(debug_arweave, {assemble_chunks, {skip, Skip}, {chunk_start, ChunkStart}, {offset, Offset}, @@ -593,7 +608,7 @@ assemble_chunks(ChunkInfos, Offset) -> }), binary:part(Data, Skip, byte_size(Data) - Skip); false -> - ?event(arweave_debug, {assemble_chunks, + ?event(debug_arweave, {assemble_chunks, {chunk_start, ChunkStart}, {offset, Offset}, {byte_size, byte_size(Data)} @@ -616,6 +631,59 @@ get_chunk(Offset, Opts) -> Path = <<"/chunk/", (hb_util:bin(Offset))/binary>>, request(<<"GET">>, Path, #{ <<"route-by">> => Offset }, Opts). +%% @doc Read and decode the bundle header index at the given global start +%% offset, returning the header size alongside the decoded index entries. +bundle_header(BundleStartOffset, Opts) -> + case hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ + <<"path">> => <<"chunk">>, + <<"offset">> => BundleStartOffset + 1 + }, + Opts + ) of + {ok, FirstChunk} -> + case ar_bundles:bundle_header_size(FirstChunk) of + invalid_bundle_header -> + {error, invalid_bundle_header}; + HeaderSize -> + case read_bundle_header(BundleStartOffset, HeaderSize, FirstChunk, Opts) of + {ok, HeaderBin} -> + case ar_bundles:decode_bundle_header(HeaderBin) of + {_Items, BundleIndex} -> + {ok, HeaderSize, BundleIndex}; + invalid_bundle_header -> + {error, invalid_bundle_header} + end; + Error -> + Error + end + end; + Error -> + Error + end. + +%% @doc Read exactly the bytes needed to decode a bundle header. +read_bundle_header(_BundleStartOffset, HeaderSize, FirstChunk, _Opts) + when HeaderSize =< byte_size(FirstChunk) -> + {ok, binary:part(FirstChunk, 0, HeaderSize)}; +read_bundle_header(BundleStartOffset, HeaderSize, FirstChunk, Opts) -> + RemainingSize = HeaderSize - byte_size(FirstChunk), + case hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ + <<"path">> => <<"chunk">>, + <<"offset">> => BundleStartOffset + byte_size(FirstChunk) + 1, + <<"length">> => RemainingSize + }, + Opts + ) of + {ok, RemainingChunk} -> + {ok, <>}; + Error -> + Error + end. + %% @doc Retrieve (and cache) block information from Arweave. If the `block' key %% is present, it is used to look up the associated block. If it is of Arweave %% block hash length (43 characters), it is used as an ID. If it is parsable as @@ -729,7 +797,7 @@ request(Method, Path, Opts) -> request(Method, Path, Extra, Opts) -> request(Method, Path, Extra, [], Opts). request(Method, Path, Extra, LogExtra, Opts) -> - ?event(arweave_debug, {request, + ?event(debug_arweave, {request, {method, Method}, {path, {explicit, Path}}, {log_extra, LogExtra}}), Res = hb_http:request( @@ -792,7 +860,7 @@ to_message(Path = <<"/tx">>, <<"POST">>, {ok, Response}, LogExtra, _Opts) -> to_message(Path = <<"/tx/", TXID/binary>>, <<"GET">>, {ok, #{ <<"body">> := Body }}, LogExtra, Opts) -> event_request(Path, <<"GET">>, 200, LogExtra), TXHeader = ar_tx:json_struct_to_tx(hb_json:decode(Body)), - ?event(arweave_debug, + ?event(debug_arweave, {arweave_tx_response, {path, {explicit, Path}}, {raw_body, {explicit, Body}}, @@ -1164,7 +1232,6 @@ setup_arweave_index_opts(TXIDs) -> arweave_index_ids => true, arweave_index_store => IndexStore }, - application:ensure_all_started(hb), % Either: Index the blocks containing the TXs... % lists:foreach( % fun(Block) -> ok = index_test_block(Block, Opts) end, @@ -1307,7 +1374,8 @@ get_tx_basic_data_exclude_data_test() -> }, Opts ), - Data = hb_ao:get(<<"data">>, RawData, Opts), + ?event(debug_test, {raw_data, RawData}), + Data = hb_ao:get(<<"body">>, RawData, Opts), StructuredWithData = Structured#{ <<"data">> => Data }, ?assert(hb_message:verify(StructuredWithData, all, Opts)), DataHash = hb_util:encode(crypto:hash(sha256, Data)), @@ -1343,7 +1411,7 @@ get_tx_data_tag_exclude_data_test() -> }, Opts ), - Data = hb_ao:get(<<"data">>, RawData, Opts), + Data = hb_ao:get(<<"body">>, RawData, Opts), StructuredWithData = Structured#{ <<"data">> => Data }, ?assert(hb_message:verify(StructuredWithData, all, Opts)), DataHash = hb_util:encode(crypto:hash(sha256, Data)), @@ -1421,7 +1489,7 @@ get_raw_range_tx_test() -> ?event(debug_test, {result, Result}), ?assertEqual( {ok, <<"{\"d">>}, - hb_maps:find(<<"data">>, Result, Opts) + hb_maps:find(<<"body">>, Result, Opts) ), {ok, Result2} = hb_ao:resolve( @@ -1441,7 +1509,7 @@ get_raw_range_tx_test() -> ), ?assertEqual( {ok, <<"ame Cr">>}, - hb_maps:find(<<"data">>, Result2, Opts) + hb_maps:find(<<"body">>, Result2, Opts) ). get_raw_range_ans104_test() -> @@ -1466,7 +1534,7 @@ get_raw_range_ans104_test() -> ?event(debug_test, {result, Result}), ?assertEqual( {ok, <<"{\n">>}, - hb_maps:find(<<"data">>, Result, Opts) + hb_maps:find(<<"body">>, Result, Opts) ), {ok, Result2} = hb_ao:resolve( @@ -1486,7 +1554,7 @@ get_raw_range_ans104_test() -> ), ?assertEqual( {ok, <<"t #972">>}, - hb_maps:find(<<"data">>, Result2, Opts) + hb_maps:find(<<"body">>, Result2, Opts) ). get_tx_rsa_nested_bundle_test() -> @@ -1844,7 +1912,6 @@ reassemble_bundle2_test() -> %% reassembles the bundle and nested items. This is also useful tool %% debugging tool to check that a bundle is present in the weave. assert_bundle_tx(TXID) -> - application:ensure_all_started(hb), Opts = #{}, {ok, #{ <<"body">> := OffsetBody }} = hb_http:request( diff --git a/src/dev_arweave_offset.erl b/src/dev_arweave_offset.erl new file mode 100644 index 000000000..ec21ac443 --- /dev/null +++ b/src/dev_arweave_offset.erl @@ -0,0 +1,489 @@ +%%% @doc A module for the Arweave device that implements the default key +%%% resolution logic. The default key returns slices of bytes inside Arweave as +%%% message representations. +-module(dev_arweave_offset). +-export([get/4]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Resolve either a message at an Arweave offset, or a direct key from the +%% base message if the key is not an integer. +get(Key, Base, Request, Opts) -> + case parse(Key) of + {ok, StartOffset, Length} -> + load_item_at_offset(StartOffset, Length, Opts); + error -> + dev_message:get(Key, Base, Request, Opts) + end. + +%% @doc Parse a path key as a global Arweave start offset. The supported syntax +%% is as follows: +%% ``` +%% Reference :: Offset-Length +%% Offset :: [Unit] +%% Length :: +%% Unit :: +%% b : The global Arweave offset in absolute bytes (default). +%% k[i][b] : The global Arweave offset in absolute kilobytes or kibibytes. +%% m[i][b] : The global Arweave offset in absolute megabytes or mebibytes. +%% g[i][b] : The global Arweave offset in absolute gigabytes or gibibytes. +%% t[i][b] : The global Arweave offset in absolute terabytes or tebibytes. +%% p[i][b] : The global Arweave offset in absolute petabytes or pebibytes. +%% e[i][b] : The global Arweave offset in absolute exabytes or exbibytes. +%% z[i][b] : The global Arweave offset in absolute zettabytes or zebibytes. +%% y[i][b] : The global Arweave offset in absolute yottabytes or yobibytes. +%% ``` +%% In the scheme above, the `i` modifier in units indicates that the unit is in +%% binary multiples of the base unit. For example, `kib` is 1024 bytes, `mib` is +%% 1024 * 1024 bytes, etc. By contrast, the `kb` unit is decimal-oriented: `kb` +%% 1000 bytes, `mb` is 1000 * 1000 bytes, etc. To aid minimization of the bytes +%% required for the references, the `b` is always implied and need not be +%% specified. +parse(Key) -> + try + {OffsetBin, Length} = + case binary:split(Key, <<"-">>) of + [Start, LengthBin] -> {Start, hb_util:int(LengthBin)}; + [Start] -> {Start, undefined} + end, + {ok, unit(OffsetBin), Length} + catch + _Class:_Error:_StackTrace -> error + end. + +%% @doc Parses and applies a unit modifier to a base value, supporting both +%% the `kb` and `kib` unit formats. +unit(Binary) -> unit(0, Binary). +unit(Complete, <<>>) -> Complete; +unit(Base, <>) when Int >= $0 andalso Int =< $9 -> + unit(Base * 10 + (Int - $0), Rest); +unit(Base, <<"b">>) -> Base; +unit(Base, <<"ki", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"b">>); +unit(Base, <<"mi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"ki">>); +unit(Base, <<"gi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"mi">>); +unit(Base, <<"ti", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"gi">>); +unit(Base, <<"pi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"ti">>); +unit(Base, <<"ei", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"pi">>); +unit(Base, <<"zi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"ei">>); +unit(Base, <<"yi", _/binary>>) when Base > 0 -> unit(Base * 1024, <<"zi">>); +unit(Base, <<"k", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"b">>); +unit(Base, <<"m", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"k">>); +unit(Base, <<"g", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"m">>); +unit(Base, <<"t", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"g">>); +unit(Base, <<"p", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"t">>); +unit(Base, <<"e", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"p">>); +unit(Base, <<"z", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"e">>); +unit(Base, <<"y", _/binary>>) when Base > 0 -> unit(Base * 1000, <<"z">>). + +%% @doc Load an ANS-104 item whose header begins at the given global offset. +%% When a length is supplied it is treated as the exact ANS-104 data length, so +%% we can skip bundle index discovery and read only the remaining payload bytes. +load_item_at_offset(ExplicitOffset, Length, Opts) when is_integer(Length) -> + maybe + {ok, _ChunkJSON, FirstChunk} ?= chunk_from_offset(ExplicitOffset, Opts), + ?event( + arweave_offset_lookup, + {loaded_explicit_offset, + {explicit_offset, ExplicitOffset}, + {length, Length} + }, + Opts + ), + load_item_from_data_size(ExplicitOffset, Length, FirstChunk, Opts) + end; +load_item_at_offset(TargetOffset, undefined, Opts) -> + maybe + {ok, StartOffset, ItemSize, FirstChunk} ?= + message_from_offset(TargetOffset, Opts), + load_item_from_serialized_size(StartOffset, ItemSize, FirstChunk, Opts) + else + false -> {error, invalid_item_size}; + Error -> Error + end. + +%% @doc Load an item when the exact ANS-104 data length is already known. +load_item_from_data_size(StartOffset, DataSize, FirstChunk, Opts) -> + maybe + {ok, HeaderSize, HeaderTX} ?= deserialize_header(FirstChunk), + load_item_from_header(StartOffset, HeaderSize, HeaderTX, DataSize, Opts) + end. + +%% @doc Load an item when its serialized size is known from the containing +%% bundle index. +load_item_from_serialized_size(StartOffset, ItemSize, FirstChunk, Opts) -> + maybe + {ok, HeaderSize, HeaderTX} ?= deserialize_header(FirstChunk), + true ?= HeaderSize =< ItemSize, + load_item_from_header( + StartOffset, + HeaderSize, + HeaderTX, + ItemSize - HeaderSize, + Opts + ) + else + false -> {error, invalid_item_size}; + Error -> Error + end. + +%% @doc Complete an item load once the header has been decoded, using any data +%% bytes that were already present after the header before reading the tail. +load_item_from_header(StartOffset, HeaderSize, HeaderTX, DataSize, Opts) -> + {HeaderData, RemainingLength} = + split_header_data(HeaderTX#tx.data, DataSize), + ?event( + arweave_offset_lookup, + {calculating_message_from_offset, + {start_offset, StartOffset}, + {header_size, HeaderSize}, + {data_size, DataSize}, + {header_data, HeaderData}, + {remaining_length, RemainingLength} + }, + Opts + ), + maybe + {ok, RemainingData} ?= + read_remaining_data( + StartOffset, + HeaderSize, + byte_size(HeaderData), + RemainingLength, + Opts + ), + FullTX = + HeaderTX#tx{ + data = << HeaderData/binary, RemainingData/binary >>, + data_size = DataSize + }, + {ok, + hb_message:convert( + FullTX, + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ) + } + end. + +%% @doc Read the chunk containing the given offset and trim it to begin at the +%% first byte of the requested item. +chunk_from_offset(StartOffset, Opts) -> + case dev_arweave:get_chunk(StartOffset + 1, Opts) of + {ok, ChunkJSON} -> + ChunkSize = hb_util:int(maps:get(<<"chunk_size">>, ChunkJSON)), + AbsEnd = hb_util:int(maps:get(<<"absolute_end_offset">>, ChunkJSON)), + Chunk = hb_util:decode(maps:get(<<"chunk">>, ChunkJSON)), + ChunkStart = AbsEnd - ChunkSize + 1, + Skip = (StartOffset + 1) - ChunkStart, + {ok, ChunkJSON, binary:part(Chunk, Skip, byte_size(Chunk) - Skip)}; + Error -> + Error + end. + +%% @doc Safe wraper for ANS-104 header deserialization. +deserialize_header(Binary) -> + try ar_bundles:deserialize_header(Binary) + catch _:_ -> {error, <<"Invalid message header">>} + end. + +%% @doc Split the bytes already present after a decoded header from those that +%% still need to be read from Arweave. +split_header_data(HeaderData, DataSize) -> + PrefixSize = min(byte_size(HeaderData), DataSize), + { + binary:part(HeaderData, 0, PrefixSize), + DataSize - PrefixSize + }. + +%% @doc Read any bytes of the data segment that were not present in the first +%% header chunk. +read_remaining_data(_StartOffset, _HeaderSize, _PrefixSize, 0, _Opts) -> + {ok, <<>>}; +read_remaining_data(StartOffset, HeaderSize, PrefixSize, Length, Opts) -> + hb_store_arweave:read_chunks(StartOffset + HeaderSize + PrefixSize, Length, Opts). + +%% @doc Locate the deepest bundled item that contains the given global offset. +message_from_offset(TargetOffset, Opts) -> + maybe + {ok, ChunkJSON, FirstChunk} ?= chunk_from_offset(TargetOffset, Opts), + message_from_offset( + TargetOffset, + bundle_start_offset(ChunkJSON), + TargetOffset, + FirstChunk, + Opts + ) + end. + +%% @doc Recover the global start offset of the containing bundle from the end +%% offset of the chunk in global space and its end offset inside the bundle. +bundle_start_offset(ChunkJSON) -> + AbsEnd = hb_util:int(maps:get(<<"absolute_end_offset">>, ChunkJSON)), + ChunkEndInBundle = + ar_merkle:extract_note( + hb_util:decode(maps:get(<<"data_path">>, ChunkJSON)) + ), + AbsEnd - ChunkEndInBundle. + +message_from_offset(TargetOffset, BundleStartOffset, KnownOffset, KnownChunk, Opts) -> + maybe + {ok, HeaderSize, BundleIndex} ?= + dev_arweave:bundle_header( + BundleStartOffset, + Opts + ), + {ok, ItemStartOffset, ItemSize} ?= + find_bundle_member( + TargetOffset, + BundleStartOffset + HeaderSize, + BundleIndex, + Opts + ), + maybe_nested_item( + TargetOffset, + ItemStartOffset, + ItemSize, + KnownOffset, + KnownChunk, + Opts + ) + end. + +%% @doc If the containing item is itself a bundle and the offset lies in its +%% data payload, recurse into its bundle header. Otherwise return the item. +maybe_nested_item( + TargetOffset, + ItemStartOffset, + ItemSize, + KnownOffset, + KnownChunk, + Opts + ) -> + maybe + {ok, FirstChunk} ?= + item_chunk(ItemStartOffset, KnownOffset, KnownChunk, Opts), + maybe_nested_item( + TargetOffset, + ItemStartOffset, + ItemSize, + FirstChunk, + KnownOffset, + KnownChunk, + Opts + ) + end. + +maybe_nested_item( + TargetOffset, + ItemStartOffset, + ItemSize, + FirstChunk, + KnownOffset, + KnownChunk, + Opts + ) -> + maybe + {ok, HeaderSize, HeaderTX} ?= deserialize_header(FirstChunk), + true ?= TargetOffset >= ItemStartOffset + HeaderSize, + true ?= dev_arweave_common:type(HeaderTX) =/= binary, + message_from_offset( + TargetOffset, + ItemStartOffset + HeaderSize, + KnownOffset, + KnownChunk, + Opts + ) + else + false -> {ok, ItemStartOffset, ItemSize, FirstChunk}; + {error, not_found} -> {ok, ItemStartOffset, ItemSize, FirstChunk}; + Error -> Error + end. + +%% @doc Reuse the first chunk we already have when the located item starts at the +%% same offset as the original request, otherwise fetch the item's first chunk. +item_chunk(ItemStartOffset, ItemStartOffset, FirstChunk, _Opts) -> + {ok, FirstChunk}; +item_chunk(ItemStartOffset, _KnownOffset, _KnownChunk, Opts) -> + case chunk_from_offset(ItemStartOffset, Opts) of + {ok, _ChunkJSON, FirstChunk} -> {ok, FirstChunk}; + Error -> Error + end. + +%% @doc Locate the bundle member containing the given offset. +find_bundle_member(TargetOffset, ItemStartOffset, _BundleIndex, Opts) + when TargetOffset < ItemStartOffset -> + ?event( + arweave_offset_lookup, + {bundle_offset_search_exceeded_bounds, + {target_offset, TargetOffset}, + {item_start_offset, ItemStartOffset} + }, + Opts + ), + {error, not_found}; +find_bundle_member(TargetOffset, ItemStartOffset, [{ID, Size} | _], Opts) + when TargetOffset < ItemStartOffset + Size -> + % The target offset is within the current bundle member. + ?event( + arweave_offset_lookup, + {resolved_bundle_member, {id, ID}, {size, Size}}, + Opts + ), + {ok, ItemStartOffset, Size}; +find_bundle_member(TargetOffset, ItemStartOffset, [{_ID, Size} | Rest], Opts) -> + find_bundle_member(TargetOffset, ItemStartOffset + Size, Rest, Opts); +find_bundle_member(_TargetOffset, _ItemStartOffset, [], _Opts) -> + {error, not_found}. + +%%% Tests + +parse_offset_test() -> + ?assertEqual({ok, 160399272861859, undefined}, parse(<<"160399272861859">>)), + ?assertEqual({ok, 160399272861859, 498852}, parse(<<"160399272861859-498852">>)), + ?assertEqual({ok, 160399273000000, undefined}, parse(<<"160399273000000">>)), + ?assertEqual({ok, 160399273000000, 498852}, parse(<<"160399273000000-498852">>)), + ?assertEqual({ok, 160399273000000, undefined}, parse(<<"160399273m">>)), + ?assertEqual({ok, 160399273000000, 498852}, parse(<<"160399273m-498852">>)), + ?assertEqual( + {ok, 1337 * 1024 * 1024 * 1024 * 1024, undefined}, + parse(<<"1337tib">>) + ), + ok. + +offset_item_cases_test() -> + Opts = #{}, + % A simple message. + assert_offset_item( + <<"160399272861859">>, + 498852, + #{ <<"content-type">> => <<"image/png">> }, + Opts + ), + % A reference with a given length. + assert_offset_item( + <<"160399272861859-498852">>, + 498852, + #{ <<"content-type">> => <<"image/png">> }, + Opts + ), + % A reference to a byte in the middle of the test message. + assert_offset_item( + <<"160399273000000">>, + 498852, + #{ <<"content-type">> => <<"image/png">> }, + Opts + ), + % A megabyte reference to the item, occurring in the middle of the item. + assert_offset_item( + <<"160399273m">>, + 498852, + #{ <<"content-type">> => <<"image/png">> }, + Opts + ), + assert_offset_item( + <<"384600234780716">>, + 856691, + #{ <<"content-type">> => <<"image/jpeg">> }, + Opts + ), + ok. + +offset_nested_item_test() -> + Opts = #{}, + TXID = <<"bndIwac23-s0K11TLC1N7z472sLGAkiOdhds87ZywoE">>, + Node = hb_http_server:start_node(), + {ok, Expected} = + hb_http:get( + Node, + <<"/~arweave@2.9/tx=", TXID/binary, "/1/2">>, + Opts + ), + {ItemStartOffset, _ItemSize} = + bundle_message_offset_from_tx(TXID, [1, 2], Opts), + assert_offset_matches(hb_util:bin(ItemStartOffset + 1), Expected, Opts). + +assert_offset_item(Path, DataSize, Tags, Opts) -> + {ok, Item} = hb_ao:resolve(#{ <<"device">> => <<"arweave@2.9">> }, Path, Opts), + TX = hb_message:convert(Item, <<"ans104@1.0">>, <<"structured@1.0">>, Opts), + ?assert(hb_message:verify(Item, all, Opts)), + ?assertEqual(DataSize, TX#tx.data_size), + ?assertEqual(DataSize, byte_size(TX#tx.data)), + maps:foreach( + fun(Key, Value) -> + ?assertEqual({ok, Value}, hb_maps:find(Key, Item, Opts)) + end, + Tags + ), + ok. + +assert_offset_matches(Path, Expected, Opts) -> + {ok, Item} = hb_ao:resolve(#{ <<"device">> => <<"arweave@2.9">> }, Path, Opts), + ExpectedTX = + hb_message:convert( + Expected, + <<"ans104@1.0">>, + <<"structured@1.0">>, + Opts + ), + TX = hb_message:convert(Item, <<"ans104@1.0">>, <<"structured@1.0">>, Opts), + ?assert(hb_message:verify(Item, all, Opts)), + ?assertEqual( + hb_message:id(Expected, signed, Opts), + hb_message:id(Item, signed, Opts) + ), + ?assertEqual(ExpectedTX#tx.data_size, TX#tx.data_size), + ok. + +bundle_message_offset_from_tx(TXID, Path, Opts) -> + {ok, #{ <<"body">> := OffsetBody }} = + hb_http:request( + #{ + <<"path">> => <<"/arweave/tx/", TXID/binary, "/offset">>, + <<"method">> => <<"GET">> + }, + Opts + ), + OffsetMsg = hb_json:decode(OffsetBody), + EndOffset = hb_util:int(maps:get(<<"offset">>, OffsetMsg)), + Size = hb_util:int(maps:get(<<"size">>, OffsetMsg)), + bundled_index_offset(EndOffset - Size, Path, Opts). + +bundled_index_offset(BundleStartOffset, [Index], Opts) -> + {ok, HeaderSize, BundleIndex} = + dev_arweave:bundle_header( + BundleStartOffset, + Opts + ), + nth_bundle_item(Index, BundleStartOffset + HeaderSize, BundleIndex); +bundled_index_offset(BundleStartOffset, [Index | Rest], Opts) -> + {ItemStartOffset, _ItemSize} = + bundled_index_offset(BundleStartOffset, [Index], Opts), + {ok, _ChunkJSON, FirstChunk} = chunk_from_offset(ItemStartOffset, Opts), + {ok, HeaderSize, _HeaderTX} = deserialize_header(FirstChunk), + bundled_index_offset(ItemStartOffset + HeaderSize, Rest, Opts). + +nth_bundle_item(1, ItemStartOffset, [{_ID, Size} | _]) -> + {ItemStartOffset, Size}; +nth_bundle_item(Index, ItemStartOffset, [{_ID, Size} | Rest]) when Index > 1 -> + nth_bundle_item(Index - 1, ItemStartOffset + Size, Rest). + +offset_as_name_resolver_lookup_test() -> + Opts = #{ + name_resolvers => [#{ <<"device">> => <<"arweave@2.9">> }], + on => + #{ + <<"request">> => [#{ <<"device">> => <<"name@1.0">> }] + } + }, + Node = hb_http_server:start_node(Opts), + {ok, Item} = + hb_http:get( + Node, + #{ + <<"path">> => <<"/">>, + <<"host">> => <<"152974576623958.localhost">> + }, + Opts + ), + ?assertEqual(<<"application/json">>, hb_ao:get(<<"content-type">>, Item, Opts)). diff --git a/src/dev_b32_name.erl b/src/dev_b32_name.erl new file mode 100644 index 000000000..798358b9e --- /dev/null +++ b/src/dev_b32_name.erl @@ -0,0 +1,303 @@ +%%% @doc Allows Arweave message IDs to be used via their base32 encoding as +%%% subdomains on a HyperBEAM node. +-module(dev_b32_name). +-export([info/1]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +info(_Opts) -> + #{ + default => fun get/4, + excludes => [<<"keys">>, <<"set">>] + }. + +%% @doc Try to resolve 52char subdomain back to its original TX ID +get(Key, _, _HookMsg, _Opts) -> + ?event({resolve_52char, {key, Key}}), + case decode(Key) of + error -> + ?event({not_base32_id, {key, Key}}), + {error, not_found}; + ID -> + ?event({resolved_52char, {key, Key}, {id, ID}}), + {ok, ID} + end. + +%% @doc If the key is a 52-character binary, attempt to decode it as base32. +%% Else, return `error`. +decode(Key) when byte_size(Key) == 52 -> + try hb_util:human_id(base32:decode(Key)) catch _:_ -> error end; +decode(_Key) -> error. + +%% @doc Convert an ID into its base32 encoded string representation. +encode(ID) when ?IS_ID(ID) -> + hb_util:bin( + string:replace( + string:to_lower( + hb_util:list(base32:encode(hb_util:native_id(ID))) + ), + "=", + "", + all + ) + ). + +%%% Tests + +%% @doc If the sudbdomain provided isn't a valid ARNS or a 52 char subdomain +%% it should return 404 instead of HyperBEAM page. +invalid_arns_and_not_52char_host_resolution_gives_404_test() -> + Opts = dev_name:test_arns_opts(), + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {error, #{<<"status">> := 404}}, + hb_http:get( + Node, + #{ + <<"path">> => <<"/">>, + <<"host">> => <<"non-existing-subdomain.localhost">> + }, + Opts + ) + ). + +%% @doc Unit test for 52 char subdomain to TX ID logic +key_to_id_test() -> + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + ?assertEqual( + <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, + decode(Subdomain) + ). + +%% @doc Resolving a 52 char subdomain without a TXID in the path should work. +empty_path_manifest_test() -> + TestPath = <<"/">>, + Opts = manifest_opts(), + %% Test to load manifest with only subdomain + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"status">> := 200, + <<"commitments">> := + #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _} + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc Loading assets from a manifest where only a 52 char subdomain is +%% provided should work. +resolve_52char_subdomain_asset_if_txid_not_present_test() -> + TestPath = <<"/assets/ArticleBlock-Dtwjc54T.js">>, + Opts = manifest_opts(), + %% Test to load asset with only subdomain (no TX ID present). + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"status">> := 200, + <<"commitments">> := + #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _} + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc Loading assets from a manifest where a 52 char subdomain and TX ID +%% is provided should work. +subdomain_matches_path_id_and_loads_asset_test() -> + TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA/assets/ArticleBlock-Dtwjc54T.js">>, + Opts = manifest_opts(), + %% Test to load asset with only subdomain (no TX ID present). + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"status">> := 200, + <<"commitments">> := + #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _} + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc Validate the behavior when a subdomain and primary path ID match. The +%% duplicated ID in the request message stream should be ignored. +subdomain_matches_path_id_test() -> + #{ id1 := ID1, opts := Opts } = test_opts(), + ?assertMatch( + {ok, 1}, + hb_http:get( + hb_http_server:start_node(Opts), + #{ + <<"path">> => <>, + <<"host">> => subdomain(ID1, Opts) + }, + Opts + ) + ). + +%% @doc Validate the behavior when a subdomain and primary path ID match. Both +%% IDs should be executed, the subdomain first then the path ID. +subdomain_does_not_match_path_id_test() -> + #{ id1 := ID1, id2 := ID2, opts := Opts } + = test_opts(), + ?assertMatch( + {error, not_found}, + hb_http:get( + hb_http_server:start_node(Opts), + #{ + <<"path">> => <>, + <<"host">> => subdomain(ID2, Opts) + }, + Opts + ) + ). + +%% @doc When both 52 char subdomain and TX ID are provided and equal, ignore +%% the TXID from the assets path. +manifest_subdomain_matches_path_id_test() -> + TestPath = <<"/42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA">>, + Opts = manifest_opts(), + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"status">> := 200, + <<"commitments">> := + #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _} + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +%% @doc When a valid 52 char subdomain TXID doesn't match the TX ID provided, +%% the subdomain TXID is loaded, and tries to access the assets path defined. +%% In this case, sinse no assets exists with this TX ID, it should load the +%% index. +manifest_subdomain_does_not_match_path_id_test() -> + TestPath = <<"/1rTy7gQuK9lJydlKqCEhtGLp2WWG-GOrVo5JdiCmaxs">>, + Opts = manifest_opts(), + Subdomain = <<"4nuojs5tw6xtfjbq47dqk6ak7n6tqyr3uxgemkq5z5vmunhxphya">>, + Node = hb_http_server:start_node(Opts), + ?assertMatch( + {ok, + #{ + <<"commitments">> := + #{ + <<"1rTy7gQuK9lJydlKqCEhtGLp2WWG-GOrVo5JdiCmaxs">> := _ + } + } + }, + hb_http:get( + Node, + #{ + <<"path">> => TestPath, + <<"host">> => <> + }, + Opts + ) + ). + +test_opts() -> + Store = [hb_test_utils:test_store()], + BaseOpts = #{ store => Store, priv_wallet => ar_wallet:new() }, + Msg1 = + #{ + <<"a">> => 1, + <<"b">> => 2, + <<"nested">> => #{ + <<"z">> => 26 + } + }, + Msg2 = + #{ + <<"a">> => 2, + <<"b">> => 4 + }, + MsgWithPath = + #{ + <<"a">> => 3, + <<"b">> => 6, + <<"c">> => 9, + <<"path">> => <<"nested">> + }, + SignedMsg3 = + hb_message:commit( + #{ <<"a">> => 3, <<"b">> => 6, <<"c">> => 9 }, + BaseOpts + ), + {ok, UnsignedID1} = hb_cache:write(Msg1, BaseOpts), + {ok, UnsignedID2} = hb_cache:write(Msg2, BaseOpts), + {ok, UnsignedIDWithPath} = hb_cache:write(MsgWithPath, BaseOpts), + {ok, _UnsignedID3} = hb_cache:write(SignedMsg3, BaseOpts), + #{ + opts => + BaseOpts#{ + store => Store, + name_resolvers => [#{ <<"device">> => <<"b32-name@1.0">> }], + on => + #{ + <<"request">> => [#{<<"device">> => <<"name@1.0">>}] + } + }, + id1 => UnsignedID1, + id2 => UnsignedID2, + id3 => SignedMsg3, + id_with_path => UnsignedIDWithPath, + messages => [Msg1, Msg2, SignedMsg3, MsgWithPath] + }. + +%% @doc Returns the subdomain for a given ID for testing purposes. +subdomain(ID, _Opts) when ?IS_ID(ID) -> + <<(encode(ID))/binary, ".localhost">>; +subdomain(ID, Opts) -> + subdomain(hb_message:id(ID, unsigned, Opts), Opts). + +%% @doc Returns `Opts' with a test environment preloaded with manifest related +%% IDs. +manifest_opts() -> + (dev_manifest:test_env_opts())#{ + name_resolvers => [#{ <<"device">> => <<"b32-name@1.0">> }], + on => + #{ + <<"request">> => + [ + #{<<"device">> => <<"name@1.0">>}, + #{<<"device">> => <<"manifest@1.0">>} + ] + } + }. diff --git a/src/dev_blacklist.erl b/src/dev_blacklist.erl index 6d9739ce9..b9a95bf56 100644 --- a/src/dev_blacklist.erl +++ b/src/dev_blacklist.erl @@ -1,9 +1,10 @@ %%% @doc A request hook device for content moderation by blacklist. %%% -%%% The node operator configures a blacklist provider via the `blacklist-provider` -%%% key in the node message options. The provider can be a message or a path that -%%% returns a message or binary. If a binary is returned from the provider, it is -%%% parsed as a newline-delimited list of IDs. +%%% The node operator configures blacklist providers via the +%%% `blacklist-providers` key (a list) in the node message options. Each provider +%%% can be a message or a path that returns a message or binary. If a binary is +%%% returned from a provider, it is parsed as a newline-delimited list of IDs. +%%% Multiple providers are merged into a single cache (union of all IDs). %%% %%% The device is intended for use as a `~hook@1.0` `on/request` handler. It %%% blocks requests when any ID present in the hook payload matches the active @@ -15,113 +16,151 @@ %%% the Arweave network: No central enforcement, but each node is capable of %%% enforcing its own content policies based on its own free choice and %%% configuration. +%%% +%%% Configuration options: +%%% - blacklist_providers: List of providers to load in AO format. +%%% - blacklist_fallback: halt or continue. +%%% - Halt waits for X milliseconds before sending 503. +%%% - Continue allow the connection to fetch while blacklist is being loaded. +%%% - blacklist_timeout: How long should the request wait for the blacklist to be +%%% loaded. +%%% - blacklist_whitelist: List of endpoint path that are always whitelisted. -module(dev_blacklist). --export([request/3, refresh/3]). +-export([request/3]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(DEFAULT_PROVIDER, - [#{ - <<"data-protocol">> => <<"content-policy">>, - <<"body">> => #{ <<"body">> => <<>> } - }] -). --define(DEFAULT_MIN_WAIT, 60). +%%% The default frequency at which the blacklist cache is refreshed in seconds. +-define(DEFAULT_REFRESH_FREQUENCY, 60 * 5). +-define(DEFAULT_REQUEST_TIMEOUT, 1000). +%% Fallback mode ptions: halt or continue +-define(DEFAULT_FALLBACK_MODE, halt). +-define(DEFAULT_WHITELIST, + [<<"/~hyperbuddy@1.0/metrics">>, + <<"/~hyperbuddy@1.0/styles.css">>, + <<"/~hyperbuddy@1.0/fonts.css">>, + <<"/~hyperbuddy@1.0/script.js">>, + <<"/~hyperbuddy@1.0/bundle.js">>]). %% @doc Hook handler: block requests that involve blacklisted IDs. request(_Base, HookReq, Opts) -> ?event({hook_req, HookReq}), - case is_match(HookReq, Opts) of - false -> - ?event(blacklist, {allowed, HookReq}, Opts), + case hb_opts:get(blacklist_providers, false, Opts) of + false -> + ?event({no_providers}), {ok, HookReq}; - ID -> - ?event(blacklist, {blocked, ID}, Opts), - { - ok, - HookReq#{ - <<"body">> => - [#{ - <<"status">> => 451, - <<"reason">> => <<"content-policy">>, - <<"blocked-id">> => ID, + _ -> + case is_match(HookReq, Opts) of + {blocked_txid, ID} -> + ?event(blacklist, {blocked, ID}, Opts), + { + ok, + HookReq#{ <<"body">> => - << - "Requested message blocked by this node's ", - "content policy. Blocked ID: ", ID/binary - >> - }] - } - } + [#{ + <<"status">> => 451, + <<"reason">> => <<"content-policy">>, + <<"blocked-id">> => ID, + <<"body">> => + << + "Requested message blocked by this node's ", + "content policy. Blocked ID: ", ID/binary + >> + }] + } + }; + Response -> + Response + end end. %% @doc Check if the message contains any blacklisted IDs. is_match(Msg, Opts) -> - maybe_refresh(Opts), - IDs = collect_ids(Msg, Opts), - MatchesFromIDs = fun(ID) -> ets:lookup(cache_table_name(Opts), ID) =/= [] end, - case lists:filter(MatchesFromIDs, IDs) of - [] -> false; - [ID|_] -> ID + WhitelistRoutes = hb_opts:get(blacklist_whitelist, ?DEFAULT_WHITELIST, Opts), + Path = hb_maps:get(<<"path">>, maps:get(<<"request">>, Msg, #{}), no_path), + case lists:member(Path, WhitelistRoutes) of + false -> + ?event({path_do_not_match_whitelist, {path, Path}}), + case ensure_cache_table(Msg, Opts) of + {ok, Msg1} -> + IDs = collect_ids(Msg1, Opts), + MatchesFromIDs = fun(ID) -> ets:lookup(cache_table_name(Opts), ID) =/= [] end, + case lists:filter(MatchesFromIDs, IDs) of + [] -> {ok, Msg1}; + [ID|_] -> {blocked_txid, ID} + end; + {error, Msg1} -> + {error, Msg1} + end; + true -> + ?event({path_match_whitelist, {path, Path}}), + {ok, Msg} end. -%% @doc Force a reload of the blacklist cache. Returns the number of newly -%% inserted IDs. -refresh(Base, Req, Opts) -> - ?event({refresh_called, {base, Base}, {req, Req}}), - maybe_refresh(Opts). - %%% Internal -%% @doc Fetch the blacklist and store the results in the cache table. -maybe_refresh(Opts) -> - ensure_cache_table(Opts), - MinWait = - hb_util:int( - hb_opts:get( - blacklist_refresh_frequency, - ?DEFAULT_MIN_WAIT, - Opts - ) +%% @doc Fetch blacklists from all configured providers and insert IDs into the +%% cache table. +fetch_and_insert_ids(Opts) -> + Total = + lists:foldl( + fun(Provider, Acc) -> + case fetch_single_provider(Provider, Opts) of + {ok, Count} -> Acc + Count; + {error, _} -> Acc + end + end, + 0, + resolve_providers(Opts) ), - Time = erlang:system_time(second), - case hb_opts:get(blacklist_last_refresh, 0, Opts) of - LastRefresh when (Time - LastRefresh) > MinWait -> - fetch_and_insert_ids(Opts), - hb_http_server:set_opts(Opts#{ blacklist_last_refresh => Time }); - _ -> - skip_update + Table = cache_table_name(Opts), + ets:insert(Table, {<<"meta/last-refresh">>, os:system_time(millisecond)}), + ?event( + {table_inserted, + {get_last_refresh, ets:lookup(Table, <<"meta/last-refresh">>)}, + {is_initialized, is_initialized(Table)} + } + ), + ?event(blacklist_short, {fetched_and_inserted_ids, Total}, Opts), + {ok, Total}. + +%% @doc Resolve the configured providers into a list. +resolve_providers(Opts) -> + case hb_opts:get(blacklist_providers, [], Opts) of + Providers when is_list(Providers) -> Providers; + _ -> [] end. -%% @doc Fetch the blacklist and insert the IDs into the cache table. -fetch_and_insert_ids(Opts) -> - ensure_cache_table(Opts), - case hb_opts:get(blacklist_provider, no_provider, Opts) of - no_provider -> {ok, 0}; - Provider -> - case execute_provider(Provider, Opts) of - {ok, Blacklist} -> - {ok, IDs} = parse_blacklist(Blacklist, Opts), - ?event({parsed_blacklist, {ids, IDs}}), - BlacklistID = hb_message:id(Blacklist, all, Opts), - ?event({update_blacklist_cache, {ids, IDs}, {blacklist_id, BlacklistID}}), - Table = cache_table_name(Opts), - {ok, insert_ids(IDs, BlacklistID, Table, Opts)}; - {error, _} = Error -> - ?event({execute_provider_error, Error}), - Error - end +%% @doc Fetch a single provider's blacklist and insert its IDs into the cache. +fetch_single_provider(Provider, Opts) -> + try + case execute_provider(Provider, Opts) of + {ok, Blacklist} -> + {ok, IDs} = parse_blacklist(Blacklist, Opts), + ?event({parsed_blacklist, {ids_lengh, length(IDs)}}), + BlacklistID = hb_message:id(Blacklist, all, Opts), + ?event({update_blacklist_cache, + {ids_lengh, length(IDs)}, {blacklist_id, BlacklistID}}), + Table = cache_table_name(Opts), + {ok, insert_ids(IDs, BlacklistID, Table, Opts)}; + {error, _} = Error -> + ?event({execute_provider_error, Error}), + Error + end + catch + Type:Reason -> + ?event({provider_fetch_error, + {type, Type}, {reason, Reason}, {provider, Provider}}), + {error, {Type, Reason}} end. %% @doc Execute the blacklist provider, returning the result. execute_provider(Provider, Opts) -> ?event({execute_provider, {provider, Provider}}), case hb_cache:ensure_loaded(Provider, Opts) of - Bin when is_binary(Bin) -> - hb_ao:resolve(#{ <<"path">> => Bin }, Opts); - Msgs when is_list(Msgs) -> - hb_ao:resolve_many(Msgs, Opts) + Bin when is_binary(Bin) -> hb_ao:resolve(#{ <<"path">> => Bin }, Opts); + Msgs when is_list(Msgs) -> hb_ao:resolve_many(Msgs, Opts) end. %% @doc Parse the blacklist body, returning a list of IDs. @@ -141,21 +180,42 @@ parse_blacklist(Body, _Opts) when is_binary(Body) -> %% @doc Parse a single line of the blacklist body, returning the ID if it is valid, %% and `false' otherwise. parse_blacklist_line(Line) -> - Trimmed = string:trim(Line, both), - case Trimmed of + case trim_ascii(Line) of <<>> -> false; <<"#", _/binary>> -> false; ID when ?IS_ID(ID) -> {true, hb_util:human_id(ID)}; _ -> false end. +%% @doc Fast ASCII-only whitespace trim (strips \r, \n, \s, \t). +%% Avoids Unicode machinery of string:trim/2 for performance. +trim_ascii(<>) when C =:= $\s; C =:= $\t; C =:= $\r; C =:= $\n -> + trim_ascii(Rest); +trim_ascii(Bin) -> + trim_ascii_right(Bin, byte_size(Bin)). + +trim_ascii_right(_, 0) -> <<>>; +trim_ascii_right(Bin, Len) -> + case binary:at(Bin, Len - 1) of + C when C =:= $\s; C =:= $\t; C =:= $\r; C =:= $\n -> + trim_ascii_right(Bin, Len - 1); + _ -> + binary:part(Bin, 0, Len) + end. + %% @doc Collect all IDs found as elements of a given message. collect_ids(Msg, Opts) -> lists:usort(collect_ids(Msg, [], Opts)). collect_ids(Bin, Acc, _Opts) when ?IS_ID(Bin) -> [hb_util:human_id(Bin) | Acc]; collect_ids(Bin, Acc, _Opts) when is_binary(Bin) -> Acc; -collect_ids(Link, Acc, Opts) when ?IS_LINK(Link) -> - collect_ids(hb_cache:ensure_loaded(Link, Opts), Acc, Opts); +collect_ids({as, _, Msg}, Acc, Opts) -> collect_ids(Msg, Acc, Opts); +collect_ids({link, ID, _}, Acc, _Opts) when ?IS_ID(ID) -> + [hb_util:human_id(ID) | Acc]; collect_ids(Msg, Acc, Opts) when is_map(Msg) -> + case hb_maps:get(<<"path">>, Msg, undefined, Opts) of + Path when ?IS_ID(Path) -> [hb_util:human_id(Path)]; + _ -> [] + end ++ + hb_maps:keys(hb_maps:get(<<"commitments">>, Msg, #{}, Opts), Opts) ++ hb_maps:fold( fun(_Key, Value, AccIn) -> collect_ids(Value, AccIn, Opts) end, Acc, @@ -181,27 +241,80 @@ insert_ids([ID | IDs], Value, Table, Opts) when ?IS_ID(ID) -> end. %% @doc Ensure the cache table exists. -ensure_cache_table(Opts) -> +ensure_cache_table(Msg, Opts) -> + %% Options: + %% - continue: Don't wait for blacklist to be initialized + %% - halt: Close connection with HTTP 503 if not initilalized + FallbackMode = hb_opts:get(blacklist_fallback, ?DEFAULT_FALLBACK_MODE, Opts), + RequestTimeout = hb_opts:get(blacklist_timeout, ?DEFAULT_REQUEST_TIMEOUT, Opts), TableName = cache_table_name(Opts), - case ets:info(TableName) of - undefined -> - ?event({creating_table, TableName}), - ets:new( + case is_initialized(TableName) of + true -> {ok, Msg}; + false -> + hb_name:singleton( TableName, - [ - named_table, - set, - public, - {read_concurrency, true}, - {write_concurrency, true} - ] + fun() -> + ?event({creating_table, TableName}), + ets:new( + TableName, + [ + named_table, + set, + public, + {read_concurrency, true}, + {write_concurrency, true} + ] + ), + ?event({table_created, TableName}), + fetch_and_insert_ids(Opts), + refresh_loop(Opts) + end ), - fetch_and_insert_ids(Opts); - _ -> - ?event({table_exists, TableName}), - ok - end, - TableName. + case FallbackMode of + continue -> {ok, Msg}; + halt -> + IsInitialized = + hb_util:wait_until( + fun() -> is_initialized(TableName) end, + RequestTimeout + ), + case IsInitialized of + true -> {ok, Msg}; + false -> + {error, Msg#{ + <<"status">> => 503, + <<"body">> => <<"Loading blacklist ...">> + }} + end + end + end. + +%% @doc Check if the cache table is initialized. We do this by checking that the +%% `meta/last-refresh' key is present, although we do not care about its value. +is_initialized(TableName) -> + ets:info(TableName) =/= undefined + andalso ets:lookup(TableName, <<"meta/last-refresh">>) =/= []. + +%% @doc Loop that periodically refreshes the blacklist cache. Runs on the +%% singleton process that is responsible for the cache ets table. +refresh_loop(Opts) -> + timer:send_after( + hb_util:int( + hb_opts:get( + blacklist_refresh_frequency, + ?DEFAULT_REFRESH_FREQUENCY, + Opts + ) + ) * 1000, + self(), + refresh + ), + receive + refresh -> + fetch_and_insert_ids(Opts), + refresh_loop(Opts); + stop -> ok + end. %% @doc Calculate the name of the cache table given the `Opts`. cache_table_name(Opts) -> @@ -212,7 +325,12 @@ cache_table_name(Opts) -> %%% Tests setup_test_env() -> - Opts0 = #{ store => hb_test_utils:test_store(), priv_wallet => hb:wallet() }, + %% We need to create a new priv_wallet to avoid conflift when starting a + %% new node from an existing priv_wallet address. + Opts0 = #{ + store => hb_test_utils:test_store(), + priv_wallet => ar_wallet:new() + }, Msg1 = hb_message:commit(#{ <<"body">> => <<"test-1">> }, Opts0), Msg2 = hb_message:commit(#{ <<"body">> => <<"test-2">> }, Opts0), Msg3 = hb_message:commit(#{ <<"body">> => <<"test-3">> }, Opts0), @@ -255,7 +373,7 @@ basic_test() -> }} = setup_test_env(), Opts1 = Opts0#{ - blacklist_provider => BlacklistID, + blacklist_providers => [BlacklistID], on => #{ <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } } @@ -275,6 +393,19 @@ basic_test() -> ), ok. +%% @doc Ensure that the default provider does not block any requests. +first_request_always_return_503_test() -> + {ok, #{ + opts := Opts0, + unsigned3 := UnsignedID3 + }} = setup_test_env(), + Opts1 = Opts0#{ blacklist_providers => [] }, + Node = hb_http_server:start_node(Opts1#{blacklist_timeout => 0}), + ?assertMatch( + {failure, #{<<"status">> := 503, <<"body">> := <<"Loading blacklist ...">>}}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ). + %% @doc Ensure that the default provider does not block any requests. default_provider_test() -> {ok, #{ @@ -282,7 +413,7 @@ default_provider_test() -> signed1 := SignedID1, unsigned3 := UnsignedID3 }} = setup_test_env(), - Opts1 = Opts0#{ blacklist_provider => ?DEFAULT_PROVIDER }, + Opts1 = Opts0#{ blacklist_providers => [] }, Node = hb_http_server:start_node(Opts1), ?assertMatch( {ok, <<"test-3">>}, @@ -310,11 +441,11 @@ blacklist_from_external_http_test() -> #{ store => RootStore, priv_wallet => ar_wallet:new(), - blacklist_provider => - << + blacklist_providers => + [<< "/~relay@1.0/call?relay-method=GET&relay-path=", BlacklistHostNode/binary, BlacklistID/binary - >>, + >>], on => #{ <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } } @@ -331,4 +462,139 @@ blacklist_from_external_http_test() -> <<"reason">> := <<"content-policy">> }}, hb_http:get(Node, SignedID1, NodeOpts) - ). \ No newline at end of file + ). + +%% @doc Test that multiple providers merge their blacklists. +multiple_providers_test() -> + {ok, #{ + opts := Opts0, + signed1 := SignedID1, + unsigned2 := UnsignedID2, + unsigned3 := UnsignedID3 + }} = setup_test_env(), + Blacklist1 = #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => <> + }, + Blacklist2 = #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => <> + }, + BlacklistMsg1 = hb_message:commit(Blacklist1, Opts0), + BlacklistMsg2 = hb_message:commit(Blacklist2, Opts0), + {ok, BlacklistID1} = hb_cache:write(BlacklistMsg1, Opts0), + {ok, BlacklistID2} = hb_cache:write(BlacklistMsg2, Opts0), + Opts1 = Opts0#{ + blacklist_providers => [BlacklistID1, BlacklistID2], + on => #{ + <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } + } + }, + Node = hb_http_server:start_node(Opts1), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, SignedID1, Opts1) + ), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, <<"/", UnsignedID2/binary>>, Opts1) + ), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + ok. + +%% @doc Test that a failing provider does not prevent other providers from +%% contributing entries. +provider_failure_resilience_test() -> + {ok, #{ + opts := Opts0, + signed1 := SignedID1, + unsigned3 := UnsignedID3, + blacklist := BlacklistID + }} = setup_test_env(), + BadProvider = <<"aaaabbbbccccddddeeeeffffgggghhhhiiiijjjjkkkk">>, + Opts1 = Opts0#{ + blacklist_providers => [BadProvider, BlacklistID], + on => #{ + <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } + } + }, + Node = hb_http_server:start_node(Opts1), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, SignedID1, Opts1) + ), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + ok. + +%% @doc Test that the blacklist cache is refreshed periodically. +refresh_periodically_test() -> + {ok, #{ + opts := Opts0 = #{ store := Store }, + signed1 := SignedID1, + unsigned3 := UnsignedID3 + }} = setup_test_env(), + InitialBlacklist = + #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => SignedID1 + }, + BlacklistMsg = hb_message:commit(InitialBlacklist, Opts0), + {ok, InitialBlacklistID} = hb_cache:write(BlacklistMsg, Opts0), + hb_store:make_link(Store, InitialBlacklistID, <<"mutable">>), + UpdatedBlacklist = + #{ + <<"data-protocol">> => <<"content-policy">>, + <<"body">> => <> + }, + UpdatedBlacklistMsg = hb_message:commit(UpdatedBlacklist, Opts0), + {ok, UpdatedBlacklistID} = hb_cache:write(UpdatedBlacklistMsg, Opts0), + hb_store:make_link(Store, InitialBlacklistID, <<"mutable">>), + Opts1 = Opts0#{ + blacklist_providers => [<<"/~cache@1.0/read?target=mutable">>], + on => #{ + <<"request">> => #{ <<"device">> => <<"blacklist@1.0">> } + }, + blacklist_refresh_frequency => 1 + }, + Node = hb_http_server:start_node(Opts1), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, SignedID1, Opts1) + ), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + hb_store:make_link(Store, UpdatedBlacklistID, <<"mutable">>), + ?assertMatch( + {ok, <<"test-3">>}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + timer:sleep(1000), + ?assertMatch( + {error, #{ <<"status">> := 451 }}, + hb_http:get(Node, <<"/", UnsignedID3/binary, "/body">>, Opts1) + ), + ok. + +%% @doc Test that parse_blacklist/2 can handle 1 million IDs within 2000ms. +parse_blacklist_performance_test() -> + GenID = fun() -> + B64 = base64:encode(crypto:strong_rand_bytes(32)), + %% base64:encode of 32 bytes = 44 chars (with 1 '=' padding). + %% Taking the first 43 chars gives a valid 43-byte binary ID. + binary:part(B64, 0, 43) + end, + IDs = [GenID() || _ <- lists:seq(1, 1000000)], + Body = iolist_to_binary(lists:join(<<"\n">>, IDs)), + Start = erlang:monotonic_time(millisecond), + {ok, Parsed} = parse_blacklist(Body, #{}), + Duration = erlang:monotonic_time(millisecond) - Start, + ?assert(length(Parsed) =:= 1000000), + ?assert(Duration =< 2000). diff --git a/src/dev_bundler.erl b/src/dev_bundler.erl index 439311959..20b3aa7f0 100644 --- a/src/dev_bundler.erl +++ b/src/dev_bundler.erl @@ -15,8 +15,11 @@ %%% available for reading instantly (`optimistically'), even before the %%% transaction is dispatched. -module(dev_bundler). --export([tx/3, item/3]). +-export([tx/3, item/3, ensure_server/1, stop_server/0, get_state/0]). +%%% Test-only exports. +-export([start_mock_gateway/1]). -include("include/hb.hrl"). +-include("include/dev_bundler.hrl"). -include_lib("eunit/include/eunit.hrl"). %%% Default options. @@ -42,7 +45,7 @@ item(_Base, Req, Opts) -> ok -> % Queue the item for bundling % (fire-and-forget, ignore errors) - ServerPID ! {item, Item}, + ServerPID ! {enqueue_item, Item}, {ok, #{ <<"id">> => ItemID, <<"timestamp">> => erlang:system_time(millisecond) @@ -97,14 +100,10 @@ cache_item(Item, Opts) -> %% @doc Return the PID of the bundler server. If the server is not running, %% it is started and registered with the name `?SERVER_NAME'. ensure_server(Opts) -> - case hb_name:lookup(?SERVER_NAME) of - undefined -> - PID = spawn(fun() -> init(Opts) end), - ?event(bundler_short, {starting_bundler_server, {pid, PID}}), - hb_name:register(?SERVER_NAME, PID), - hb_name:lookup(?SERVER_NAME); - PID -> PID - end. + hb_name:singleton( + ?SERVER_NAME, + fun() -> init(Opts) end + ). stop_server() -> case hb_name:lookup(?SERVER_NAME) of @@ -114,59 +113,79 @@ stop_server() -> hb_name:unregister(?SERVER_NAME) end. +%% @doc Return the current bundler server state for tests. +get_state() -> + case hb_name:lookup(?SERVER_NAME) of + undefined -> undefined; + PID -> + PID ! {get_state, self(), Ref = make_ref()}, + receive + {state, Ref, State} -> State + after 1000 -> timeout + end + end. + %% @doc Initialize the bundler server. init(Opts) -> - % Start the dispatcher to recover any in-progress bundles - dev_bundler_dispatch:ensure_dispatcher(Opts), - % Recover any unbundled items from cache - {UnbundledItems, RecoveredBytes} = recover_unbundled_items(Opts), - InitialState = #{ - max_size => hb_opts:get( - bundler_max_size, ?DEFAULT_MAX_SIZE, Opts), - max_idle_time => hb_opts:get( - bundler_max_idle_time, ?DEFAULT_MAX_IDLE_TIME, Opts), - max_items => hb_opts:get( - bundler_max_items, ?DEFAULT_MAX_ITEMS, Opts), - queue => UnbundledItems, - bytes => RecoveredBytes - }, - % If recovered items are ready to dispatch, do so immediately - State = maybe_dispatch(InitialState, Opts), - server(State, Opts). - -%% @doc Recover unbundled items from cache and calculate their total size. -%% Returns {Items, TotalBytes}. -recover_unbundled_items(Opts) -> - UnbundledItems = dev_bundler_cache:load_unbundled_items(Opts), - ?event(bundler_short, {recovered_unbundled_items, length(UnbundledItems)}), - % Calculate total bytes for recovered items - RecoveredBytes = lists:foldl( - fun(Item, Acc) -> - Acc + erlang:external_size(Item) + NumWorkers = hb_opts:get(bundler_workers, ?DEFAULT_NUM_WORKERS, Opts), + Workers = lists:map( + fun(_) -> + WorkerPID = spawn_link(fun dev_bundler_task:worker_loop/0), + {WorkerPID, idle} end, - 0, - UnbundledItems + lists:seq(1, NumWorkers) ), - {UnbundledItems, RecoveredBytes}. + InitialState = #state{ + max_size = hb_opts:get(bundler_max_size, ?DEFAULT_MAX_SIZE, Opts), + max_idle_time = hb_opts:get( + bundler_max_idle_time, ?DEFAULT_MAX_IDLE_TIME, Opts), + max_items = hb_opts:get(bundler_max_items, ?DEFAULT_MAX_ITEMS, Opts), + queue = [], + bytes = 0, + workers = maps:from_list(Workers), + task_queue = queue:new(), + bundles = #{}, + opts = Opts + }, + dev_bundler_recovery:recover_unbundled_items(self(), Opts), + dev_bundler_recovery:recover_bundles(self(), Opts), + server(assign_tasks(InitialState), Opts). -%% @doc The main loop of the bundler server. Simply waits for messages to be -%% added to the queue, and then dispatches them when the queue is large enough. -server(State = #{ max_idle_time := MaxIdleTime }, Opts) -> +%% @doc The main loop of the bundler server. +server(State = #state{max_idle_time = MaxIdleTime}, Opts) -> receive - {item, Item} -> - server(maybe_dispatch(add_to_queue(Item, State, Opts), Opts), Opts); + {enqueue_item, Item} -> + State1 = add_to_queue(Item, State, Opts), + server(assign_tasks(maybe_dispatch(State1)), Opts); + {recover_bundle, CommittedTX, Items} -> + State1 = recover_bundle(CommittedTX, Items, State), + server(assign_tasks(State1), Opts); + {task_complete, WorkerPID, Task, Result} -> + State1 = handle_task_complete(WorkerPID, Task, Result, State), + server(assign_tasks(State1), Opts); + {task_failed, WorkerPID, Task, Reason} -> + State1 = handle_task_failed(WorkerPID, Task, Reason, State), + server(assign_tasks(State1), Opts); + {retry_task, Task} -> + State1 = enqueue_task(Task, State), + server(assign_tasks(State1), Opts); + {get_state, From, Ref} -> + From ! {state, Ref, State}, + server(State, Opts); stop -> + maps:foreach( + fun(WorkerPID, _) -> WorkerPID ! stop end, + State#state.workers + ), exit(normal) after MaxIdleTime -> - Q = maps:get(queue, State), - dev_bundler_dispatch:dispatch(Q, Opts), - server(State#{ queue => [] }, Opts) + server(assign_tasks(dispatch_queue(State)), Opts) end. -%% @doc Add an item to the queue. Update the state with the new queue +%% @doc Add an enqueue_item to the queue. Update the state with the new queue %% and approximate total byte size of the queue. %% Note: Item has already been verified and cached before reaching here. -add_to_queue(Item, State = #{ queue := Queue, bytes := Bytes }, Opts) -> +add_to_queue(Item, State = #state{queue = Queue, bytes = Bytes}, Opts) -> ItemSize = erlang:external_size(Item), NewQueue = [Item | Queue], NewBytes = Bytes + ItemSize, @@ -176,28 +195,20 @@ add_to_queue(Item, State = #{ queue := Queue, bytes := Bytes }, Opts) -> {queue_size, length(NewQueue)}, {queue_bytes, NewBytes} }), - State#{ - queue => NewQueue, - bytes => NewBytes - }. + State#state{queue = NewQueue, bytes = NewBytes}. %% @doc Dispatch the queue if it is ready. %% Only dispatches up to max_items at a time to respect the limit. -maybe_dispatch(State = #{queue := Q, max_items := MaxItems}, Opts) -> - case dispatchable(State, Opts) of +maybe_dispatch(State = #state{queue = Q, max_items = MaxItems}) -> + case dispatchable(State) of true -> - % Only dispatch up to max_items, keep the rest in queue {ToDispatch, Remaining} = split_queue(Q, MaxItems), - dev_bundler_dispatch:dispatch(ToDispatch, Opts), - % Recalculate bytes for remaining items - RemainingBytes = lists:foldl( - fun(Item, Acc) -> Acc + erlang:external_size(Item) end, - 0, - Remaining - ), - NewState = State#{queue => Remaining, bytes => RemainingBytes}, - % Check if we should dispatch again (in case we have more than max_items) - maybe_dispatch(NewState, Opts); + State1 = create_bundle(ToDispatch, State), + NewState = State1#state{ + queue = Remaining, + bytes = queue_bytes(Remaining) + }, + maybe_dispatch(NewState); false -> State end. @@ -209,15 +220,247 @@ split_queue(Queue, MaxItems) -> {ToDispatch, Remaining}. %% @doc Returns whether the queue is dispatchable. -dispatchable(#{ queue := Q, max_items := MaxLen }, _Opts) - when length(Q) >= MaxLen -> +dispatchable(#state{queue = Q, max_items = MaxLen}) when length(Q) >= MaxLen -> true; -dispatchable(#{ bytes := Bytes, max_size := MaxSize }, _Opts) - when Bytes >= MaxSize -> +dispatchable(#state{bytes = Bytes, max_size = MaxSize}) when Bytes >= MaxSize -> true; -dispatchable(_State, _Opts) -> +dispatchable(_State) -> false. +%% @doc Return the total size of a queue of items. +queue_bytes(Items) -> + lists:foldl( + fun(Item, Acc) -> Acc + erlang:external_size(Item) end, + 0, + Items + ). + +%% @doc Dispatch all currently queued items immediately. +dispatch_queue(State = #state{queue = []}) -> + State; +dispatch_queue(State = #state{queue = Queue}) -> + create_bundle(Queue, State#state{queue = [], bytes = 0}). + +%% @doc Create a bundle and enqueue its initial post task. +create_bundle([], State) -> + State; +create_bundle(Items, State = #state{bundles = Bundles, opts = Opts}) -> + BundleID = make_ref(), + Bundle = #bundle{ + id = BundleID, + items = Items, + status = initializing, + tx = undefined, + proofs = #{}, + start_time = erlang:timestamp() + }, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle, Bundles) + }, + ?event( + bundler_short, + {dispatching_bundle, + {timestamp, dev_bundler_task:format_timestamp()}, + {bundle_id, BundleID}, + {num_items, length(Items)} + } + ), + Task = #task{ + bundle_id = BundleID, + type = post_tx, + data = Items, + opts = Opts + }, + enqueue_task(Task, State1). + +%% @doc Enqueue a task for worker execution. +enqueue_task(Task, State = #state{task_queue = Queue}) -> + State#state{task_queue = queue:in(Task, Queue)}. + +%% @doc Assign pending tasks to all idle workers. +assign_tasks(State = #state{workers = Workers}) -> + IdleWorkers = maps:filter( + fun(_, Status) -> Status =:= idle end, + Workers + ), + assign_tasks(maps:keys(IdleWorkers), State). + +assign_tasks([], State) -> + State; +assign_tasks([WorkerPID | Rest], State = #state{workers = Workers, task_queue = Queue}) -> + case queue:out(Queue) of + {{value, Task}, Queue1} -> + WorkerPID ! {execute_task, self(), Task}, + State1 = State#state{ + task_queue = Queue1, + workers = maps:put(WorkerPID, {busy, Task}, Workers) + }, + assign_tasks(Rest, State1); + {empty, _} -> + State + end. + +%% @doc Handle successful task completion. +handle_task_complete(WorkerPID, Task, Result, State = #state{ + workers = Workers, + bundles = Bundles + }) -> + #task{bundle_id = BundleID} = Task, + ?event(debug_bundler, dev_bundler_task:log_task(task_complete, Task, [])), + State1 = State#state{ + workers = maps:put(WorkerPID, idle, Workers) + }, + case maps:get(BundleID, Bundles, undefined) of + undefined -> + ?event(bundler_short, {bundle_not_found, BundleID}), + State1; + Bundle -> + task_completed(Task, Bundle, Result, State1) + end. + +%% @doc Handle task failure and schedule a retry. +handle_task_failed(WorkerPID, Task, Reason, State = #state{ + workers = Workers, + opts = Opts + }) -> + RetryCount = Task#task.retry_count, + BaseDelay = hb_opts:get( + retry_base_delay_ms, ?DEFAULT_RETRY_BASE_DELAY_MS, Opts), + MaxDelay = hb_opts:get( + retry_max_delay_ms, ?DEFAULT_RETRY_MAX_DELAY_MS, Opts), + Jitter = hb_opts:get(retry_jitter, ?DEFAULT_RETRY_JITTER, Opts), + BaseDelayWithBackoff = min(BaseDelay * (1 bsl RetryCount), MaxDelay), + JitterFactor = (rand:uniform() * 2 - 1) * Jitter, + Delay = round(BaseDelayWithBackoff * (1 + JitterFactor)), + ?event( + bundler_short, + dev_bundler_task:log_task(task_failed_retrying, Task, [ + {reason, {explicit, Reason}}, + {retry_count, RetryCount}, + {delay_ms, Delay} + ]) + ), + Task1 = Task#task{retry_count = RetryCount + 1}, + erlang:send_after(Delay, self(), {retry_task, Task1}), + State#state{ + workers = maps:put(WorkerPID, idle, Workers) + }. + +%% @doc Apply task completion effects to server state. +task_completed(#task{bundle_id = BundleID, type = post_tx}, Bundle, CommittedTX, State) -> + Bundles = State#state.bundles, + Opts = State#state.opts, + Bundle1 = Bundle#bundle{status = tx_posted, tx = CommittedTX}, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle1, Bundles) + }, + BuildProofsTask = #task{ + bundle_id = BundleID, + type = build_proofs, + data = CommittedTX, + opts = Opts + }, + enqueue_task(BuildProofsTask, State1); +task_completed(#task{bundle_id = BundleID, type = build_proofs}, Bundle, Proofs, State) -> + Bundles = State#state.bundles, + Opts = State#state.opts, + case Proofs of + [] -> + bundle_complete(Bundle, State); + _ -> + ProofsMap = maps:from_list([ + {maps:get(offset, Proof), #proof{proof = Proof, status = pending}} + || Proof <- Proofs + ]), + Bundle1 = Bundle#bundle{ + proofs = ProofsMap, + status = proofs_built + }, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle1, Bundles) + }, + lists:foldl( + fun(ProofData, StateAcc) -> + ProofTask = #task{ + bundle_id = BundleID, + type = post_proof, + data = ProofData, + opts = Opts + }, + enqueue_task(ProofTask, StateAcc) + end, + State1, + Proofs + ) + end; +task_completed( + #task{bundle_id = BundleID, type = post_proof, data = ProofData}, + Bundle, + _Result, + State + ) -> + Bundles = State#state.bundles, + Offset = maps:get(offset, ProofData), + Proofs = Bundle#bundle.proofs, + Proofs1 = maps:update_with( + Offset, + fun(Proof) -> Proof#proof{status = seeded} end, + Proofs + ), + Bundle1 = Bundle#bundle{proofs = Proofs1}, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle1, Bundles) + }, + AllSeeded = lists:all( + fun(#proof{status = Status}) -> Status =:= seeded end, + maps:values(Proofs1) + ), + case AllSeeded of + true -> + bundle_complete(Bundle1, State1); + false -> + State1 + end. + +%% @doc Mark a bundle as complete and remove it from state. +bundle_complete(Bundle, State = #state{opts = Opts}) -> + ok = dev_bundler_cache:complete_tx(Bundle#bundle.tx, Opts), + ElapsedTime = + timer:now_diff(erlang:timestamp(), Bundle#bundle.start_time) / 1000000, + ?event( + bundler_short, + {bundle_complete, + {bundle_id, Bundle#bundle.id}, + {timestamp, dev_bundler_task:format_timestamp()}, + {tx, {explicit, hb_message:id(Bundle#bundle.tx, signed, Opts)}}, + {elapsed_time_s, ElapsedTime} + } + ), + State#state{bundles = maps:remove(Bundle#bundle.id, State#state.bundles)}. + +%% @doc Recover a single bundle and enqueue any follow-up work. +recover_bundle(CommittedTX, Items, State = #state{opts = Opts}) -> + BundleID = make_ref(), + Bundle = #bundle{ + id = BundleID, + items = Items, + status = tx_posted, + tx = CommittedTX, + proofs = #{}, + start_time = erlang:timestamp() + }, + Bundles = State#state.bundles, + State1 = State#state{ + bundles = maps:put(BundleID, Bundle, Bundles) + }, + Task = #task{ + bundle_id = BundleID, + type = build_proofs, + data = CommittedTX, + opts = Opts + }, + enqueue_task(Task, State1). + %%%=================================================================== %%% Tests %%%=================================================================== @@ -242,7 +485,7 @@ nested_bundle_test() -> ClientOpts = #{}, NodeOpts2 = maps:merge(NodeOpts, #{ bundler_max_items => 3 }), Node = hb_http_server:start_node(NodeOpts2#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store() }), %% Upload 3 data items across 4 chunks. @@ -289,7 +532,7 @@ tx_error_test() -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1 }), @@ -320,7 +563,7 @@ unsigned_dataitem_test() -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), debug_print => false }), @@ -352,7 +595,7 @@ idle_test() -> ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ bundler_max_idle_time => 400, - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store() }), % Test posting each of the supported signature types @@ -410,7 +653,7 @@ dispatch_blocking_test() -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 3 }), @@ -451,29 +694,6 @@ dispatch_blocking_test() -> stop_test_servers(ServerHandle) end. -recover_unbundled_items_test() -> - Opts = #{store => hb_test_utils:test_store()}, - % Create and cache some items - Item1 = hb_message:convert(new_data_item(1, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - Item2 = hb_message:convert(new_data_item(2, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - Item3 = hb_message:convert(new_data_item(3, 10), <<"structured@1.0">>, <<"ans104@1.0">>, Opts), - ok = dev_bundler_cache:write_item(Item1, Opts), - ok = dev_bundler_cache:write_item(Item2, Opts), - ok = dev_bundler_cache:write_item(Item3, Opts), - % Bundle Item2 with a fake TX - FakeTX = ar_tx:sign(#tx{format = 2, tags = [{<<"test">>, <<"tx">>}]}, hb:wallet()), - StructuredTX = hb_message:convert(FakeTX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), - ok = dev_bundler_cache:write_tx(StructuredTX, [Item2], Opts), - % Now recover unbundled items - {RecoveredItems, RecoveredBytes} = recover_unbundled_items(Opts), - ?assertEqual(3924, RecoveredBytes), - RecoveredItems2 = [ - hb_message:with_commitments( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) - || Item <- RecoveredItems], - ?assertEqual(lists:sort([Item1, Item3]), lists:sort(RecoveredItems2)), - ok. - %% @doc Test that items are recovered and posted while respecting the %% max_items limit. recover_respects_max_items_test() -> @@ -487,7 +707,7 @@ recover_respects_max_items_test() -> % Use max_items of 3, so 10 items should dispatch as 3+3+3+1 MaxItems = 3, Opts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => MaxItems }, @@ -516,6 +736,458 @@ recover_respects_max_items_test() -> stop_test_servers(ServerHandle) end. +complete_task_sequence_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 2, + retry_base_delay_ms => 100, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [ + new_structured_data_item(1, 10, Opts), + new_structured_data_item(2, 10, Opts) + ], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + ?assertEqual(1, length(TXs)), + Proofs = hb_mock_server:get_requests(chunk, 1, ServerHandle), + ?assertEqual(1, length(Proofs)), + State = get_state(), + ?assertNotEqual(undefined, State), + ?assertNotEqual(timeout, State), + Workers = State#state.workers, + IdleWorkers = [ + PID + || {PID, Status} <- maps:to_list(Workers), Status =:= idle + ], + ?assertEqual(maps:size(Workers), length(IdleWorkers)), + Queue = State#state.task_queue, + ?assert(queue:is_empty(Queue)), + Bundles = State#state.bundles, + ?assertEqual(0, maps:size(Bundles)), + ok + after + stop_test_servers(ServerHandle) + end. + +recover_bundles_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + chunk => fun(_Req) -> + timer:sleep(250), + {200, <<"OK">>} + end, + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store() + }, + hb_http_server:start_node(Opts), + Item1 = new_structured_data_item(1, 10, Opts), + Item2 = new_structured_data_item(2, 10, Opts), + Item3 = new_structured_data_item(3, 10, Opts), + ok = dev_bundler_cache:write_item(Item1, Opts), + ok = dev_bundler_cache:write_item(Item2, Opts), + ok = dev_bundler_cache:write_item(Item3, Opts), + TX = dev_bundler_task:data_items_to_tx( + lists:reverse([Item1, Item2, Item3]), Opts), + CommittedTX = hb_message:convert( + TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), + ok = dev_bundler_cache:write_tx(CommittedTX, [Item1, Item2, Item3], Opts), + Item4 = new_structured_data_item(4, 10, Opts), + ok = dev_bundler_cache:write_item(Item4, Opts), + TX2 = dev_bundler_task:data_items_to_tx( + lists:reverse([Item4]), Opts), + CommittedTX2 = hb_message:convert( + TX2, <<"structured@1.0">>, <<"tx@1.0">>, Opts), + ok = dev_bundler_cache:write_tx(CommittedTX2, [Item4], Opts), + ok = dev_bundler_cache:complete_tx(CommittedTX2, Opts), + ensure_server(Opts), + State = get_state(), + ?assertNotEqual(undefined, State), + ?assertNotEqual(timeout, State), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle, 200), + ?assertEqual([], TXs), + ?assert( + hb_util:wait_until( + fun() -> + dev_bundler_cache:load_bundle_states(Opts) =:= [] + end, + 2000 + ) + ), + FinalState = get_state(), + ?assertEqual(0, maps:size(FinalState#state.bundles)), + ok + after + stop_test_servers(ServerHandle) + end. + +post_tx_price_failure_retry_test() -> + Anchor = rand:bytes(32), + FailCount = 3, + setup_test_counter(price_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => fun(_Req) -> + Count = increment_test_counter(price_attempts_counter) - 1, + case Count < FailCount of + true -> {500, <<"error">>}; + false -> {200, <<"12345">>} + end + end, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 50, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + ?assertEqual(1, length(TXs)), + FinalCount = get_test_counter(price_attempts_counter), + ?assertEqual(FailCount + 1, FinalCount), + ok + after + cleanup_test_counter(price_attempts_counter), + stop_test_servers(ServerHandle) + end. + +post_tx_anchor_failure_retry_test() -> + Price = 12345, + FailCount = 3, + setup_test_counter(anchor_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => fun(_Req) -> + Count = increment_test_counter(anchor_attempts_counter) - 1, + case Count < FailCount of + true -> {500, <<"error">>}; + false -> {200, hb_util:encode(rand:bytes(32))} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 50, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + ?assertEqual(1, length(TXs)), + FinalCount = get_test_counter(anchor_attempts_counter), + ?assertEqual(FailCount + 1, FinalCount), + ok + after + cleanup_test_counter(anchor_attempts_counter), + stop_test_servers(ServerHandle) + end. + +post_tx_post_failure_retry_test() -> + Anchor = rand:bytes(32), + Price = 12345, + FailCount = 4, + setup_test_counter(tx_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + Count = increment_test_counter(tx_attempts_counter) - 1, + case Count < FailCount of + true -> {400, <<"Transaction verification failed">>}; + false -> {200, <<"OK">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 50, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, FailCount + 1, ServerHandle), + ?assertEqual(FailCount + 1, length(TXs)), + FinalCount = get_test_counter(tx_attempts_counter), + ?assertEqual(FailCount + 1, FinalCount), + ok + after + cleanup_test_counter(tx_attempts_counter), + stop_test_servers(ServerHandle) + end. + +post_proof_failure_retry_test() -> + Anchor = rand:bytes(32), + Price = 12345, + FailCount = 2, + setup_test_counter(chunk_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + chunk => fun(_Req) -> + Count = increment_test_counter(chunk_attempts_counter) - 1, + case Count < FailCount of + true -> {500, <<"error">>}; + false -> {200, <<"OK">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 50, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, floor(4.5 * ?DATA_CHUNK_SIZE), Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), + ?assertEqual(1, length(TXs)), + Chunks = hb_mock_server:get_requests(chunk, FailCount + 5, ServerHandle), + ?assertEqual(FailCount + 5, length(Chunks)), + FinalCount = get_test_counter(chunk_attempts_counter), + ?assertEqual(FailCount + 5, FinalCount), + ok + after + cleanup_test_counter(chunk_attempts_counter), + stop_test_servers(ServerHandle) + end. + +rapid_dispatch_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + timer:sleep(100), + {200, <<"OK">>} + end + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + bundler_workers => 3 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + lists:foreach( + fun(I) -> + Items = [new_structured_data_item(I, 10, Opts)], + submit_test_items(Items, Opts) + end, + lists:seq(1, 10) + ), + TXs = hb_mock_server:get_requests(tx, 10, ServerHandle), + ?assertEqual(10, length(TXs)), + ok + after + stop_test_servers(ServerHandle) + end. + +one_bundle_fails_others_continue_test() -> + Anchor = rand:bytes(32), + Price = 12345, + setup_test_counter(mixed_attempts_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + Count = increment_test_counter(mixed_attempts_counter) - 1, + case Count of + 0 -> {200, <<"OK">>}; + _ -> {400, <<"fail">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 100, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items1 = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items1, Opts), + Items2 = [new_structured_data_item(2, 10, Opts)], + submit_test_items(Items2, Opts), + TXs = hb_mock_server:get_requests(tx, 5, ServerHandle), + ?assert(length(TXs) >= 5, length(TXs)), + ok + after + cleanup_test_counter(mixed_attempts_counter), + stop_test_servers(ServerHandle) + end. + +parallel_task_execution_test() -> + Anchor = rand:bytes(32), + Price = 12345, + SleepTime = 120, + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + chunk => fun(_Req) -> + timer:sleep(SleepTime), + {200, <<"OK">>} + end + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + bundler_workers => 5 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + lists:foreach( + fun(I) -> + Items = [new_structured_data_item(I, 10, Opts)], + submit_test_items(Items, Opts) + end, + lists:seq(1, 10) + ), + StartTime = erlang:system_time(millisecond), + Chunks = hb_mock_server:get_requests(chunk, 10, ServerHandle), + ElapsedTime = erlang:system_time(millisecond) - StartTime, + ?assertEqual(10, length(Chunks)), + ?assert(ElapsedTime < 2000, "ElapsedTime: " ++ integer_to_list(ElapsedTime)), + ok + after + stop_test_servers(ServerHandle) + end. + +exponential_backoff_timing_test() -> + Anchor = rand:bytes(32), + Price = 12345, + FailCount = 5, + setup_test_counter(backoff_cap_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + Timestamp = erlang:system_time(millisecond), + Attempt = increment_test_counter(backoff_cap_counter), + Count = Attempt - 1, + add_test_attempt_timestamp(backoff_cap_counter, Attempt, Timestamp), + case Count < FailCount of + true -> {400, <<"fail">>}; + false -> {200, <<"OK">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 100, + retry_max_delay_ms => 500, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items, Opts), + TXs = hb_mock_server:get_requests(tx, FailCount + 1, ServerHandle, 5000), + ?assertEqual(FailCount + 1, length(TXs)), + Timestamps = test_attempt_timestamps(backoff_cap_counter), + ?assertEqual(6, length(Timestamps)), + [T1, T2, T3, T4, T5, T6] = Timestamps, + Delay1 = T2 - T1, + Delay2 = T3 - T2, + Delay3 = T4 - T3, + Delay4 = T5 - T4, + Delay5 = T6 - T5, + ?assert(Delay1 >= 70 andalso Delay1 =< 200, Delay1), + ?assert(Delay2 >= 150 andalso Delay2 =< 300, Delay2), + ?assert(Delay3 >= 300 andalso Delay3 =< 500, Delay3), + ?assert(Delay4 >= 400 andalso Delay4 =< 700, Delay4), + ?assert(Delay5 >= 400 andalso Delay5 =< 700, Delay5), + ok + after + cleanup_test_counter(backoff_cap_counter), + stop_test_servers(ServerHandle) + end. + +independent_task_retry_counts_test() -> + Anchor = rand:bytes(32), + Price = 12345, + setup_test_counter(independent_retry_counter), + {ServerHandle, NodeOpts} = start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)}, + tx => fun(_Req) -> + Count = increment_test_counter(independent_retry_counter) - 1, + case Count < 2 of + true -> {400, <<"fail">>}; + false -> {200, <<"OK">>} + end + end + }), + try + Opts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store(), + bundler_max_items => 1, + retry_base_delay_ms => 100, + retry_jitter => 0 + }, + hb_http_server:start_node(Opts), + ensure_server(Opts), + Items1 = [new_structured_data_item(1, 10, Opts)], + submit_test_items(Items1, Opts), + hb_mock_server:get_requests(tx, 3, ServerHandle), + Items2 = [new_structured_data_item(2, 10, Opts)], + submit_test_items(Items2, Opts), + TotalAttempts = 4, + TXs = hb_mock_server:get_requests(tx, TotalAttempts, ServerHandle), + ?assertEqual(TotalAttempts, length(TXs)), + ok + after + cleanup_test_counter(independent_retry_counter), + stop_test_servers(ServerHandle) + end. + invalid_item_test() -> Anchor = rand:bytes(32), Price = 12345, @@ -526,7 +1198,7 @@ invalid_item_test() -> try ClientOpts = #{}, TestOpts = NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store() }, Node = hb_http_server:start_node(TestOpts#{ @@ -538,7 +1210,7 @@ invalid_item_test() -> data = <<"testdata">>, tags = [{<<"tag1">>, <<"value1">>}] }, - hb:wallet() + ar_wallet:new() ), % Tamper with the data after signing (this invalidates the signature) TamperedItem = Item#tx{data = <<"tampereddata">>}, @@ -572,7 +1244,7 @@ cache_write_failure_test() -> data = <<"testdata">>, tags = [{<<"tag1">>, <<"value1">>}] }, - hb:wallet() + ar_wallet:new() ), StructuredItem = hb_message:convert( Item, <<"structured@1.0">>, <<"ans104@1.0">>, GoodOpts), @@ -584,14 +1256,12 @@ cache_write_failure_test() -> <<"error">> := <<"cache_write_failed">>}}, Result), ok after - stop_server(), - dev_bundler_dispatch:stop_dispatcher() + stop_server() end. stop_test_servers(ServerHandle) -> hb_mock_server:stop(ServerHandle), - stop_server(), - dev_bundler_dispatch:stop_dispatcher(). + stop_server(). test_bundle(Opts) -> Anchor = rand:bytes(32), @@ -607,7 +1277,7 @@ test_bundle(Opts) -> ClientOpts = #{}, NodeOpts2 = maps:merge(NodeOpts, Opts), Node = hb_http_server:start_node(NodeOpts2#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store() }), %% Upload 3 data items across 4 chunks. @@ -636,7 +1306,7 @@ test_api_error(Responses) -> try ClientOpts = #{}, Node = hb_http_server:start_node(NodeOpts#{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => hb_test_utils:test_store(), bundler_max_items => 1 }), @@ -658,7 +1328,25 @@ test_api_error(Responses) -> end. new_data_item(Index, Size) -> - new_data_item(Index, Size, hb:wallet()). + new_data_item(Index, Size, ar_wallet:new()). + +new_structured_data_item(Index, Size, Opts) -> + hb_message:convert( + new_data_item(Index, Size), + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ). + +submit_test_items([], _Opts) -> + ok; +submit_test_items(Items, Opts) -> + lists:foreach( + fun(Item) -> + ?assertMatch({ok, _}, item(#{}, Item, Opts)) + end, + Items + ). new_data_item(Index, Size, Wallet) -> Data = rand:bytes(Size), @@ -780,4 +1468,35 @@ start_mock_gateway(Responses) -> } ] }, - {ServerHandle, NodeOpts}. \ No newline at end of file + {ServerHandle, NodeOpts}. + +setup_test_counter(Table) -> + cleanup_test_counter(Table), + ets:new(Table, [named_table, public, set]), + ok. + +cleanup_test_counter(Table) -> + case ets:info(Table) of + undefined -> ok; + _ -> ets:delete(Table), ok + end. + +increment_test_counter(Table) -> + ets:update_counter(Table, Table, {2, 1}, {Table, 0}). + +get_test_counter(Table) -> + case ets:lookup(Table, Table) of + [{_, Value}] -> Value; + [] -> 0 + end. + +add_test_attempt_timestamp(Table, Attempt, Timestamp) -> + ets:insert(Table, {{Table, Attempt}, Timestamp}). + +test_attempt_timestamps(Table) -> + TimestampEntries = [ + {Attempt, Timestamp} + || {{Prefix1, Attempt}, Timestamp} <- ets:tab2list(Table), + Prefix1 =:= Table + ], + [Timestamp || {_, Timestamp} <- lists:sort(TimestampEntries)]. \ No newline at end of file diff --git a/src/dev_bundler_cache.erl b/src/dev_bundler_cache.erl index e0df60d6a..ae538c933 100644 --- a/src/dev_bundler_cache.erl +++ b/src/dev_bundler_cache.erl @@ -7,17 +7,18 @@ %%% %%% Recovery flow: %%% 1. Load unbundled items (where bundle = <<>>) back into dev_bundler queue -%%% 2. Load TX states and reconstruct dev_bundler_dispatch bundles +%%% 2. Load TX states and reconstruct in-progress bundler bundles %%% 3. Enqueue appropriate tasks based on status -module(dev_bundler_cache). -export([ write_item/2, write_tx/3, complete_tx/2, - load_unbundled_items/1, load_bundle_states/1, - load_bundled_items/2, - load_tx/2 + load_tx/2, + load_items/2, + load_items/4, + list_item_ids/1 ]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -55,11 +56,13 @@ get_item_bundle(Item, Opts) when is_map(Item) -> %% @doc Construct the pseudopath for an item's bundle reference. %% Item should be a structured message. item_path(Item, Opts) when is_map(Item) -> + item_path(item_id(Item, Opts), Opts); +item_path(ItemID, Opts) when is_binary(ItemID) -> Store = hb_opts:get(store, no_viable_store, Opts), hb_store:path(Store, [ ?BUNDLER_PREFIX, <<"item">>, - item_id(Item, Opts), + ItemID, <<"bundle">> ]). @@ -78,7 +81,8 @@ write_tx(TX, Items, Opts) when is_map(TX) -> ok = link_item_to_tx(Item, TX, Opts) end, Items - ). + ), + ok. complete_tx(TX, Opts) -> set_tx_status(TX, <<"complete">>, Opts). @@ -86,7 +90,7 @@ complete_tx(TX, Opts) -> %% @doc Set the status of a bundle TX. set_tx_status(TX, Status, Opts) -> Path = tx_path(TX, Opts), - ?event(bundler_debug, {set_tx_status, {path, Path}, {status, Status}}), + ?event(debug_bundler, {set_tx_status, {path, Path}, {status, Status}}), write_pseudopath(Path, Status, Opts). %% @doc Get the status of a bundle TX. @@ -110,57 +114,6 @@ tx_path(TX, Opts) -> %%% Recovery operations -%% @doc Load all unbundled items (where bundle = <<>>) from cache. -%% Returns list of actual Item messages for re-queuing. -load_unbundled_items(Opts) -> - Store = hb_opts:get(store, no_viable_store, Opts), - ItemsPath = hb_store:path(Store, [?BUNDLER_PREFIX, <<"item">>]), - % List all item IDs - ItemIDs = case hb_cache:list(ItemsPath, Opts) of - [] -> []; - List -> List - end, - ?event(bundler_short, - {recovering_all_unbundled_items, length(ItemIDs)} - ), - % Filter for unbundled items and load them - lists:filtermap( - fun(ItemIDStr) -> - % Read the bundle pseudopath directly - BundlePath = hb_store:path(Store, [ - ?BUNDLER_PREFIX, - <<"item">>, - ItemIDStr, - <<"bundle">> - ]), - case read_pseudopath(BundlePath, Opts) of - {ok, <<>>} -> - % Unbundled item - load it fully (resolve all links) - case hb_cache:read(ItemIDStr, Opts) of - {ok, Item} -> - FullyLoadedItem = hb_cache:ensure_all_loaded(Item, Opts), - ?event(bundler_short, - {recovered_unbundled_item, - {id, {string, ItemIDStr}} - } - ), - {true, FullyLoadedItem}; - _ -> - ?event(bundler_short, - {failed_to_recover_unbundled_item, - {id, {string, ItemIDStr}} - } - ), - false - end; - _ -> - % Already bundled or not found - false - end - end, - ItemIDs - ). - %% @doc Load all bundle TX states from cache. %% Returns list of {TXID, Status} tuples. load_bundle_states(Opts) -> @@ -181,7 +134,7 @@ load_bundle_states(Opts) -> <<"complete">> -> false; % Skip completed bundles Status -> ?event( - bundler_debug, + debug_bundler, {loaded_tx_state, {id, {string, TXID}}, {status, Status} @@ -193,66 +146,12 @@ load_bundle_states(Opts) -> TXIDs ). -%% @doc Load all data items associated with a bundle TX. -%% Uses the item pseudopaths to find items with matching tx-id. -load_bundled_items(TXID, Opts) -> - Store = hb_opts:get(store, no_viable_store, Opts), - ItemsPath = hb_store:path(Store, [?BUNDLER_PREFIX, <<"item">>]), - % List all item IDs - ItemIDs = case hb_cache:list(ItemsPath, Opts) of - [] -> []; - List -> List - end, - ?event(bundler_short, {recovering_bundled_items, - {count, length(ItemIDs)}}), - % Filter for items belonging to this TX and load them - lists:filtermap( - fun(ItemIDStr) -> - % Read the bundle pseudopath directly - BundlePath = hb_store:path(Store, [ - ?BUNDLER_PREFIX, - <<"item">>, - ItemIDStr, - <<"bundle">> - ]), - case read_pseudopath(BundlePath, Opts) of - {ok, BundleTXID} when BundleTXID =:= TXID -> - % This item belongs to our bundle - load it fully (resolve all links) - case hb_cache:read(ItemIDStr, Opts) of - {ok, Item} -> - FullyLoadedItem = hb_cache:ensure_all_loaded(Item, Opts), - ?event( - bundler_debug, - {loaded_tx_item, - {tx_id, {explicit, TXID}}, - {item_id, {explicit, ItemIDStr}} - } - ), - {true, FullyLoadedItem}; - _ -> - ?event( - error, - {failed_to_load_tx_item, - {tx_id, {explicit, TXID}}, - {item_id, {explicit, ItemIDStr}} - } - ), - false - end; - _ -> - % Doesn't belong to this bundle or not found - false - end - end, - ItemIDs - ). - %% @doc Load a TX from cache by its ID. load_tx(TXID, Opts) -> - ?event(bundler_debug, {load_tx, {tx_id, {explicit, TXID}}}), + ?event(debug_bundler, {load_tx, {tx_id, {explicit, TXID}}}), case hb_cache:read(TXID, Opts) of {ok, TX} -> - ?event(bundler_debug, {loaded_tx, {tx_id, {explicit, TXID}}}), + ?event(debug_bundler, {loaded_tx, {tx_id, {explicit, TXID}}}), hb_cache:ensure_all_loaded(TX, Opts); _ -> ?event(error, {failed_to_load_tx, {tx_id, {explicit, TXID}}}), @@ -265,7 +164,10 @@ load_tx(TXID, Opts) -> %% @doc Write a value to a pseudopath. write_pseudopath(Path, Value, Opts) -> Store = hb_opts:get(store, no_viable_store, Opts), - hb_store:write(Store, Path, Value). + Result = hb_store:write(Store, Path, Value), + % force a flush to disk + hb_store:read(Store, Path), + Result. %% @doc Read a value from a pseudopath. read_pseudopath(Path, Opts) -> @@ -275,6 +177,47 @@ read_pseudopath(Path, Opts) -> _ -> not_found end. +%% @doc List all cached bundler item IDs. +list_item_ids(Opts) -> + Store = hb_opts:get(store, no_viable_store, Opts), + ItemsPath = hb_store:path(Store, [?BUNDLER_PREFIX, <<"item">>]), + case hb_cache:list(ItemsPath, Opts) of + [] -> []; + List -> List + end. + +%% @doc Load all items whose bundle pseudopath matches BundleID. +load_items(BundleID, Opts) -> + load_items( + BundleID, + Opts, + fun(_ItemID, _Item) -> ok end, + fun(_ItemID) -> ok end + ). + +%% @doc Load all items whose bundle pseudopath matches BundleID and invoke callbacks. +load_items(BundleID, Opts, OnLoaded, OnFailed) -> + lists:filtermap( + fun(ItemID) -> + BundlePath = item_path(ItemID, Opts), + case read_pseudopath(BundlePath, Opts) of + {ok, BundleID} -> + case hb_cache:read(ItemID, Opts) of + {ok, Item} -> + FullyLoadedItem = hb_cache:ensure_all_loaded(Item, Opts), + OnLoaded(ItemID, FullyLoadedItem), + {true, FullyLoadedItem}; + _ -> + OnFailed(ItemID), + false + end; + _ -> + false + end + end, + list_item_ids(Opts) + ). + %%% Tests basic_cache_test() -> @@ -306,7 +249,7 @@ load_unbundled_items_test() -> % Link item2 to a bundle, leave others unbundled ok = write_tx(TX, [Item2], Opts), % Load unbundled items - UnbundledItems1 = load_unbundled_items(Opts), + UnbundledItems1 = load_items(<<>>, Opts), UnbundledItems2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, @@ -317,6 +260,17 @@ load_unbundled_items_test() -> ?assertEqual(lists:sort([Item1, Item3]), UnbundledItems3), ok. +recovered_items_relink_to_original_bundle_path_test() -> + Opts = #{store => hb_test_utils:test_store()}, + Item = new_data_item(1, <<"data1">>, Opts), + ok = write_item(Item, Opts), + [RecoveredItem] = load_items(<<>>, Opts), + TX = new_tx(1, Opts), + ok = write_tx(TX, [RecoveredItem], Opts), + ?assertEqual(tx_id(TX, Opts), get_item_bundle(Item, Opts)), + ?assertEqual([], load_items(<<>>, Opts)), + ok. + load_bundle_states_test() -> Opts = #{store => hb_test_utils:test_store()}, TX1 = new_tx(1, Opts), @@ -348,7 +302,7 @@ load_bundled_items_test() -> ok = write_tx(TX1, [Item1, Item2], Opts), ok = write_tx(TX2, [Item3], Opts), % Load items for bundle 1 - Bundle1Items1 = load_bundled_items(tx_id(TX1, Opts), Opts), + Bundle1Items1 = load_items(tx_id(TX1, Opts), Opts), Bundle1Items2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, @@ -357,7 +311,7 @@ load_bundled_items_test() -> Bundle1Items3 = lists:sort(Bundle1Items2), ?assertEqual(lists:sort([Item1, Item2]), Bundle1Items3), % Load items for bundle 2 - Bundle2Items1 = load_bundled_items(tx_id(TX2, Opts), Opts), + Bundle2Items1 = load_items(tx_id(TX2, Opts), Opts), Bundle2Items2 = [ hb_message:with_commitments( #{ <<"commitment-device">> => <<"ans104@1.0">> }, @@ -375,17 +329,18 @@ load_bundled_items_test() -> %% L4IItem1 (leaf) %% L4IItem2 (leaf) bundler_optimistic_cache_test() -> + Wallet = ar_wallet:new(), L3Item = ar_bundles:sign_item( #tx{ data = <<"l3item">>, tags = [{<<"idx">>, <<"1">>}] }, - hb:wallet() + Wallet ), L4Item1 = ar_bundles:sign_item( #tx{ data = <<"l4item1">>, tags = [{<<"idx">>, <<"2.1">>}] }, - hb:wallet() + Wallet ), L4Item2 = ar_bundles:sign_item( #tx{ data = <<"l4item2">>, tags = [{<<"idx">>, <<"2.2">>}] }, - hb:wallet() + Wallet ), % L3Bundle is itself a bundle wrapping the two L4 leaves. {undefined, L3BundlePayload} = ar_bundles:serialize_bundle( @@ -399,7 +354,7 @@ bundler_optimistic_cache_test() -> {<<"idx">>, <<"2">>} ] }, - hb:wallet() + Wallet ), {undefined, L2BundlePayload} = ar_bundles:serialize_bundle( list, [L3Item, L3Bundle], false), @@ -411,7 +366,7 @@ bundler_optimistic_cache_test() -> {<<"Bundle-Version">>, <<"2.0.0">>} ] }, - hb:wallet() + Wallet ), % Compute signed IDs for all items before posting. L2BundleID = hb_util:encode(ar_bundles:id(L2Bundle, signed)), @@ -421,7 +376,7 @@ bundler_optimistic_cache_test() -> L4Item2ID = hb_util:encode(ar_bundles:id(L4Item2, signed)), % Start a real node with LMDB and POST the serialized bundle wrapper over HTTP. Node = hb_http_server:start_node(#{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet, store => hb_test_utils:test_store(hb_store_lmdb) }), try @@ -458,11 +413,7 @@ bundler_optimistic_cache_test() -> ), ok after - case hb_name:lookup(bundler_server) of - undefined -> ok; - PID -> PID ! stop, hb_name:unregister(bundler_server) - end, - dev_bundler_dispatch:stop_dispatcher() + dev_bundler:stop_server() end. new_data_item(Index, SizeOrData, Opts) -> diff --git a/src/dev_bundler_dispatch.erl b/src/dev_bundler_dispatch.erl deleted file mode 100644 index 2c286091c..000000000 --- a/src/dev_bundler_dispatch.erl +++ /dev/null @@ -1,1118 +0,0 @@ -%%% @doc A dispatcher for the bundler device (dev_bundler). This module -%%% manages a worker pool to handle bundle building, TX posting, proof -%%% generation, and chunk seeding. Failed tasks are automatically re-queued -%%% for immediate retry until successful. --module(dev_bundler_dispatch). --export([dispatch/2, ensure_dispatcher/1, stop_dispatcher/0]). --include("include/hb.hrl"). --include_lib("eunit/include/eunit.hrl"). - -%%% State record for the dispatcher process. --record(state, { - workers, % Map of WorkerPID => idle | {busy, Task} - task_queue, % Queue of pending tasks - bundles, % Map of BundleID => #bundle{} - opts % Configuration options -}). - -%%% Task record representing work to be done by a worker. --record(task, { - bundle_id, % ID of the bundle this task belongs to - type, % Task type: post_tx | build_proofs | post_proof - data, % Task-specific data (map) - opts, % Configuration options - retry_count = 0 % Number of times this task has been retried -}). - -%%% Proof record to track individual proof seeding status. --record(proof, { - proof, % The proof data (chunk, merkle path, etc) - status % pending | seeded -}). - -%%% Bundle record to track bundle progress through the dispatch pipeline. --record(bundle, { - id, % Unique bundle identifier - items, % List of dataitems to bundle - status, % Current state (initializing, tx_built, tx_posted, proofs_built) - tx, % The built/signed transaction - proofs, % Map of offset => #proof{} records - start_time % The time the bundle was started -}). - -%%% Default options. --define(DISPATCHER_NAME, bundler_dispatcher). --define(DEFAULT_NUM_WORKERS, 20). --define(DEFAULT_RETRY_BASE_DELAY_MS, 1000). --define(DEFAULT_RETRY_MAX_DELAY_MS, 600000). % 10 minutes --define(DEFAULT_RETRY_JITTER, 0.25). % ±25% jitter - -%% @doc Dispatch the queue. -dispatch([], _Opts) -> - ok; -dispatch(Items, Opts) -> - PID = ensure_dispatcher(Opts), - PID ! {dispatch, Items}. - -%% @doc Return the PID of the dispatch server. If the server is not running, -%% it is started and registered with the name `?SERVER_NAME'. -ensure_dispatcher(Opts) -> - case hb_name:lookup(?DISPATCHER_NAME) of - undefined -> - PID = spawn(fun() -> init(Opts) end), - ?event(bundler_short, {starting_dispatcher, {pid, PID}}), - hb_name:register(?DISPATCHER_NAME, PID), - hb_name:lookup(?DISPATCHER_NAME); - PID -> PID - end. - -stop_dispatcher() -> - case hb_name:lookup(?DISPATCHER_NAME) of - undefined -> ok; - PID -> - PID ! stop, - hb_name:unregister(?DISPATCHER_NAME) - end. - -get_state() -> - case hb_name:lookup(?DISPATCHER_NAME) of - undefined -> undefined; - PID -> - PID ! {get_state, self(), Ref = make_ref()}, - receive - {state, Ref, State} -> State - after 1000 -> timeout - end - end. - -%% @doc Initialize the dispatcher with worker pool. -init(Opts) -> - NumWorkers = hb_opts:get(bundler_workers, ?DEFAULT_NUM_WORKERS, Opts), - Workers = lists:map( - fun(_) -> - WorkerPID = spawn_link(fun() -> worker_loop() end), - {WorkerPID, idle} - end, - lists:seq(1, NumWorkers) - ), - State = #state{ - workers = maps:from_list(Workers), - task_queue = queue:new(), - bundles = #{}, - opts = Opts - }, - % Recover any in-progress bundles from cache - State1 = recover_bundles(State), - dispatcher(assign_tasks(State1)). - -%% @doc The main loop of the dispatcher. Manages task queue and worker pool. -dispatcher(State) -> - receive - {dispatch, Items} -> - % Create a new bundle and queue the post_tx task - Opts = State#state.opts, - BundleID = make_ref(), - Bundle = #bundle{ - id = BundleID, - items = Items, - status = initializing, - tx = undefined, - proofs = #{}, - start_time = erlang:timestamp() - }, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle, State#state.bundles) - }, - ?event(bundler_short, - {dispatching_bundle, - {timestamp, format_timestamp()}, - {bundle_id, BundleID}, - {num_items, length(Items)} - } - ), - Task = #task{bundle_id = BundleID, type = post_tx, data = Items, opts = Opts}, - State2 = enqueue_task(Task, State1), - % Assign tasks to idle workers - dispatcher(assign_tasks(State2)); - {task_complete, WorkerPID, Task, Result} -> - State1 = handle_task_complete(WorkerPID, Task, Result, State), - dispatcher(assign_tasks(State1)); - {task_failed, WorkerPID, Task, Reason} -> - State1 = handle_task_failed(WorkerPID, Task, Reason, State), - dispatcher(assign_tasks(State1)); - {retry_task, Task} -> - % Re-enqueue the task after backoff delay - State1 = enqueue_task(Task, State), - dispatcher(assign_tasks(State1)); - {get_state, From, Ref} -> - From ! {state, Ref, State}, - dispatcher(State); - stop -> - % Stop all workers - maps:foreach( - fun(WorkerPID, _) -> WorkerPID ! stop end, - State#state.workers - ), - exit(normal) - end. - -%% @doc Enqueue a task to the task queue. -enqueue_task(Task, State) -> - Queue = State#state.task_queue, - State#state{task_queue = queue:in(Task, Queue)}. - -%% @doc Format a task for logging. -format_task(#task{bundle_id = BundleID, type = post_tx, data = DataItems}) -> - {post_tx, {timestamp, format_timestamp()}, {bundle, BundleID}, - {num_items, length(DataItems)}}; -format_task(#task{bundle_id = BundleID, type = build_proofs, data = CommittedTX}) -> - {build_proofs, {timestamp, format_timestamp()}, {bundle, BundleID}, - {tx, {explicit, hb_message:id(CommittedTX, signed, #{})}}}; -format_task(#task{bundle_id = BundleID, type = post_proof, data = Proof}) -> - Offset = maps:get(offset, Proof), - {post_proof, {timestamp, format_timestamp()}, {bundle, BundleID}, - {offset, Offset}}. - -%% @doc Format erlang:timestamp() as a user-friendly RFC3339 string with milliseconds. -format_timestamp() -> - {MegaSecs, Secs, MicroSecs} = erlang:timestamp(), - Millisecs = (MegaSecs * 1000000 + Secs) * 1000 + (MicroSecs div 1000), - calendar:system_time_to_rfc3339(Millisecs, [{unit, millisecond}, {offset, "Z"}]). - -%% @doc Assign tasks to all idle workers until no idle workers -%% or no tasks remain. -assign_tasks(State) -> - IdleWorkers = maps:filter( - fun(_, Status) -> Status =:= idle end, - State#state.workers), - assign_tasks(maps:keys(IdleWorkers), State). - -assign_tasks([], State) -> - % No more idle workers - State; -assign_tasks([WorkerPID | Rest], State) -> - Workers = State#state.workers, - Queue = State#state.task_queue, - case queue:out(Queue) of - {{value, Task}, Queue1} -> - % Assign task to this worker - WorkerPID ! {execute_task, self(), Task}, - State1 = State#state{ - task_queue = Queue1, - workers = maps:put(WorkerPID, {busy, Task}, Workers) - }, - % Continue with remaining idle workers - assign_tasks(Rest, State1); - {empty, _} -> - % No more tasks, stop - State - end. - -handle_task_complete(WorkerPID, Task, Result, State) -> - Workers = State#state.workers, - Bundles = State#state.bundles, - #task{bundle_id = BundleID} = Task, - ?event(bundler_debug, {task_complete, format_task(Task)}), - % Update worker to idle - State1 = State#state{ - workers = maps:put(WorkerPID, idle, Workers) - }, - case maps:get(BundleID, Bundles, undefined) of - undefined -> - ?event(bundler_short, {bundle_not_found, BundleID}), - State1; - Bundle -> - task_completed(Task, Bundle, Result, State1) - end. - -handle_task_failed(WorkerPID, Task, Reason, State) -> - Workers = State#state.workers, - Opts = State#state.opts, - RetryCount = Task#task.retry_count, - % Calculate exponential backoff delay - BaseDelay = hb_opts:get(retry_base_delay_ms, ?DEFAULT_RETRY_BASE_DELAY_MS, Opts), - MaxDelay = hb_opts:get(retry_max_delay_ms, ?DEFAULT_RETRY_MAX_DELAY_MS, Opts), - Jitter = hb_opts:get(retry_jitter, ?DEFAULT_RETRY_JITTER, Opts), - % Compute base delay with exponential backoff: min(base * 2^retry_count, max_delay) - BaseDelayWithBackoff = min(BaseDelay * (1 bsl RetryCount), MaxDelay), - % Apply jitter: delay * (1 + random(-jitter, +jitter)) - % This distributes the delay across [delay * (1-jitter), delay * (1+jitter)] - JitterFactor = (rand:uniform() * 2 - 1) * Jitter, % Random value in [-jitter, +jitter] - Delay = round(BaseDelayWithBackoff * (1 + JitterFactor)), - ?event( - bundler_short, - {task_failed_retrying, format_task(Task), - {reason, {explicit, Reason}}, - {retry_count, RetryCount}, {delay_ms, Delay} - } - ), - % Update worker to idle - State1 = State#state{ - workers = maps:put(WorkerPID, idle, Workers) - }, - % Increment retry count and schedule delayed retry - Task1 = Task#task{retry_count = RetryCount + 1}, - erlang:send_after(Delay, self(), {retry_task, Task1}), - State1. - -task_completed(#task{bundle_id = BundleID, type = post_tx}, Bundle, CommittedTX, State) -> - Bundles = State#state.bundles, - Opts = State#state.opts, - Bundle1 = Bundle#bundle{status = tx_posted, tx = CommittedTX}, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle1, Bundles) - }, - BuildProofsTask = #task{ - bundle_id = BundleID, type = build_proofs, - data = CommittedTX, opts = Opts}, - enqueue_task(BuildProofsTask, State1); - -task_completed(#task{bundle_id = BundleID, type = build_proofs}, Bundle, Proofs, State) -> - Bundles = State#state.bundles, - Opts = State#state.opts, - case Proofs of - [] -> - % No proofs, bundle complete - bundle_complete(Bundle, State); - _ -> - % Proofs built, wrap each in a proof record with offset as key - ProofsMap = maps:from_list([ - {maps:get(offset, P), #proof{proof = P, status = pending}} || P <- Proofs - ]), - Bundle1 = Bundle#bundle{ - proofs = ProofsMap, - status = proofs_built - }, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle1, Bundles) - }, - % Enqueue all post_proof tasks - lists:foldl( - fun(ProofData, S) -> - ProofTask = #task{ - bundle_id = BundleID, - type = post_proof, - data = ProofData, - opts = Opts - }, - enqueue_task(ProofTask, S) - end, - State1, - Proofs - ) - end; - -task_completed(#task{bundle_id = BundleID, type = post_proof, data = ProofData}, Bundle, _Result, State) -> - Bundles = State#state.bundles, - Offset = maps:get(offset, ProofData), - Proofs = Bundle#bundle.proofs, - Proofs1 = maps:update_with( - Offset, - fun(P) -> P#proof{status = seeded} end, - Proofs - ), - Bundle1 = Bundle#bundle{proofs = Proofs1}, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle1, Bundles) - }, - % Check if all proofs are seeded - AllSeeded = lists:all( - fun(#proof{status = Status}) -> Status =:= seeded end, - maps:values(Proofs1) - ), - case AllSeeded of - true -> - bundle_complete(Bundle, State1); - false -> - State1 - end. - -%% @doc Mark a bundle as complete and remove it from state. -bundle_complete(Bundle, State) -> - Opts = State#state.opts, - ok = dev_bundler_cache:complete_tx(Bundle#bundle.tx, Opts), - ElapsedTime = - timer:now_diff(erlang:timestamp(), Bundle#bundle.start_time) / 1000000, - ?event(bundler_short, {bundle_complete, {bundle_id, Bundle#bundle.id}, - {timestamp, format_timestamp()}, - {tx, {explicit, hb_message:id(Bundle#bundle.tx, signed, Opts)}}, - {elapsed_time_s, ElapsedTime}}), - State#state{bundles = maps:remove(Bundle#bundle.id, State#state.bundles)}. - -%%% Recovery - -%% @doc Recover in-progress bundles from cache after a crash. -recover_bundles(State) -> - Opts = State#state.opts, - % Reconstruct bundles and enqueue appropriate tasks - lists:foldl( - fun({TXID, Status}, StateAcc) -> - recover_bundle(TXID, Status, StateAcc) - end, - State, - dev_bundler_cache:load_bundle_states(Opts) - ). - -%% @doc Recover a single bundle based on its cached state. -recover_bundle(TXID, Status, State) -> - Opts = State#state.opts, - ?event(bundler_short, {recovering_bundle, - {tx_id, {explicit, TXID}}, - {status, Status} - }), - try - % Load the TX and its items - CommittedTX = dev_bundler_cache:load_tx(TXID, Opts), - Items = dev_bundler_cache:load_bundled_items(TXID, Opts), - % Create a new bundle record - BundleID = make_ref(), - Bundle = #bundle{ - id = BundleID, - items = Items, - status = tx_posted, - tx = CommittedTX, - proofs = #{}, - start_time = erlang:timestamp() - }, - % Add bundle to state - Bundles = State#state.bundles, - State1 = State#state{ - bundles = maps:put(BundleID, Bundle, Bundles) - }, - - % Enqueue appropriate task based on status - Task = #task{ - bundle_id = BundleID, type = build_proofs, - data = CommittedTX, opts = Opts}, - enqueue_task(Task, State1) - catch - _:Error:Stack -> - ?event(bundler_short, {failed_to_recover_bundle, - {tx_id, {explicit, TXID}}, - {error, Error}, - {stack, Stack} - }), - % Skip this bundle and continue - State - end. - -%%% Worker implementation - -%% @doc Worker loop - executes tasks and reports back to dispatcher. -worker_loop() -> - receive - {execute_task, DispatcherPID, Task} -> - case execute_task(Task) of - {ok, Value} -> - DispatcherPID ! {task_complete, self(), Task, Value}; - {error, Reason} -> - DispatcherPID ! {task_failed, self(), Task, Reason} - end, - - worker_loop(); - stop -> - exit(normal) - end. - -%% @doc Execute a specific task. -execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> - try - ?event(bundler_debug, {execute_task, format_task(Task)}), - % Get price and anchor - {ok, TX} = dev_codec_tx:to(lists:reverse(Items), #{}, #{}), - DataSize = TX#tx.data_size, - PriceResult = get_price(DataSize, Opts), - AnchorResult = get_anchor(Opts), - case {PriceResult, AnchorResult} of - {{ok, Price}, {ok, Anchor}} -> - % Sign the TX - Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), - SignedTX = ar_tx:sign(TX#tx{ anchor = Anchor, reward = Price }, Wallet), - % Convert and post - Committed = hb_message:convert( - SignedTX, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, - Opts), - ?event(bundler_short, {posting_tx, - {tx, {explicit, hb_message:id(Committed, signed, Opts)}}}), - PostTXResponse = hb_ao:resolve( - #{ <<"device">> => <<"arweave@2.9">> }, - Committed#{ - <<"path">> => <<"/tx">>, - <<"method">> => <<"POST">> - }, - Opts - ), - case PostTXResponse of - {ok, _Result} -> - dev_bundler_cache:write_tx( - Committed, - Items, - Opts - ), - {ok, Committed}; - {_, ErrorReason} -> {error, ErrorReason} - end; - {PriceErr, AnchorErr} -> - ?event(bundle_short, {post_tx_failed, - format_task(Task), - {price, PriceErr}, - {anchor, AnchorErr}}), - {error, {PriceErr, AnchorErr}} - end - catch - _:Err:_Stack -> - ?event(bundle_short, {post_tx_failed, - format_task(Task), - {error, Err}}), - {error, Err} - end; - -execute_task(#task{type = build_proofs, data = CommittedTX, opts = Opts} = Task) -> - try - ?event(bundler_debug, {execute_task, format_task(Task)}), - % Calculate chunks and proofs - TX = hb_message:convert( - CommittedTX, <<"tx@1.0">>, <<"structured@1.0">>, Opts), - Data = TX#tx.data, - DataRoot = TX#tx.data_root, - DataSize = TX#tx.data_size, - Mode = ar_tx:chunking_mode(TX#tx.format), - Chunks = ar_tx:chunk_binary(Mode, ?DATA_CHUNK_SIZE, Data), - ?event(bundler_short, {building_proofs, - {bundle, Task#task.bundle_id}, - {data_size, DataSize}, - {num_chunks, length(Chunks)}}), - SizeTaggedChunks = ar_tx:chunks_to_size_tagged_chunks(Chunks), - SizeTaggedChunkIDs = ar_tx:sized_chunks_to_sized_chunk_ids(SizeTaggedChunks), - {_Root, DataTree} = ar_merkle:generate_tree(SizeTaggedChunkIDs), - % Build proof list - Proofs = lists:filtermap( - fun({Chunk, Offset}) -> - case Chunk of - <<>> -> false; - _ -> - DataPath = ar_merkle:generate_path( - DataRoot, Offset - 1, DataTree), - Proof = #{ - chunk => Chunk, - data_path => DataPath, - offset => Offset - 1, - data_size => DataSize, - data_root => DataRoot - }, - {true, Proof} - end - end, - SizeTaggedChunks - ), - % -1 because the `?event(...)' macro increments the counter by 1. - hb_event:increment(bundler_short, built_proofs, length(Proofs) - 1), - ?event( - bundler_short, - {built_proofs, - {bundle, Task#task.bundle_id}, - {num_proofs, length(Proofs)} - }, - Opts - ), - {ok, Proofs} - catch - _:Err:_Stack -> - ?event(bundler_short, {build_proofs_failed, - format_task(Task), - {error, Err}}), - {error, Err} - end; - -execute_task(#task{type = post_proof, data = Proof, opts = Opts} = Task) -> - #{chunk := Chunk, data_path := DataPath, offset := Offset, - data_size := DataSize, data_root := DataRoot} = Proof, - ?event(bundler_debug, {execute_task, format_task(Task)}), - Request = #{ - <<"chunk">> => hb_util:encode(Chunk), - <<"data_path">> => hb_util:encode(DataPath), - <<"offset">> => integer_to_binary(Offset), - <<"data_size">> => integer_to_binary(DataSize), - <<"data_root">> => hb_util:encode(DataRoot) - }, - try - Serialized = hb_json:encode(Request), - Response = dev_arweave:post_json_chunk(Serialized, Opts), - case Response of - {ok, _} -> {ok, proof_posted}; - {error, Reason} -> {error, Reason} - end - catch - _:Err:_Stack -> - ?event(bundler_short, {post_proof_failed, - format_task(Task), - {error, Err}}), - {error, Err} - end. - -get_price(DataSize, Opts) -> - hb_ao:resolve( - #{ <<"device">> => <<"arweave@2.9">> }, - #{ <<"path">> => <<"/price">>, <<"size">> => DataSize }, - Opts - ). - -get_anchor(Opts) -> - hb_ao:resolve( - #{ <<"device">> => <<"arweave@2.9">> }, - #{ <<"path">> => <<"/tx_anchor">> }, - Opts - ). - -%%%=================================================================== -%%% Tests -%%%=================================================================== - -complete_task_sequence_test() -> - Anchor = rand:bytes(32), - Price = 12345, - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)} - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 100, - retry_jitter => 0 - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts), new_data_item(2, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to be posted - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), - ?assertEqual(1, length(TXs)), - % Wait for chunk to be posted - Proofs = hb_mock_server:get_requests(chunk, 1, ServerHandle), - ?assertEqual(1, length(Proofs)), - % Verify dispatcher state - State = get_state(), - ?assertNotEqual(undefined, State), - ?assertNotEqual(timeout, State), - % All workers should be idle - Workers = State#state.workers, - IdleWorkers = [PID || {PID, Status} <- maps:to_list(Workers), Status =:= idle], - ?assertEqual(maps:size(Workers), length(IdleWorkers)), - % Task queue should be empty - Queue = State#state.task_queue, - ?assert(queue:is_empty(Queue)), - % Bundle should be completed and removed - Bundles = State#state.bundles, - ?assertEqual(0, maps:size(Bundles)), - ok - after - cleanup_dispatcher(ServerHandle) - end. - -post_tx_price_failure_retry_test() -> - Anchor = rand:bytes(32), - FailCount = 3, - setup_test_counter(price_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => fun(_Req) -> - Count = increment_test_counter(price_attempts_counter) - 1, - case Count < FailCount of - true -> {500, <<"error">>}; - false -> {200, <<"12345">>} - end - end, - tx_anchor => {200, hb_util:encode(Anchor)} - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 50, - retry_jitter => 0 - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to eventually be posted - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), - ?assertEqual(1, length(TXs)), - % Verify it retried multiple times - FinalCount = get_test_counter(price_attempts_counter), - ?assertEqual(FailCount+1, FinalCount), - ok - after - cleanup_test_counter(price_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -post_tx_anchor_failure_retry_test() -> - Price = 12345, - FailCount = 3, - setup_test_counter(anchor_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => fun(_Req) -> - Count = increment_test_counter(anchor_attempts_counter) - 1, - case Count < FailCount of - true -> {500, <<"error">>}; - false -> {200, hb_util:encode(rand:bytes(32))} - end - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 50, - retry_jitter => 0 - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to eventually be posted - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), - ?assertEqual(1, length(TXs)), - % Verify it retried multiple times - FinalCount = get_test_counter(anchor_attempts_counter), - ?assertEqual(FailCount+1, FinalCount), - ok - after - cleanup_test_counter(anchor_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -post_tx_post_failure_retry_test() -> - Anchor = rand:bytes(32), - Price = 12345, - FailCount = 4, - setup_test_counter(tx_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - Count = increment_test_counter(tx_attempts_counter) - 1, - case Count < FailCount of - true -> {400, <<"Transaction verification failed">>}; - false -> {200, <<"OK">>} - end - end - }), - try - % Use short retry delays for testing. - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 50, - retry_jitter => 0 % Disable jitter for deterministic tests - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to eventually succeed - TXs = hb_mock_server:get_requests(tx, FailCount+1, ServerHandle), - ?assertEqual(FailCount+1, length(TXs)), - % Verify final attempt succeeded - FinalCount = get_test_counter(tx_attempts_counter), - ?assertEqual(FailCount+1, FinalCount), - ok - after - cleanup_test_counter(tx_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -post_proof_failure_retry_test() -> - Anchor = rand:bytes(32), - Price = 12345, - FailCount = 2, - setup_test_counter(chunk_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - chunk => fun(_Req) -> - Count = increment_test_counter(chunk_attempts_counter) - 1, - case Count < FailCount of - true -> {500, <<"error">>}; - false -> {200, <<"OK">>} - end - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 50, - retry_jitter => 0 - }, - hb_http_server:start_node(Opts), - % Large enough for multiple chunks - Items = [new_data_item(1, floor(4.5 * ?DATA_CHUNK_SIZE), Opts)], - dispatch(Items, Opts), - % Wait for TX - TXs = hb_mock_server:get_requests(tx, 1, ServerHandle), - ?assertEqual(1, length(TXs)), - % Wait for chunks to eventually succeed - Chunks = hb_mock_server:get_requests(chunk, FailCount+5, ServerHandle), - ?assertEqual( FailCount+5, length(Chunks)), - % Verify retries happened - FinalCount = get_test_counter(chunk_attempts_counter), - ?assertEqual(FailCount+5, FinalCount), - ok - after - cleanup_test_counter(chunk_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -empty_dispatch_test() -> - Opts = #{}, - dispatch([], Opts), - % Should not crash - ok. - -rapid_dispatch_test() -> - Anchor = rand:bytes(32), - Price = 12345, - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - timer:sleep(100), - {200, <<"OK">>} - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - bundler_workers => 3 - }, - hb_http_server:start_node(Opts), - % Dispatch 10 bundles rapidly - lists:foreach( - fun(I) -> - Items = [new_data_item(I, 10, Opts)], - dispatch(Items, Opts) - end, - lists:seq(1, 10) - ), - - % Wait for all 10 TXs - TXs = hb_mock_server:get_requests(tx, 10, ServerHandle), - ?assertEqual(10, length(TXs)), - ok - after - cleanup_dispatcher(ServerHandle) - end. - -one_bundle_fails_others_continue_test() -> - Anchor = rand:bytes(32), - Price = 12345, - setup_test_counter(mixed_attempts_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - % First TX succeeds, all following attempts fail. - Count = increment_test_counter(mixed_attempts_counter) - 1, - case Count of - 0 -> {200, <<"OK">>}; - _ -> {400, <<"fail">>} - end - end - }), - try - % Use short retry delays for testing (100ms base, with exponential backoff) - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 100, - retry_jitter => 0 % Disable jitter for deterministic tests - }, - hb_http_server:start_node(Opts), - % Dispatch first bundle (will keep failing) - Items1 = [new_data_item(1, 10, Opts)], - dispatch(Items1, Opts), - % Dispatch second bundle (will succeed) - Items2 = [new_data_item(2, 10, Opts)], - dispatch(Items2, Opts), - % Wait for at least 5 TX attempts (1 success + multiple retries) - TXs = hb_mock_server:get_requests(tx, 5, ServerHandle), - ?assert(length(TXs) >= 5, length(TXs)), - ok - after - cleanup_test_counter(mixed_attempts_counter), - cleanup_dispatcher(ServerHandle) - end. - -parallel_task_execution_test() -> - Anchor = rand:bytes(32), - Price = 12345, - SleepTime = 120, - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - chunk => fun(_Req) -> - timer:sleep(SleepTime), - {200, <<"OK">>} - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - bundler_workers => 5 - }, - hb_http_server:start_node(Opts), - % Dispatch 3 bundles, each with 2 chunks - lists:foreach( - fun(I) -> - Items = [new_data_item(I, 10, Opts)], - dispatch(Items, Opts) - end, - lists:seq(1, 10) - ), - % With 3 workers and 1s delay, 10 chunks should complete in ~2s not 9s - StartTime = erlang:system_time(millisecond), - Chunks = hb_mock_server:get_requests(chunk, 10, ServerHandle), - ElapsedTime = erlang:system_time(millisecond) - StartTime, - ?assertEqual(10, length(Chunks)), - % Should take ~2-3 seconds with parallelism, not 9+ - ?assert(ElapsedTime < 2000, "ElapsedTime: " ++ integer_to_list(ElapsedTime)), - ok - after - cleanup_dispatcher(ServerHandle) - end. - -exponential_backoff_timing_test() -> - Anchor = rand:bytes(32), - Price = 12345, - FailCount = 5, - setup_test_counter(backoff_cap_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - Timestamp = erlang:system_time(millisecond), - Attempt = increment_test_counter(backoff_cap_counter), - Count = Attempt - 1, - % Store timestamp by attempt number. - add_test_attempt_timestamp(backoff_cap_counter, Attempt, Timestamp), - case Count < FailCount of - true -> {400, <<"fail">>}; - false -> {200, <<"OK">>} - end - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 100, - retry_max_delay_ms => 500, % Cap at 500ms - retry_jitter => 0 % Disable jitter for deterministic tests - }, - hb_http_server:start_node(Opts), - Items = [new_data_item(1, 10, Opts)], - dispatch(Items, Opts), - % Wait for TX to eventually succeed - TXs = hb_mock_server:get_requests(tx, FailCount+1, ServerHandle, 5000), - ?assertEqual(FailCount+1, length(TXs)), - % Verify backoff respects cap - Timestamps = test_attempt_timestamps(backoff_cap_counter), - ?assertEqual(6, length(Timestamps)), - [T1, T2, T3, T4, T5, T6] = Timestamps, - % Calculate actual delays - Delay1 = T2 - T1, - Delay2 = T3 - T2, - Delay3 = T4 - T3, - Delay4 = T5 - T4, - Delay5 = T6 - T5, - % Expected: ~100ms, ~200ms, ~400ms, ~500ms (capped), ~500ms (capped) - ?assert(Delay1 >= 70 andalso Delay1 =< 200, Delay1), - ?assert(Delay2 >= 150 andalso Delay2 =< 300, Delay2), - ?assert(Delay3 >= 300 andalso Delay3 =< 500, Delay3), - ?assert(Delay4 >= 400 andalso Delay4 =< 700, Delay4), - ?assert(Delay5 >= 400 andalso Delay5 =< 700, Delay5), - ok - after - cleanup_test_counter(backoff_cap_counter), - cleanup_dispatcher(ServerHandle) - end. - -independent_task_retry_counts_test() -> - Anchor = rand:bytes(32), - Price = 12345, - setup_test_counter(independent_retry_counter), - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)}, - tx => fun(_Req) -> - % Use request ordering to distinguish bundles - % First 3 requests are bundle1 (fail, fail, succeed) - % 4th request is bundle2 (succeed) - Count = increment_test_counter(independent_retry_counter) - 1, - case Count < 2 of - true -> {400, <<"fail">>}; % First 2 attempts fail - false -> {200, <<"OK">>} % Rest succeed - end - end - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store(), - retry_base_delay_ms => 100, - retry_jitter => 0 % Disable jitter for deterministic tests - }, - hb_http_server:start_node(Opts), - % Dispatch first bundle (will fail twice and retry) - Items1 = [new_data_item(1, 10, Opts)], - dispatch(Items1, Opts), - % Wait a bit for first bundle to start failing - hb_mock_server:get_requests(tx, 3, ServerHandle), - % Dispatch second bundle (will succeed on first try since we're past the 2 failures) - Items2 = [new_data_item(2, 10, Opts)], - dispatch(Items2, Opts), - % Verify we got all TX requests logged - TotalAttempts = 4, - TXs = hb_mock_server:get_requests(tx, TotalAttempts, ServerHandle), - ?assertEqual(TotalAttempts, length(TXs)), - ok - after - cleanup_test_counter(independent_retry_counter), - cleanup_dispatcher(ServerHandle) - end. - -recover_bundles_test() -> - Anchor = rand:bytes(32), - Price = 12345, - {ServerHandle, NodeOpts} = start_mock_gateway(#{ - price => {200, integer_to_binary(Price)}, - tx_anchor => {200, hb_util:encode(Anchor)} - }), - try - Opts = NodeOpts#{ - priv_wallet => hb:wallet(), - store => hb_test_utils:test_store() - }, - hb_http_server:start_node(Opts), - % Create some test items - Item1 = new_data_item(1, 10, Opts), - Item2 = new_data_item(2, 10, Opts), - Item3 = new_data_item(3, 10, Opts), - % Write items to cache as unbundled - ok = dev_bundler_cache:write_item(Item1, Opts), - ok = dev_bundler_cache:write_item(Item2, Opts), - ok = dev_bundler_cache:write_item(Item3, Opts), - % Create a bundle TX and cache it with posted status - {ok, TX} = dev_codec_tx:to(lists:reverse([Item1, Item2, Item3]), #{}, #{}), - CommittedTX = hb_message:convert(TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts), - ok = dev_bundler_cache:write_tx(CommittedTX, [Item1, Item2, Item3], Opts), - % Create a second bundle that is already complete (should not be recovered) - Item4 = new_data_item(4, 10, Opts), - ok = dev_bundler_cache:write_item(Item4, Opts), - {ok, TX2} = dev_codec_tx:to(lists:reverse([Item4]), #{}, #{}), - CommittedTX2 = hb_message:convert(TX2, <<"structured@1.0">>, <<"tx@1.0">>, Opts), - ok = dev_bundler_cache:write_tx(CommittedTX2, [Item4], Opts), - ok = dev_bundler_cache:complete_tx(CommittedTX2, Opts), - % Now initialize dispatcher which should recover only the posted bundle - ensure_dispatcher(Opts), - State = get_state(), - % Get the recovered bundle (should only be 1, not the completed one) - ?assertEqual(1, maps:size(State#state.bundles)), - [Bundle] = maps:values(State#state.bundles), - ?assertNotEqual(undefined, Bundle#bundle.start_time), - ?assertEqual(#{}, Bundle#bundle.proofs), - RecoveredItems = [ - hb_message:with_commitments( - #{ <<"commitment-device">> => <<"ans104@1.0">> }, Item, Opts) - || Item <- Bundle#bundle.items], - ?assertEqual( - lists:sort([Item1, Item2, Item3]), - lists:sort(RecoveredItems)), - ?assertEqual(tx_posted, Bundle#bundle.status), - ?assert(hb_message:verify(Bundle#bundle.tx)), - ?assertEqual( - hb_message:id(CommittedTX, signed, Opts), - hb_message:id(Bundle#bundle.tx, signed, Opts)), - ok - after - cleanup_dispatcher(ServerHandle) - end. - -%%% Test Helper Functions - -new_data_item(Index, Size, Opts) -> - Data = rand:bytes(Size), - Tag = <<"tag", (integer_to_binary(Index))/binary>>, - Value = <<"value", (integer_to_binary(Index))/binary>>, - Item = ar_bundles:sign_item( - #tx{ - data = Data, - tags = [{Tag, Value}] - }, - hb:wallet() - ), - hb_message:convert(Item, <<"structured@1.0">>, <<"ans104@1.0">>, Opts). - -start_mock_gateway(Responses) -> - DefaultResponse = {200, <<>>}, - Endpoints = [ - {"/chunk", chunk, maps:get(chunk, Responses, DefaultResponse)}, - {"/tx", tx, maps:get(tx, Responses, DefaultResponse)}, - {"/price/:size", price, maps:get(price, Responses, DefaultResponse)}, - {"/tx_anchor", tx_anchor, maps:get(tx_anchor, Responses, DefaultResponse)} - ], - {ok, MockServer, ServerHandle} = hb_mock_server:start(Endpoints), - NodeOpts = #{ - gateway => MockServer, - routes => [ - #{ - <<"template">> => <<"/arweave">>, - <<"node">> => #{ - <<"match">> => <<"^/arweave">>, - <<"with">> => MockServer, - <<"opts">> => #{http_client => httpc, protocol => http2} - } - } - ] - }, - {ServerHandle, NodeOpts}. - -cleanup_dispatcher(ServerHandle) -> - stop_dispatcher(), - timer:sleep(10), % Ensure dispatcher fully stops - hb_mock_server:stop(ServerHandle). - -setup_test_counter(Table) -> - cleanup_test_counter(Table), - ets:new(Table, [named_table, public, set]), - ok. - -cleanup_test_counter(Table) -> - case ets:info(Table) of - undefined -> ok; - _ -> ets:delete(Table), ok - end. - -increment_test_counter(Table) -> - ets:update_counter(Table, Table, {2, 1}, {Table, 0}). - -get_test_counter(Table) -> - case ets:lookup(Table, Table) of - [{_, Value}] -> Value; - [] -> 0 - end. - -add_test_attempt_timestamp(Table, Attempt, Timestamp) -> - ets:insert(Table, {{Table, Attempt}, Timestamp}). - -test_attempt_timestamps(Table) -> - TimestampEntries = [ - {Attempt, Timestamp} - || {{Prefix1, Attempt}, Timestamp} <- ets:tab2list(Table), - Prefix1 =:= Table - ], - [Timestamp || {_, Timestamp} <- lists:sort(TimestampEntries)]. diff --git a/src/dev_bundler_recovery.erl b/src/dev_bundler_recovery.erl new file mode 100644 index 000000000..915b734bd --- /dev/null +++ b/src/dev_bundler_recovery.erl @@ -0,0 +1,278 @@ +%%% @doc Logic for handling bundler recocery on node restart. +%%% +%%% When a bundler is running it will cache the state of each uploaded item +%%% or bundle as it move through the bundling and upload process. If the node +%%% is restarted before it can finish including all uploaded items in a bundle, +%%% or finish seeding all bundles in process, the recovery process will ensure +%%% that the data in process is recovered and resumed. +-module(dev_bundler_recovery). +-export([ + recover_unbundled_items/2, + recover_bundles/2 +]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Spawn a process to recover unbundled items. +recover_unbundled_items(ServerPID, Opts) -> + spawn(fun() -> do_recover_unbundled_items(ServerPID, Opts) end). + +%% @doc Spawn a process to recover in-progress bundles. +recover_bundles(ServerPID, Opts) -> + spawn(fun() -> do_recover_bundles(ServerPID, Opts) end). + +do_recover_unbundled_items(ServerPID, Opts) -> + try + ?event(bundler_short, {recover_unbundled_items_start}), + UnbundledItems = dev_bundler_cache:load_items( + <<>>, + Opts, + fun(ItemID, Item) -> + ?event( + bundler_short, + {recovered_unbundled_item, + {id, {string, ItemID}} + } + ), + ServerPID ! {enqueue_item, Item} + end, + fun(ItemID) -> + ?event( + bundler_short, + {failed_to_recover_unbundled_item, + {id, {string, ItemID}} + } + ) + end + ), + ?event(bundler_short, {recover_unbundled_items_complete, + {count, length(UnbundledItems)}}), + ok + catch + _:Error:Stack -> + ?event( + error, + {recover_unbundled_items_failed, + {error, Error}, + {stack, Stack} + }, + Opts + ) + end. + +do_recover_bundles(ServerPID, Opts) -> + try + BundleStates = dev_bundler_cache:load_bundle_states(Opts), + ?event(bundler_short, {recover_bundles_start, + {count, length(BundleStates)}}), + lists:foreach( + fun({TXID, Status}) -> + recover_bundle(ServerPID, TXID, Status, Opts) + end, + BundleStates + ), + ?event(bundler_short, {recover_bundles_complete, + {count, length(BundleStates)}}), + ok + catch + _:Error:Stack -> + ?event( + error, + {recover_bundles_failed, + {error, Error}, + {stack, Stack} + }, + Opts + ) + end. + +recover_bundle(ServerPID, TXID, Status, Opts) -> + ?event( + bundler_short, + {recovering_bundle, + {tx_id, {explicit, TXID}}, + {status, Status} + } + ), + try + CommittedTX = dev_bundler_cache:load_tx(TXID, Opts), + case CommittedTX of + not_found -> + throw(tx_not_found); + _ -> + Items = dev_bundler_cache:load_items( + TXID, + Opts, + fun(ItemID, _Item) -> + ?event( + debug_bundler, + {loaded_bundle_item, + {tx_id, {explicit, TXID}}, + {item_id, {explicit, ItemID}} + } + ) + end, + fun(ItemID) -> + ?event( + error, + {failed_to_load_bundle_item, + {tx_id, {explicit, TXID}}, + {item_id, {explicit, ItemID}} + }, + Opts + ), + throw({failed_to_load_bundle_item, ItemID}) + end + ), + ServerPID ! {recover_bundle, CommittedTX, Items} + end + catch + _:Error:Stack -> + ?event( + error, + {failed_to_recover_bundle, + {tx_id, {explicit, TXID}}, + {error, Error}, + {stack, Stack} + }, + Opts + ) + end. + +%%%=================================================================== +%%% Tests +%%%=================================================================== + +recover_unbundled_items_test() -> + Opts = #{store => hb_test_utils:test_store()}, + Item1 = new_data_item(1, 10, Opts), + Item2 = new_data_item(2, 10, Opts), + Item3 = new_data_item(3, 10, Opts), + ok = dev_bundler_cache:write_item(Item1, Opts), + ok = dev_bundler_cache:write_item(Item2, Opts), + ok = dev_bundler_cache:write_item(Item3, Opts), + FakeTX = new_bundle_tx([Item2], Opts), + ok = dev_bundler_cache:write_tx(FakeTX, [Item2], Opts), + recover_unbundled_items(self(), Opts), + RecoveredItems = receive_enqueue_items(2), + RecoveredItems1 = normalize_items(RecoveredItems, Opts), + ?assertEqual( + lists:sort([Item1, Item3]), + lists:sort(RecoveredItems1) + ). + +recover_bundles_skips_complete_test() -> + Opts = #{store => hb_test_utils:test_store()}, + Item1 = new_data_item(1, 10, Opts), + Item2 = new_data_item(2, 10, Opts), + Item3 = new_data_item(3, 10, Opts), + ok = dev_bundler_cache:write_item(Item1, Opts), + ok = dev_bundler_cache:write_item(Item2, Opts), + ok = dev_bundler_cache:write_item(Item3, Opts), + PostedTX = new_bundle_tx([Item1, Item2], Opts), + CompletedTX = new_bundle_tx([Item3], Opts), + ok = dev_bundler_cache:write_tx(PostedTX, [Item1, Item2], Opts), + ok = dev_bundler_cache:write_tx(CompletedTX, [Item3], Opts), + ok = dev_bundler_cache:complete_tx(CompletedTX, Opts), + recover_bundles(self(), Opts), + {RecoveredTX, RecoveredItems} = receive_recovered_bundle(), + RecoveredItems1 = normalize_items(RecoveredItems, Opts), + ?assertEqual( + hb_message:id(PostedTX, signed, Opts), + hb_message:id(RecoveredTX, signed, Opts) + ), + ?assertEqual( + lists:sort([Item1, Item2]), + lists:sort(RecoveredItems1) + ), + receive + {recover_bundle, _, _} -> + erlang:error(unexpected_second_recovered_bundle) + after 200 -> + ok + end. + +recover_bundles_failed_bundle_items_continue_test() -> + Opts = #{ + store => hb_test_utils:test_store(), + debug_print => false + }, + ValidItem = new_data_item(1, 10, Opts), + ok = dev_bundler_cache:write_item(ValidItem, Opts), + ValidTX = new_bundle_tx([ValidItem], Opts), + ok = dev_bundler_cache:write_tx(ValidTX, [ValidItem], Opts), + BrokenTX = new_bundle_tx([], Opts), + ok = dev_bundler_cache:write_tx(BrokenTX, [], Opts), + MissingItemID = <<"missing-item">>, + ok = write_missing_item_bundle(MissingItemID, BrokenTX, Opts), + recover_bundles(self(), Opts), + {RecoveredTX, RecoveredItems} = receive_recovered_bundle(), + RecoveredItems1 = normalize_items(RecoveredItems, Opts), + ?assertEqual( + hb_message:id(ValidTX, signed, Opts), + hb_message:id(RecoveredTX, signed, Opts) + ), + ?assertEqual([ValidItem], RecoveredItems1), + receive + {recover_bundle, _, _} -> + erlang:error(unexpected_broken_bundle_recovered) + after 200 -> + ok + end. + +receive_enqueue_items(Count) -> + receive_enqueue_items(Count, []). + +receive_enqueue_items(0, Items) -> + lists:reverse(Items); +receive_enqueue_items(Count, Items) -> + receive + {enqueue_item, Item} -> + receive_enqueue_items(Count - 1, [Item | Items]) + after 1000 -> + erlang:error({missing_enqueue_items, Count}) + end. + +receive_recovered_bundle() -> + receive + {recover_bundle, CommittedTX, Items} -> + {CommittedTX, Items} + after 1000 -> + erlang:error(missing_recovered_bundle) + end. + +normalize_items(Items, Opts) -> + [ + hb_message:with_commitments( + #{ <<"commitment-device">> => <<"ans104@1.0">> }, + Item, + Opts + ) + || Item <- Items + ]. + +write_missing_item_bundle(ItemID, TX, Opts) -> + Store = hb_opts:get(store, no_viable_store, Opts), + Path = hb_store:path(Store, [ + <<"~bundler@1.0">>, + <<"item">>, + ItemID, + <<"bundle">> + ]), + hb_store:write(Store, Path, hb_message:id(TX, signed, Opts)). + +new_data_item(Index, Size, Opts) -> + Tag = <<"tag", (integer_to_binary(Index))/binary>>, + Value = <<"value", (integer_to_binary(Index))/binary>>, + Item = ar_bundles:sign_item( + #tx{ + data = rand:bytes(Size), + tags = [{Tag, Value}] + }, + hb:wallet() + ), + hb_message:convert(Item, <<"structured@1.0">>, <<"ans104@1.0">>, Opts). + +new_bundle_tx(Items, Opts) -> + TX = dev_bundler_task:data_items_to_tx(lists:reverse(Items), Opts), + hb_message:convert(TX, <<"structured@1.0">>, <<"tx@1.0">>, Opts). diff --git a/src/dev_bundler_task.erl b/src/dev_bundler_task.erl new file mode 100644 index 000000000..93ee6e39b --- /dev/null +++ b/src/dev_bundler_task.erl @@ -0,0 +1,378 @@ +%%% @doc Implements the different bundling primitives: +%%% - post_tx: Building and posting an L1 transaction +%%% - build_proofs:Chunking up the bundle data and building the chunk proofs +%%% - post_proof: Seeding teh chunks to the Arweave network +-module(dev_bundler_task). +-export([worker_loop/0, log_task/3, format_timestamp/0]). +%%% Test-only exports. +-export([data_items_to_tx/2]). +-include("include/hb.hrl"). +-include("include/dev_bundler.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% @doc Worker loop - executes tasks and reports back to dispatcher. +worker_loop() -> + receive + {execute_task, DispatcherPID, Task} -> + case execute_task(Task) of + {ok, Value} -> + DispatcherPID ! {task_complete, self(), Task, Value}; + {error, Reason} -> + DispatcherPID ! {task_failed, self(), Task, Reason} + end, + + worker_loop(); + stop -> + exit(normal) + end. + +%% @doc Execute a specific task. +execute_task(#task{type = post_tx, data = Items, opts = Opts} = Task) -> + try + ?event(debug_bundler, log_task(executing_task, Task, [])), + case build_signed_tx(Items, Opts) of + {ok, SignedTX} -> + Committed = hb_message:convert( + SignedTX, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, + Opts), + ?event(bundler_short, log_task(posting_tx, + Task, + [{tx, {explicit, hb_message:id(Committed, signed, Opts)}}] + )), + PostTXResponse = dev_arweave:post_tx_header( + SignedTX, + Opts + ), + case PostTXResponse of + {ok, _Result} -> + dev_bundler_cache:write_tx( + Committed, + Items, + Opts + ), + {ok, Committed}; + {_, ErrorReason} -> {error, ErrorReason} + end; + {error, {PriceErr, AnchorErr}} -> + ?event(bundler_short, + log_task(task_failed, Task, [ + {price, PriceErr}, + {anchor, AnchorErr} + ])), + {error, {PriceErr, AnchorErr}} + end + catch + _:Err:Stack -> + ?event(bundler_short, log_task(task_failed, Task, [{error, Err}])), + ?event(bundler_upload_error, + log_task(task_failed, Task, [{error, Err}, {trace, Stack}])), + {error, Err} + end; + +execute_task(#task{type = build_proofs, data = CommittedTX, opts = Opts} = Task) -> + try + ?event(debug_bundler, log_task(executing_task, Task, [])), + % Calculate chunks and proofs + TX = hb_message:convert( + CommittedTX, <<"tx@1.0">>, <<"structured@1.0">>, Opts), + Data = TX#tx.data, + DataRoot = TX#tx.data_root, + DataSize = TX#tx.data_size, + Mode = ar_tx:chunking_mode(TX#tx.format), + Chunks = ar_tx:chunk_binary(Mode, ?DATA_CHUNK_SIZE, Data), + ?event(bundler_short, {building_proofs, + {bundle, Task#task.bundle_id}, + {data_size, DataSize}, + {num_chunks, length(Chunks)}}), + SizeTaggedChunks = ar_tx:chunks_to_size_tagged_chunks(Chunks), + SizeTaggedChunkIDs = ar_tx:sized_chunks_to_sized_chunk_ids(SizeTaggedChunks), + {_Root, DataTree} = ar_merkle:generate_tree(SizeTaggedChunkIDs), + % Build proof list + Proofs = lists:filtermap( + fun({Chunk, Offset}) -> + case Chunk of + <<>> -> false; + _ -> + DataPath = ar_merkle:generate_path( + DataRoot, Offset - 1, DataTree), + Proof = #{ + chunk => Chunk, + data_path => DataPath, + offset => Offset - 1, + data_size => DataSize, + data_root => DataRoot + }, + {true, Proof} + end + end, + SizeTaggedChunks + ), + % -1 because the `?event(...)' macro increments the counter by 1. + hb_event:increment(bundler_short, built_proofs, length(Proofs) - 1), + ?event( + bundler_short, + {built_proofs, + {bundle, Task#task.bundle_id}, + {num_proofs, length(Proofs)} + }, + Opts + ), + {ok, Proofs} + catch + _:Err:_Stack -> + ?event(bundler_short, log_task(task_failed, Task, [{error, Err}])), + {error, Err} + end; + +execute_task(#task{type = post_proof, data = Proof, opts = Opts} = Task) -> + #{chunk := Chunk, data_path := DataPath, offset := Offset, + data_size := DataSize, data_root := DataRoot} = Proof, + ?event(debug_bundler, log_task(executing_task, Task, [])), + Request = #{ + <<"chunk">> => hb_util:encode(Chunk), + <<"data_path">> => hb_util:encode(DataPath), + <<"offset">> => integer_to_binary(Offset), + <<"data_size">> => integer_to_binary(DataSize), + <<"data_root">> => hb_util:encode(DataRoot) + }, + try + Serialized = hb_json:encode(Request), + Response = dev_arweave:post_json_chunk(Serialized, Opts), + case Response of + {ok, _} -> {ok, proof_posted}; + {error, Reason} -> {error, Reason} + end + catch + _:Err:_Stack -> + ?event(bundler_short, log_task(task_failed, Task, [{error, Err}])), + {error, Err} + end. + +%% @doc Build and sign a bundle TX without posting it. +build_signed_tx(Items, Opts) -> + TX = data_items_to_tx(Items, Opts), + DataSize = TX#tx.data_size, + PriceResult = get_price(DataSize, Opts), + AnchorResult = get_anchor(Opts), + case {PriceResult, AnchorResult} of + {{ok, Price}, {ok, Anchor}} -> + Wallet = hb_opts:get(priv_wallet, no_viable_wallet, Opts), + SignedTX = + dev_arweave_common:normalize( + ar_tx:sign( + TX#tx{anchor = Anchor, reward = Price}, + Wallet + ) + ), + {ok, SignedTX}; + {PriceErr, AnchorErr} -> + {error, {PriceErr, AnchorErr}} + end. + +data_items_to_tx(Items, Opts) -> + List = lists:map( + fun(Item) -> + hb_message:convert( + Item, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + <<"structured@1.0">>, + Opts + ) + end, + lists:reverse(Items)), + dev_arweave_common:normalize(#tx{ + format = 2, + data = List + }). + +get_price(DataSize, Opts) -> + hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ <<"path">> => <<"/price">>, <<"size">> => DataSize }, + Opts + ). + +get_anchor(Opts) -> + hb_ao:resolve( + #{ <<"device">> => <<"arweave@2.9">> }, + #{ <<"path">> => <<"/tx_anchor">> }, + Opts + ). + +%%%=================================================================== +%%% Logging +%%%=================================================================== + +%% @doc Return a complete task event tuple for logging. +log_task(Event, Task, ExtraLogs) -> + erlang:list_to_tuple([Event | format_task(Task) ++ ExtraLogs]). + +%% @doc Format a task for logging. +format_task(#task{bundle_id = BundleID, type = post_tx, data = DataItems}) -> + [ + {task_type, post_tx}, + {timestamp, format_timestamp()}, + {bundle, BundleID}, + {num_items, length(DataItems)} + ]; +format_task(#task{bundle_id = BundleID, type = build_proofs, data = CommittedTX}) -> + [ + {task_type, build_proofs}, + {timestamp, format_timestamp()}, + {bundle, BundleID}, + {tx, {explicit, hb_message:id(CommittedTX, signed, #{})}} + ]; +format_task(#task{bundle_id = BundleID, type = post_proof, data = Proof}) -> + Offset = maps:get(offset, Proof), + [ + {task_type, post_proof}, + {timestamp, format_timestamp()}, + {bundle, BundleID}, + {offset, Offset} + ]. + +%% @doc Format erlang:timestamp() as a user-friendly RFC3339 string with milliseconds. +format_timestamp() -> + {MegaSecs, Secs, MicroSecs} = erlang:timestamp(), + Millisecs = (MegaSecs * 1000000 + Secs) * 1000 + (MicroSecs div 1000), + calendar:system_time_to_rfc3339(Millisecs, [{unit, millisecond}, {offset, "Z"}]). + +build_signed_tx_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = dev_bundler:start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + TestOpts = NodeOpts#{ + priv_wallet => ar_wallet:new(), + store => hb_test_utils:test_store() + }, + try + Timestamp = 12344567, + ListValue = [<<"a">>, <<"b">>, <<"c">>], + StructuredItems = [ + #{ + <<"body">> => <<"body1">>, + <<"tag1">> => <<"value1">>, + <<"timestamp">> => Timestamp + }, + #{ + <<"body">> => <<"body3">>, + <<"tag3">> => <<"value3">>, + <<"list">> => ListValue + }, + #{ + <<"body">> => <<"body2">>, + <<"tag2">> => <<"value2">> + } + ], + Items = [ + hb_message:commit( + Item, + TestOpts, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true } + ) + || Item <- StructuredItems], + {ok, SignedTX} = build_signed_tx(Items, TestOpts), + ?assert(ar_tx:verify(SignedTX)), + ?assertEqual(Anchor, SignedTX#tx.anchor), + ?assertEqual(Price, SignedTX#tx.reward), + ?event(debug_test, {signed_tx, SignedTX}), + BundledTX = ar_bundles:deserialize(SignedTX), + ?event(debug_test, {bundled_tx, BundledTX}), + BundledItems = hb_util:numbered_keys_to_list(BundledTX#tx.data, #{}), + lists:foreach( + fun(Item) -> + ?assert(ar_bundles:verify_item(Item)) + end, + BundledItems + ), + BundledStructuredItems = [ + hb_message:convert( + Item, + <<"structured@1.0">>, + <<"ans104@1.0">>, + TestOpts + ) + || Item <- BundledItems], + ?assertEqual(lists:reverse(Items), BundledStructuredItems), + ok + after + hb_mock_server:stop(ServerHandle) + end. + +build_signed_tx_on_arbundles_js_test() -> + Anchor = rand:bytes(32), + Price = 12345, + {ServerHandle, NodeOpts} = dev_bundler:start_mock_gateway(#{ + price => {200, integer_to_binary(Price)}, + tx_anchor => {200, hb_util:encode(Anchor)} + }), + TestOpts = NodeOpts#{ + priv_wallet => hb:wallet(), + store => hb_test_utils:test_store() + }, + try + % Load an arweave.js-created dataitem + Item = ar_bundles:deserialize( + hb_util:ok( + file:read_file(<<"test/arbundles.js/ans104-item.bundle">>) + ) + ), + ?event(debug_test, {item, Item}), + ?assert(ar_bundles:verify_item(Item)), + % Load an arweave.js-created list bundle + {ok, Bin} = file:read_file(<<"test/arbundles.js/ans104-list-bundle.bundle">>), + BundledItem = ar_bundles:sign_item(#tx{ + format = ans104, + data = Bin, + data_size = byte_size(Bin), + tags = [ + {<<"Bundle-Format">>, <<"binary">>}, + {<<"Bundle-Version">>, <<"2.0.0">>} + ] + }, hb:wallet()), + ?event(debug_test, {bundled_item, BundledItem}), + ?assert(ar_bundles:verify_item(BundledItem)), + % Convert both dataitems to structured messages + ItemStructured = hb_message:convert(Item, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + TestOpts), + ?event(debug_test, {item_structured, ItemStructured}), + ?assert(hb_message:verify(ItemStructured, all, TestOpts)), + BundledItemStructured = hb_message:convert(BundledItem, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, + TestOpts), + ?event(debug_test, {bundled_item_structured, BundledItemStructured}), + ?assert(hb_message:verify(BundledItemStructured, all, TestOpts)), + % Use build_signed_tx/2 to mimic the bundler worker logic. + {ok, SignedTX} = build_signed_tx( + [ItemStructured, BundledItemStructured], + TestOpts + ), + ?event(debug_test, {signed_tx, SignedTX}), + ?assert(ar_tx:verify(SignedTX)), + % Convert the signed TX to a structured message + StructuredTX = hb_message:convert(SignedTX, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, + TestOpts), + % ?event(debug_test, {structured_tx, StructuredTX}), + ?assert(hb_message:verify(StructuredTX, all, TestOpts)), + % Convert back to an L1 TX + SignedTXRoundtrip = hb_message:convert(StructuredTX, + #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, + #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + TestOpts), + ?event(debug_test, {signed_tx_roundtrip, SignedTXRoundtrip}), + ?assert(ar_tx:verify(SignedTXRoundtrip)), + ?assertEqual(SignedTX, SignedTXRoundtrip), + ok + after + hb_mock_server:stop(ServerHandle) + end. \ No newline at end of file diff --git a/src/dev_codec_tx.erl b/src/dev_codec_tx.erl index 46f91bc93..9ac8750a4 100644 --- a/src/dev_codec_tx.erl +++ b/src/dev_codec_tx.erl @@ -147,18 +147,6 @@ to(RawTABM, Req, Opts) when is_map(RawTABM) -> enforce_valid_tx(FinalTX), ?event({to_result, FinalTX}), {ok, FinalTX}; -%% @doc List of ans104 items is bundled into a single L1 transaction. -to(RawList, Req, Opts) when is_list(RawList) -> - List = lists:map( - fun(Item) -> hb_util:ok(dev_codec_ans104:to(Item, Req, Opts)) end, - RawList), - TX = #tx{ - format = 2, - data = List - }, - Bundle = dev_arweave_common:normalize(TX), - ?event({to_result, Bundle}), - {ok, Bundle}; to(Other, _Req, _Opts) -> throw({invalid_tx, Other}). @@ -1521,61 +1509,28 @@ test_bundle_uncommitted(Encode, Decode) -> end, ok. -bundle_list_test() -> - % Load an arweave.js-created dataitem - Item = ar_bundles:deserialize( - hb_util:ok( - file:read_file(<<"test/arbundles.js/ans104-item.bundle">>) - ) - ), - ?event(debug_test, {item, Item}), - ?assert(ar_bundles:verify_item(Item)), - % Load an arweave.js-created list bundle - {ok, Bin} = file:read_file(<<"test/arbundles.js/ans104-list-bundle.bundle">>), - BundledItem = ar_bundles:sign_item(#tx{ - format = ans104, - data = Bin, - data_size = byte_size(Bin), - tags = [ - {<<"Bundle-Format">>, <<"binary">>}, - {<<"Bundle-Version">>, <<"2.0.0">>} - ] - }, hb:wallet()), - ?event(debug_test, {bundled_item, BundledItem}), - ?assert(ar_bundles:verify_item(BundledItem)), - % Convert both dataitems to structured messages - ItemStructured = hb_message:convert(Item, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, - #{}), - ?event(debug_test, {item_structured, ItemStructured}), - ?assert(hb_message:verify(ItemStructured, all, #{})), - BundledItemStructured = hb_message:convert(BundledItem, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => true }, - #{}), - ?event(debug_test, {bundled_item_structured, BundledItemStructured}), - ?assert(hb_message:verify(BundledItemStructured, all, #{})), - % Use dev_codec_tx:to(List) to create a L1 TX bundle. We use this - % interface to mimic the logic used in dev_bundler - {ok, BundledTX} = dev_codec_tx:to( - [ItemStructured, BundledItemStructured], #{}, #{}), - SignedTX = ar_tx:sign(BundledTX, hb:wallet()), - ?event(debug_test, {signed_tx, SignedTX}), - ?assert(ar_tx:verify(SignedTX)), - % Convert the signed TX to a structured message - StructuredTX = hb_message:convert(SignedTX, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, +%% Disabled test that captures an issue we will face if we want to support +%% ao-types on tx's. +list_aotypes_test_disabled() -> + Items = [ + #{ <<"tag1">> => <<"value1">> }, + #{ <<"tag2">> => <<"value2">> }, + #{ <<"tag3">> => <<"value3">> } + ], + TX = hb_message:convert( + Items, #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, + <<"structured@1.0">>, #{}), - ?event(debug_test, {structured_tx, StructuredTX}), - ?assert(hb_message:verify(StructuredTX, all, #{})), - % Convert back to an L1 TX - SignedTXRoundtrip = hb_message:convert(StructuredTX, - #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true }, - #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true }, + Anchor = crypto:strong_rand_bytes(32), + % SignedTX = ar_tx:sign( + % TX#tx{ anchor = Anchor, reward = 100 }, + % hb:wallet()), + % ?event(debug_test, {signed_tx, SignedTX}), + StructuredTX = hb_message:convert( + TX#tx{ anchor = Anchor, reward = 100 }, + <<"structured@1.0">>, + <<"tx@1.0">>, #{}), - ?event(debug_test, {signed_tx_roundtrip, SignedTXRoundtrip}), - ?assert(ar_tx:verify(SignedTXRoundtrip)), - ?assertEqual(SignedTX, SignedTXRoundtrip), + ?event(debug_test, {structured_tx, StructuredTX}), ok. \ No newline at end of file diff --git a/src/dev_copycat_arweave.erl b/src/dev_copycat_arweave.erl index c48e51af1..970b3d9b4 100644 --- a/src/dev_copycat_arweave.erl +++ b/src/dev_copycat_arweave.erl @@ -28,22 +28,24 @@ arweave(_Base, Request, Opts) -> parse_range(Request, Opts) -> From = case hb_maps:find(<<"from">>, Request, Opts) of - {ok, Height} -> - RequestedFrom = hb_util:int(Height), - case RequestedFrom < 0 of - true -> latest_height(Opts) + RequestedFrom; - false -> RequestedFrom - end; + {ok, FromHeight} -> normalize_height(FromHeight, Opts); error -> latest_height(Opts) end, To = case hb_maps:find(<<"to">>, Request, Opts) of - {ok, ToHeight} -> hb_util:int(ToHeight); + {ok, ToHeight} -> normalize_height(ToHeight, Opts); error -> undefined end, {From, To}. +normalize_height(Height, Opts) -> + RequestedHeight = hb_util:int(Height), + case RequestedHeight < 0 of + true -> latest_height(Opts) + RequestedHeight; + false -> RequestedHeight + end. + latest_height(Opts) -> case hb_ao:resolve( <>, @@ -113,7 +115,7 @@ list_index_blocks(Current, To, Opts, Acc) -> end. fetch_block_header(Height, Opts) -> - ?event(copycat_debug, {fetching_block, Height}), + ?event(debug_copycat, {fetching_block, Height}), observe_event(<<"block_header">>, fun() -> hb_ao:resolve( << @@ -190,7 +192,7 @@ fetch_and_process_block(Current, To, Opts) -> process_block(BlockRes, Current, To, Opts) -> case BlockRes of {ok, Block} -> - ?event(copycat_debug, {{processing_block, Current}, + ?event(debug_copycat, {{processing_block, Current}, {indep_hash, hb_maps:get(<<"indep_hash">>, Block, <<>>)}}), case maybe_index_ids(Block, Opts) of {block_skipped, Results} -> @@ -284,7 +286,7 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> TXID = hb_util:encode(TX#tx.id), TXEndOffset = BlockStartOffset + EndOffset, TXStartOffset = TXEndOffset - TX#tx.data_size, - ?event(copycat_debug, {writing_index, + ?event(debug_copycat, {writing_index, {id, {explicit, TXID}}, {offset, TXStartOffset}, {size, TX#tx.data_size} @@ -301,8 +303,12 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> case is_bundle_tx(TX, Opts) of false -> #{items_count => 0, bundle_count => 0, skipped_count => 0}; true -> - ?event(copycat_debug, {fetching_bundle_header, - {tx_id, {explicit, TXID}}, + % Lightweight processing of block transactions to depth 2. We + % can avoid loading the full L1 TX data into memory, and instead + % only load the bundle header. But as a result we're unable to + % recurse any deeper than L2 dataitems. + ?event(debug_copycat, {fetching_bundle_header, + {tx_id, {string, TXID}}, {tx_end_offset, TXEndOffset}, {tx_data_size, TX#tx.data_size} }), @@ -310,7 +316,7 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> TXEndOffset, TX#tx.data_size, Opts ), case BundleRes of - {ok, {BundleIndex, HeaderSize}} -> + {ok, HeaderSize, BundleIndex} -> % Batch event tracking: measure total time and count for all write_offset calls {TotalTime, {_, ItemsCount}} = timer:tc(fun() -> lists:foldl( @@ -328,6 +334,11 @@ process_tx({{TX, _TXDataRoot}, EndOffset}, BlockStartOffset, Opts) -> BundleIndex ) end), + ?event(debug_copycat, + {bundle_items_indexed, + {tx_id, {string, TXID}}, + {items_count, ItemsCount} + }), % Single event increment for the batch record_event_metrics(<<"item_indexed">>, ItemsCount, TotalTime), #{items_count => ItemsCount, bundle_count => 1, skipped_count => 0}; @@ -371,55 +382,9 @@ is_bundle_tx(TX, _Opts) -> download_bundle_header(EndOffset, Size, Opts) -> observe_event(<<"bundle_header">>, fun() -> - StartOffset = EndOffset - Size + 1, - case hb_ao:resolve( - << - ?ARWEAVE_DEVICE/binary, - "/chunk&offset=", - (hb_util:bin(StartOffset))/binary - >>, - Opts - ) of - {ok, FirstChunk} -> - % Most bundle headers can fit in a single chunk, but those with - % thousands of items might require multiple chunks to fully - % represent the item index. - HeaderSize = ar_bundles:bundle_header_size(FirstChunk), - case header_chunk(HeaderSize, FirstChunk, StartOffset, Opts) of - {ok, BundleHeader} -> - {_ItemsBin, BundleIndex} = - ar_bundles:decode_bundle_header(BundleHeader), - {ok, {BundleIndex, HeaderSize}}; - Error -> - Error - end; - Error -> - Error - end + dev_arweave:bundle_header(EndOffset - Size, Opts) end). -header_chunk(invalid_bundle_header, _FirstChunk, _StartOffset, _Opts) -> - {error, invalid_bundle_header}; -header_chunk(HeaderSize, FirstChunk, _StartOffset, _Opts) - when HeaderSize =< byte_size(FirstChunk) -> - {ok, FirstChunk}; -header_chunk(HeaderSize, FirstChunk, StartOffset, Opts) -> - Res = - hb_ao:resolve( - << - ?ARWEAVE_DEVICE/binary, - "/chunk&offset=", - (hb_util:bin(StartOffset + byte_size(FirstChunk)))/binary, - "&length=", - (hb_util:bin(HeaderSize - byte_size(FirstChunk)))/binary - >>, - Opts - ), - case Res of - {ok, OtherChunks} -> {ok, <>}; - Other -> Other - end. - resolve_tx_headers(TXIDs, Opts) -> Results = parallel_map( TXIDs, @@ -439,7 +404,7 @@ resolve_tx_headers(TXIDs, Opts) -> resolve_tx_header(TXID, Opts) -> try - ?event(copycat_debug, {fetching_tx, {explicit, TXID}}), + ?event(debug_copycat, {fetching_tx, {explicit, TXID}}), ResolveRes = observe_event(<<"tx_header">>, fun() -> hb_ao:resolve( << @@ -579,7 +544,7 @@ small_bundle_header_test() -> OffsetMsg = hb_json:decode(OffsetBody), EndOffset = hb_util:int(maps:get(<<"offset">>, OffsetMsg)), Size = hb_util:int(maps:get(<<"size">>, OffsetMsg)), - {ok, {BundleIndex, HeaderSize}} = + {ok, HeaderSize, BundleIndex} = download_bundle_header(EndOffset, Size, Opts), ?assertEqual(1704, length(BundleIndex)), ?assertEqual(109088, HeaderSize), @@ -600,7 +565,7 @@ large_bundle_header_test() -> OffsetMsg = hb_json:decode(OffsetBody), EndOffset = hb_util:int(maps:get(<<"offset">>, OffsetMsg)), Size = hb_util:int(maps:get(<<"size">>, OffsetMsg)), - {ok, {BundleIndex, HeaderSize}} = + {ok, HeaderSize, BundleIndex} = download_bundle_header(EndOffset, Size, Opts), ?assertEqual(15000, length(BundleIndex)), ?assertEqual(960032, HeaderSize), @@ -940,16 +905,21 @@ auto_stop_partial_index_test() -> ?assertNot(has_any_indexed_tx(Block-1, Opts)), ok. -negative_from_parse_range_test() -> +negative_parse_range_test() -> {_TestStore, _StoreOpts, Opts} = setup_index_opts(), {ok, Tip} = hb_ao:resolve( <>, Opts ), - {From, To} = parse_range(#{ <<"from">> => <<"-3">> }, Opts), - ?assertEqual(hb_util:int(Tip) - 3, From), - ?assertEqual(undefined, To), + {NegativeFrom, UndefinedTo} = + parse_range(#{ <<"from">> => <<"-3">> }, Opts), + ?assertEqual(hb_util:int(Tip) - 3, NegativeFrom), + ?assertEqual(undefined, UndefinedTo), + {PositiveFrom, NegativeTo} = + parse_range(#{ <<"from">> => <<"10">>, <<"to">> => <<"-3">> }, Opts), + ?assertEqual(10, PositiveFrom), + ?assertEqual(hb_util:int(Tip) - 3, NegativeTo), ok. negative_from_index_test() -> diff --git a/src/dev_copycat_graphql.erl b/src/dev_copycat_graphql.erl index df8c9d47e..9f0752ec6 100644 --- a/src/dev_copycat_graphql.erl +++ b/src/dev_copycat_graphql.erl @@ -270,7 +270,7 @@ default_query(Parts) -> %% @doc Run node for testing run_test_node() -> Store = hb_test_utils:test_store(), - Opts = #{ store => Store, priv_wallet => hb:wallet() }, + Opts = #{ store => Store, priv_wallet => ar_wallet:new() }, Node = hb_http_server:start_node(Opts), {Node ,Opts}. %% @doc Basic test to test copycat device @@ -450,4 +450,4 @@ fetch_scheduler_location_test() -> ?assert(is_integer(Data)), ?assert(Data > 0), ?event({schedulers_indexed, Data}), - ok. + ok. \ No newline at end of file diff --git a/src/dev_hyperbuddy.erl b/src/dev_hyperbuddy.erl index 18c70abf5..0f6303412 100644 --- a/src/dev_hyperbuddy.erl +++ b/src/dev_hyperbuddy.erl @@ -1,16 +1,20 @@ %%% @doc A device that renders a REPL-like interface for AO-Core via HTML. -module(dev_hyperbuddy). --export([info/0, format/3, return_file/2, return_error/2]). +-export([info/1, format/3, return_file/2, return_error/2]). -export([metrics/3, events/3]). -export([throw/3]). -include_lib("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -%% @doc Export an explicit list of files via http. -info() -> +%% @doc Export an explicit list of files via http. Filenames added to the +%% `hyperbuddy_serve' key of the node message will be served as static files. +%% Each filename must point to a path relative to the HyperBEAM instance's +%% build subdirectory as follows: `priv/html/hyperbuddy@1.0'. +info(Opts) -> + ServedRoutes = hb_maps:get(hyperbuddy_serve, Opts, #{}, Opts), #{ default => fun serve/4, - routes => #{ + serve => ServedRoutes#{ % Default message viewer page: <<"index">> => <<"index.html">>, <<"bundle.js">> => <<"bundle.js">>, @@ -141,20 +145,18 @@ throw(_Msg, _Req, Opts) -> end. %% @doc Serve a file from the priv directory. Only serves files that are explicitly -%% listed in the `routes' field of the `info/0' return value. +%% listed in the `routes' field of the `info/1' return value. serve(<<"keys">>, M1, _M2, Opts) -> dev_message:keys(M1, Opts); serve(<<"set">>, M1, M2, Opts) -> dev_message:set(M1, M2, Opts); serve(Key, _, _, Opts) -> ?event({hyperbuddy_serving, Key}), - Routes = hb_maps:get(routes, info(), no_routes, Opts), - case hb_maps:get(Key, Routes, undefined, Opts) of - undefined -> {error, not_found}; - Filename -> return_file(Filename) + ServeRoutes = hb_maps:get(serve, info(Opts), #{}, Opts), + case hb_maps:find(Key, ServeRoutes, Opts) of + {ok, Filename} -> return_file(<<"hyperbuddy@1.0">>, Filename, #{}); + error -> {error, not_found} end. %% @doc Read a file from disk and serve it as a static HTML page. -return_file(Name) -> - return_file(<<"hyperbuddy@1.0">>, Name, #{}). return_file(Device, Name) -> return_file(Device, Name, #{}). return_file(Device, Name, Template) -> @@ -173,7 +175,9 @@ return_file(Device, Name, Template) -> <<".css">> -> <<"text/css">>; <<".png">> -> <<"image/png">>; <<".ico">> -> <<"image/x-icon">>; - <<".ttf">> -> <<"font/ttf">> + <<".ttf">> -> <<"font/ttf">>; + <<".json">> -> <<"application/json">>; + _ -> <<"text/plain">> end } }; @@ -197,6 +201,7 @@ apply_template(Body, Template) when is_map(Template) -> apply_template(Body, []) -> Body; apply_template(Body, [{Key, Value} | Rest]) -> + ?event(debug_apply_template, {key, Key, value, Value}), apply_template( re:replace( Body, @@ -221,4 +226,25 @@ return_templated_file_test() -> ?assertNotEqual( binary:match(Body, <<"This is an error message.">>), nomatch - ). \ No newline at end of file + ). + +return_custom_json_test() -> + ?assertMatch( + {ok, + #{ + <<"body">> := JSONBin, + <<"content-type">> := <<"application/json">> + } + } when byte_size(JSONBin) > 0, + hb_ao:resolve( + #{ + <<"device">> => <<"hyperbuddy@1.0">> + }, + <<"custom.json">>, + #{ + hyperbuddy_serve => #{ + <<"custom.json">> => <<"test.json">> + } + } + ) + ). diff --git a/src/dev_location.erl b/src/dev_location.erl index 433a8afab..42ba3061b 100644 --- a/src/dev_location.erl +++ b/src/dev_location.erl @@ -199,7 +199,7 @@ default_url(Opts) -> case hb_opts:get(location_url, not_found, Opts) of not_found -> Port = hb_util:bin(hb_opts:get(port, 8734, Opts)), - Host = hb_opts:get(host, <<"localhost">>, Opts), + Host = hb_opts:get(node_host, <<"localhost">>, Opts), Protocol = hb_opts:get(protocol, http1, Opts), ProtoStr = case Protocol of diff --git a/src/dev_manifest.erl b/src/dev_manifest.erl index b30cc6c0d..395cc2492 100644 --- a/src/dev_manifest.erl +++ b/src/dev_manifest.erl @@ -2,6 +2,8 @@ %%% https://specs.ar.io/?tx=lXLd0OPwo-dJLB_Amz5jgIeDhiOkjXuM3-r0H_aiNj0 -module(dev_manifest). -export([index/3, info/0, request/3]). +%%% Public test exports +-export([test_env_opts/0]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -15,7 +17,7 @@ info() -> %% @doc Return the fallback index page when the manifest itself is requested. index(M1, M2, Opts) -> - ?event(debug_manifest, {index_request, {m1, M1}, {m2, M2}}), + ?event(debug_manifest, {index_request, {base, M1}, {request, M2}}, Opts), case route(<<"index">>, M1, M2, Opts) of {ok, Index} -> ?event({manifest_index_returned, Index}), @@ -64,15 +66,10 @@ route(ID, _, _, Opts) when ?IS_ID(ID) -> route(Key, M1, M2, Opts) -> ?event(debug_manifest, {manifest_lookup, {key, Key}, {m1, M1}, {m2, {explicit, M2}}}), {ok, Manifest} = manifest(M1, M2, Opts), - Res = hb_ao:get( - <<"paths/", Key/binary>>, - {as, <<"message@1.0">>, Manifest}, - Opts - ), - ?event({manifest_lookup_result, {res, Res}}), - case Res of - not_found -> - %% Support materialized view in some JavaScript frameworks + {ok, Res} = maps:find(<<"paths">>, Manifest), + case maps:get(Key, Res, no_path_match) of + no_path_match -> + % Support materialized view in some JavaScript frameworks. case hb_opts:get(manifest_404, fallback, Opts) of error -> ?event({manifest_404_error, {key, Key}}), @@ -81,9 +78,11 @@ route(Key, M1, M2, Opts) -> ?event({manifest_fallback, {key, Key}}), route(<<"index">>, M1, M2, Opts) end; - _ -> - ?event({manifest_lookup_success, {key, Key}}), - {ok, Res} + Result -> + ?event({manifest_lookup_success, {key, Key}, {result, Result}}), + try {ok, hb_cache:ensure_loaded(Result, Opts)} + catch _:_:_ -> {error, not_found} + end end. %% @doc Implement the `on/request' hook for the `manifest@1.0' device, finding @@ -91,59 +90,90 @@ route(Key, M1, M2, Opts) -> %% `manifest@1.0' before execution. Allowing `/ID/path` style access for old data. request(Base, Req, Opts) -> ?event({on_req_manifest_detector, {base, Base}, {req, Req}}), - case hb_maps:find(<<"body">>, Req, Opts) of - {ok, [PrimaryMsg|Rest]} -> - case maybe_cast_manifest(PrimaryMsg, Opts) of - {ok, CastedMsg} -> - %% For to go to index if no key provided. - Rest2 = case Rest of - [] -> [#{<<"path">> => <<"index">>}]; - _ -> Rest - end, - {ok, Req#{ <<"body">> => [CastedMsg|Rest2] }}; - Error -> - ?event({manifest_not_cast, {error, Error}}), - Error - end; - _ -> + maybe + {ok, [PrimaryMsg|Rest]} ?= hb_maps:find(<<"body">>, Req, Opts), + {ok, Loaded} ?= load(PrimaryMsg, Opts), + ?event(debug_manifest, {loaded, Loaded}), + % Must handle three cases: + % 1. The maybe_cast is not a manifest, so we return the *loaded* request, + % such that the work to load it is not wasted. + % 2. The maybe_cast is a manifest, and there are no other elements of + % the path, so we add the `index' path and return. + % 3. The maybe_cast is a manifest, and there are other elements of + % the path, so we return the original request sequence with the first + % message replaced with the casted manifest. + case {Rest, maybe_cast_manifest(Loaded, Opts)} of + {_, ignored} -> + ?event( + debug_manifest, + {non_manifest_returning_loaded, {loaded, Loaded}, {rest, Rest}}), + {ok, Req#{ <<"body">> => [Loaded|Rest] }}; + {[], {ok, Casted}} -> + ?event(debug_manifest, {manifest_returning_index, {req, Req}}), + {ok, Req#{ <<"body">> => [Casted, #{<<"path">> => <<"index">>}] }}; + {_, {ok, Casted}} -> + ?event(debug_manifest, {manifest_returning_subpath, {req, Req}}), + {ok, Req#{ <<"body">> => [Casted|Rest] }} + end + else + {error, not_found} -> + ?event(debug_manifest, {not_found_on_load, {req, Req}}), + { + error, + #{ + <<"status">> => 404, + <<"body">> => <<"Not Found">> + } + }; + Error -> + ?event(debug_manifest, {request_ignored, {unexpected, Error}}), + % On other errors, we return the original request. {ok, Req} end. %% @doc Cast a message to `manifest@1.0` if it has the correct content-type but %% no other device is specified. -maybe_cast_manifest(ID, Opts) when ?IS_ID(ID) -> +load(Msg, _Opts) when is_map(Msg) -> {ok, Msg}; +load(List, _Opts) when is_list(List) -> skip; +load({as, _, _}, _Opts) -> skip; +load(ID, Opts) when ?IS_ID(ID) -> case hb_cache:read(ID, Opts) of - {ok, Msg} -> maybe_cast_manifest(Msg, Opts); + {ok, Msg} -> load(Msg, Opts); _ -> - ?event(maybe_cast_manifest, {message_not_found, {id, ID}}), + ?event(debug_maybe_cast_manifest, {message_load_failed, {id, ID}}), {error, not_found} end; -maybe_cast_manifest(Msg, Opts) when is_map(Msg) orelse ?IS_LINK(Msg) -> +load(Msg, Opts) when ?IS_LINK(Msg) -> + try load(hb_cache:ensure_loaded(Msg, Opts), Opts) + catch + _ -> + ?event(debug_maybe_cast_manifest, {message_load_failed, {link, Msg}}), + {error, not_found} + end. + +maybe_cast_manifest(Msg, Opts) -> case hb_maps:find(<<"device">>, Msg, Opts) of - {ok, X} when X == <<"manifest@1.0">> orelse X == <<"message@1.0">> -> - {ok, Msg}; + {ok, X} when X == <<"manifest@1.0">> -> {ok, Msg}; _ -> case hb_maps:find(<<"content-type">>, Msg, Opts) of - {ok, <<"application/x.arweave-manifest+json">>} -> - ?event(maybe_cast_manifest, {manifest_casting, {msg, Msg}}), + {ok, <<"application/x.arweave-manifest", _/binary>>} -> + ?event(debug_maybe_cast_manifest, {manifest_casting, {msg, Msg}}), {ok, {as, <<"manifest@1.0">>, Msg}}; - _ -> - {ok, Msg} + _IgnoredContentType -> + ignored end - end; -maybe_cast_manifest(Msg, _Opts) -> - ?event(maybe_cast_manifest, {message_is_not_manifest, {msg, Msg}}), - {ok, Msg}. + end. %% @doc Find and deserialize a manifest from the given base, returning a %% message with the `~manifest@1.0' device. manifest(Base, _Req, Opts) -> JSON = - hb_ao:get_first( + hb_maps:get_first( [ - {{as, <<"message@1.0">>, Base}, [<<"data">>]}, - {{as, <<"message@1.0">>, Base}, [<<"body">>]} + {Base, <<"data">>}, + {Base, <<"body">>} ], + not_found, Opts ), FlatManifest = #{ <<"paths">> := FlatPaths } = hb_json:decode(JSON), @@ -154,7 +184,7 @@ manifest(Base, _Req, Opts) -> %% @doc Generate a nested message of links to content from a parsed (and %% structured) manifest. -linkify(#{ <<"id">> := ID }, Opts) -> +linkify(#{ <<"id">> := ID }, Opts) when is_binary(ID) -> LinkOptsBase = (maps:with([store], Opts))#{ scope => [local, remote]}, {link, ID, LinkOptsBase#{ <<"type">> => <<"link">>, <<"lazy">> => false }}; linkify(Manifest, Opts) when is_map(Manifest) -> @@ -223,7 +253,6 @@ resolve_test() -> hb_http:get(Node, << ManifestID/binary, "/nested/page2" >>, Opts)), % Making the same requests to a node with the `request' hook enabled should % yield the same results. - ?hr(), ?event({legacy_manifest_id, LegacyManifestID}), ?assertMatch( {ok, #{ <<"body">> := <<"Page 1">> }}, @@ -281,7 +310,7 @@ create_generic_manifest(Opts) -> %% @doc Download the manifest raw data. %% NOTE: This test requests data to arweave node -manifest_download_via_raw_endpoint_test() -> +manifest_download_via_raw_endpoint_test_ignore() -> Opts = #{ arweave_index_ids => true, store => [ @@ -318,13 +347,7 @@ manifest_download_via_raw_endpoint_test() -> %% @doc Accessing `/TXID` of a manifest transaction should access the index key. manifest_inner_redirect_test() -> - %% Define the store - LmdbStore = hb_test_utils:test_store(), - %% Load transaction information to the store - load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - %% Start node - Opts = #{store => LmdbStore}, + Opts = test_env_opts(), Node = hb_http_server:start_node(Opts), %% Request manifest to node. ?assertMatch( @@ -338,11 +361,7 @@ manifest_inner_redirect_test() -> %% @doc Accessing `/TXID/assets/ArticleBlock-Dtwjc54T.js` should return valid message. access_key_path_in_manifest_test() -> - LmdbStore = hb_test_utils:test_store(), - load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - load_and_store(LmdbStore, <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">>), - Opts = #{store => LmdbStore}, + Opts = test_env_opts(), Node = hb_http_server:start_node(Opts), ?assertMatch( {ok, #{<<"commitments">> := #{<<"oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4">> := _ }}}, @@ -356,10 +375,7 @@ access_key_path_in_manifest_test() -> %% This works with `not_found.js` but doesn't follow the logic if under a %% folder structure, like `assets/not_found.js . manifest_should_fallback_on_not_found_path_test() -> - LmdbStore = hb_test_utils:test_store(), - load_and_store(LmdbStore, <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>), - load_and_store(LmdbStore, <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>), - Opts = #{store => LmdbStore}, + Opts = test_env_opts(), Node = hb_http_server:start_node(Opts), ?assertMatch( {ok, #{<<"commitments">> := #{<<"Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM">> := _ }}}, @@ -370,19 +386,34 @@ manifest_should_fallback_on_not_found_path_test() -> ) ). -%% @doc Load ans104 binary files to a store. -load_and_store(LmdbStore, File) -> - Opts = #{}, - {ok, SerializedItem} = - file:read_file( - hb_util:bin( - <<"test/arbundles.js/ans-104-manifest-", File/binary>> +%% @doc Returns `Opts' with the test manifest fixture flow used by `dev_b32_name'. +test_env_opts() -> + TempStore = hb_test_utils:test_store(), + BaseOpts = + #{ + store => + [ + TempStore + #{<<"store-module">> => hb_store_gateway} + ] + }, + lists:foreach( + fun(Ref) -> + hb_test_utils:preload( + BaseOpts, + <<"test/arbundles.js/ans-104-manifest-", Ref/binary>> ) - ), - Message = hb_message:convert( - ar_bundles:deserialize(SerializedItem), - <<"structured@1.0">>, - <<"ans104@1.0">>, - Opts + end, + [ + <<"42jky7O3rzKkMOfHBXgK-304YjulzEYqHc9qyjT3efA.bin">>, + <<"index-Tqh6oIS2CLUaDY11YUENlvvHmDim1q16pMyXAeSKsFM.bin">>, + <<"item-oLnQY-EgiYRg9XyO7yZ_mC0Ehy7TFR3UiDhFvxcohC4.bin">> + ] ), - _ = hb_cache:write(Message, #{store => LmdbStore}). \ No newline at end of file + BaseOpts#{ + on => + #{ + <<"request">> => + [#{<<"device">> => <<"manifest@1.0">>}] + } + }. diff --git a/src/dev_message.erl b/src/dev_message.erl index 4cd72ee8b..857845e8e 100644 --- a/src/dev_message.erl +++ b/src/dev_message.erl @@ -635,7 +635,7 @@ set(Base, NewValuesMsg, Opts) -> ), % Base message with keys-to-unset removed BaseValues = hb_maps:without(UnsetKeys, Base, Opts), - ?event(message_set, + ?event(debug_message_set, {performing_set, {conflicting_keys, ConflictingKeys}, {keys_to_unset, UnsetKeys}, @@ -979,4 +979,4 @@ test_verify(KeyType) -> #{ <<"path">> => <<"verify">>, <<"body">> => Signed }, #{ hashpath => ignore } ) - ). + ). \ No newline at end of file diff --git a/src/dev_name.erl b/src/dev_name.erl index 9e4dab29e..f7c275a7f 100644 --- a/src/dev_name.erl +++ b/src/dev_name.erl @@ -5,6 +5,8 @@ %%% first resolver that matches. -module(dev_name). -export([info/1, request/3]). +%%% Public helpers. +-export([test_arns_opts/0]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -34,12 +36,20 @@ resolve(Key, _, Req, Opts) -> false -> {ok, Resolved}; true -> - hb_cache:read(Resolved, Opts) + maybe_load_resolved(Resolved, Opts) end; - not_found -> - not_found + not_found -> not_found end. +%% @doc Load a resolved name target if it is a cache reference, otherwise +%% return the resolved value directly. +maybe_load_resolved(Resolved, Opts) when ?IS_ID(Resolved) -> + hb_cache:read(Resolved, Opts); +maybe_load_resolved(Resolved, Opts) when ?IS_LINK(Resolved) -> + {ok, hb_cache:ensure_loaded(Resolved, Opts)}; +maybe_load_resolved(Resolved, _Opts) -> + {ok, Resolved}. + %% @doc Find the first resolver that matches the key and return its value. match_resolver(_Key, [], _Opts) -> not_found; @@ -75,15 +85,14 @@ request(HookMsg, HookReq, Opts) -> maybe {ok, Req} ?= hb_maps:find(<<"request">>, HookReq, Opts), {ok, Host} ?= hb_maps:find(<<"host">>, Req, Opts), - {ok, Name} ?= name_from_host(Host, hb_opts:get(host, no_host, Opts)), - {ok, ResolvedMsg} ?= resolve(Name, HookMsg, #{}, Opts), + {ok, Name} ?= name_from_host(Host, hb_opts:get(node_host, no_host, Opts)), + {ok, ResolvedMsg} ?= resolve(Name, HookMsg, HookReq, Opts), ModReq = - case hb_maps:find(<<"body">>, HookReq, Opts) of - {ok, [OldBase|Rest]} -> - [overlay_loaded(OldBase, ResolvedMsg, Opts)|Rest]; - {ok, []} -> - [ResolvedMsg] - end, + maybe_append_named_message( + ResolvedMsg, + hb_util:ok(hb_maps:find(<<"body">>, HookReq, Opts)), + Opts + ), ?event( {request_with_prepended_path, {name, Name}, @@ -95,17 +104,60 @@ request(HookMsg, HookReq, Opts) -> {ok, #{ <<"body">> => ModReq }} else Reason -> - ?event({request_hook_skip, {reason, Reason}, {hook_req, HookReq}}), - {ok, HookReq} + case maps:get(<<"body">>, HookReq, []) of + [] -> + ?event({request_hook_404, root_path}), + % No path provided should return 404 if not resolved + % (via name resolvers or 52 char subdomain) + {error, #{<<"status">> => 404, <<"body">> => <<"Not Found">>}}; + _ -> + ?event({request_hook_skip, {reason, Reason}, {hook_req, HookReq}}), + {ok, HookReq} + end + end. + +%% @doc After finding a hit for a named message, we should ensure that it is the +%% base message for the evaluation. If it is already present in the request, +%% however, we should not add it twice. Instead, we must add the version that +%% is loaded (if applicable). +%% +%% Eg: +%% base32IDA.hyperbeam/ -> [IDA] +%% base32IDA.hyperbeam/base64urlIDA/xyz -> [IDA, xyz] +%% base32IDA.hyperbeam/base64urlIDB/xyz -> [IDA, IDB, xyz] +maybe_append_named_message(ResolvedMsg, [], _Opts) -> [ResolvedMsg]; +maybe_append_named_message(ResolvedMsg, OldReq = [OldBase|ReqMsgsRest], Opts) -> + case permissive_id(OldBase, Opts) == permissive_id(ResolvedMsg, Opts) of + true when is_map(OldBase) or is_list(OldBase) -> OldReq; + true -> [ResolvedMsg|ReqMsgsRest]; + false -> + case is_map(OldBase) andalso hb_maps:get(<<"path">>, OldBase, not_found, Opts) of + not_found -> + ?event( + {skipping_old_base, + {old_base, OldBase}, + {resolved_msg, ResolvedMsg} + } + ), + [ResolvedMsg|ReqMsgsRest]; + _ -> [ResolvedMsg, OldBase|ReqMsgsRest] + end end. +%% @doc Takes a message or resolution request (`as` or `resolve`) -- whether in +%% the form of an ID, link, or loaded map -- and returns its ID. +permissive_id(ID, _Opts) when ?IS_ID(ID) -> ID; +permissive_id({link, ID, _LinkOpts}, _Opts) -> ID; +permissive_id({as, _Device, Msg}, Opts) -> permissive_id(Msg, Opts); +permissive_id(Msg, Opts) when is_map(Msg) -> hb_message:id(Msg, signed, Opts). + %% @doc Takes a request-given host and the host value in the node message and %% returns only the name component of the host, if it is present. If no name is %% present, an empty binary is returned. name_from_host(Host, no_host) -> - case hd(binary:split(Host, <<".">>)) of - <<>> -> {error, <<"No name found in `Host`.">>}; - Name -> {ok, Name} + case binary:split(Host, <<".">>, [global, trim_all]) of + [_Host] -> {error, <<"No subdomain found in `Host: ", Host/binary, "`.">>}; + [Name|_] -> {ok, Name} end; name_from_host(ReqHost, RawNodeHost) -> NodeHost = uri_string:parse(RawNodeHost), @@ -113,18 +165,11 @@ name_from_host(ReqHost, RawNodeHost) -> WithoutNodeHost = binary:replace( ReqHost, - maps:get(host, uri_string:parse(RawNodeHost)), + maps:get(host, NodeHost), <<>> ), name_from_host(WithoutNodeHost, no_host). -%% @doc Merge the base message with the resolved message, ensuring that `~` as -%% device specifiers are preserved. -overlay_loaded({as, DevID, Base}, Resolved, Opts) -> - {as, DevID, hb_maps:merge(Base, Resolved, Opts)}; -overlay_loaded(Base, Resolved, Opts) -> - hb_maps:merge(Base, Resolved, Opts). - %%% Tests. no_resolvers_test() -> @@ -232,10 +277,9 @@ load_and_execute_test() -> %% @doc Return an `Opts` for an environment with the default ARNS name export %% and a temporary store for the test. -arns_opts() -> +test_arns_opts() -> JSONNames = <<"G_gb7SAgogHMtmqycwaHaC6uC-CZ3akACdFv5PUaEE8">>, Path = <>, - hb_http_server:start_node(#{}), TempStore = hb_test_utils:test_store(), #{ store => @@ -256,13 +300,13 @@ arns_opts() -> %% @doc Names from JSON test. arns_json_snapshot_test() -> - Opts = arns_opts(), + Opts = test_arns_opts(), ?assertMatch( - {ok, <<"application/pdf">>}, + {ok, <<"text/html">>}, hb_ao:resolve_many( [ #{ <<"device">> => <<"name@1.0">> }, - #{ <<"path">> => <<"draft-17_whitepaper">> }, + #{ <<"path">> => <<"001_permabytes">>, <<"load">> => true }, <<"content-type">> ], Opts @@ -270,15 +314,15 @@ arns_json_snapshot_test() -> ). arns_host_resolution_test() -> - Opts = arns_opts(), + Opts = test_arns_opts(), Node = hb_http_server:start_node(Opts), ?assertMatch( - {ok, <<"application/pdf">>}, + {ok, <<"text/html">>}, hb_http:get( Node, #{ <<"path">> => <<"content-type">>, - <<"host">> => <<"draft-17_whitepaper">> + <<"host">> => <<"001_permabytes.localhost">> }, Opts ) diff --git a/src/dev_process.erl b/src/dev_process.erl index bec7cb058..12a124a14 100644 --- a/src/dev_process.erl +++ b/src/dev_process.erl @@ -461,7 +461,7 @@ store_result(ForceSnapshot, ProcID, Slot, Res, Req, Opts) -> false -> Res; true -> ?event( - compute_debug, + debug_compute, {snapshotting, {proc_id, ProcID}, {slot, Slot}}, Opts ), diff --git a/src/dev_process_worker.erl b/src/dev_process_worker.erl index e6bd9ab90..4789f1263 100644 --- a/src/dev_process_worker.erl +++ b/src/dev_process_worker.erl @@ -106,13 +106,13 @@ await(Worker, GroupName, Base, Req, Opts) -> receive {resolved, _, GroupName, {slot, RecvdSlot}, Res} when RecvdSlot == TargetSlot orelse TargetSlot == any -> - ?event(compute_debug, {notified_of_resolution, + ?event(debug_compute, {notified_of_resolution, {target, TargetSlot}, {group, GroupName} }), Res; {resolved, _, GroupName, {slot, RecvdSlot}, _Res} -> - ?event(compute_debug, {waiting_again, + ?event(debug_compute, {waiting_again, {target, TargetSlot}, {recvd, RecvdSlot}, {worker, Worker}, @@ -120,7 +120,7 @@ await(Worker, GroupName, Base, Req, Opts) -> }), await(Worker, GroupName, Base, Req, Opts); {'DOWN', _R, process, Worker, _Reason} -> - ?event(compute_debug, + ?event(debug_compute, {leader_died, {group, GroupName}, {leader, Worker}, diff --git a/src/dev_push.erl b/src/dev_push.erl index 6ebfa1d63..9c260f017 100644 --- a/src/dev_push.erl +++ b/src/dev_push.erl @@ -363,7 +363,7 @@ push_downstream_remote(TargetID, NextSlotOnProc, Origin, RawOpts) -> {ok, NewOpts} -> NewOpts; _ -> RawOpts end, - Self = hb_opts:get(host, host_not_specified, Opts), + Self = hb_opts:get(node_host, host_not_specified, Opts), ?event(remote_push, {push_downstream_remote, {target, TargetID}, @@ -1263,7 +1263,7 @@ nested_push_prompts_encoding_change() -> cache_control => <<"always">>, store => hb_opts:get(store) }, - ?event(push_debug, {opts, Opts}), + ?event(debug_push, {opts, Opts}), Base = dev_process_test_vectors:aos_process(Opts), hb_cache:write(Base, Opts), {ok, SchedInit} = diff --git a/src/dev_query.erl b/src/dev_query.erl index 82c0bc77f..4dc43a09d 100644 --- a/src/dev_query.erl +++ b/src/dev_query.erl @@ -215,7 +215,7 @@ query_match_key(Path, Opts) -> %% @doc Return test options with a test store. test_setup() -> Store = hb_test_utils:test_store(), - Opts = #{ store => Store, priv_wallet => hb:wallet() }, + Opts = #{ store => Store, priv_wallet => ar_wallet:new() }, % Write a simple message. hb_cache:write( #{ @@ -365,4 +365,4 @@ http_test() -> Opts ), ?assertEqual(<<"binary-value">>, hb_maps:get(<<"basic">>, Msg, Opts)), - ok. + ok. \ No newline at end of file diff --git a/src/dev_query_test_vectors.erl b/src/dev_query_test_vectors.erl index 91f6d8dd9..0f6f37371 100644 --- a/src/dev_query_test_vectors.erl +++ b/src/dev_query_test_vectors.erl @@ -69,7 +69,7 @@ write_test_message_with_recipient(Recipient, Opts) -> simple_blocks_query_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -114,7 +114,7 @@ simple_blocks_query_test() -> block_by_height_query_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -165,7 +165,7 @@ block_by_height_query_test() -> simple_ans104_query_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -202,7 +202,7 @@ simple_ans104_query_test() -> Node, Query, #{ - <<"owners">> => [hb:address()] + <<"owners">> => [hb:address(Wallet)] }, Opts ), @@ -232,7 +232,7 @@ simple_ans104_query_test() -> transactions_query_tags_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -295,7 +295,7 @@ transactions_query_tags_test() -> transactions_query_owners_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -327,7 +327,7 @@ transactions_query_owners_test() -> Node, Query, #{ - <<"owners">> => [hb:address()] + <<"owners">> => [hb:address(Wallet)] }, Opts ), @@ -357,7 +357,7 @@ transactions_query_owners_test() -> transactions_query_recipients_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -422,7 +422,7 @@ transactions_query_recipients_test() -> transactions_query_ids_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -484,7 +484,7 @@ transactions_query_ids_test() -> transactions_query_combined_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -521,7 +521,7 @@ transactions_query_combined_test() -> Node, Query, #{ - <<"owners">> => [hb:address()], + <<"owners">> => [hb:address(Wallet)], <<"ids">> => [ExpectedID] }, Opts @@ -552,7 +552,7 @@ transactions_query_combined_test() -> transaction_query_by_id_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -602,7 +602,7 @@ transaction_query_by_id_test() -> transaction_query_full_test() -> Opts = #{ - priv_wallet => SenderKey = hb:wallet(), + priv_wallet => SenderKey = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -679,7 +679,7 @@ transaction_query_full_test() -> transaction_query_not_found_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => ar_wallet:new(), store => [hb_test_utils:test_store()] }, Res = @@ -715,7 +715,7 @@ transaction_query_not_found_test() -> transaction_query_with_anchor_test() -> Opts = #{ - priv_wallet => hb:wallet(), + priv_wallet => Wallet = ar_wallet:new(), store => [hb_test_utils:test_store()] }, Node = hb_http_server:start_node(Opts), @@ -727,7 +727,7 @@ transaction_query_with_anchor_test() -> anchor = AnchorID = crypto:strong_rand_bytes(32), data = <<"test-data">> }, - hb:wallet() + Wallet ), <<"structured@1.0">>, <<"ans104@1.0">>, diff --git a/src/dev_rate_limit.erl b/src/dev_rate_limit.erl new file mode 100644 index 000000000..7d4f90ba0 --- /dev/null +++ b/src/dev_rate_limit.erl @@ -0,0 +1,264 @@ +%%% @doc A basic rate limiter device. It is intended for use as a `~hook@1.0` +%%% `on/request` handler. It limits the number of requests per time period from a +%%% given IP address, returning a 429 status code and response if the limit is +%%% exceeded. +%%% +%%% The device can be configured with the following node message options: +%%% +%%% ``` +%%% rate_limit_requests: The maximum number of requests per period from a +%%% given user. +%%% Default: 1000. +%%% rate_limit_period: The rate at which peer's fully recharge balances. +%%% Default: 60 (unit: seconds). +%%% rate_limit_max: The maximum `balance' that a peer may hold. +%%% Default: 1000. +%%% rate_limit_min: The minimum `balance' that a peer may hold. +%%% Default: -1000. +%%% rate_limit_exempt: A list of peer IDs that are exempt from the limit. +%%% Default: []. +%%% ``` +%%% +%%% Notably, the `balance` of a user -- in terms of their available limit -- may +%%% become _negative_ if they continue to make calls even after exceeding their +%%% limit. The effect of this is that users that make too many requests to the +%%% server repeatedly simply receive no further service. The `rate_limit_min` +%%% option can be used to specify the minimum balance that users will hit. Any +%%% further requests are rejected but do not diminish their balance further. +-module(dev_rate_limit). +-export([request/3]). +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(LOOKUP_TIMEOUT, 1000). +-define(DEFAULT_MAX, 1_000). +-define(DEFAULT_MIN, -1_000). +-define(DEFAULT_REQS, 1000). +-define(DEFAULT_PERIOD, 60). + +%% @doc `on/request' handler that triggers rate limit counting and returns a +%% 429 status code and response if the limit is exceeded. The response includes +%% a `retry-after' header that indicates the number of seconds the client should +%% wait before making the next request. +request(_, Msg, Opts) -> + ?event(rate_limit, {request, {msg, Msg}}), + Reference = request_reference(hb_maps:get(<<"request">>, Msg, #{}, Opts), Opts), + case is_limited(Reference, Opts) of + {true, Balance} -> + ?event( + rate_limit, + {rate_limit_exceeded, {caller, Reference}, {balance, Balance}} + ), + RechargeRate = + hb_opts:get(rate_limit_requests, ?DEFAULT_REQS, Opts) / + hb_opts:get(rate_limit_period, ?DEFAULT_PERIOD, Opts), + RawRetryAfter = ceil(abs(Balance) / RechargeRate), % ...seconds + % If the node config specifies a `min` balance of `0`, callers may + % have a non-negative balance but still be rate-limited. In this case, + % we bump the `retry-after` to 1 second so as not to confuse the + % caller. + RetryAfter = + if RawRetryAfter =< 0.0 -> 1; + true -> RawRetryAfter + end, + RetryAfterBin = hb_util:bin(RetryAfter), + ?event( + rate_limit, + {rate_limit_exceeded, + {caller, Reference}, + {balance, Balance}, + {retry_after, RetryAfterBin} + } + ), + % Transform the given request into a request to return a 429 status + % code and response. + {error, + #{ + <<"status">> => 429, + <<"reason">> => <<"rate-limited">>, + <<"body">> => <<"Rate limit exceeded.">>, + <<"retry-after">> => RetryAfterBin + } + }; + false -> + ?event(rate_limit, {rate_limit_allowed, {caller, Reference}}), + {ok, Msg} + end. + +%% @doc The singleton ID of the rate limiter server. This allows us to run +%% multiple rate limiters on the same node if needed, each with its own +%% configuration, but with all of the callers sharing the same rate limiter +%% server. +server_id(Opts) -> + {?MODULE, hb_util:human_id(hb_opts:get(priv_wallet, undefined, Opts))}. + +%% @doc Determine the reference of the caller. Presently only the `ip` form +%% may be used to identify the caller. +request_reference(Msg, Opts) -> hb_private:get(<<"ip">>, Msg, Opts). + +%% @doc Check if the caller is limited according to the current state of the +%% rate limiter server. +is_limited(Reference, Opts) -> + PID = ensure_rate_limiter_started(Opts), + PID ! {request, self(), Reference}, + receive + {incremented, Balance} when Balance > 0 -> false; + {incremented, Balance} when Balance =< 0 -> {true, Balance} + after ?LOOKUP_TIMEOUT -> + ?event(warning, {rate_limit_timeout, restarting}), + hb_name:unregister(server_id(Opts)), + is_limited(Reference, Opts) + end. + +%% @doc Ensure that the rate limiter server is started and return the PID of +%% the server. In the event of two instanteous spawns, one of the new processes +%% will fail with an error and the other will succeed. The effect to the caller +%% is the same: A rate limiter is available to query. +ensure_rate_limiter_started(Opts) -> + ServerID = server_id(Opts), + hb_name:singleton( + ServerID, + fun() -> start_server(ServerID, Opts) end + ). + +start_server(ServerID, Opts) -> + % Exit the process if we cannot register the server ID. + Reqs = hb_opts:get(rate_limit_requests, ?DEFAULT_REQS, Opts), + Period = hb_opts:get(rate_limit_period, ?DEFAULT_PERIOD, Opts), + Max = hb_opts:get(rate_limit_max, ?DEFAULT_MAX, Opts), + Min = hb_opts:get(rate_limit_min, ?DEFAULT_MIN, Opts), + Exempt = hb_opts:get(rate_limit_exempt, [], Opts), + ?event( + rate_limit, + {started_rate_limiter, + {server_id, ServerID}, + {reqs, Reqs}, + {period, Period}, + {max, Max}, + {min, Min}, + {exempt, Exempt} + } + ), + server_loop( + #{ + reqs => Reqs, + period => Period, + max => Max, + min => Min, + peers => #{ Ref => infinity || Ref <- Exempt } + } + ). + +%% @doc The main loop of the rate limiter server. Only responds to two messages: +%% - `{request, Self, Reference}': Debit the account of the given reference by 1. +%% - `{balance, PID, Reference}': Return the current balance of the given reference. +%% The `balance` call is not presently used, but seems sensible to have. +server_loop(State) -> + ?event({server_loop, {state, State}}), + receive + {request, PID, Reference} -> + NewState = debit(Reference, 1, State, Now = erlang:system_time(millisecond)), + ?event({state_after_debit, NewState}), + Balance = account_balance(Reference, NewState, Now), + ?event( + rate_limit_short, + {rate_limit_debited, {target, Reference}, {balance, Balance}} + ), + PID ! {incremented, Balance}, + server_loop(NewState); + {balance, PID, Reference} -> + PID ! {balance, account_balance(Reference, State)}, + server_loop(State) + end. + +%% @doc Debit the account of the given reference by the given quantity. +debit(Ref, Amount, State = #{ peers := Peers, min := Min }, Now) -> + case account_balance(Ref, State, Now) of + infinity -> State; + Balance -> + State#{ + peers => + Peers#{ + Ref => + #{ + balance => max(Min, Balance - Amount), + last => Now + } + } + } + end. + +%% @doc Calculate the current balance for a user, including unused capacity +%% accrued since the last interaction. +account_balance(Reference, State) -> + account_balance(Reference, State, erlang:system_time(millisecond)). +account_balance( + Reference, + #{ max := Max, reqs := Reqs, period := Period, peers := Peers }, + Time + ) -> + ?event({account_balance, {target, Reference}, {peers, Peers}, {time, Time}}), + case maps:get(Reference, Peers, not_found) of + infinity -> infinity; + not_found -> Max; + #{ balance := Balance, last := LastInteraction } -> + RechargeRate = Reqs / (Period * 1000), + RechargedSinceLast = (Time - LastInteraction) * RechargeRate, + min(Max, Balance + RechargedSinceLast) + end. + +%%% Tests + +rate_limit_test() -> + ServerOpts = #{ + rate_limit_requests => 2, + rate_limit_period => 1, + rate_limit_max => 2, + on => + #{ + <<"request">> => + #{ + <<"device">> => <<"rate-limit@1.0">> + } + } + }, + ServerNode = hb_http_server:start_node(ServerOpts), + ?assertMatch( + {ok, _}, + hb_http:get(ServerNode, <<"id">>, #{}) + ), + ?debug_wait(100), + ?assertMatch( + {ok, _}, + hb_http:get(ServerNode, <<"id">>, #{}) + ), + ?debug_wait(100), + ?assertMatch( + {error, #{ <<"status">> := 429 }}, + hb_http:get(ServerNode, <<"id">>, #{}) + ). + +rate_limit_reset_test() -> + ServerOpts = #{ + rate_limit_requests => 2, + rate_limit_period => 1, + rate_limit_max => 2, + rate_limit_min => 0, + rate_limit_exempt => [], + on => + #{ + <<"request">> => + #{ + <<"device">> => <<"rate-limit@1.0">> + } + } + }, + ServerNode = hb_http_server:start_node(ServerOpts), + ?assertMatch({ok, _}, hb_http:get(ServerNode, <<"id">>, #{})), + ?assertMatch({ok, _}, hb_http:get(ServerNode, <<"id">>, #{})), + ?assertMatch( + {error, #{ <<"status">> := 429 }}, + hb_http:get(ServerNode, <<"id">>, #{}) + ), + timer:sleep(1_000), + ?assertMatch({ok, _}, hb_http:get(ServerNode, <<"id">>, #{})). \ No newline at end of file diff --git a/src/dev_scheduler.erl b/src/dev_scheduler.erl index bb85c2cba..3d165342a 100644 --- a/src/dev_scheduler.erl +++ b/src/dev_scheduler.erl @@ -211,7 +211,7 @@ find_next_assignment(Base, Req, _Schedule, LastSlot, Opts) -> end, case LocalCacheRes of {ok, Worker, Assignment} -> - ?event(next_debug, + ?event(debug_next, {in_cache, {slot, LastSlot + 1}, {assignment, Assignment} diff --git a/src/dev_whois.erl b/src/dev_whois.erl index 61011c489..a30bf09fd 100644 --- a/src/dev_whois.erl +++ b/src/dev_whois.erl @@ -17,7 +17,7 @@ echo(_, Req, Opts) -> node(_, _, Opts) -> case ensure_host(Opts) of {ok, NewOpts} -> - {ok, hb_opts:get(host, <<"unknown">>, NewOpts)}; + {ok, hb_opts:get(node_host, <<"unknown">>, NewOpts)}; Error -> Error end. @@ -25,12 +25,12 @@ node(_, _, Opts) -> %% @doc Return the node message ensuring that the host is set. If it is not, we %% attempt to find the host information from the specified bootstrap node. ensure_host(Opts) -> - case hb_opts:get(host, <<"unknown">>, Opts) of + case hb_opts:get(node_host, <<"unknown">>, Opts) of <<"unknown">> -> case bootstrap_node_echo(Opts) of {ok, Host} -> % Set the host information in the persisted node message. - hb_http_server:set_opts(NewOpts = Opts#{ host => Host }), + hb_http_server:set_opts(NewOpts = Opts#{ node_host => Host }), {ok, NewOpts}; Error -> Error diff --git a/src/hb.erl b/src/hb.erl index c6cce5c75..a7b2cc041 100644 --- a/src/hb.erl +++ b/src/hb.erl @@ -91,7 +91,7 @@ -export([no_prod/3]). -export([read/1, read/2, debug_wait/4]). %%% Node wallet and address management: --export([address/0, wallet/0, wallet/1]). +-export([address/0, address/1, wallet/0, wallet/1]). -include("include/hb.hrl"). %% @doc Initialize system-wide settings for the hyperbeam node. diff --git a/src/hb_ao.erl b/src/hb_ao.erl index f26e29f28..8ab031b45 100644 --- a/src/hb_ao.erl +++ b/src/hb_ao.erl @@ -142,7 +142,7 @@ resolve(Base, Req, Opts) -> {stage, 1, prepare_multimessage_resolution, {path_parts, PathParts}} ), MessagesToExec = [ Req#{ <<"path">> => Path } || Path <- PathParts ], - ?event(ao_core, + ?event(debug_ao_core, {stage, 1, prepare_multimessage_resolution, @@ -164,7 +164,7 @@ resolve_many([ID], Opts) when ?IS_ID(ID) -> % 2. The main AO-Core logic looks for linkages between message input % pairs and outputs. With only a single ID, there is not a valid pairing % to use in looking up a cached result. - ?event(ao_core, {stage, na, resolve_directly_to_id, ID, {opts, Opts}}, Opts), + ?event(debug_ao_core, {stage, na, resolve_directly_to_id, ID, {opts, Opts}}, Opts), try {ok, ensure_message_loaded(ID, Opts)} catch _:_:_ -> {error, not_found} end; @@ -190,20 +190,20 @@ resolve_many({as, DevID, Msg}, Opts) -> resolve_many([{resolve, Subres}], Opts) -> resolve_many(Subres, Opts); resolve_many(MsgList, Opts) -> - ?event(ao_core, {resolve_many, MsgList}, Opts), + ?event(debug_ao_core, {resolve_many, MsgList}, Opts), Res = do_resolve_many(MsgList, Opts), - ?event(ao_core, {resolve_many_complete, {res, Res}, {reqs, MsgList}}, Opts), + ?event(debug_ao_core, {resolve_many_complete, {res, Res}, {reqs, MsgList}}, Opts), Res. do_resolve_many([], _Opts) -> {failure, <<"Attempted to resolve an empty message sequence.">>}; do_resolve_many([Res], Opts) -> - ?event(ao_core, {stage, 11, resolve_complete, Res}), + ?event(debug_ao_core, {stage, 11, resolve_complete, Res}), hb_cache:ensure_loaded(maybe_force_message(Res, Opts), Opts); do_resolve_many([Base, Req | MsgList], Opts) -> - ?event(ao_core, {stage, 0, resolve_many, {base, Base}, {req, Req}}), + ?event(debug_ao_core, {stage, 0, resolve_many, {base, Base}, {req, Req}}), case resolve_stage(1, Base, Req, Opts) of {ok, Res} -> - ?event(ao_core, + ?event(debug_ao_core, { stage, 13, @@ -216,19 +216,19 @@ do_resolve_many([Base, Req | MsgList], Opts) -> do_resolve_many([Res | MsgList], Opts); Res -> % The result is not a resolvable message. Return it. - ?event(ao_core, {stage, 13, resolve_many_terminating_early, Res}), + ?event(debug_ao_core, {stage, 13, resolve_many_terminating_early, Res}), maybe_force_message(Res, Opts) end. resolve_stage(1, Link, Req, Opts) when ?IS_LINK(Link) -> % If the first message is a link, we should load the message and % continue with the resolution. - ?event(ao_core, {stage, 1, resolve_base_link, {link, Link}}, Opts), + ?event(debug_ao_core, {stage, 1, resolve_base_link, {link, Link}}, Opts), resolve_stage(1, hb_cache:ensure_loaded(Link, Opts), Req, Opts); resolve_stage(1, Base, Link, Opts) when ?IS_LINK(Link) -> % If the second message is a link, we should load the message and % continue with the resolution. - ?event(ao_core, {stage, 1, resolve_req_link, {link, Link}}, Opts), + ?event(debug_ao_core, {stage, 1, resolve_req_link, {link, Link}}, Opts), resolve_stage(1, Base, hb_cache:ensure_loaded(Link, Opts), Opts); resolve_stage(1, {as, DevID, Ref}, Req, Opts) when ?IS_ID(Ref) orelse ?IS_LINK(Ref) -> % Normalize `as' requests with a raw ID or link as the path. Links will be @@ -237,27 +237,27 @@ resolve_stage(1, {as, DevID, Ref}, Req, Opts) when ?IS_ID(Ref) orelse ?IS_LINK(R resolve_stage(1, {as, DevID, Link}, Req, Opts) when ?IS_LINK(Link) -> % If the first message is an `as' with a link, we should load the message and % continue with the resolution. - ?event(ao_core, {stage, 1, resolve_base_as_link, {link, Link}}, Opts), + ?event(debug_ao_core, {stage, 1, resolve_base_as_link, {link, Link}}, Opts), resolve_stage(1, {as, DevID, hb_cache:ensure_loaded(Link, Opts)}, Req, Opts); resolve_stage(1, {as, DevID, Raw = #{ <<"path">> := ID }}, Req, Opts) when ?IS_ID(ID) -> % If the first message is an `as' with an ID, we should load the message and % apply the non-path elements of the sub-request to it. - ?event(ao_core, {stage, 1, subresolving_with_load, {dev, DevID}, {id, ID}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolving_with_load, {dev, DevID}, {id, ID}}, Opts), RemBase = hb_maps:without([<<"path">>], Raw, Opts), - ?event(subresolution, {loading_message, {id, ID}, {params, RemBase}}, Opts), + ?event(debug_subresolution, {loading_message, {id, ID}, {params, RemBase}}, Opts), Baseb = ensure_message_loaded(ID, Opts), - ?event(subresolution, {loaded_message, {msg, Baseb}}, Opts), + ?event(debug_subresolution, {loaded_message, {msg, Baseb}}, Opts), Basec = hb_maps:merge(Baseb, RemBase, Opts), - ?event(subresolution, {merged_message, {msg, Basec}}, Opts), + ?event(debug_subresolution, {merged_message, {msg, Basec}}, Opts), Based = set(Basec, <<"device">>, DevID, Opts), - ?event(subresolution, {loaded_parameterized_message, {msg, Based}}, Opts), + ?event(debug_subresolution, {loaded_parameterized_message, {msg, Based}}, Opts), resolve_stage(1, Based, Req, Opts); resolve_stage(1, Raw = {as, DevID, SubReq}, Req, Opts) -> % Set the device of the message to the specified one and resolve the sub-path. % As this is the first message, we will then continue to execute the request % on the result. - ?event(ao_core, {stage, 1, subresolving_base, {dev, DevID}, {subreq, SubReq}}, Opts), - ?event(subresolution, {as, {dev, DevID}, {subreq, SubReq}, {req, Req}}), + ?event(debug_ao_core, {stage, 1, subresolving_base, {dev, DevID}, {subreq, SubReq}}, Opts), + ?event(debug_subresolution, {as, {dev, DevID}, {subreq, SubReq}, {req, Req}}), case subresolve(SubReq, DevID, SubReq, Opts) of {ok, SubRes} -> % The subresolution has returned a new message. Continue with it. @@ -276,7 +276,7 @@ resolve_stage(1, RawBase, ReqOuter = #{ <<"path">> := {as, DevID, ReqInner} }, O % Set the device to the specified `DevID' and resolve the message. Merging % the `ReqInner' into the `ReqOuter' message first. We return the result % of the sub-resolution directly. - ?event(ao_core, {stage, 1, subresolving_from_request, {dev, DevID}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolving_from_request, {dev, DevID}}, Opts), LoadedInner = ensure_message_loaded(ReqInner, Opts), Req = hb_maps:merge( @@ -296,17 +296,17 @@ resolve_stage(1, RawBase, ReqOuter = #{ <<"path">> := {as, DevID, ReqInner} }, O resolve_stage(1, {resolve, Subres}, Req, Opts) -> % If the first message is a `{resolve, Subres}' tuple, we should execute it % directly, then apply the request to the result. - ?event(ao_core, {stage, 1, subresolving_base_message, {subres, Subres}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolving_base_message, {subres, Subres}}, Opts), % Unlike the `request' case for pre-subresolutions, we do not need to unset % the `force_message' option, because the result should be a message, anyway. % If it is not, it is more helpful to have the message placed into the `body' % of a result, which can then be executed upon. case resolve_many(Subres, Opts) of {ok, Base} -> - ?event(ao_core, {stage, 1, subresolve_success, {new_base, Base}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolve_success, {new_base, Base}}, Opts), resolve_stage(1, Base, Req, Opts); OtherRes -> - ?event(ao_core, + ?event(debug_ao_core, {stage, 1, subresolve_failed, @@ -321,7 +321,7 @@ resolve_stage(1, Base, {resolve, Subres}, Opts) -> % execute the subresolution directly to gain the underlying `Req' for % our execution. We assume that the subresolution is already in a normalized, % executable form, so we pass it to `resolve_many' for execution. - ?event(ao_core, {stage, 1, subresolving_request_message, {subres, Subres}}, Opts), + ?event(debug_ao_core, {stage, 1, subresolving_request_message, {subres, Subres}}, Opts), % We make sure to unset the `force_message' option so that if the subresolution % returns a literal, the rest of `resolve' will normalize it to a path. case resolve_many(Subres, maps:without([force_message], Opts)) of @@ -348,31 +348,31 @@ resolve_stage(1, Base, {resolve, Subres}, Opts) -> end; resolve_stage(1, Base, Req, Opts) when is_list(Base) -> % Normalize lists to numbered maps (base=1) if necessary. - ?event(ao_core, {stage, 1, list_normalize}, Opts), + ?event(debug_ao_core, {stage, 1, list_normalize}, Opts), resolve_stage(1, normalize_keys(Base, Opts), Req, Opts ); resolve_stage(1, Base, NonMapReq, Opts) when not is_map(NonMapReq) -> - ?event(ao_core, {stage, 1, path_normalize}), + ?event(debug_ao_core, {stage, 1, path_normalize}), resolve_stage(1, Base, #{ <<"path">> => NonMapReq }, Opts); resolve_stage(1, RawBase, RawReq, Opts) -> % Normalize the path to a private key containing the list of remaining % keys to resolve. - ?event(ao_core, {stage, 1, normalize}, Opts), + ?event(debug_ao_core, {stage, 1, normalize}, Opts), Base = normalize_keys(RawBase, Opts), Req = normalize_keys(RawReq, Opts), resolve_stage(2, Base, Req, Opts); resolve_stage(2, Base, Req, Opts) -> - ?event(ao_core, {stage, 2, cache_lookup}, Opts), + ?event(debug_ao_core, {stage, 2, cache_lookup}, Opts), % Lookup request in the cache. If we find a result, return it. % If we do not find a result, we continue to the next stage, % unless the cache lookup returns `halt' (the user has requested that we % only return a result if it is already in the cache). case hb_cache_control:maybe_lookup(Base, Req, Opts) of {ok, Res} -> - ?event(ao_core, {stage, 2, cache_hit, {res, Res}, {opts, Opts}}, Opts), + ?event(debug_ao_core, {stage, 2, cache_hit, {res, Res}, {opts, Opts}}, Opts), {ok, Res}; {continue, NewBase, NewReq} -> resolve_stage(3, NewBase, NewReq, Opts); @@ -381,10 +381,10 @@ resolve_stage(2, Base, Req, Opts) -> resolve_stage(3, Base, Req, Opts) when not is_map(Base) or not is_map(Req) -> % Validation check: If the messages are not maps, we cannot find a key % in them, so return not_found. - ?event(ao_core, {stage, 3, validation_check_type_error}, Opts), + ?event(debug_ao_core, {stage, 3, validation_check_type_error}, Opts), {error, not_found}; resolve_stage(3, Base, Req, Opts) -> - ?event(ao_core, {stage, 3, validation_check}, Opts), + ?event(debug_ao_core, {stage, 3, validation_check}, Opts), % Validation checks: If `paranoid_message_verification' is enabled, we should % verify the base and request messages prior to execution. hb_message:paranoid_verify( @@ -398,7 +398,7 @@ resolve_stage(3, Base, Req, Opts) -> ), resolve_stage(4, Base, Req, Opts); resolve_stage(4, Base, Req, Opts) -> - ?event(ao_core, {stage, 4, persistent_resolver_lookup}, Opts), + ?event(debug_ao_core, {stage, 4, persistent_resolver_lookup}, Opts), % Persistent-resolver lookup: Search for local (or Distributed % Erlang cluster) processes that are already performing the execution. % Before we search for a live executor, we check if the device specifies @@ -461,7 +461,7 @@ resolve_stage(4, Base, Req, Opts) -> end end. resolve_stage(5, Base, Req, ExecName, Opts) -> - ?event(ao_core, {stage, 5, device_lookup}, Opts), + ?event(debug_ao_core, {stage, 5, device_lookup}, Opts), % Device lookup: Find the Erlang function that should be utilized to % execute Req on Base. {ResolvedFunc, NewOpts} = @@ -528,7 +528,7 @@ resolve_stage(5, Base, Req, ExecName, Opts) -> end, resolve_stage(6, ResolvedFunc, Base, Req, ExecName, NewOpts). resolve_stage(6, Func, Base, Req, ExecName, Opts) -> - ?event(ao_core, {stage, 6, ExecName, execution}, Opts), + ?event(debug_ao_core, {stage, 6, ExecName, execution}, Opts), % Execution. ExecOpts = execution_opts(Opts), Args = @@ -542,7 +542,7 @@ resolve_stage(6, Func, Base, Req, ExecName, Opts) -> TruncatedArgs = hb_ao_device:truncate_args(Func, Args), MsgRes = maybe_profiled_apply(Func, TruncatedArgs, Base, Req, Opts), ?event( - ao_result, + debug_ao_result, { ao_result, {exec_name, ExecName}, @@ -597,7 +597,7 @@ resolve_stage(6, Func, Base, Req, ExecName, Opts) -> ), resolve_stage(7, Base, Req, Res, ExecName, Opts); resolve_stage(7, Base, Req, {St, Res}, ExecName, Opts = #{ on := On = #{ <<"step">> := _ }}) -> - ?event(ao_core, {stage, 7, ExecName, executing_step_hook, {on, On}}, Opts), + ?event(debug_ao_core, {stage, 7, ExecName, executing_step_hook, {on, On}}, Opts), % If the `step' hook is defined, we execute it. Note: This function clause % matches directly on the `on' key of the `Opts' map. This is in order to % remove the expensive lookup check that would otherwise be performed on every @@ -623,18 +623,18 @@ resolve_stage(7, Base, Req, {St, Res}, ExecName, Opts = #{ on := On = #{ <<"step Error end; resolve_stage(7, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 7, ExecName, no_step_hook}, Opts), + ?event(debug_ao_core, {stage, 7, ExecName, no_step_hook}, Opts), resolve_stage(8, Base, Req, Res, ExecName, Opts); resolve_stage(8, Base, Req, {ok, {resolve, Sublist}}, ExecName, Opts) -> - ?event(ao_core, {stage, 8, ExecName, subresolve_result}, Opts), + ?event(debug_ao_core, {stage, 8, ExecName, subresolve_result}, Opts), % If the result is a `{resolve, Sublist}' tuple, we need to execute it % as a sub-resolution. resolve_stage(9, Base, Req, resolve_many(Sublist, Opts), ExecName, Opts); resolve_stage(8, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), + ?event(debug_ao_core, {stage, 8, ExecName, no_subresolution_necessary}, Opts), resolve_stage(9, Base, Req, Res, ExecName, Opts); resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> - ?event(ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), + ?event(debug_ao_core, {stage, 9, ExecName, generate_hashpath}, Opts), % Cryptographic linking. Now that we have generated the result, we % need to cryptographically link the output to its input via a hashpath. resolve_stage(10, Base, Req, @@ -663,7 +663,7 @@ resolve_stage(9, Base, Req, {ok, Res}, ExecName, Opts) when is_map(Res) -> Opts ); resolve_stage(9, Base, Req, {Status, Res}, ExecName, Opts) when is_map(Res) -> - ?event(ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), + ?event(debug_ao_core, {stage, 9, ExecName, abnormal_status_reset_hashpath}, Opts), ?event(hashpath, {resetting_hashpath_res, {base, Base}, {req, Req}, {opts, Opts}}), % Skip cryptographic linking and reset the hashpath if the result is abnormal. Priv = hb_private:from_message(Res), @@ -672,27 +672,27 @@ resolve_stage(9, Base, Req, {Status, Res}, ExecName, Opts) when is_map(Res) -> {Status, Res#{ <<"priv">> => maps:without([<<"hashpath">>], Priv) }}, ExecName, Opts); resolve_stage(9, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), + ?event(debug_ao_core, {stage, 9, ExecName, non_map_result_skipping_hash_path}, Opts), % Skip cryptographic linking and continue if we don't have a map that can have % a hashpath at all. resolve_stage(10, Base, Req, Res, ExecName, Opts); resolve_stage(10, Base, Req, {ok, Res}, ExecName, Opts) -> - ?event(ao_core, {stage, 10, ExecName, result_caching}, Opts), + ?event(debug_ao_core, {stage, 10, ExecName, result_caching}, Opts), % Result caching: Optionally, cache the result of the computation locally. hb_cache_control:maybe_store(Base, Req, Res, Opts), resolve_stage(11, Base, Req, {ok, Res}, ExecName, Opts); resolve_stage(10, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), + ?event(debug_ao_core, {stage, 10, ExecName, abnormal_status_skip_caching}, Opts), % Skip result caching if the result is abnormal. resolve_stage(11, Base, Req, Res, ExecName, Opts); resolve_stage(11, Base, Req, Res, ExecName, Opts) -> - ?event(ao_core, {stage, 11, ExecName}, Opts), + ?event(debug_ao_core, {stage, 11, ExecName}, Opts), % Notify processes that requested the resolution while we were executing and % unregister ourselves from the group. hb_persistent:unregister_notify(ExecName, Req, Res, Opts), resolve_stage(12, Base, Req, Res, ExecName, Opts); resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> - ?event(ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), + ?event(debug_ao_core, {stage, 12, ExecName, maybe_spawn_worker}, Opts), % Check if we should fork out a new worker process for the current execution case {is_map(Res), hb_opts:get(spawn_worker, false, Opts#{ prefer => local })} of {A, B} when (A == false) or (B == false) -> @@ -704,7 +704,7 @@ resolve_stage(12, _Base, _Req, {ok, Res} = Res, ExecName, Opts) -> Res end; resolve_stage(12, _Base, _Req, OtherRes, ExecName, Opts) -> - ?event(ao_core, {stage, 12, ExecName, abnormal_status_skip_spawning}, Opts), + ?event(debug_ao_core, {stage, 12, ExecName, abnormal_status_skip_spawning}, Opts), OtherRes. %% @doc Execute a sub-resolution. @@ -880,7 +880,7 @@ error_infinite(Base, Req, Opts) -> error_execution(ExecGroup, Req, Whence, {Class, Exception, Stacktrace}, Opts) -> Error = {error, Whence, {Class, Exception, Stacktrace}}, hb_persistent:unregister_notify(ExecGroup, Req, Error, Opts), - ?event(ao_core, {handle_error, Error, {opts, Opts}}, Opts), + ?event(debug_ao_core, {handle_error, Error, {opts, Opts}}, Opts), case hb_opts:get(error_strategy, throw, Opts) of throw -> erlang:raise(Class, Exception, Stacktrace); _ -> Error diff --git a/src/hb_ao_device.erl b/src/hb_ao_device.erl index 22ed65f5c..ffbf5b027 100644 --- a/src/hb_ao_device.erl +++ b/src/hb_ao_device.erl @@ -439,11 +439,11 @@ do_is_direct_key_access(error, Key, Opts) -> do_is_direct_key_access(<<"message@1.0">>, Key, _Opts) -> not lists:member(Key, ?MESSAGE_KEYS); do_is_direct_key_access(Dev, NormKey, Opts) -> - ?event(read_cached, {calculating_info, {device, Dev}}), + ?event(debug_read_cached, {calculating_info, {device, Dev}}), case info(#{ <<"device">> => Dev}, Opts) of Info = #{ exports := Exports } when not is_map_key(handler, Info) andalso not is_map_key(default, Info) -> - ?event(read_cached, + ?event(debug_read_cached, {exports, {device, Dev}, {key, NormKey}, diff --git a/src/hb_ao_test_vectors.erl b/src/hb_ao_test_vectors.erl index a9677790a..dae90fff7 100644 --- a/src/hb_ao_test_vectors.erl +++ b/src/hb_ao_test_vectors.erl @@ -841,6 +841,10 @@ load_as_test(Opts) -> <<"device">> => <<"test-device@1.0">>, <<"test_func">> => #{ <<"test_key">> => <<"MESSAGE">> } }, + % There is a race condition where we write to the store and a + % reset happens making not read the written value. + % Lower this number can still produce flaky test + timer:sleep(100), {ok, ID} = hb_cache:write(Msg, Opts), {ok, ReadMsg} = hb_cache:read(ID, Opts), ?assert(hb_message:match(Msg, ReadMsg, primary, Opts)), @@ -969,7 +973,7 @@ step_hook_test(InitOpts) -> #{ <<"step">> => fun(_, Req, _) -> - ?event(ao_core, {step_hook, {self(), Ref}}), + ?event(debug_ao_core, {step_hook, {self(), Ref}}), Self ! {step, Ref}, {ok, Req} end @@ -1149,4 +1153,4 @@ benchmark_set_multiple_deep_test(Opts) -> <<"Set two keys operations:">>, ?BENCHMARK_ITERATIONS, Time - ). \ No newline at end of file + ). diff --git a/src/hb_cache.erl b/src/hb_cache.erl index 2968e7792..6980bf4f2 100644 --- a/src/hb_cache.erl +++ b/src/hb_cache.erl @@ -123,7 +123,7 @@ ensure_loaded(Ref, Link = {link, ID, LinkOpts = #{ <<"lazy">> := true }}, RawOpt end, case CacheReadResult of {ok, LoadedMsg} -> - ?event(caching, + ?event(debug_caching, {lazy_loaded, {link, ID}, {msg, LoadedMsg}, diff --git a/src/hb_event.erl b/src/hb_event.erl index 2507ceb75..426d73d50 100644 --- a/src/hb_event.erl +++ b/src/hb_event.erl @@ -6,8 +6,8 @@ -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(OVERLOAD_QUEUE_LENGTH, 10000). --define(MAX_MEMORY, 1_000_000_000). % 1GB +-define(OVERLOAD_QUEUE_LENGTH, 10_000). +-define(MAX_MEMORY, 50_000_000). % 50 MB -define(MAX_EVENT_NAME_LENGTH, 100). -ifdef(NO_EVENTS). @@ -74,6 +74,8 @@ increment(debug_linkify, _Message, _Opts, _Count) -> ignored; increment(debug_id, _Message, _Opts, _Count) -> ignored; increment(debug_enc, _Message, _Opts, _Count) -> ignored; increment(debug_commitments, _Message, _Opts, _Count) -> ignored; +increment(message_set, _Message, _Opts, _Count) -> ignored; +increment(read_cached, _Message, _Opts, _Count) -> ignored; increment(ao_core, _Message, _Opts, _Count) -> ignored; increment(ao_internal, _Message, _Opts, _Count) -> ignored; increment(ao_devices, _Message, _Opts, _Count) -> ignored; @@ -148,43 +150,47 @@ raw_counters() -> []. -else. raw_counters() -> - ets:tab2list(prometheus_counter_table). + ets:match_object( + prometheus_counter_table, + {{default, <<"event">>, '_', '_'}, '_', '_'} + ). -endif. %% @doc Find the event server, creating it if it doesn't exist. We cache the %% result in the process dictionary to avoid looking it up multiple times. find_event_server() -> - case erlang:get({event_server, ?MODULE}) of - {cached, Pid} -> Pid; - undefined -> - PID = - case hb_name:lookup(?MODULE) of - Pid when is_pid(Pid) -> Pid; - undefined -> - NewServer = spawn(fun() -> server() end), - hb_name:register(?MODULE, NewServer), - NewServer - end, - erlang:put({event_server, ?MODULE}, {cached, PID}), - PID - end. + hb_name:singleton(?MODULE, fun() -> server() end). server() -> - await_prometheus_started(), - prometheus_counter:declare( + hb_prometheus:ensure_started(), + ensure_event_counter(), + handle_events(). + +ensure_event_counter() -> + hb_prometheus:declare( + counter, [ {name, <<"event">>}, {help, <<"AO-Core execution events">>}, {labels, [topic, event]} - ]), - handle_events(). + ]). + handle_events() -> + handle_events(0). +handle_events(N) -> receive - {increment, TopicBin, EventName, Count} -> + {increment, Topic, Event, Count} -> + BatchCount = 0, + prometheus_counter:inc(<<"event">>, [Topic, Event], Count + BatchCount), + check_overload({Topic, Event}, N), + handle_events(N + 1) + end. + +check_overload(Last, N) -> + case N rem 1000 of + 0 -> case erlang:process_info(self(), message_queue_len) of {message_queue_len, Len} when Len > ?OVERLOAD_QUEUE_LENGTH -> - % Print a warning, but do so less frequently the more - % overloaded the system is. {memory, MemorySize} = erlang:process_info(self(), memory), case rand:uniform(max(1000, Len - ?OVERLOAD_QUEUE_LENGTH)) of 1 -> @@ -192,14 +198,12 @@ handle_events() -> {warning, prometheus_event_queue_overloading, {queue, Len}, - {current_message, EventName}, + {last_event, Last}, {memory_bytes, MemorySize} } ); _ -> ignored end, - % If the size of this process is too large, exit such that - % we can be restarted by the next caller. case MemorySize of MemorySize when MemorySize > ?MAX_MEMORY -> ?debug_print( @@ -207,26 +211,15 @@ handle_events() -> prometheus_event_queue_terminating_on_memory_overload, {queue, Len}, {memory_bytes, MemorySize}, - {current_message, EventName} + {last_event, Last} } ), exit(memory_overload); _ -> no_action end; _ -> ignored - end, - prometheus_counter:inc(<<"event">>, [TopicBin, EventName], Count), - handle_events() - end. - -%% @doc Delay the event server until prometheus is started. -await_prometheus_started() -> - receive - Msg -> - case application:get_application(prometheus) of - undefined -> await_prometheus_started(); - _ -> self() ! Msg, ok - end + end; + _ -> ok end. parse_name(Name) when is_tuple(Name) -> @@ -239,6 +232,8 @@ parse_name(Name) no_event_name; parse_name(Name) when is_list(Name) -> iolist_to_binary(Name); +parse_name(Name) when is_binary(Name) -> + Name; parse_name(_) -> no_event_name. %%% Benchmark tests @@ -282,3 +277,137 @@ benchmark_increment_test() -> hb_test_utils:benchmark_print(<<"Incremented">>, <<"events">>, Iterations), ?assert(Iterations >= 1000), ok. + +-ifdef(NO_EVENTS). +benchmark_drain_rate_test() -> ok. +batch_correctness_test() -> ok. +overload_checks_past_first_thousand_test() -> ok. +-else. +benchmark_drain_rate_test() -> + NumKeys = 50, + NumEvents = 100000, + log(warmup, {warmup, 0}), + timer:sleep(100), + EventPid = hb_name:lookup(?MODULE), + wait_drain(EventPid, 5000), + erlang:suspend_process(EventPid), + Keys = + [ + { + hb_util:bin([<<"corr-topic-">>, hb_util:int(K)]), + hb_util:bin([<<"corr-event-">>, hb_util:int(K)]) + } + || + K <- lists:seq(1, NumKeys) + ], + fill_mailbox(EventPid, NumEvents, Keys), + erlang:resume_process(EventPid), + {DrainTime, _} = + timer:tc( + fun() -> + wait_drain(EventPid, 30000) + end + ), + DrainRate = round(NumEvents / (max(1, DrainTime) / 1_000_000)), + hb_test_utils:benchmark_print( + <<"Drained">>, + <<"events">>, + DrainRate, + 1 + ), + ?assert(DrainRate >= 10000), + ok. + +batch_correctness_test() -> + log(warmup, {warmup, 0}), + timer:sleep(100), + EventPid = hb_name:lookup(?MODULE), + wait_drain(EventPid, 5000), + NumKeys = 5, + N = 30_000, + Keys = [{list_to_binary("corr_topic_" ++ integer_to_list(K)), + list_to_binary("corr_event_" ++ integer_to_list(K))} + || K <- lists:seq(1, NumKeys)], + Before = counters(), + BeforeCounts = [{T, E, deep_get([T, E], Before, 0)} || {T, E} <- Keys], + erlang:suspend_process(EventPid), + lists:foreach(fun(I) -> + {T, E} = lists:nth((I rem NumKeys) + 1, Keys), + EventPid ! {increment, T, E, 1} + end, lists:seq(1, N)), + erlang:resume_process(EventPid), + wait_drain(EventPid, 30000), + After = counters(), + PerKey = N div NumKeys, + lists:foreach(fun({T, E, BeforeVal}) -> + AfterVal = deep_get([T, E], After, 0), + ?assertEqual(PerKey, AfterVal - BeforeVal) + end, BeforeCounts), + ok. + +overload_checks_past_first_thousand_test() -> + {EventPid, Ref} = + spawn_monitor( + fun() -> + hb_prometheus:ensure_started(), + ensure_event_counter(), + handle_events(1000) + end + ), + erlang:suspend_process(EventPid), + Topic = lists:duplicate(256, $a), + Event = lists:duplicate(256, $b), + lists:foreach( + fun(_) -> + EventPid ! {increment, Topic, Event, 1} + end, + lists:seq(1, ?OVERLOAD_QUEUE_LENGTH + 100) + ), + {message_queue_len, QueueLen} = + erlang:process_info(EventPid, message_queue_len), + {memory, MemorySize} = erlang:process_info(EventPid, memory), + ?assert(QueueLen > ?OVERLOAD_QUEUE_LENGTH), + ?assert(MemorySize > ?MAX_MEMORY), + erlang:resume_process(EventPid), + receive + {'DOWN', Ref, process, EventPid, memory_overload} -> + ok; + {'DOWN', Ref, process, EventPid, Reason} -> + ?assertEqual(memory_overload, Reason) + after 5000 -> + exit(EventPid, kill), + error(memory_overload_not_triggered) + end. + +deep_get([Group, Name], Map, Default) -> + case maps:get(Group, Map, undefined) of + undefined -> Default; + Inner -> maps:get(Name, Inner, Default) + end. + +%% @doc Fill the event server mailbox with a list of keys. Rotate the keys to +%% ensure that we are testing the event server's ability to handle many different +%% types of event. +fill_mailbox(_Pid, 0, _Keys) -> ok; +fill_mailbox(Pid, N, Keys = [{Topic, Event}|_]) -> + Pid ! {increment, Topic, Event, 1}, + fill_mailbox(Pid, N - 1, hb_util:shuffle(Keys)). + +wait_drain(Pid, Timeout) -> + Deadline = erlang:monotonic_time(millisecond) + Timeout, + wait_drain_loop(Pid, Deadline). + +wait_drain_loop(Pid, Deadline) -> + case erlang:process_info(Pid, message_queue_len) of + {message_queue_len, 0} -> ok; + {message_queue_len, _} -> + case erlang:monotonic_time(millisecond) >= Deadline of + true -> error(drain_timeout); + false -> + timer:sleep(10), + wait_drain_loop(Pid, Deadline) + end; + undefined -> + error(event_server_dead) + end. +-endif. diff --git a/src/hb_format.erl b/src/hb_format.erl index b05f4b4c3..ee07ee0ed 100644 --- a/src/hb_format.erl +++ b/src/hb_format.erl @@ -401,9 +401,9 @@ escape_format(Else) -> Else. %% @doc Format an error message as a string. error(ErrorMsg, Opts) -> - Type = hb_ao:get(<<"type">>, ErrorMsg, <<"">>, Opts), - Details = hb_ao:get(<<"details">>, ErrorMsg, <<"">>, Opts), - Stacktrace = hb_ao:get(<<"stacktrace">>, ErrorMsg, <<"">>, Opts), + Type = hb_maps:get(<<"type">>, ErrorMsg, <<"[No type]">>, Opts), + Details = hb_maps:get(<<"details">>, ErrorMsg, <<"[No details]">>, Opts), + Stacktrace = hb_maps:get(<<"stacktrace">>, ErrorMsg, <<"[No trace]">>, Opts), hb_util:bin( [ <<"Termination type: '">>, Type, diff --git a/src/hb_gateway_client.erl b/src/hb_gateway_client.erl index e29479144..8826bfc39 100644 --- a/src/hb_gateway_client.erl +++ b/src/hb_gateway_client.erl @@ -110,14 +110,19 @@ data(ID, Opts) -> }, case hb_http:request(Req, Opts) of {ok, Res} -> + Data = + case hb_maps:find(<<"data">>, Res, Opts) of + {ok, D} -> D; + _ -> hb_ao:get(<<"body">>, Res, <<>>, Opts) + end, ?event(gateway, {data, {id, ID}, {response, Res}, - {body, hb_ao:get(<<"body">>, Res, <<>>, Opts)} + {data, Data} } ), - {ok, hb_ao:get(<<"body">>, Res, <<>>, Opts)}; + {ok, Data}; Res -> ?event(gateway, {request_error, {id, ID}, {response, Res}}), {error, no_viable_gateway} diff --git a/src/hb_http.erl b/src/hb_http.erl index 799554835..fefaf4f26 100644 --- a/src/hb_http.erl +++ b/src/hb_http.erl @@ -129,7 +129,6 @@ request(Method, Peer, Path, RawMessage, Opts) -> Error -> Error end. - request_response(Method, Peer, Path, Response, Duration, Opts) -> {_ErlStatus, Status, Headers, Body} = Response, @@ -166,7 +165,7 @@ request_response(Method, Peer, Path, Response, Duration, Opts) -> % constructed from the header key-value pair list. HeaderMap = hb_maps:merge(hb_maps:from_list(Headers), MaybeSetCookie, Opts), NormHeaderMap = hb_ao:normalize_keys(HeaderMap, Opts), - ?event(http_outbound, + ?event(debug_http_outbound, {normalized_response_headers, {norm_header_map, NormHeaderMap}}, Opts ), @@ -186,7 +185,7 @@ request_response(Method, Peer, Path, Response, Duration, Opts) -> Key when is_binary(Key) -> Msg = http_response_to_httpsig(Status, NormHeaderMap, Body, Opts), ?event( - http_outbound, + debug_http_outbound, {result_is_single_key, {key, Key}, {msg, Msg}}, Opts ), @@ -227,10 +226,9 @@ request_response(Method, Peer, Path, Response, Duration, Opts) -> ) end. - %% @doc Convert an HTTP response to a message. outbound_result_to_message(<<"ans104@1.0">>, Status, Headers, Body, Opts) -> - ?event(http_outbound, + ?event(debug_http_outbound, {result_is_ans104, {headers, Headers}, {body, Body}}, Opts ), @@ -261,10 +259,21 @@ outbound_result_to_message(<<"ans104@1.0">>, Status, Headers, Body, Opts) -> outbound_result_to_message(<<"httpsig@1.0">>, Status, Headers, Body, Opts) end; outbound_result_to_message(<<"httpsig@1.0">>, Status, Headers, Body, Opts) -> - ?event(http_outbound, {result_is_httpsig, {body, Body}}, Opts), + ?event(debug_http_outbound, {result_is_httpsig, {body, Body}}, Opts), { hb_http_client:response_status_to_atom(Status), http_response_to_httpsig(Status, Headers, Body, Opts) + }; +outbound_result_to_message(Codec, Status, Headers, Body, Opts) -> + ?event(debug, {headers, Headers}), + { + hb_http_client:response_status_to_atom(Status), + hb_message:convert( + Body, + <<"structured@1.0">>, + Codec, + Opts + ) }. %% @doc Convert a HTTP response to a httpsig message. @@ -302,7 +311,7 @@ route_to_request(M, {ok, #{ <<"uri">> := XPath, <<"opts">> := ReqOpts}}, Opts) - % The request is a direct HTTP URL, so we need to split the path into a % host and path. URI = uri_string:parse(XPath), - ?event(http_outbound, {parsed_uri, {uri, {explicit, URI}}}), + ?event(debug_http_outbound, {parsed_uri, {uri, {explicit, URI}}}), Method = hb_ao:get(<<"method">>, M, <<"GET">>, Opts), % We must remove the path and host from the message, because they are not % valid for outbound requests. The path is retrieved from the route, and @@ -328,10 +337,10 @@ route_to_request(M, {ok, #{ <<"uri">> := XPath, <<"opts">> := ReqOpts}}, Opts) - Query -> [<<"?", Query/binary>>] end, Path = iolist_to_binary(PathParts), - ?event(http_outbound, {parsed_req, {node, Node}, {method, Method}, {path, Path}}), + ?event(debug_http_outbound, {parsed_req, {node, Node}, {method, Method}, {path, Path}}), {ok, Method, Node, Path, MsgWithoutMeta, hb_util:deep_merge(Opts, ReqOpts, Opts)}; route_to_request(M, {ok, Routes}, Opts) -> - ?event(http_outbound, {found_routes, {req, M}, {routes, Routes}}), + ?event(debug_http_outbound, {found_routes, {req, M}, {routes, Routes}}), % The result is a route, so we leave it to `request' to handle it. Path = hb_ao:get(<<"path">>, M, <<"/">>, Opts), Method = hb_ao:get(<<"method">>, M, <<"GET">>, Opts), @@ -362,7 +371,7 @@ prepare_request(Format, Method, Peer, Path, RawMessage, Opts) -> Opts ), {ok, CookieReset} = dev_codec_cookie:reset(Message, Opts), - ?event(http, {cookie_lines, CookieLines}), + ?event(debug_http, {cookie_lines, CookieLines}), { #{ <<"cookie">> => CookieLines }, CookieReset @@ -411,8 +420,8 @@ prepare_request(Format, Method, Peer, Path, RawMessage, Opts) -> ), Body = hb_maps:get(<<"body">>, FullEncoding, <<>>, Opts), Headers = hb_maps:without([<<"body">>], FullEncoding, Opts), - ?event(http, {request_headers, {explicit, {headers, Headers}}}), - ?event(http, {request_body, {explicit, {body, Body}}}), + ?event(debug_http, {request_headers, {explicit, {headers, Headers}}}), + ?event(debug_http, {request_body, {explicit, {body, Body}}}), hb_maps:merge( ReqBase, #{ headers => maps:merge(MaybeCookie, Headers), body => Body }, @@ -491,7 +500,7 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> ReqHdr = cowboy_req:header(<<"access-control-request-headers">>, Req, <<"">>), HeadersWithCors = add_cors_headers(HeadersBeforeCors, ReqHdr, Opts), EncodedHeaders = hb_private:reset(HeadersWithCors), - ?event(http, + ?event(debug_http, {http_replying, {status, {explicit, Status}}, {path, hb_maps:get(<<"path">>, Req, undefined_path, Opts)}, @@ -502,33 +511,43 @@ reply(InitReq, TABMReq, RawStatus, RawMessage, Opts) -> ), ReqBeforeStream = Req#{ resp_headers => EncodedHeaders }, PostStreamReq = cowboy_req:stream_reply(Status, #{}, ReqBeforeStream), - cowboy_req:stream_body(EncodedBody, nofin, PostStreamReq), + Fin = + case should_finalize_stream(Status, EncodedBody) of + true -> fin; + false -> nofin + end, + cowboy_req:stream_body(EncodedBody, Fin, PostStreamReq), EndTime = os:system_time(millisecond), ReqDuration = EndTime - hb_maps:get(start_time, Req, undefined, Opts), ReplyDuration = EndTime - ReplyStartTime, record_request_metric( - ReqDuration * 1000000, - ReplyDuration * 1000000, - Status + ReqDuration * 1000000, + ReplyDuration * 1000000, + Status ), - ?event(http, {reply_headers, {explicit, PostStreamReq}}), + ?event(debug_http, {reply_headers, {explicit, PostStreamReq}}), ?event(http_server_short, {sent, {status, Status}, - {duration, ReqDuration}, + {duration, EndTime - hb_maps:get(start_time, Req, undefined, Opts)}, + {body_size, byte_size(EncodedBody)}, {method, cowboy_req:method(Req)}, + {ip, {string, real_ip(Req, Opts)}}, {path, {string, uri_string:percent_decode( hb_maps:get(<<"path">>, TABMReq, <<"[NO PATH]">>, Opts) ) } - }, - {body_size, byte_size(EncodedBody)} + } } ), {ok, PostStreamReq, no_state}. +%% @doc Determine if the stream should be finalized. +should_finalize_stream(429, _EncodedBody) -> true; +should_finalize_stream(_, _EncodedBody) -> false. + %% @doc Handle replying with cookies if the message contains them. Returns the %% new Cowboy `Req` object, and the message with the cookies removed. Both %% `set-cookie' and `cookie' fields are treated as viable sources of cookies. @@ -597,7 +616,7 @@ add_cors_headers(Msg, ReqHdr, Opts) -> %% @doc Generate the headers and body for a HTTP response message. encode_reply(Status, TABMReq, Message, Opts) -> Codec = accept_to_codec(TABMReq, Message, Opts), - ?event(http, {encoding_reply, {codec, Codec}, {message, Message}}), + ?event(debug_http, {encoding_reply, {codec, Codec}, {message, Message}}), BaseHdrs = hb_maps:merge( #{ @@ -613,7 +632,7 @@ encode_reply(Status, TABMReq, Message, Opts) -> hb_util:atom( hb_maps:get(<<"accept-bundle">>, TABMReq, false, Opts) ), - ?event(http, + ?event(debug_http, {encoding_reply, {status, Status}, {codec, Codec}, @@ -847,7 +866,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> error -> default_codec(Opts) end end, - ?event(http, + ?event(debug_http, {parsing_req, {path, FullPath}, {query, QueryKeys}, @@ -858,7 +877,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> ?event({req_to_tabm_singleton, {codec, Codec}}), case Codec of <<"httpsig@1.0">> -> - ?event( + ?event(debug_http, {req_to_tabm_singleton, {request, {explicit, Req}, {body, {string, Body}} @@ -897,7 +916,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> ), case ar_tx:verify(TX) of true -> - ?event(tx, {valid_tx_signature, TX}), + ?event(debug_tx, {valid_tx_signature, TX}), StructuredTX = hb_message:convert( TX, @@ -911,7 +930,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> end; Codec -> % Assume that the codec stores the encoded message in the `body' field. - ?event(http, {decoding_body, {codec, Codec}, {body, {string, Body}}}), + ?event(debug_http, {decoding_body, {codec, Codec}, {body, {string, Body}}}), Decoded = hb_message:convert( Body, @@ -920,7 +939,7 @@ req_to_tabm_singleton(Req, Body, Opts) -> Opts ), ReqMessage = hb_maps:merge(PrimitiveMsg, Decoded, Opts), - ?event( + ?event(debug_http, {verifying_encoded_message, {codec, Codec}, {body, {string, Body}}, @@ -951,7 +970,7 @@ httpsig_to_tabm_singleton(PrimMsg, Req, Body, Opts) -> ), Opts ), - ?event(http, {decoded, Decoded}, Opts), + ?event(debug_http, {decoded, Decoded}, Opts), ForceSignedRequests = hb_opts:get(force_signed_requests, false, Opts), case (not ForceSignedRequests) orelse hb_message:verify(Decoded, all, Opts) of true -> @@ -991,7 +1010,7 @@ httpsig_to_tabm_singleton(PrimMsg, Req, Body, Opts) -> %% 1. The path in the message %% 2. The path in the request URI normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> - ?event({adding_method_and_path_from_request, {explicit, Req}}), + ?event(debug_http, {adding_method_and_path_from_request, {explicit, Req}}), Method = cowboy_req:method(Req), MsgPath = hb_maps:get( @@ -1061,39 +1080,43 @@ normalize_unsigned(PrimMsg, Req = #{ headers := RawHeaders }, Msg, Opts) -> <<"">> -> hb_message:without_unless_signed(<<"body">>, WithCookie, Opts); _ -> WithCookie end, + RealIP = real_ip(Req, Opts), WithPeer = case hb_maps:get(<<"ao-peer-port">>, NormalBody, undefined, Opts) of undefined -> NormalBody; P2PPort -> - % Calculate the peer address from the request. We honor the - % `x-real-ip' header if it is present. - RealIP = - case hb_maps:get(<<"x-real-ip">>, RawHeaders, undefined, Opts) of - undefined -> - {{A, B, C, D}, _} = cowboy_req:peer(Req), - hb_util:bin( - io_lib:format( - "~b.~b.~b.~b", - [A, B, C, D] - ) - ); - IP -> IP - end, Peer = <>, (hb_message:without_unless_signed(<<"ao-peer-port">>, NormalBody, Opts))#{ <<"ao-peer">> => Peer } end, + WithPrivIP = hb_private:set(WithPeer, <<"ip">>, RealIP, Opts), % Add device from PrimMsg if present - case maps:get(<<"device">>, PrimMsg, not_found) of - not_found -> WithPeer; - Device -> WithPeer#{<<"device">> => Device} + WithDevice = case maps:get(<<"device">>, PrimMsg, not_found) of + not_found -> WithPrivIP; + Device -> WithPrivIP#{<<"device">> => Device} + end, + Host = cowboy_req:host(Req), + WithDevice#{<<"host">> => Host}. + +%% @doc Determine the caller, honoring the `x-real-ip' header if present. +real_ip(Req = #{ headers := RawHeaders }, Opts) -> + case hb_maps:get(<<"x-real-ip">>, RawHeaders, undefined, Opts) of + undefined -> + {{A, B, C, D}, _} = cowboy_req:peer(Req), + hb_util:bin( + io_lib:format( + "~b.~b.~b.~b", + [A, B, C, D] + ) + ); + IP -> IP end. %%% Metrics init_prometheus() -> hb_prometheus:declare(histogram, [ - {name, http_request_server_reply_duration_seconds}, + {name, http_server_encoding_duration_seconds}, {labels, [status_code]}, {buckets, [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, @@ -1108,7 +1131,7 @@ init_prometheus() -> } ]), hb_prometheus:declare(histogram, [ - {name, http_request_server_duration_seconds}, + {name, http_server_duration_seconds}, {labels, [status_code]}, {buckets, [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, @@ -1124,21 +1147,16 @@ init_prometheus() -> record_request_metric(TotalDuration, ReplyDuration, StatusCode) -> spawn( fun() -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - prometheus_histogram:observe( - http_request_server_duration_seconds, - [StatusCode], - TotalDuration - ), - prometheus_histogram:observe( - http_request_server_reply_duration_seconds, - [StatusCode], - ReplyDuration - ) - end, - ok + hb_prometheus:observe( + TotalDuration, + http_server_duration_seconds, + [StatusCode] + ), + hb_prometheus:observe( + ReplyDuration, + http_server_encoding_duration_seconds, + [StatusCode] + ) end ). diff --git a/src/hb_http_client.erl b/src/hb_http_client.erl index d04de1f1c..c17a6f6f9 100644 --- a/src/hb_http_client.erl +++ b/src/hb_http_client.erl @@ -3,24 +3,36 @@ -module(hb_http_client). -behaviour(gen_server). -include("include/hb.hrl"). --export([start_link/1, init_prometheus/0, response_status_to_atom/1, request/2]). --export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]). +-include("include/hb_opts.hrl"). +-include("include/hb_http_client.hrl"). +%% Public API +-export([request/2, response_status_to_atom/1, setup_conn/1]). +%% GenServer +-export([start_link/1, init/1]). +-export([handle_cast/2, handle_call/3, handle_info/2, terminate/2]). +-export([init_prometheus/0]). -record(state, { - pid_by_peer = #{}, - status_by_pid = #{}, opts = #{} }). --define(DEFAULT_RETRIES, 0). --define(DEFAULT_RETRY_TIME, 1000). --define(DEFAULT_KEEPALIVE_TIMEOUT, 60_000). --define(DEFAULT_CONNECT_TIMEOUT, 60_000). - %%% ================================================================== %%% Public interface. %%% ================================================================== +%% @doc Use Opts to configure connection pool size. +setup_conn(Opts) -> + ConnPoolReadSize = hb_maps:get(conn_pool_read_size,Opts, ?DEFAULT_CONN_POOL_READ_SIZE), + ConnPoolWriteSize = hb_maps:get(conn_pool_write_size,Opts, ?DEFAULT_CONN_POOL_WRITE_SIZE), + ?event( + connection_pool, + {conn, + {pool_read_num, ConnPoolReadSize}, + {pool_write_num, ConnPoolWriteSize} + } + ), + persistent_term:put(?CONN_TERM, {ConnPoolReadSize, ConnPoolWriteSize}). + start_link(Opts) -> gen_server:start_link({local, ?MODULE}, ?MODULE, Opts, []). @@ -43,13 +55,13 @@ request(Args, RemainingRetries, Opts) -> StatusAtom = response_status_to_atom(Status), RetryResponses = hb_opts:get(http_retry_response, [], Opts), case lists:member(StatusAtom, RetryResponses) of - true -> maybe_retry(RemainingRetries, Args, Response, Opts); + true -> maybe_retry(RemainingRetries, Args, Response, Opts); false -> Response end end. do_request(Args, Opts) -> - case hb_opts:get(http_client, gun, Opts) of + case hb_opts:get(http_client, ?DEFAULT_HTTP_CLIENT, Opts) of gun -> gun_req(Args, Opts); httpc -> httpc_req(Args, Opts) end. @@ -88,12 +100,12 @@ httpc_req(Args, Opts) -> body := Body } = Args, ?event({httpc_req, Args}), - {Host, Port} = parse_peer(Peer, Opts), + {ok, {Host, Port}} = parse_peer(Peer, Opts), Scheme = case Port of 443 -> "https"; _ -> "http" end, - ?event(http_client, {httpc_req, {explicit, Args}}), + ?event(debug_http_client, {httpc_req, {explicit, Args}}), URL = binary_to_list(iolist_to_binary([Scheme, "://", Host, ":", integer_to_binary(Port), Path])), FilteredHeaders = hb_maps:without([<<"content-type">>, <<"cookie">>], Headers, Opts), HeaderKV = @@ -133,18 +145,18 @@ httpc_req(Args, Opts) -> end, ?event({http_client_outbound, Method, URL, Request}), HTTPCOpts = [{full_result, true}, {body_format, binary}], - StartTime = os:system_time(millisecond), + StartTime = os:system_time(native), case httpc:request(Method, Request, [], HTTPCOpts) of {ok, {{_, Status, _}, RawRespHeaders, RespBody}} -> download_metric(RespBody), - EndTime = os:system_time(millisecond), + EndTime = os:system_time(native), RespHeaders = [ {list_to_binary(Key), list_to_binary(Value)} || {Key, Value} <- RawRespHeaders ], - ?event(http_client, {httpc_resp, Status, RespHeaders, RespBody}), + ?event(debug_http_client, {httpc_resp, Status, RespHeaders, RespBody}), record_duration(#{ <<"request-method">> => method_to_bin(Method), <<"request-path">> => hb_util:bin(Path), @@ -164,11 +176,12 @@ gun_req(Args, Opts) -> gun_req(Args, ReestablishedConnection, Opts) -> StartTime = os:system_time(native), #{ peer := Peer, path := Path, method := Method } = Args, + ConnType = get_connection_type(Method), Response = - case catch gen_server:call(?MODULE, {get_connection, Args, Opts}, infinity) of + case get_connection(Peer, ConnType, Args, Opts) of {ok, PID} -> ar_rate_limiter:throttle(Peer, Path, Opts), - case do_gun_request(PID, Args, Opts) of + case do_gun_request(PID, Args, hb_opts:mimic_default_types(Opts, existing, Opts)) of {error, Error} when Error == {shutdown, normal}; Error == noproc -> case ReestablishedConnection of @@ -181,6 +194,7 @@ gun_req(Args, ReestablishedConnection, Opts) -> {'EXIT', _} -> {error, client_error}; Error -> + ?event(http_client, {gun_error, Error}), Error end, EndTime = os:system_time(native), @@ -201,44 +215,104 @@ gun_req(Args, ReestablishedConnection, Opts) -> end, Response. -%% @doc Record the duration of the request in an async process. We write the -%% data to prometheus if the application is enabled, as well as invoking the -%% `http_monitor' if appropriate. -record_duration(Details, Opts) -> - spawn( - fun() -> - % First, write to prometheus if it is enabled. Prometheus works - % only with strings as lists, so we encode the data before granting - % it. - GetFormat = - fun - (<<"request-category">>) -> - path_to_category(maps:get(<<"request-path">>, Details)); - (Key) -> - hb_util:list(maps:get(Key, Details)) - end, - case application:get_application(prometheus) of - undefined -> ok; - _ -> - prometheus_histogram:observe( - http_request_duration_seconds, - lists:map( - GetFormat, - [ - <<"request-method">>, - <<"status-class">>, - <<"request-category">> - ] - ), - maps:get(<<"duration">>, Details) - ) - end, - maybe_invoke_monitor( - Details#{ <<"path">> => <<"duration">> }, - Opts - ) - end - ). +%% Connection Pool Logic + +init_ets_tables() -> + init_ets_table(?CONNECTIONS_ETS), + init_ets_table(?CONN_STATUS_ETS), + init_counter_ets_table(?CONN_COUNTER_ETS). + +init_ets_table(Table) -> + case ets:whereis(Table) of + undefined -> + ets:new(Table, [ + named_table, + public, + set, + {read_concurrency, true}, + {write_concurrency, true} + ]); + _ -> + ok + end. + +init_counter_ets_table(Table) -> + case ets:whereis(Table) of + undefined -> + ets:new(Table, [ + named_table, + public, + set, + {write_concurrency, true} + ]); + _ -> + ok + end. + + +get_connection_type(<<"GET">>) -> read; +get_connection_type(<<"get">>) -> read; +get_connection_type(<<"HEAD">>) -> read; +get_connection_type(<<"head">>) -> read; +get_connection_type(get) -> read; +get_connection_type(head) -> read; +get_connection_type(_) -> write. + +%% @doc Get the pool size for a connection type. +get_pool_size(read) -> + {ReadSize, _} = persistent_term:get(?CONN_TERM, {?DEFAULT_CONN_POOL_READ_SIZE, ?DEFAULT_CONN_POOL_WRITE_SIZE}), + ReadSize; +get_pool_size(write) -> + {_, WriteSize} = persistent_term:get(?CONN_TERM, {?DEFAULT_CONN_POOL_READ_SIZE, ?DEFAULT_CONN_POOL_WRITE_SIZE}), + WriteSize. + +%% @doc Get the next connection index using round-robin selection. +%% Uses ets:update_counter for atomic increment. +get_next_conn_index(Peer, ConnType) -> + PoolSize = get_pool_size(ConnType), + CounterKey = {Peer, ConnType}, + %% Atomically increment and wrap around using update_counter + %% If key doesn't exist, it will be created with default 0 + try + Index = ets:update_counter(?CONN_COUNTER_ETS, CounterKey, {2, 1, PoolSize, 1}), + Index + catch + error:badarg -> + %% Key doesn't exist, initialize it + ets:insert_new(?CONN_COUNTER_ETS, {CounterKey, 1}), + 1 + end. + +%% @doc Get a connection for a peer+type, using ETS for fast lookup. +%% If no connection exists, it will be created via the gen_server. +%% Uses round-robin to distribute requests across the connection pool. +get_connection(Peer, ConnType, Args, Opts) -> + PoolSize = get_pool_size(ConnType), + ConnIndex = get_next_conn_index(Peer, ConnType), + ConnKey = {Peer, ConnType, ConnIndex}, + get_connection_by_key(ConnKey, PoolSize, Args, Opts, 0). + +%% @doc Try to get a connection by key, with fallback to other pool connections. +get_connection_by_key(ConnKey, PoolSize, Args, Opts, Attempts) when Attempts < PoolSize -> + case ets:lookup(?CONNECTIONS_ETS, ConnKey) of + [{ConnKey, PID}] -> + %% Found a connection, check if it's still alive and connected + case ets:lookup(?CONN_STATUS_ETS, PID) of + [{PID, connected, _MonitorRef, _ConnKey}] -> + {ok, PID}; + [{PID, {connecting, _}, _MonitorRef, _ConnKey}] -> + %% Connection is being established, wait for it via gen_server + catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, 10_000); + [] -> + %% Status not found, connection might be dead, create new one + catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, 10_000) + end; + [] -> + %% No connection, create one via gen_server + catch gen_server:call(?MODULE, {get_connection, ConnKey, Args, Opts}, 10_000) + end; +get_connection_by_key(_ConnKey, _PoolSize, _Args, _Opts, _Attempts) -> + {error, no_available_connection}. %% @doc Invoke the HTTP monitor message with AO-Core, if it is set in the %% node message key. We invoke the given message with the `body' set to a signed @@ -276,7 +350,7 @@ maybe_invoke_monitor(Details, Opts) -> % execute. ReqMsgs = hb_singleton:from(Req, Opts), Res = hb_ao:resolve_many(ReqMsgs, Opts), - ?event(http_monitor, {resolved_monitor, Res}) + ?event(debug_http_monitor, {resolved_monitor, Res}) end. %%% ================================================================== @@ -284,6 +358,7 @@ maybe_invoke_monitor(Details, Opts) -> %%% ================================================================== init(Opts) -> + init_ets_tables(), case hb_opts:get(prometheus, not hb_features:test(), Opts) of true -> ?event({starting_prometheus_application, @@ -308,88 +383,33 @@ init(Opts) -> false -> {ok, #state{ opts = Opts }} end. -init_prometheus() -> - hb_prometheus:declare(counter, [ - {name, gun_requests_total}, - {labels, [http_method, status_class, category]}, - { - help, - "The total number of GUN requests." - } - ]), - hb_prometheus:declare(gauge, [{name, outbound_connections}, - {help, "The current number of the open outbound network connections"}]), - hb_prometheus:declare(histogram, [ - {name, http_request_duration_seconds}, - {buckets, [0.01, 0.1, 0.5, 1, 5, 10, 30, 60]}, - {labels, [http_method, status_class, category]}, - { - help, - "The total duration of an hb_http_client:req call. This includes more than" - " just the GUN request itself (e.g. establishing a connection, " - "throttling, etc...)" - } - ]), - hb_prometheus:declare(histogram, [ - {name, http_client_get_chunk_duration_seconds}, - {buckets, [0.1, 1, 10, 60]}, - {labels, [status_class, peer]}, - { - help, - "The total duration of an HTTP GET chunk request made to a peer." - } - ]), - hb_prometheus:declare(counter, [ - {name, http_client_downloaded_bytes_total}, - {help, "The total amount of bytes requested via HTTP, per remote endpoint"} - ]), - hb_prometheus:declare(counter, [ - {name, http_client_uploaded_bytes_total}, - {help, "The total amount of bytes posted via HTTP, per remote endpoint"} - ]), - ok. - -handle_call({get_connection, Args, Opts}, From, - #state{ pid_by_peer = PIDPeer, status_by_pid = StatusByPID } = State) -> - Peer = hb_maps:get(peer, Args, undefined, Opts), - case hb_maps:get(Peer, PIDPeer, not_found, Opts) of - not_found -> - {ok, PID} = open_connection(Args, hb_maps:merge(State#state.opts, Opts, Opts)), - MonitorRef = monitor(process, PID), - PIDPeer2 = hb_maps:put(Peer, PID, PIDPeer, Opts), - StatusByPID2 = - hb_maps:put( - PID, - {{connecting, [{From, Args}]}, MonitorRef, Peer}, - StatusByPID, - Opts - ), - { - reply, - {ok, PID}, - State#state{ - pid_by_peer = PIDPeer2, - status_by_pid = StatusByPID2 - } - }; - PID -> - case hb_maps:get(PID, StatusByPID, undefined, Opts) of - {{connecting, PendingRequests}, MonitorRef, Peer} -> - StatusByPID2 = - hb_maps:put(PID, - { - {connecting, [{From, Args} | PendingRequests]}, - MonitorRef, - Peer - }, - StatusByPID, - Opts - ), - {noreply, State#state{ status_by_pid = StatusByPID2 }}; - {connected, _MonitorRef, Peer} -> - {reply, {ok, PID}, State} - end - end; +handle_call({get_connection, ConnKey, Args, Opts}, From, State) -> + ArgsOpts = maps:get(opts, Args, #{}), + HttpOpts = maps:get(opts, hb_opts:mimic_default_types(Opts, existing, #{deep => true}), #{}), + MergedHttpOpts = maps:merge(ArgsOpts, HttpOpts), + MergedArgs = Args#{opts => MergedHttpOpts}, + %% ConnKey = {Peer, ConnType, Index} where ConnType is 'read' or 'write' + %% and Index is 1..PoolSize for round-robin distribution + %% Double-check ETS to handle race conditions + case ets:lookup(?CONNECTIONS_ETS, ConnKey) of + [{ConnKey, PID}] -> + %% Connection exists, check status + case ets:lookup(?CONN_STATUS_ETS, PID) of + [{PID, connected, _MonitorRef, _ConnKey}] -> + {reply, {ok, PID}, State}; + [{PID, {connecting, PendingRequests}, MonitorRef, ConnKey}] -> + %% Add to pending requests list + ets:insert(?CONN_STATUS_ETS, {PID, {connecting, [{From, MergedArgs} | PendingRequests]}, MonitorRef, ConnKey}), + {noreply, State}; + [] -> + %% Status not found, PID is stale - remove and create new + ets:delete(?CONNECTIONS_ETS, ConnKey), + create_new_connection(ConnKey, MergedArgs, From, State) + end; + [] -> + %% No connection exists, create one + create_new_connection(ConnKey, MergedArgs, From, State) + end; handle_call(Request, _From, State) -> ?event(warning, {unhandled_call, {module, ?MODULE}, {request, Request}}), @@ -399,31 +419,32 @@ handle_cast(Cast, State) -> ?event(warning, {unhandled_cast, {module, ?MODULE}, {cast, Cast}}), {noreply, State}. -handle_info({gun_up, PID, _Protocol}, #state{ status_by_pid = StatusByPID } = State) -> - case hb_maps:get(PID, StatusByPID, not_found) of - not_found -> +handle_info({gun_up, PID, Protocol}, State) -> + case ets:lookup(?CONN_STATUS_ETS, PID) of + [] -> %% A connection timeout should have occurred. {noreply, State}; - {{connecting, PendingRequests}, MonitorRef, Peer} -> - [gen_server:reply(ReplyTo, {ok, PID}) || {ReplyTo, _} <- PendingRequests], - StatusByPID2 = hb_maps:put(PID, {connected, MonitorRef, Peer}, StatusByPID), - inc_prometheus_gauge(outbound_connections), - {noreply, State#state{ status_by_pid = StatusByPID2 }}; - {connected, _MonitorRef, Peer} -> + [{PID, {connecting, PendingRequests}, MonitorRef, ConnKey}] -> + ?event(http_client, {gun_up, {protocol, Protocol}, {conn_key, ConnKey}}), + [gen_server:reply(ReplyTo, {ok, PID}) || {ReplyTo, _} <- PendingRequests], + ets:insert(?CONN_STATUS_ETS, {PID, connected, MonitorRef, ConnKey}), + hb_prometheus:inc(gauge, outbound_connections), + {noreply, State}; + [{PID, connected, _MonitorRef, ConnKey}] -> ?event(warning, - {gun_up_pid_already_exists, {peer, Peer}}), + {gun_up_pid_already_exists, {conn_key, ConnKey}}), {noreply, State} end; -handle_info({gun_error, PID, Reason}, - #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> - case hb_maps:get(PID, StatusByPID, not_found) of - not_found -> +handle_info({gun_error, PID, Reason}, State) -> + case ets:lookup(?CONN_STATUS_ETS, PID) of + [] -> ?event(warning, {gun_connection_error_with_unknown_pid}), {noreply, State}; - {Status, _MonitorRef, Peer} -> - PIDByPeer2 = hb_maps:remove(Peer, PIDByPeer), - StatusByPID2 = hb_maps:remove(PID, StatusByPID), + [{PID, Status, MonitorRef, ConnKey}] -> + ets:delete(?CONNECTIONS_ETS, ConnKey), + ets:delete(?CONN_STATUS_ETS, PID), + demonitor(MonitorRef, [flush]), Reason2 = case Reason of timeout -> @@ -437,109 +458,103 @@ handle_info({gun_error, PID, Reason}, {connecting, PendingRequests} -> reply_error(PendingRequests, Reason2); connected -> - dec_prometheus_gauge(outbound_connections), + hb_prometheus:dec(gauge, outbound_connections), ok end, gun:shutdown(PID), - ?event({connection_error, {reason, Reason}}), - {noreply, State#state{ status_by_pid = StatusByPID2, pid_by_peer = PIDByPeer2 }} + ?event(http_client, {connection_error, {conn_key, ConnKey}, {reason, Reason}}), + {noreply, State} end; -handle_info({gun_down, PID, Protocol, Reason, _KilledStreams}, - #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> - case hb_maps:get(PID, StatusByPID, not_found) of - not_found -> +handle_info({gun_down, PID, Protocol, Reason, _KilledStreams}, State) -> + case ets:lookup(?CONN_STATUS_ETS, PID) of + [] -> ?event(warning, {gun_connection_down_with_unknown_pid, {protocol, Protocol}}), - {noreply, State}; - {Status, _MonitorRef, Peer} -> - PIDByPeer2 = hb_maps:remove(Peer, PIDByPeer), - StatusByPID2 = hb_maps:remove(PID, StatusByPID), - Reason2 = - case Reason of - {Type, _} -> - Type; - _ -> - Reason - end, + {noreply, State}; + [{PID, Status, MonitorRef, ConnKey}] -> + ets:delete(?CONNECTIONS_ETS, ConnKey), + ets:delete(?CONN_STATUS_ETS, PID), + demonitor(MonitorRef, [flush]), + Reason2 = + case Reason of + {Type, _} -> + Type; + _ -> + Reason + end, case Status of {connecting, PendingRequests} -> reply_error(PendingRequests, Reason2); _ -> - dec_prometheus_gauge(outbound_connections), + hb_prometheus:dec(gauge,outbound_connections), ok end, - {noreply, - State#state{ - status_by_pid = StatusByPID2, - pid_by_peer = PIDByPeer2 - } - } + gun:shutdown(PID), + ?event(http_outbound, {gun_shutdown_after_down, {conn_key, ConnKey}, {protocol, Protocol}}), + {noreply, State} end; -handle_info({'DOWN', _Ref, process, PID, Reason}, - #state{ pid_by_peer = PIDByPeer, status_by_pid = StatusByPID } = State) -> - case hb_maps:get(PID, StatusByPID, not_found) of - not_found -> - {noreply, State}; - {Status, _MonitorRef, Peer} -> - PIDByPeer2 = hb_maps:remove(Peer, PIDByPeer), - StatusByPID2 = hb_maps:remove(PID, StatusByPID), +handle_info({'DOWN', _Ref, process, PID, Reason}, State) -> + case ets:lookup(?CONN_STATUS_ETS, PID) of + [] -> + {noreply, State}; + [{PID, Status, MonitorRef, ConnKey}] -> + ets:delete(?CONNECTIONS_ETS, ConnKey), + ets:delete(?CONN_STATUS_ETS, PID), + demonitor(MonitorRef, [flush]), case Status of {connecting, PendingRequests} -> reply_error(PendingRequests, Reason); _ -> - dec_prometheus_gauge(outbound_connections), + hb_prometheus:dec(gauge, outbound_connections), ok end, - {noreply, - State#state{ - status_by_pid = StatusByPID2, - pid_by_peer = PIDByPeer2 - } - } - end; + {noreply, State} + end; handle_info(Message, State) -> ?event(warning, {unhandled_info, {module, ?MODULE}, {message, Message}}), {noreply, State}. -terminate(Reason, #state{ status_by_pid = StatusByPID }) -> +terminate(Reason, _State) -> ?event(info,{http_client_terminating, {reason, Reason}}), - hb_maps:map(fun(PID, _Status) -> gun:shutdown(PID) end, StatusByPID), + ets:foldl( + fun({PID, _Status, MonitorRef, _ConnKey}, Acc) -> + gun:shutdown(PID), + demonitor(MonitorRef, [flush]), + Acc + end, + ok, + ?CONN_STATUS_ETS + ), ok. %%% ================================================================== %%% Private functions. %%% ================================================================== -%% @doc Safe wrapper for prometheus_gauge:inc/2. -inc_prometheus_gauge(Name) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - try prometheus_gauge:inc(Name) - catch _:_ -> - init_prometheus(), - prometheus_gauge:inc(Name) - end +%% @doc Create a new connection and store it in ETS. +create_new_connection(ConnKey, Args, _From, State) -> + MergedOpts = hb_maps:merge(State#state.opts, hb_maps:get(opts, Args, #{}), #{}), + case open_connection(Args, MergedOpts) of + {ok, PID} -> + MonitorRef = monitor(process, PID), + ets:insert(?CONNECTIONS_ETS, {ConnKey, PID}), + ets:insert(?CONN_STATUS_ETS, + {PID, {connecting, []}, MonitorRef, ConnKey}), + {reply, {ok, PID}, State}; + {error, _} = Err -> + {reply, Err, State} end. -%% @doc Safe wrapper for prometheus_gauge:dec/2. -dec_prometheus_gauge(Name) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> prometheus_gauge:dec(Name) - end. - -inc_prometheus_counter(Name, Labels, Value) -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> prometheus_counter:inc(Name, Labels, Value) +open_connection(#{ peer := Peer }, Opts) -> + case parse_peer(Peer, Opts) of + {error, _} = Err -> Err; + {ok, {Host, Port}} -> open_connection_gun(Host, Port, Peer, Opts) end. -open_connection(#{ peer := Peer }, Opts) -> - {Host, Port} = parse_peer(Peer, Opts), +open_connection_gun(Host, Port, Peer, Opts) -> ?event(http_outbound, {parsed_peer, {peer, Peer}, {host, Host}, {port, Port}}), BaseGunOpts = #{ @@ -547,7 +562,7 @@ open_connection(#{ peer := Peer }, Opts) -> #{ keepalive => hb_opts:get( - http_keepalive, + http_client_keepalive, ?DEFAULT_KEEPALIVE_TIMEOUT, Opts ) @@ -555,7 +570,7 @@ open_connection(#{ peer := Peer }, Opts) -> retry => 0, connect_timeout => hb_opts:get( - http_connect_timeout, + http_client_connect_timeout, ?DEFAULT_CONNECT_TIMEOUT, Opts ) @@ -574,7 +589,8 @@ open_connection(#{ peer := Peer }, Opts) -> GunOpts = case Proto = hb_opts:get(protocol, DefaultProto, Opts) of http3 -> BaseGunOpts#{protocols => [http3], transport => quic}; - _ -> BaseGunOpts + http2 -> BaseGunOpts#{protocols => [http2]}; + http1 -> BaseGunOpts#{protocols => [http]} end, ?event(http_outbound, {gun_open, @@ -590,15 +606,17 @@ parse_peer(Peer, Opts) -> Parsed = uri_string:parse(Peer), case Parsed of #{ host := Host, port := Port } -> - {hb_util:list(Host), Port}; + {ok, {hb_util:list(Host), Port}}; URI = #{ host := Host } -> - { + {ok, { hb_util:list(Host), case hb_maps:get(scheme, URI, undefined, Opts) of <<"https">> -> 443; _ -> hb_opts:get(port, 8734, Opts) end - } + }}; + _ -> + {error, {bad_peer, Peer}} end. reply_error([], _Reason) -> @@ -611,45 +629,10 @@ reply_error([PendingRequest | PendingRequests], Reason) -> gen_server:reply(ReplyTo, {error, Reason}), reply_error(PendingRequests, Reason). -record_response_status(Method, Response) -> - record_response_status(Method, Response, undefined). -record_response_status(Method, Response, Path) -> - inc_prometheus_counter(gun_requests_total, - [ - hb_util:list(method_to_bin(Method)), - hb_util:list(get_status_class(Response)), - hb_util:list(path_to_category(Path)) - ], - 1 - ). - -method_to_bin(get) -> - <<"GET">>; -method_to_bin(post) -> - <<"POST">>; -method_to_bin(put) -> - <<"PUT">>; -method_to_bin(head) -> - <<"HEAD">>; -method_to_bin(delete) -> - <<"DELETE">>; -method_to_bin(connect) -> - <<"CONNECT">>; -method_to_bin(options) -> - <<"OPTIONS">>; -method_to_bin(trace) -> - <<"TRACE">>; -method_to_bin(patch) -> - <<"PATCH">>; -method_to_bin(Method) when is_binary(Method) -> - Method; -method_to_bin(_) -> - <<"unknown">>. - do_gun_request(PID, Args, Opts) -> Timer = inet:start_timer( - hb_opts:get(http_request_send_timeout, no_request_send_timeout, Opts) + hb_opts:get(http_client_send_timeout, no_request_send_timeout, Opts) ), Method = hb_maps:get(method, Args, undefined, Opts), Path = hb_maps:get(path, Args, undefined, Opts), @@ -682,9 +665,13 @@ do_gun_request(PID, Args, Opts) -> Ref = gun:request(PID, Method, Path, Headers, Body), ResponseArgs = #{ - pid => PID, stream_ref => Ref, - timer => Timer, limit => hb_maps:get(limit, Args, infinity, Opts), - counter => 0, acc => [], start => os:system_time(microsecond), + pid => PID, + stream_ref => Ref, + timer => Timer, + limit => hb_maps:get(limit, Args, infinity, Opts), + counter => 0, + acc => [], + start => os:system_time(microsecond), is_peer_request => hb_maps:get(is_peer_request, Args, true, Opts) }, Response = await_response(hb_maps:merge(Args, ResponseArgs, Opts), Opts), @@ -734,23 +721,28 @@ await_response(Args, Opts) -> }; {error, timeout} = Response -> record_response_status(Method, Response, Path), + ?event(http_outbound, {gun_cancel, {path, Path}}), gun:cancel(PID, Ref), - log(warn, gun_await_process_down, Args, Response, Opts), + log(warning, gun_await_process_down, Args, timeout, Opts), Response; + {error,{connection_error,{stream_closed, Message}}} = Response -> + ?event(http_outbound, {gun_cancel, {path, Path}, {message, Message}}), + gun:cancel(PID, Ref), + Response; {error, Reason} = Response when is_tuple(Reason) -> record_response_status(Method, Response, Path), - log(warn, gun_await_process_down, Args, Reason, Opts), + log(warning, gun_await_process_down, Args, Reason, Opts), Response; Response -> record_response_status(Method, Response, Path), - log(warn, gun_await_unknown, Args, Response, Opts), + log(warning, gun_await_unknown, Args, Response, Opts), Response end. %% @doc Debug `http` state logging. log(Type, Event, #{method := Method, peer := Peer, path := Path}, Reason, Opts) -> ?event( - http, + Type, {gun_log, {type, Type}, {event, Event}, @@ -763,9 +755,101 @@ log(Type, Event, #{method := Method, peer := Peer, path := Path}, Reason, Opts) ), ok. -%% @doc Record instances of downloaded bytes from the remote server. +%% Metrics + +init_prometheus() -> + hb_prometheus:declare(counter, [ + {name, gun_requests_total}, + {labels, [http_method, status_class, category]}, + { + help, + "The total number of GUN requests." + } + ]), + hb_prometheus:declare(gauge, [{name, outbound_connections}, + {help, "The current number of the open outbound network connections"}]), + hb_prometheus:declare(histogram, [ + {name, http_client_duration_seconds}, + {buckets, [0.01, 0.1, 0.5, 1, 5, 10, 30, 60]}, + {labels, [http_method, status_class, category]}, + { + help, + "The total duration of an hb_http_client:req call. This includes more than" + " just the GUN request itself (e.g. establishing a connection, " + "throttling, etc...)" + } + ]), + hb_prometheus:declare(histogram, [ + {name, http_client_get_chunk_duration_seconds}, + {buckets, [0.1, 1, 10, 60]}, + {labels, [status_class, peer]}, + { + help, + "The total duration of an HTTP GET chunk request made to a peer." + } + ]), + hb_prometheus:declare(counter, [ + {name, http_client_downloaded_bytes_total}, + {help, "The total amount of bytes requested via HTTP, per remote endpoint"} + ]), + hb_prometheus:declare(counter, [ + {name, http_client_uploaded_bytes_total}, + {help, "The total amount of bytes posted via HTTP, per remote endpoint"} + ]), + ?event(started), + ok. + +%% @doc Record the duration of the request in an async process. We write the +%% data to prometheus if the application is enabled, as well as invoking the +%% `http_monitor' if appropriate. +record_duration(Details, Opts) -> + spawn( + fun() -> + % Prometheus works only with strings as lists, so we encode the + % data before granting it. + GetFormat = + fun + (<<"request-category">>) -> + path_to_category(maps:get(<<"request-path">>, Details)); + (Key) -> + hb_util:list(maps:get(Key, Details)) + end, + Labels = lists:map( + GetFormat, + [ + <<"request-method">>, + <<"status-class">>, + <<"request-category">> + ]), + hb_prometheus:observe( + maps:get(<<"duration">>, Details), + http_client_duration_seconds, + Labels + ), + maybe_invoke_monitor( + Details#{ <<"path">> => <<"duration">> }, + Opts + ) + end + ). + +record_response_status(Method, Response) -> + record_response_status(Method, Response, undefined). +record_response_status(Method, Response, Path) -> + hb_prometheus:inc( + counter, + gun_requests_total, + [ + hb_util:list(method_to_bin(Method)), + hb_util:list(get_status_class(Response)), + hb_util:list(path_to_category(Path)) + ], + 1 + ). + download_metric(Data) -> - inc_prometheus_counter( + hb_prometheus:inc( + counter, http_client_downloaded_bytes_total, [], byte_size(Data) @@ -777,7 +861,7 @@ upload_metric(#{method := Method, body := Body}) when is_atom(Method) -> upload_metric(#{ method := <<"POST">>, body := Body}) -> upload_metric(Body); upload_metric(#{ method := <<"PUT">>, body := Body}) -> upload_metric(Body); upload_metric(Body) when is_binary(Body) -> - inc_prometheus_counter( + hb_prometheus:inc(counter, http_client_uploaded_bytes_total, [], byte_size(Body) @@ -785,11 +869,34 @@ upload_metric(Body) when is_binary(Body) -> upload_metric(_) -> ok. +method_to_bin(get) -> + <<"GET">>; +method_to_bin(post) -> + <<"POST">>; +method_to_bin(put) -> + <<"PUT">>; +method_to_bin(head) -> + <<"HEAD">>; +method_to_bin(delete) -> + <<"DELETE">>; +method_to_bin(connect) -> + <<"CONNECT">>; +method_to_bin(options) -> + <<"OPTIONS">>; +method_to_bin(trace) -> + <<"TRACE">>; +method_to_bin(patch) -> + <<"PATCH">>; +method_to_bin(Method) when is_binary(Method) -> + Method; +method_to_bin(_) -> + <<"unknown">>. + % @doc Return the HTTP status class label for cowboy_requests_total and % gun_requests_total metrics. get_status_class({ok, {{Status, _}, _, _, _, _}}) -> get_status_class(Status); -get_status_class({ok, Status, _RespHeaders, _Body}) -> +get_status_class({ok, Status, _RespondeHeaders, _Body}) -> get_status_class(Status); get_status_class({error, connection_closed}) -> <<"connection-closed">>; @@ -871,4 +978,4 @@ path_to_category(Path) -> <<"/~cache@1.0/read", _/binary>> -> <<"Remote Read">>; undefined -> <<"unknown">>; _ -> <<"unknown">> - end. + end. \ No newline at end of file diff --git a/src/hb_http_client_sup.erl b/src/hb_http_client_sup.erl index 54c060610..39d08d006 100644 --- a/src/hb_http_client_sup.erl +++ b/src/hb_http_client_sup.erl @@ -13,6 +13,7 @@ -define(CHILD(I, Type, Opts), {I, {I, start_link, Opts}, permanent, ?SHUTDOWN_TIMEOUT, Type, [I]}). start_link(Opts) -> + hb_prometheus:ensure_started(), supervisor:start_link({local, ?MODULE}, ?MODULE, Opts). init(Opts) -> diff --git a/src/hb_http_client_tests.erl b/src/hb_http_client_tests.erl new file mode 100644 index 000000000..6bc1921ea --- /dev/null +++ b/src/hb_http_client_tests.erl @@ -0,0 +1,92 @@ +-module(hb_http_client_tests). +-include("include/hb.hrl"). +-include("include/hb_http_client.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +%% Regression test: create_new_connection used to reply to From immediately +%% AND store From in PendingRequests. gun_up would then reply again, +%% leaving orphan {Ref, {ok, PID}} messages in the caller's mailbox. +%% Fixed by storing an empty pending list when replying immediately. +orphan_message_leak_test_() -> + {timeout, 60, fun() -> + application:ensure_all_started(hb), + flush_mailbox(), + Peer = <<"https://arweave.net">>, + Args = #{ + peer => Peer, + path => <<"/info">>, + method => <<"GET">>, + headers => #{}, + body => <<>> + }, + Opts = #{http_client => gun, http_retry => 0}, + {ok, 200, _, _} = hb_http_client:request(Args, Opts), + timer:sleep(2000), + Orphans = flush_mailbox(), + ?event(http_client_tests, {orphaned_messages, {length, length(Orphans)}}), + ?assertEqual(0, length(Orphans), + "No orphan messages should be left in caller mailbox") + end}. + +unreachable_peer_hang_test_() -> + {timeout, 30, fun() -> + application:ensure_all_started(hb), + Peer = <<"http://192.0.2.1:1984">>, + Args = #{ + peer => Peer, + path => <<"/info">>, + method => <<"GET">>, + headers => #{}, + body => <<>> + }, + Opts = #{http_client => gun, http_retry => 0}, + T0 = erlang:monotonic_time(millisecond), + Result = hb_http_client:request(Args, Opts), + Elapsed = erlang:monotonic_time(millisecond) - T0, + ?event(http_client_tests, + {unreachable_peer_result, {result, Result}, {elapsed, Elapsed}} + ), + ?assertMatch({error, _}, Result), + ?assert(Elapsed >= 4000 andalso Elapsed =< 15000, + "Should block for ~5s connect_timeout, not infinity") + end}. + +bad_peer_survives_test_() -> + {timeout, 30, fun() -> + application:ensure_all_started(hb), + ?assert(erlang:whereis(hb_http_client) =/= undefined), + ValidArgs = #{ + peer => <<"https://arweave.net">>, + path => <<"/info">>, + method => <<"GET">>, + headers => #{}, + body => <<>> + }, + Opts = #{http_client => gun, http_retry => 0}, + {ok, 200, _, _} = hb_http_client:request(ValidArgs, Opts), + BadArgs = ValidArgs#{peer => <<"not-a-valid-uri">>}, + BadResult = hb_http_client:request(BadArgs, Opts), + ?event(http_client_tests, {bad_peer_result, BadResult}), + ?assertMatch({error, _}, BadResult), + timer:sleep(500), + ?assert(erlang:whereis(hb_http_client) =/= undefined, + "gen_server must survive a bad peer URI"), + {ok, 200, _, _} = hb_http_client:request(ValidArgs, Opts), + ?event(http_client_tests, follow_up_request_to_valid_peer_succeeded) + end}. + +flush_mailbox() -> + flush_mailbox([]). +flush_mailbox(Acc) -> + receive + Msg -> flush_mailbox([Msg | Acc]) + after 0 -> + lists:reverse(Acc) + end. + +summarize({caught, C, R}) when is_tuple(R) -> + {caught, C, element(1, R)}; +summarize({caught, C, R}) -> + {caught, C, R}; +summarize(Other) -> + Other. diff --git a/src/hb_http_conn_monitor.erl b/src/hb_http_conn_monitor.erl new file mode 100644 index 000000000..86c05113a --- /dev/null +++ b/src/hb_http_conn_monitor.erl @@ -0,0 +1,109 @@ +%%% @doc A gen_server that monitors gun connection mailbox sizes and reports +%%% metrics to Prometheus. This provides visibility into connection health +%%% and potential backpressure issues. +-module(hb_http_conn_monitor). +-behaviour(gen_server). +-include("include/hb.hrl"). +-include("include/hb_http_client.hrl"). +-export([start_link/1]). +-export([init/1, handle_cast/2, handle_call/3, handle_info/2, terminate/2]). + +-define(MONITORING_INTERVAL_MS, 5000). + +%%% ================================================================== +%%% Public interface. +%%% ================================================================== + +start_link(Opts) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, Opts, []). + +%%% ================================================================== +%%% gen_server callbacks. +%%% ================================================================== + +init(Opts) -> + case hb_opts:get(prometheus, not hb_features:test(), Opts) of + true -> + init_prometheus(), + erlang:send_after(?MONITORING_INTERVAL_MS, self(), conn_mailbox_monitoring); + false -> + no_op + end, + {ok, #{}}. + +handle_call(Request, _From, State) -> + ?event(warning, {unhandled_call, {module, ?MODULE}, {request, Request}}), + {reply, ok, State}. + +handle_cast(Cast, State) -> + ?event(warning, {unhandled_cast, {module, ?MODULE}, {cast, Cast}}), + {noreply, State}. + +handle_info(conn_mailbox_monitoring, State) -> + spawn(fun() -> + case ets:whereis(?CONNECTIONS_ETS) of + undefined -> + ok; + _ -> + ets:foldl( + fun({ConnKey, ConnPID}, Acc) -> + sample_conn_pid(ConnKey, ConnPID), + Acc + end, + ok, + ?CONNECTIONS_ETS + ) + end + end), + erlang:send_after(?MONITORING_INTERVAL_MS, self(), conn_mailbox_monitoring), + {noreply, State}; + +handle_info(Message, State) -> + ?event(warning, {unhandled_info, {module, ?MODULE}, {message, Message}}), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +%%% ================================================================== +%%% Private functions. +%%% ================================================================== + +init_prometheus() -> + application:ensure_all_started([prometheus]), + case application:get_application(prometheus) of + undefined -> + ok; + _ -> + try + prometheus_gauge:new([ + {name, gun_mailbox_size}, + {labels, [conn_id]}, + {help, "Gun connection mailbox size"} + ]), + ok + catch + error:{mf_already_exists, _, _} -> + %% Metric already registered, this is fine + ok + end + end. + +sample_conn_pid(ConnKey, ConnPID) -> + case process_info(ConnPID, message_queue_len) of + {message_queue_len, Len} -> + report(ConnKey, Len); + undefined -> + ok + end. + +report(ConnKey, Value) -> + ConnKeyString = conn_key_string(ConnKey), + prometheus_gauge:set( + gun_mailbox_size, + [ConnKeyString], + Value). + +conn_key_string({Peer, ConnType, Index}) -> + iolist_to_binary([hb_util:bin(Peer), "_", atom_to_binary(ConnType), "_", integer_to_binary(Index)]). + diff --git a/src/hb_http_multi.erl b/src/hb_http_multi.erl index 95129d8ae..c4dae4299 100644 --- a/src/hb_http_multi.erl +++ b/src/hb_http_multi.erl @@ -88,7 +88,7 @@ request(Config, Method, Path, Message, Opts) -> Opts ) end, - ?event(http, {multirequest_results, {admissible_results, AdmissibleResults}, {all_responses, AllResponses}}), + ?event(debug_http, {multirequest_results, {admissible_results, AdmissibleResults}, {all_responses, AllResponses}}), case AdmissibleResults of [] -> {error, {no_viable_responses, AllResponses}}; Results -> if Responses == 1 -> hd(Results); true -> Results end @@ -155,7 +155,7 @@ serial_multirequest([Node|Nodes], Remaining, Method, Path, Message, Admissible, {ErlStatus, Res} = hb_http:request(Method, Node, Path, Message, Opts), case is_admissible(ErlStatus, Res, Admissible, Statuses, Opts) of true -> - ?event(http, {admissible_status, {response, Res}}), + ?event(debug_http, {admissible_status, {response, Res}}), {AdmissibleAcc, AllAcc} = serial_multirequest( Nodes, Remaining - 1, @@ -168,7 +168,7 @@ serial_multirequest([Node|Nodes], Remaining, Method, Path, Message, Admissible, ), {[{ErlStatus, Res} | AdmissibleAcc], [{ErlStatus, Res} | AllAcc]}; false -> - ?event(http, {inadmissible_status, {response, Res}}), + ?event(debug_http, {inadmissible_status, {response, Res}}), {AdmissibleAcc, AllAcc} = serial_multirequest( Nodes, Remaining, diff --git a/src/hb_http_server.erl b/src/hb_http_server.erl index ea949ebc0..8d2935eab 100644 --- a/src/hb_http_server.erl +++ b/src/hb_http_server.erl @@ -36,6 +36,7 @@ start() -> hb_opts:default_message_with_env(), Loaded ), + hb_http_client:setup_conn(MergedConfig), %% Apply store defaults before starting store StoreOpts = hb_opts:get(store, no_store, MergedConfig), StoreDefaults = hb_opts:get(store_defaults, #{}, MergedConfig), @@ -77,6 +78,7 @@ start(Opts) -> ]), hb:init(), BaseOpts = set_default_opts(Opts), + ok = hb_process_sampler:ensure_started(BaseOpts), {ok, Listener, _Port} = new_server(BaseOpts), {ok, Listener}. @@ -124,7 +126,7 @@ print_greeter(Config, PrivWallet) -> io_lib:format( "http://~s:~p", [ - hb_opts:get(host, <<"localhost">>, Config), + hb_opts:get(node_host, <<"localhost">>, Config), hb_opts:get(port, 8734, Config) ] ) @@ -196,7 +198,7 @@ new_server(RawNodeMsg) -> % Attempt to start the prometheus application, if possible. try application:ensure_all_started([prometheus, prometheus_cowboy, prometheus_ranch]), - prometheus_registry:register_collectors([hb_metrics_collector, prometheus_ranch_collector]), + prometheus_registry:register_collectors([hb_metrics_collector]), ProtoOpts#{ metrics_callback => fun prometheus_cowboy2_instrumenter:observe/1, @@ -364,7 +366,7 @@ cors_reply(Req, _ServerID) -> <<"access-control-allow-methods">> => <<"GET, POST, PUT, DELETE, OPTIONS, PATCH">> }, Req), - ?event(http_debug, {cors_reply, {req, Req}, {req2, Req2}}), + ?event(debug_http, {cors_reply, {req, Req}, {req2, Req2}}), {ok, Req2, no_state}. %% @doc Handle all non-CORS preflight requests as AO-Core requests. Execution @@ -571,6 +573,7 @@ start_node(Opts) -> hb:init(), hb_sup:start_link(Opts), ServerOpts = set_default_opts(Opts), + ok = hb_process_sampler:ensure_started(ServerOpts), {ok, _Listener, Port} = new_server(ServerOpts), <<"http://localhost:", (hb_util:bin(Port))/binary, "/">>. diff --git a/src/hb_maps.erl b/src/hb_maps.erl index 391e018dc..bcb38d2d7 100644 --- a/src/hb_maps.erl +++ b/src/hb_maps.erl @@ -17,7 +17,7 @@ %%% yourself from the inevitable issues that will arise from using this %%% module without understanding the full implications. You have been warned. -module(hb_maps). --export([get/2, get/3, get/4, put/3, put/4, find/2, find/3]). +-export([get/2, get/3, get/4, get_first/2, get_first/3, put/3, put/4, find/2, find/3]). -export([is_key/2, is_key/3, keys/1, keys/2, values/1, values/2]). -export([map/2, map/3, filter/2, filter/3, filtermap/2, filtermap/3]). -export([fold/3, fold/4, take/2, take/3, size/1, size/2]). @@ -26,6 +26,27 @@ -export([from_list/1, to_list/1, to_list/2]). -include_lib("eunit/include/eunit.hrl"). +%%% HyperBEAM-specific functions + +-spec get_first( + Paths :: [{Base :: map() | binary(), Path :: binary()}], + Opts :: map() +) -> term(). +get_first(Paths, Opts) -> + get_first(Paths, not_found, Opts). + +-spec get_first( + Paths :: [{Base :: map() | binary(), Path :: binary()}], + Default :: term(), + Opts :: map() +) -> term(). +get_first([], Default, _Opts) -> Default; +get_first([{Base, Path}|Paths], Default, Opts) -> + case find(Path, Base, Opts) of + {ok, Value} -> Value; + error -> get_first(Paths, Default, Opts) + end. + -spec get(Key :: term(), Map :: map()) -> term(). get(Key, Map) -> get(Key, Map, undefined). diff --git a/src/hb_message.erl b/src/hb_message.erl index b34267811..13fb10e37 100644 --- a/src/hb_message.erl +++ b/src/hb_message.erl @@ -694,7 +694,7 @@ unsafe_match(RawMap1, RawMap2, Mode, Path, Opts) -> fun(Key) -> lists:member(Key, Keys1) end, Keys1 ), - ?event(match, + ?event(debug_match, {match, {keys1, Keys1}, {keys2, Keys2}, @@ -709,7 +709,7 @@ unsafe_match(RawMap1, RawMap2, Mode, Path, Opts) -> lists:all( fun(<<"commitments">>) -> true; (Key) -> - ?event(match, {matching_key, Key}), + ?event(debug_match, {matching_key, Key}), Val1 = hb_ao:normalize_keys( hb_maps:get(Key, NormMap1, not_found, Opts), diff --git a/src/hb_metrics_collector.erl b/src/hb_metrics_collector.erl index 18804b04b..afb2be508 100644 --- a/src/hb_metrics_collector.erl +++ b/src/hb_metrics_collector.erl @@ -23,7 +23,7 @@ collect_mf(_Registry, Callback) -> ) ), - SystemLoad = cpu_sup:avg5(), + SystemLoad = safe_avg5(), Callback( create_gauge( @@ -58,5 +58,30 @@ collect_metrics(process_uptime_seconds, Uptime) -> %%==================================================================== %% Private Functions %%==================================================================== + +%% @doc Wrapper around cpu_sup:avg5/0 with a 2-second timeout. +%% cpu_sup:avg5/0 uses an infinity timeout to os_mon internally; +%% if the port program stalls, it blocks the Prometheus scrape indefinitely. +%% On timeout, the worker is killed to avoid leaking blocked processes. +safe_avg5() -> + Ref = make_ref(), + Self = self(), + {Pid, MonRef} = spawn_monitor(fun() -> Self ! {Ref, catch cpu_sup:avg5()} end), + receive + {Ref, Load} when is_integer(Load) -> + erlang:demonitor(MonRef, [flush]), + Load; + {Ref, _} -> + erlang:demonitor(MonRef, [flush]), + 0; + {'DOWN', MonRef, process, Pid, _} -> + 0 + after 2000 -> + exit(Pid, kill), + erlang:demonitor(MonRef, [flush]), + receive {Ref, _} -> ok after 0 -> ok end, + 0 + end. + create_gauge(Name, Help, Data) -> prometheus_model_helpers:create_mf(Name, Help, gauge, ?MODULE, Data). \ No newline at end of file diff --git a/src/hb_name.erl b/src/hb_name.erl index e85c05162..1aecccbf9 100644 --- a/src/hb_name.erl +++ b/src/hb_name.erl @@ -5,6 +5,7 @@ %%% There can only ever be one registrant for a given name at a time. -module(hb_name). -export([start/0, register/1, register/2, unregister/1, lookup/1, all/0]). +-export([singleton/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). -define(NAME_TABLE, hb_name_registry). @@ -58,6 +59,45 @@ unregister(Name) -> ets:delete(?NAME_TABLE, Name), ok. +%% @doc Atomic singleton lookup/spawn+register operation. +%% +%% Multiple callers may simultanoeously invoke this function, but the PID of +%% only one surviving surivor will be registered and returned to all callers. +%% If the given function crashes on spawn, then the operation will retry the +%% operation until they successfully spawn and register a process -- which will +%% promptly fail. The result is that the intended semantics are still preserved: +%% Calling `singleton' will always return the PID of a process that owns that name +%% at the time of return. +singleton(Name, Fun) -> + case lookup(Name) of + Registered when is_pid(Registered) -> Registered; + undefined -> singleton_spawn(Name, Fun) + end. + +%% @doc Perform the actual atomic spawn+register operation. +singleton_spawn(Name, Fun) -> + start(), + Parent = self(), + ReadyRef = make_ref(), + PID = + spawn( + fun() -> + Spawned = self(), + case catch ?MODULE:register(Name, Spawned) of + ok -> + Parent ! {spawned, ReadyRef, Spawned}, + Fun(); + _ -> + Parent ! {spawn_failed, ReadyRef}, + ok + end + end + ), + receive + {spawned, ReadyRef, PID} -> PID; + {spawn_failed, ReadyRef} -> singleton(Name, Fun) + end. + %%% @doc Lookup a name -> PID. lookup(Name) when is_atom(Name) -> case whereis(Name) of @@ -118,6 +158,14 @@ atom_test() -> term_test() -> basic_test({term, os:timestamp()}). +singleton_returns_spawned_pid_test() -> + Name = {singleton, os:timestamp()}, + Pid = singleton(Name, fun() -> receive stop -> ok end end), + ?assertEqual(Pid, lookup(Name)), + ?assertNotEqual(self(), Pid), + Pid ! stop, + hb_name:unregister(Name). + concurrency_test() -> Name = {concurrent_test, os:timestamp()}, SuccessCount = length([R || R <- spawn_test_workers(Name), R =:= ok]), diff --git a/src/hb_opts.erl b/src/hb_opts.erl index ce8f64ad3..33a469dc6 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -18,6 +18,7 @@ -export([ensure_node_history/2]). -export([check_required_opts/2]). -include("include/hb.hrl"). +-include("include/hb_opts.hrl"). -include("include/hb_arweave_nodes.hrl"). %%% Environment variables that can be used to override the default message. @@ -42,6 +43,8 @@ -ifndef(TEST). -define(DEFAULT_NAME_RESOLVERS, [ + #{ <<"device">> => <<"arweave@2.9">> }, + #{ <<"device">> => <<"b32-name@1.0">> }, << "G_gb7SAgogHMtmqycwaHaC6uC-CZ3akACdFv5PUaEE8", "~json@1.0/deserialize&target=data" @@ -63,6 +66,7 @@ <<"store-module">> => hb_store_lmdb }). -define(DEFAULT_GATEWAY, <<"https://arweave.net">>). +-define(DEFAULT_HTTP_OPTS, #{http_client => ?DEFAULT_HTTP_CLIENT, protocol => http2}). -define(ENV_KEYS, #{ priv_key_location => {"HB_KEY", "hyperbeam-key.json"}, @@ -136,7 +140,7 @@ default_message() -> initialized => true, %% What HTTP client should the node use? %% Options: gun, httpc - http_client => gun, + http_client => ?DEFAULT_HTTP_CLIENT, %% Scheduling mode: Determines when the SU should inform the recipient %% that an assignment has been scheduled for a message. %% Options: aggressive(!), local_confirmation, remote_confirmation, @@ -162,6 +166,7 @@ default_message() -> #{<<"name">> => <<"apply@1.0">>, <<"module">> => dev_apply}, #{<<"name">> => <<"auth-hook@1.0">>, <<"module">> => dev_auth_hook}, #{<<"name">> => <<"ans104@1.0">>, <<"module">> => dev_codec_ans104}, + #{<<"name">> => <<"b32-name@1.0">>, <<"module">> => dev_b32_name}, #{<<"name">> => <<"blacklist@1.0">>, <<"module">> => dev_blacklist}, #{<<"name">> => <<"bundler@1.0">>, <<"module">> => dev_bundler}, #{<<"name">> => <<"compute@1.0">>, <<"module">> => dev_cu}, @@ -201,6 +206,7 @@ default_message() -> #{<<"name">> => <<"profile@1.0">>, <<"module">> => dev_profile}, #{<<"name">> => <<"push@1.0">>, <<"module">> => dev_push}, #{<<"name">> => <<"query@1.0">>, <<"module">> => dev_query}, + #{<<"name">> => <<"rate-limit@1.0">>, <<"module">> => dev_rate_limit}, #{<<"name">> => <<"relay@1.0">>, <<"module">> => dev_relay}, #{<<"name">> => <<"router@1.0">>, <<"module">> => dev_router}, #{<<"name">> => <<"scheduler@1.0">>, <<"module">> => dev_scheduler}, @@ -232,11 +238,13 @@ default_message() -> trusted_device_signers => [], %% What should the node do if a client error occurs? client_error_strategy => throw, - %% HTTP request options - http_connect_timeout => 5000, - http_keepalive => 120000, - http_request_send_timeout => 300_000, + %% HTTP client request options + http_client_connect_timeout => 5000, + http_client_keepalive => 120000, + http_client_send_timeout => 300_000, port => 8734, + process_sampler => true, + process_sampler_interval => 15000, wasm_allow_aot => false, %% Options for the relay device relay_http_client => httpc, @@ -265,7 +273,7 @@ default_message() -> debug_trace_type => ?DEFAULT_TRACE_TYPE, short_trace_len => 20, debug_show_priv => if_present, - debug_resolve_links => true, + debug_resolve_links => false, debug_print_fail_mode => long, trusted => #{}, snp_enforced_keys => [ @@ -319,15 +327,14 @@ default_message() -> <<"path">> => <<"^/arweave/chunk">>, <<"method">> => <<"GET">> }, - <<"nodes">> => - ?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES, + <<"nodes">> => add_opts(?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES), <<"strategy">> => <<"Shuffled-Range">>, <<"choose">> => length( ?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES ), - <<"parallel">> => 4, + <<"parallel">> => 1, <<"responses">> => 1, <<"stop-after">> => true, <<"admissible-status">> => 200 @@ -338,8 +345,7 @@ default_message() -> <<"path">> => <<"^/arweave/chunk">>, <<"method">> => <<"POST">> }, - <<"nodes">> => - ?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES, + <<"nodes">> => add_opts(?ARWEAVE_BOOTSTRAP_DATA_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES), <<"strategy">> => <<"Shuffled-Range">>, <<"choose">> => length( @@ -357,8 +363,7 @@ default_message() -> <<"path">> => <<"^/arweave/tx">>, <<"method">> => <<"POST">> }, - <<"nodes">> => - ?ARWEAVE_BOOTSTRAP_CHAIN_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES, + <<"nodes">> => add_opts(?ARWEAVE_BOOTSTRAP_CHAIN_NODES ++ ?ARWEAVE_BOOTSTRAP_TIP_NODES), <<"parallel">> => true, <<"responses">> => 3, <<"stop-after">> => false, @@ -374,11 +379,11 @@ default_message() -> <<"opts">> => #{ http_client => httpc, protocol => http2 } } }, - %% General Arweave requests: race both chain nodes, take + %% General Arweave requests: race all chain nodes, take %% the first 200. #{ <<"template">> => <<"^/arweave">>, - <<"nodes">> => ?ARWEAVE_BOOTSTRAP_CHAIN_NODES, + <<"nodes">> => add_opts(?ARWEAVE_BOOTSTRAP_CHAIN_NODES), <<"parallel">> => true, <<"stop-after">> => 1, <<"admissible-status">> => 200 @@ -437,7 +442,7 @@ default_message() -> % responses are not verifiable. ans104_trust_gql => true, % Number of chunks to fetch in parallel when loading a TX or dataitem. - chunk_fetch_concurrency => 10, + arweave_chunk_fetch_concurrency => 5, http_extra_opts => #{ force_message => true, @@ -454,6 +459,9 @@ default_message() -> on => #{ <<"request">> => [ + #{ + <<"device">> => <<"rate-limit@1.0">> + }, #{ <<"device">> => <<"auth-hook@1.0">>, <<"path">> => <<"request">>, @@ -472,6 +480,9 @@ default_message() -> }, #{ <<"device">> => <<"manifest@1.0">> + }, + #{ + <<"device">> => <<"blacklist@1.0">> } ] }, @@ -862,6 +873,17 @@ ensure_node_history(Opts, RequiredOpts) -> {error, validation_failed} end. +%% @doc Util to add opts to nodes. +add_opts(Items) -> + add_opts(Items, ?DEFAULT_HTTP_OPTS). +add_opts(Items, Opts) -> + lists:map( + fun (Item) when is_map(Item) -> + Item#{<<"opts">> => Opts} + end, + Items + ). + %%% Tests -ifdef(TEST). @@ -899,14 +921,14 @@ global_preference_test() -> load_flat_test() -> % File contents: % port: 1234 - % host: https://ao.computer + % node_host: https://ao.computer % await-inprogress: false {ok, Conf} = load("test/config.flat", #{}), ?event({loaded, {explicit, Conf}}), % Ensure we convert types as expected. ?assertEqual(1234, hb_maps:get(port, Conf)), % A binary - ?assertEqual(<<"https://ao.computer">>, hb_maps:get(host, Conf)), + ?assertEqual(<<"https://ao.computer">>, hb_maps:get(node_host, Conf)), % An atom, where the key contained a header-key `-' rather than a `_'. ?assertEqual(false, hb_maps:get(await_inprogress, Conf)). @@ -916,7 +938,7 @@ load_json_test() -> ?assertEqual(1234, hb_maps:get(port, Conf)), ?assertEqual(9001, hb_maps:get(example, Conf)), % A binary - ?assertEqual(<<"https://ao.computer">>, hb_maps:get(host, Conf)), + ?assertEqual(<<"https://ao.computer">>, hb_maps:get(node_host, Conf)), % An atom, where the key contained a header-key `-' rather than a `_'. ?assertEqual(false, hb_maps:get(await_inprogress, Conf)), % Ensure that a store with `ao-types' is loaded correctly. diff --git a/src/hb_process_sampler.erl b/src/hb_process_sampler.erl new file mode 100644 index 000000000..42a5f92be --- /dev/null +++ b/src/hb_process_sampler.erl @@ -0,0 +1,284 @@ +%%% @doc Sample BEAM process state for diagnostics. +-module(hb_process_sampler). + +-export([ensure_started/1]). + +-include("include/hb.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-define(DEFAULT_SAMPLE_PROCESSES_INTERVAL, 15000). + +%% @doc Ensure the process sampler singleton is started if enabled. +ensure_started(Opts) -> + ProcessSamplerEnabled = + hb_opts:get(process_sampler, not hb_features:test(), Opts) + andalso hb_opts:get(prometheus, not hb_features:test(), Opts), + ?event(process_sampler, {process_sampler_enabled, ProcessSamplerEnabled}), + case ProcessSamplerEnabled of + true -> + _ = hb_name:singleton(?MODULE, fun() -> start(Opts) end), + ok; + false -> + ok + end. + +%% @doc Initialize the process sampler and enter its receive loop. +start(Opts) -> + ?event(process_sampler, {starting_process_sampler, + {interval, hb_opts:get(process_sampler_interval, ?DEFAULT_SAMPLE_PROCESSES_INTERVAL, Opts)}}), + hb_prometheus:ensure_started(), + schedule_process_sample(Opts), + loop( + #{ + opts => Opts + } + ). + +%% @doc Receive loop for the process sampler. +loop(State = #{ opts := Opts }) -> + receive + sample_processes -> + sample_processes(State), + schedule_process_sample(Opts), + loop(State); + Message -> + ?event(warning, {unhandled_info, {module, ?MODULE}, {message, Message}}), + loop(State) + end. + +%% @doc Schedule the next process sample if enabled. +schedule_process_sample(Opts) -> + case hb_opts:get( + process_sampler_interval, + ?DEFAULT_SAMPLE_PROCESSES_INTERVAL, + Opts + ) of + Interval when is_integer(Interval) andalso Interval > 0 -> + erlang:send_after(Interval, self(), sample_processes); + _ -> + ok + end. + +%% @doc Sample all BEAM processes and report aggregate metrics. +sample_processes(#{ opts := Opts }) -> + StartTime = erlang:monotonic_time(), + try + Processes = erlang:processes(), + ProcessData = + lists:filtermap( + fun(PID) -> process_function(PID, Opts) end, + Processes + ), + ProcessMetrics = accumulate_process_metrics(ProcessData), + report_process_metrics(ProcessMetrics), + EndTime = erlang:monotonic_time(), + ElapsedTime = + erlang:convert_time_unit( + EndTime - StartTime, + native, + microsecond + ), + ?event( + process_sampler, + {sample_processes, + {processes, length(Processes)}, + {elapsed_ms, ElapsedTime / 1000} + }, + Opts + ) + catch + Class:Reason:Stacktrace -> + ?event( + warning, + {process_sampler_failed, + {class, Class}, + {reason, Reason}, + {stacktrace, {trace, Stacktrace}} + }, + Opts + ) + end. + +%% @doc Sum process memory, reductions, and mailbox sizes by process name. +accumulate_process_metrics(ProcessData) -> + lists:foldl( + fun({_Status, ProcessName, Memory, Reductions, MsgQueueLen}, Acc) -> + {MemoryTotal, ReductionsTotal, MsgQueueLenTotal} = + maps:get(ProcessName, Acc, {0, 0, 0}), + maps:put( + ProcessName, + { + MemoryTotal + Memory, + ReductionsTotal + Reductions, + MsgQueueLenTotal + MsgQueueLen + }, + Acc + ) + end, + #{}, + ProcessData + ). + +%% @doc Report aggregate process metrics to Prometheus. +report_process_metrics(ProcessMetrics) -> + reset_process_info_metric(), + maps:foreach( + fun(ProcessName, {Memory, Reductions, MsgQueueLen}) -> + prometheus_gauge:set(process_info, [ProcessName, <<"memory">>], Memory), + prometheus_gauge:set( + process_info, + [ProcessName, <<"reductions">>], + Reductions + ), + prometheus_gauge:set( + process_info, + [ProcessName, <<"message_queue">>], + MsgQueueLen + ) + end, + ProcessMetrics + ), + report_memory_metrics(). + +%% @doc Recreate the per-process metric family to clear exited-process labels. +reset_process_info_metric() -> + _ = prometheus_gauge:deregister(process_info), + ok = + prometheus_gauge:new( + [ + {name, process_info}, + {labels, [process, type]}, + {help, + "Sampling info about active processes." + " Only set when process_sampler is enabled."} + ] + ). + +%% @doc Report BEAM memory totals through the process_info metric family. +report_memory_metrics() -> + prometheus_gauge:set( + process_info, + [<<"total">>, <<"memory">>], + erlang:memory(total) + ), + prometheus_gauge:set( + process_info, + [<<"processes">>, <<"memory">>], + erlang:memory(processes) + ), + prometheus_gauge:set( + process_info, + [<<"processes_used">>, <<"memory">>], + erlang:memory(processes_used) + ), + prometheus_gauge:set( + process_info, + [<<"system">>, <<"memory">>], + erlang:memory(system) + ), + prometheus_gauge:set( + process_info, + [<<"atom">>, <<"memory">>], + erlang:memory(atom) + ), + prometheus_gauge:set( + process_info, + [<<"atom_used">>, <<"memory">>], + erlang:memory(atom_used) + ), + prometheus_gauge:set( + process_info, + [<<"binary">>, <<"memory">>], + erlang:memory(binary) + ), + prometheus_gauge:set( + process_info, + [<<"code">>, <<"memory">>], + erlang:memory(code) + ), + prometheus_gauge:set( + process_info, + [<<"ets">>, <<"memory">>], + erlang:memory(ets) + ). + +%% @doc Sample a single process and return aggregate data for it. +process_function(PID, _Opts) -> + case process_info( + PID, + [ + current_stacktrace, + registered_name, + status, + memory, + reductions, + message_queue_len + ] + ) of + [{current_stacktrace, Stack}, + {registered_name, Name}, + {status, Status}, + {memory, Memory}, + {reductions, Reductions}, + {message_queue_len, MsgQueueLen}] -> + ProcessName = process_name(Name, Stack), + {true, {Status, ProcessName, Memory, Reductions, MsgQueueLen}}; + _ -> + false + end. + +%% @doc Resolve a readable process name from its registration or stack. +process_name([], []) -> + <<"unknown">>; +process_name([], Stack) -> + InitialCall = initial_call(lists:reverse(Stack)), + M = element(1, InitialCall), + F = element(2, InitialCall), + A = element(3, InitialCall), + << + (hb_util:bin(M))/binary, + ":", + (hb_util:bin(F))/binary, + "/", + (hb_util:bin(A))/binary + >>; +process_name(Name, _Stack) -> + hb_util:bin(Name). + +%% @doc Determine the initial stack frame for an anonymous process. +initial_call([]) -> + {unknown, unknown, 0}; +initial_call([{proc_lib, init_p_do_apply, _A, _Location} | Stack]) -> + initial_call(Stack); +initial_call([InitialCall | _Stack]) -> + InitialCall. + +%%% Tests + +%% @doc Ensure anonymous process names fall back to their initial call. +process_name_from_stack_test() -> + ?assertEqual( + <<"hb_http_server:init/2">>, + process_name( + [], + [ + {hb_http_server, init, 2, []}, + {proc_lib, init_p_do_apply, 3, []} + ] + ) + ). + +%% @doc Ensure registered names are returned directly. +process_name_registered_test() -> + ?assertEqual(<<"my_proc">>, process_name(my_proc, [])). + +%% @doc Ensure aggregate process metrics are summed by process name. +accumulate_process_metrics_test() -> + Metrics = + accumulate_process_metrics( + [ + {running, <<"worker">>, 10, 20, 1}, + {running, <<"worker">>, 5, 3, 2} + ] + ), + ?assertEqual({15, 23, 3}, maps:get(<<"worker">>, Metrics)). diff --git a/src/hb_prometheus.erl b/src/hb_prometheus.erl index f4aa717ae..8b4c3e7a8 100644 --- a/src/hb_prometheus.erl +++ b/src/hb_prometheus.erl @@ -1,23 +1,49 @@ %%% @doc HyperBEAM wrapper for Prometheus metrics. -module(hb_prometheus). -export([ensure_started/0, declare/2, measure_and_report/2, measure_and_report/3]). +-export([observe/2, observe/3, inc/2, inc/3, inc/4, dec/2, dec/3, dec/4]). -%% @doc Ensure the Prometheus application has been started. +%% @doc Ensure the Prometheus application has been started. Caches startup +%% failure with a timestamp to avoid repeated blocking ensure_all_started +%% calls on hot paths, but retries after a cooldown period. ensure_started() -> - case application:get_application(prometheus) of - undefined -> + case is_started() of + true -> ok; + false -> application:ensure_all_started( [prometheus, prometheus_cowboy, prometheus_ranch] ), - ok; - _ -> ok + wait_for_prometheus_started() + end. + +%% @doc Lazy wait for prometheus to come up, after we have started the application. +wait_for_prometheus_started() -> + case is_started() of + true -> ok; + false -> + timer:sleep(1), + wait_for_prometheus_started() end. +%% @doc Check if prometheus has been started. +%% The application itself may return `ok` to Erlang before it is actually ready +%% for use, so we wait for the `ets` table to be created instead. +is_started() -> + Info = ets:info(prometheus_registry_table), + Info =/= undefined. + %% @doc Declare a new Prometheus metric in a replay-safe manner. declare(Type, Metric) -> - ok = ensure_started(), - try do_declare(Type, Metric) - catch error:mfa_already_exists -> ok + case ensure_started() of + ok -> + try do_declare(Type, Metric) + catch + error:mfa_already_exists -> + ok; + error:{mf_already_exists, _, _} -> + ok + end; + _ -> ok end. do_declare(histogram, Metric) -> prometheus_histogram:declare(Metric); @@ -26,16 +52,59 @@ do_declare(gauge, Metric) -> prometheus_gauge:declare(Metric); do_declare(Type, _Metric) -> throw({unsupported_metric_type, Type}). %% @doc Measure function duration and report metric, ensuring that the Prometheus -%% application has been started first. -measure_and_report(Fun, Metric) -> +%% application has been started first. If Prometheus is unavailable, the function +%% is executed without measurement. +measure_and_report(Fun, Metric) when is_function(Fun) -> measure_and_report(Fun, Metric, []). -measure_and_report(Fun, Metric, Labels) -> - ok = ensure_started(), +measure_and_report(Fun, Metric, Labels) when is_function(Fun) -> Start = erlang:monotonic_time(), try Fun() after DurationNative = erlang:monotonic_time() - Start, - try prometheus_histogram:observe(Metric, Labels, DurationNative) - catch _:_ -> ok - end + observe(DurationNative, Metric, Labels) + end. + +observe(Duration, Metric) when is_integer(Duration) -> + observe(Duration, Metric, []). +observe(Duration, Metric, Labels) when is_integer(Duration) -> + case ensure_started() of + ok -> + try prometheus_histogram:observe(Metric, Labels, Duration) + catch _:_ -> ok + end; + _ -> + ok end. + +inc(Type, Metrics) -> + inc(Type, Metrics, []). +inc(Type, Metrics, Labels) -> + inc(Type, Metrics, Labels, 1). +inc(Type, Metrics, Labels, Value) -> + case ensure_started() of + ok -> + try do_inc(Type, Metrics, Labels, Value) + catch error:mfa_already_exists -> ok + end; + _ -> ok + end. +do_inc(counter, Name, Labels, Value) -> + prometheus_counter:inc(Name, Labels, Value); +do_inc(gauge, Name, Labels, Value) -> + prometheus_gauge:inc(Name, Labels, Value). + +dec(Type, Metrics) -> + dec(Type, Metrics, []). +dec(Type, Metrics, Labels) -> + dec(Type, Metrics, Labels, 1). +dec(Type, Metrics, Labels, Value) -> + case ensure_started() of + ok -> + try do_dec(Type, Metrics, Labels, Value) + catch error:mfa_already_exists -> ok + end; + _ -> ok + end. + +do_dec(gauge, Name, Labels, Value) -> + prometheus_gauge:dec(Name, Labels, Value). \ No newline at end of file diff --git a/src/hb_store_arweave.erl b/src/hb_store_arweave.erl index fec49492e..d1739edeb 100644 --- a/src/hb_store_arweave.erl +++ b/src/hb_store_arweave.erl @@ -77,24 +77,25 @@ read_offset(#{ <<"index-store">> := IndexStore }, ID) -> {ok, OffsetBinary} -> {Version, CodecName, StartOffset, Length} = hb_store_arweave_offset:decode(OffsetBinary), - record_partition_metric(StartOffset), {ok, #{ <<"version">> => Version, <<"codec-device">> => CodecName, <<"start-offset">> => StartOffset, <<"length">> => Length }}; - _ -> not_found + _ -> + not_found end; read_offset(_, _) -> not_found. %% @doc Read the data at the given key, reading the `local-store' first if %% available. -read(StoreOpts, ID) -> +read(StoreOpts, ID) when ?IS_ID(ID) -> case hb_store_remote_node:read_local_cache(StoreOpts, ID) of {ok, Message} -> {ok, Message}; not_found -> do_read(StoreOpts, ID) - end. + end; +read(_, _) -> not_found. %% @doc Read the data at the given key, reading the provided Arweave index store %% as a source of offsets. After offsets have been found, the data is loaded @@ -128,6 +129,7 @@ do_read(StoreOpts, ID) -> {length, Length} } ), + record_partition_metric(StartOffset, ok), Loaded; {error, Reason} -> ?event( @@ -141,6 +143,7 @@ do_read(StoreOpts, ID) -> {reason, Reason} } ), + record_partition_metric(StartOffset, not_found), if Reason =:= not_found -> not_found; true -> {error, Reason} end @@ -252,21 +255,15 @@ write_offset( hb_store:write(IndexStore, hb_store_arweave_offset:path(ID), Value). %% @doc Record the partition that data is found in when it is requested. -record_partition_metric(Offset) when is_integer(Offset) -> - spawn( - fun () -> - case application:get_application(prometheus) of - undefined -> ok; - _ -> - Partition = Offset div ?PARTITION_SIZE, - prometheus_counter:inc( - hb_store_arweave_requests_partition, - [Partition], - 1 - ) - end - end - ). +record_partition_metric(Offset, Result) when is_integer(Offset) -> + spawn(fun() -> + hb_prometheus:inc( + counter, + hb_store_arweave_requests_partition, + [Offset div ?PARTITION_SIZE, hb_util:bin(Result)], + 1 + ) + end). %% @doc Initialize the Prometheus metrics for the Arweave store. Executed on %% `start/1' of the store. @@ -292,7 +289,7 @@ init_prometheus() -> counter, [ {name, hb_store_arweave_requests_partition}, - {labels, [partition]}, + {labels, [partition, result]}, {help, "Partition where chunks are being requested"} ] ), diff --git a/src/hb_store_gateway.erl b/src/hb_store_gateway.erl index 9799b3a14..ea992a30b 100644 --- a/src/hb_store_gateway.erl +++ b/src/hb_store_gateway.erl @@ -475,7 +475,8 @@ verifiability_test() -> ?assert(hb_message:verify(Structured)). %% @doc Reading an unsupported signature type transaction should fail -failure_to_process_message_test() -> +%% TODO: Enable when we find a TX that we don't support +failure_to_process_message_test_disabled() -> hb_http_server:start_node(#{}), ?assertEqual(failure, hb_cache:read( diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index acbe5980b..ae813c2d1 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -613,7 +613,11 @@ reset(Opts) -> %% @doc Increment the hit metrics for the current store's name. name_hit_metrics(Name) -> - prometheus_counter:inc(hb_store_lmdb_hit, [Name], 1). + hb_prometheus:inc( + counter, + hb_store_lmdb_hit, + [Name], + 1). init_prometheus() -> hb_prometheus:declare(histogram, [ @@ -998,29 +1002,29 @@ isolated_type_debug_test() -> % 2. Create nested groups for "commitments" and "other-test-key" CommitmentsPath = <>, OtherKeyPath = <>, - ?event(isolated_debug, {creating_nested_groups, CommitmentsPath, OtherKeyPath}), + ?event(debug_isolated, {creating_nested_groups, CommitmentsPath, OtherKeyPath}), make_group(StoreOpts, CommitmentsPath), make_group(StoreOpts, OtherKeyPath), % 3. Add some actual data within those groups write(StoreOpts, <>, <<"signature_data_1">>), write(StoreOpts, <>, <<"nested_value">>), % 4. Test type detection on the nested paths - ?event(isolated_debug, {testing_main_message_type}), + ?event(debug_isolated, {testing_main_message_type}), MainType = type(StoreOpts, MessageID), - ?event(isolated_debug, {main_message_type, MainType}), - ?event(isolated_debug, {testing_commitments_type}), + ?event(debug_isolated, {main_message_type, MainType}), + ?event(debug_isolated, {testing_commitments_type}), CommitmentsType = type(StoreOpts, CommitmentsPath), - ?event(isolated_debug, {commitments_type, CommitmentsType}), - ?event(isolated_debug, {testing_other_key_type}), + ?event(debug_isolated, {commitments_type, CommitmentsType}), + ?event(debug_isolated, {testing_other_key_type}), OtherKeyType = type(StoreOpts, OtherKeyPath), - ?event(isolated_debug, {other_key_type, OtherKeyType}), + ?event(debug_isolated, {other_key_type, OtherKeyType}), % 5. Test what happens when reading these nested paths - ?event(isolated_debug, {reading_commitments_directly}), + ?event(debug_isolated, {reading_commitments_directly}), CommitmentsResult = read(StoreOpts, CommitmentsPath), - ?event(isolated_debug, {commitments_read_result, CommitmentsResult}), - ?event(isolated_debug, {reading_other_key_directly}), + ?event(debug_isolated, {commitments_read_result, CommitmentsResult}), + ?event(debug_isolated, {reading_other_key_directly}), OtherKeyResult = read(StoreOpts, OtherKeyPath), - ?event(isolated_debug, {other_key_read_result, OtherKeyResult}), + ?event(debug_isolated, {other_key_read_result, OtherKeyResult}), stop(StoreOpts). %% @doc Test that list function resolves links correctly diff --git a/src/hb_store_remote_node.erl b/src/hb_store_remote_node.erl index c769c553e..90890e9c3 100644 --- a/src/hb_store_remote_node.erl +++ b/src/hb_store_remote_node.erl @@ -52,6 +52,8 @@ type(Opts = #{ <<"node">> := Node }, Key) -> %% @param Opts A map of options (including node configuration). %% @param Key The key to read. %% @returns {ok, Msg} on success or not_found if the key is missing. +read(#{ <<"only-ids">> := true }, Key) when not ?IS_ID(Key) -> + not_found; read(Opts = #{ <<"node">> := Node }, Key) -> ?event(store_remote_node, {executing_read, {node, Node}, {key, Key}}), HTTPRes = @@ -70,7 +72,8 @@ read(Opts = #{ <<"node">> := Node }, Key) -> {error, _Err} -> ?event(store_remote_node, {read_not_found, {key, Key}}), not_found - end. + end; +read(_, _) -> not_found. %% @doc Cache the data if the cache is enabled. The `local-store' option may %% either be `false' or a store definition to use as the local cache. Additional @@ -227,4 +230,25 @@ read_test() -> #{ <<"store-module">> => hb_store_remote_node, <<"node">> => Node } ], {ok, RetrievedMsg} = hb_cache:read(ID, #{ store => RemoteStore }), - ?assertMatch(#{ <<"test-key">> := Rand }, hb_cache:ensure_all_loaded(RetrievedMsg)). \ No newline at end of file + ?assertMatch(#{ <<"test-key">> := Rand }, hb_cache:ensure_all_loaded(RetrievedMsg)). + +read_only_ids_test() -> + LocalStore = hb_test_utils:test_store(), + hb_store:reset(LocalStore), + {ok, ID} = + hb_cache:write( + <<"message">>, + #{ store => LocalStore } + ), + Node = + hb_http_server:start_node( + #{ + store => LocalStore + } + ), + RemoteStore = [ + #{ <<"store-module">> => hb_store_remote_node, + <<"node">> => Node, + <<"only-ids">> => true } + ], + ?assertEqual(not_found, hb_cache:read(ID, #{ store => RemoteStore })). diff --git a/src/hb_sup.erl b/src/hb_sup.erl index 28b0176eb..aaea1c38b 100644 --- a/src/hb_sup.erl +++ b/src/hb_sup.erl @@ -32,7 +32,11 @@ init(Opts) -> type => worker, modules => [hb_http_client] }, - {ok, {SupFlags, [GunChild | StoreChildren]}}. + HttpMailBoxMonitoringChild = #{ + id => hb_http_conn_monitor, + start => {hb_http_conn_monitor, start_link, [Opts]} + }, + {ok, {SupFlags, [HttpMailBoxMonitoringChild | [GunChild | StoreChildren]]}}. %% @doc Generate a child spec for stores in the given Opts. store_children(Store) when not is_list(Store) -> diff --git a/src/hb_test_utils.erl b/src/hb_test_utils.erl index 02e36a1a8..2deb1ca44 100644 --- a/src/hb_test_utils.erl +++ b/src/hb_test_utils.erl @@ -7,6 +7,7 @@ -export([benchmark/1, benchmark/2, benchmark/3, benchmark_iterations/2]). -export([benchmark_print/2, benchmark_print/3, benchmark_print/4]). -export([compare_events/3, compare_events/4, compare_events/5]). +-export([preload/2]). -include("include/hb.hrl"). -include_lib("eunit/include/eunit.hrl"). @@ -261,3 +262,16 @@ format_time(Time) when is_integer(Time) -> hb_util:human_int(Time) ++ "s"; format_time(Time) -> hb_util:human_int(Time * 1000) ++ "ms". + +%% @doc Load ans104 binary files to a store. +preload(Opts, File) -> + {ok, SerializedItem} = file:read_file(hb_util:bin(File)), + Message = + hb_message:convert( + ar_bundles:deserialize(SerializedItem), + <<"structured@1.0">>, + <<"ans104@1.0">>, + Opts + ), + hb_cache:write(Message, Opts). + diff --git a/src/hb_util.erl b/src/hb_util.erl index 7e4db770e..653224ce2 100644 --- a/src/hb_util.erl +++ b/src/hb_util.erl @@ -20,7 +20,7 @@ -export([maybe_throw/2]). -export([is_hb_module/1, is_hb_module/2, all_hb_modules/0]). -export([ok/1, ok/2, until/1, until/2, until/3, wait_until/2]). --export([count/2, mean/1, stddev/1, variance/1, weighted_random/1]). +-export([count/2, mean/1, stddev/1, variance/1, weighted_random/1, shuffle/1]). -export([unique/1]). -export([split_depth_string_aware/2, split_depth_string_aware_single/2]). -export([unquote/1, split_escaped_single/2]). diff --git a/src/html/hyperbuddy@1.0/index.html b/src/html/hyperbuddy@1.0/index.html index 049ae14fa..aa2f6ef7f 100644 --- a/src/html/hyperbuddy@1.0/index.html +++ b/src/html/hyperbuddy@1.0/index.html @@ -5,10 +5,10 @@ HyperBEAM - +
- + diff --git a/src/include/dev_bundler.hrl b/src/include/dev_bundler.hrl new file mode 100644 index 000000000..c601785e0 --- /dev/null +++ b/src/include/dev_bundler.hrl @@ -0,0 +1,40 @@ +%%% Shared state and task records for the bundler server and workers. + +-record(state, { + max_size, + max_idle_time, + max_items, + queue, + bytes, + workers, + task_queue, + bundles, + opts +}). + +-record(task, { + bundle_id, + type, + data, + opts, + retry_count = 0 +}). + +-record(proof, { + proof, + status +}). + +-record(bundle, { + id, + items, + status, + tx, + proofs, + start_time +}). + +-define(DEFAULT_NUM_WORKERS, 20). +-define(DEFAULT_RETRY_BASE_DELAY_MS, 1000). +-define(DEFAULT_RETRY_MAX_DELAY_MS, 600000). +-define(DEFAULT_RETRY_JITTER, 0.25). diff --git a/src/include/hb_arweave_nodes.hrl b/src/include/hb_arweave_nodes.hrl index d7ea8e3bf..73cba8b53 100644 --- a/src/include/hb_arweave_nodes.hrl +++ b/src/include/hb_arweave_nodes.hrl @@ -6,16 +6,14 @@ <<"min">> => 0, <<"max">> => 57_600_000_000_000, <<"center">> => 28_800_000_000_000, - <<"with">> => <<"http://data-1.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-1.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 0, <<"max">> => 57_600_000_000_000, <<"center">> => 28_800_000_000_000, - <<"with">> => <<"http://data-13.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-13.arweave.xyz:1984">> }, %% Partitions 0-3 #{ @@ -23,8 +21,7 @@ <<"min">> => 0, <<"max">> => 14_400_000_000_000, <<"center">> => 7_200_000_000_000, - <<"with">> => <<"http://data-2.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-2.arweave.xyz:1984">> }, %% Partitions 4-7 #{ @@ -32,8 +29,7 @@ <<"min">> => 14_400_000_000_000, <<"max">> => 28_800_000_000_000, <<"center">> => 21_600_000_000_000, - <<"with">> => <<"http://data-3.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-3.arweave.xyz:1984">> }, %% Partitions 8-11 #{ @@ -41,8 +37,7 @@ <<"min">> => 28_800_000_000_000, <<"max">> => 43_200_000_000_000, <<"center">> => 36_000_000_000_000, - <<"with">> => <<"http://data-4.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-4.arweave.xyz:1984">> }, %% Partitions 12-15 #{ @@ -50,8 +45,7 @@ <<"min">> => 43_200_000_000_000, <<"max">> => 57_600_000_000_000, <<"center">> => 50_400_000_000_000, - <<"with">> => <<"http://data-5.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-5.arweave.xyz:1984">> }, %% Partitions 16-31 #{ @@ -59,32 +53,28 @@ <<"min">> => 57_600_000_000_000, <<"max">> => 115_200_000_000_000, <<"center">> => 86_400_000_000_000, - <<"with">> => <<"http://data-2.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-2.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 57_600_000_000_000, <<"max">> => 115_200_000_000_000, <<"center">> => 86_400_000_000_000, - <<"with">> => <<"http://data-3.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-3.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 57_600_000_000_000, <<"max">> => 115_200_000_000_000, <<"center">> => 86_400_000_000_000, - <<"with">> => <<"http://data-14.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-14.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 57_600_000_000_000, <<"max">> => 115_200_000_000_000, <<"center">> => 86_400_000_000_000, - <<"with">> => <<"http://data-15.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-15.arweave.xyz:1984">> }, %% Partitions 32-47 #{ @@ -92,32 +82,28 @@ <<"min">> => 115_200_000_000_000, <<"max">> => 172_800_000_000_000, <<"center">> => 144_000_000_000_000, - <<"with">> => <<"http://data-4.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-4.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 115_200_000_000_000, <<"max">> => 172_800_000_000_000, <<"center">> => 144_000_000_000_000, - <<"with">> => <<"http://data-5.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-5.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 115_200_000_000_000, <<"max">> => 172_800_000_000_000, <<"center">> => 144_000_000_000_000, - <<"with">> => <<"http://data-16.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-16.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 115_200_000_000_000, <<"max">> => 172_800_000_000_000, <<"center">> => 144_000_000_000_000, - <<"with">> => <<"http://data-17.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://data-17.arweave.xyz:1984">> } % Exclude these data nodes for now since their partitions are covered % by the tip nodes (and the tip nodes are faster to read from). @@ -190,40 +176,35 @@ <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-1.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-1.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-2.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-2.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-3.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-3.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-4.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-4.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, <<"min">> => 172_800_000_000_000, <<"max">> => 388_800_000_000_000, <<"center">> => 280_800_000_000_000, - <<"with">> => <<"http://tip-5.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://tip-5.arweave.xyz:1984">> } ]). @@ -231,12 +212,14 @@ [ #{ <<"match">> => <<"^/arweave">>, - <<"with">> => <<"http://chain-1.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://chain-3.arweave.xyz:1984">> }, #{ <<"match">> => <<"^/arweave">>, - <<"with">> => <<"http://chain-2.arweave.xyz:1984">>, - <<"opts">> => #{ http_client => gun, protocol => http2 } + <<"with">> => <<"http://chain-1.arweave.xyz:1984">> + }, + #{ + <<"match">> => <<"^/arweave">>, + <<"with">> => <<"http://chain-2.arweave.xyz:1984">> } ]). diff --git a/src/include/hb_http_client.hrl b/src/include/hb_http_client.hrl new file mode 100644 index 000000000..0b2c0e9bf --- /dev/null +++ b/src/include/hb_http_client.hrl @@ -0,0 +1,17 @@ +-define(DEFAULT_RETRIES, 0). +-define(DEFAULT_RETRY_TIME, 1000). +-define(DEFAULT_KEEPALIVE_TIMEOUT, 60_000). +-define(DEFAULT_CONNECT_TIMEOUT, 60_000). + +%% Connection Pool +-define(DEFAULT_CONN_POOL_READ_SIZE, 3). +-define(DEFAULT_CONN_POOL_WRITE_SIZE, 3). +%% Keep track of available connections +-define(CONNECTIONS_ETS, hb_http_client_connections). +%% Used to keep status of the connection +-define(CONN_STATUS_ETS, hb_http_client_conn_status). +%% Used for Round-robin connection +-define(CONN_COUNTER_ETS, hb_http_client_conn_counter). +%% Used to load connection pool configuration +-define(CONN_TERM, connection_pool_size). + diff --git a/src/include/hb_opts.hrl b/src/include/hb_opts.hrl new file mode 100644 index 000000000..a0eb30283 --- /dev/null +++ b/src/include/hb_opts.hrl @@ -0,0 +1,2 @@ +-define(DEFAULT_HTTP_CLIENT, gun). + diff --git a/test/arbundles.js/upload-items.js b/test/arbundles.js/upload-items.js index f5e629f51..9bc63d475 100644 --- a/test/arbundles.js/upload-items.js +++ b/test/arbundles.js/upload-items.js @@ -4,11 +4,11 @@ const { ArweaveSigner, createData } = require("@dha-team/arbundles"); // Configuration const BUNDLER_URL = "http://localhost:8734"; -const WALLET_PATH = "../../hyperbeam-key.json"; +const DEFAULT_WALLET = "../../hyperbeam-key.json"; const CONCURRENT_UPLOADS = 100; // Number of parallel uploads -async function performanceTest(itemCount, bytesPerItem = 0) { - const wallet = require(WALLET_PATH); +async function performanceTest(walletPath, itemCount, bytesPerItem = 0) { + const wallet = require(path.resolve(walletPath)); const signer = new ArweaveSigner(wallet); const endpoint = `${BUNDLER_URL}/~bundler@1.0/item?codec-device=ans104@1.0`; @@ -68,6 +68,7 @@ async function performanceTest(itemCount, bytesPerItem = 0) { const uploadPromises = batch.map(async (item) => { try { + console.log(`Posting data item: ${item.id}`); const response = await fetch(endpoint, { method: "POST", headers: { @@ -132,24 +133,28 @@ async function performanceTest(itemCount, bytesPerItem = 0) { // Main execution if (require.main === module) { - const itemCount = parseInt(process.argv[2], 10); - const bytesPerItem = parseInt(process.argv[3], 10) || 0; - + // If the first arg looks like a number, treat it as itemCount and use the default wallet + const firstIsNumber = !isNaN(parseInt(process.argv[2], 10)); + const walletPath = firstIsNumber ? DEFAULT_WALLET : (process.argv[2] || DEFAULT_WALLET); + const itemCount = parseInt(firstIsNumber ? process.argv[2] : process.argv[3], 10); + const bytesPerItem = parseInt(firstIsNumber ? process.argv[3] : process.argv[4], 10) || 0; + if (!itemCount || itemCount < 1 || isNaN(itemCount)) { - console.error("Usage: node upload-items.js [bytes_per_item]"); + console.error("Usage: node upload-items.js [wallet_path] [bytes_per_item]"); console.error(""); console.error("Arguments:"); + console.error(" wallet_path - Path to Arweave wallet JSON (default: ../../hyperbeam-key.json)"); console.error(" number_of_items - Number of data items to create and upload"); console.error(" bytes_per_item - Minimum size of each item in bytes (optional)"); console.error(""); console.error("Examples:"); console.error(" node upload-items.js 100"); - console.error(" node upload-items.js 100 1024 # 100 items, ~1KB each"); - console.error(" node upload-items.js 50 10485760 # 50 items, ~10MB each"); + console.error(" node upload-items.js 100 1024"); + console.error(" node upload-items.js /path/to/wallet.json 100 1024"); process.exit(1); } - performanceTest(itemCount, bytesPerItem) + performanceTest(walletPath, itemCount, bytesPerItem) .then(() => { process.exit(0); }) diff --git a/test/config.flat b/test/config.flat index be8c0d876..a28c640d9 100644 --- a/test/config.flat +++ b/test/config.flat @@ -1,3 +1,3 @@ port: 1234 -host: https://ao.computer +node_host: https://ao.computer await-inprogress: false \ No newline at end of file diff --git a/test/config.json b/test/config.json index 0be064746..37c2e2c96 100644 --- a/test/config.json +++ b/test/config.json @@ -1,7 +1,7 @@ { "port": 1234, "example": 9001, - "host": "https://ao.computer", + "node_host": "https://ao.computer", "await_inprogress": false, "store": [ {