diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index c66e68a47..a616c7dbc 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -12,3 +12,5 @@ export * from "./graphql/modules/MerkleWitnessResolver"; export * from "./graphql/modules/LinkedMerkleWitnessResolver"; export * from "./graphql/VanillaGraphqlModules"; export * from "./metrics/OpenTelemetryServer"; +export * from "./metrics/OpenTelemetryTracer"; +export * from "./metrics/ModularizedInstrumentation"; diff --git a/packages/api/src/metrics/ModularizedInstrumentation.ts b/packages/api/src/metrics/ModularizedInstrumentation.ts new file mode 100644 index 000000000..ac18b0632 --- /dev/null +++ b/packages/api/src/metrics/ModularizedInstrumentation.ts @@ -0,0 +1,97 @@ +import { injectable, injectAll } from "tsyringe"; +import { PollInstrumentation, PushInstrumentation } from "@proto-kit/sequencer"; +import { InstrumentationBase } from "@opentelemetry/instrumentation"; +import { mapSequential, splitArray } from "@proto-kit/common"; + +const INSTRUMENTATION_PREFIX = "protokit"; + +@injectable() +export class ModularizedInstrumentation extends InstrumentationBase<{}> { + // private readonly pushInstrumentations: PushInstrumentation[]; + + // private readonly pollInstrumentations: PollInstrumentation[]; + + public constructor( + @injectAll("Instrumentation") + private readonly instrumentations: ( + | PushInstrumentation + | PollInstrumentation + )[] + ) { + super("protokit", "canary", {}); + } + + public async start() { + const { pushInstrumentations } = this.splitInstrumentations(); + await mapSequential(pushInstrumentations, async (i) => await i.start()); + } + + public splitInstrumentations() { + // PushInstrumentation is abstract, therefore we can filter like this, while Poll + // isn't. But all remainders are of that type (bcs of the annotation) so it's fine + const split = splitArray(this.instrumentations, (i) => + i instanceof PushInstrumentation ? "push" : "poll" + ); + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + const pushInstrumentations = (split.push as PushInstrumentation[]) ?? []; + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + const pollInstrumentations = (split.poll as PollInstrumentation[]) ?? []; + + return { + pushInstrumentations, + pollInstrumentations, + }; + } + + public initialize() { + const { pushInstrumentations, pollInstrumentations } = + this.splitInstrumentations(); + + pollInstrumentations.forEach((instrumentation) => { + const observableCounter = this.meter.createObservableCounter( + `${INSTRUMENTATION_PREFIX}_${instrumentation.name}`, + { + description: instrumentation.description, + } + ); + + this.meter.addBatchObservableCallback( + async (observableResult) => { + const metric = await instrumentation.poll(); + + observableResult.observe(observableCounter, metric); + }, + [observableCounter] + ); + }); + + pushInstrumentations.forEach((instrumentation) => { + const gauges = instrumentation.names.map((name) => { + const gauge = this.meter.createGauge( + `${INSTRUMENTATION_PREFIX}_${name}`, + { + description: instrumentation.description, + } + ); + return [name, gauge] as const; + }); + + instrumentation.setPushFn((name, v) => { + const gauge = gauges.find(([candidate]) => candidate === name); + gauge![1].record(v); + }); + }); + } + + // Called when a new `MeterProvider` is set + // the Meter (result of @opentelemetry/api's getMeter) is + // available as this.meter within this method + // eslint-disable-next-line no-underscore-dangle + override _updateMetricInstruments() { + if (this.instrumentations !== undefined) { + this.initialize(); + } + } + + init() {} +} diff --git a/packages/api/src/metrics/OpenTelemetryServer.ts b/packages/api/src/metrics/OpenTelemetryServer.ts index e83e32d96..bb871c79a 100644 --- a/packages/api/src/metrics/OpenTelemetryServer.ts +++ b/packages/api/src/metrics/OpenTelemetryServer.ts @@ -16,8 +16,8 @@ import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api"; import { inject } from "tsyringe"; import { dependencyFactory, DependencyRecord, log } from "@proto-kit/common"; -import { SequencerInstrumentation } from "./SequencerInstrumentation"; import { OpenTelemetryTracer } from "./OpenTelemetryTracer"; +import { ModularizedInstrumentation } from "./ModularizedInstrumentation"; export type OpenTelemetryServerConfig = { metrics?: { @@ -54,9 +54,8 @@ export class OpenTelemetryServer extends SequencerModule { - private blockProduced: (height: number) => void = () => {}; - - public constructor( - @inject("BlockTrigger", { isOptional: true }) - trigger: BlockTriggerBase | undefined, - @inject("Mempool", { isOptional: true }) - private readonly mempool: PrivateMempool | undefined - ) { - super("protokit", "canary", {}); - if (trigger !== undefined) { - trigger.events.on("block-produced", (block) => { - this.blockProduced(parseInt(block.height.toString(), 10)); - }); - } - } - - // Called when a new `MeterProvider` is set - // the Meter (result of @opentelemetry/api's getMeter) is - // available as this.meter within this method - // eslint-disable-next-line no-underscore-dangle - override _updateMetricInstruments() { - const { mempool } = this; - - if (mempool !== undefined) { - const mempoolSize = this.meter.createObservableCounter( - "protokit_mempool_size", - { - description: "The size of the mempool", - } - ); - - this.meter.addBatchObservableCallback( - async (observableResult) => { - const mempoolLength = await mempool.length(); - - observableResult.observe(mempoolSize, mempoolLength); - }, - [mempoolSize] - ); - } - - const blockHeight = this.meter.createGauge("protokit_block_height"); - this.blockProduced = (height) => blockHeight.record(height); - } - - init() {} -} diff --git a/packages/sequencer/src/index.ts b/packages/sequencer/src/index.ts index 4946ec580..77fee6d6a 100644 --- a/packages/sequencer/src/index.ts +++ b/packages/sequencer/src/index.ts @@ -136,3 +136,6 @@ export * from "./appChain/AppChain"; export * from "./appChain/AppChainModule"; export * from "./appChain/AreProofsEnabledFactory"; export * from "./appChain/SharedDependencyFactory"; +export * from "./metrics/Instrumentation"; +export * from "./metrics/BlockProductionInstrumentation"; +export * from "./metrics/MempoolInstrumentation"; diff --git a/packages/sequencer/src/mempool/private/PrivateMempool.ts b/packages/sequencer/src/mempool/private/PrivateMempool.ts index e3c2f6d70..edc0ddce1 100644 --- a/packages/sequencer/src/mempool/private/PrivateMempool.ts +++ b/packages/sequencer/src/mempool/private/PrivateMempool.ts @@ -1,4 +1,4 @@ -import { EventEmitter, log, noop } from "@proto-kit/common"; +import { dependencyFactory, EventEmitter, log, noop } from "@proto-kit/common"; import { inject } from "tsyringe"; import type { Mempool, MempoolEvents } from "../Mempool"; @@ -14,6 +14,7 @@ import { trace } from "../../logging/trace"; import { IncomingMessagesService } from "../../settlement/messages/IncomingMessagesService"; import { MempoolSorting } from "../sorting/MempoolSorting"; import { DefaultMempoolSorting } from "../sorting/DefaultMempoolSorting"; +import { MempoolInstrumentation } from "../../metrics/MempoolInstrumentation"; type PrivateMempoolConfig = { type?: "hybrid" | "private" | "based"; @@ -21,6 +22,7 @@ type PrivateMempoolConfig = { }; @sequencerModule() +@dependencyFactory() export class PrivateMempool extends SequencerModule implements Mempool @@ -43,6 +45,14 @@ export class PrivateMempool this.mempoolSorting = mempoolSorting ?? new DefaultMempoolSorting(); } + public static dependencies() { + return { + MempoolInstrumentation: { + useClass: MempoolInstrumentation, + }, + }; + } + private type() { return this.config.type ?? "hybrid"; } diff --git a/packages/sequencer/src/metrics/BlockProductionInstrumentation.ts b/packages/sequencer/src/metrics/BlockProductionInstrumentation.ts new file mode 100644 index 000000000..1f3600339 --- /dev/null +++ b/packages/sequencer/src/metrics/BlockProductionInstrumentation.ts @@ -0,0 +1,37 @@ +import { inject, injectable } from "tsyringe"; + +import type { BlockTriggerBase } from "../protocol/production/trigger/BlockTrigger"; + +import { instrumentation, PushInstrumentation } from "./Instrumentation"; + +@instrumentation() +@injectable() +export class BlockProductionInstrumentation extends PushInstrumentation { + names = ["block_height", "block_result", "batch_height"]; + + description = "L2 block height"; + + public constructor( + @inject("BlockTrigger") + private readonly trigger: BlockTriggerBase + ) { + super(); + } + + public async start() { + this.trigger.events.on("block-produced", (block) => { + this.pushValue("block_height", parseInt(block.height.toString(), 10)); + }); + + this.trigger.events.on("block-metadata-produced", (block) => { + this.pushValue( + "block_result", + parseInt(block.block.height.toString(), 10) + ); + }); + + this.trigger.events.on("batch-produced", (batch) => { + this.pushValue("batch_height", parseInt(batch.height.toString(), 10)); + }); + } +} diff --git a/packages/sequencer/src/metrics/Instrumentation.ts b/packages/sequencer/src/metrics/Instrumentation.ts new file mode 100644 index 000000000..f5ed0cbad --- /dev/null +++ b/packages/sequencer/src/metrics/Instrumentation.ts @@ -0,0 +1,41 @@ +import { implement, Startable } from "@proto-kit/common"; + +export function instrumentation() { + return implement( + "Instrumentation" + ); +} + +export interface InstrumentationModule { + name: string; + + description?: string; +} + +export interface PollInstrumentation extends InstrumentationModule { + poll(): Promise; +} + +export abstract class PushInstrumentation implements Startable { + abstract names: string[]; + + abstract description?: string; + + private pushFn?: (name: string, value: number) => void; + + public setPushFn(f: (name: string, value: number) => void) { + this.pushFn = f; + } + + protected pushValue(name: string, value: number) { + if (this.pushFn === undefined) { + throw new Error("Pushfn not initialized"); + } + if (!this.names.includes(name)) { + throw new Error(`Name ${name} not specified in declared name list`); + } + this.pushFn(name, value); + } + + abstract start(): Promise; +} diff --git a/packages/sequencer/src/metrics/MempoolInstrumentation.ts b/packages/sequencer/src/metrics/MempoolInstrumentation.ts new file mode 100644 index 000000000..f4fca673c --- /dev/null +++ b/packages/sequencer/src/metrics/MempoolInstrumentation.ts @@ -0,0 +1,22 @@ +import { inject, injectable } from "tsyringe"; + +import type { PrivateMempool } from "../mempool/private/PrivateMempool"; + +import { instrumentation, PollInstrumentation } from "./Instrumentation"; + +@instrumentation() +@injectable() +export class MempoolInstrumentation implements PollInstrumentation { + name = "mempool_size"; + + description = "The size of the mempool"; + + public constructor( + @inject("Mempool") + private readonly mempool: PrivateMempool + ) {} + + async poll() { + return await this.mempool.length(); + } +} diff --git a/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts b/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts index 36c01a75c..33e8bfae4 100644 --- a/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts +++ b/packages/sequencer/src/protocol/production/trigger/TimedBlockTrigger.ts @@ -1,5 +1,5 @@ import { inject, injectable } from "tsyringe"; -import { log } from "@proto-kit/common"; +import { dependencyFactory, log } from "@proto-kit/common"; import { closeable, Closeable } from "../../../sequencer/builder/Closeable"; import { BatchProducerModule } from "../BatchProducerModule"; @@ -13,6 +13,7 @@ import { } from "../../../settlement/BridgingModule"; import { ensureNotBusy } from "../../../helpers/BusyGuard"; import { SequencerStartupModule } from "../../../sequencer/SequencerStartupModule"; +import { BlockProductionInstrumentation } from "../../../metrics/BlockProductionInstrumentation"; import { BlockTriggerBase } from "./BlockTrigger"; @@ -26,6 +27,7 @@ export interface TimedBlockTriggerConfig { @injectable() @closeable() +@dependencyFactory() export class TimedBlockTrigger extends BlockTriggerBase implements Closeable @@ -59,6 +61,14 @@ export class TimedBlockTrigger ); } + public static dependencies() { + return { + BlockProductionInstrumentation: { + useClass: BlockProductionInstrumentation, + }, + }; + } + public async start(): Promise { log.info("Starting timed block trigger"); const { settlementInterval, blockInterval } = this.config; diff --git a/packages/stack/src/scripts/graphql/server.ts b/packages/stack/src/scripts/graphql/server.ts index 7b22f4319..57f526a75 100644 --- a/packages/stack/src/scripts/graphql/server.ts +++ b/packages/stack/src/scripts/graphql/server.ts @@ -174,6 +174,10 @@ export async function startServer() { }, metrics: { enabled: true, + prometheus: { + port: 9464, + host: "0.0.0.0", + }, }, }, @@ -239,7 +243,7 @@ export async function startServer() { let nonce = Number(as?.nonce.toString() ?? "0"); setInterval(async () => { - const random = Math.floor(Math.random() * 5); + const random = 0; //Math.floor(Math.random() * 5); await mapSequential(range(0, random), async () => { const tx = await appChain.transaction( priv.toPublicKey(),