1use std::{
12 future::Future,
13 pin::Pin,
14 sync::{Arc, OnceLock},
15};
16
17use alloy_consensus::BlockHeader;
18use arb_payload::ArbEngineTypes;
19use arb_primitives::ArbPrimitives;
20use futures::{FutureExt, StreamExt, stream::FusedStream, stream_select};
21use reth_chainspec::{EthChainSpec, EthereumHardforks};
22use reth_engine_tree::{
23 chain::{ChainEvent, FromOrchestrator},
24 engine::{EngineApiKind, EngineApiRequest, EngineRequestHandler},
25 tree::TreeConfig,
26};
27use reth_engine_util::EngineMessageStreamExt;
28use reth_exex::ExExManagerHandle;
29use reth_network::{NetworkSyncUpdater, SyncState, types::BlockRangeUpdate};
30use reth_network_api::BlockDownloaderProvider;
31use reth_node_api::{
32 BuiltPayload, ConsensusEngineHandle, FullNodeTypes, NodeTypes, NodeTypesWithDBAdapter,
33};
34use reth_node_builder::{
35 AddOns, AddOnsContext, FullNode, LaunchContext, LaunchNode, NodeAdapter,
36 NodeBuilderWithComponents, NodeComponents, NodeComponentsBuilder, NodeHandle, NodeTypesAdapter,
37 common::{Attached, LaunchContextWith, WithConfigs},
38 hooks::NodeHooks,
39 rpc::{EngineShutdown, EngineValidatorAddOn, EngineValidatorBuilder, RethRpcAddOns, RpcHandle},
40 setup::build_networked_pipeline,
41};
42use reth_node_core::{
43 dirs::{ChainPath, DataDirPath},
44 exit::NodeExitFuture,
45 primitives::Head,
46};
47use reth_node_events::node;
48use reth_provider::{
49 BlockNumReader, StorageSettingsCache,
50 providers::{BlockchainProvider, NodeTypesForProvider},
51};
52use reth_tasks::TaskExecutor;
53use reth_tokio_util::EventSender;
54use reth_tracing::tracing::{debug, error, info};
55use reth_trie_db::ChangesetCache;
56use tokio::sync::{mpsc::unbounded_channel, oneshot};
57use tokio_stream::wrappers::UnboundedReceiverStream;
58
59use crate::{
60 engine::{TreeSender, build_arb_engine_orchestrator},
61 error::LauncherError,
62};
63
64static TREE_SENDER: OnceLock<TreeSender<ArbEngineTypes, ArbPrimitives>> = OnceLock::new();
65static ENGINE_HANDLE: OnceLock<ConsensusEngineHandle<ArbEngineTypes>> = OnceLock::new();
66static FLUSH_HANDLE: OnceLock<FlushHandle> = OnceLock::new();
67static PARALLEL_STATE_ROOT_FN: OnceLock<ParallelStateRootFn> = OnceLock::new();
68
69pub enum PersistenceRequest {
71 Flush(FlushRequest),
72 Unwind {
73 target: u64,
74 done: crossbeam_channel::Sender<Result<(), LauncherError>>,
75 },
76}
77
78pub struct FlushRequest {
80 pub blocks: Vec<reth_chain_state::ExecutedBlock<ArbPrimitives>>,
81 pub last_num_hash: alloy_eips::BlockNumHash,
82}
83
84pub struct FlushResult {
86 pub last_num_hash: alloy_eips::BlockNumHash,
87 pub count: usize,
88 pub duration: std::time::Duration,
89}
90
91struct FlushHandle {
93 sender: std::sync::mpsc::Sender<PersistenceRequest>,
94 result_rx: crossbeam_channel::Receiver<FlushResult>,
95 flush_done: Arc<tokio::sync::Notify>,
96}
97
98type ParallelStateRootFn = Box<
100 dyn Fn(
101 Arc<reth_trie_common::TrieInputSorted>,
102 reth_trie_common::prefix_set::TriePrefixSets,
103 )
104 -> Result<(alloy_primitives::B256, reth_trie::updates::TrieUpdates), LauncherError>
105 + Send
106 + Sync,
107>;
108
109pub fn tree_sender() -> Option<&'static TreeSender<ArbEngineTypes, ArbPrimitives>> {
110 TREE_SENDER.get()
111}
112
113pub fn engine_handle() -> Option<&'static ConsensusEngineHandle<ArbEngineTypes>> {
114 ENGINE_HANDLE.get()
115}
116
117pub fn start_flush(request: FlushRequest) {
119 if let Some(handle) = FLUSH_HANDLE.get()
120 && let Err(e) = handle.sender.send(PersistenceRequest::Flush(request))
121 {
122 error!(target: "reth::cli", "Failed to send flush request: {e}");
123 }
124}
125
126pub fn start_unwind(target: u64) -> Option<crossbeam_channel::Receiver<Result<(), LauncherError>>> {
131 let handle = FLUSH_HANDLE.get()?;
132 let (done_tx, done_rx) = crossbeam_channel::bounded(1);
133 if let Err(e) = handle.sender.send(PersistenceRequest::Unwind {
134 target,
135 done: done_tx,
136 }) {
137 error!(target: "reth::cli", "Failed to send unwind request: {e}");
138 return None;
139 }
140 Some(done_rx)
141}
142
143pub fn try_flush_result() -> Option<FlushResult> {
145 FLUSH_HANDLE
146 .get()
147 .and_then(|handle| handle.result_rx.try_recv().ok())
148}
149
150pub fn flush_notifier() -> Option<Arc<tokio::sync::Notify>> {
152 FLUSH_HANDLE.get().map(|handle| handle.flush_done.clone())
153}
154
155pub fn compute_parallel_state_root(
156 overlay: Arc<reth_trie_common::TrieInputSorted>,
157 prefix_sets: reth_trie_common::prefix_set::TriePrefixSets,
158) -> Result<(alloy_primitives::B256, reth_trie::updates::TrieUpdates), LauncherError> {
159 let f = PARALLEL_STATE_ROOT_FN
160 .get()
161 .ok_or(LauncherError::ParallelStateRootNotInitialized)?;
162 f(overlay, prefix_sets)
163}
164
165#[derive(Debug)]
170pub struct ArbEngineLauncher {
171 pub ctx: LaunchContext,
172 pub engine_tree_config: TreeConfig,
173}
174
175impl ArbEngineLauncher {
176 pub const fn new(
177 task_executor: TaskExecutor,
178 data_dir: ChainPath<DataDirPath>,
179 engine_tree_config: TreeConfig,
180 ) -> Self {
181 Self {
182 ctx: LaunchContext::new(task_executor, data_dir),
183 engine_tree_config,
184 }
185 }
186
187 async fn launch_node<T, CB, AO>(
190 self,
191 target: NodeBuilderWithComponents<T, CB, AO>,
192 ) -> eyre::Result<NodeHandle<NodeAdapter<T, CB::Components>, AO>>
193 where
194 T: FullNodeTypes<
195 Types: NodeTypesForProvider<Payload = ArbEngineTypes, Primitives = ArbPrimitives>,
196 Provider = BlockchainProvider<
197 NodeTypesWithDBAdapter<<T as FullNodeTypes>::Types, <T as FullNodeTypes>::DB>,
198 >,
199 >,
200 CB: NodeComponentsBuilder<T>,
201 AO: RethRpcAddOns<NodeAdapter<T, CB::Components>>
202 + EngineValidatorAddOn<NodeAdapter<T, CB::Components>>,
203 {
204 let Self {
205 ctx,
206 engine_tree_config,
207 } = self;
208 let NodeBuilderWithComponents {
209 adapter: NodeTypesAdapter { database },
210 rocksdb_provider,
211 components_builder,
212 add_ons:
213 AddOns {
214 hooks,
215 exexs: installed_exex,
216 add_ons,
217 },
218 config,
219 } = target;
220 let NodeHooks {
221 on_component_initialized,
222 on_node_started,
223 ..
224 } = hooks;
225
226 let changeset_cache = ChangesetCache::new();
227
228 let ctx = ctx
229 .with_configured_globals(engine_tree_config.reserved_cpu_cores())
230 .with_loaded_toml_config(config)?
231 .with_resolved_peers()?
232 .attach(database.clone())
233 .with_adjusted_configs()
234 .with_provider_factory::<_, <CB::Components as NodeComponents<T>>::Evm>(
235 changeset_cache.clone(),
236 rocksdb_provider,
237 )
238 .await?
239 .inspect(|_| {
240 info!(target: "reth::cli", "Database opened");
241 })
242 .with_prometheus_server()
243 .await?
244 .inspect(|this| {
245 debug!(target: "reth::cli", chain=%this.chain_id(), genesis=?this.genesis_hash(), "Initializing genesis");
246 })
247 .with_genesis()?
248 .inspect(
249 |this: &LaunchContextWith<
250 Attached<WithConfigs<<T::Types as NodeTypes>::ChainSpec>, _>,
251 >| {
252 info!(target: "reth::cli", "\n{}", this.chain_spec().display_hardforks());
253 let settings = this.provider_factory().cached_storage_settings();
254 info!(target: "reth::cli", ?settings, "Loaded storage settings");
255 },
256 )
257 .with_metrics_task()
258 .with_blockchain_db::<T, _>(move |provider_factory| {
259 Ok(BlockchainProvider::new(provider_factory)?)
260 })?
261 .with_components(components_builder, on_component_initialized)
262 .await?;
263
264 let maybe_exex_manager_handle = ctx.launch_exex(installed_exex).await?;
265
266 let network_handle = ctx.components().network().clone();
267 let network_client = network_handle.fetch_client().await?;
268 let (consensus_engine_tx, consensus_engine_rx) = unbounded_channel();
269
270 let node_config = ctx.node_config();
271
272 network_handle.update_sync_state(SyncState::Syncing);
273
274 let max_block = ctx.max_block(network_client.clone()).await?;
275
276 let static_file_producer = ctx.static_file_producer();
277 let static_file_producer_events = static_file_producer.lock().events();
278 info!(target: "reth::cli", "StaticFileProducer initialized");
279
280 let consensus = Arc::new(ctx.components().consensus().clone());
281
282 let pipeline = build_networked_pipeline(
283 &ctx.toml_config().stages,
284 network_client.clone(),
285 consensus.clone(),
286 ctx.provider_factory().clone(),
287 ctx.task_executor(),
288 ctx.sync_metrics_tx(),
289 ctx.prune_config(),
290 max_block,
291 static_file_producer,
292 ctx.components().evm_config().clone(),
293 maybe_exex_manager_handle
294 .clone()
295 .unwrap_or_else(ExExManagerHandle::empty),
296 ctx.era_import_source(),
297 )?;
298
299 pipeline.move_to_static_files()?;
300
301 let pipeline_events = pipeline.events();
302
303 let mut pruner_builder = ctx.pruner_builder();
304 if let Some(exex_manager_handle) = &maybe_exex_manager_handle {
305 pruner_builder =
306 pruner_builder.finished_exex_height(exex_manager_handle.finished_height());
307 }
308 let pruner = pruner_builder.build_with_provider_factory(ctx.provider_factory().clone());
309 let pruner_events = pruner.events();
310 info!(target: "reth::cli", prune_config=?ctx.prune_config(), "Pruner initialized");
311
312 let event_sender = EventSender::default();
313
314 let beacon_engine_handle = ConsensusEngineHandle::new(consensus_engine_tx.clone());
315
316 let jwt_secret = ctx.auth_jwt_secret()?;
317
318 let add_ons_ctx = AddOnsContext {
319 node: ctx.node_adapter().clone(),
320 config: ctx.node_config(),
321 beacon_engine_handle: beacon_engine_handle.clone(),
322 jwt_secret,
323 engine_events: event_sender.clone(),
324 };
325 let validator_builder = add_ons.engine_validator_builder();
326
327 let engine_validator = validator_builder
328 .clone()
329 .build_tree_validator(
330 &add_ons_ctx,
331 engine_tree_config.clone(),
332 changeset_cache.clone(),
333 )
334 .await?;
335
336 let consensus_engine_stream = UnboundedReceiverStream::from(consensus_engine_rx)
337 .maybe_skip_fcu(node_config.debug.skip_fcu)
338 .maybe_skip_new_payload(node_config.debug.skip_new_payload)
339 .maybe_reorg(
340 ctx.blockchain_db().clone(),
341 ctx.components().evm_config().clone(),
342 || async {
343 let reorg_cache = ChangesetCache::new();
344 validator_builder
345 .build_tree_validator(&add_ons_ctx, engine_tree_config.clone(), reorg_cache)
346 .await
347 },
348 node_config.debug.reorg_frequency,
349 node_config.debug.reorg_depth,
350 )
351 .await?
352 .maybe_store_messages(node_config.debug.engine_api_store.clone());
353
354 let engine_kind = if ctx.chain_spec().is_optimism() {
355 EngineApiKind::OpStack
356 } else {
357 EngineApiKind::Ethereum
358 };
359
360 {
364 use reth_provider::{DatabaseProviderFactory, SaveBlocksMode};
365 use reth_storage_api::{BlockExecutionWriter, DBProvider};
366
367 let pf = ctx.provider_factory().clone();
368 let (req_tx, req_rx) = std::sync::mpsc::channel::<PersistenceRequest>();
369 let (res_tx, res_rx) = crossbeam_channel::bounded::<FlushResult>(1);
370 let flush_done = Arc::new(tokio::sync::Notify::new());
371 let flush_done_thread = flush_done.clone();
372
373 std::thread::Builder::new()
374 .name("arb-persistence".into())
375 .spawn(move || {
376 while let Ok(req) = req_rx.recv() {
377 match req {
378 PersistenceRequest::Flush(flush) => {
379 let start = std::time::Instant::now();
380 let count = flush.blocks.len();
381 let last = flush.last_num_hash;
382
383 let result = (|| -> Result<(), LauncherError> {
384 let provider_rw = pf.database_provider_rw()?;
385 provider_rw.save_blocks(flush.blocks, SaveBlocksMode::Full)?;
386 provider_rw.commit()?;
387 Ok(())
388 })();
389
390 match result {
391 Ok(()) => {
392 let _ = res_tx.send(FlushResult {
393 last_num_hash: last,
394 count,
395 duration: start.elapsed(),
396 });
397 }
398 Err(e) => {
399 error!(target: "reth::cli", "Background flush failed: {e}");
400 let _ = res_tx.send(FlushResult {
401 last_num_hash: last,
402 count: 0, duration: start.elapsed(),
404 });
405 }
406 }
407 flush_done_thread.notify_one();
408 }
409 PersistenceRequest::Unwind { target, done } => {
410 let start = std::time::Instant::now();
411 let result = (|| -> Result<(), LauncherError> {
412 let provider_rw = pf.database_provider_rw()?;
413 provider_rw.remove_block_and_execution_above(target)?;
414 provider_rw.commit()?;
415 Ok(())
416 })();
417 match &result {
418 Ok(()) => info!(
419 target: "reth::cli",
420 target,
421 duration_ms = start.elapsed().as_millis(),
422 "Persisted unwind complete"
423 ),
424 Err(e) => error!(
425 target: "reth::cli",
426 target,
427 err = %e,
428 "Persisted unwind failed"
429 ),
430 }
431 let _ = done.send(result);
432 }
433 }
434 }
435 })
436 .expect("failed to spawn persistence thread");
437
438 let _ = FLUSH_HANDLE.set(FlushHandle {
439 sender: req_tx,
440 result_rx: res_rx,
441 flush_done,
442 });
443 }
444
445 {
446 use reth_chain_state::{
447 AnchoredTrieInput, ComputedTrieData, DeferredTrieData, LazyOverlay,
448 };
449 use reth_provider::providers::OverlayStateProviderFactory;
450 use reth_trie_parallel::root::ParallelStateRoot;
451
452 let pf = ctx.provider_factory().clone();
453 let runtime = ctx.task_executor().clone();
454 let changeset_cache_for_root = changeset_cache.clone();
455
456 let state_root_fn: ParallelStateRootFn = Box::new(move |overlay, prefix_sets| {
457 let anchor_hash = alloy_primitives::B256::ZERO;
458 let computed = ComputedTrieData {
459 hashed_state: Arc::clone(&overlay.state),
460 trie_updates: Arc::clone(&overlay.nodes),
461 anchored_trie_input: Some(AnchoredTrieInput {
462 anchor_hash,
463 trie_input: overlay,
464 }),
465 };
466 let lazy = LazyOverlay::new(anchor_hash, vec![DeferredTrieData::ready(computed)]);
467 let factory =
468 OverlayStateProviderFactory::new(pf.clone(), changeset_cache_for_root.clone())
469 .with_lazy_overlay(Some(lazy));
470
471 ParallelStateRoot::new(factory, prefix_sets, runtime.clone())
472 .incremental_root_with_updates()
473 .map_err(LauncherError::from)
474 });
475 let _ = PARALLEL_STATE_ROOT_FN.set(state_root_fn);
476 }
477
478 let (mut orchestrator, arb_tree_sender) = build_arb_engine_orchestrator(
479 engine_kind,
480 consensus.clone(),
481 network_client.clone(),
482 Box::pin(consensus_engine_stream),
483 pipeline,
484 ctx.task_executor().clone(),
485 ctx.provider_factory().clone(),
486 ctx.blockchain_db().clone(),
487 pruner,
488 ctx.components().payload_builder_handle().clone(),
489 engine_validator,
490 engine_tree_config,
491 ctx.sync_metrics_tx(),
492 ctx.components().evm_config().clone(),
493 changeset_cache,
494 ctx.task_executor().clone(),
495 );
496
497 let _ = TREE_SENDER.set(arb_tree_sender);
498 let _ = ENGINE_HANDLE.set(beacon_engine_handle.clone());
499 info!(target: "reth::cli", "Arbitrum engine tree sender and handle captured");
500
501 info!(target: "reth::cli", "Consensus engine initialized");
502
503 #[allow(clippy::needless_continue)]
504 let events = stream_select!(
505 event_sender.new_listener().map(Into::into),
506 pipeline_events.map(Into::into),
507 ctx.consensus_layer_events(),
508 pruner_events.map(Into::into),
509 static_file_producer_events.map(Into::into),
510 );
511
512 ctx.task_executor().spawn_critical_task(
513 "events task",
514 Box::pin(node::handle_events(
515 Some(Box::new(ctx.components().network().clone())),
516 Some(ctx.head().number),
517 events,
518 )),
519 );
520
521 let RpcHandle {
522 rpc_server_handles,
523 rpc_registry,
524 engine_events,
525 beacon_engine_handle,
526 engine_shutdown: _,
527 } = add_ons.launch_add_ons(add_ons_ctx).await?;
528
529 let (engine_shutdown, shutdown_rx) = EngineShutdown::new();
530
531 let initial_target = ctx.initial_backfill_target()?;
532 let mut built_payloads = ctx
533 .components()
534 .payload_builder_handle()
535 .subscribe()
536 .await
537 .map_err(|e| eyre::eyre!("Failed to subscribe to payload builder events: {:?}", e))?
538 .into_built_payload_stream()
539 .fuse();
540
541 let chainspec = ctx.chain_spec();
542 let provider = ctx.blockchain_db().clone();
543 let (exit, rx) = oneshot::channel();
544 let terminate_after_backfill = ctx.terminate_after_initial_backfill();
545 let startup_sync_state_idle = ctx.node_config().debug.startup_sync_state_idle;
546
547 info!(target: "reth::cli", "Starting consensus engine");
548 let consensus_engine = move |mut on_graceful_shutdown| async move {
549 if let Some(initial_target) = initial_target {
550 debug!(target: "reth::cli", %initial_target, "start backfill sync");
551 orchestrator.start_backfill_sync(initial_target);
552 } else if startup_sync_state_idle {
553 network_handle.update_sync_state(SyncState::Idle);
554 }
555
556 let mut res = Ok(());
557 let mut shutdown_rx = shutdown_rx.fuse();
558
559 loop {
560 tokio::select! {
561 event = orchestrator.next() => {
562 let Some(event) = event else { break };
563 debug!(target: "reth::cli", "Event: {event}");
564 match event {
565 ChainEvent::BackfillSyncFinished => {
566 if terminate_after_backfill {
567 debug!(target: "reth::cli", "Terminating after initial backfill");
568 break
569 }
570 if startup_sync_state_idle {
571 network_handle.update_sync_state(SyncState::Idle);
572 }
573 }
574 ChainEvent::BackfillSyncStarted => {
575 network_handle.update_sync_state(SyncState::Syncing);
576 }
577 ChainEvent::FatalError => {
578 error!(target: "reth::cli", "Fatal error in consensus engine");
579 res = Err(eyre::eyre!("Fatal error in consensus engine"));
580 break
581 }
582 ChainEvent::Handler(ev) => {
583 if let Some(head) = ev.canonical_header() {
584 network_handle.update_sync_state(SyncState::Idle);
585 let head_block = Head {
586 number: head.number(),
587 hash: head.hash(),
588 difficulty: head.difficulty(),
589 timestamp: head.timestamp(),
590 total_difficulty: chainspec.final_paris_total_difficulty()
591 .filter(|_| chainspec.is_paris_active_at_block(head.number()))
592 .unwrap_or_default(),
593 };
594 network_handle.update_status(head_block);
595
596 let updated = BlockRangeUpdate {
597 earliest: provider.earliest_block_number().unwrap_or_default(),
598 latest: head.number(),
599 latest_hash: head.hash(),
600 };
601 network_handle.update_block_range(updated);
602 }
603 event_sender.notify(ev);
604 }
605 }
606 }
607 payload = built_payloads.select_next_some(), if !built_payloads.is_terminated() => {
608 if let Some(executed_block) = payload.executed_block() {
609 debug!(target: "reth::cli", block=?executed_block.recovered_block.num_hash(), "inserting built payload");
610 orchestrator.handler_mut().handler_mut().on_event(EngineApiRequest::InsertExecutedBlock(executed_block.into_executed_payload()).into());
611 }
612 }
613 shutdown_req = &mut shutdown_rx => {
614 if let Ok(req) = shutdown_req {
615 debug!(target: "reth::cli", "received engine shutdown request");
616 orchestrator.handler_mut().handler_mut().on_event(
617 FromOrchestrator::Terminate { tx: req.done_tx }.into()
618 );
619 }
620 }
621 _guard = &mut on_graceful_shutdown => {
622 debug!(target: "reth::cli", "shutdown signal received, terminating engine");
626 let (done_tx, done_rx) = oneshot::channel();
627 orchestrator.handler_mut().handler_mut().on_event(
628 FromOrchestrator::Terminate { tx: done_tx }.into()
629 );
630 let _ = done_rx.await;
631 break;
632 }
633 }
634 }
635
636 let _ = exit.send(res);
637 };
638 ctx.task_executor()
639 .spawn_critical_with_graceful_shutdown_signal("consensus engine", consensus_engine);
640
641 let engine_events_for_ethstats = engine_events.new_listener();
642
643 let full_node = FullNode {
644 evm_config: ctx.components().evm_config().clone(),
645 pool: ctx.components().pool().clone(),
646 network: ctx.components().network().clone(),
647 provider: ctx.node_adapter().provider.clone(),
648 payload_builder_handle: ctx.components().payload_builder_handle().clone(),
649 task_executor: ctx.task_executor().clone(),
650 config: ctx.node_config().clone(),
651 data_dir: ctx.data_dir().clone(),
652 add_ons_handle: RpcHandle {
653 rpc_server_handles,
654 rpc_registry,
655 engine_events,
656 beacon_engine_handle,
657 engine_shutdown,
658 },
659 };
660 on_node_started.on_event(FullNode::clone(&full_node))?;
661
662 ctx.spawn_ethstats(engine_events_for_ethstats).await?;
663
664 let handle = NodeHandle {
665 node_exit_future: NodeExitFuture::new(async { rx.await? }),
666 node: full_node,
667 };
668
669 Ok(handle)
670 }
671}
672
673impl<T, CB, AO> LaunchNode<NodeBuilderWithComponents<T, CB, AO>> for ArbEngineLauncher
674where
675 T: FullNodeTypes<
676 Types: NodeTypesForProvider<Payload = ArbEngineTypes, Primitives = ArbPrimitives>,
677 Provider = BlockchainProvider<
678 NodeTypesWithDBAdapter<<T as FullNodeTypes>::Types, <T as FullNodeTypes>::DB>,
679 >,
680 >,
681 CB: NodeComponentsBuilder<T> + 'static,
682 AO: RethRpcAddOns<NodeAdapter<T, CB::Components>>
683 + EngineValidatorAddOn<NodeAdapter<T, CB::Components>>
684 + 'static,
685{
686 type Node = NodeHandle<NodeAdapter<T, CB::Components>, AO>;
687 type Future = Pin<Box<dyn Future<Output = eyre::Result<Self::Node>> + Send>>;
688
689 fn launch_node(self, target: NodeBuilderWithComponents<T, CB, AO>) -> Self::Future {
690 Box::pin(self.launch_node(target))
691 }
692}