openvm_circuit/system/cuda/merkle_tree/
mod.rs

1use std::{ffi::c_void, sync::Arc};
2
3use openvm_circuit::{
4    arch::{MemoryConfig, ADDR_SPACE_OFFSET, DEFAULT_BLOCK_SIZE},
5    system::memory::{merkle::MemoryMerkleCols, TimestampedEquipartition},
6    utils::next_power_of_two_or_zero,
7};
8use openvm_cuda_backend::{base::DeviceMatrix, prelude::F, GpuBackend};
9use openvm_cuda_common::{
10    copy::{cuda_memcpy_on, MemCopyD2H, MemCopyH2D},
11    d_buffer::DeviceBuffer,
12    stream::{CudaEvent, GpuDeviceCtx},
13};
14use openvm_stark_backend::{
15    p3_maybe_rayon::prelude::{IntoParallelIterator, ParallelIterator},
16    p3_util::log2_ceil_usize,
17    prover::AirProvingContext,
18};
19use p3_field::PrimeCharacteristicRing;
20
21use super::{poseidon2::SharedBuffer, Poseidon2PeripheryChipGPU, DIGEST_WIDTH};
22
23pub mod cuda;
24use cuda::merkle_tree::*;
25
26type H = [F; DIGEST_WIDTH];
27/// Width of `((u32, u32), TimestampedValues<F, DEFAULT_BLOCK_SIZE>)` in u32 units.
28/// = 2 (key) + 1 (timestamp) + DEFAULT_BLOCK_SIZE (values)
29pub const TIMESTAMPED_BLOCK_WIDTH: usize = 3 + DEFAULT_BLOCK_SIZE;
30/// Width of `((u32, u32), TimestampedValues<F, DIGEST_WIDTH>)` in u32 units.
31/// = 2 (key) + 1 (timestamp) + DIGEST_WIDTH (values)
32pub const MERKLE_TOUCHED_BLOCK_WIDTH: usize = 3 + DIGEST_WIDTH;
33
34/// A Merkle subtree stored in a single flat buffer, combining a vertical path and a heap-ordered
35/// binary tree.
36///
37/// Memory layout:
38/// - The first `path_len` elements form a vertical path (one node per level), used when the actual
39///   size is smaller than the max size.
40/// - The remaining elements store the subtree nodes in heap-order (breadth-first), with `size`
41///   leaves and `2 * size - 1` total nodes.
42///
43/// All GPU work is issued on the subtree's `GpuDeviceCtx` stream.
44/// `build_completion_event` records when the build kernels finish so that downstream consumers can
45/// synchronize.
46pub struct MemoryMerkleSubTree {
47    build_completion_event: Option<CudaEvent>,
48    pub buf: DeviceBuffer<H>,
49    pub height: usize,
50    pub path_len: usize,
51}
52
53impl MemoryMerkleSubTree {
54    /// Constructs a new Merkle subtree with a vertical path and heap-ordered tree.
55    /// The buffer is sized based on the actual address space and the maximum size.
56    ///
57    /// `addr_space_size` is the number of leaf digest nodes necessary for this address space. The
58    /// `max_size` is the number of leaf digest nodes in the full balanced tree dictated by
59    /// `addr_space_height` from the `MemoryConfig`.
60    ///
61    /// `addr_space_size` must be a power of two or zero.
62    /// `max_size` must be a power of two.
63    pub fn new(addr_space_size: usize, max_size: usize, device_ctx: &GpuDeviceCtx) -> Self {
64        assert!(
65            addr_space_size == 0 || addr_space_size.is_power_of_two(),
66            "The actual address space size must be a power of two"
67        );
68        assert!(
69            max_size.is_power_of_two(),
70            "Max address space size must be a power of two"
71        );
72        if addr_space_size == 0 {
73            let mut res = MemoryMerkleSubTree::dummy();
74            res.height = log2_ceil_usize(max_size);
75            return res;
76        }
77        let height = log2_ceil_usize(addr_space_size);
78        let path_len = log2_ceil_usize(max_size).checked_sub(height).unwrap();
79        tracing::debug!(
80            "Creating a subtree buffer, size is {} (addr space size is {})",
81            path_len + (2 * addr_space_size - 1),
82            addr_space_size
83        );
84        let buf =
85            DeviceBuffer::<H>::with_capacity_on(path_len + (2 * addr_space_size - 1), device_ctx);
86
87        Self {
88            build_completion_event: None,
89            height,
90            buf,
91            path_len,
92        }
93    }
94
95    pub fn dummy() -> Self {
96        Self {
97            build_completion_event: None,
98            height: 0,
99            buf: DeviceBuffer::new(),
100            path_len: 0,
101        }
102    }
103
104    /// Builds the Merkle subtree on the provided `GpuDeviceCtx` stream.
105    /// Also reconstructs the vertical path if `path_len > 0`, and records a completion event.
106    ///
107    /// Here `addr_space_idx` is the address space _shifted_ by ADDR_SPACE_OFFSET = 1
108    pub fn build_async(
109        &mut self,
110        d_data: &DeviceBuffer<u8>,
111        addr_space_idx: usize,
112        zero_hash: &DeviceBuffer<H>,
113        device_ctx: &GpuDeviceCtx,
114    ) {
115        let event = CudaEvent::new().unwrap();
116        if self.buf.is_empty() {
117            self.buf = DeviceBuffer::with_capacity_on(1, device_ctx);
118            unsafe {
119                cuda_memcpy_on::<true, true>(
120                    self.buf.as_mut_raw_ptr(),
121                    zero_hash.as_ptr().add(self.height) as *mut c_void,
122                    size_of::<H>(),
123                    device_ctx,
124                )
125                .unwrap();
126                event.record(device_ctx.stream.as_raw()).unwrap();
127            }
128        } else {
129            unsafe {
130                build_merkle_subtree(
131                    d_data,
132                    1 << self.height,
133                    &self.buf,
134                    self.path_len,
135                    addr_space_idx as u32,
136                    device_ctx.stream.as_raw(),
137                )
138                .unwrap();
139
140                if self.path_len > 0 {
141                    restore_merkle_subtree_path(
142                        &self.buf,
143                        zero_hash,
144                        self.path_len,
145                        self.height + self.path_len,
146                        device_ctx.stream.as_raw(),
147                    )
148                    .unwrap();
149                }
150                event.record(device_ctx.stream.as_raw()).unwrap();
151            }
152        }
153        self.build_completion_event = Some(event);
154    }
155
156    /// Returns the bounds [start, end) of the layer at the given depth.
157    /// These bounds correspond to the indices of the layer in the buffer.
158    /// depth: 0 = root, 1 = root's children, ..., height-1 = leaves
159    pub fn layer_bounds(&self, depth: usize) -> (usize, usize) {
160        let global_height = self.height + self.path_len;
161        assert!(
162            depth < global_height,
163            "Depth {depth} out of bounds for height {global_height}",
164        );
165        if depth >= self.path_len {
166            // depth is within the heap-ordered subtree
167            let d = depth - self.path_len;
168            let start = self.path_len + ((1 << d) - 1);
169            let end = self.path_len + ((1 << (d + 1)) - 1);
170            (start, end)
171        } else {
172            // vertical path layer: single node per level
173            (depth, depth + 1)
174        }
175    }
176}
177
178/// A Memory Merkle tree composed of independent subtrees (one per address space),
179/// each built asynchronously and finalized into a top-level Merkle root.
180///
181/// Layout:
182/// - The memory is split across multiple `MemoryMerkleSubTree` instances, one per address space.
183/// - The top-level tree is formed by hashing all subtree roots into a single buffer (`top_roots`).
184///     - top_roots layout: \[root, hash(root_addr_space_1, root_addr_space_2),
185///       hash(root_addr_space_3), hash(root_addr_space_4), ...\]
186///     - if we have > 4 address spaces, top_roots will be extended with the next hash, etc.
187///
188/// Execution:
189/// - Subtrees are built on the tree's `GpuDeviceCtx` stream.
190/// - The final root is computed after all subtrees complete on that same stream.
191pub struct MemoryMerkleTree {
192    pub device_ctx: GpuDeviceCtx,
193    pub subtrees: Vec<MemoryMerkleSubTree>,
194    pub top_roots: DeviceBuffer<H>,
195    zero_hash: DeviceBuffer<H>,
196    pub height: usize,
197    pub hasher_buffer: SharedBuffer<F>,
198    mem_config: MemoryConfig,
199    pub(crate) top_roots_host: Vec<H>,
200}
201
202impl MemoryMerkleTree {
203    /// Creates a full Merkle tree with one subtree per address space.
204    /// Initializes all buffers and precomputes the zero hash chain.
205    pub fn new(
206        mem_config: MemoryConfig,
207        hasher_chip: Arc<Poseidon2PeripheryChipGPU>,
208        device_ctx: GpuDeviceCtx,
209    ) -> Self {
210        let addr_space_sizes = mem_config
211            .addr_spaces
212            .iter()
213            .map(|ashc| {
214                assert!(
215                    ashc.num_cells % DIGEST_WIDTH == 0,
216                    "the number of cells must be divisible by `DIGEST_WIDTH`"
217                );
218                ashc.num_cells / DIGEST_WIDTH
219            })
220            .collect::<Vec<_>>();
221        assert!(!(addr_space_sizes.is_empty()), "Invalid config");
222
223        let num_addr_spaces = addr_space_sizes.len() - ADDR_SPACE_OFFSET as usize;
224        assert!(
225            num_addr_spaces.is_power_of_two(),
226            "Number of address spaces must be a one plus power of two"
227        );
228        for &sz in addr_space_sizes.iter().take(ADDR_SPACE_OFFSET as usize) {
229            assert!(
230                sz == 0,
231                "The first `ADDR_SPACE_OFFSET` address spaces are assumed to be empty"
232            );
233        }
234
235        let label_max_bits = mem_config.pointer_max_bits - log2_ceil_usize(DIGEST_WIDTH);
236
237        let zero_hash = DeviceBuffer::<H>::with_capacity_on(label_max_bits + 1, &device_ctx);
238        let top_roots = DeviceBuffer::<H>::with_capacity_on(2 * num_addr_spaces - 1, &device_ctx);
239        unsafe {
240            calculate_zero_hash(&zero_hash, label_max_bits, device_ctx.stream.as_raw()).unwrap();
241        }
242
243        Self {
244            device_ctx,
245            subtrees: Vec::new(),
246            top_roots,
247            height: label_max_bits + log2_ceil_usize(num_addr_spaces),
248            zero_hash,
249            hasher_buffer: hasher_chip.shared_buffer(),
250            mem_config,
251            top_roots_host: vec![],
252        }
253    }
254
255    pub fn mem_config(&self) -> &MemoryConfig {
256        &self.mem_config
257    }
258
259    /// Starts construction of the specified address space's Merkle subtree.
260    /// Uses internal zero hashes and launches kernels on the tree's `GpuDeviceCtx` stream.
261    ///
262    /// Here `addr_space` is the _unshifted_ address space, so `addr_space = 0` is the immediate
263    /// address space, which should be ignored.
264    ///
265    /// **Note:** the caller MUST ENSURE that `d_data` lives long enough to be there
266    /// when the enqueued task actually starts.
267    pub fn build_async(&mut self, d_data: &DeviceBuffer<u8>, addr_space: usize) {
268        if addr_space < ADDR_SPACE_OFFSET as usize {
269            return;
270        }
271        let addr_space_idx = addr_space - ADDR_SPACE_OFFSET as usize;
272        if addr_space < self.mem_config.addr_spaces.len() && addr_space_idx == self.subtrees.len() {
273            let mut subtree = MemoryMerkleSubTree::new(
274                self.mem_config.addr_spaces[addr_space].num_cells / DIGEST_WIDTH,
275                1 << (self.zero_hash.len() - 1), /* label_max_bits */
276                &self.device_ctx,
277            );
278            subtree.build_async(d_data, addr_space_idx, &self.zero_hash, &self.device_ctx);
279            self.subtrees.push(subtree);
280        } else {
281            panic!("Invalid address space index");
282        }
283    }
284
285    /// Finalizes the Merkle tree by collecting all subtree roots and computing the final root.
286    /// All subtree builds were issued on the same `GpuDeviceCtx` stream, so stream ordering
287    /// guarantees they are complete before the finalize kernel runs.
288    pub fn finalize(&mut self) {
289        let roots: Vec<usize> = self
290            .subtrees
291            .iter()
292            .map(|subtree| subtree.buf.as_ptr() as usize)
293            .collect();
294        let d_roots = roots.to_device_on(&self.device_ctx).unwrap();
295
296        unsafe {
297            finalize_merkle_tree(
298                &d_roots,
299                &self.top_roots,
300                self.subtrees.len(),
301                self.device_ctx.stream.as_raw(),
302            )
303            .unwrap();
304        }
305    }
306
307    /// Drops all massive buffers to free memory. Used at the end of an execution segment.
308    ///
309    /// Synchronizes the tree's `GpuDeviceCtx` stream before deallocating buffers and destroying
310    /// events.
311    pub fn drop_subtrees(&mut self) {
312        self.device_ctx.stream.synchronize().unwrap();
313        self.subtrees.clear();
314    }
315
316    /// Updates the tree and returns the merkle trace.
317    pub fn update_with_touched_blocks(
318        &mut self,
319        unpadded_height: usize,
320        d_touched_blocks: &DeviceBuffer<u32>, // consists of (as, ptr, ts, [F; DIGEST_WIDTH])
321        empty_touched_blocks: bool,
322    ) -> AirProvingContext<GpuBackend> {
323        let mut public_values = self.top_roots.to_host_on(&self.device_ctx).unwrap()[0].to_vec();
324        // .to_host() calls cudaEventSynchronize on the D2H memcpy, which also means all subtree
325        // events are now completed, so we can clean up the events.
326        for subtree in &mut self.subtrees {
327            subtree.build_completion_event = None;
328        }
329        let merkle_trace = {
330            let width = MemoryMerkleCols::<u8, DIGEST_WIDTH>::width();
331            let padded_height = next_power_of_two_or_zero(unpadded_height);
332            let output =
333                DeviceMatrix::<F>::with_capacity_on(padded_height, width, &self.device_ctx);
334            output.buffer().fill_zero_on(&self.device_ctx).unwrap();
335
336            let actual_heights = self.subtrees.iter().map(|s| s.height).collect::<Vec<_>>();
337            let subtrees_pointers = self
338                .subtrees
339                .iter()
340                .map(|st| st.buf.as_ptr() as usize)
341                .collect::<Vec<_>>()
342                .to_device_on(&self.device_ctx)
343                .unwrap();
344            unsafe {
345                update_merkle_tree(
346                    &output,
347                    &subtrees_pointers,
348                    &self.top_roots,
349                    &self.zero_hash,
350                    d_touched_blocks,
351                    self.height - log2_ceil_usize(self.subtrees.len()),
352                    &actual_heights,
353                    unpadded_height,
354                    &self.hasher_buffer,
355                    &self.device_ctx,
356                )
357                .unwrap();
358            }
359
360            if empty_touched_blocks {
361                // The trace is small then
362                let mut output_vec = output.buffer().to_host_on(&self.device_ctx).unwrap();
363                output_vec[unpadded_height - 1 + (width - 2) * padded_height] = F::ONE; // left_direction_different
364                output_vec[unpadded_height - 1 + (width - 1) * padded_height] = F::ONE; // right_direction_different
365                DeviceMatrix::new(
366                    Arc::new(output_vec.to_device_on(&self.device_ctx).unwrap()),
367                    padded_height,
368                    width,
369                )
370            } else {
371                output
372            }
373        };
374        self.top_roots_host = self.top_roots.to_host_on(&self.device_ctx).unwrap();
375        public_values.extend(self.top_roots_host[0]);
376
377        AirProvingContext::new(Vec::new(), merkle_trace, public_values)
378    }
379
380    /// An auxiliary function to calculate the required number of rows for the merkle trace.
381    /// Generic over BLOCK_SIZE since only addresses are used, not values.
382    pub fn calculate_unpadded_height<const BLOCK_SIZE: usize>(
383        &self,
384        touched_memory: &TimestampedEquipartition<F, BLOCK_SIZE>,
385    ) -> usize {
386        let md = self.mem_config.memory_dimensions();
387        let tree_height = md.overall_height();
388        let shift_address = |(sp, ptr): (u32, u32)| (sp, ptr / DIGEST_WIDTH as u32);
389        2 * if touched_memory.is_empty() {
390            tree_height
391        } else {
392            tree_height
393                + (0..(touched_memory.len() - 1))
394                    .into_par_iter()
395                    .map(|i| {
396                        let x = md.label_to_index(shift_address(touched_memory[i].0));
397                        let y = md.label_to_index(shift_address(touched_memory[i + 1].0));
398                        let xor = x ^ y;
399                        if xor == 0 {
400                            0
401                        } else {
402                            xor.ilog2() as usize
403                        }
404                    })
405                    .sum::<usize>()
406        }
407    }
408}
409
410impl Drop for MemoryMerkleTree {
411    fn drop(&mut self) {
412        self.drop_subtrees();
413    }
414}
415
416#[cfg(test)]
417mod tests {
418    use std::sync::Arc;
419
420    use openvm_circuit::{
421        arch::{vm_poseidon2_config, AddressSpaceHostLayout, MemoryCellType, MemoryConfig},
422        system::{
423            cuda::merkle_tree::MERKLE_TOUCHED_BLOCK_WIDTH,
424            memory::{
425                merkle::MerkleTree,
426                online::{GuestMemory, LinearMemory},
427                AddressMap, TimestampedValues,
428            },
429            poseidon2::Poseidon2PeripheryChip,
430        },
431    };
432    use openvm_cuda_backend::prelude::F;
433    use openvm_cuda_common::{
434        common::get_device,
435        copy::{MemCopyD2H, MemCopyH2D},
436        d_buffer::DeviceBuffer,
437        stream::{CudaStream, GpuDeviceCtx, StreamGuard},
438    };
439    use openvm_instructions::{
440        riscv::{RV32_MEMORY_AS, RV32_REGISTER_AS},
441        DEFERRAL_AS,
442    };
443    use openvm_stark_sdk::utils::create_seeded_rng;
444    use p3_field::{PrimeCharacteristicRing, PrimeField32};
445    use rand::Rng;
446
447    use super::MemoryMerkleTree;
448    use crate::system::cuda::{Poseidon2PeripheryChipGPU, DIGEST_WIDTH};
449
450    #[test]
451    fn test_cuda_merkle_tree_cpu_gpu_root_equivalence() {
452        let mut rng = create_seeded_rng();
453        let mem_config = {
454            let mut addr_spaces = MemoryConfig::empty_address_space_configs(5);
455            let max_cells = 1 << 16;
456            addr_spaces[RV32_REGISTER_AS as usize].num_cells = 32 * size_of::<u32>();
457            addr_spaces[RV32_MEMORY_AS as usize].num_cells = max_cells;
458            addr_spaces[DEFERRAL_AS as usize].num_cells = max_cells;
459            MemoryConfig::new(2, addr_spaces, max_cells.ilog2() as usize, 29, 17)
460        };
461
462        let mut initial_memory = GuestMemory::new(AddressMap::from_mem_config(&mem_config));
463        for (idx, space) in mem_config.addr_spaces.iter().enumerate() {
464            unsafe {
465                match space.layout {
466                    MemoryCellType::Null => {}
467                    MemoryCellType::U8 => {
468                        for i in 0..space.num_cells {
469                            initial_memory.write::<u8, 1>(
470                                idx as u32,
471                                i as u32,
472                                [rng.random_range(0..space.layout.size()) as u8],
473                            );
474                        }
475                    }
476                    MemoryCellType::U16 => {
477                        for i in 0..space.num_cells {
478                            initial_memory.write::<u16, 1>(
479                                idx as u32,
480                                i as u32,
481                                [rng.random_range(0..space.layout.size()) as u16],
482                            );
483                        }
484                    }
485                    MemoryCellType::U32 => {
486                        for i in 0..space.num_cells {
487                            initial_memory.write::<u32, 1>(
488                                idx as u32,
489                                i as u32,
490                                [rng.random_range(0..space.layout.size()) as u32],
491                            );
492                        }
493                    }
494                    MemoryCellType::F { .. } => {
495                        for i in 0..space.num_cells {
496                            initial_memory.write::<F, 1>(
497                                idx as u32,
498                                i as u32,
499                                [F::from_u32(rng.random_range(0..F::ORDER_U32))],
500                            );
501                        }
502                    }
503                }
504            }
505        }
506
507        let device_ctx = GpuDeviceCtx {
508            device_id: get_device().unwrap() as u32,
509            stream: StreamGuard::new(CudaStream::new_non_blocking().unwrap()),
510        };
511        let gpu_hasher_chip = Arc::new(Poseidon2PeripheryChipGPU::new(
512            1, // sbox_regs
513            device_ctx.clone(),
514        ));
515        let mut gpu_merkle_tree = MemoryMerkleTree::new(
516            mem_config.clone(),
517            gpu_hasher_chip.clone(),
518            device_ctx.clone(),
519        );
520        let mem_slices = initial_memory
521            .memory
522            .get_memory()
523            .iter()
524            .map(|mem| {
525                let mem_slice = mem.as_slice();
526                if !mem_slice.is_empty() {
527                    mem_slice.to_device_on(&gpu_merkle_tree.device_ctx).unwrap()
528                } else {
529                    DeviceBuffer::new()
530                }
531            })
532            .collect::<Vec<_>>();
533        for (i, mem_slice) in mem_slices.iter().enumerate() {
534            gpu_merkle_tree.build_async(mem_slice, i);
535        }
536        gpu_merkle_tree.finalize();
537
538        let cpu_hasher_chip = Poseidon2PeripheryChip::new(vm_poseidon2_config(), 3);
539        let mut cpu_merkle_tree = MerkleTree::<F, DIGEST_WIDTH>::from_memory(
540            &initial_memory.memory,
541            &mem_config.memory_dimensions(),
542            &cpu_hasher_chip,
543        );
544
545        assert_eq!(
546            cpu_merkle_tree.root(),
547            gpu_merkle_tree
548                .top_roots
549                .to_host_on(&gpu_merkle_tree.device_ctx)
550                .unwrap()[0]
551        );
552        eprintln!("{:?}", cpu_merkle_tree.root());
553        eprintln!(
554            "{:?}",
555            gpu_merkle_tree
556                .top_roots
557                .to_host_on(&gpu_merkle_tree.device_ctx)
558                .unwrap()[0]
559        );
560
561        // Now we add some touched memory
562        // We don't care about the memory layout and whatnot, because neither implementation uses
563        // any special form of the touched blocks
564        let touched_ptrs = mem_config
565            .addr_spaces
566            .iter()
567            .enumerate()
568            .flat_map(|(i, cnf)| {
569                let mut ptrs = Vec::new();
570                for j in 0..(cnf.num_cells / DIGEST_WIDTH) {
571                    if rng.random_bool(0.333) {
572                        ptrs.push((i as u32, (j * DIGEST_WIDTH) as u32));
573                    }
574                }
575                ptrs
576            })
577            .collect::<Vec<_>>();
578        let new_data = touched_ptrs
579            .iter()
580            .map(|_| std::array::from_fn(|_| F::from_u32(rng.random_range(0..F::ORDER_U32))))
581            .collect::<Vec<[F; DIGEST_WIDTH]>>();
582        assert!(!touched_ptrs.is_empty());
583        cpu_merkle_tree.finalize(
584            &cpu_hasher_chip,
585            &(touched_ptrs
586                .iter()
587                .copied()
588                .zip(new_data.iter().copied())
589                .collect()),
590            &mem_config.memory_dimensions(),
591        );
592        let touched_blocks = touched_ptrs
593            .into_iter()
594            .zip(new_data)
595            .map(|(address, data)| {
596                (
597                    address,
598                    TimestampedValues {
599                        timestamp: rng.random_range(0..(1u32 << mem_config.timestamp_max_bits)),
600                        values: data,
601                    },
602                )
603            })
604            .collect::<Vec<_>>();
605        let mut merkle_records =
606            Vec::<u32>::with_capacity(touched_blocks.len() * MERKLE_TOUCHED_BLOCK_WIDTH);
607        for (address, ts_values) in &touched_blocks {
608            let (address_space, ptr) = *address;
609            merkle_records.push(address_space);
610            merkle_records.push(ptr);
611            merkle_records.push(ts_values.timestamp);
612            for &v in &ts_values.values {
613                merkle_records.push(unsafe { std::mem::transmute::<F, u32>(v) });
614            }
615        }
616        let d_touched_blocks = merkle_records
617            .to_device_on(&gpu_merkle_tree.device_ctx)
618            .unwrap();
619
620        let unpadded_height = gpu_merkle_tree.calculate_unpadded_height(&touched_blocks);
621        gpu_hasher_chip.prepare_records(unpadded_height);
622        gpu_merkle_tree.update_with_touched_blocks(unpadded_height, &d_touched_blocks, false);
623
624        assert_eq!(
625            cpu_merkle_tree.root(),
626            gpu_merkle_tree
627                .top_roots
628                .to_host_on(&gpu_merkle_tree.device_ctx)
629                .unwrap()[0]
630        );
631        eprintln!("{:?}", cpu_merkle_tree.root());
632        eprintln!(
633            "{:?}",
634            gpu_merkle_tree
635                .top_roots
636                .to_host_on(&gpu_merkle_tree.device_ctx)
637                .unwrap()[0]
638        );
639    }
640}