1use std::{ffi::c_void, sync::Arc};
2
3use openvm_circuit::{
4 arch::{MemoryConfig, ADDR_SPACE_OFFSET, DEFAULT_BLOCK_SIZE},
5 system::memory::{merkle::MemoryMerkleCols, TimestampedEquipartition},
6 utils::next_power_of_two_or_zero,
7};
8use openvm_cuda_backend::{base::DeviceMatrix, prelude::F, GpuBackend};
9use openvm_cuda_common::{
10 copy::{cuda_memcpy_on, MemCopyD2H, MemCopyH2D},
11 d_buffer::DeviceBuffer,
12 stream::{CudaEvent, GpuDeviceCtx},
13};
14use openvm_stark_backend::{
15 p3_maybe_rayon::prelude::{IntoParallelIterator, ParallelIterator},
16 p3_util::log2_ceil_usize,
17 prover::AirProvingContext,
18};
19use p3_field::PrimeCharacteristicRing;
20
21use super::{poseidon2::SharedBuffer, Poseidon2PeripheryChipGPU, DIGEST_WIDTH};
22
23pub mod cuda;
24use cuda::merkle_tree::*;
25
26type H = [F; DIGEST_WIDTH];
27pub const TIMESTAMPED_BLOCK_WIDTH: usize = 3 + DEFAULT_BLOCK_SIZE;
30pub const MERKLE_TOUCHED_BLOCK_WIDTH: usize = 3 + DIGEST_WIDTH;
33
34pub struct MemoryMerkleSubTree {
47 build_completion_event: Option<CudaEvent>,
48 pub buf: DeviceBuffer<H>,
49 pub height: usize,
50 pub path_len: usize,
51}
52
53impl MemoryMerkleSubTree {
54 pub fn new(addr_space_size: usize, max_size: usize, device_ctx: &GpuDeviceCtx) -> Self {
64 assert!(
65 addr_space_size == 0 || addr_space_size.is_power_of_two(),
66 "The actual address space size must be a power of two"
67 );
68 assert!(
69 max_size.is_power_of_two(),
70 "Max address space size must be a power of two"
71 );
72 if addr_space_size == 0 {
73 let mut res = MemoryMerkleSubTree::dummy();
74 res.height = log2_ceil_usize(max_size);
75 return res;
76 }
77 let height = log2_ceil_usize(addr_space_size);
78 let path_len = log2_ceil_usize(max_size).checked_sub(height).unwrap();
79 tracing::debug!(
80 "Creating a subtree buffer, size is {} (addr space size is {})",
81 path_len + (2 * addr_space_size - 1),
82 addr_space_size
83 );
84 let buf =
85 DeviceBuffer::<H>::with_capacity_on(path_len + (2 * addr_space_size - 1), device_ctx);
86
87 Self {
88 build_completion_event: None,
89 height,
90 buf,
91 path_len,
92 }
93 }
94
95 pub fn dummy() -> Self {
96 Self {
97 build_completion_event: None,
98 height: 0,
99 buf: DeviceBuffer::new(),
100 path_len: 0,
101 }
102 }
103
104 pub fn build_async(
109 &mut self,
110 d_data: &DeviceBuffer<u8>,
111 addr_space_idx: usize,
112 zero_hash: &DeviceBuffer<H>,
113 device_ctx: &GpuDeviceCtx,
114 ) {
115 let event = CudaEvent::new().unwrap();
116 if self.buf.is_empty() {
117 self.buf = DeviceBuffer::with_capacity_on(1, device_ctx);
118 unsafe {
119 cuda_memcpy_on::<true, true>(
120 self.buf.as_mut_raw_ptr(),
121 zero_hash.as_ptr().add(self.height) as *mut c_void,
122 size_of::<H>(),
123 device_ctx,
124 )
125 .unwrap();
126 event.record(device_ctx.stream.as_raw()).unwrap();
127 }
128 } else {
129 unsafe {
130 build_merkle_subtree(
131 d_data,
132 1 << self.height,
133 &self.buf,
134 self.path_len,
135 addr_space_idx as u32,
136 device_ctx.stream.as_raw(),
137 )
138 .unwrap();
139
140 if self.path_len > 0 {
141 restore_merkle_subtree_path(
142 &self.buf,
143 zero_hash,
144 self.path_len,
145 self.height + self.path_len,
146 device_ctx.stream.as_raw(),
147 )
148 .unwrap();
149 }
150 event.record(device_ctx.stream.as_raw()).unwrap();
151 }
152 }
153 self.build_completion_event = Some(event);
154 }
155
156 pub fn layer_bounds(&self, depth: usize) -> (usize, usize) {
160 let global_height = self.height + self.path_len;
161 assert!(
162 depth < global_height,
163 "Depth {depth} out of bounds for height {global_height}",
164 );
165 if depth >= self.path_len {
166 let d = depth - self.path_len;
168 let start = self.path_len + ((1 << d) - 1);
169 let end = self.path_len + ((1 << (d + 1)) - 1);
170 (start, end)
171 } else {
172 (depth, depth + 1)
174 }
175 }
176}
177
178pub struct MemoryMerkleTree {
192 pub device_ctx: GpuDeviceCtx,
193 pub subtrees: Vec<MemoryMerkleSubTree>,
194 pub top_roots: DeviceBuffer<H>,
195 zero_hash: DeviceBuffer<H>,
196 pub height: usize,
197 pub hasher_buffer: SharedBuffer<F>,
198 mem_config: MemoryConfig,
199 pub(crate) top_roots_host: Vec<H>,
200}
201
202impl MemoryMerkleTree {
203 pub fn new(
206 mem_config: MemoryConfig,
207 hasher_chip: Arc<Poseidon2PeripheryChipGPU>,
208 device_ctx: GpuDeviceCtx,
209 ) -> Self {
210 let addr_space_sizes = mem_config
211 .addr_spaces
212 .iter()
213 .map(|ashc| {
214 assert!(
215 ashc.num_cells % DIGEST_WIDTH == 0,
216 "the number of cells must be divisible by `DIGEST_WIDTH`"
217 );
218 ashc.num_cells / DIGEST_WIDTH
219 })
220 .collect::<Vec<_>>();
221 assert!(!(addr_space_sizes.is_empty()), "Invalid config");
222
223 let num_addr_spaces = addr_space_sizes.len() - ADDR_SPACE_OFFSET as usize;
224 assert!(
225 num_addr_spaces.is_power_of_two(),
226 "Number of address spaces must be a one plus power of two"
227 );
228 for &sz in addr_space_sizes.iter().take(ADDR_SPACE_OFFSET as usize) {
229 assert!(
230 sz == 0,
231 "The first `ADDR_SPACE_OFFSET` address spaces are assumed to be empty"
232 );
233 }
234
235 let label_max_bits = mem_config.pointer_max_bits - log2_ceil_usize(DIGEST_WIDTH);
236
237 let zero_hash = DeviceBuffer::<H>::with_capacity_on(label_max_bits + 1, &device_ctx);
238 let top_roots = DeviceBuffer::<H>::with_capacity_on(2 * num_addr_spaces - 1, &device_ctx);
239 unsafe {
240 calculate_zero_hash(&zero_hash, label_max_bits, device_ctx.stream.as_raw()).unwrap();
241 }
242
243 Self {
244 device_ctx,
245 subtrees: Vec::new(),
246 top_roots,
247 height: label_max_bits + log2_ceil_usize(num_addr_spaces),
248 zero_hash,
249 hasher_buffer: hasher_chip.shared_buffer(),
250 mem_config,
251 top_roots_host: vec![],
252 }
253 }
254
255 pub fn mem_config(&self) -> &MemoryConfig {
256 &self.mem_config
257 }
258
259 pub fn build_async(&mut self, d_data: &DeviceBuffer<u8>, addr_space: usize) {
268 if addr_space < ADDR_SPACE_OFFSET as usize {
269 return;
270 }
271 let addr_space_idx = addr_space - ADDR_SPACE_OFFSET as usize;
272 if addr_space < self.mem_config.addr_spaces.len() && addr_space_idx == self.subtrees.len() {
273 let mut subtree = MemoryMerkleSubTree::new(
274 self.mem_config.addr_spaces[addr_space].num_cells / DIGEST_WIDTH,
275 1 << (self.zero_hash.len() - 1), &self.device_ctx,
277 );
278 subtree.build_async(d_data, addr_space_idx, &self.zero_hash, &self.device_ctx);
279 self.subtrees.push(subtree);
280 } else {
281 panic!("Invalid address space index");
282 }
283 }
284
285 pub fn finalize(&mut self) {
289 let roots: Vec<usize> = self
290 .subtrees
291 .iter()
292 .map(|subtree| subtree.buf.as_ptr() as usize)
293 .collect();
294 let d_roots = roots.to_device_on(&self.device_ctx).unwrap();
295
296 unsafe {
297 finalize_merkle_tree(
298 &d_roots,
299 &self.top_roots,
300 self.subtrees.len(),
301 self.device_ctx.stream.as_raw(),
302 )
303 .unwrap();
304 }
305 }
306
307 pub fn drop_subtrees(&mut self) {
312 self.device_ctx.stream.synchronize().unwrap();
313 self.subtrees.clear();
314 }
315
316 pub fn update_with_touched_blocks(
318 &mut self,
319 unpadded_height: usize,
320 d_touched_blocks: &DeviceBuffer<u32>, empty_touched_blocks: bool,
322 ) -> AirProvingContext<GpuBackend> {
323 let mut public_values = self.top_roots.to_host_on(&self.device_ctx).unwrap()[0].to_vec();
324 for subtree in &mut self.subtrees {
327 subtree.build_completion_event = None;
328 }
329 let merkle_trace = {
330 let width = MemoryMerkleCols::<u8, DIGEST_WIDTH>::width();
331 let padded_height = next_power_of_two_or_zero(unpadded_height);
332 let output =
333 DeviceMatrix::<F>::with_capacity_on(padded_height, width, &self.device_ctx);
334 output.buffer().fill_zero_on(&self.device_ctx).unwrap();
335
336 let actual_heights = self.subtrees.iter().map(|s| s.height).collect::<Vec<_>>();
337 let subtrees_pointers = self
338 .subtrees
339 .iter()
340 .map(|st| st.buf.as_ptr() as usize)
341 .collect::<Vec<_>>()
342 .to_device_on(&self.device_ctx)
343 .unwrap();
344 unsafe {
345 update_merkle_tree(
346 &output,
347 &subtrees_pointers,
348 &self.top_roots,
349 &self.zero_hash,
350 d_touched_blocks,
351 self.height - log2_ceil_usize(self.subtrees.len()),
352 &actual_heights,
353 unpadded_height,
354 &self.hasher_buffer,
355 &self.device_ctx,
356 )
357 .unwrap();
358 }
359
360 if empty_touched_blocks {
361 let mut output_vec = output.buffer().to_host_on(&self.device_ctx).unwrap();
363 output_vec[unpadded_height - 1 + (width - 2) * padded_height] = F::ONE; output_vec[unpadded_height - 1 + (width - 1) * padded_height] = F::ONE; DeviceMatrix::new(
366 Arc::new(output_vec.to_device_on(&self.device_ctx).unwrap()),
367 padded_height,
368 width,
369 )
370 } else {
371 output
372 }
373 };
374 self.top_roots_host = self.top_roots.to_host_on(&self.device_ctx).unwrap();
375 public_values.extend(self.top_roots_host[0]);
376
377 AirProvingContext::new(Vec::new(), merkle_trace, public_values)
378 }
379
380 pub fn calculate_unpadded_height<const BLOCK_SIZE: usize>(
383 &self,
384 touched_memory: &TimestampedEquipartition<F, BLOCK_SIZE>,
385 ) -> usize {
386 let md = self.mem_config.memory_dimensions();
387 let tree_height = md.overall_height();
388 let shift_address = |(sp, ptr): (u32, u32)| (sp, ptr / DIGEST_WIDTH as u32);
389 2 * if touched_memory.is_empty() {
390 tree_height
391 } else {
392 tree_height
393 + (0..(touched_memory.len() - 1))
394 .into_par_iter()
395 .map(|i| {
396 let x = md.label_to_index(shift_address(touched_memory[i].0));
397 let y = md.label_to_index(shift_address(touched_memory[i + 1].0));
398 let xor = x ^ y;
399 if xor == 0 {
400 0
401 } else {
402 xor.ilog2() as usize
403 }
404 })
405 .sum::<usize>()
406 }
407 }
408}
409
410impl Drop for MemoryMerkleTree {
411 fn drop(&mut self) {
412 self.drop_subtrees();
413 }
414}
415
416#[cfg(test)]
417mod tests {
418 use std::sync::Arc;
419
420 use openvm_circuit::{
421 arch::{vm_poseidon2_config, AddressSpaceHostLayout, MemoryCellType, MemoryConfig},
422 system::{
423 cuda::merkle_tree::MERKLE_TOUCHED_BLOCK_WIDTH,
424 memory::{
425 merkle::MerkleTree,
426 online::{GuestMemory, LinearMemory},
427 AddressMap, TimestampedValues,
428 },
429 poseidon2::Poseidon2PeripheryChip,
430 },
431 };
432 use openvm_cuda_backend::prelude::F;
433 use openvm_cuda_common::{
434 common::get_device,
435 copy::{MemCopyD2H, MemCopyH2D},
436 d_buffer::DeviceBuffer,
437 stream::{CudaStream, GpuDeviceCtx, StreamGuard},
438 };
439 use openvm_instructions::{
440 riscv::{RV32_MEMORY_AS, RV32_REGISTER_AS},
441 DEFERRAL_AS,
442 };
443 use openvm_stark_sdk::utils::create_seeded_rng;
444 use p3_field::{PrimeCharacteristicRing, PrimeField32};
445 use rand::Rng;
446
447 use super::MemoryMerkleTree;
448 use crate::system::cuda::{Poseidon2PeripheryChipGPU, DIGEST_WIDTH};
449
450 #[test]
451 fn test_cuda_merkle_tree_cpu_gpu_root_equivalence() {
452 let mut rng = create_seeded_rng();
453 let mem_config = {
454 let mut addr_spaces = MemoryConfig::empty_address_space_configs(5);
455 let max_cells = 1 << 16;
456 addr_spaces[RV32_REGISTER_AS as usize].num_cells = 32 * size_of::<u32>();
457 addr_spaces[RV32_MEMORY_AS as usize].num_cells = max_cells;
458 addr_spaces[DEFERRAL_AS as usize].num_cells = max_cells;
459 MemoryConfig::new(2, addr_spaces, max_cells.ilog2() as usize, 29, 17)
460 };
461
462 let mut initial_memory = GuestMemory::new(AddressMap::from_mem_config(&mem_config));
463 for (idx, space) in mem_config.addr_spaces.iter().enumerate() {
464 unsafe {
465 match space.layout {
466 MemoryCellType::Null => {}
467 MemoryCellType::U8 => {
468 for i in 0..space.num_cells {
469 initial_memory.write::<u8, 1>(
470 idx as u32,
471 i as u32,
472 [rng.random_range(0..space.layout.size()) as u8],
473 );
474 }
475 }
476 MemoryCellType::U16 => {
477 for i in 0..space.num_cells {
478 initial_memory.write::<u16, 1>(
479 idx as u32,
480 i as u32,
481 [rng.random_range(0..space.layout.size()) as u16],
482 );
483 }
484 }
485 MemoryCellType::U32 => {
486 for i in 0..space.num_cells {
487 initial_memory.write::<u32, 1>(
488 idx as u32,
489 i as u32,
490 [rng.random_range(0..space.layout.size()) as u32],
491 );
492 }
493 }
494 MemoryCellType::F { .. } => {
495 for i in 0..space.num_cells {
496 initial_memory.write::<F, 1>(
497 idx as u32,
498 i as u32,
499 [F::from_u32(rng.random_range(0..F::ORDER_U32))],
500 );
501 }
502 }
503 }
504 }
505 }
506
507 let device_ctx = GpuDeviceCtx {
508 device_id: get_device().unwrap() as u32,
509 stream: StreamGuard::new(CudaStream::new_non_blocking().unwrap()),
510 };
511 let gpu_hasher_chip = Arc::new(Poseidon2PeripheryChipGPU::new(
512 1, device_ctx.clone(),
514 ));
515 let mut gpu_merkle_tree = MemoryMerkleTree::new(
516 mem_config.clone(),
517 gpu_hasher_chip.clone(),
518 device_ctx.clone(),
519 );
520 let mem_slices = initial_memory
521 .memory
522 .get_memory()
523 .iter()
524 .map(|mem| {
525 let mem_slice = mem.as_slice();
526 if !mem_slice.is_empty() {
527 mem_slice.to_device_on(&gpu_merkle_tree.device_ctx).unwrap()
528 } else {
529 DeviceBuffer::new()
530 }
531 })
532 .collect::<Vec<_>>();
533 for (i, mem_slice) in mem_slices.iter().enumerate() {
534 gpu_merkle_tree.build_async(mem_slice, i);
535 }
536 gpu_merkle_tree.finalize();
537
538 let cpu_hasher_chip = Poseidon2PeripheryChip::new(vm_poseidon2_config(), 3);
539 let mut cpu_merkle_tree = MerkleTree::<F, DIGEST_WIDTH>::from_memory(
540 &initial_memory.memory,
541 &mem_config.memory_dimensions(),
542 &cpu_hasher_chip,
543 );
544
545 assert_eq!(
546 cpu_merkle_tree.root(),
547 gpu_merkle_tree
548 .top_roots
549 .to_host_on(&gpu_merkle_tree.device_ctx)
550 .unwrap()[0]
551 );
552 eprintln!("{:?}", cpu_merkle_tree.root());
553 eprintln!(
554 "{:?}",
555 gpu_merkle_tree
556 .top_roots
557 .to_host_on(&gpu_merkle_tree.device_ctx)
558 .unwrap()[0]
559 );
560
561 let touched_ptrs = mem_config
565 .addr_spaces
566 .iter()
567 .enumerate()
568 .flat_map(|(i, cnf)| {
569 let mut ptrs = Vec::new();
570 for j in 0..(cnf.num_cells / DIGEST_WIDTH) {
571 if rng.random_bool(0.333) {
572 ptrs.push((i as u32, (j * DIGEST_WIDTH) as u32));
573 }
574 }
575 ptrs
576 })
577 .collect::<Vec<_>>();
578 let new_data = touched_ptrs
579 .iter()
580 .map(|_| std::array::from_fn(|_| F::from_u32(rng.random_range(0..F::ORDER_U32))))
581 .collect::<Vec<[F; DIGEST_WIDTH]>>();
582 assert!(!touched_ptrs.is_empty());
583 cpu_merkle_tree.finalize(
584 &cpu_hasher_chip,
585 &(touched_ptrs
586 .iter()
587 .copied()
588 .zip(new_data.iter().copied())
589 .collect()),
590 &mem_config.memory_dimensions(),
591 );
592 let touched_blocks = touched_ptrs
593 .into_iter()
594 .zip(new_data)
595 .map(|(address, data)| {
596 (
597 address,
598 TimestampedValues {
599 timestamp: rng.random_range(0..(1u32 << mem_config.timestamp_max_bits)),
600 values: data,
601 },
602 )
603 })
604 .collect::<Vec<_>>();
605 let mut merkle_records =
606 Vec::<u32>::with_capacity(touched_blocks.len() * MERKLE_TOUCHED_BLOCK_WIDTH);
607 for (address, ts_values) in &touched_blocks {
608 let (address_space, ptr) = *address;
609 merkle_records.push(address_space);
610 merkle_records.push(ptr);
611 merkle_records.push(ts_values.timestamp);
612 for &v in &ts_values.values {
613 merkle_records.push(unsafe { std::mem::transmute::<F, u32>(v) });
614 }
615 }
616 let d_touched_blocks = merkle_records
617 .to_device_on(&gpu_merkle_tree.device_ctx)
618 .unwrap();
619
620 let unpadded_height = gpu_merkle_tree.calculate_unpadded_height(&touched_blocks);
621 gpu_hasher_chip.prepare_records(unpadded_height);
622 gpu_merkle_tree.update_with_touched_blocks(unpadded_height, &d_touched_blocks, false);
623
624 assert_eq!(
625 cpu_merkle_tree.root(),
626 gpu_merkle_tree
627 .top_roots
628 .to_host_on(&gpu_merkle_tree.device_ctx)
629 .unwrap()[0]
630 );
631 eprintln!("{:?}", cpu_merkle_tree.root());
632 eprintln!(
633 "{:?}",
634 gpu_merkle_tree
635 .top_roots
636 .to_host_on(&gpu_merkle_tree.device_ctx)
637 .unwrap()[0]
638 );
639 }
640}