jit/mmu.rs
1//! The MMU separates guest addresses and permissions from unsafe host pointers used by JIT code.
2//!
3//! Linux mapping rules: https://man7.org/linux/man-pages/man2/mmap.2.html
4//! Rust C layout rules: https://doc.rust-lang.org/reference/type-layout.html#the-c-representation
5
6use crate::{DEFAULT_STACK_SIZE, DEFAULT_STACK_TOP, JitResult, PAGE_SIZE, ProgramImage};
7
8use std::io;
9use std::ops::{BitOr, BitOrAssign};
10
11#[derive(Clone, Copy, Debug, PartialEq, Eq)]
12/// Uses Linux protection bits so mapping operations need no second permission model.
13pub struct PagePerms(u8);
14
15/*
16
17 wip - siehe notion
18
19 todo
20 ----
21 - schnitstelle nach außen
22 - magic values
23 - größen
24
25*/
26
27// todo -- nochmal neu
28impl PagePerms {
29 // One bit represents one independent permission. These values also match Linux PROT_* values.
30 // Bit 0 has value 1, bit 1 has value 2, and bit 2 has value 4.
31 /// Allows guest reads from a page.
32 pub const READ: Self = Self(1 << 0);
33 /// Allows guest writes to a page.
34 pub const WRITE: Self = Self(1 << 1);
35 /// Allows guest instruction fetches from a page.
36 pub const EXEC: Self = Self(1 << 2);
37 // pub const RWX: Self
38
39 /// Creates permissions that deny all guest accesses.
40 pub const fn empty() -> Self {
41 // Zero has no permission bits set. It represents a mapped page with no allowed access.
42 Self(0)
43 }
44
45 /// Returns the three permission bits used by generated code.
46 pub const fn bits(self) -> u8 {
47 // Generated code needs the compact bit value because it cannot call this Rust type.
48 self.0
49 }
50
51 /// Creates permissions and removes bits that this MMU does not support.
52 pub const fn from_bits(bits: u8) -> Self {
53 // Ignore unknown protection bits so they cannot enter the JIT ABI.
54 // Binary 111 keeps only the low READ, WRITE, and EXEC bits.
55 Self(bits & 0b111)
56 }
57
58 /// Tests whether all permissions in `other` are present.
59 pub const fn contains(self, other: Self) -> bool {
60 // AND removes all bits that are not requested. Equality means that every requested bit exists.
61 (self.0 & other.0) == other.0
62 }
63}
64
65impl BitOr for PagePerms {
66 type Output = Self;
67
68 fn bitor(self, rhs: Self) -> Self::Output {
69 // OR combines independent permissions without changing permissions that already exist.
70 // This trait lets callers write `READ | WRITE` instead of calling a helper function.
71 Self(self.0 | rhs.0)
72 }
73}
74
75impl BitOrAssign for PagePerms {
76 fn bitor_assign(&mut self, rhs: Self) {
77 // This trait gives `|=` the same bit-combine rule as `|`.
78 self.0 |= rhs.0;
79 }
80}
81
82// ist shared siehe emitter
83#[repr(C)]
84#[derive(Clone, Copy)]
85/// Keeps the host and generated-code page layout identical across the C ABI.
86// `repr(C)` fixes field order. The default Rust layout does not give this guarantee.
87pub struct GuestPageAbi {
88 /// Gives the aligned guest address of the first byte in this page.
89 pub guest_base: u64,
90 /// Gives the compact [`PagePerms`] bits checked by generated code.
91 pub perms: u8,
92 // Seven bytes fill the space after `perms`. The next pointer then starts on an 8-byte boundary.
93 /// Keeps `data` at the offset that the generated ABI expects.
94 pub _padding: [u8; 7],
95 // Generated code uses this pointer after it has checked the guest address and permission.
96 /// Points to the first host byte of this guest page.
97 pub data: *mut u8,
98}
99
100#[repr(C)]
101#[derive(Clone, Copy)]
102/// Caches one guest-to-host translation to avoid a page search on common accesses.
103pub struct GuestTranslationAbi {
104 /// Identifies the guest page held in this cache entry.
105 pub guest_base: u64,
106 // A write uses this index to record the owning page in the dirty-page list.
107 /// Identifies the page in the ABI page array for dirty tracking.
108 pub page_index: usize,
109 /// Copies the page permissions so a cache hit still checks access rights.
110 pub perms: u8,
111 // Explicit padding gives this host type the same layout as the type in generated Rust code.
112 /// Keeps `data` at the offset that the generated ABI expects.
113 pub _padding: [u8; 7],
114 // This pointer is null until the cache contains a valid translation.
115 /// Points to the first host byte, or is null when this entry is empty.
116 pub data: *mut u8,
117}
118
119impl Default for GuestTranslationAbi {
120 fn default() -> Self {
121 // A null data pointer marks an empty cache entry. Guest address zero can then be cached safely.
122 Self {
123 guest_base: 0,
124 page_index: 0,
125 perms: 0,
126 _padding: [0; 7],
127 data: std::ptr::null_mut(),
128 }
129 }
130}
131
132#[repr(C)]
133/// Exposes only stable pointers and lengths because generated code cannot use Rust containers.
134pub struct GuestMemoryAbi {
135 // This raw pointer refers to a contiguous `abi_pages` vector owned by `GuestMemory`.
136 /// Points to the sorted page descriptions used by generated binary search.
137 pub pages: *const GuestPageAbi,
138 // Generated binary search uses the count to stay inside the raw page array.
139 /// Limits generated searches to valid entries in `pages`.
140 pub page_count: usize,
141 // Publish the selected page size in the mirrored ABI. The current template embeds the same constant.
142 /// Gives the page size that defines guest page offsets.
143 pub page_size: usize,
144 // Sixteen entries give a small power-of-two cache. Generated code can select a slot with `& 15`.
145 /// Keeps recent translations so common accesses do not need binary search.
146 pub translations: [GuestTranslationAbi; 16],
147 // todo -- von gamozo
148 // The list lets restore visit only changed pages. The bitmap records each page once.
149 /// Points to page indices changed since the current snapshot baseline.
150 pub dirty_pages: *mut usize,
151 // Only this prefix of `dirty_pages` contains page indices for the current snapshot interval.
152 /// Gives the valid prefix length of `dirty_pages`.
153 pub dirty_count: usize,
154 /// Points to one bit per page so each dirty page enters the list once.
155 pub dirty_bitmap: *mut u64,
156}
157
158#[derive(Clone)]
159// A boxed page keeps its data address stable when the sorted page vector moves its entries.
160struct OwnedPage {
161 // The base has page alignment. An offset in this page is always smaller than `PAGE_SIZE`.
162 base: u64,
163 perms: PagePerms,
164 data: Box<[u8; PAGE_SIZE]>,
165}
166
167/// Owns guest pages and maintains the pointer-based view used by compiled regions.
168pub struct GuestMemory {
169 // Pages stay in guest-address order because all lookup paths use binary search.
170 pages: Vec<OwnedPage>,
171 // This second vector has the simple C layout that generated code can read.
172 abi_pages: Vec<GuestPageAbi>,
173 // Restore visits only these changed page indices instead of copying all guest memory.
174 dirty_pages: Vec<usize>,
175 // One bit records each page at most once, even when generated code writes it many times.
176 dirty_bitmap: Vec<u64>,
177 abi: GuestMemoryAbi,
178 // A true value requires fresh raw pointers before generated code enters the MMU ABI.
179 abi_dirty: bool, // gamozo siehe
180 // A revision change means that snapshot page indices can no longer be trusted.
181 mapping_revision: u64,
182 /// Gives the first address after the current guest heap.
183 // `brk` is the first address after the current guest heap.
184 pub brk: u64,
185 /// Gives the fixed exclusive stack top used to build the initial process stack.
186 // The fixed top is exclusive. Stack data grows toward lower guest addresses.
187 pub stack_top: u64,
188}
189
190// fast reset -- tofo optimze?
191impl Clone for GuestMemory {
192
193 fn clone(&self) -> Self {
194 // Rebuild all ABI pointers because pointers in the source memory refer to its allocations.
195 // A derived `Clone` would copy stale raw pointers and could make generated code use old memory.
196 let mut memory = Self {
197 pages: self.pages.clone(),
198 abi_pages: Vec::new(),
199 dirty_pages: Vec::new(),
200 dirty_bitmap: Vec::new(),
201 abi: GuestMemoryAbi {
202 pages: std::ptr::null(),
203 page_count: 0,
204 page_size: PAGE_SIZE,
205 translations: [GuestTranslationAbi::default(); 16],
206 dirty_pages: std::ptr::null_mut(),
207 dirty_count: 0,
208 dirty_bitmap: std::ptr::null_mut(),
209 },
210 abi_dirty: true,
211 mapping_revision: self.mapping_revision,
212 brk: self.brk,
213 stack_top: self.stack_top,
214 };
215 memory.sync_abi();
216 memory
217 }
218}
219
220impl Default for GuestMemory {
221 fn default() -> Self {
222 // Keep one construction path so `new` and `default` cannot get different ABI state.
223 Self::new()
224 }
225}
226
227// todo - 4096 passt?
228impl GuestMemory {
229 /// Starts with no mappings so ELF loading can define all initial permissions.
230 pub fn new() -> Self {
231 Self {
232 pages: Vec::new(),
233 abi_pages: Vec::new(),
234 dirty_pages: Vec::new(),
235 dirty_bitmap: Vec::new(),
236 abi: GuestMemoryAbi {
237 pages: std::ptr::null(),
238 page_count: 0,
239 page_size: PAGE_SIZE,
240 translations: [GuestTranslationAbi::default(); 16],
241 dirty_pages: std::ptr::null_mut(),
242 dirty_count: 0,
243 dirty_bitmap: std::ptr::null_mut(),
244 },
245 abi_dirty: true,
246 mapping_revision: 0,
247 brk: 0, // heap
248 stack_top: 0,
249 }
250 }
251
252 /// Maps one program image and creates its fixed high guest stack.
253 pub fn from_program(program: &ProgramImage) -> JitResult<Self> {
254 // Create all initial process mappings in one place so the runtime gets a complete ABI view.
255 let mut memory = Self::new();
256
257 memory.map_program(program)?;
258
259 // A fixed high stack keeps it separate from the image, heap, and mmap area.
260 memory.map_zeroed(
261 DEFAULT_STACK_TOP - DEFAULT_STACK_SIZE,
262 DEFAULT_STACK_SIZE,
263 PagePerms::READ | PagePerms::WRITE,
264 )?;
265
266 memory.stack_top = DEFAULT_STACK_TOP;
267 memory.sync_abi();
268
269 // dbg!(memory);
270
271 Ok(memory)
272 }
273
274 // todo -- wichtig. anders machen
275 /// Refreshes raw pointers only when mappings changed, which keeps normal dispatch inexpensive.
276 pub fn as_abi_mut(&mut self) -> *mut GuestMemoryAbi {
277 if self.abi_dirty {
278 self.sync_abi();
279 }
280 // Rust converts this mutable reference to a raw pointer for the generated C ABI.
281 // The `GuestMemory` box must stay alive while generated code uses this pointer.
282 &mut self.abi
283 }
284
285 /// Maps the main image at its linked addresses and sets the initial heap end.
286 pub fn map_program(&mut self, program: &ProgramImage) -> JitResult<()> {
287 // A zero bias keeps the linked virtual addresses of the main executable unchanged.
288 self.map_program_at(program, 0)?;
289 // Linux starts `brk` after the last load segment.
290 self.brk = program.brk_base;
291
292 Ok(())
293 }
294
295 /// Maps an image with an address bias so a position-independent loader can coexist with it.
296 pub fn map_program_at(&mut self, program: &ProgramImage, bias: u64) -> JitResult<()> {
297 // The bias relocates a position-independent interpreter without changing the stored
298 // ELF segment metadata.
299 for segment in &program.segments {
300 // `?` stops this function and forwards the error if checked addition fails.
301 let base = bias
302 .checked_add(segment.vaddr)
303 .ok_or_else(|| io::Error::other("segment address overflow"))?;
304
305 self.map_bytes(base, &segment.bytes, segment.perms)?;
306
307 // ELF memory beyond the file payload is BSS and must start as zero.
308 if segment.mem_size > segment.file_size {
309 self.map_zeroed(
310 base + segment.file_size,
311 segment.mem_size - segment.file_size,
312 segment.perms,
313 )?;
314 }
315 }
316
317 Ok(())
318 }
319
320 /// Ensures that all pages in a byte range exist, start with zero, and include `perms`.
321 pub fn map_zeroed(&mut self, base: u64, size: u64, perms: PagePerms) -> JitResult<()> {
322 if size == 0 {
323 // A zero-length ELF BSS or mapping has no pages and is a successful no-op.
324 return Ok(());
325 }
326
327 // Guest addresses are untrusted. Checked addition prevents an address wrap to zero.
328 let end = base
329 .checked_add(size)
330 .ok_or_else(|| io::Error::other("overflow"))?;
331
332 let mut page_base = align_down(base, PAGE_SIZE as u64);
333 // Subtract one because an end on a page boundary does not include the next page.
334 let last_page = align_down(end.saturating_sub(1), PAGE_SIZE as u64);
335
336 // Include partial edge pages because guest mappings do not have to start at page boundaries.
337 while page_base <= last_page {
338 let page = self.ensure_page(page_base, perms);
339 // A segment can share an edge page with another segment. Keep permissions from both.
340 page.perms |= perms;
341 // Saturation prevents wraparound from turning the loop back to address zero.
342 page_base = page_base.saturating_add(PAGE_SIZE as u64);
343 }
344
345 Ok(())
346 }
347
348 /// Copies image bytes into mapped pages without requiring guest write permission.
349 pub fn map_bytes(&mut self, base: u64, bytes: &[u8], perms: PagePerms) -> JitResult<()> {
350 // let mut page_base = align_down(base, PAGE_SIZE as u64);
351
352 // Write through the loader path because ELF segments can start or end inside a shared page.
353 for (offset, byte) in bytes.iter().enumerate() {
354 let addr = base
355 .checked_add(offset as u64)
356 .ok_or_else(|| io::Error::other("overflow"))?;
357
358 // Loading must initialize read-only and executable pages without a guest write.
359 self.write_mapped_byte(addr, *byte, perms)?;
360 }
361 Ok(())
362 }
363
364 /// Replaces complete pages so fixed mappings cannot retain old bytes or permissions.
365 pub fn replace_mapping(
366 &mut self,
367 base: u64,
368 size: u64,
369 perms: PagePerms,
370 bytes: &[u8],
371 ) -> JitResult<()> {
372 // Linux MAP_FIXED and munmap require a page-aligned start address.
373 if size == 0 || !base.is_multiple_of(PAGE_SIZE as u64) {
374 return Err(io::Error::other("invalid mapping").into());
375 }
376
377 // Remove old bytes and permissions so a fixed mmap cannot expose stale contents.
378 self.unmap(base, size)?;
379 self.map_zeroed(base, size, perms)?;
380
381 // Ignore file bytes after the mapping size so they cannot enter the next guest page.
382 for (offset, byte) in bytes.iter().take(size as usize).enumerate() {
383 self.write_mapped_byte(base + offset as u64, *byte, perms)?;
384 }
385
386 Ok(())
387 }
388
389 /// Replaces permissions on each mapped page in an aligned range.
390 ///
391 /// The function returns `false` for an invalid range or a missing page. It can change an
392 /// earlier page before it finds a missing later page.
393 pub fn protect(&mut self, base: u64, size: u64, perms: PagePerms) -> bool {
394 // An invalid range changes nothing. A missing later page can leave earlier pages changed.
395 let Some((start, end)) = page_range(base, size) else {
396 return false;
397 };
398
399 let mut page_base = start;
400
401 // perf binary search
402 while page_base < end {
403 let Some(idx) = self
404 .pages
405 .binary_search_by_key(&page_base, |page| page.base)
406 .ok()
407 else {
408 return false;
409 };
410
411 if self.pages[idx].perms != perms {
412 self.pages[idx].perms = perms;
413 // Generated translation entries include permissions and must be invalidated.
414 self.mapping_changed();
415 }
416 // Each iteration changes exactly one page in the requested range.
417 page_base += PAGE_SIZE as u64;
418 }
419
420 true
421 }
422
423 /// Removes all pages in an aligned, half-open range.
424 pub fn unmap(&mut self, base: u64, size: u64) -> JitResult<()> {
425 // Use one half-open page range so removal follows the same boundary rules as protection.
426 let Some((start, end)) = page_range(base, size) else {
427 return Err(io::Error::other("keine valid mapping ").into());
428 };
429
430 let old_len = self.pages.len();
431 // `retain` keeps pages outside the range and drops pages inside the range.
432 self.pages
433 .retain(|page| page.base < start || page.base >= end);
434 if self.pages.len() != old_len {
435 self.mapping_changed();
436 }
437
438 // dbg!(self.pages);
439 Ok(())
440 }
441
442 /// Finds the first page-aligned free range at or after `start`.
443 pub fn find_free_range(&self, start: u64, size: u64) -> Option<u64> {
444 // Both values need page alignment because Linux creates complete-page mappings.
445 let size = align_up(size, PAGE_SIZE as u64);
446 let mut base = align_up(start, PAGE_SIZE as u64);
447
448 // Sorted pages make the first sufficient gap a deterministic mmap result.
449 for page in &self.pages {
450 if page.base < base {
451 continue;
452 }
453 if base.checked_add(size)? <= page.base {
454 return Some(base);
455 }
456 // Continue after this occupied page. Checked addition rejects address-space overflow.
457 base = page.base.checked_add(PAGE_SIZE as u64)?;
458 }
459
460 Some(base)
461 }
462
463 /// Tests whether an aligned, nonempty range has no mapped page.
464 pub fn range_is_free(&self, base: u64, size: u64) -> bool {
465 let Some((start, end)) = page_range(base, size) else {
466 return false;
467 };
468 // `all` returns true only when no page overlaps the requested half-open range.
469 self.pages
470 .iter()
471 .all(|page| page.base < start || page.base >= end)
472 }
473
474 // todo -- code dup
475 /// Loads at most eight little-endian bytes when the full range is readable.
476 pub fn load_le(&self, addr: u64, size: usize) -> Option<u64> {
477 let mut value = 0u64;
478
479 // Callers must request at most eight bytes because the result container is u64.
480 for idx in 0..size {
481 // Here, `?` changes any invalid address or denied read into `None` for the full load.
482 let byte = self.read_byte(addr.checked_add(idx as u64)?, PagePerms::READ)?;
483 // One byte has 8 bits. Little endian puts the byte at the lowest address in bits 0..7.
484 value |= (byte as u64) << (idx * 8);
485 }
486
487 Some(value)
488 }
489
490 /// Copies a readable guest range and returns `None` if any byte is unavailable.
491 pub fn load_bytes(&self, addr: u64, size: usize) -> Option<Vec<u8>> {
492 let mut result = Vec::with_capacity(size);
493 let mut current = addr;
494
495 // Copy one page span at a time to reduce repeated searches for large buffers.
496 while result.len() < size {
497 let page = self.find_page(current)?;
498 if !page.perms.contains(PagePerms::READ) {
499 return None;
500 }
501
502 let offset = (current - page.base) as usize;
503 // Stop at this page end or the requested buffer end, whichever comes first.
504 let count = (PAGE_SIZE - offset).min(size - result.len());
505 result.extend_from_slice(&page.data[offset..offset + count]);
506 current = current.checked_add(count as u64)?;
507 }
508
509 Some(result)
510 }
511
512 /// Fetches one little-endian 16-bit instruction parcel with execute permission.
513 pub fn fetch_u16(&self, addr: u64) -> Option<u16> {
514 // Instruction fetch needs execute permission even when the page is not readable.
515 let bytes = [
516 self.read_byte(addr, PagePerms::EXEC)?,
517 self.read_byte(addr.checked_add(1)?, PagePerms::EXEC)?,
518 ];
519 // RISC-V code in this runtime is little endian, so array byte 0 is the low byte.
520 Some(u16::from_le_bytes(bytes))
521 }
522
523 /// Fetches one little-endian 32-bit instruction word with execute permission.
524 pub fn fetch_u32(&self, addr: u64) -> Option<u32> {
525 // Fetch each byte separately because a four-byte instruction can cross a page boundary.
526 let bytes = [
527 self.read_byte(addr, PagePerms::EXEC)?,
528 self.read_byte(addr.checked_add(1)?, PagePerms::EXEC)?,
529 self.read_byte(addr.checked_add(2)?, PagePerms::EXEC)?,
530 self.read_byte(addr.checked_add(3)?, PagePerms::EXEC)?,
531 ];
532 // `from_le_bytes` gives the same instruction word on little- and big-endian hosts.
533 Some(u32::from_le_bytes(bytes))
534 }
535
536 /// Finds the exclusive end of consecutive executable pages that contain `addr`.
537 pub fn executable_end(&self, addr: u64) -> Option<u64> {
538 let mut page = self.find_page(addr)?;
539 if !page.perms.contains(PagePerms::EXEC) {
540 return None;
541 }
542
543 let mut end = page.base + PAGE_SIZE as u64;
544 // Stop at the first gap or non-executable page so AOT lifting stays in valid code.
545 while let Some(next) = self.find_page(end) {
546 page = next;
547 if !page.perms.contains(PagePerms::EXEC) {
548 break;
549 }
550 end += PAGE_SIZE as u64;
551 }
552 Some(end)
553 }
554
555 /// Stores at most eight little-endian bytes and records each changed page.
556 ///
557 /// The store is not atomic. It can change initial bytes before a later byte fails.
558 pub fn store_le(&mut self, addr: u64, size: usize, value: u64) -> bool {
559 // This byte loop is not atomic. A later fault does not undo bytes already written.
560 // Callers must request at most eight bytes because the source container is u64.
561 for idx in 0..size {
562 // Move the selected byte to bits 0..7, then mask off all higher bits with 0xff.
563 let byte = ((value >> (idx * 8)) & 0xff) as u8;
564
565 let Some(byte_addr) = addr.checked_add(idx as u64) else {
566 return false;
567 };
568
569 if self.write_byte(byte_addr, byte, PagePerms::WRITE).is_none() {
570 return false;
571 }
572 }
573 true
574 }
575
576 /// Copies bytes to writable guest pages and records each changed page.
577 ///
578 /// The store is not atomic. It can change an initial page span before a later span fails.
579 pub fn store_bytes(&mut self, addr: u64, bytes: &[u8]) -> bool {
580 let mut written = 0;
581 let mut current = addr;
582
583 // This operation can write an initial page span before a later span fails.
584 // Copy contiguous page spans because syscall buffers can be much larger than one value.
585 while written < bytes.len() {
586 let page_base = align_down(current, PAGE_SIZE as u64);
587 let Ok(index) = self
588 .pages
589 .binary_search_by_key(&page_base, |page| page.base)
590 else {
591 return false;
592 };
593
594 if !self.pages[index].perms.contains(PagePerms::WRITE) {
595 return false;
596 }
597 let offset = (current - page_base) as usize;
598 // One copy ends at a page boundary or at the input end.
599 let count = (PAGE_SIZE - offset).min(bytes.len() - written);
600 self.pages[index].data[offset..offset + count]
601 .copy_from_slice(&bytes[written..written + count]);
602
603 self.mark_dirty(index);
604 written += count;
605 let Some(next) = current.checked_add(count as u64) else {
606 return false;
607 };
608 current = next;
609 }
610 true
611 }
612
613 // todo -- refactor quatsch
614 pub(crate) fn clear_dirty(&mut self) {
615 // Start a new dirty interval and force the next memory access to revalidate its translation.
616 self.abi.dirty_count = 0;
617 self.dirty_bitmap.fill(0);
618 self.clear_translations();
619 }
620
621 pub(crate) fn restore_dirty_from(&mut self, snapshot: &GuestMemory) -> bool {
622 // Dirty-page restore is safe only when page indices still identify the same mappings.
623 if self.mapping_revision != snapshot.mapping_revision
624 || self.pages.len() != snapshot.pages.len()
625 // `zip` compares page pairs at the same sorted index in both memories.
626 || self
627 .pages
628 .iter()
629 .zip(&snapshot.pages)
630 .any(|(page, saved)| page.base != saved.base || page.perms != saved.perms)
631 {
632 return false;
633 }
634
635 for slot in 0..self.abi.dirty_count {
636 // Each list entry names one page that changed after the snapshot.
637 let index = self.dirty_pages[slot];
638 self.pages[index]
639 .data
640 .copy_from_slice(&snapshot.pages[index].data[..]);
641 }
642 self.brk = snapshot.brk;
643 self.stack_top = snapshot.stack_top;
644 self.clear_dirty();
645 true
646 }
647
648 /// Tests execute permission for the page that contains `addr`.
649 pub fn can_execute(&self, addr: u64) -> bool {
650 self.find_page(addr)
651 .map(|page| page.perms.contains(PagePerms::EXEC))
652 .unwrap_or(false)
653 }
654
655 /// Returns adjacent pages with equal permissions as half-open mapping ranges.
656 pub fn mappings(&self) -> Vec<(u64, u64, PagePerms)> {
657 // Each returned tuple uses a half-open range: `start` is included and `end` is not.
658 let mut mappings = Vec::new();
659 for page in &self.pages {
660 // Merge adjacent pages with equal permissions to give process-style mapping ranges.
661 if let Some((_, end, perms)) = mappings.last_mut()
662 && *end == page.base
663 && *perms == page.perms
664 {
665 *end += PAGE_SIZE as u64;
666 } else {
667 mappings.push((page.base, page.base + PAGE_SIZE as u64, page.perms));
668 }
669 }
670 mappings
671 }
672
673 fn ensure_page(&mut self, base: u64, perms: PagePerms) -> &mut OwnedPage {
674 // Keep pages sorted because runtime and generated lookups both use binary search.
675 match self.pages.binary_search_by_key(&base, |page| page.base) {
676 Ok(idx) => {
677 if !self.pages[idx].perms.contains(perms) {
678 self.pages[idx].perms |= perms;
679 self.mapping_changed();
680 }
681 &mut self.pages[idx]
682 }
683
684 Err(idx) => {
685 // Binary search returns the insertion index in `Err`, so insertion keeps sort order.
686 self.mapping_changed();
687 self.pages.insert(
688 idx,
689 OwnedPage {
690 base,
691 perms,
692 // Anonymous mappings and ELF BSS require zero-filled initial bytes.
693 data: Box::new([0; PAGE_SIZE]),
694 },
695 );
696 &mut self.pages[idx]
697 }
698 }
699 }
700
701 fn find_page(&self, addr: u64) -> Option<&OwnedPage> {
702 // Remove the offset bits so every address in one page has the same search key.
703 let page_base = align_down(addr, PAGE_SIZE as u64);
704
705 self.pages
706 .binary_search_by_key(&page_base, |page| page.base)
707 .ok()
708 .map(|idx| &self.pages[idx])
709 }
710
711 fn read_byte(&self, addr: u64, perms: PagePerms) -> Option<u8> {
712 let page = self.find_page(addr)?;
713
714 if !page.perms.contains(perms) {
715 return None;
716 }
717
718 // Subtraction changes a guest address into an index in the page allocation.
719 let offset = (addr - page.base) as usize;
720 Some(page.data[offset])
721 }
722
723 fn write_byte(&mut self, addr: u64, byte: u8, perms: PagePerms) -> Option<()> {
724 // Page alignment removes the low 12 address bits when pages have 4096 bytes.
725 let page_base = align_down(addr, PAGE_SIZE as u64);
726
727
728 let index = self
729 .pages
730 .binary_search_by_key(&page_base, |page| page.base)
731 .ok()?;
732 let page = &mut self.pages[index];
733
734 if !page.perms.contains(perms) {
735 return None;
736 }
737
738 let offset = (addr - page.base) as usize;
739 page.data[offset] = byte;
740
741 self.mark_dirty(index);
742 Some(())
743 }
744
745 fn write_mapped_byte(&mut self, addr: u64, byte: u8, perms: PagePerms) -> JitResult<()> {
746 let page_base = align_down(addr, PAGE_SIZE as u64);
747 let page = self.ensure_page(page_base, perms);
748 page.perms |= perms;
749
750 let offset = (addr - page.base) as usize;
751 // Image loading does not mark pages dirty. A later snapshot records these bytes as baseline.
752 page.data[offset] = byte;
753
754 Ok(())
755 }
756
757 // todo wichtig
758 fn sync_abi(&mut self) {
759 // Build pointer arrays only after page storage is in its final position for this revision.
760 self.abi_pages = self
761 .pages
762 // Mutable iteration is needed because `as_mut_ptr` creates pointers to page bytes.
763 .iter_mut()
764 .map(|page| GuestPageAbi {
765 guest_base: page.base,
766 perms: page.perms.bits(),
767 _padding: [0; 7],
768 data: page.data.as_mut_ptr(),
769 })
770 .collect();
771
772 // The list needs at most one entry for each page.
773 self.dirty_pages.resize(self.pages.len(), 0);
774 // One u64 bitmap word has 64 bits, so it tracks 64 pages. Round up for a partial word.
775 self.dirty_bitmap.resize(self.pages.len().div_ceil(64), 0);
776 // A rebuilt ABI starts a new dirty interval, so no old page index can remain set.
777 self.dirty_bitmap.fill(0);
778
779 self.abi.pages = self.abi_pages.as_ptr();
780 self.abi.page_count = self.abi_pages.len();
781 self.abi.page_size = PAGE_SIZE;
782 // These pointers stay valid until a mapping change can resize their owner vectors.
783 self.abi.dirty_pages = self.dirty_pages.as_mut_ptr();
784 self.abi.dirty_count = 0;
785 self.abi.dirty_bitmap = self.dirty_bitmap.as_mut_ptr();
786 self.clear_translations();
787 self.abi_dirty = false;
788 }
789
790 fn mark_dirty(&mut self, index: usize) {
791 if self.abi_dirty {
792 self.sync_abi();
793 }
794 // Integer division selects the group of 64 pages.
795 let word = index / 64;
796 // The remainder selects one bit in that group. Shifting 1 creates its mask.
797 let bit = 1u64 << (index % 64);
798 // Record each page once so restore cost depends on changed pages, not write count.
799 if self.dirty_bitmap[word] & bit == 0 {
800 self.dirty_bitmap[word] |= bit;
801 self.dirty_pages[self.abi.dirty_count] = index;
802 self.abi.dirty_count += 1;
803 }
804
805 //dbg!(self.dirty_pages);
806 }
807
808 fn mapping_changed(&mut self) {
809 // A mapping change can invalidate pointers. A permission change makes the ABI page view stale.
810 // Both changes also invalidate cached translations.
811 self.abi_dirty = true;
812 // Only revision equality matters. Wrapping avoids a panic after an extreme number of changes.
813 self.mapping_revision = self.mapping_revision.wrapping_add(1);
814 self.clear_translations();
815 }
816
817 fn clear_translations(&mut self) {
818 // Array repeat syntax copies the empty entry into all sixteen direct-cache slots.
819 self.abi.translations = [GuestTranslationAbi::default(); 16];
820 }
821}
822
823pub(crate) fn align_down(value: u64, align: u64) -> u64 {
824 // Page and stack alignments are powers of two, so a mask avoids division.
825 // For a power-of-two alignment, `align - 1` has all offset bits set.
826 // NOT changes it to a mask that keeps only the aligned address bits.
827 value & !(align - 1)
828}
829
830pub(crate) fn align_up(value: u64, align: u64) -> u64 {
831 if value == 0 {
832 // Handle zero first because the general formula subtracts one.
833 0
834 } else {
835 // Set all low offset bits with OR, then add one to reach the next aligned value.
836 // Subtract one first so an already aligned value stays unchanged.
837 // The caller must ensure that the rounded result fits in `u64`.
838 ((value - 1) | (align - 1)) + 1
839 }
840}
841
842fn page_range(base: u64, size: u64) -> Option<(u64, u64)> {
843 // The start must have page alignment. The end rounds up because a partial last page is included.
844 if size == 0 || !base.is_multiple_of(PAGE_SIZE as u64) {
845 return None;
846 }
847
848 // `?` rejects overflow in `base + size`. The rounded end must also fit in `u64`.
849 // `align_up` has an unchecked final addition, so callers must keep the range below that limit.
850 let end = align_up(base.checked_add(size)?, PAGE_SIZE as u64);
851 Some((base, end))
852}