1use crate::{DEFAULT_STACK_SIZE, DEFAULT_STACK_TOP, JitResult, PAGE_SIZE, ProgramImage};
2
3use std::io;
4use std::ops::{BitOr, BitOrAssign};
5
6#[derive(Clone, Copy, Debug, PartialEq, Eq)]
7pub struct PagePerms(u8);
8
9impl PagePerms {
10 pub const READ: Self = Self(1 << 0);
11 pub const WRITE: Self = Self(1 << 1);
12 pub const EXEC: Self = Self(1 << 2);
13 pub const fn empty() -> Self {
16 Self(0)
17 }
18
19 pub const fn bits(self) -> u8 {
20 self.0
21 }
22
23 pub const fn from_bits(bits: u8) -> Self {
24 Self(bits & 0b111)
25 }
26
27 pub const fn contains(self, other: Self) -> bool {
28 (self.0 & other.0) == other.0
29 }
30}
31
32impl BitOr for PagePerms {
33 type Output = Self;
34
35 fn bitor(self, rhs: Self) -> Self::Output {
36 Self(self.0 | rhs.0)
37 }
38}
39
40impl BitOrAssign for PagePerms {
41 fn bitor_assign(&mut self, rhs: Self) {
42 self.0 |= rhs.0;
43 }
44}
45
46use crate::{GuestMemoryAbi, GuestPageAbi, GuestTranslationAbi};
48
49#[derive(Clone)]
50struct OwnedPage {
51 base: u64,
52 perms: PagePerms,
53 data: Box<[u8; PAGE_SIZE]>,
54}
55
56pub struct GuestMemory {
57 pages: Vec<OwnedPage>,
58 abi_pages: Vec<GuestPageAbi>,
59 dirty_pages: Vec<usize>,
60 dirty_bitmap: Vec<u64>,
61 abi: GuestMemoryAbi,
62 abi_dirty: bool,
63 mapping_revision: u64,
64 baseline: u64,
65 pub brk: u64,
66 pub brk_floor: u64,
67 pub stack_top: u64,
68}
69
70impl Clone for GuestMemory {
71 fn clone(&self) -> Self {
72 let mut memory = Self {
73 pages: self.pages.clone(),
74 abi_pages: Vec::new(),
75 dirty_pages: Vec::new(),
76 dirty_bitmap: Vec::new(),
77
78 abi: GuestMemoryAbi {
79 pages: std::ptr::null(),
80 page_count: 0,
81 page_size: PAGE_SIZE,
82 translations: [GuestTranslationAbi::default(); crate::TRANSLATION_SLOTS],
83 dirty_pages: std::ptr::null_mut(),
84 dirty_count: 0,
85 code_dirty: false,
86 dirty_bitmap: std::ptr::null_mut(),
87 },
88
89 abi_dirty: true,
90 mapping_revision: self.mapping_revision,
91 baseline: self.baseline,
92 brk: self.brk,
93 brk_floor: self.brk_floor,
94 stack_top: self.stack_top,
95 };
96
97 memory.sync_abi();
98
99 if !self.abi_dirty {
100 memory.dirty_pages.clone_from(&self.dirty_pages);
101 memory.dirty_bitmap.clone_from(&self.dirty_bitmap);
102 memory.abi.dirty_pages = memory.dirty_pages.as_mut_ptr();
103 memory.abi.dirty_bitmap = memory.dirty_bitmap.as_mut_ptr();
104 memory.abi.dirty_count = self.abi.dirty_count;
105 }
106
107 memory.abi.code_dirty = self.abi.code_dirty;
108
109 memory
110 }
111}
112
113impl Default for GuestMemory {
114 fn default() -> Self {
115 Self::new()
116 }
117}
118
119impl GuestMemory {
120 pub fn new() -> Self {
121 Self {
122 pages: Vec::new(),
123 abi_pages: Vec::new(),
124 dirty_pages: Vec::new(),
125 dirty_bitmap: Vec::new(),
126
127 abi: GuestMemoryAbi {
128 pages: std::ptr::null(),
129 page_count: 0,
130 page_size: PAGE_SIZE,
131 translations: [GuestTranslationAbi::default(); crate::TRANSLATION_SLOTS],
132 dirty_pages: std::ptr::null_mut(),
133 dirty_count: 0,
134 code_dirty: false,
135 dirty_bitmap: std::ptr::null_mut(),
136 },
137
138 abi_dirty: true,
139 mapping_revision: 0,
140 baseline: 0,
141 brk_floor: 0,
142 brk: 0, stack_top: 0,
144 }
145 }
146
147 pub fn from_program(program: &ProgramImage) -> JitResult<Self> {
148 let mut memory = Self::new();
149
150 memory.map_program(program)?;
151
152 memory.map_zeroed(
153 DEFAULT_STACK_TOP - DEFAULT_STACK_SIZE,
154 DEFAULT_STACK_SIZE,
155 PagePerms::READ | PagePerms::WRITE,
156 )?;
157
158 memory.stack_top = DEFAULT_STACK_TOP;
159 memory.sync_abi();
160
161 Ok(memory)
162 }
163
164 pub fn as_abi_mut(&mut self) -> *mut GuestMemoryAbi {
165 if self.abi_dirty {
166 self.sync_abi();
167 }
168 &mut self.abi
169 }
170
171 pub fn map_program(&mut self, program: &ProgramImage) -> JitResult<()> {
172 self.map_program_at(program, 0)?;
173 self.brk = program.brk_base;
174 self.brk_floor = self.brk;
175
176 Ok(())
177 }
178
179 pub fn map_program_at(&mut self, program: &ProgramImage, bias: u64) -> JitResult<()> {
180 for segment in &program.segments {
181 let base = bias.checked_add(segment.vaddr).ok_or_else(|| io::Error::other("segment address overflow"))?;
182
183 self.map_bytes(base, &segment.bytes, segment.perms)?;
184
185 if segment.mem_size > segment.file_size {
186 self.map_zeroed(base + segment.file_size, segment.mem_size - segment.file_size, segment.perms)?;
187 }
188 }
189
190 Ok(())
191 }
192
193 pub fn map_zeroed(&mut self, base: u64, size: u64, perms: PagePerms) -> JitResult<()> {
194 if size == 0 {
195 return Ok(());
196 }
197
198 let end =
199 align_up(base.checked_add(size).ok_or_else(|| io::Error::other("mapping overflow"))?, PAGE_SIZE as u64)
200 .ok_or_else(|| io::Error::other("mapping overflow"))?;
201
202 let start = align_down(base, PAGE_SIZE as u64);
203
204 if !self.can_map(base, size) {
205 return Err(io::Error::other("guest memory limit exceeded").into());
206 }
207
208 for page_base in (start..end).step_by(PAGE_SIZE) {
209 self.ensure_page(page_base, perms);
210 }
211
212 Ok(())
213 }
214
215 pub fn map_bytes(&mut self, base: u64, bytes: &[u8], perms: PagePerms) -> JitResult<()> {
216 self.map_zeroed(base, bytes.len() as u64, perms)?;
217
218 let mut address = base;
219 let mut remaining = bytes;
220
221 while !remaining.is_empty() {
222 let page_base = align_down(address, PAGE_SIZE as u64);
223
224 let index = self.pages.binary_search_by_key(&page_base, |page| page.base).unwrap();
225
226 let page = &mut self.pages[index];
227 let offset = (address - page_base) as usize;
228 let count = remaining.len().min(PAGE_SIZE - offset);
229
230 page.data[offset..offset + count].copy_from_slice(&remaining[..count]);
231
232 self.abi.code_dirty |= page.perms.contains(PagePerms::EXEC);
233
234 if self.baseline != 0 {
235 self.mark_dirty(index);
236 }
237
238 address += count as u64;
239 remaining = &remaining[count..];
240 }
241 Ok(())
242 }
243
244 pub fn can_map(&self, base: u64, size: u64) -> bool {
245 let Some(end) = base.checked_add(size).and_then(|end| align_up(end, PAGE_SIZE as u64)) else {
246 return false;
247 };
248
249 let start = align_down(base, PAGE_SIZE as u64);
250
251 let existing = self.pages.iter().filter(|p| p.base >= start && p.base < end).count() as u64;
252
253 self.pages.len() as u64 + (end - start) / PAGE_SIZE as u64 - existing
255 <= crate::MAX_GUEST_MEMORY / PAGE_SIZE as u64
256 }
257
258 pub fn replace_mapping(&mut self, base: u64, size: u64, perms: PagePerms, bytes: &[u8]) -> JitResult<()> {
259 if size == 0 || !base.is_multiple_of(PAGE_SIZE as u64) || !self.can_map(base, size) {
260 return Err(io::Error::other("invalid mapping").into());
261 }
262
263 self.unmap(base, size)?;
264 self.map_zeroed(base, size, perms)?;
265
266 self.map_bytes(base, &bytes[..bytes.len().min(size as usize)], perms)?;
267
268 Ok(())
269 }
270
271 pub fn protect(&mut self, base: u64, size: u64, perms: PagePerms) -> bool {
272 let Some((start, end)) = page_range(base, size) else {
273 return false;
274 };
275
276 let mut page_base = start;
277
278 while page_base < end {
280 let Some(idx) = self.pages.binary_search_by_key(&page_base, |page| page.base).ok() else {
281 return false;
282 };
283
284 if self.pages[idx].perms != perms {
285 let code_changed = (self.pages[idx].perms | perms).contains(PagePerms::EXEC);
286
287 self.pages[idx].perms = perms;
288
289 self.mapping_changed(code_changed);
290 }
291 page_base += PAGE_SIZE as u64;
292 }
293
294 true
295 }
296
297 pub fn unmap(&mut self, base: u64, size: u64) -> JitResult<()> {
298 let Some((start, end)) = page_range(base, size) else {
299 return Err(io::Error::other("keine valid mapping ").into());
300 };
301
302 let code_changed =
303 self.pages.iter().any(|page| page.base >= start && page.base < end && page.perms.contains(PagePerms::EXEC));
304
305 let old_len = self.pages.len();
306 self.pages.retain(|page| page.base < start || page.base >= end);
307
308 if self.pages.len() != old_len {
309 self.mapping_changed(code_changed);
310 }
311
312 Ok(())
313 }
314
315 pub fn find_free_range(&self, start: u64, size: u64) -> Option<u64> {
316 let size = align_up(size, PAGE_SIZE as u64)?;
317 let mut base = align_up(start, PAGE_SIZE as u64)?;
318
319 for page in &self.pages {
320 if page.base < base {
321 continue;
322 }
323 if base.checked_add(size)? <= page.base {
324 return Some(base);
325 }
326 base = page.base.checked_add(PAGE_SIZE as u64)?;
327 }
328
329 base.checked_add(size)?;
330 Some(base)
331 }
332
333 pub fn range_is_free(&self, base: u64, size: u64) -> bool {
334 let Some((start, end)) = page_range(base, size) else {
335 return false;
336 };
337 self.pages.iter().all(|page| page.base < start || page.base >= end)
338 }
339
340 pub fn load_le(&self, addr: u64, size: usize) -> Option<u64> {
341 let mut value = 0u64;
342
343 for idx in 0..size {
344 let byte = self.read_byte(addr.checked_add(idx as u64)?, PagePerms::READ)?;
345 value |= (byte as u64) << (idx * 8);
346 }
347
348 Some(value)
349 }
350
351 pub fn check_range(&self, addr: u64, size: usize, perms: PagePerms) -> bool {
352 let Some(end) = addr.checked_add(size as u64) else {
353 return false;
354 };
355
356 let mut current = addr;
357
358 while current < end {
359 let Some(page) = self.find_page(current) else {
360 return false;
361 };
362
363 if !page.perms.contains(perms) {
364 return false;
365 }
366
367 current = (page.base.saturating_add(PAGE_SIZE as u64)).min(end);
368 }
369
370 true
371 }
372
373 pub(crate) fn take_code_dirty(&mut self) -> bool {
374 std::mem::take(&mut self.abi.code_dirty)
375 }
376
377 pub fn load_bytes(&self, addr: u64, size: usize) -> Option<Vec<u8>> {
378 if size as u64 > crate::MAX_GUEST_MEMORY || !self.check_range(addr, size, PagePerms::READ) {
379 return None;
380 }
381
382 let mut result = Vec::new();
383 result.try_reserve_exact(size).ok()?;
384
385 let mut current = addr;
386
387 while result.len() < size {
388 let page = self.find_page(current)?;
389 if !page.perms.contains(PagePerms::READ) {
390 return None;
391 }
392
393 let offset = (current - page.base) as usize;
394 let count = (PAGE_SIZE - offset).min(size - result.len());
395 result.extend_from_slice(&page.data[offset..offset + count]);
396 current = current.checked_add(count as u64)?;
397 }
398
399 Some(result)
400 }
401
402 pub fn fetch_u16(&self, addr: u64) -> Option<u16> {
403 let bytes = [self.read_byte(addr, PagePerms::EXEC)?, self.read_byte(addr.checked_add(1)?, PagePerms::EXEC)?];
404 Some(u16::from_le_bytes(bytes))
405 }
406
407 pub fn fetch_u32(&self, addr: u64) -> Option<u32> {
408 let bytes = [
409 self.read_byte(addr, PagePerms::EXEC)?,
410 self.read_byte(addr.checked_add(1)?, PagePerms::EXEC)?,
411 self.read_byte(addr.checked_add(2)?, PagePerms::EXEC)?,
412 self.read_byte(addr.checked_add(3)?, PagePerms::EXEC)?,
413 ];
414 Some(u32::from_le_bytes(bytes))
415 }
416
417 pub fn executable_end(&self, addr: u64) -> Option<u64> {
418 let mut page = self.find_page(addr)?;
419 if !page.perms.contains(PagePerms::EXEC) {
420 return None;
421 }
422
423 let mut end = page.base + PAGE_SIZE as u64;
424 while let Some(next) = self.find_page(end) {
425 page = next;
426 if !page.perms.contains(PagePerms::EXEC) {
427 break;
428 }
429 end += PAGE_SIZE as u64;
430 }
431 Some(end)
432 }
433
434 pub fn store_le(&mut self, addr: u64, size: usize, value: u64) -> bool {
435 for idx in 0..size {
436 let byte = ((value >> (idx * 8)) & 0xff) as u8;
437
438 let Some(byte_addr) = addr.checked_add(idx as u64) else {
439 return false;
440 };
441
442 if self.write_byte(byte_addr, byte, PagePerms::WRITE).is_none() {
443 return false;
444 }
445 }
446 true
447 }
448
449 pub fn store_bytes(&mut self, addr: u64, bytes: &[u8]) -> bool {
450 let mut written = 0;
451 let mut current = addr;
452
453 while written < bytes.len() {
454 let page_base = align_down(current, PAGE_SIZE as u64);
455 let Ok(index) = self.pages.binary_search_by_key(&page_base, |page| page.base) else {
456 return false;
457 };
458
459 if !self.pages[index].perms.contains(PagePerms::WRITE) {
460 return false;
461 }
462 let offset = (current - page_base) as usize;
463 let count = (PAGE_SIZE - offset).min(bytes.len() - written);
464 self.pages[index].data[offset..offset + count].copy_from_slice(&bytes[written..written + count]);
465
466 self.abi.code_dirty |= self.pages[index].perms.contains(PagePerms::EXEC);
467
468 self.mark_dirty(index);
469 written += count;
470
471 let Some(next) = current.checked_add(count as u64) else {
472 return false;
473 };
474 current = next;
475 }
476 true
477 }
478
479 pub(crate) fn snapshot(&mut self) -> Self {
480 static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
482
483 self.baseline = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
484
485 self.clear_dirty();
486 self.clone()
487 }
488
489 pub(crate) fn clear_dirty(&mut self) {
490 self.abi.dirty_count = 0;
491 self.dirty_bitmap.fill(0);
492 self.clear_translations();
493 }
494
495 pub(crate) fn restore_dirty_from(&mut self, snapshot: &GuestMemory) -> bool {
496 if self.baseline != snapshot.baseline
497 || self.mapping_revision != snapshot.mapping_revision
498 || self.pages.len() != snapshot.pages.len()
499 {
500 return false;
501 }
502
503 for slot in 0..self.abi.dirty_count {
504 let index = self.dirty_pages[slot];
505
506 self.abi.code_dirty |= self.pages[index].perms.contains(PagePerms::EXEC);
507
508 self.pages[index].data.copy_from_slice(&snapshot.pages[index].data[..]);
509 }
510
511 self.brk = snapshot.brk;
512 self.brk_floor = snapshot.brk_floor;
513 self.stack_top = snapshot.stack_top;
514 self.clear_dirty();
515 true
516 }
517
518 pub fn can_execute(&self, addr: u64) -> bool {
519 self.find_page(addr).map(|page| page.perms.contains(PagePerms::EXEC)).unwrap_or(false)
520 }
521
522 pub fn mappings(&self) -> Vec<(u64, u64, PagePerms)> {
523 let mut mappings = Vec::new();
524 for page in &self.pages {
525 if let Some((_, end, perms)) = mappings.last_mut()
526 && *end == page.base
527 && *perms == page.perms
528 {
529 *end += PAGE_SIZE as u64;
530 } else {
531 mappings.push((page.base, page.base + PAGE_SIZE as u64, page.perms));
532 }
533 }
534 mappings
535 }
536
537 pub fn mapped_pages(&self) -> impl Iterator<Item = (u64, PagePerms, &[u8])> {
538 self.pages.iter().map(|page| (page.base, page.perms, page.data.as_slice()))
539 }
540
541 fn ensure_page(&mut self, base: u64, perms: PagePerms) -> &mut OwnedPage {
542 match self.pages.binary_search_by_key(&base, |page| page.base) {
543 Ok(idx) => {
544 if !self.pages[idx].perms.contains(perms) {
545 self.pages[idx].perms |= perms;
546 self.mapping_changed(self.pages[idx].perms.contains(PagePerms::EXEC));
547 }
548 &mut self.pages[idx]
549 }
550
551 Err(idx) => {
552 self.mapping_changed(perms.contains(PagePerms::EXEC));
553 self.pages.insert(idx, OwnedPage { base, perms, data: Box::new([0; PAGE_SIZE]) });
554 &mut self.pages[idx]
555 }
556 }
557 }
558
559 fn find_page(&self, addr: u64) -> Option<&OwnedPage> {
560 let page_base = align_down(addr, PAGE_SIZE as u64);
561
562 self.pages.binary_search_by_key(&page_base, |page| page.base).ok().map(|idx| &self.pages[idx])
563 }
564
565 fn read_byte(&self, addr: u64, perms: PagePerms) -> Option<u8> {
566 let page = self.find_page(addr)?;
567
568 if !page.perms.contains(perms) {
569 return None;
570 }
571
572 let offset = (addr - page.base) as usize;
573 Some(page.data[offset])
574 }
575
576 fn write_byte(&mut self, addr: u64, byte: u8, perms: PagePerms) -> Option<()> {
577 let page_base = align_down(addr, PAGE_SIZE as u64);
578
579 let index = self.pages.binary_search_by_key(&page_base, |page| page.base).ok()?;
580 let page = &mut self.pages[index];
581
582 if !page.perms.contains(perms) {
583 return None;
584 }
585
586 let offset = (addr - page.base) as usize;
587 page.data[offset] = byte;
588
589 self.abi.code_dirty |= page.perms.contains(PagePerms::EXEC);
590
591 self.mark_dirty(index);
592 Some(())
593 }
594
595 fn sync_abi(&mut self) {
597 self.abi_pages = self
598 .pages
599 .iter_mut()
600 .map(|page| GuestPageAbi {
601 guest_base: page.base,
602 perms: page.perms.bits(),
603 _padding: [0; 7],
604 data: page.data.as_mut_ptr(),
605 })
606 .collect();
607
608 self.dirty_pages.resize(self.pages.len(), 0);
609 self.dirty_bitmap.resize(self.pages.len().div_ceil(64), 0);
610 self.dirty_bitmap.fill(0);
611
612 self.abi.pages = self.abi_pages.as_ptr();
613 self.abi.page_count = self.abi_pages.len();
614 self.abi.page_size = PAGE_SIZE;
615 self.abi.dirty_pages = self.dirty_pages.as_mut_ptr();
616 self.abi.dirty_count = 0;
617 self.abi.dirty_bitmap = self.dirty_bitmap.as_mut_ptr();
618 self.clear_translations();
619 self.abi_dirty = false;
620 }
621
622 fn mark_dirty(&mut self, index: usize) {
623 if self.abi_dirty {
624 self.sync_abi();
625 }
626 let word = index / 64;
627 let bit = 1u64 << (index % 64);
628 if self.dirty_bitmap[word] & bit == 0 {
629 self.dirty_bitmap[word] |= bit;
630 self.dirty_pages[self.abi.dirty_count] = index;
631 self.abi.dirty_count += 1;
632 }
633 }
634
635 fn mapping_changed(&mut self, code_changed: bool) {
636 self.abi.code_dirty |= code_changed;
637 self.abi_dirty = true;
638 self.mapping_revision = self.mapping_revision.wrapping_add(1);
639 self.clear_translations();
640 }
641
642 fn clear_translations(&mut self) {
643 self.abi.translations = [GuestTranslationAbi::default(); crate::TRANSLATION_SLOTS];
644 }
645}
646
647pub(crate) fn align_down(value: u64, align: u64) -> u64 {
648 value & !(align - 1)
649}
650
651pub(crate) fn align_up(value: u64, align: u64) -> Option<u64> {
652 value.checked_add(align - 1).map(|value| value & !(align - 1))
653}
654
655fn page_range(base: u64, size: u64) -> Option<(u64, u64)> {
656 if size == 0 || !base.is_multiple_of(PAGE_SIZE as u64) {
657 return None;
658 }
659
660 let end = align_up(base.checked_add(size)?, PAGE_SIZE as u64)?;
661 Some((base, end))
662}