Skip to main content

jit/
mmu.rs

1use crate::{DEFAULT_STACK_SIZE, DEFAULT_STACK_TOP, JitResult, PAGE_SIZE, ProgramImage};
2
3use std::io;
4use std::ops::{BitOr, BitOrAssign};
5
6#[derive(Clone, Copy, Debug, PartialEq, Eq)]
7pub struct PagePerms(u8);
8
9impl PagePerms {
10    pub const READ: Self = Self(1 << 0);
11    pub const WRITE: Self = Self(1 << 1);
12    pub const EXEC: Self = Self(1 << 2);
13    // pub const RWX: Self
14
15    pub const fn empty() -> Self {
16        Self(0)
17    }
18
19    pub const fn bits(self) -> u8 {
20        self.0
21    }
22
23    pub const fn from_bits(bits: u8) -> Self {
24        Self(bits & 0b111)
25    }
26
27    pub const fn contains(self, other: Self) -> bool {
28        (self.0 & other.0) == other.0
29    }
30}
31
32impl BitOr for PagePerms {
33    type Output = Self;
34
35    fn bitor(self, rhs: Self) -> Self::Output {
36        Self(self.0 | rhs.0)
37    }
38}
39
40impl BitOrAssign for PagePerms {
41    fn bitor_assign(&mut self, rhs: Self) {
42        self.0 |= rhs.0;
43    }
44}
45
46// rebuild ABI page table after changes
47use crate::{GuestMemoryAbi, GuestPageAbi, GuestTranslationAbi};
48
49#[derive(Clone)]
50struct OwnedPage {
51    base: u64,
52    perms: PagePerms,
53    data: Box<[u8; PAGE_SIZE]>,
54}
55
56pub struct GuestMemory {
57    pages: Vec<OwnedPage>,
58    abi_pages: Vec<GuestPageAbi>,
59    dirty_pages: Vec<usize>,
60    dirty_bitmap: Vec<u64>,
61    abi: GuestMemoryAbi,
62    abi_dirty: bool,
63    mapping_revision: u64,
64    baseline: u64,
65    pub brk: u64,
66    pub brk_floor: u64,
67    pub stack_top: u64,
68}
69
70impl Clone for GuestMemory {
71    fn clone(&self) -> Self {
72        let mut memory = Self {
73            pages: self.pages.clone(),
74            abi_pages: Vec::new(),
75            dirty_pages: Vec::new(),
76            dirty_bitmap: Vec::new(),
77
78            abi: GuestMemoryAbi {
79                pages: std::ptr::null(),
80                page_count: 0,
81                page_size: PAGE_SIZE,
82                translations: [GuestTranslationAbi::default(); crate::TRANSLATION_SLOTS],
83                dirty_pages: std::ptr::null_mut(),
84                dirty_count: 0,
85                code_dirty: false,
86                dirty_bitmap: std::ptr::null_mut(),
87            },
88
89            abi_dirty: true,
90            mapping_revision: self.mapping_revision,
91            baseline: self.baseline,
92            brk: self.brk,
93            brk_floor: self.brk_floor,
94            stack_top: self.stack_top,
95        };
96
97        memory.sync_abi();
98
99        if !self.abi_dirty {
100            memory.dirty_pages.clone_from(&self.dirty_pages);
101            memory.dirty_bitmap.clone_from(&self.dirty_bitmap);
102            memory.abi.dirty_pages = memory.dirty_pages.as_mut_ptr();
103            memory.abi.dirty_bitmap = memory.dirty_bitmap.as_mut_ptr();
104            memory.abi.dirty_count = self.abi.dirty_count;
105        }
106
107        memory.abi.code_dirty = self.abi.code_dirty;
108
109        memory
110    }
111}
112
113impl Default for GuestMemory {
114    fn default() -> Self {
115        Self::new()
116    }
117}
118
119impl GuestMemory {
120    pub fn new() -> Self {
121        Self {
122            pages: Vec::new(),
123            abi_pages: Vec::new(),
124            dirty_pages: Vec::new(),
125            dirty_bitmap: Vec::new(),
126
127            abi: GuestMemoryAbi {
128                pages: std::ptr::null(),
129                page_count: 0,
130                page_size: PAGE_SIZE,
131                translations: [GuestTranslationAbi::default(); crate::TRANSLATION_SLOTS],
132                dirty_pages: std::ptr::null_mut(),
133                dirty_count: 0,
134                code_dirty: false,
135                dirty_bitmap: std::ptr::null_mut(),
136            },
137
138            abi_dirty: true,
139            mapping_revision: 0,
140            baseline: 0,
141            brk_floor: 0,
142            brk: 0, // heap
143            stack_top: 0,
144        }
145    }
146
147    pub fn from_program(program: &ProgramImage) -> JitResult<Self> {
148        let mut memory = Self::new();
149
150        memory.map_program(program)?;
151
152        memory.map_zeroed(
153            DEFAULT_STACK_TOP - DEFAULT_STACK_SIZE,
154            DEFAULT_STACK_SIZE,
155            PagePerms::READ | PagePerms::WRITE,
156        )?;
157
158        memory.stack_top = DEFAULT_STACK_TOP;
159        memory.sync_abi();
160
161        Ok(memory)
162    }
163
164    pub fn as_abi_mut(&mut self) -> *mut GuestMemoryAbi {
165        if self.abi_dirty {
166            self.sync_abi();
167        }
168        &mut self.abi
169    }
170
171    pub fn map_program(&mut self, program: &ProgramImage) -> JitResult<()> {
172        self.map_program_at(program, 0)?;
173        self.brk = program.brk_base;
174        self.brk_floor = self.brk;
175
176        Ok(())
177    }
178
179    pub fn map_program_at(&mut self, program: &ProgramImage, bias: u64) -> JitResult<()> {
180        for segment in &program.segments {
181            let base = bias.checked_add(segment.vaddr).ok_or_else(|| io::Error::other("segment address overflow"))?;
182
183            self.map_bytes(base, &segment.bytes, segment.perms)?;
184
185            if segment.mem_size > segment.file_size {
186                self.map_zeroed(base + segment.file_size, segment.mem_size - segment.file_size, segment.perms)?;
187            }
188        }
189
190        Ok(())
191    }
192
193    pub fn map_zeroed(&mut self, base: u64, size: u64, perms: PagePerms) -> JitResult<()> {
194        if size == 0 {
195            return Ok(());
196        }
197
198        let end =
199            align_up(base.checked_add(size).ok_or_else(|| io::Error::other("mapping overflow"))?, PAGE_SIZE as u64)
200                .ok_or_else(|| io::Error::other("mapping overflow"))?;
201
202        let start = align_down(base, PAGE_SIZE as u64);
203
204        if !self.can_map(base, size) {
205            return Err(io::Error::other("guest memory limit exceeded").into());
206        }
207
208        for page_base in (start..end).step_by(PAGE_SIZE) {
209            self.ensure_page(page_base, perms);
210        }
211
212        Ok(())
213    }
214
215    pub fn map_bytes(&mut self, base: u64, bytes: &[u8], perms: PagePerms) -> JitResult<()> {
216        self.map_zeroed(base, bytes.len() as u64, perms)?;
217
218        let mut address = base;
219        let mut remaining = bytes;
220
221        while !remaining.is_empty() {
222            let page_base = align_down(address, PAGE_SIZE as u64);
223
224            let index = self.pages.binary_search_by_key(&page_base, |page| page.base).unwrap();
225
226            let page = &mut self.pages[index];
227            let offset = (address - page_base) as usize;
228            let count = remaining.len().min(PAGE_SIZE - offset);
229
230            page.data[offset..offset + count].copy_from_slice(&remaining[..count]);
231
232            self.abi.code_dirty |= page.perms.contains(PagePerms::EXEC);
233
234            if self.baseline != 0 {
235                self.mark_dirty(index);
236            }
237
238            address += count as u64;
239            remaining = &remaining[count..];
240        }
241        Ok(())
242    }
243
244    pub fn can_map(&self, base: u64, size: u64) -> bool {
245        let Some(end) = base.checked_add(size).and_then(|end| align_up(end, PAGE_SIZE as u64)) else {
246            return false;
247        };
248
249        let start = align_down(base, PAGE_SIZE as u64);
250
251        let existing = self.pages.iter().filter(|p| p.base >= start && p.base < end).count() as u64;
252
253        // todo -
254        self.pages.len() as u64 + (end - start) / PAGE_SIZE as u64 - existing
255            <= crate::MAX_GUEST_MEMORY / PAGE_SIZE as u64
256    }
257
258    pub fn replace_mapping(&mut self, base: u64, size: u64, perms: PagePerms, bytes: &[u8]) -> JitResult<()> {
259        if size == 0 || !base.is_multiple_of(PAGE_SIZE as u64) || !self.can_map(base, size) {
260            return Err(io::Error::other("invalid mapping").into());
261        }
262
263        self.unmap(base, size)?;
264        self.map_zeroed(base, size, perms)?;
265
266        self.map_bytes(base, &bytes[..bytes.len().min(size as usize)], perms)?;
267
268        Ok(())
269    }
270
271    pub fn protect(&mut self, base: u64, size: u64, perms: PagePerms) -> bool {
272        let Some((start, end)) = page_range(base, size) else {
273            return false;
274        };
275
276        let mut page_base = start;
277
278        // perf binary search
279        while page_base < end {
280            let Some(idx) = self.pages.binary_search_by_key(&page_base, |page| page.base).ok() else {
281                return false;
282            };
283
284            if self.pages[idx].perms != perms {
285                let code_changed = (self.pages[idx].perms | perms).contains(PagePerms::EXEC);
286
287                self.pages[idx].perms = perms;
288
289                self.mapping_changed(code_changed);
290            }
291            page_base += PAGE_SIZE as u64;
292        }
293
294        true
295    }
296
297    pub fn unmap(&mut self, base: u64, size: u64) -> JitResult<()> {
298        let Some((start, end)) = page_range(base, size) else {
299            return Err(io::Error::other("keine valid mapping ").into());
300        };
301
302        let code_changed =
303            self.pages.iter().any(|page| page.base >= start && page.base < end && page.perms.contains(PagePerms::EXEC));
304
305        let old_len = self.pages.len();
306        self.pages.retain(|page| page.base < start || page.base >= end);
307
308        if self.pages.len() != old_len {
309            self.mapping_changed(code_changed);
310        }
311
312        Ok(())
313    }
314
315    pub fn find_free_range(&self, start: u64, size: u64) -> Option<u64> {
316        let size = align_up(size, PAGE_SIZE as u64)?;
317        let mut base = align_up(start, PAGE_SIZE as u64)?;
318
319        for page in &self.pages {
320            if page.base < base {
321                continue;
322            }
323            if base.checked_add(size)? <= page.base {
324                return Some(base);
325            }
326            base = page.base.checked_add(PAGE_SIZE as u64)?;
327        }
328
329        base.checked_add(size)?;
330        Some(base)
331    }
332
333    pub fn range_is_free(&self, base: u64, size: u64) -> bool {
334        let Some((start, end)) = page_range(base, size) else {
335            return false;
336        };
337        self.pages.iter().all(|page| page.base < start || page.base >= end)
338    }
339
340    pub fn load_le(&self, addr: u64, size: usize) -> Option<u64> {
341        let mut value = 0u64;
342
343        for idx in 0..size {
344            let byte = self.read_byte(addr.checked_add(idx as u64)?, PagePerms::READ)?;
345            value |= (byte as u64) << (idx * 8);
346        }
347
348        Some(value)
349    }
350
351    pub fn check_range(&self, addr: u64, size: usize, perms: PagePerms) -> bool {
352        let Some(end) = addr.checked_add(size as u64) else {
353            return false;
354        };
355
356        let mut current = addr;
357
358        while current < end {
359            let Some(page) = self.find_page(current) else {
360                return false;
361            };
362
363            if !page.perms.contains(perms) {
364                return false;
365            }
366
367            current = (page.base.saturating_add(PAGE_SIZE as u64)).min(end);
368        }
369
370        true
371    }
372
373    pub(crate) fn take_code_dirty(&mut self) -> bool {
374        std::mem::take(&mut self.abi.code_dirty)
375    }
376
377    pub fn load_bytes(&self, addr: u64, size: usize) -> Option<Vec<u8>> {
378        if size as u64 > crate::MAX_GUEST_MEMORY || !self.check_range(addr, size, PagePerms::READ) {
379            return None;
380        }
381
382        let mut result = Vec::new();
383        result.try_reserve_exact(size).ok()?;
384
385        let mut current = addr;
386
387        while result.len() < size {
388            let page = self.find_page(current)?;
389            if !page.perms.contains(PagePerms::READ) {
390                return None;
391            }
392
393            let offset = (current - page.base) as usize;
394            let count = (PAGE_SIZE - offset).min(size - result.len());
395            result.extend_from_slice(&page.data[offset..offset + count]);
396            current = current.checked_add(count as u64)?;
397        }
398
399        Some(result)
400    }
401
402    pub fn fetch_u16(&self, addr: u64) -> Option<u16> {
403        let bytes = [self.read_byte(addr, PagePerms::EXEC)?, self.read_byte(addr.checked_add(1)?, PagePerms::EXEC)?];
404        Some(u16::from_le_bytes(bytes))
405    }
406
407    pub fn fetch_u32(&self, addr: u64) -> Option<u32> {
408        let bytes = [
409            self.read_byte(addr, PagePerms::EXEC)?,
410            self.read_byte(addr.checked_add(1)?, PagePerms::EXEC)?,
411            self.read_byte(addr.checked_add(2)?, PagePerms::EXEC)?,
412            self.read_byte(addr.checked_add(3)?, PagePerms::EXEC)?,
413        ];
414        Some(u32::from_le_bytes(bytes))
415    }
416
417    pub fn executable_end(&self, addr: u64) -> Option<u64> {
418        let mut page = self.find_page(addr)?;
419        if !page.perms.contains(PagePerms::EXEC) {
420            return None;
421        }
422
423        let mut end = page.base + PAGE_SIZE as u64;
424        while let Some(next) = self.find_page(end) {
425            page = next;
426            if !page.perms.contains(PagePerms::EXEC) {
427                break;
428            }
429            end += PAGE_SIZE as u64;
430        }
431        Some(end)
432    }
433
434    pub fn store_le(&mut self, addr: u64, size: usize, value: u64) -> bool {
435        for idx in 0..size {
436            let byte = ((value >> (idx * 8)) & 0xff) as u8;
437
438            let Some(byte_addr) = addr.checked_add(idx as u64) else {
439                return false;
440            };
441
442            if self.write_byte(byte_addr, byte, PagePerms::WRITE).is_none() {
443                return false;
444            }
445        }
446        true
447    }
448
449    pub fn store_bytes(&mut self, addr: u64, bytes: &[u8]) -> bool {
450        let mut written = 0;
451        let mut current = addr;
452
453        while written < bytes.len() {
454            let page_base = align_down(current, PAGE_SIZE as u64);
455            let Ok(index) = self.pages.binary_search_by_key(&page_base, |page| page.base) else {
456                return false;
457            };
458
459            if !self.pages[index].perms.contains(PagePerms::WRITE) {
460                return false;
461            }
462            let offset = (current - page_base) as usize;
463            let count = (PAGE_SIZE - offset).min(bytes.len() - written);
464            self.pages[index].data[offset..offset + count].copy_from_slice(&bytes[written..written + count]);
465
466            self.abi.code_dirty |= self.pages[index].perms.contains(PagePerms::EXEC);
467
468            self.mark_dirty(index);
469            written += count;
470
471            let Some(next) = current.checked_add(count as u64) else {
472                return false;
473            };
474            current = next;
475        }
476        true
477    }
478
479    pub(crate) fn snapshot(&mut self) -> Self {
480        // theoretisch kann ein snapshot von einem anderen run kommen
481        static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
482
483        self.baseline = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
484
485        self.clear_dirty();
486        self.clone()
487    }
488
489    pub(crate) fn clear_dirty(&mut self) {
490        self.abi.dirty_count = 0;
491        self.dirty_bitmap.fill(0);
492        self.clear_translations();
493    }
494
495    pub(crate) fn restore_dirty_from(&mut self, snapshot: &GuestMemory) -> bool {
496        if self.baseline != snapshot.baseline
497            || self.mapping_revision != snapshot.mapping_revision
498            || self.pages.len() != snapshot.pages.len()
499        {
500            return false;
501        }
502
503        for slot in 0..self.abi.dirty_count {
504            let index = self.dirty_pages[slot];
505
506            self.abi.code_dirty |= self.pages[index].perms.contains(PagePerms::EXEC);
507
508            self.pages[index].data.copy_from_slice(&snapshot.pages[index].data[..]);
509        }
510
511        self.brk = snapshot.brk;
512        self.brk_floor = snapshot.brk_floor;
513        self.stack_top = snapshot.stack_top;
514        self.clear_dirty();
515        true
516    }
517
518    pub fn can_execute(&self, addr: u64) -> bool {
519        self.find_page(addr).map(|page| page.perms.contains(PagePerms::EXEC)).unwrap_or(false)
520    }
521
522    pub fn mappings(&self) -> Vec<(u64, u64, PagePerms)> {
523        let mut mappings = Vec::new();
524        for page in &self.pages {
525            if let Some((_, end, perms)) = mappings.last_mut()
526                && *end == page.base
527                && *perms == page.perms
528            {
529                *end += PAGE_SIZE as u64;
530            } else {
531                mappings.push((page.base, page.base + PAGE_SIZE as u64, page.perms));
532            }
533        }
534        mappings
535    }
536
537    pub fn mapped_pages(&self) -> impl Iterator<Item = (u64, PagePerms, &[u8])> {
538        self.pages.iter().map(|page| (page.base, page.perms, page.data.as_slice()))
539    }
540
541    fn ensure_page(&mut self, base: u64, perms: PagePerms) -> &mut OwnedPage {
542        match self.pages.binary_search_by_key(&base, |page| page.base) {
543            Ok(idx) => {
544                if !self.pages[idx].perms.contains(perms) {
545                    self.pages[idx].perms |= perms;
546                    self.mapping_changed(self.pages[idx].perms.contains(PagePerms::EXEC));
547                }
548                &mut self.pages[idx]
549            }
550
551            Err(idx) => {
552                self.mapping_changed(perms.contains(PagePerms::EXEC));
553                self.pages.insert(idx, OwnedPage { base, perms, data: Box::new([0; PAGE_SIZE]) });
554                &mut self.pages[idx]
555            }
556        }
557    }
558
559    fn find_page(&self, addr: u64) -> Option<&OwnedPage> {
560        let page_base = align_down(addr, PAGE_SIZE as u64);
561
562        self.pages.binary_search_by_key(&page_base, |page| page.base).ok().map(|idx| &self.pages[idx])
563    }
564
565    fn read_byte(&self, addr: u64, perms: PagePerms) -> Option<u8> {
566        let page = self.find_page(addr)?;
567
568        if !page.perms.contains(perms) {
569            return None;
570        }
571
572        let offset = (addr - page.base) as usize;
573        Some(page.data[offset])
574    }
575
576    fn write_byte(&mut self, addr: u64, byte: u8, perms: PagePerms) -> Option<()> {
577        let page_base = align_down(addr, PAGE_SIZE as u64);
578
579        let index = self.pages.binary_search_by_key(&page_base, |page| page.base).ok()?;
580        let page = &mut self.pages[index];
581
582        if !page.perms.contains(perms) {
583            return None;
584        }
585
586        let offset = (addr - page.base) as usize;
587        page.data[offset] = byte;
588
589        self.abi.code_dirty |= page.perms.contains(PagePerms::EXEC);
590
591        self.mark_dirty(index);
592        Some(())
593    }
594
595    // todo -
596    fn sync_abi(&mut self) {
597        self.abi_pages = self
598            .pages
599            .iter_mut()
600            .map(|page| GuestPageAbi {
601                guest_base: page.base,
602                perms: page.perms.bits(),
603                _padding: [0; 7],
604                data: page.data.as_mut_ptr(),
605            })
606            .collect();
607
608        self.dirty_pages.resize(self.pages.len(), 0);
609        self.dirty_bitmap.resize(self.pages.len().div_ceil(64), 0);
610        self.dirty_bitmap.fill(0);
611
612        self.abi.pages = self.abi_pages.as_ptr();
613        self.abi.page_count = self.abi_pages.len();
614        self.abi.page_size = PAGE_SIZE;
615        self.abi.dirty_pages = self.dirty_pages.as_mut_ptr();
616        self.abi.dirty_count = 0;
617        self.abi.dirty_bitmap = self.dirty_bitmap.as_mut_ptr();
618        self.clear_translations();
619        self.abi_dirty = false;
620    }
621
622    fn mark_dirty(&mut self, index: usize) {
623        if self.abi_dirty {
624            self.sync_abi();
625        }
626        let word = index / 64;
627        let bit = 1u64 << (index % 64);
628        if self.dirty_bitmap[word] & bit == 0 {
629            self.dirty_bitmap[word] |= bit;
630            self.dirty_pages[self.abi.dirty_count] = index;
631            self.abi.dirty_count += 1;
632        }
633    }
634
635    fn mapping_changed(&mut self, code_changed: bool) {
636        self.abi.code_dirty |= code_changed;
637        self.abi_dirty = true;
638        self.mapping_revision = self.mapping_revision.wrapping_add(1);
639        self.clear_translations();
640    }
641
642    fn clear_translations(&mut self) {
643        self.abi.translations = [GuestTranslationAbi::default(); crate::TRANSLATION_SLOTS];
644    }
645}
646
647pub(crate) fn align_down(value: u64, align: u64) -> u64 {
648    value & !(align - 1)
649}
650
651pub(crate) fn align_up(value: u64, align: u64) -> Option<u64> {
652    value.checked_add(align - 1).map(|value| value & !(align - 1))
653}
654
655fn page_range(base: u64, size: u64) -> Option<(u64, u64)> {
656    if size == 0 || !base.is_multiple_of(PAGE_SIZE as u64) {
657        return None;
658    }
659
660    let end = align_up(base.checked_add(size)?, PAGE_SIZE as u64)?;
661    Some((base, end))
662}