Skip to main content

jit/
runtime.rs

1use crate::mmu::{align_down, align_up};
2use crate::process::{Process, ProcessSnapshot, guest_errno, host_errno, open_beneath};
3use crate::remi::compile_jit_multi;
4use crate::*;
5
6use liil::{blocks::direct_successors, nnil::Nnil};
7
8use std::{
9    collections::{BTreeMap, BTreeSet, VecDeque},
10    fs, io,
11    os::unix::fs::FileExt as _,
12    path::{Path, PathBuf},
13    sync::{
14        Arc,
15        atomic::{AtomicBool, Ordering},
16    },
17    time::{Duration, Instant},
18};
19
20#[path = "setup.rs"]
21mod setup;
22
23#[path = "syscall.rs"]
24mod syscall;
25
26type AuxEntry = (u64, u64);
27
28#[derive(Debug, Clone)]
29pub struct RemuConfig {
30    pub argv: Vec<String>,
31    pub envp: Vec<String>,
32    pub sysroot: Option<PathBuf>,
33    pub workdir: Option<PathBuf>,
34    pub jit_opt_level: u8, // 0, 1, 2
35    pub jit_cache_dir: Option<PathBuf>,
36    pub raw_jit: bool,
37}
38
39impl Default for RemuConfig {
40    fn default() -> Self {
41        Self {
42            argv: Vec::new(),
43            envp: Vec::new(),
44            sysroot: None,
45            workdir: None,
46            jit_opt_level: 2,
47            jit_cache_dir: Some(PathBuf::from("jit-tmp/cache")),
48            raw_jit: false,
49        }
50    }
51}
52
53pub struct RuntimeSnapshot {
54    memory: GuestMemory,
55    state: GuestState,
56    host_process: ProcessSnapshot,
57    mmap_base: u64,
58}
59
60pub struct JitRuntime<L>
61where
62    L: Fn(&GuestMemory, u64) -> JitResult<Nnil>,
63{
64    program: ProgramImage,
65    process: RemuConfig,
66    memory: Box<GuestMemory>,
67
68    cache: BTreeMap<u64, CompiledJit>,
69    jit_entries: Vec<JitEntryAbi>,
70
71    initial_pc: u64,
72    loader_base: u64,
73    state: GuestState,
74    host_process: Process,
75    mmap_base: u64,
76
77    jit_output_dir: PathBuf,
78    verbose: bool,
79
80    compile_count: usize,
81    compile_time: Duration,
82    execute_time: Duration,
83
84    control: Box<GuestControl>,
85    breakpoints: Vec<u64>,
86    pause: Arc<AtomicBool>,
87    initial_snapshot: Option<RuntimeSnapshot>,
88
89    last_syscall: Option<[u64; 8]>,
90    lift_region: L,
91}
92
93impl<L> JitRuntime<L>
94where
95    L: Fn(&GuestMemory, u64) -> JitResult<Nnil>,
96{
97    pub fn new(program: ProgramImage, lift_region: L) -> JitResult<Self> {
98        Self::init_runtime(program, RemuConfig::default(), Path::new("output").to_path_buf(), lift_region)
99    }
100
101    pub fn cache_len(&self) -> usize {
102        self.cache.len()
103    }
104
105    pub fn program(&self) -> &ProgramImage {
106        &self.program
107    }
108
109    pub fn state(&self) -> &GuestState {
110        &self.state
111    }
112
113    pub fn state_mut(&mut self) -> &mut GuestState {
114        &mut self.state
115    }
116
117    pub fn memory(&self) -> &GuestMemory {
118        &self.memory
119    }
120
121    pub fn memory_mut(&mut self) -> &mut GuestMemory {
122        self.state.memory = self.memory.as_abi_mut();
123        &mut self.memory
124    }
125
126    pub fn set_entry_pc(&mut self, pc: u64) {
127        self.initial_pc = pc;
128        self.state.pc = pc;
129    }
130
131    pub fn set_verbose(&mut self, verbose: bool) {
132        self.verbose = verbose;
133    }
134
135    pub fn process_config(&self) -> &RemuConfig {
136        &self.process
137    }
138
139    pub fn pause_flag(&self) -> Arc<AtomicBool> {
140        Arc::clone(&self.pause)
141    }
142
143    pub fn add_breakpoint(&mut self, address: u64) {
144        if let Err(index) = self.breakpoints.binary_search(&address) {
145            self.breakpoints.insert(index, address);
146        }
147    }
148
149    pub fn remove_breakpoint(&mut self, address: u64) {
150        self.breakpoints.retain(|value| *value != address);
151    }
152
153    pub fn clear_breakpoints(&mut self) {
154        self.breakpoints.clear();
155    }
156
157    pub fn breakpoints(&self) -> &[u64] {
158        &self.breakpoints
159    }
160
161    pub fn trace(&self) -> Vec<u64> {
162        let count = self.control.trace_pos.min(DEBUG_SLOTS);
163        let start = self.control.trace_pos.saturating_sub(count);
164
165        (start..self.control.trace_pos).map(|index| self.control.trace[index % DEBUG_SLOTS]).collect()
166    }
167
168    pub fn last_syscall(&self) -> Option<[u64; 8]> {
169        self.last_syscall
170    }
171
172    pub fn lift_at(&self, pc: u64) -> JitResult<Nnil> {
173        (self.lift_region)(&self.memory, pc)
174    }
175
176    pub fn precompile(&mut self) -> JitResult<usize> {
177        if self.memory.take_code_dirty() {
178            self.invalidate_cache();
179        }
180
181        let mut pc = self.initial_pc;
182
183        let end = self.memory.executable_end(pc).ok_or_else(|| io::Error::other("entry is not executable?"))?;
184
185        let mut regions = Vec::new();
186
187        while pc < end {
188            let nnil = match (self.lift_region)(&self.memory, pc) {
189                Ok(nnil) => nnil,
190                Err(error) => {
191                    if self.verbose {
192                        eprintln!("precompile stopped at 0x{pc:x}: {error}");
193                    }
194
195                    break;
196                }
197            };
198
199            if nnil.pc() <= pc {
200                break;
201            }
202
203            pc = nnil.pc();
204            regions.push(nnil);
205        }
206
207        let compiled = match self.compile_regions(&regions) {
208            Ok(compiled) => compiled,
209            Err(error) => {
210                if self.verbose {
211                    eprintln!("precompile failed: {error}");
212                }
213
214                return Ok(0);
215            }
216        };
217
218        let count = compiled.len();
219        self.cache.extend(compiled);
220
221        self.sync_jit_entries();
222        Ok(count)
223    }
224
225    pub fn run(&mut self) -> JitResult<ExitReason> {
226        self.state.control = std::ptr::null_mut();
227        self.run_inner()
228    }
229
230    pub fn run_for_blocks(&mut self, limit: u64) -> JitResult<ExitReason> {
231        self.control.breakpoints = self.breakpoints.as_ptr();
232        self.control.breakpoint_count = self.breakpoints.len();
233        self.control.blocks_left = limit;
234        self.control.limited = 1;
235
236        self.control.skip_breakpoint =
237            if self.state.exit_reason == ExitReason::Breakpoint { self.state.pc } else { u64::MAX };
238
239        self.state.control = &mut *self.control;
240
241        let result = self.run_inner();
242
243        self.state.control = std::ptr::null_mut();
244        self.control.limited = 0;
245
246        self.pause.store(false, Ordering::Relaxed);
247        result
248    }
249
250    fn run_inner(&mut self) -> JitResult<ExitReason> {
251        if self.state.pc == 0 {
252            self.state.pc = self.initial_pc;
253        }
254
255        loop {
256            self.state.memory = self.memory.as_abi_mut();
257
258            if self.memory.take_code_dirty() {
259                self.invalidate_cache();
260            }
261
262            self.state.exit_reason = ExitReason::None;
263
264            self.state.fault_addr = 0;
265            self.state.fault_access = AccessKind::None;
266
267            if self.verbose {
268                println!("neuer pc=0x{:x}", self.state.pc)
269            }
270
271            if !self.memory.can_execute(self.state.pc) {
272                self.state.exit_reason = ExitReason::ExecFault;
273                self.state.fault_addr = self.state.pc;
274                self.state.fault_access = AccessKind::Execute;
275
276                return Ok(self.finished(ExitReason::ExecFault));
277            }
278
279            if !self.cache.contains_key(&self.state.pc) {
280                self.compile_region(self.state.pc)?;
281            } else if self.verbose {
282                println!("jit-cache-hit für pc=0x{:x}", self.state.pc);
283            }
284
285            let compiled = self.cache.get(&self.state.pc).ok_or_else(|| io::Error::other("cache fehler"))?;
286
287            let exit_reason = if self.verbose {
288                let started = Instant::now();
289                let reason = unsafe { compiled.run(&mut self.state) };
290
291                self.execute_time += started.elapsed();
292                reason
293            } else {
294                unsafe { compiled.run(&mut self.state) }
295            };
296
297            match exit_reason {
298                ExitReason::IndirectBranch => continue,
299                ExitReason::Syscall => {
300                    if self.verbose {
301                        self.log_syscall();
302                    }
303
304                    if self.handle_syscall() {
305                        continue;
306                    }
307
308                    if self.state.exit_reason == ExitReason::GuestExit {
309                        return Ok(self.finished(ExitReason::GuestExit));
310                    }
311
312                    return Ok(self.finished(ExitReason::Syscall));
313                }
314
315                other => return Ok(self.finished(other)),
316            }
317        }
318    }
319
320    fn finished(&self, reason: ExitReason) -> ExitReason {
321        if self.verbose {
322            println!(
323                "jit stats: {} rustc calls, {:.2?} compiling, {:.2?} executing",
324                self.compile_count, self.compile_time, self.execute_time
325            );
326        }
327
328        reason
329    }
330
331    fn compile_region(&mut self, pc: u64) -> JitResult<()> {
332        let regions = self.collect_lazy_batch(pc, 128)?;
333
334        let compiled = match self.compile_regions(&regions) {
335            Ok(compiled) => compiled,
336            Err(error) if regions.len() > 1 => {
337                if self.verbose {
338                    eprintln!("batch at 0x{pc:x} failed: {error}; compiling requested region");
339                }
340
341                self.compile_regions(&regions[..1])?
342            }
343            Err(error) => return Err(error),
344        };
345
346        self.cache.extend(compiled);
347
348        self.sync_jit_entries();
349        Ok(())
350    }
351
352    fn compile_regions(&mut self, regions: &[Nnil]) -> JitResult<Vec<(u64, CompiledJit)>> {
353        let started = Instant::now();
354
355        let compiled = compile_jit_multi(
356            regions,
357            &self.jit_output_dir,
358            self.process.jit_opt_level,
359            self.process.jit_cache_dir.as_deref(),
360            self.process.raw_jit,
361        )?;
362
363        if let Some((_, first)) = compiled.first() {
364            if first.compiled_now() {
365                self.compile_count += 1;
366                self.compile_time += started.elapsed();
367            }
368
369            if self.verbose {
370                println!(
371                    "JIT: {} regions, compiled={}, {}",
372                    compiled.len(),
373                    first.compiled_now(),
374                    first.source_path().display()
375                );
376            }
377        }
378        Ok(compiled)
379    }
380
381    fn collect_lazy_batch(&self, start_pc: u64, max_regions: usize) -> JitResult<Vec<Nnil>> {
382        self.memory
383            .executable_end(start_pc)
384            .ok_or_else(|| io::Error::other(format!("entry pc 0x{start_pc:x} is not executable")))?;
385
386        let mut queue = VecDeque::from([start_pc]);
387        let mut seen = BTreeSet::new();
388        let mut regions = Vec::new();
389
390        while let Some(pc) = queue.pop_front()
391            && regions.len() < max_regions
392        {
393            if !seen.insert(pc) || !self.memory.can_execute(pc) {
394                continue;
395            }
396
397            let nnil = match (self.lift_region)(&self.memory, pc) {
398                Ok(nnil) => nnil,
399                Err(_) if pc != start_pc => continue,
400                Err(error) => return Err(error),
401            };
402
403            let next_pc = nnil.pc();
404
405            if next_pc <= pc {
406                return Err(
407                    io::Error::other(format!("compiling pc 0x{pc:x} kein progress (next pc 0x{next_pc:x})")).into()
408                );
409            }
410
411            for successor in direct_successors(&nnil) {
412                if !seen.contains(&successor) {
413                    queue.push_back(successor);
414                }
415            }
416
417            if !self.cache.contains_key(&pc) {
418                regions.push(nnil);
419            }
420        }
421
422        Ok(regions)
423    }
424
425    fn invalidate_cache(&mut self) {
426        self.cache.clear();
427        self.sync_jit_entries();
428    }
429
430    fn sync_jit_entries(&mut self) {
431        self.jit_entries = self
432            .cache
433            .iter()
434            .map(|(&guest_pc, compiled)| JitEntryAbi { guest_pc, execute: compiled.entry() })
435            .collect();
436
437        self.state.jit_entries = self.jit_entries.as_ptr();
438        self.state.jit_entry_count = self.jit_entries.len();
439    }
440}