1use crate::mmu::{align_down, align_up};
2use crate::process::{Process, ProcessSnapshot, guest_errno, host_errno, open_beneath};
3use crate::remi::compile_jit_multi;
4use crate::*;
5
6use liil::{blocks::direct_successors, nnil::Nnil};
7
8use std::{
9 collections::{BTreeMap, BTreeSet, VecDeque},
10 fs, io,
11 os::unix::fs::FileExt as _,
12 path::{Path, PathBuf},
13 sync::{
14 Arc,
15 atomic::{AtomicBool, Ordering},
16 },
17 time::{Duration, Instant},
18};
19
20#[path = "setup.rs"]
21mod setup;
22
23#[path = "syscall.rs"]
24mod syscall;
25
26type AuxEntry = (u64, u64);
27
28#[derive(Debug, Clone)]
29pub struct RemuConfig {
30 pub argv: Vec<String>,
31 pub envp: Vec<String>,
32 pub sysroot: Option<PathBuf>,
33 pub workdir: Option<PathBuf>,
34 pub jit_opt_level: u8, pub jit_cache_dir: Option<PathBuf>,
36 pub raw_jit: bool,
37}
38
39impl Default for RemuConfig {
40 fn default() -> Self {
41 Self {
42 argv: Vec::new(),
43 envp: Vec::new(),
44 sysroot: None,
45 workdir: None,
46 jit_opt_level: 2,
47 jit_cache_dir: Some(PathBuf::from("jit-tmp/cache")),
48 raw_jit: false,
49 }
50 }
51}
52
53pub struct RuntimeSnapshot {
54 memory: GuestMemory,
55 state: GuestState,
56 host_process: ProcessSnapshot,
57 mmap_base: u64,
58}
59
60pub struct JitRuntime<L>
61where
62 L: Fn(&GuestMemory, u64) -> JitResult<Nnil>,
63{
64 program: ProgramImage,
65 process: RemuConfig,
66 memory: Box<GuestMemory>,
67
68 cache: BTreeMap<u64, CompiledJit>,
69 jit_entries: Vec<JitEntryAbi>,
70
71 initial_pc: u64,
72 loader_base: u64,
73 state: GuestState,
74 host_process: Process,
75 mmap_base: u64,
76
77 jit_output_dir: PathBuf,
78 verbose: bool,
79
80 compile_count: usize,
81 compile_time: Duration,
82 execute_time: Duration,
83
84 control: Box<GuestControl>,
85 breakpoints: Vec<u64>,
86 pause: Arc<AtomicBool>,
87 initial_snapshot: Option<RuntimeSnapshot>,
88
89 last_syscall: Option<[u64; 8]>,
90 lift_region: L,
91}
92
93impl<L> JitRuntime<L>
94where
95 L: Fn(&GuestMemory, u64) -> JitResult<Nnil>,
96{
97 pub fn new(program: ProgramImage, lift_region: L) -> JitResult<Self> {
98 Self::init_runtime(program, RemuConfig::default(), Path::new("output").to_path_buf(), lift_region)
99 }
100
101 pub fn cache_len(&self) -> usize {
102 self.cache.len()
103 }
104
105 pub fn program(&self) -> &ProgramImage {
106 &self.program
107 }
108
109 pub fn state(&self) -> &GuestState {
110 &self.state
111 }
112
113 pub fn state_mut(&mut self) -> &mut GuestState {
114 &mut self.state
115 }
116
117 pub fn memory(&self) -> &GuestMemory {
118 &self.memory
119 }
120
121 pub fn memory_mut(&mut self) -> &mut GuestMemory {
122 self.state.memory = self.memory.as_abi_mut();
123 &mut self.memory
124 }
125
126 pub fn set_entry_pc(&mut self, pc: u64) {
127 self.initial_pc = pc;
128 self.state.pc = pc;
129 }
130
131 pub fn set_verbose(&mut self, verbose: bool) {
132 self.verbose = verbose;
133 }
134
135 pub fn process_config(&self) -> &RemuConfig {
136 &self.process
137 }
138
139 pub fn pause_flag(&self) -> Arc<AtomicBool> {
140 Arc::clone(&self.pause)
141 }
142
143 pub fn add_breakpoint(&mut self, address: u64) {
144 if let Err(index) = self.breakpoints.binary_search(&address) {
145 self.breakpoints.insert(index, address);
146 }
147 }
148
149 pub fn remove_breakpoint(&mut self, address: u64) {
150 self.breakpoints.retain(|value| *value != address);
151 }
152
153 pub fn clear_breakpoints(&mut self) {
154 self.breakpoints.clear();
155 }
156
157 pub fn breakpoints(&self) -> &[u64] {
158 &self.breakpoints
159 }
160
161 pub fn trace(&self) -> Vec<u64> {
162 let count = self.control.trace_pos.min(DEBUG_SLOTS);
163 let start = self.control.trace_pos.saturating_sub(count);
164
165 (start..self.control.trace_pos).map(|index| self.control.trace[index % DEBUG_SLOTS]).collect()
166 }
167
168 pub fn last_syscall(&self) -> Option<[u64; 8]> {
169 self.last_syscall
170 }
171
172 pub fn lift_at(&self, pc: u64) -> JitResult<Nnil> {
173 (self.lift_region)(&self.memory, pc)
174 }
175
176 pub fn precompile(&mut self) -> JitResult<usize> {
177 if self.memory.take_code_dirty() {
178 self.invalidate_cache();
179 }
180
181 let mut pc = self.initial_pc;
182
183 let end = self.memory.executable_end(pc).ok_or_else(|| io::Error::other("entry is not executable?"))?;
184
185 let mut regions = Vec::new();
186
187 while pc < end {
188 let nnil = match (self.lift_region)(&self.memory, pc) {
189 Ok(nnil) => nnil,
190 Err(error) => {
191 if self.verbose {
192 eprintln!("precompile stopped at 0x{pc:x}: {error}");
193 }
194
195 break;
196 }
197 };
198
199 if nnil.pc() <= pc {
200 break;
201 }
202
203 pc = nnil.pc();
204 regions.push(nnil);
205 }
206
207 let compiled = match self.compile_regions(®ions) {
208 Ok(compiled) => compiled,
209 Err(error) => {
210 if self.verbose {
211 eprintln!("precompile failed: {error}");
212 }
213
214 return Ok(0);
215 }
216 };
217
218 let count = compiled.len();
219 self.cache.extend(compiled);
220
221 self.sync_jit_entries();
222 Ok(count)
223 }
224
225 pub fn run(&mut self) -> JitResult<ExitReason> {
226 self.state.control = std::ptr::null_mut();
227 self.run_inner()
228 }
229
230 pub fn run_for_blocks(&mut self, limit: u64) -> JitResult<ExitReason> {
231 self.control.breakpoints = self.breakpoints.as_ptr();
232 self.control.breakpoint_count = self.breakpoints.len();
233 self.control.blocks_left = limit;
234 self.control.limited = 1;
235
236 self.control.skip_breakpoint =
237 if self.state.exit_reason == ExitReason::Breakpoint { self.state.pc } else { u64::MAX };
238
239 self.state.control = &mut *self.control;
240
241 let result = self.run_inner();
242
243 self.state.control = std::ptr::null_mut();
244 self.control.limited = 0;
245
246 self.pause.store(false, Ordering::Relaxed);
247 result
248 }
249
250 fn run_inner(&mut self) -> JitResult<ExitReason> {
251 if self.state.pc == 0 {
252 self.state.pc = self.initial_pc;
253 }
254
255 loop {
256 self.state.memory = self.memory.as_abi_mut();
257
258 if self.memory.take_code_dirty() {
259 self.invalidate_cache();
260 }
261
262 self.state.exit_reason = ExitReason::None;
263
264 self.state.fault_addr = 0;
265 self.state.fault_access = AccessKind::None;
266
267 if self.verbose {
268 println!("neuer pc=0x{:x}", self.state.pc)
269 }
270
271 if !self.memory.can_execute(self.state.pc) {
272 self.state.exit_reason = ExitReason::ExecFault;
273 self.state.fault_addr = self.state.pc;
274 self.state.fault_access = AccessKind::Execute;
275
276 return Ok(self.finished(ExitReason::ExecFault));
277 }
278
279 if !self.cache.contains_key(&self.state.pc) {
280 self.compile_region(self.state.pc)?;
281 } else if self.verbose {
282 println!("jit-cache-hit für pc=0x{:x}", self.state.pc);
283 }
284
285 let compiled = self.cache.get(&self.state.pc).ok_or_else(|| io::Error::other("cache fehler"))?;
286
287 let exit_reason = if self.verbose {
288 let started = Instant::now();
289 let reason = unsafe { compiled.run(&mut self.state) };
290
291 self.execute_time += started.elapsed();
292 reason
293 } else {
294 unsafe { compiled.run(&mut self.state) }
295 };
296
297 match exit_reason {
298 ExitReason::IndirectBranch => continue,
299 ExitReason::Syscall => {
300 if self.verbose {
301 self.log_syscall();
302 }
303
304 if self.handle_syscall() {
305 continue;
306 }
307
308 if self.state.exit_reason == ExitReason::GuestExit {
309 return Ok(self.finished(ExitReason::GuestExit));
310 }
311
312 return Ok(self.finished(ExitReason::Syscall));
313 }
314
315 other => return Ok(self.finished(other)),
316 }
317 }
318 }
319
320 fn finished(&self, reason: ExitReason) -> ExitReason {
321 if self.verbose {
322 println!(
323 "jit stats: {} rustc calls, {:.2?} compiling, {:.2?} executing",
324 self.compile_count, self.compile_time, self.execute_time
325 );
326 }
327
328 reason
329 }
330
331 fn compile_region(&mut self, pc: u64) -> JitResult<()> {
332 let regions = self.collect_lazy_batch(pc, 128)?;
333
334 let compiled = match self.compile_regions(®ions) {
335 Ok(compiled) => compiled,
336 Err(error) if regions.len() > 1 => {
337 if self.verbose {
338 eprintln!("batch at 0x{pc:x} failed: {error}; compiling requested region");
339 }
340
341 self.compile_regions(®ions[..1])?
342 }
343 Err(error) => return Err(error),
344 };
345
346 self.cache.extend(compiled);
347
348 self.sync_jit_entries();
349 Ok(())
350 }
351
352 fn compile_regions(&mut self, regions: &[Nnil]) -> JitResult<Vec<(u64, CompiledJit)>> {
353 let started = Instant::now();
354
355 let compiled = compile_jit_multi(
356 regions,
357 &self.jit_output_dir,
358 self.process.jit_opt_level,
359 self.process.jit_cache_dir.as_deref(),
360 self.process.raw_jit,
361 )?;
362
363 if let Some((_, first)) = compiled.first() {
364 if first.compiled_now() {
365 self.compile_count += 1;
366 self.compile_time += started.elapsed();
367 }
368
369 if self.verbose {
370 println!(
371 "JIT: {} regions, compiled={}, {}",
372 compiled.len(),
373 first.compiled_now(),
374 first.source_path().display()
375 );
376 }
377 }
378 Ok(compiled)
379 }
380
381 fn collect_lazy_batch(&self, start_pc: u64, max_regions: usize) -> JitResult<Vec<Nnil>> {
382 self.memory
383 .executable_end(start_pc)
384 .ok_or_else(|| io::Error::other(format!("entry pc 0x{start_pc:x} is not executable")))?;
385
386 let mut queue = VecDeque::from([start_pc]);
387 let mut seen = BTreeSet::new();
388 let mut regions = Vec::new();
389
390 while let Some(pc) = queue.pop_front()
391 && regions.len() < max_regions
392 {
393 if !seen.insert(pc) || !self.memory.can_execute(pc) {
394 continue;
395 }
396
397 let nnil = match (self.lift_region)(&self.memory, pc) {
398 Ok(nnil) => nnil,
399 Err(_) if pc != start_pc => continue,
400 Err(error) => return Err(error),
401 };
402
403 let next_pc = nnil.pc();
404
405 if next_pc <= pc {
406 return Err(
407 io::Error::other(format!("compiling pc 0x{pc:x} kein progress (next pc 0x{next_pc:x})")).into()
408 );
409 }
410
411 for successor in direct_successors(&nnil) {
412 if !seen.contains(&successor) {
413 queue.push_back(successor);
414 }
415 }
416
417 if !self.cache.contains_key(&pc) {
418 regions.push(nnil);
419 }
420 }
421
422 Ok(regions)
423 }
424
425 fn invalidate_cache(&mut self) {
426 self.cache.clear();
427 self.sync_jit_entries();
428 }
429
430 fn sync_jit_entries(&mut self) {
431 self.jit_entries = self
432 .cache
433 .iter()
434 .map(|(&guest_pc, compiled)| JitEntryAbi { guest_pc, execute: compiled.entry() })
435 .collect();
436
437 self.state.jit_entries = self.jit_entries.as_ptr();
438 self.state.jit_entry_count = self.jit_entries.len();
439 }
440}