1use crate::{
4 DEBUG_SLOTS, ExecuteFn, ExitReason, FP_REG_BASE, GUEST_REG_COUNT, GUEST_TMP_COUNT, GuestState,
5 INTEGER_ZERO_REG, JitResult, PAGE_SIZE,
6};
7
8use liil::nnil::{Address, AmoOp, Insn, Label, Nnil, NnilInstruction};
9
10use std::collections::{BTreeMap, BTreeSet};
11use std::fmt::Write as _;
12use std::fs;
13use std::io;
14use std::path::{Path, PathBuf};
15use std::process::Command;
16use std::sync::{Arc, OnceLock};
17use std::time::{SystemTime, UNIX_EPOCH};
18
19use libloading::Library;
22
23enum LoadedJitCode {
34 Shared(Library),
36 #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
37 Raw(RawCode),
39}
40
41struct LoadedJitArtifact {
43 code: LoadedJitCode,
45 source_path: PathBuf,
47 library_path: PathBuf,
48 compiled_now: bool,
50}
51
52pub struct CompiledJit {
54 artifact: Arc<LoadedJitArtifact>,
55 execute: ExecuteFn,
56}
57
58impl CompiledJit {
59 pub fn source_path(&self) -> &Path {
61 &self.artifact.source_path
62 }
63
64 pub fn library_path(&self) -> &Path {
66 &self.artifact.library_path
67 }
68
69 pub(crate) fn compiled_now(&self) -> bool {
70 self.artifact.compiled_now
71 }
72
73 pub(crate) fn entry(&self) -> ExecuteFn {
74 self.execute
75 }
76
77 pub(crate) unsafe fn run(&self, state: &mut GuestState) -> ExitReason {
78 unsafe { (self.execute)(state as *mut GuestState) }
80 }
81}
82
83#[derive(Debug, Clone, Copy)]
84struct BasicBlock {
85 addr: Address,
87 start_insn: usize,
89 end_insn: usize,
90 fallthrough: Option<Address>,
92 next_pc: u64,
94}
95
96#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98enum LoadMode {
99 Raw,
100 ZeroExtend,
101 SignExtend,
102}
103
104#[derive(Debug, Clone, Copy, PartialEq, Eq)]
105enum Signedness {
106 Signed,
108 Unsigned,
109}
110
111struct RustEmitter<'a> {
112 nnil: &'a Nnil,
114 blocks: Vec<BasicBlock>,
115 block_names: BTreeMap<Address, String>,
117 unsigned_markers: BTreeSet<u64>,
119}
120
121pub fn compile_jit(
123 nnil: &Nnil,
124 output_dir: &Path,
125 opt_level: u8,
126 cache_dir: Option<&Path>,
127 raw_jit: bool,
128) -> JitResult<CompiledJit> {
129 let rust_code = generate_rust_code(nnil)?;
131 let artifact = compile_generated_artifact(
132 rust_code,
133 output_dir,
134 "generated",
135 opt_level,
136 cache_dir,
137 raw_jit,
138 )?;
139 compiled_symbol_from_artifact(&artifact, b"execute")
140}
141
142pub fn compile_jit_multi(
145 regions: &[Nnil],
146 output_dir: &Path,
147 artifact_prefix: &str,
148 opt_level: u8,
149 cache_dir: Option<&Path>,
150 raw_jit: bool,
151) -> JitResult<Vec<(u64, CompiledJit)>> {
152 if regions.is_empty() {
154 return Ok(Vec::new());
155 }
156
157 let (rust_code, exports) = generate_bundle_rust_code(regions)?;
158 let artifact = compile_generated_artifact(
159 rust_code,
160 output_dir,
161 artifact_prefix,
162 opt_level,
163 cache_dir,
164 raw_jit,
165 )?;
166
167 let mut compiled = Vec::with_capacity(exports.len());
169
170 for (entry_pc, export_name) in exports {
171 let jit = compiled_symbol_from_artifact(&artifact, export_name.as_bytes())?;
172 compiled.push((entry_pc, jit));
173 }
174
175 Ok(compiled)
176}
177
178fn compile_generated_artifact(
179 rust_code: String,
180 output_dir: &Path,
181 artifact_prefix: &str,
182 opt_level: u8,
183 cache_dir: Option<&Path>,
184 raw_jit: bool,
185) -> JitResult<Arc<LoadedJitArtifact>> {
186 if raw_jit {
188 return compile_raw_artifact(rust_code, output_dir, artifact_prefix, opt_level, cache_dir);
189 }
190
191
192 let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
195 let (source_path, library_path, compile_source, compile_library) =
196 if let Some(cache_dir) = cache_dir {
197 fs::create_dir_all(cache_dir)?;
198 let key = cache_key(&rust_code, opt_level, false)?;
199 let source_path = cache_dir.join(format!("jit-{key}.rs"));
200 let library_path = cache_dir.join(format!("libjit-{key}.so"));
201
202 if library_path.is_file()
205 && fs::read_to_string(&source_path).ok().as_deref() == Some(rust_code.as_str())
206 && let Ok(library) = unsafe { Library::new(&library_path) }
207 {
208 return Ok(Arc::new(LoadedJitArtifact {
209 code: LoadedJitCode::Shared(library),
210 source_path,
211 library_path,
212 compiled_now: false,
213 }));
214 }
215
216 let unique = format!("{}-{timestamp}", std::process::id());
218 (
219 source_path,
220 library_path,
221 cache_dir.join(format!("jit-temp-{unique}.rs")),
222 cache_dir.join(format!("libjit-temp-{unique}.so")),
223 )
224 } else {
225 fs::create_dir_all(output_dir)?;
226 let source_path = output_dir.join(format!("{artifact_prefix}-{timestamp}.rs"));
227 let library_path = output_dir.join(format!("lib{artifact_prefix}-{timestamp}.so"));
228 (
229 source_path.clone(),
230 library_path.clone(),
231 source_path,
232 library_path,
233 )
234 };
235
236 fs::write(&compile_source, rust_code)?;
238
239 let mut command = Command::new("rustc");
241 command
246 .args(["--edition=2021", "--crate-type", "cdylib", "-C"])
247 .arg(format!("opt-level={opt_level}"))
248 .args(["-C", "codegen-units=1", "-C", "panic=abort"]); if cfg!(target_os = "macos") {
251 command.args(["-C", "link-arg=-lSystem"]);
253 }
254 if cfg!(target_os = "linux") {
255 command.args(["-C", "link-arg=-Wl,-z,defs"]);
258 }
259
260 let status = command
261 .arg("-o")
262 .arg(&compile_library)
263 .arg(&compile_source)
264 .status()?;
265
266 if !status.success() {
267 if cache_dir.is_some() {
268 let _ = fs::remove_file(&compile_source);
269 let _ = fs::remove_file(&compile_library);
270 }
271 return Err(io::Error::other("rustc failed to generate JIT").into());
272 }
273
274 if cache_dir.is_some() {
275 fs::rename(&compile_source, &source_path)?;
278 fs::rename(&compile_library, &library_path)?;
279 }
280
281 let library = unsafe { Library::new(&library_path)? };
283 Ok(Arc::new(LoadedJitArtifact {
284 code: LoadedJitCode::Shared(library),
285 source_path,
286 library_path,
287 compiled_now: true,
288 }))
289}
290
291#[cfg(not(all(target_os = "linux", target_arch = "x86_64")))]
292fn compile_raw_artifact(
293 _rust_code: String,
294 _output_dir: &Path,
295 _artifact_prefix: &str,
296 _opt_level: u8,
297 _cache_dir: Option<&Path>,
298) -> JitResult<Arc<LoadedJitArtifact>> {
299 Err(io::Error::other("raw JIT nur auf Linux x86-64").into())
302}
303
304#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
305fn compile_raw_artifact(
306 rust_code: String,
307 output_dir: &Path,
308 artifact_prefix: &str,
309 opt_level: u8,
310 cache_dir: Option<&Path>,
311) -> JitResult<Arc<LoadedJitArtifact>> {
312
313 let timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
315 let (source_path, artifact_path, compile_source, compile_artifact) =
316 if let Some(cache_dir) = cache_dir {
317 fs::create_dir_all(cache_dir)?;
318 let key = cache_key(&rust_code, opt_level, true)?;
319 let source_path = cache_dir.join(format!("jit-{key}.rs"));
320 let artifact_path = cache_dir.join(format!("jit-{key}.raw"));
321
322 if artifact_path.is_file()
325 && fs::read_to_string(&source_path).ok().as_deref() == Some(rust_code.as_str())
326 && let Ok(bytes) = fs::read(&artifact_path)
327 && let Ok((image, symbols)) = read_raw_cache(&bytes)
328 && let Ok(code) = RawCode::new(&image, symbols)
329 {
330 return Ok(Arc::new(LoadedJitArtifact {
331 code: LoadedJitCode::Raw(code),
332 source_path,
333 library_path: artifact_path,
334 compiled_now: false,
335 }));
336 }
337
338 let unique = format!("{}-{timestamp}", std::process::id());
340 (
341 source_path,
342 artifact_path,
343 cache_dir.join(format!("jit-temp-{unique}.rs")),
344 cache_dir.join(format!("jit-temp-{unique}.raw")),
345 )
346 } else {
347 fs::create_dir_all(output_dir)?;
348 let source_path = output_dir.join(format!("{artifact_prefix}-{timestamp}.rs"));
349 let artifact_path = output_dir.join(format!("{artifact_prefix}-{timestamp}.raw"));
350 (
351 source_path.clone(),
352 artifact_path.clone(),
353 source_path,
354 artifact_path,
355 )
356 };
357
358 let object_path = compile_source.with_extension("o");
361 let elf_path = compile_source.with_extension("elf");
362 let script_path = compile_source.with_extension("ld");
363 fs::write(&compile_source, rust_code)?;
364 fs::write(&script_path, RAW_LINKER_SCRIPT)?;
365
366 let rustc_status = Command::new("rustc")
372 .args(["--edition=2021", "--crate-type", "lib", "--emit=obj", "-C"])
373 .arg(format!("opt-level={opt_level}"))
374 .args([
375 "-C",
376 "codegen-units=1",
377 "-C",
378 "panic=abort",
379 "-C",
380 "relocation-model=pic",
381 "-o",
382 ])
383 .arg(&object_path)
384 .arg(&compile_source)
385 .status()?;
386
387
388 if !rustc_status.success() {
389 clean_raw_temps(
390 &compile_source,
391 &object_path,
392 &elf_path,
393 &script_path,
394 cache_dir,
395 );
396 return Err(io::Error::other("rustc failed to generate raw JIT").into());
397 }
398
399 let ld_status = Command::new("ld")
403 .args(["-static", "-nostdlib", "--build-id=none", "-T"])
404 .arg(&script_path)
405 .arg("-o")
406 .arg(&elf_path)
407 .arg(&object_path)
408 .status()?;
409
410 if !ld_status.success() {
411 clean_raw_temps(
412 &compile_source,
413 &object_path,
414 &elf_path,
415 &script_path,
416 cache_dir,
417 );
418 return Err(io::Error::other("GNU ld failed to link raw JIT object").into());
419 }
420
421 let (image, symbols) = parse_raw_elf(&fs::read(&elf_path)?)?;
424 fs::write(&compile_artifact, write_raw_cache(&image, &symbols))?;
425 let code = RawCode::new(&image, symbols)?;
426
427 let _ = fs::remove_file(&object_path);
428 let _ = fs::remove_file(&elf_path);
429 let _ = fs::remove_file(&script_path);
430 if cache_dir.is_some() {
431 fs::rename(&compile_source, &source_path)?;
432 fs::rename(&compile_artifact, &artifact_path)?;
433 }
434
435 Ok(Arc::new(LoadedJitArtifact {
436 code: LoadedJitCode::Raw(code),
437 source_path,
438 library_path: artifact_path,
439 compiled_now: true,
440 }))
441}
442
443#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
445const RAW_LINKER_SCRIPT: &str = r#"
454SECTIONS {
455 . = 0;
456 .text : ALIGN(16) { *(.text .text.*) }
457 .rodata : ALIGN(16) { *(.rodata .rodata.*) }
458 .data : ALIGN(16) { *(.data .data.*) *(.got .got.*) }
459 .bss : ALIGN(16) { *(.bss .bss.*) *(COMMON) }
460 /DISCARD/ : { *(.eh_frame .eh_frame.*) *(.comment) *(.note .note.*) }
461}
462"#;
463
464#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
465fn clean_raw_temps(source: &Path, object: &Path, elf: &Path, script: &Path, cache: Option<&Path>) {
466 if cache.is_some() {
468 let _ = fs::remove_file(source);
469 }
470 let _ = fs::remove_file(object);
471 let _ = fs::remove_file(elf);
472 let _ = fs::remove_file(script);
473}
474
475fn cache_key(rust_code: &str, opt_level: u8, raw_jit: bool) -> JitResult<String> {
477 const CACHE_VERSION: &str = "4-moin";
480 const SHARED_FLAGS: &str = "edition=2021;crate-type=cdylib;codegen-units=1;panic=abort;macos=-lSystem;linux=-Wl,-z,defs";
482 const RAW_FLAGS: &str = "edition=2021;crate-type=lib;emit=obj;codegen-units=1;panic=abort;relocation-model=pic;ld=-static,-nostdlib,--build-id=none";
483
484 let version = "rustcversiontodo";
485 let mut hash = 0xcbf29ce484222325u64;
496
497 for part in [
500 CACHE_VERSION,
501 if raw_jit { RAW_FLAGS } else { SHARED_FLAGS },
502 std::env::consts::ARCH,
503 std::env::consts::OS,
504 &version,
505 if raw_jit { "raw" } else { "shared" },
507 &opt_level.to_string(),
508 rust_code,
509 ] {
510 for byte in part.bytes().chain(std::iter::once(0xff)) {
512 hash ^= byte as u64;
514 hash = hash.wrapping_mul(0x100000001b3);
516 }
517 }
518 Ok(format!("{hash:016x}"))
520}
521
522#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
555struct RawCode {
556 base: *mut u8,
558 length: usize,
560 symbols: BTreeMap<Vec<u8>, usize>,
562}
563
564#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
566impl RawCode {
567 fn new(image: &[u8], symbols: BTreeMap<Vec<u8>, usize>) -> JitResult<Self> {
568 if image.is_empty() {
569 return Err(io::Error::other("raw JIT image is empty").into());
570 }
571 let length = image.len().div_ceil(PAGE_SIZE) * PAGE_SIZE;
573 let base = unsafe {
579 mmap(
580 std::ptr::null_mut(),
581 length,
582 PROT_READ | PROT_WRITE,
583 MAP_PRIVATE | MAP_ANONYMOUS,
584 -1,
585 0,
586 )
587 };
588 if base as isize == -1 {
590 return Err(io::Error::last_os_error().into());
591 }
592 unsafe {
595 std::ptr::copy_nonoverlapping(image.as_ptr(), base.cast::<u8>(), image.len());
596 }
597 if unsafe { mprotect(base, length, PROT_READ | PROT_EXEC) } != 0 {
601 let error = io::Error::last_os_error();
602 unsafe {
603 munmap(base, length);
604 }
605 return Err(error.into());
606 }
607 Ok(Self {
608 base: base.cast(),
609 length,
610 symbols,
611 })
612 }
613
614 fn entry(&self, name: &[u8]) -> JitResult<ExecuteFn> {
615 let offset = self.symbols.get(name).ok_or_else(|| {
617 io::Error::other(format!(
618 "raw JIT has no exported symbol {}",
619 String::from_utf8_lossy(name)
620 ))
621 })?;
622 if *offset >= self.length {
623 return Err(io::Error::other("WICHTIG - <raw JIT symbol is outside its image").into());
624 }
625 Ok(unsafe { std::mem::transmute::<*mut u8, ExecuteFn>(self.base.add(*offset)) })
628 }
629}
630
631#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
632impl Drop for RawCode {
633 fn drop(&mut self) {
634 unsafe {
636 munmap(self.base.cast(), self.length);
637 }
638 }
639}
640
641#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
649const PROT_READ: i32 = 1;
650#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
651const PROT_WRITE: i32 = 2;
652#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
653const PROT_EXEC: i32 = 4;
654#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
655const MAP_PRIVATE: i32 = 2;
656#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
657const MAP_ANONYMOUS: i32 = 0x20;
658
659#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
660unsafe extern "C" {
662 fn mmap(
663 address: *mut std::ffi::c_void,
664 length: usize,
665 protection: i32,
666 flags: i32,
667 fd: i32,
668 offset: i64,
669 ) -> *mut std::ffi::c_void;
670 fn mprotect(address: *mut std::ffi::c_void, length: usize, protection: i32) -> i32;
671 fn munmap(address: *mut std::ffi::c_void, length: usize) -> i32;
672}
673
674#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
676fn parse_raw_elf(bytes: &[u8]) -> JitResult<(Vec<u8>, BTreeMap<Vec<u8>, usize>)> {
677 if bytes
682 .get(..16)
683 .is_none_or(|ident| ident[..4] != *b"\x7fELF" || ident[4] != 2 || ident[5] != 1)
684 || elf_u16(bytes, 18)? != 62
685 {
686 return Err(io::Error::other("raw JIT linker output is not x86-64 ELF64").into());
687 }
688
689 let phoff = elf_u64(bytes, 32)? as usize;
692 let phentsize = elf_u16(bytes, 54)? as usize;
693 let phnum = elf_u16(bytes, 56)? as usize;
694
695 if phentsize < 56 {
697 return Err(io::Error::other("raw JIT has invalid program headers").into());
698 }
699
700 let mut loads = Vec::new();
702 let mut image_start = u64::MAX;
705 let mut image_end = 0u64;
706
707 for index in 0..phnum {
708 let offset = phoff
710 .checked_add(
711 index
712 .checked_mul(phentsize)
713 .ok_or_else(|| io::Error::other("ELF overflow"))?,
714 )
715 .ok_or_else(|| io::Error::other("ELF overflow"))?;
716 if elf_u32(bytes, offset)? != 1 {
719 continue;
720 }
721 let file_offset = elf_u64(bytes, offset + 8)?;
723 let vaddr = elf_u64(bytes, offset + 16)?;
724 let file_size = elf_u64(bytes, offset + 32)?;
725 let memory_size = elf_u64(bytes, offset + 40)?;
726 if file_size > memory_size {
727 return Err(io::Error::other("raw JIT has an invalid load segment").into());
728 }
729 image_start = image_start.min(vaddr);
731 image_end = image_end.max(
732 vaddr
733 .checked_add(memory_size)
734 .ok_or_else(|| io::Error::other("raw JIT image overflow"))?,
735 );
736 loads.push((file_offset, vaddr, file_size));
737 }
738
739 if loads.is_empty() || image_end <= image_start {
740 return Err(io::Error::other("raw JIT has no loadable code").into());
741 }
742
743 let image_len = usize::try_from(image_end - image_start)?;
745 let mut image = vec![0u8; image_len];
746
747 for (file_offset, vaddr, file_size) in loads {
748 let source_start = usize::try_from(file_offset)?;
750 let source_end = source_start
751 .checked_add(usize::try_from(file_size)?)
752 .ok_or_else(|| io::Error::other("raw JIT segment overflow"))?;
753 let target_start = usize::try_from(vaddr - image_start)?;
755 let target_end = target_start
756 .checked_add(usize::try_from(file_size)?)
757 .ok_or_else(|| io::Error::other("raw JIT segment overflow"))?;
758 image
759 .get_mut(target_start..target_end)
760 .ok_or_else(|| io::Error::other("raw JIT segment is outside image???"))?
761 .copy_from_slice(
762 bytes
763 .get(source_start..source_end)
764 .ok_or_else(|| io::Error::other("truncated raw JIT segment"))?,
765 );
766 }
767
768 let shoff = elf_u64(bytes, 40)? as usize;
774 let shentsize = elf_u16(bytes, 58)? as usize;
775 let shnum = elf_u16(bytes, 60)? as usize;
776
777 if shentsize < 64 {
779 return Err(io::Error::other("raw JIT has invalid section headers").into());
780 }
781 let mut symbols = BTreeMap::new();
782 for index in 0..shnum {
783 let section = shoff
785 .checked_add(
786 index
787 .checked_mul(shentsize)
788 .ok_or_else(|| io::Error::other("ELF overflow??"))?,
789 )
790 .ok_or_else(|| io::Error::other("ELF overflow??"))?;
791 if elf_u32(bytes, section + 4)? != 2 {
794 continue;
795 }
796 let symbol_offset = elf_u64(bytes, section + 24)? as usize;
798 let symbol_size = elf_u64(bytes, section + 32)? as usize;
799 let string_index = elf_u32(bytes, section + 40)? as usize;
800 let entry_size = elf_u64(bytes, section + 56)? as usize;
801
802 if entry_size < 24 || string_index >= shnum {
804 return Err(io::Error::other("raw JIT has invalid symbols").into());
805 }
806 let strings = shoff + string_index * shentsize;
808 let strings_offset = elf_u64(bytes, strings + 24)? as usize;
809 let strings_size = elf_u64(bytes, strings + 32)? as usize;
810
811 let string_data = bytes
812 .get(strings_offset..strings_offset + strings_size)
813 .ok_or_else(|| io::Error::other("truncated raw JIT string table"))?;
814
815 for symbol in (symbol_offset..symbol_offset + symbol_size).step_by(entry_size) {
816 let name_offset = elf_u32(bytes, symbol)? as usize;
819 let info = *bytes
820 .get(symbol + 4)
821 .ok_or_else(|| io::Error::other("truncated raw JIT symbol"))?;
822 let section_index = elf_u16(bytes, symbol + 6)?;
823
824 if section_index == 0 || info >> 4 == 0 || name_offset >= string_data.len() {
827 continue;
828 }
829
830 let name_end = string_data[name_offset..]
832 .iter()
833 .position(|byte| *byte == 0)
834 .map(|length| name_offset + length)
835 .ok_or_else(|| io::Error::other("invalid raw JIT symbol name"))?;
836 let name = &string_data[name_offset..name_end];
837 if !name.starts_with(b"execute") {
839 continue;
840 }
841 let value = elf_u64(bytes, symbol + 8)?;
842 let offset = usize::try_from(
845 value
846 .checked_sub(image_start)
847 .ok_or_else(|| io::Error::other("raw JIT symbol is outside image"))?,
848 )?;
849 if offset >= image.len() {
850 return Err(io::Error::other("raw JIT symbol is outside image??").into());
851 }
852 symbols.insert(name.to_vec(), offset);
853 }
854 }
855 if symbols.is_empty() {
856 return Err(io::Error::other("raw JIT has no exported entry symbols").into());
857 }
858
859 Ok((image, symbols))
861}
862
863#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
864fn write_raw_cache(image: &[u8], symbols: &BTreeMap<Vec<u8>, usize>) -> Vec<u8> {
865 let mut bytes = Vec::new();
866
867 bytes.extend_from_slice(b"REMURAW3");
873 bytes.extend_from_slice(&(image.len() as u64).to_le_bytes());
874 bytes.extend_from_slice(&(symbols.len() as u32).to_le_bytes());
875 for (name, offset) in symbols {
876 bytes.extend_from_slice(&(name.len() as u16).to_le_bytes());
877 bytes.extend_from_slice(name);
878 bytes.extend_from_slice(&(*offset as u64).to_le_bytes());
879 }
880 bytes.extend_from_slice(image);
881 bytes
882}
883
884#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
886fn read_raw_cache(bytes: &[u8]) -> JitResult<(Vec<u8>, BTreeMap<Vec<u8>, usize>)> {
887 if bytes.get(..8) != Some(b"REMURAW3") {
889 return Err(io::Error::other("invalid raw JIT cache header").into());
890 }
891 let image_len = usize::try_from(elf_u64(bytes, 8)?)?;
893 let symbol_count = elf_u32(bytes, 16)? as usize;
894 let mut cursor = 20usize;
895 let mut symbols = BTreeMap::new();
896
897 for _ in 0..symbol_count {
899 let name_len = elf_u16(bytes, cursor)? as usize;
901 cursor += 2;
902 let name = bytes
903 .get(cursor..cursor + name_len)
904 .ok_or_else(|| io::Error::other("truncated raw JIT cache"))?
905 .to_vec();
906 cursor += name_len;
908 let offset = usize::try_from(elf_u64(bytes, cursor)?)?;
909 cursor += 8;
911 if offset >= image_len {
912 return Err(io::Error::other("invalid raw JIT cache symbol").into());
913 }
914 symbols.insert(name, offset);
915 }
916
917 let image = bytes
918 .get(cursor..cursor + image_len)
919 .ok_or_else(|| io::Error::other("truncated raw JIT cache image"))?
920 .to_vec();
921
922 if cursor + image_len != bytes.len() || symbols.is_empty() {
924 return Err(io::Error::other("invalid raw JIT cache size").into());
925 }
926 Ok((image, symbols))
927}
928
929#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
930fn elf_u16(bytes: &[u8], offset: usize) -> JitResult<u16> {
931 Ok(u16::from_le_bytes(
934 bytes
935 .get(offset..offset + 2)
936 .ok_or_else(|| io::Error::other("truncated ELF"))?
937 .try_into()?,
938 ))
939}
940
941#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
942fn elf_u32(bytes: &[u8], offset: usize) -> JitResult<u32> {
943 Ok(u32::from_le_bytes(
946 bytes
947 .get(offset..offset + 4)
948 .ok_or_else(|| io::Error::other("truncated ELF"))?
949 .try_into()?,
950 ))
951}
952
953#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
954fn elf_u64(bytes: &[u8], offset: usize) -> JitResult<u64> {
955 Ok(u64::from_le_bytes(
958 bytes
959 .get(offset..offset + 8)
960 .ok_or_else(|| io::Error::other("truncated ELF"))?
961 .try_into()?,
962 ))
963}
964
965fn compiled_symbol_from_artifact(
966 artifact: &Arc<LoadedJitArtifact>,
967 symbol_name: &[u8],
968) -> JitResult<CompiledJit> {
969 let execute = match &artifact.code {
971 LoadedJitCode::Shared(library) => unsafe { *library.get::<ExecuteFn>(symbol_name)? },
973 #[cfg(all(target_os = "linux", target_arch = "x86_64"))]
974 LoadedJitCode::Raw(code) => code.entry(symbol_name)?,
975 };
976
977 Ok(CompiledJit {
979 artifact: Arc::clone(artifact),
980 execute,
981 })
982}
983
984fn generate_rust_code(nnil: &Nnil) -> JitResult<String> {
985 let entry = nnil
987 .entrypoint()
988 .ok_or_else(|| io::Error::other("NNIL ist leer"))?;
989
990 RustEmitter::new(nnil, &format!("region_{:x}", entry.0))?.generate()
991}
992
993fn generate_bundle_rust_code(regions: &[Nnil]) -> JitResult<(String, Vec<(u64, String)>)> {
994 let mut code = genjittemplate();
996 let mut exports = Vec::with_capacity(regions.len());
997 let mut emitters = Vec::with_capacity(regions.len());
998
999 for (index, region) in regions.iter().enumerate() {
1000 let entry = region
1001 .entrypoint()
1002 .ok_or_else(|| io::Error::other("NNIL ist leer"))?;
1003
1004 let export_name = format!("execute_entry_{:x}", entry.0);
1006 let prefix = format!("bundle_{index}_{:x}", entry.0);
1008
1009 emitters.push(RustEmitter::new(region, &prefix)?);
1010 exports.push((entry.0, export_name));
1011 }
1012
1013 let linked_blocks = emitters
1016 .iter()
1017 .flat_map(|emitter| {
1018 emitter
1019 .block_names
1020 .iter()
1021 .map(|(&address, name)| (address, name.clone()))
1022 })
1023 .collect::<BTreeMap<_, _>>();
1024
1025 for (emitter, (_, export_name)) in emitters.iter().zip(&exports) {
1026 emitter.emit_region(export_name, &mut code, &linked_blocks)?;
1027 }
1028
1029 Ok((code, exports))
1030}
1031
1032fn genjittemplate() -> String {
1033 let mut code = String::new();
1038
1039 code.push_str("#![no_std]\n");
1043 code.push_str("#![feature(explicit_tail_calls)]\n");
1047 code.push_str("#![allow(incomplete_features)]\n");
1048 code.push_str("#![allow(non_snake_case)]\n");
1051 code.push_str("#![allow(dead_code)]\n");
1052 code.push_str("#![allow(unused_variables)]\n");
1053 code.push_str("use core::sync::atomic::{AtomicBool, Ordering};\n\n");
1055 code.push_str("#[panic_handler]\n");
1059 code.push_str("fn panic(_: &core::panic::PanicInfo) -> ! {\n");
1060 code.push_str(" #[cfg(target_arch = \"x86_64\")]\n");
1062 code.push_str(" unsafe { core::arch::asm!(\"ud2\", options(noreturn)); }\n");
1063 code.push_str(" #[cfg(target_arch = \"aarch64\")]\n");
1065 code.push_str(" unsafe { core::arch::asm!(\"brk #0\", options(noreturn)); }\n");
1066 code.push_str(" #[allow(unreachable_code)] loop { core::hint::spin_loop(); }\n");
1068 code.push_str("}\n\n");
1069 code.push_str("#[cfg(target_os = \"linux\")]\n");
1072 code.push_str("#[no_mangle]\n");
1073 code.push_str("pub extern \"C\" fn rust_eh_personality() {}\n\n");
1074
1075 code.push_str("#[repr(u8)]\n");
1077 code.push_str("#[derive(Clone, Copy, Debug, PartialEq, Eq)]\n");
1078 code.push_str("pub enum AccessKind {\n");
1079 code.push_str(" None = 0,\n");
1082 code.push_str(" Read = 1,\n");
1083 code.push_str(" Write = 2,\n");
1084 code.push_str(" Execute = 3,\n");
1085 code.push_str("}\n\n");
1086
1087 code.push_str("#[repr(C)]\n");
1089 code.push_str("#[derive(Clone, Copy, Debug, PartialEq, Eq)]\n");
1090 code.push_str("pub enum ExitReason {\n");
1093 code.push_str(" None,\n");
1094 code.push_str(" IndirectBranch,\n");
1095 code.push_str(" ReadFault,\n");
1096 code.push_str(" WriteFault,\n");
1097 code.push_str(" ExecFault,\n");
1098 code.push_str(" UnhandledHostCall,\n");
1099 code.push_str(" GuestExit,\n");
1100 code.push_str(" Syscall,\n");
1101 code.push_str(" Sysbreak,\n");
1102 code.push_str(" Breakpoint,\n");
1103 code.push_str(" Paused,\n");
1104 code.push_str(" BlockLimit,\n");
1105 code.push_str("}\n\n");
1106
1107 code.push_str("#[repr(C)]\n");
1108 code.push_str("pub struct GuestControl {\n");
1110 code.push_str(" pub pause: *const AtomicBool,\n");
1112 writeln!(code, " pub breakpoints: [u64; {DEBUG_SLOTS}],").unwrap();
1114 code.push_str(" pub breakpoint_count: usize,\n");
1115 code.push_str(" pub skip_breakpoint: u64,\n");
1118 code.push_str(" pub blocks_left: u64,\n");
1120 code.push_str(" pub limited: u8,\n");
1121 writeln!(code, " pub trace: [u64; {DEBUG_SLOTS}],").unwrap();
1123 code.push_str(" pub trace_pos: usize,\n");
1124 code.push_str("}\n\n");
1125
1126 code.push_str("#[repr(C)]\n");
1127 code.push_str("pub struct GuestPageAbi {\n");
1129 code.push_str(" pub guest_base: u64,\n");
1130 code.push_str(" pub perms: u8,\n");
1131 code.push_str(" pub _padding: [u8; 7],\n");
1134 code.push_str(" pub data: *mut u8,\n");
1135 code.push_str("}\n\n");
1136
1137 code.push_str("#[repr(C)]\n");
1138 code.push_str("pub struct GuestTranslationAbi {\n");
1140 code.push_str(" pub guest_base: u64,\n");
1141 code.push_str(" pub page_index: usize,\n");
1143 code.push_str(" pub perms: u8,\n");
1144 code.push_str(" pub _padding: [u8; 7],\n");
1146 code.push_str(" pub data: *mut u8,\n");
1147 code.push_str("}\n\n");
1148
1149 code.push_str("#[repr(C)]\n");
1150 code.push_str("pub struct GuestMemoryAbi {\n");
1152 code.push_str(" pub pages: *const GuestPageAbi,\n");
1153 code.push_str(" pub page_count: usize,\n");
1154 code.push_str(" pub page_size: usize,\n");
1156 code.push_str(" pub translations: [GuestTranslationAbi; 16],\n");
1159 code.push_str(" pub dirty_pages: *mut usize,\n");
1161 code.push_str(" pub dirty_count: usize,\n");
1162 code.push_str(" pub dirty_bitmap: *mut u64,\n");
1163 code.push_str("}\n\n");
1164
1165 code.push_str(
1167 "pub type ExecuteFn = unsafe extern \"C\" fn(*mut GuestState) -> ExitReason;\n\n",
1168 );
1169 code.push_str("#[repr(C)]\n");
1170 code.push_str("pub struct JitEntryAbi {\n");
1172 code.push_str(" pub guest_pc: u64,\n");
1173 code.push_str(" pub execute: ExecuteFn,\n");
1174 code.push_str("}\n\n");
1175
1176 code.push_str("#[repr(C)]\n");
1177 code.push_str("pub struct GuestState {\n");
1179 writeln!(code, " pub exit_reason: ExitReason,").unwrap();
1180 writeln!(code, " pub pc: u64,").unwrap();
1181 writeln!(code, " pub regs: [u64; {}],", GUEST_REG_COUNT).unwrap();
1183 writeln!(code, " pub tmps: [u64; {}],", GUEST_TMP_COUNT).unwrap();
1185 code.push_str(" pub memory: *mut GuestMemoryAbi,\n");
1186 code.push_str(" pub fault_addr: u64,\n");
1188 code.push_str(" pub fault_access: AccessKind,\n");
1189 code.push_str(" pub control: *mut GuestControl,\n");
1190 code.push_str(" pub jit_entries: *const JitEntryAbi,\n");
1192 code.push_str(" pub jit_entry_count: usize,\n");
1193 code.push_str("}\n\n");
1194
1195 code.push_str("#[inline(always)]\n");
1198 code.push_str("unsafe extern \"C\" fn dispatch_jit(state: *mut GuestState) -> ExitReason {\n");
1199 code.push_str(" let pc = (*state).pc;\n");
1200 code.push_str(" let mut left = 0usize;\n");
1202 code.push_str(" let mut right = (*state).jit_entry_count;\n");
1203 code.push_str(" while left < right {\n");
1204 code.push_str(" let mid = left + (right - left) / 2;\n");
1206 code.push_str(" let entry = &*((*state).jit_entries.add(mid));\n");
1208 code.push_str(" if entry.guest_pc < pc { left = mid + 1; } else { right = mid; }\n");
1210 code.push_str(" }\n");
1211 code.push_str(" if left < (*state).jit_entry_count {\n");
1212 code.push_str(" let entry = &*((*state).jit_entries.add(left));\n");
1213 code.push_str(" if entry.guest_pc == pc { become (entry.execute)(state); }\n");
1215 code.push_str(" }\n");
1216 code.push_str(" (*state).exit_reason = ExitReason::IndirectBranch;\n");
1217 code.push_str(" ExitReason::IndirectBranch\n");
1218 code.push_str("}\n\n");
1219
1220 code.push_str("#[inline(always)]\n");
1225 code.push_str("unsafe fn read_reg(state: *mut GuestState, reg: usize) -> u64 {\n");
1226 writeln!(
1227 code,
1228 "unsafe {{ if reg == {} {{ 0 }} else {{ *(*state).regs.get_unchecked(reg) }} }}",
1229 INTEGER_ZERO_REG
1230 )
1231 .unwrap();
1232 code.push_str("}\n\n");
1233
1234 code.push_str("#[inline(always)]\n");
1235 code.push_str(
1236 "unsafe fn control_block(state: *mut GuestState, pc: u64) -> Option<ExitReason> {\n",
1237 );
1238 code.push_str(" let control = (*state).control;\n");
1240 code.push_str(" if control.is_null() { return None; }\n");
1241 code.push_str(" if !(*control).pause.is_null() && (*(*control).pause).load(Ordering::Relaxed) { return Some(ExitReason::Paused); }\n");
1244 code.push_str(" if (*control).skip_breakpoint == pc {\n");
1245 code.push_str(" (*control).skip_breakpoint = u64::MAX;\n");
1247 code.push_str(" } else if core::slice::from_raw_parts((*control).breakpoints.as_ptr(), (*control).breakpoint_count).binary_search(&pc).is_ok() {\n");
1249 code.push_str(" return Some(ExitReason::Breakpoint);\n");
1250 code.push_str(" }\n");
1251 code.push_str(" if (*control).limited != 0 {\n");
1253 code.push_str(
1254 " if (*control).blocks_left == 0 { return Some(ExitReason::BlockLimit); }\n",
1255 );
1256 code.push_str(" (*control).blocks_left -= 1;\n");
1258 code.push_str(" }\n");
1259 writeln!(
1260 code,
1261 " let trace_index = (*control).trace_pos % {DEBUG_SLOTS};"
1262 )
1263 .unwrap();
1264 code.push_str(" *(*control).trace.get_unchecked_mut(trace_index) = pc;\n");
1266 code.push_str(" (*control).trace_pos = (*control).trace_pos.wrapping_add(1);\n");
1268 code.push_str(" None\n");
1269 code.push_str("}\n\n");
1270
1271 code.push_str("#[inline(always)]\n");
1274 code.push_str("unsafe fn write_reg(state: *mut GuestState, reg: usize, value: u64) {\n");
1275 writeln!(
1276 code,
1277 "unsafe {{ if reg != {} {{ *(*state).regs.get_unchecked_mut(reg) = value; }} }}",
1278 INTEGER_ZERO_REG
1279 )
1280 .unwrap();
1281 code.push_str("}\n\n");
1282
1283 code.push_str("#[inline(always)]\n");
1284 code.push_str("unsafe fn set_fault(state: *mut GuestState, addr: u64, access: AccessKind) {\n");
1285 code.push_str(" (*state).fault_addr = addr;\n");
1287 code.push_str(" (*state).fault_access = access;\n");
1288 code.push_str("}\n\n");
1289
1290 code.push_str("#[inline(always)]\n");
1291 writeln!(code, "fn zext(value: u64, bytes: usize) -> u64 {{").unwrap();
1294 code.push_str(" if bytes >= 8 { value } else { value & ((1u64 << (bytes * 8)) - 1) }\n");
1295 code.push_str("}\n\n");
1296
1297 code.push_str("#[inline(always)]\n");
1298 code.push_str("fn sext(value: u64, bytes: usize) -> u64 {\n");
1301 code.push_str(" if bytes >= 8 {\n");
1302 code.push_str(" value\n");
1303 code.push_str(" } else {\n");
1304 code.push_str(" let shift = 64 - (bytes * 8) as u32;\n");
1306 code.push_str(" (((value << shift) as i64) >> shift) as u64\n");
1307 code.push_str(" }\n");
1308 code.push_str("}\n\n");
1309
1310 code.push_str("#[inline(always)]\n");
1311 code.push_str(
1313 "fn slt_signed(lhs: u64, rhs: u64) -> u64 { ((lhs as i64) < (rhs as i64)) as u64 }\n\n",
1314 );
1315 code.push_str("#[inline(always)]\n");
1316 code.push_str("fn slt_unsigned(lhs: u64, rhs: u64) -> u64 { (lhs < rhs) as u64 }\n\n");
1317
1318 code.push_str("#[inline(always)]\n");
1319 code.push_str("fn div_signed(lhs: u64, rhs: u64) -> u64 {\n");
1324 code.push_str(" let lhs = lhs as i64;\n");
1325 code.push_str(" let rhs = rhs as i64;\n");
1326 code.push_str(" if rhs == 0 {\n");
1327 code.push_str(" u64::MAX\n");
1329 code.push_str(" } else if lhs == i64::MIN && rhs == -1 {\n");
1330 code.push_str(" lhs as u64\n");
1331 code.push_str(" } else {\n");
1332 code.push_str(" lhs.wrapping_div(rhs) as u64\n");
1333 code.push_str(" }\n");
1334 code.push_str("}\n\n");
1335
1336 code.push_str("#[inline(always)]\n");
1337 code.push_str("fn div_unsigned(lhs: u64, rhs: u64) -> u64 { if rhs == 0 { u64::MAX } else { lhs / rhs } }\n\n");
1339 code.push_str("#[inline(always)]\n");
1340 code.push_str("fn rem_signed(lhs: u64, rhs: u64) -> u64 {\n");
1343 code.push_str(" let lhs = lhs as i64;\n");
1344 code.push_str(" let rhs = rhs as i64;\n");
1345 code.push_str(" if rhs == 0 {\n");
1346 code.push_str(" lhs as u64\n");
1347 code.push_str(" } else if lhs == i64::MIN && rhs == -1 {\n");
1348 code.push_str(" 0\n");
1349 code.push_str(" } else {\n");
1350 code.push_str(" lhs.wrapping_rem(rhs) as u64\n");
1351 code.push_str(" }\n");
1352 code.push_str("}\n\n");
1353 code.push_str("#[inline(always)]\n");
1354 code.push_str(
1356 "fn rem_unsigned(lhs: u64, rhs: u64) -> u64 { if rhs == 0 { lhs } else { lhs % rhs } }\n\n",
1357 );
1358
1359 code.push_str("#[inline(always)]\n");
1360 code.push_str("fn mulh_signed_signed(lhs: u64, rhs: u64) -> u64 { (((lhs as i64 as i128) * (rhs as i64 as i128)) >> 64) as u64 }\n\n");
1364 code.push_str("#[inline(always)]\n");
1365 code.push_str("fn mulh_signed_unsigned(lhs: u64, rhs: u64) -> u64 { (((lhs as i64 as i128) * (rhs as u64 as i128)) >> 64) as u64 }\n\n");
1367 code.push_str("#[inline(always)]\n");
1368 code.push_str("fn mulh_unsigned_unsigned(lhs: u64, rhs: u64) -> u64 { ((lhs as u128 * rhs as u128) >> 64) as u64 }\n\n");
1370
1371 code.push_str("#[inline(always)]\n");
1372 code.push_str("unsafe fn mark_dirty(memory: *mut GuestMemoryAbi, page_index: usize) {\n");
1374 code.push_str(" let word = (*memory).dirty_bitmap.add(page_index / 64);\n");
1376 code.push_str(" let mask = 1u64 << (page_index % 64);\n");
1378 code.push_str(" if *word & mask == 0 {\n");
1380 code.push_str(" *word |= mask;\n");
1381 code.push_str(" *(*memory).dirty_pages.add((*memory).dirty_count) = page_index;\n");
1383 code.push_str(" (*memory).dirty_count += 1;\n");
1384 code.push_str(" }\n");
1385 code.push_str("}\n\n");
1386
1387 code.push_str("#[cold]\n");
1391 code.push_str("#[inline(never)]\n");
1392 code.push_str("unsafe fn translate_slow(memory: *mut GuestMemoryAbi, page_base: u64, offset: usize, required_perm: u8) -> *mut u8 {\n");
1393 code.push_str(" let mut left = 0usize;\n");
1395 code.push_str(" let mut right = (*memory).page_count;\n");
1396 code.push_str(" while left < right {\n");
1397 code.push_str(" let mid = left + (right - left) / 2;\n");
1399 code.push_str(" let page = &*((*memory).pages.add(mid));\n");
1402 code.push_str(
1403 " if page.guest_base < page_base { left = mid + 1; } else { right = mid; }\n",
1404 );
1405 code.push_str(" }\n");
1406 code.push_str(" if left < (*memory).page_count {\n");
1407 code.push_str(" let page = &*((*memory).pages.add(left));\n");
1408 code.push_str(" if page.guest_base == page_base && (page.perms & required_perm) == required_perm {\n");
1409 writeln!(
1413 code,
1414 " let slot = ((page_base / {}u64) as usize) & 15;",
1415 PAGE_SIZE
1416 )
1417 .unwrap();
1418 code.push_str(" let cached = (*memory).translations.get_unchecked_mut(slot);\n");
1419 code.push_str(" cached.guest_base = page_base;\n");
1421 code.push_str(" cached.page_index = left;\n");
1422 code.push_str(" cached.perms = page.perms;\n");
1423 code.push_str(" cached.data = page.data;\n");
1424 code.push_str(" if required_perm & 2 != 0 { mark_dirty(memory, left); }\n");
1426 code.push_str(" return page.data.add(offset);\n");
1428 code.push_str(" }\n");
1429 code.push_str(" }\n");
1430 code.push_str(" core::ptr::null_mut()\n");
1431 code.push_str("}\n\n");
1432
1433 code.push_str("#[inline(always)]\n");
1436 code.push_str("unsafe fn translate(memory: *mut GuestMemoryAbi, addr: u64, required_perm: u8) -> *mut u8 {\n");
1437 code.push_str(" if memory.is_null() {\n");
1438 code.push_str(" return core::ptr::null_mut();\n");
1439 code.push_str(" }\n");
1440 writeln!(
1443 code,
1444 " let page_base = addr & !(({}u64) - 1);",
1445 PAGE_SIZE
1446 )
1447 .unwrap();
1448 writeln!(
1450 code,
1451 " let offset = (addr & (({}u64) - 1)) as usize;",
1452 PAGE_SIZE
1453 )
1454 .unwrap();
1455 writeln!(
1458 code,
1459 " let slot = ((page_base / {}u64) as usize) & 15;",
1460 PAGE_SIZE
1461 )
1462 .unwrap();
1463 code.push_str(" let cached = (*memory).translations.get_unchecked_mut(slot);\n");
1464 code.push_str(" if !cached.data.is_null() && cached.guest_base == page_base && (cached.perms & required_perm) == required_perm {\n");
1466 code.push_str(" if required_perm & 2 != 0 { mark_dirty(memory, cached.page_index); }\n");
1468 code.push_str(" return cached.data.add(offset);\n");
1469 code.push_str(" }\n");
1470 code.push_str(" translate_slow(memory, page_base, offset, required_perm)\n");
1471 code.push_str("}\n\n");
1472
1473 code.push_str("#[inline(always)]\n");
1477 code.push_str(
1478 "unsafe fn load_le(state: *mut GuestState, addr: u64, size: usize) -> Option<u64> {\n",
1479 );
1480 writeln!(
1483 code,
1484 " if size <= 8 && (addr as usize & ({} - 1)) + size <= {} {{",
1485 PAGE_SIZE, PAGE_SIZE
1486 )
1487 .unwrap();
1488 code.push_str(" let ptr = translate((*state).memory, addr, 1);\n");
1490 code.push_str(
1491 " if ptr.is_null() { set_fault(state, addr, AccessKind::Read); return None; }\n",
1492 );
1493 code.push_str(" return Some(match size {\n");
1496 code.push_str(" 1 => *ptr as u64,\n");
1497 code.push_str(
1498 " 2 => u16::from_le(core::ptr::read_unaligned(ptr.cast::<u16>())) as u64,\n",
1499 );
1500 code.push_str(
1501 " 4 => u32::from_le(core::ptr::read_unaligned(ptr.cast::<u32>())) as u64,\n",
1502 );
1503 code.push_str(" 8 => u64::from_le(core::ptr::read_unaligned(ptr.cast::<u64>())),\n");
1504 code.push_str(" _ => 0,\n");
1506 code.push_str(" });\n");
1507 code.push_str(" }\n");
1508 code.push_str(" let mut value = 0u64;\n");
1510 code.push_str(" for idx in 0..size {\n");
1511 code.push_str(" let Some(byte_addr) = addr.checked_add(idx as u64) else {\n");
1513 code.push_str(" set_fault(state, addr, AccessKind::Read);\n");
1514 code.push_str(" return None;\n");
1515 code.push_str(" };\n");
1516 code.push_str(" let ptr = translate((*state).memory, byte_addr, 1);\n");
1518 code.push_str(" if ptr.is_null() {\n");
1519 code.push_str(" set_fault(state, byte_addr, AccessKind::Read);\n");
1520 code.push_str(" return None;\n");
1521 code.push_str(" }\n");
1522 code.push_str(" value |= (*ptr as u64) << (idx * 8);\n");
1525 code.push_str(" }\n");
1526 code.push_str(" Some(value)\n");
1527 code.push_str("}\n\n");
1528
1529 code.push_str("#[inline(always)]\n");
1530 code.push_str("unsafe fn store_le(state: *mut GuestState, addr: u64, size: usize, value: u64) -> bool {\n");
1532 writeln!(
1533 code,
1534 " if size <= 8 && (addr as usize & ({} - 1)) + size <= {} {{",
1535 PAGE_SIZE, PAGE_SIZE
1536 )
1537 .unwrap();
1538 code.push_str(" let ptr = translate((*state).memory, addr, 2);\n");
1540 code.push_str(
1541 " if ptr.is_null() { set_fault(state, addr, AccessKind::Write); return false; }\n",
1542 );
1543 code.push_str(" match size {\n");
1545 code.push_str(" 1 => *ptr = value as u8,\n");
1546 code.push_str(
1547 " 2 => core::ptr::write_unaligned(ptr.cast::<u16>(), (value as u16).to_le()),\n",
1548 );
1549 code.push_str(
1550 " 4 => core::ptr::write_unaligned(ptr.cast::<u32>(), (value as u32).to_le()),\n",
1551 );
1552 code.push_str(
1553 " 8 => core::ptr::write_unaligned(ptr.cast::<u64>(), value.to_le()),\n",
1554 );
1555 code.push_str(" _ => return false,\n");
1557 code.push_str(" }\n");
1558 code.push_str(" return true;\n");
1559 code.push_str(" }\n");
1560 code.push_str(" for idx in 0..size {\n");
1563 code.push_str(" let Some(byte_addr) = addr.checked_add(idx as u64) else {\n");
1565 code.push_str(" set_fault(state, addr, AccessKind::Write);\n");
1566 code.push_str(" return false;\n");
1567 code.push_str(" };\n");
1568 code.push_str(" let ptr = translate((*state).memory, byte_addr, 2);\n");
1570 code.push_str(" if ptr.is_null() {\n");
1571 code.push_str(" set_fault(state, byte_addr, AccessKind::Write);\n");
1572 code.push_str(" return false;\n");
1573 code.push_str(" }\n");
1574 code.push_str(" *ptr = ((value >> (idx * 8)) & 0xff) as u8;\n");
1577 code.push_str(" }\n");
1578 code.push_str(" true\n");
1579 code.push_str("}\n\n");
1580
1581 code.push_str("#[inline(always)]\n");
1582 code.push_str("fn fdiv_bits(size: usize, lhs: u64, rhs: u64) -> u64 {\n");
1586 code.push_str(" match size {\n");
1587 code.push_str(" 4 => (f32::from_bits(lhs as u32) / f32::from_bits(rhs as u32)).to_bits() as u64,\n");
1589 code.push_str(" 8 => (f64::from_bits(lhs) / f64::from_bits(rhs)).to_bits(),\n");
1590 code.push_str(" _ => unsafe { core::hint::unreachable_unchecked() },\n");
1593 code.push_str(" }\n");
1594 code.push_str("}\n\n");
1595
1596 code.push_str("#[inline(always)]\n");
1597 code.push_str(
1603 "fn int_to_float_bits(src: u64, src_size: usize, signed: bool, dst_size: usize) -> u64 {\n",
1604 );
1605 code.push_str(" match (src_size, signed, dst_size) {\n");
1606 code.push_str(" (4, false, 8) => (src as u32 as f64).to_bits(),\n");
1607 code.push_str(" (4, true, 8) => ((src as i32) as f64).to_bits(),\n");
1608 code.push_str(" (8, false, 8) => (src as f64).to_bits(),\n");
1609 code.push_str(" (8, true, 8) => ((src as i64) as f64).to_bits(),\n");
1610 code.push_str(" _ => unsafe { core::hint::unreachable_unchecked() },\n");
1613 code.push_str(" }\n");
1614 code.push_str("}\n\n");
1615
1616 code
1617}
1618
1619impl<'a> RustEmitter<'a> {
1620 fn new(nnil: &'a Nnil, name_prefix: &str) -> JitResult<Self> {
1621 let blocks = collect_blocks(nnil)?;
1623
1624 let block_names = blocks
1626 .iter()
1627 .map(|block| {
1628 (
1629 block.addr,
1630 format!("{name_prefix}_block_{:x}", block.addr.0),
1631 )
1632 })
1633 .collect::<BTreeMap<_, _>>();
1634
1635 let unsigned_markers = nnil
1638 .insns()
1639 .iter()
1640 .filter_map(|insn| match insn {
1641 NnilInstruction::Unsigned { dst } => Some(dst.0),
1642 _ => None,
1643 })
1644 .collect::<BTreeSet<_>>();
1645
1646 Ok(Self {
1647 nnil,
1648 blocks,
1649 block_names,
1650 unsigned_markers,
1651 })
1652 }
1653
1654 fn generate(&self) -> JitResult<String> {
1655 let mut code = genjittemplate();
1657 self.emit_region("execute", &mut code, &self.block_names)?;
1658 Ok(code)
1659 }
1660
1661 fn emit_region(
1662 &self,
1663 export_name: &str,
1664 code: &mut String,
1665 linked_blocks: &BTreeMap<Address, String>,
1666 ) -> JitResult<()> {
1667 let entry = self
1668 .nnil
1669 .entrypoint()
1670 .ok_or_else(|| io::Error::other("cannot JIT an empty NNIL region"))?;
1671 for block in &self.blocks {
1674 self.emit_block(block, code, linked_blocks)?;
1675 }
1676
1677 code.push_str("#[no_mangle]\n");
1680 writeln!(
1681 code,
1682 "pub unsafe extern \"C\" fn {export_name}(state: *mut GuestState) -> ExitReason {{"
1683 )
1684 .unwrap();
1685 code.push_str(" if state.is_null() {\n");
1687 code.push_str(" return ExitReason::IndirectBranch;\n");
1688 code.push_str(" }\n");
1689 writeln!(
1691 code,
1692 " if (*state).pc == 0 {{ (*state).pc = {}; }}",
1693 entry.0
1694 )
1695 .unwrap();
1696 code.push_str(" match (*state).pc {\n");
1699
1700 for block in &self.blocks {
1701 writeln!(
1702 code,
1703 " {} => become {}(state),",
1704 block.addr.0,
1705 self.block_name(block.addr)?
1706 )
1707 .unwrap();
1708 }
1709
1710 code.push_str(" _ => {\n");
1712 code.push_str(" (*state).exit_reason = ExitReason::IndirectBranch;\n");
1713 code.push_str(" ExitReason::IndirectBranch\n");
1714 code.push_str(" }\n");
1715 code.push_str(" }\n");
1716 code.push_str("}\n\n");
1717
1718 Ok(())
1719
1720 }
1722
1723 fn emit_block(
1724 &self,
1725 block: &BasicBlock,
1726 out: &mut String,
1727 linked_blocks: &BTreeMap<Address, String>,
1728 ) -> JitResult<()> {
1729 writeln!(
1731 out,
1732 "unsafe extern \"C\" fn {}(state: *mut GuestState) -> ExitReason {{",
1733 self.block_name(block.addr)?
1734 )
1735 .unwrap();
1736
1737 writeln!(out, " (*state).pc = {};", block.addr.0).unwrap();
1740 out.push_str(" if let Some(reason) = control_block(state, (*state).pc) {\n");
1741 out.push_str(" (*state).exit_reason = reason;\n");
1742 out.push_str(" return reason;\n");
1743 out.push_str(" }\n");
1744
1745 let mut terminated = false;
1747
1748 for insn_idx in block.start_insn..block.end_insn {
1749 if self.emit_insn(block, insn_idx, out, linked_blocks)? {
1750 terminated = true;
1751 }
1752 }
1753
1754 if !terminated {
1757 if let Some(next) = block.fallthrough {
1758 writeln!(out, " (*state).pc = {};", next.0).unwrap();
1759 self.write_branch_or_tailcall(next, out, 4, linked_blocks)?;
1761 } else {
1762 writeln!(out, " (*state).pc = {};", block.next_pc).unwrap();
1764 out.push_str(" become dispatch_jit(state);\n");
1765 }
1766 }
1767
1768 out.push_str("}\n\n");
1769 Ok(())
1770 }
1771
1772 fn emit_insn(
1773 &self,
1774 block: &BasicBlock,
1775 insn_idx: usize,
1776 out: &mut String,
1777 linked_blocks: &BTreeMap<Address, String>,
1778 ) -> JitResult<bool> {
1779 let insn = &self.nnil.insns()[insn_idx];
1780
1781 match insn {
1784 NnilInstruction::Imm { imm } => {
1785 writeln!(out, " let v_{insn_idx}: u64 = ({imm}i64) as u64;").unwrap();
1787 }
1788
1789 NnilInstruction::Reg { reg } => {
1790 self.ensure_reg(*reg)?;
1792 writeln!(out, " let v_{insn_idx}: u64 = read_reg(state, {});", reg).unwrap();
1793 }
1794
1795 NnilInstruction::TmpReg { reg } => {
1796 self.ensure_tmp(*reg)?;
1798 writeln!(
1799 out,
1800 " let v_{insn_idx}: u64 = *(*state).tmps.get_unchecked({});",
1801 reg
1802 )
1803 .unwrap();
1804 }
1805
1806 NnilInstruction::Add { lhs, rhs } => {
1807 writeln!(
1811 out,
1812 " let v_{insn_idx}: u64 = v_{}.wrapping_add(v_{});",
1813 lhs.0, rhs.0
1814 )
1815 .unwrap();
1816 }
1817
1818 NnilInstruction::Sub { lhs, rhs } => {
1819 writeln!(
1821 out,
1822 " let v_{insn_idx}: u64 = v_{}.wrapping_sub(v_{});",
1823 lhs.0, rhs.0
1824 )
1825 .unwrap();
1826 }
1827
1828 NnilInstruction::Mul { lhs, rhs } => {
1829 writeln!(
1831 out,
1832 " let v_{insn_idx}: u64 = v_{}.wrapping_mul(v_{});",
1833 lhs.0, rhs.0
1834 )
1835 .unwrap();
1836 }
1837
1838 NnilInstruction::Mulh { lhs, rhs } => {
1839 let expr = match (self.signedness(*lhs), self.signedness(*rhs)) {
1843 (Signedness::Signed, Signedness::Signed) => {
1844 format!("mulh_signed_signed(v_{}, v_{})", lhs.0, rhs.0)
1845 }
1846 (Signedness::Signed, Signedness::Unsigned) => {
1847 format!("mulh_signed_unsigned(v_{}, v_{})", lhs.0, rhs.0)
1848 }
1849 (Signedness::Unsigned, Signedness::Unsigned) => {
1850 format!("mulh_unsigned_unsigned(v_{}, v_{})", lhs.0, rhs.0)
1851 }
1852 (Signedness::Unsigned, Signedness::Signed) => {
1853 format!("mulh_signed_unsigned(v_{}, v_{})", rhs.0, lhs.0)
1854 }
1855 };
1856 writeln!(out, " let v_{insn_idx}: u64 = {expr};").unwrap();
1857 }
1858
1859 NnilInstruction::Div { lhs, rhs } => {
1860 let expr = match self.binary_signedness(*lhs, *rhs) {
1862 Signedness::Signed => format!("div_signed(v_{}, v_{})", lhs.0, rhs.0),
1863 Signedness::Unsigned => format!("div_unsigned(v_{}, v_{})", lhs.0, rhs.0),
1864 };
1865 writeln!(out, " let v_{insn_idx}: u64 = {expr};").unwrap();
1866 }
1867
1868 NnilInstruction::Mod { lhs, rhs } => {
1869 let expr = match self.binary_signedness(*lhs, *rhs) {
1871 Signedness::Signed => format!("rem_signed(v_{}, v_{})", lhs.0, rhs.0),
1872 Signedness::Unsigned => format!("rem_unsigned(v_{}, v_{})", lhs.0, rhs.0),
1873 };
1874 writeln!(out, " let v_{insn_idx}: u64 = {expr};").unwrap();
1875 }
1876
1877 NnilInstruction::And { lhs, rhs } => {
1878 writeln!(
1880 out,
1881 " let v_{insn_idx}: u64 = v_{} & v_{};",
1882 lhs.0, rhs.0
1883 )
1884 .unwrap();
1885 }
1886
1887 NnilInstruction::Or { lhs, rhs } => {
1888 writeln!(
1890 out,
1891 " let v_{insn_idx}: u64 = v_{} | v_{};",
1892 lhs.0, rhs.0
1893 )
1894 .unwrap();
1895 }
1896
1897 NnilInstruction::Xor { lhs, rhs } => {
1898 writeln!(
1900 out,
1901 " let v_{insn_idx}: u64 = v_{} ^ v_{};",
1902 lhs.0, rhs.0
1903 )
1904 .unwrap();
1905 }
1906
1907 NnilInstruction::Sll { lhs, rhs } => {
1908 writeln!(
1912 out,
1913 " let v_{insn_idx}: u64 = v_{}.wrapping_shl((v_{} & 63) as u32);",
1914 lhs.0, rhs.0
1915 )
1916 .unwrap();
1917 }
1918
1919 NnilInstruction::Srl { lhs, rhs } => {
1938 writeln!(
1940 out,
1941 " let v_{insn_idx}: u64 = v_{}.wrapping_shr((v_{} & 63) as u32);",
1942 lhs.0, rhs.0
1943 )
1944 .unwrap();
1945 }
1946
1947 NnilInstruction::Sra { lhs, rhs } => {
1948 writeln!(out, " let v_{insn_idx}: u64 = ((v_{} as i64).wrapping_shr((v_{} & 63) as u32)) as u64;", lhs.0, rhs.0).unwrap();
1952 }
1953
1954 NnilInstruction::Slt { lhs, rhs } => {
1955 let expr = match self.binary_signedness(*lhs, *rhs) {
1957 Signedness::Signed => format!("slt_signed(v_{}, v_{})", lhs.0, rhs.0),
1958 Signedness::Unsigned => format!("slt_unsigned(v_{}, v_{})", lhs.0, rhs.0),
1959 };
1960 writeln!(out, " let v_{insn_idx}: u64 = {expr};").unwrap();
1961 }
1962 NnilInstruction::Mov { dst, src } | NnilInstruction::Set { dst, src } => {
1963 writeln!(out, " let v_{insn_idx}: u64 = v_{};", src.0).unwrap();
1965 self.emit_write_target(*dst, format!("v_{insn_idx}"), out)?;
1966 }
1967
1968 NnilInstruction::Load { dst, src, size } => {
1969 self.ensure_load_store_size(*size)?;
1971 let load_expr = format!(
1972 "match load_le(state, v_{}, {}) {{ Some(value) => value, None => {{ (*state).pc = {}; (*state).exit_reason = ExitReason::ReadFault; return ExitReason::ReadFault; }} }}",
1973 src.0, size, block.addr.0
1974 );
1975 writeln!(out, " let raw_{insn_idx}: u64 = {load_expr};").unwrap();
1976 let value_expr = match self.load_mode(insn_idx, *dst) {
1979 LoadMode::Raw | LoadMode::ZeroExtend => format!("zext(raw_{insn_idx}, {size})"),
1980 LoadMode::SignExtend => format!("sext(raw_{insn_idx}, {size})"),
1981 };
1982 writeln!(out, " let v_{insn_idx}: u64 = {value_expr};").unwrap();
1983 self.emit_write_target(*dst, format!("v_{insn_idx}"), out)?;
1984 }
1985
1986 NnilInstruction::Store { dst, src, size } => {
1987 self.ensure_load_store_size(*size)?;
1988 writeln!(out, " let v_{insn_idx}: u64 = v_{};", src.0).unwrap();
1990 writeln!(
1991 out,
1992 " if !store_le(state, v_{}, {}, v_{insn_idx}) {{",
1993 dst.0, size
1994 )
1995 .unwrap();
1996 writeln!(out, " (*state).pc = {};", block.addr.0).unwrap();
1998 out.push_str(" (*state).exit_reason = ExitReason::WriteFault;\n");
1999 out.push_str(" return ExitReason::WriteFault;\n");
2000 out.push_str(" }\n");
2001 }
2002
2003 NnilInstruction::Sext { size, dst } => {
2004 writeln!(
2006 out,
2007 " let v_{insn_idx}: u64 = sext(v_{}, {});",
2008 dst.0, size
2009 )
2010 .unwrap();
2011 }
2012
2013 NnilInstruction::Zext { size, dst } => {
2014 writeln!(
2016 out,
2017 " let v_{insn_idx}: u64 = zext(v_{}, {});",
2018 dst.0, size
2019 )
2020 .unwrap();
2021 }
2022
2023 NnilInstruction::Signed { dst } | NnilInstruction::Unsigned { dst } => {
2024 writeln!(out, " let v_{insn_idx}: u64 = v_{};", dst.0).unwrap();
2026 }
2027
2028 NnilInstruction::Fdiv { lhs, rhs, size } => {
2029 writeln!(
2031 out,
2032 " let v_{insn_idx}: u64 = fdiv_bits({}, v_{}, v_{});",
2033 size, lhs.0, rhs.0
2034 )
2035 .unwrap();
2036 }
2037
2038 NnilInstruction::IntToFloat {
2039 src,
2040 src_size,
2041 signed,
2042 dst_size,
2043 } => {
2044 writeln!(
2046 out,
2047 " let v_{insn_idx}: u64 = int_to_float_bits(v_{}, {}, {}, {});",
2048 src.0, src_size, signed, dst_size
2049 )
2050 .unwrap();
2051 }
2052
2053 NnilInstruction::Fence { .. } => {
2055 writeln!(out, " let v_{insn_idx}: u64 = 0;").unwrap();
2058 }
2059
2060 NnilInstruction::Jr { dst } | NnilInstruction::Jmp { dst } => {
2062 if let Some(target) = self.direct_jump_target(*dst) {
2064 writeln!(out, " (*state).pc = {};", target.0).unwrap();
2065 if let Some(name) = linked_blocks.get(&target) {
2068 writeln!(out, " become {name}(state);").unwrap();
2069 } else {
2070 out.push_str(" become dispatch_jit(state);\n");
2071 }
2072 } else {
2073 writeln!(out, " (*state).pc = v_{};", dst.0).unwrap();
2075 out.push_str(" become dispatch_jit(state);\n");
2076 }
2077 return Ok(true);
2078 }
2079
2080 NnilInstruction::Beq {
2081 lhs,
2082 rhs,
2083 ttgt,
2084 ftgt,
2085 } => {
2086 self.emit_branch(
2088 format!("v_{} == v_{}", lhs.0, rhs.0),
2089 *ttgt,
2090 *ftgt,
2091 out,
2092 linked_blocks,
2093 )?;
2094 return Ok(true);
2095 }
2096
2097 NnilInstruction::Bne {
2098 lhs,
2099 rhs,
2100 ttgt,
2101 ftgt,
2102 } => {
2103 self.emit_branch(
2105 format!("v_{} != v_{}", lhs.0, rhs.0),
2106 *ttgt,
2107 *ftgt,
2108 out,
2109 linked_blocks,
2110 )?;
2111 return Ok(true);
2112 }
2113
2114 NnilInstruction::Blt {
2115 lhs,
2116 rhs,
2117 ttgt,
2118 ftgt,
2119 } => {
2120 let cond = match self.binary_signedness(*lhs, *rhs) {
2122 Signedness::Signed => format!("(v_{} as i64) < (v_{} as i64)", lhs.0, rhs.0),
2123 Signedness::Unsigned => format!("v_{} < v_{}", lhs.0, rhs.0),
2124 };
2125 self.emit_branch(cond, *ttgt, *ftgt, out, linked_blocks)?;
2126 return Ok(true);
2127 }
2128
2129 NnilInstruction::Bgt {
2130 lhs,
2131 rhs,
2132 ttgt,
2133 ftgt,
2134 } => {
2135 let cond = match self.binary_signedness(*lhs, *rhs) {
2137 Signedness::Signed => format!("(v_{} as i64) > (v_{} as i64)", lhs.0, rhs.0),
2138 Signedness::Unsigned => format!("v_{} > v_{}", lhs.0, rhs.0),
2139 };
2140 self.emit_branch(cond, *ttgt, *ftgt, out, linked_blocks)?;
2141 return Ok(true);
2142 }
2143
2144 NnilInstruction::Syscall {} => {
2145 writeln!(out, " (*state).pc = {};", block.next_pc).unwrap();
2148 out.push_str(" (*state).exit_reason = ExitReason::Syscall;\n");
2149 out.push_str(" return ExitReason::Syscall;\n");
2150 return Ok(true);
2151 }
2152
2153 NnilInstruction::Sysbreak {} => {
2154 writeln!(out, " (*state).pc = {};", block.next_pc).unwrap();
2156 out.push_str(" (*state).exit_reason = ExitReason::Sysbreak;\n");
2157 out.push_str(" return ExitReason::Sysbreak;\n");
2158 return Ok(true);
2159 }
2160
2161 NnilInstruction::Lr { dst, src, size } => {
2162 self.ensure_load_store_size(*size)?;
2165 writeln!(
2166 out,
2167 " let raw_{insn_idx}: u64 = match load_le(state, v_{}, {}) {{ Some(value) => value, None => {{ (*state).pc = {}; (*state).exit_reason = ExitReason::ReadFault; return ExitReason::ReadFault; }} }};",
2168 src.0, size, block.addr.0
2169 ).unwrap();
2170 let value_expr = if *size < 8 {
2173 format!("sext(raw_{insn_idx}, {})", size)
2174 } else {
2175 format!("raw_{insn_idx}")
2176 };
2177 writeln!(out, " let v_{insn_idx}: u64 = {value_expr};").unwrap();
2178 self.emit_write_target(*dst, format!("v_{insn_idx}"), out)?;
2179 }
2180
2181 NnilInstruction::Sc {
2182 dst,
2183 src,
2184 addr,
2185 size,
2186 } => {
2187 self.ensure_load_store_size(*size)?;
2188 let store_val = if *size < 8 {
2191 format!("v_{} & ((1u64 << ({} * 8)) - 1)", src.0, size)
2192 } else {
2193 format!("v_{}", src.0)
2194 };
2195 writeln!(out, " let v_{insn_idx}: u64 = 0;").unwrap();
2198 writeln!(
2199 out,
2200 " if !store_le(state, v_{}, {}, {}) {{",
2201 addr.0, size, store_val
2202 )
2203 .unwrap();
2204 writeln!(out, " (*state).pc = {};", block.addr.0).unwrap();
2205 out.push_str(" (*state).exit_reason = ExitReason::WriteFault;\n");
2206 out.push_str(" return ExitReason::WriteFault;\n");
2207 out.push_str(" }\n");
2208 self.emit_write_target(*dst, format!("v_{insn_idx}"), out)?;
2209 }
2210
2211 NnilInstruction::Amo {
2212 op,
2213 dst,
2214 src,
2215 addr,
2216 size,
2217 } => {
2218 self.ensure_load_store_size(*size)?;
2221 writeln!(
2222 out,
2223 " let raw_{insn_idx}: u64 = match load_le(state, v_{}, {}) {{ Some(value) => value, None => {{ (*state).pc = {}; (*state).exit_reason = ExitReason::ReadFault; return ExitReason::ReadFault; }} }};",
2224 addr.0, size, block.addr.0
2225 ).unwrap();
2226
2227 let loaded = if *size < 8 {
2229 format!("sext(raw_{insn_idx}, {})", size)
2230 } else {
2231 format!("raw_{insn_idx}")
2232 };
2233 let rhs = if *size < 8 {
2236 format!("sext(zext(v_{}, {}), {})", src.0, size, size)
2237 } else {
2238 format!("v_{}", src.0)
2239 };
2240
2241 let result_expr = match op {
2244 AmoOp::Swap => format!("v_{}", src.0),
2245 AmoOp::Add => format!("{loaded}.wrapping_add({rhs})"),
2246 AmoOp::Xor => format!("{loaded} ^ {rhs}"),
2247 AmoOp::And => format!("{loaded} & {rhs}"),
2248 AmoOp::Or => format!("{loaded} | {rhs}"),
2249 AmoOp::Min => format!(
2250 "if ({loaded} as i64) < ({rhs} as i64) {{ {loaded} }} else {{ {rhs} }}"
2251 ),
2252 AmoOp::Max => format!(
2253 "if ({loaded} as i64) > ({rhs} as i64) {{ {loaded} }} else {{ {rhs} }}"
2254 ),
2255 AmoOp::Minu => format!("if {loaded} < {rhs} {{ {loaded} }} else {{ {rhs} }}"),
2256 AmoOp::Maxu => format!("if {loaded} > {rhs} {{ {loaded} }} else {{ {rhs} }}"),
2257 };
2258
2259 let store_val = if *size < 8 {
2261 format!("{result_expr} & ((1u64 << ({} * 8)) - 1)", size)
2262 } else {
2263 result_expr
2264 };
2265 writeln!(
2266 out,
2267 " if !store_le(state, v_{}, {}, {}) {{",
2268 addr.0, size, store_val
2269 )
2270 .unwrap();
2271 writeln!(out, " (*state).pc = {};", block.addr.0).unwrap();
2272 out.push_str(" (*state).exit_reason = ExitReason::WriteFault;\n");
2273 out.push_str(" return ExitReason::WriteFault;\n");
2274 out.push_str(" }\n");
2275
2276 writeln!(out, " let v_{insn_idx}: u64 = {loaded};").unwrap();
2278 self.emit_write_target(*dst, format!("v_{insn_idx}"), out)?;
2279 }
2280 }
2281
2282 Ok(false)
2283 }
2284
2285 fn emit_branch(
2286 &self,
2287 condition: String,
2288 ttgt: Label,
2289 ftgt: Label,
2290 out: &mut String,
2291 linked_blocks: &BTreeMap<Address, String>,
2292 ) -> JitResult<()> {
2293 let ttgt_addr = self.label_address(ttgt)?;
2295 let ftgt_addr = self.label_address(ftgt)?;
2296
2297 writeln!(out, " if {condition} {{").unwrap();
2298 writeln!(out, " (*state).pc = {};", ttgt_addr.0).unwrap();
2299
2300 self.write_branch_or_tailcall(ttgt_addr, out, 8, linked_blocks)?;
2302 out.push_str(" } else {\n");
2303
2304 writeln!(out, " (*state).pc = {};", ftgt_addr.0).unwrap();
2307
2308 self.write_branch_or_tailcall(ftgt_addr, out, 8, linked_blocks)?;
2309 out.push_str(" }\n");
2310 Ok(())
2311 }
2312
2313 fn emit_write_target(&self, dst: Insn, value_expr: String, out: &mut String) -> JitResult<()> {
2314 match self.nnil.insns()[dst.0 as usize] {
2317 NnilInstruction::Reg { reg } => {
2318 self.ensure_reg(reg)?;
2319 writeln!(out, " write_reg(state, {}, {});", reg, value_expr).unwrap();
2320 }
2321 NnilInstruction::TmpReg { reg } => {
2322 self.ensure_tmp(reg)?;
2323 writeln!(
2325 out,
2326 " *(*state).tmps.get_unchecked_mut({}) = {};",
2327 reg, value_expr
2328 )
2329 .unwrap();
2330 }
2331 ref other => {
2332 return Err(
2333 io::Error::other(format!("unexpected in emitwritetarget: {other:?}")).into(),
2334 );
2335 }
2336 }
2337 Ok(())
2338 }
2339
2340 fn block_name(&self, addr: Address) -> JitResult<&str> {
2341 self.block_names
2343 .get(&addr)
2344 .map(String::as_str)
2345 .ok_or_else(|| {
2346 io::Error::other(format!("missing block for address 0x{:x}", addr.0)).into()
2347 })
2348 }
2349
2350 fn label_address(&self, label: Label) -> JitResult<Address> {
2351 self.nnil
2353 .label_address(label)
2354 .ok_or_else(|| io::Error::other(format!("unknown label {:?}", label)).into())
2355 }
2356
2357 fn write_branch_or_tailcall(
2359 &self,
2360 addr: Address,
2361 out: &mut String,
2362 indent: usize,
2363 linked_blocks: &BTreeMap<Address, String>,
2364 ) -> JitResult<()> {
2365 let padding = " ".repeat(indent);
2368
2369 if let Some(name) = linked_blocks.get(&addr) {
2373 writeln!(out, "{padding}become {name}(state);").unwrap();
2374 } else {
2375 writeln!(out, "{padding}become dispatch_jit(state);").unwrap();
2376 }
2377 Ok(())
2378 }
2379
2380 fn direct_jump_target(&self, dst: Insn) -> Option<Address> {
2381 match self.nnil.insns()[dst.0 as usize] {
2383 NnilInstruction::Imm { imm } if imm >= 0 => Some(Address(imm as u64)),
2384 _ => None,
2385 }
2386 }
2387
2388 fn load_mode(&self, load_idx: usize, dst: Insn) -> LoadMode {
2389 if matches!(self.nnil.insns()[dst.0 as usize], NnilInstruction::Reg { reg } if reg as usize >= FP_REG_BASE)
2392 {
2393 return LoadMode::Raw;
2394 }
2395
2396 if self.unsigned_markers.contains(&(load_idx as u64)) {
2398 LoadMode::ZeroExtend
2399 } else {
2400 LoadMode::SignExtend
2401 }
2402 }
2403
2404 fn signedness(&self, insn: Insn) -> Signedness {
2405 match self.nnil.insns()[insn.0 as usize] {
2407 NnilInstruction::Unsigned { .. } => Signedness::Unsigned,
2408 _ => Signedness::Signed,
2409 }
2410 }
2411
2412 fn binary_signedness(&self, lhs: Insn, rhs: Insn) -> Signedness {
2413 if self.signedness(lhs) == Signedness::Unsigned
2415 || self.signedness(rhs) == Signedness::Unsigned
2416 {
2417 Signedness::Unsigned
2418 } else {
2419 Signedness::Signed
2420 }
2421 }
2422
2423 fn ensure_reg(&self, reg: u64) -> JitResult<()> {
2424 if reg as usize >= GUEST_REG_COUNT {
2426 Err(io::Error::other(format!("register {} exceeds register file", reg)).into())
2427 } else {
2428 Ok(())
2429 }
2430 }
2431
2432 fn ensure_tmp(&self, reg: u64) -> JitResult<()> {
2433 if reg as usize >= GUEST_TMP_COUNT {
2435 Err(io::Error::other(format!("temporary register {} exceeds tmp file", reg)).into())
2436 } else {
2437 Ok(())
2438 }
2439 }
2440
2441 fn ensure_load_store_size(&self, size: u64) -> JitResult<()> {
2442 match size {
2445 1 | 2 | 4 | 8 => Ok(()),
2446 other => Err(io::Error::other(format!("memory size unsupported: {}", other)).into()),
2447 }
2448 }
2449}
2450
2451fn collect_blocks(nnil: &Nnil) -> JitResult<Vec<BasicBlock>> {
2452 let starts = nnil
2454 .instruction_starts()
2455 .iter()
2456 .map(|(addr, insn)| (*addr, insn.0 as usize))
2457 .collect::<Vec<_>>();
2458
2459 if starts.is_empty() {
2460 return Err(io::Error::other("leere NNIL!").into());
2461 }
2462
2463 let mut blocks = Vec::new();
2465 let mut block_start_addr = starts[0].0;
2466 let mut block_start_insn = starts[0].1;
2467
2468 for (idx, &(addr, insn_start)) in starts.iter().enumerate() {
2471 let next = starts.get(idx + 1).copied();
2472
2473 let insn_end = next
2475 .map(|(_, next_start)| next_start)
2476 .unwrap_or_else(|| nnil.insns().len());
2477
2478 let guest_insn = &nnil.insns()[insn_start..insn_end];
2479
2480 let terminates = is_terminator(guest_insn);
2482
2483 let next_is_label = next
2485 .map(|(next_addr, _)| nnil.label_for_address(next_addr).is_some())
2486 .unwrap_or(false);
2487
2488 let next_pc = next
2490 .map(|(next_addr, _)| next_addr.0)
2491 .unwrap_or_else(|| nnil.pc());
2492 let should_end = terminates || next.is_none() || next_is_label;
2493
2494 if should_end {
2495 let fallthrough = if terminates {
2497 None
2498 } else {
2499 next.map(|(next_addr, _)| next_addr)
2500 };
2501
2502 blocks.push(BasicBlock {
2503 addr: block_start_addr,
2504 start_insn: block_start_insn,
2505 end_insn: insn_end,
2506 fallthrough,
2507 next_pc,
2508 });
2509
2510 if let Some((next_addr, next_start)) = next {
2513 block_start_addr = next_addr;
2515 block_start_insn = next_start;
2516 }
2517 }
2518
2519 let _ = addr;
2521 }
2522
2523 Ok(blocks)
2524}
2525
2526fn is_terminator(insns: &[NnilInstruction]) -> bool {
2528 matches!(
2531 insns.last(),
2532 Some(
2533 NnilInstruction::Jr { .. }
2534 | NnilInstruction::Jmp { .. }
2535 | NnilInstruction::Beq { .. }
2536 | NnilInstruction::Bne { .. }
2537 | NnilInstruction::Blt { .. }
2538 | NnilInstruction::Bgt { .. }
2539 | NnilInstruction::Syscall {}
2540 | NnilInstruction::Sysbreak {}
2541 )
2542 )
2543}