1use crate::{JitResult, PAGE_SIZE};
2
3use goblin::elf::{Elf, header::EM_X86_64, program_header::PT_LOAD};
4use libloading::Library;
5
6use std::{
7 collections::{BTreeMap, hash_map::DefaultHasher},
8 fs,
9 hash::{Hash, Hasher},
10 io,
11 path::{Path, PathBuf},
12 process::Command,
13 sync::Arc,
14 time::{SystemTime, UNIX_EPOCH},
15};
16
17#[derive(Clone, Copy, Hash, PartialEq, Eq)]
18pub enum CompileMode {
19 Scalar, Raw,
21 Vector,
22}
23
24pub struct Artifact {
25 code: Code,
26 pub source_path: PathBuf,
27 pub library_path: PathBuf,
28 pub compiled_now: bool,
29}
30
31enum Code {
32 Shared(Library), Raw(RawCode), }
35
36impl Artifact {
37 pub unsafe fn symbol(&self, name: &[u8]) -> JitResult<*const ()> {
38 match &self.code {
39 Code::Shared(library) => Ok(unsafe { *library.get::<*const ()>(name)? }),
40 Code::Raw(code) => {
41 let offset = code
42 .symbols
43 .get(std::str::from_utf8(name)?)
44 .ok_or_else(|| io::Error::other("missing raw JIT entry"))?;
45 Ok(unsafe { code.base.add(*offset).cast() })
46 }
47 }
48 }
49}
50
51const COMMON: &[&str] = &["--edition=2021", "--cfg=remu_generated", "-C", "codegen-units=1", "-C", "panic=abort"];
52
53const SHARED: &[&str] = &["--crate-type=cdylib", "-C", "linker-flavor=ld", "-C", "link-arg=-zdefs"];
54
55const RAW: &[&str] = &["--crate-type=lib", "--emit=obj", "-C", "relocation-model=pic"];
56
57const VECTOR: &str = "target-feature=+avx512f,+avx512dq,+avx512bw";
58
59const LINK: &[&str] = &["-static", "-nostdlib", "--build-id=none"];
60
61const LINK_SCRIPT: &str = r#"SECTIONS {
63 . = 0;
64 .text : ALIGN(16) { *(.text .text.*) }
65 .rodata : ALIGN(16) { *(.rodata .rodata.*) }
66 .data : ALIGN(16) { *(.data .data.*) *(.got .got.*) }
67 .bss : ALIGN(16) { *(.bss .bss.*) *(COMMON) }
68 /DISCARD/ : { *(.eh_frame .eh_frame.*) *(.comment) *(.note .note.*) }
69}"#;
70
71pub fn compile(
72 source: &str,
73 output: &Path,
74 cache: Option<&Path>,
75 opt: u8,
76 mode: CompileMode,
77) -> JitResult<Arc<Artifact>> {
78 let mut hash = DefaultHasher::new();
80 (
81 source,
82 opt,
83 mode,
84 COMMON,
85 SHARED,
86 RAW,
87 VECTOR,
88 LINK,
89 LINK_SCRIPT,
90 )
94 .hash(&mut hash);
95
96 let root = cache.unwrap_or(output).join("artifacts-v1");
97
98 fs::create_dir_all(&root)?;
99
100 let target = root.join(format!("{:016x}", hash.finish()));
101
102 if cache.is_some() {
103 if let Ok(artifact) = load(&target, source, mode, false) {
104 return Ok(artifact);
105 }
106
107 if target.exists() {
108 fs::remove_dir_all(&target)?;
109 }
110 }
111
112 let stamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_nanos();
113 let temporary = root.join(format!("build-{}-{stamp}", std::process::id()));
114 fs::create_dir(&temporary)?;
115
116 let source_path = temporary.join("generated.rs");
117 fs::write(&source_path, source)?;
118
119 let mut rustc = Command::new("rustc");
121
122 rustc.args(COMMON).args(["-C", &format!("opt-level={opt}")]);
123 rustc.args(if mode == CompileMode::Raw { RAW } else { SHARED });
124
125 if mode != CompileMode::Raw {
126 rustc.args(["-C", "linker=ld"]);
129 }
130
131 if mode == CompileMode::Vector {
132 rustc.args(["-C", VECTOR]);
133 }
134
135 rustc.arg("-o").arg(temporary.join(if mode == CompileMode::Raw { "code.o" } else { "code.so" })).arg(&source_path);
136
137 run(&mut rustc)?;
138
139 if mode == CompileMode::Raw {
140 let script = temporary.join("layout.ld");
142 let elf_path = temporary.join("code.elf");
143
144 fs::write(&script, LINK_SCRIPT)?;
145
146 run(Command::new("ld")
148 .args(LINK)
149 .arg("-T")
150 .arg(script)
151 .arg("-o")
152 .arg(&elf_path)
153 .arg(temporary.join("code.o")))?;
154
155 let (image, symbols) = raw_image(&fs::read(elf_path)?)?;
156
157 fs::write(temporary.join("code.bin"), image)?;
158
159 let index = symbols.iter().map(|(name, offset)| format!("{offset:x} {name}\n")).collect::<String>();
160
161 fs::write(temporary.join("symbols.txt"), index)?;
162
163 for name in ["code.o", "code.elf", "layout.ld"] {
164 fs::remove_file(temporary.join(name))?;
165 }
166 }
167
168 let mut artifact = load(&temporary, source, mode, true)?;
169
170 if cache.is_some() {
171 match fs::rename(&temporary, &target) {
172 Ok(()) => {
173 let artifact = Arc::get_mut(&mut artifact).unwrap();
174
175 artifact.source_path = target.join("generated.rs");
176 artifact.library_path = target.join(if mode == CompileMode::Raw { "code.bin" } else { "code.so" });
177 }
178
179 Err(error) => {
180 if load(&target, source, mode, false).is_err() {
182 return Err(error.into());
183 }
184 }
185 }
186 }
187
188 Ok(artifact)
189}
190
191fn run(command: &mut Command) -> JitResult<()> {
192 if !command.status()?.success() {
193 return Err(io::Error::other(format!("command failed: {command:?}")).into());
194 }
195 Ok(())
196}
197
198fn load(
199 dir: &Path,
200 _source: &str, mode: CompileMode,
202 compiled_now: bool,
203) -> JitResult<Arc<Artifact>> {
204 let source_path = dir.join("generated.rs");
205
206 let library_path = dir.join(if mode == CompileMode::Raw { "code.bin" } else { "code.so" });
211
212 let code = if mode == CompileMode::Raw {
213 let image = fs::read(&library_path)?;
214 let mut symbols = BTreeMap::new();
215
216 for line in fs::read_to_string(dir.join("symbols.txt"))?.lines() {
217 let (offset, name) = line.split_once(' ').ok_or_else(|| io::Error::other("invalid raw symbol index"))?;
218
219 let offset = usize::from_str_radix(offset, 16)?;
220
221 if offset >= image.len() {
222 return Err(io::Error::other("raw entry outside image").into());
223 }
224
225 symbols.insert(name.to_owned(), offset);
226 }
227
228 if symbols.is_empty() {
229 return Err(io::Error::other("raw image has no entry??").into());
230 }
231
232 Code::Raw(RawCode::new(&image, symbols)?)
233 } else {
234 Code::Shared(unsafe { Library::new(&library_path)? })
235 };
236
237 Ok(Arc::new(Artifact { code, source_path, library_path, compiled_now }))
238}
239
240fn raw_image(bytes: &[u8]) -> JitResult<(Vec<u8>, BTreeMap<String, usize>)> {
241 let elf = Elf::parse(bytes)?;
243
244 if !elf.is_64 || !elf.little_endian || elf.header.e_machine != EM_X86_64 {
245 return Err(io::Error::other("expected x86-64 ELF64 linker output").into());
246 }
247
248 let loads = elf.program_headers.iter().filter(|ph| ph.p_type == PT_LOAD).collect::<Vec<_>>();
249
250 let base = loads.iter().map(|ph| ph.p_vaddr).min().ok_or_else(|| io::Error::other("no raw load segments"))?;
251
252 let mut end = base;
253
254 for ph in &loads {
255 if ph.p_filesz > ph.p_memsz {
256 return Err(io::Error::other("invalid raw load segment").into());
257 }
258
259 end = end.max(ph.p_vaddr.checked_add(ph.p_memsz).ok_or_else(|| io::Error::other("raw image overflow"))?);
260 }
261
262 let mut image = vec![0; usize::try_from(end - base)?];
264
265 for ph in loads {
266 let file_end =
267 ph.p_offset.checked_add(ph.p_filesz).ok_or_else(|| io::Error::other("raw file range overflow"))?;
268
269 let data = bytes
270 .get(usize::try_from(ph.p_offset)?..usize::try_from(file_end)?)
271 .ok_or_else(|| io::Error::other("truncated raw load segment"))?;
272
273 let offset = usize::try_from(ph.p_vaddr - base)?;
274 image[offset..offset + data.len()].copy_from_slice(data);
275 }
276
277 let mut symbols = BTreeMap::new();
278
279 for symbol in elf.syms.iter().filter(|s| s.st_shndx != 0 && s.st_bind() != 0) {
280 let Some(name) = elf.strtab.get_at(symbol.st_name).filter(|s| s.starts_with("execute")) else {
281 continue;
282 };
283
284 if !(base..end).contains(&symbol.st_value) {
285 return Err(io::Error::other("raw entry outside image").into());
286 }
287
288 symbols.insert(name.to_owned(), usize::try_from(symbol.st_value - base)?);
289 }
290
291 Ok((image, symbols))
292}
293
294struct RawCode {
295 base: *mut u8,
296 length: usize,
297 symbols: BTreeMap<String, usize>,
298}
299
300impl RawCode {
301 fn new(image: &[u8], symbols: BTreeMap<String, usize>) -> JitResult<Self> {
302 if image.is_empty() {
303 return Err(io::Error::other("raw JIT image is empty").into());
304 }
305
306 let length = image.len().div_ceil(PAGE_SIZE) * PAGE_SIZE;
307
308 let base =
310 unsafe { mmap(std::ptr::null_mut(), length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) };
311
312 if base as isize == -1 {
313 return Err(io::Error::last_os_error().into());
314 }
315
316 unsafe {
317 std::ptr::copy_nonoverlapping(image.as_ptr(), base.cast::<u8>(), image.len());
318 }
319
320 if unsafe { mprotect(base, length, PROT_READ | PROT_EXEC) } != 0 {
321 let error = io::Error::last_os_error();
322
323 unsafe {
324 munmap(base, length);
325 }
326
327 return Err(error.into());
328 }
329
330 Ok(Self { base: base.cast(), length, symbols })
331 }
332}
333
334impl Drop for RawCode {
335 fn drop(&mut self) {
336 unsafe {
337 munmap(self.base.cast(), self.length);
338 }
339 }
340}
341
342const PROT_READ: i32 = 1;
343const PROT_WRITE: i32 = 2;
344const PROT_EXEC: i32 = 4;
345const MAP_PRIVATE: i32 = 2;
346const MAP_ANONYMOUS: i32 = 0x20;
347
348unsafe extern "C" {
349 fn mmap(
350 address: *mut std::ffi::c_void,
351 length: usize,
352 protection: i32,
353 flags: i32,
354 fd: i32,
355 offset: i64,
356 ) -> *mut std::ffi::c_void;
357
358 fn mprotect(address: *mut std::ffi::c_void, length: usize, protection: i32) -> i32;
359 fn munmap(address: *mut std::ffi::c_void, length: usize) -> i32;
360}