1use liil::blocks::direct_successors;
6use liil::nnil::{Address, Nnil};
7
8mod emitter;
9mod mmu;
10mod syscall;
11
12use emitter::CompiledVectorBlock;
13
14use jit::{GUEST_REG_COUNT, GUEST_TMP_COUNT, GuestMemory, GuestState, INTEGER_ZERO_REG, PAGE_SIZE, PagePerms};
15
16use mmu::{MemorySnapshot, VectorMemory};
17mod abi;
18
19pub use abi::{COVERAGE_BITS, FaultAccess, LANES, LaneExit};
20
21use abi::{COVERAGE_WORDS, VectorStateAbi};
22use syscall::{LaneProcess, LaneProcessSnapshot};
23
24use std::collections::{BTreeMap, BTreeSet, VecDeque};
25use std::error::Error;
26use std::fs;
27use std::io;
28use std::path::PathBuf;
29
30pub use mmu::BytePerms;
31
32const DEFAULT_BATCH_SIZE: usize = 64;
33
34pub type VectorResult<T> = Result<T, Box<dyn Error>>;
35
36#[derive(Debug, Clone)]
37pub struct VectorConfig {
38 pub output_dir: PathBuf,
39 pub cache_dir: Option<PathBuf>,
40 pub opt_level: u8,
41 pub batch_size: usize,
42 pub argv: Vec<String>,
43 pub sysroot: Option<PathBuf>,
44 pub workdir: Option<PathBuf>,
45}
46
47impl Default for VectorConfig {
48 fn default() -> Self {
49 Self {
50 output_dir: PathBuf::from("output"),
51 cache_dir: Some(PathBuf::from("jit-tmp/vector-cache")),
52 opt_level: 2,
53 batch_size: DEFAULT_BATCH_SIZE,
54 argv: Vec::new(),
55 sysroot: None,
56 workdir: None,
57 }
58 }
59}
60
61impl LaneExit {
62 fn from_raw(value: u8) -> Self {
63 [
64 Self::Running,
65 Self::ReadFault,
66 Self::WriteFault,
67 Self::ExecFault,
68 Self::Syscall,
69 Self::Sysbreak,
70 Self::BlockLimit,
71 Self::Unsupported,
72 Self::GuestExit,
73 Self::UnsupportedCodeDivergence,
74 ]
75 .get(value as usize)
76 .copied()
77 .unwrap_or(Self::Unsupported)
78 }
79}
80
81impl FaultAccess {
82 fn from_raw(value: u8) -> Self {
83 [Self::None, Self::Read, Self::Write, Self::Execute].get(value as usize).copied().unwrap_or(Self::None)
84 }
85}
86
87#[derive(Debug)]
88pub struct LaneResult {
89 pub exit: LaneExit,
90 pub pc: u64,
91
92 pub fault_address: u64,
93 pub fault_access: FaultAccess,
94
95 pub blocks: u64,
96 pub exit_status: Option<u32>,
97 pub last_syscall: Option<[u64; 8]>,
98 pub registers: [u64; GUEST_REG_COUNT],
99
100 pub coverage: Box<[u64]>, }
102
103struct StateSnapshot {
104 regs: [[u64; LANES]; GUEST_REG_COUNT],
105 tmps: [[u64; LANES]; GUEST_TMP_COUNT],
106 pcs: [u64; LANES],
107 previous_pcs: [u64; LANES],
108}
109
110pub struct VectorSnapshot {
111 state: StateSnapshot,
112 memory: MemorySnapshot,
113 processes: [LaneProcessSnapshot; LANES],
114}
115
116pub struct LaneMut<'a> {
117 lane: usize,
118 state: &'a mut VectorStateAbi,
119 memory: &'a mut VectorMemory,
120}
121
122impl LaneMut<'_> {
123 pub fn lane(&self) -> usize {
124 self.lane
125 }
126
127 pub fn pc(&self) -> u64 {
128 self.state.pcs[self.lane]
129 }
130
131 pub fn set_pc(&mut self, pc: u64) {
132 self.state.pcs[self.lane] = pc;
133 }
134
135 pub fn register(&self, register: usize) -> Option<u64> {
136 self.state.regs.get(register).map(|values| values[self.lane])
137 }
138
139 pub fn set_register(&mut self, register: usize, value: u64) -> VectorResult<()> {
140 if register >= GUEST_REG_COUNT {
141 return Err(io::Error::other("guest register ist nicht im register file").into());
142 }
143
144 if register != INTEGER_ZERO_REG {
145 self.state.regs[register][self.lane] = value;
146 }
147
148 Ok(())
149 }
150
151 pub fn write_memory(&mut self, address: u64, bytes: &[u8]) -> VectorResult<()> {
152 if !self.memory.write_lane_host(self.lane, address, bytes) {
153 return Err(io::Error::other("write_memory mit lane host failed").into());
154 }
155
156 Ok(())
157 }
158
159 pub fn read_memory(&mut self, address: u64, bytes: &mut [u8]) -> VectorResult<()> {
160 if !self.memory.read_lane(self.lane, address, bytes) {
161 return Err(io::Error::other("input read is outside vector memory").into());
162 }
163
164 Ok(())
165 }
166
167 pub fn set_permissions(&mut self, address: u64, size: usize, permissions: BytePerms) -> VectorResult<()> {
168 if permissions.contains(BytePerms::EXEC) {
169 return Err(io::Error::other("execute permission nicht erlaubt").into());
170 }
171
172 if !self.memory.set_permissions(self.lane, address, size, permissions) {
173 return Err(io::Error::other("permission range is outside vector memory").into());
174 }
175
176 Ok(())
177 }
178}
179
180pub struct VectorRuntime<L>
181where
182 L: Fn(&GuestMemory, u64) -> VectorResult<Nnil>,
183{
184 lift_memory: GuestMemory,
185 memory: VectorMemory,
186 state: Box<VectorStateAbi>,
187
188 coverage: Box<[u64]>, cache: BTreeMap<u64, CompiledVectorBlock>,
191 executable_pages: BTreeMap<u64, Box<[u8]>>,
192 processes: [LaneProcess; LANES],
193 config: VectorConfig,
194
195 lift_block: L,
196}
197
198impl<L> VectorRuntime<L>
199where
200 L: Fn(&GuestMemory, u64) -> VectorResult<Nnil>,
201{
202 pub fn from_template(
203 memory: &GuestMemory,
204 state: &GuestState,
205 lift_block: L,
206 mut config: VectorConfig,
207 ) -> VectorResult<Self> {
208 require_avx512()?;
209
210 if memory.mapped_pages().count() as u64 * PAGE_SIZE as u64 > jit::MAX_GUEST_MEMORY / 4 {
211 return Err(io::Error::other("vector memory limit exceeded").into());
212 }
213
214 if let Some(sysroot) = &config.sysroot {
223 config.sysroot = Some(fs::canonicalize(sysroot).map_err(|error| {
224 io::Error::other(format!("cannot open vector sysroot {}: {error}", sysroot.display()))
225 })?);
226 }
227
228 let workdir = config.workdir.clone().unwrap_or(std::env::current_dir()?);
229
230 let workdir = fs::canonicalize(&workdir)
231 .map_err(|error| io::Error::other(format!("cannot open vector workdir {}: {error}", workdir.display())))?;
232
233 if !workdir.is_dir() {
234 return Err(io::Error::other("vector workdir is not a directory!!").into());
235 }
236
237 config.workdir = Some(workdir.clone());
238
239 let mut executable_pages = BTreeMap::new();
240
241 for (base, perms, bytes) in memory.mapped_pages() {
242 if perms.contains(PagePerms::EXEC) && perms.contains(PagePerms::WRITE) {
243 return Err(
244 io::Error::other(format!("vector memory page 0x{base:x} is writable and executable")).into()
245 );
246 }
247
248 if perms.contains(PagePerms::EXEC) {
249 executable_pages.insert(base, bytes.to_vec().into_boxed_slice());
250 }
251 }
252
253 let mut regs = [[0; LANES]; GUEST_REG_COUNT];
254 let mut tmps = [[0; LANES]; GUEST_TMP_COUNT];
255
256 for (register, values) in regs.iter_mut().enumerate() {
257 values.fill(state.regs[register]);
258 }
259
260 for (register, values) in tmps.iter_mut().enumerate() {
261 values.fill(state.tmps[register]);
262 }
263
264 let mut vector_memory = VectorMemory::from_guest(memory);
265
266 let processes = (0..LANES)
267 .map(|lane| {
268 LaneProcess::new(
269 &workdir,
270 lane,
271 memory.brk,
272 config.sysroot.clone(),
273 config.argv.first().cloned().unwrap_or_default(),
274 )
275 })
276 .collect::<Result<Vec<_>, _>>()?
277 .try_into()
278 .map_err(|_| io::Error::other("vector process count is not 8!!"))?;
279
280 let mut coverage = vec![0; LANES * COVERAGE_WORDS].into_boxed_slice();
281
282 let vector_state = Box::new(VectorStateAbi {
283 regs,
284 tmps,
285 pcs: [state.pc; LANES],
286 previous_pcs: [state.pc; LANES],
287 last_edges: [u64::MAX; LANES],
288
289 fault_addresses: [0; LANES],
290 block_counts: [0; LANES],
291 block_limits: [u64::MAX; LANES],
292
293 memory: vector_memory.as_opaque_mut(),
294 coverage: coverage.as_mut_ptr(), load: VectorMemory::load_fn(),
297 store: VectorMemory::store_fn(),
298 exec: VectorMemory::exec_fn(),
299
300 exec_cache: vector_memory.exec_cache_mut(),
301 reasons: [LaneExit::Running as u8; LANES],
302 fault_access: [FaultAccess::None as u8; LANES],
303
304 live_mask: u8::MAX,
305 active_mask: 0,
306 next_lane: 0,
307 pending_syscall_mask: 0,
308 _padding: [0; 12],
309 });
310
311 Ok(Self {
312 lift_memory: memory.clone(),
313 memory: vector_memory,
314 state: vector_state,
315 coverage,
316 cache: BTreeMap::new(),
317 executable_pages,
318 processes,
319 config,
320 lift_block,
321 })
322 }
323
324 pub fn snapshot(&mut self) -> VectorResult<VectorSnapshot> {
325 let processes = self
326 .processes
327 .iter_mut()
328 .map(LaneProcess::snapshot)
329 .collect::<Result<Vec<_>, _>>()?
330 .try_into()
331 .map_err(|_| io::Error::other("vector process snapshot count is not eight"))?;
332
333 Ok(VectorSnapshot {
334 state: StateSnapshot {
335 regs: self.state.regs,
336 tmps: self.state.tmps,
337 pcs: self.state.pcs,
338 previous_pcs: self.state.previous_pcs,
339 },
340
341 memory: self.memory.snapshot(),
342 processes,
343 })
344 }
345
346 pub fn run_batch<F>(
347 &mut self,
348 snapshot: &VectorSnapshot,
349 inputs: [&[u8]; LANES],
350 block_limit: u64,
351 mut inject: F,
352 ) -> VectorResult<[LaneResult; LANES]>
353 where
354 F: FnMut(usize, &[u8], &mut LaneMut<'_>) -> VectorResult<()>,
355 {
356 self.restore(snapshot)?;
357
358 self.coverage.fill(0);
359
360 self.state.last_edges.fill(u64::MAX);
361 self.state.coverage = self.coverage.as_mut_ptr();
362 self.state.reasons.fill(LaneExit::Running as u8);
363
364 self.state.fault_access.fill(FaultAccess::None as u8);
365 self.state.fault_addresses.fill(0);
366
367 self.state.block_counts.fill(0);
368 self.state.block_limits.fill(block_limit);
369
370 self.state.live_mask = u8::MAX;
371 self.state.active_mask = 0;
372 self.state.next_lane = 0;
373 self.state.pending_syscall_mask = 0;
374
375 for process in &mut self.processes {
376 process.exit_status = None;
377 process.last_syscall = None;
378 }
379
380 for lane in 0..LANES {
381 let mut lane_state = LaneMut { lane, state: &mut self.state, memory: &mut self.memory };
382
383 inject(lane, inputs[lane], &mut lane_state)?;
384 }
385
386 self.sync_abi();
387 self.run()?;
388
389 Ok(std::array::from_fn(|lane| {
390 let mut registers = [0; GUEST_REG_COUNT];
391
392 for (register, value) in registers.iter_mut().enumerate() {
393 *value = self.state.regs[register][lane];
394 }
395
396 let coverage_start = lane * COVERAGE_WORDS;
397
398 LaneResult {
399 exit: LaneExit::from_raw(self.state.reasons[lane]),
400 pc: self.state.pcs[lane],
401
402 fault_address: self.state.fault_addresses[lane],
403 fault_access: FaultAccess::from_raw(self.state.fault_access[lane]),
404
405 blocks: self.state.block_counts[lane],
406 exit_status: self.processes[lane].exit_status,
407 last_syscall: self.processes[lane].last_syscall,
408 registers,
409
410 coverage: self.coverage[coverage_start..coverage_start + COVERAGE_WORDS].to_vec().into_boxed_slice(),
411 }
412 }))
413 }
414
415 pub fn read_lane_memory(&mut self, lane: usize, address: u64, bytes: &mut [u8]) -> VectorResult<()> {
416 if !self.memory.read_lane(lane, address, bytes) {
417 return Err(io::Error::other("memory read failed").into());
418 }
419
420 Ok(())
421 }
422
423 pub fn cache_len(&self) -> usize {
424 self.cache.len()
425 }
426
427 fn restore(&mut self, snapshot: &VectorSnapshot) -> VectorResult<()> {
428 for (process, saved) in self.processes.iter_mut().zip(&snapshot.processes) {
429 process.restore(saved)?;
430 }
431
432 self.memory.restore(&snapshot.memory);
433
434 self.state.regs = snapshot.state.regs;
435 self.state.tmps = snapshot.state.tmps;
436 self.state.pcs = snapshot.state.pcs;
437 self.state.previous_pcs = snapshot.state.previous_pcs;
438
439 self.sync_abi();
440 Ok(())
441 }
442
443 fn sync_abi(&mut self) {
444 self.state.memory = self.memory.as_opaque_mut();
445 self.state.coverage = self.coverage.as_mut_ptr();
446 self.state.load = VectorMemory::load_fn();
447 self.state.store = VectorMemory::store_fn();
448 self.state.exec = VectorMemory::exec_fn();
449 self.state.exec_cache = self.memory.exec_cache_mut();
450
451 }
453
454 fn run(&mut self) -> VectorResult<()> {
455 while self.state.live_mask != 0 {
456 let lane = self.select_lane().unwrap();
457 let pc = self.state.pcs[lane];
458
459 let mut group = 0u8;
460
461 for candidate in 0..LANES {
462 if self.state.live_mask & (1 << candidate) != 0 && self.state.pcs[candidate] == pc {
463 group |= 1 << candidate;
464 }
465 }
466
467 let mut active = group;
468
469 for candidate in 0..LANES {
470 let bit = 1 << candidate;
471
472 if active & bit == 0 {
473 continue;
474 }
475
476 if self.state.block_counts[candidate] >= self.state.block_limits[candidate] {
477 self.stop_lane(candidate, LaneExit::BlockLimit, FaultAccess::None, 0);
478 active &= !bit;
479
480 continue;
481 }
482
483 if let Err(address) = self.memory.check_execute(candidate, pc, 1) {
484 self.stop_lane(candidate, LaneExit::ExecFault, FaultAccess::Execute, address);
485
486 active &= !bit;
487 continue;
488 }
489 }
490
491 if active != 0 {
492 if !self.ensure_compiled(pc)? {
493 for candidate in 0..LANES {
494 if active & (1 << candidate) != 0 {
495 self.stop_lane(candidate, LaneExit::Unsupported, FaultAccess::None, pc);
496 }
497 }
498
499 continue;
500 }
501
502 self.state.active_mask = active;
503 self.state.next_lane = ((lane + 1) % LANES) as u8;
504
505 let block = self.cache.get(&pc).unwrap();
506
507 unsafe { block.run(&mut self.state, pc) };
509
510 self.service_pending_syscalls()?;
511 }
512
513 self.state.active_mask = 0;
514 }
515 Ok(())
516 }
517
518 fn select_lane(&self) -> Option<usize> {
519 (0..LANES)
520 .map(|offset| (self.state.next_lane as usize + offset) % LANES)
521 .find(|lane| self.state.live_mask & (1 << lane) != 0)
522 }
523
524 fn stop_lane(&mut self, lane: usize, reason: LaneExit, access: FaultAccess, address: u64) {
525 self.state.reasons[lane] = reason as u8;
526 self.state.fault_access[lane] = access as u8;
527 self.state.fault_addresses[lane] = address;
528 self.state.live_mask &= !(1 << lane);
529 self.state.active_mask &= !(1 << lane);
530 }
531
532 pub(crate) fn prepare_executable_mapping(
533 &mut self,
534 _lane: usize,
535 base: u64,
536 size: u64,
537 perms: BytePerms,
538 bytes: &[u8],
539 ) -> VectorResult<Result<(), u64>> {
540 if !perms.contains(BytePerms::EXEC) {
541 return Ok(Ok(()));
542 }
543
544 let Some(aligned_size) = size.checked_add(PAGE_SIZE as u64 - 1).map(|value| value & !((PAGE_SIZE as u64) - 1))
545 else {
546 return Ok(Err(base));
547 };
548
549 if perms.contains(BytePerms::WRITE)
550 || size == 0
551 || !base.is_multiple_of(PAGE_SIZE as u64)
552 || bytes.len() != aligned_size as usize
553 {
554 return Ok(Err(base));
555 }
556
557 for (page_index, page_bytes) in bytes.chunks_exact(PAGE_SIZE).enumerate() {
558 let page_base = base + (page_index * PAGE_SIZE) as u64;
559
560 if let Some(canonical) = self.executable_pages.get(&page_base)
561 && canonical.as_ref() != page_bytes
562 {
563 let offset = canonical.iter().zip(page_bytes).position(|(left, right)| left != right).unwrap();
564
565 return Ok(Err(page_base + offset as u64));
566 }
567 }
568
569 let mut published = false;
570 let page_perms = PagePerms::from_bits(perms.bits());
571
572 for (page_index, page_bytes) in bytes.chunks_exact(PAGE_SIZE).enumerate() {
573 let page_base = base + (page_index * PAGE_SIZE) as u64;
574
575 if self.executable_pages.contains_key(&page_base) {
576 continue;
577 }
578
579 self.lift_memory.replace_mapping(page_base, PAGE_SIZE as u64, page_perms, page_bytes)?;
580
581 self.executable_pages.insert(page_base, page_bytes.to_vec().into_boxed_slice());
582 published = true;
583 }
584
585 if published {
586 self.cache.clear();
587 }
588
589 Ok(Ok(()))
592 }
593
594 fn ensure_compiled(&mut self, pc: u64) -> VectorResult<bool> {
595 if self.cache.contains_key(&pc) {
596 return Ok(true);
597 }
598
599 let Some(regions) = self.collect_batch(pc)? else {
600 return Ok(false);
601 };
602
603 let compiled = emitter::compile_bundle(®ions, &self.config)?;
604
605 for (entry, block) in compiled {
606 self.cache.insert(entry, block);
607 }
608
609 Ok(true)
610 }
611
612 fn collect_batch(&self, start_pc: u64) -> VectorResult<Option<Vec<Nnil>>> {
613 let mut queue = VecDeque::from([start_pc]);
614 let mut seen = BTreeSet::new();
615 let mut regions = Vec::new();
616
617 while let Some(pc) = queue.pop_front()
618 && regions.len() < self.config.batch_size
619 {
620 if !seen.insert(pc) || self.cache.contains_key(&pc) {
621 continue;
622 }
623
624 let nnil = match (self.lift_block)(&self.lift_memory, pc) {
625 Ok(nnil) => nnil,
626 Err(_) if pc != start_pc => continue,
627 Err(_) => return Ok(None),
628 };
629
630 if nnil.entrypoint() != Some(Address(pc)) || nnil.pc() <= pc {
631 return Err(io::Error::other("lifter returned an invalid vector block").into());
632 }
633
634 for successor in direct_successors(&nnil) {
635 if !seen.contains(&successor) {
636 queue.push_back(successor);
637 }
638 }
639
640 regions.push(nnil);
641 }
642
643 if regions.is_empty() {
644 return Err(io::Error::other("vector compile batch leer?").into());
645 }
646
647 Ok(Some(regions))
648 }
649}
650
651fn require_avx512() -> VectorResult<()> {
652 #[cfg(target_arch = "x86_64")]
653 {
654 if std::is_x86_feature_detected!("avx512f")
655 && std::is_x86_feature_detected!("avx512dq")
656 && std::is_x86_feature_detected!("avx512bw")
657 {
658 return Ok(());
659 }
660 }
661
662 Err(io::Error::other("AVX-512 backend requires x86-64 AVX512F, AVX512DQ, and AVX512BW").into())
663}