Skip to main content

vector/
lib.rs

1//! - [RISC-V psABI](https://riscv-non-isa.github.io/riscv-elf-psabi-doc/)
2//! - [Linux generic syscall table](https://github.com/torvalds/linux/blob/master/include/uapi/asm-generic/unistd.h)
3//! - [Intel Intrinsics Guide](https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html)
4
5use liil::blocks::direct_successors;
6use liil::nnil::{Address, Nnil};
7
8mod emitter;
9mod mmu;
10mod syscall;
11
12use emitter::CompiledVectorBlock;
13
14use jit::{GUEST_REG_COUNT, GUEST_TMP_COUNT, GuestMemory, GuestState, INTEGER_ZERO_REG, PAGE_SIZE, PagePerms};
15
16use mmu::{MemorySnapshot, VectorMemory};
17mod abi;
18
19pub use abi::{COVERAGE_BITS, FaultAccess, LANES, LaneExit};
20
21use abi::{COVERAGE_WORDS, VectorStateAbi};
22use syscall::{LaneProcess, LaneProcessSnapshot};
23
24use std::collections::{BTreeMap, BTreeSet, VecDeque};
25use std::error::Error;
26use std::fs;
27use std::io;
28use std::path::PathBuf;
29
30pub use mmu::BytePerms;
31
32const DEFAULT_BATCH_SIZE: usize = 64;
33
34pub type VectorResult<T> = Result<T, Box<dyn Error>>;
35
36#[derive(Debug, Clone)]
37pub struct VectorConfig {
38    pub output_dir: PathBuf,
39    pub cache_dir: Option<PathBuf>,
40    pub opt_level: u8,
41    pub batch_size: usize,
42    pub argv: Vec<String>,
43    pub sysroot: Option<PathBuf>,
44    pub workdir: Option<PathBuf>,
45}
46
47impl Default for VectorConfig {
48    fn default() -> Self {
49        Self {
50            output_dir: PathBuf::from("output"),
51            cache_dir: Some(PathBuf::from("jit-tmp/vector-cache")),
52            opt_level: 2,
53            batch_size: DEFAULT_BATCH_SIZE,
54            argv: Vec::new(),
55            sysroot: None,
56            workdir: None,
57        }
58    }
59}
60
61impl LaneExit {
62    fn from_raw(value: u8) -> Self {
63        [
64            Self::Running,
65            Self::ReadFault,
66            Self::WriteFault,
67            Self::ExecFault,
68            Self::Syscall,
69            Self::Sysbreak,
70            Self::BlockLimit,
71            Self::Unsupported,
72            Self::GuestExit,
73            Self::UnsupportedCodeDivergence,
74        ]
75        .get(value as usize)
76        .copied()
77        .unwrap_or(Self::Unsupported)
78    }
79}
80
81impl FaultAccess {
82    fn from_raw(value: u8) -> Self {
83        [Self::None, Self::Read, Self::Write, Self::Execute].get(value as usize).copied().unwrap_or(Self::None)
84    }
85}
86
87#[derive(Debug)]
88pub struct LaneResult {
89    pub exit: LaneExit,
90    pub pc: u64,
91
92    pub fault_address: u64,
93    pub fault_access: FaultAccess,
94
95    pub blocks: u64,
96    pub exit_status: Option<u32>,
97    pub last_syscall: Option<[u64; 8]>,
98    pub registers: [u64; GUEST_REG_COUNT],
99
100    pub coverage: Box<[u64]>, // todo
101}
102
103struct StateSnapshot {
104    regs: [[u64; LANES]; GUEST_REG_COUNT],
105    tmps: [[u64; LANES]; GUEST_TMP_COUNT],
106    pcs: [u64; LANES],
107    previous_pcs: [u64; LANES],
108}
109
110pub struct VectorSnapshot {
111    state: StateSnapshot,
112    memory: MemorySnapshot,
113    processes: [LaneProcessSnapshot; LANES],
114}
115
116pub struct LaneMut<'a> {
117    lane: usize,
118    state: &'a mut VectorStateAbi,
119    memory: &'a mut VectorMemory,
120}
121
122impl LaneMut<'_> {
123    pub fn lane(&self) -> usize {
124        self.lane
125    }
126
127    pub fn pc(&self) -> u64 {
128        self.state.pcs[self.lane]
129    }
130
131    pub fn set_pc(&mut self, pc: u64) {
132        self.state.pcs[self.lane] = pc;
133    }
134
135    pub fn register(&self, register: usize) -> Option<u64> {
136        self.state.regs.get(register).map(|values| values[self.lane])
137    }
138
139    pub fn set_register(&mut self, register: usize, value: u64) -> VectorResult<()> {
140        if register >= GUEST_REG_COUNT {
141            return Err(io::Error::other("guest register ist nicht im register file").into());
142        }
143
144        if register != INTEGER_ZERO_REG {
145            self.state.regs[register][self.lane] = value;
146        }
147
148        Ok(())
149    }
150
151    pub fn write_memory(&mut self, address: u64, bytes: &[u8]) -> VectorResult<()> {
152        if !self.memory.write_lane_host(self.lane, address, bytes) {
153            return Err(io::Error::other("write_memory mit lane host failed").into());
154        }
155
156        Ok(())
157    }
158
159    pub fn read_memory(&mut self, address: u64, bytes: &mut [u8]) -> VectorResult<()> {
160        if !self.memory.read_lane(self.lane, address, bytes) {
161            return Err(io::Error::other("input read is outside vector memory").into());
162        }
163
164        Ok(())
165    }
166
167    pub fn set_permissions(&mut self, address: u64, size: usize, permissions: BytePerms) -> VectorResult<()> {
168        if permissions.contains(BytePerms::EXEC) {
169            return Err(io::Error::other("execute permission nicht erlaubt").into());
170        }
171
172        if !self.memory.set_permissions(self.lane, address, size, permissions) {
173            return Err(io::Error::other("permission range is outside vector memory").into());
174        }
175
176        Ok(())
177    }
178}
179
180pub struct VectorRuntime<L>
181where
182    L: Fn(&GuestMemory, u64) -> VectorResult<Nnil>,
183{
184    lift_memory: GuestMemory,
185    memory: VectorMemory,
186    state: Box<VectorStateAbi>,
187
188    coverage: Box<[u64]>, // todo
189
190    cache: BTreeMap<u64, CompiledVectorBlock>,
191    executable_pages: BTreeMap<u64, Box<[u8]>>,
192    processes: [LaneProcess; LANES],
193    config: VectorConfig,
194
195    lift_block: L,
196}
197
198impl<L> VectorRuntime<L>
199where
200    L: Fn(&GuestMemory, u64) -> VectorResult<Nnil>,
201{
202    pub fn from_template(
203        memory: &GuestMemory,
204        state: &GuestState,
205        lift_block: L,
206        mut config: VectorConfig,
207    ) -> VectorResult<Self> {
208        require_avx512()?;
209
210        if memory.mapped_pages().count() as u64 * PAGE_SIZE as u64 > jit::MAX_GUEST_MEMORY / 4 {
211            return Err(io::Error::other("vector memory limit exceeded").into());
212        }
213
214        // if config.opt_level > 2 {
215        //     return Err(io::Error::other("vector optimization level must be 0, 1, or 2").into());
216        // }
217
218        // if config.batch_size == 0 {
219        //     return Err(io::Error::other("vector batch size must not be zero").into());
220        // }
221
222        if let Some(sysroot) = &config.sysroot {
223            config.sysroot = Some(fs::canonicalize(sysroot).map_err(|error| {
224                io::Error::other(format!("cannot open vector sysroot {}: {error}", sysroot.display()))
225            })?);
226        }
227
228        let workdir = config.workdir.clone().unwrap_or(std::env::current_dir()?);
229
230        let workdir = fs::canonicalize(&workdir)
231            .map_err(|error| io::Error::other(format!("cannot open vector workdir {}: {error}", workdir.display())))?;
232
233        if !workdir.is_dir() {
234            return Err(io::Error::other("vector workdir is not a directory!!").into());
235        }
236
237        config.workdir = Some(workdir.clone());
238
239        let mut executable_pages = BTreeMap::new();
240
241        for (base, perms, bytes) in memory.mapped_pages() {
242            if perms.contains(PagePerms::EXEC) && perms.contains(PagePerms::WRITE) {
243                return Err(
244                    io::Error::other(format!("vector memory page 0x{base:x} is writable and executable")).into()
245                );
246            }
247
248            if perms.contains(PagePerms::EXEC) {
249                executable_pages.insert(base, bytes.to_vec().into_boxed_slice());
250            }
251        }
252
253        let mut regs = [[0; LANES]; GUEST_REG_COUNT];
254        let mut tmps = [[0; LANES]; GUEST_TMP_COUNT];
255
256        for (register, values) in regs.iter_mut().enumerate() {
257            values.fill(state.regs[register]);
258        }
259
260        for (register, values) in tmps.iter_mut().enumerate() {
261            values.fill(state.tmps[register]);
262        }
263
264        let mut vector_memory = VectorMemory::from_guest(memory);
265
266        let processes = (0..LANES)
267            .map(|lane| {
268                LaneProcess::new(
269                    &workdir,
270                    lane,
271                    memory.brk,
272                    config.sysroot.clone(),
273                    config.argv.first().cloned().unwrap_or_default(),
274                )
275            })
276            .collect::<Result<Vec<_>, _>>()?
277            .try_into()
278            .map_err(|_| io::Error::other("vector process count is not 8!!"))?;
279
280        let mut coverage = vec![0; LANES * COVERAGE_WORDS].into_boxed_slice();
281
282        let vector_state = Box::new(VectorStateAbi {
283            regs,
284            tmps,
285            pcs: [state.pc; LANES],
286            previous_pcs: [state.pc; LANES],
287            last_edges: [u64::MAX; LANES],
288
289            fault_addresses: [0; LANES],
290            block_counts: [0; LANES],
291            block_limits: [u64::MAX; LANES],
292
293            memory: vector_memory.as_opaque_mut(),
294            coverage: coverage.as_mut_ptr(), // tpdo
295
296            load: VectorMemory::load_fn(),
297            store: VectorMemory::store_fn(),
298            exec: VectorMemory::exec_fn(),
299
300            exec_cache: vector_memory.exec_cache_mut(),
301            reasons: [LaneExit::Running as u8; LANES],
302            fault_access: [FaultAccess::None as u8; LANES],
303
304            live_mask: u8::MAX,
305            active_mask: 0,
306            next_lane: 0,
307            pending_syscall_mask: 0,
308            _padding: [0; 12],
309        });
310
311        Ok(Self {
312            lift_memory: memory.clone(),
313            memory: vector_memory,
314            state: vector_state,
315            coverage,
316            cache: BTreeMap::new(),
317            executable_pages,
318            processes,
319            config,
320            lift_block,
321        })
322    }
323
324    pub fn snapshot(&mut self) -> VectorResult<VectorSnapshot> {
325        let processes = self
326            .processes
327            .iter_mut()
328            .map(LaneProcess::snapshot)
329            .collect::<Result<Vec<_>, _>>()?
330            .try_into()
331            .map_err(|_| io::Error::other("vector process snapshot count is not eight"))?;
332
333        Ok(VectorSnapshot {
334            state: StateSnapshot {
335                regs: self.state.regs,
336                tmps: self.state.tmps,
337                pcs: self.state.pcs,
338                previous_pcs: self.state.previous_pcs,
339            },
340
341            memory: self.memory.snapshot(),
342            processes,
343        })
344    }
345
346    pub fn run_batch<F>(
347        &mut self,
348        snapshot: &VectorSnapshot,
349        inputs: [&[u8]; LANES],
350        block_limit: u64,
351        mut inject: F,
352    ) -> VectorResult<[LaneResult; LANES]>
353    where
354        F: FnMut(usize, &[u8], &mut LaneMut<'_>) -> VectorResult<()>,
355    {
356        self.restore(snapshot)?;
357
358        self.coverage.fill(0);
359
360        self.state.last_edges.fill(u64::MAX);
361        self.state.coverage = self.coverage.as_mut_ptr();
362        self.state.reasons.fill(LaneExit::Running as u8);
363
364        self.state.fault_access.fill(FaultAccess::None as u8);
365        self.state.fault_addresses.fill(0);
366
367        self.state.block_counts.fill(0);
368        self.state.block_limits.fill(block_limit);
369
370        self.state.live_mask = u8::MAX;
371        self.state.active_mask = 0;
372        self.state.next_lane = 0;
373        self.state.pending_syscall_mask = 0;
374
375        for process in &mut self.processes {
376            process.exit_status = None;
377            process.last_syscall = None;
378        }
379
380        for lane in 0..LANES {
381            let mut lane_state = LaneMut { lane, state: &mut self.state, memory: &mut self.memory };
382
383            inject(lane, inputs[lane], &mut lane_state)?;
384        }
385
386        self.sync_abi();
387        self.run()?;
388
389        Ok(std::array::from_fn(|lane| {
390            let mut registers = [0; GUEST_REG_COUNT];
391
392            for (register, value) in registers.iter_mut().enumerate() {
393                *value = self.state.regs[register][lane];
394            }
395
396            let coverage_start = lane * COVERAGE_WORDS;
397
398            LaneResult {
399                exit: LaneExit::from_raw(self.state.reasons[lane]),
400                pc: self.state.pcs[lane],
401
402                fault_address: self.state.fault_addresses[lane],
403                fault_access: FaultAccess::from_raw(self.state.fault_access[lane]),
404
405                blocks: self.state.block_counts[lane],
406                exit_status: self.processes[lane].exit_status,
407                last_syscall: self.processes[lane].last_syscall,
408                registers,
409
410                coverage: self.coverage[coverage_start..coverage_start + COVERAGE_WORDS].to_vec().into_boxed_slice(),
411            }
412        }))
413    }
414
415    pub fn read_lane_memory(&mut self, lane: usize, address: u64, bytes: &mut [u8]) -> VectorResult<()> {
416        if !self.memory.read_lane(lane, address, bytes) {
417            return Err(io::Error::other("memory read failed").into());
418        }
419
420        Ok(())
421    }
422
423    pub fn cache_len(&self) -> usize {
424        self.cache.len()
425    }
426
427    fn restore(&mut self, snapshot: &VectorSnapshot) -> VectorResult<()> {
428        for (process, saved) in self.processes.iter_mut().zip(&snapshot.processes) {
429            process.restore(saved)?;
430        }
431
432        self.memory.restore(&snapshot.memory);
433
434        self.state.regs = snapshot.state.regs;
435        self.state.tmps = snapshot.state.tmps;
436        self.state.pcs = snapshot.state.pcs;
437        self.state.previous_pcs = snapshot.state.previous_pcs;
438
439        self.sync_abi();
440        Ok(())
441    }
442
443    fn sync_abi(&mut self) {
444        self.state.memory = self.memory.as_opaque_mut();
445        self.state.coverage = self.coverage.as_mut_ptr();
446        self.state.load = VectorMemory::load_fn();
447        self.state.store = VectorMemory::store_fn();
448        self.state.exec = VectorMemory::exec_fn();
449        self.state.exec_cache = self.memory.exec_cache_mut();
450
451        // dbg!(self.state.memory);
452    }
453
454    fn run(&mut self) -> VectorResult<()> {
455        while self.state.live_mask != 0 {
456            let lane = self.select_lane().unwrap();
457            let pc = self.state.pcs[lane];
458
459            let mut group = 0u8;
460
461            for candidate in 0..LANES {
462                if self.state.live_mask & (1 << candidate) != 0 && self.state.pcs[candidate] == pc {
463                    group |= 1 << candidate;
464                }
465            }
466
467            let mut active = group;
468
469            for candidate in 0..LANES {
470                let bit = 1 << candidate;
471
472                if active & bit == 0 {
473                    continue;
474                }
475
476                if self.state.block_counts[candidate] >= self.state.block_limits[candidate] {
477                    self.stop_lane(candidate, LaneExit::BlockLimit, FaultAccess::None, 0);
478                    active &= !bit;
479
480                    continue;
481                }
482
483                if let Err(address) = self.memory.check_execute(candidate, pc, 1) {
484                    self.stop_lane(candidate, LaneExit::ExecFault, FaultAccess::Execute, address);
485
486                    active &= !bit;
487                    continue;
488                }
489            }
490
491            if active != 0 {
492                if !self.ensure_compiled(pc)? {
493                    for candidate in 0..LANES {
494                        if active & (1 << candidate) != 0 {
495                            self.stop_lane(candidate, LaneExit::Unsupported, FaultAccess::None, pc);
496                        }
497                    }
498
499                    continue;
500                }
501
502                self.state.active_mask = active;
503                self.state.next_lane = ((lane + 1) % LANES) as u8;
504
505                let block = self.cache.get(&pc).unwrap();
506
507                // topdo
508                unsafe { block.run(&mut self.state, pc) };
509
510                self.service_pending_syscalls()?;
511            }
512
513            self.state.active_mask = 0;
514        }
515        Ok(())
516    }
517
518    fn select_lane(&self) -> Option<usize> {
519        (0..LANES)
520            .map(|offset| (self.state.next_lane as usize + offset) % LANES)
521            .find(|lane| self.state.live_mask & (1 << lane) != 0)
522    }
523
524    fn stop_lane(&mut self, lane: usize, reason: LaneExit, access: FaultAccess, address: u64) {
525        self.state.reasons[lane] = reason as u8;
526        self.state.fault_access[lane] = access as u8;
527        self.state.fault_addresses[lane] = address;
528        self.state.live_mask &= !(1 << lane);
529        self.state.active_mask &= !(1 << lane);
530    }
531
532    pub(crate) fn prepare_executable_mapping(
533        &mut self,
534        _lane: usize,
535        base: u64,
536        size: u64,
537        perms: BytePerms,
538        bytes: &[u8],
539    ) -> VectorResult<Result<(), u64>> {
540        if !perms.contains(BytePerms::EXEC) {
541            return Ok(Ok(()));
542        }
543
544        let Some(aligned_size) = size.checked_add(PAGE_SIZE as u64 - 1).map(|value| value & !((PAGE_SIZE as u64) - 1))
545        else {
546            return Ok(Err(base));
547        };
548
549        if perms.contains(BytePerms::WRITE)
550            || size == 0
551            || !base.is_multiple_of(PAGE_SIZE as u64)
552            || bytes.len() != aligned_size as usize
553        {
554            return Ok(Err(base));
555        }
556
557        for (page_index, page_bytes) in bytes.chunks_exact(PAGE_SIZE).enumerate() {
558            let page_base = base + (page_index * PAGE_SIZE) as u64;
559
560            if let Some(canonical) = self.executable_pages.get(&page_base)
561                && canonical.as_ref() != page_bytes
562            {
563                let offset = canonical.iter().zip(page_bytes).position(|(left, right)| left != right).unwrap();
564
565                return Ok(Err(page_base + offset as u64));
566            }
567        }
568
569        let mut published = false;
570        let page_perms = PagePerms::from_bits(perms.bits());
571
572        for (page_index, page_bytes) in bytes.chunks_exact(PAGE_SIZE).enumerate() {
573            let page_base = base + (page_index * PAGE_SIZE) as u64;
574
575            if self.executable_pages.contains_key(&page_base) {
576                continue;
577            }
578
579            self.lift_memory.replace_mapping(page_base, PAGE_SIZE as u64, page_perms, page_bytes)?;
580
581            self.executable_pages.insert(page_base, page_bytes.to_vec().into_boxed_slice());
582            published = true;
583        }
584
585        if published {
586            self.cache.clear();
587        }
588
589        //dbg!(page_perms);
590
591        Ok(Ok(()))
592    }
593
594    fn ensure_compiled(&mut self, pc: u64) -> VectorResult<bool> {
595        if self.cache.contains_key(&pc) {
596            return Ok(true);
597        }
598
599        let Some(regions) = self.collect_batch(pc)? else {
600            return Ok(false);
601        };
602
603        let compiled = emitter::compile_bundle(&regions, &self.config)?;
604
605        for (entry, block) in compiled {
606            self.cache.insert(entry, block);
607        }
608
609        Ok(true)
610    }
611
612    fn collect_batch(&self, start_pc: u64) -> VectorResult<Option<Vec<Nnil>>> {
613        let mut queue = VecDeque::from([start_pc]);
614        let mut seen = BTreeSet::new();
615        let mut regions = Vec::new();
616
617        while let Some(pc) = queue.pop_front()
618            && regions.len() < self.config.batch_size
619        {
620            if !seen.insert(pc) || self.cache.contains_key(&pc) {
621                continue;
622            }
623
624            let nnil = match (self.lift_block)(&self.lift_memory, pc) {
625                Ok(nnil) => nnil,
626                Err(_) if pc != start_pc => continue,
627                Err(_) => return Ok(None),
628            };
629
630            if nnil.entrypoint() != Some(Address(pc)) || nnil.pc() <= pc {
631                return Err(io::Error::other("lifter returned an invalid vector block").into());
632            }
633
634            for successor in direct_successors(&nnil) {
635                if !seen.contains(&successor) {
636                    queue.push_back(successor);
637                }
638            }
639
640            regions.push(nnil);
641        }
642
643        if regions.is_empty() {
644            return Err(io::Error::other("vector compile batch leer?").into());
645        }
646
647        Ok(Some(regions))
648    }
649}
650
651fn require_avx512() -> VectorResult<()> {
652    #[cfg(target_arch = "x86_64")]
653    {
654        if std::is_x86_feature_detected!("avx512f")
655            && std::is_x86_feature_detected!("avx512dq")
656            && std::is_x86_feature_detected!("avx512bw")
657        {
658            return Ok(());
659        }
660    }
661
662    Err(io::Error::other("AVX-512 backend requires x86-64 AVX512F, AVX512DQ, and AVX512BW").into())
663}