nub_host_kvm/hypervisor/mod.rs
1/*
2Copyright 2025 The Hyperlight Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17/// Abstracts over different hypervisor register representations
18pub(crate) mod regs;
19
20pub(crate) mod virtual_machine;
21
22pub(crate) mod hyperlight_vm;
23
24use std::fmt::Debug;
25#[cfg(kvm)]
26use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, Ordering};
27#[cfg(kvm)]
28use std::time::Duration;
29
30/// A trait for platform-specific interrupt handle implementation details
31pub(crate) trait InterruptHandleImpl: InterruptHandle {
32 /// Set the thread ID for the vcpu thread
33 #[cfg(kvm)]
34 fn set_tid(&self);
35
36 /// Set the running state
37 fn set_running(&self);
38
39 /// Clear the running state
40 fn clear_running(&self);
41
42 /// Mark the handle as dropped
43 fn set_dropped(&self);
44
45 /// Check if cancellation was requested
46 fn is_cancelled(&self) -> bool;
47
48 /// Clear the cancellation request flag
49 fn clear_cancel(&self);
50
51 /// Check if debug interrupt was requested (always returns false when gdb feature is disabled)
52 fn is_debug_interrupted(&self) -> bool;
53}
54
55/// A trait for handling interrupts to a sandbox's vcpu
56pub trait InterruptHandle: Send + Sync + Debug {
57 /// Interrupt the corresponding sandbox from running.
58 ///
59 /// - If this is called while the the sandbox currently executing a guest function call, it will interrupt the sandbox and return `true`.
60 /// - If this is called while the sandbox is not running (for example before or after calling a guest function), it will do nothing and return `false`.
61 ///
62 /// # Note
63 /// This function will block for the duration of the time it takes for the vcpu thread to be interrupted.
64 fn kill(&self) -> bool;
65
66 /// Returns true if the corresponding sandbox has been dropped
67 fn dropped(&self) -> bool;
68}
69
70#[cfg(kvm)]
71#[derive(Debug)]
72pub(super) struct LinuxInterruptHandle {
73 /// Atomic value packing vcpu execution state.
74 ///
75 /// Bit layout:
76 /// - Bit 2: DEBUG_INTERRUPT_BIT - set when debugger interrupt is requested
77 /// - Bit 1: RUNNING_BIT - set when vcpu is actively running
78 /// - Bit 0: CANCEL_BIT - set when cancellation has been requested
79 ///
80 /// CANCEL_BIT persists across vcpu exits/re-entries within a single `VirtualCPU::run()` call
81 /// (e.g., during host function calls), but is cleared at the start of each new `VirtualCPU::run()` call.
82 state: AtomicU8,
83
84 /// Thread ID where the vcpu is running.
85 ///
86 /// Note: Multiple VMs may have the same `tid` (same thread runs multiple sandboxes sequentially),
87 /// but at most one VM will have RUNNING_BIT set at any given time.
88 tid: AtomicU64,
89
90 /// Whether the corresponding VM has been dropped.
91 dropped: AtomicBool,
92
93 /// Delay between retry attempts when sending signals to interrupt the vcpu.
94 retry_delay: Duration,
95
96 /// Offset from SIGRTMIN for the signal used to interrupt the vcpu thread.
97 sig_rt_min_offset: u8,
98}
99
100#[cfg(kvm)]
101impl LinuxInterruptHandle {
102 const RUNNING_BIT: u8 = 1 << 1;
103 const CANCEL_BIT: u8 = 1 << 0;
104
105 /// Get the running, cancel and debug flags atomically.
106 ///
107 /// # Memory Ordering
108 /// Uses `Acquire` ordering to synchronize with the `Release` in `set_running()` and `kill()`.
109 /// This ensures that when we observe running=true, we also see the correct `tid` value.
110 fn get_running_cancel_debug(&self) -> (bool, bool, bool) {
111 let state = self.state.load(Ordering::Acquire);
112 let running = state & Self::RUNNING_BIT != 0;
113 let cancel = state & Self::CANCEL_BIT != 0;
114 let debug = false;
115 (running, cancel, debug)
116 }
117
118 fn send_signal(&self) -> bool {
119 let signal_number = libc::SIGRTMIN() + self.sig_rt_min_offset as libc::c_int;
120 let mut sent_signal = false;
121
122 loop {
123 let (running, cancel, debug) = self.get_running_cancel_debug();
124
125 // Check if we should continue sending signals
126 // Exit if not running OR if neither cancel nor debug_interrupt is set
127 let should_continue = running && (cancel || debug);
128
129 if !should_continue {
130 break;
131 }
132
133 tracing::info!("Sending signal to kill vcpu thread...");
134 sent_signal = true;
135 // Acquire ordering to synchronize with the Release store in set_tid()
136 // This ensures we see the correct tid value for the currently running vcpu
137 unsafe {
138 libc::pthread_kill(self.tid.load(Ordering::Acquire) as _, signal_number);
139 }
140 std::thread::sleep(self.retry_delay);
141 }
142
143 sent_signal
144 }
145}
146
147#[cfg(kvm)]
148impl InterruptHandleImpl for LinuxInterruptHandle {
149 fn set_tid(&self) {
150 // Release ordering to synchronize with the Acquire load of `running` in send_signal()
151 // This ensures that when send_signal() observes RUNNING_BIT=true (via Acquire),
152 // it also sees the correct tid value stored here
153 self.tid
154 .store(unsafe { libc::pthread_self() as u64 }, Ordering::Release);
155 }
156
157 fn set_running(&self) {
158 // Release ordering to ensure that the tid store (which uses Release)
159 // is visible to any thread that observes running=true via Acquire ordering.
160 // This prevents the interrupt thread from reading a stale tid value.
161 self.state.fetch_or(Self::RUNNING_BIT, Ordering::Release);
162 }
163
164 fn is_cancelled(&self) -> bool {
165 // Acquire ordering to synchronize with the Release in kill()
166 // This ensures we see the cancel flag set by the interrupt thread
167 self.state.load(Ordering::Acquire) & Self::CANCEL_BIT != 0
168 }
169
170 fn clear_cancel(&self) {
171 // Release ordering to ensure that any operations from the previous run()
172 // are visible to other threads. While this is typically called by the vcpu thread
173 // at the start of run(), the VM itself can move between threads across guest calls.
174 self.state.fetch_and(!Self::CANCEL_BIT, Ordering::Release);
175 }
176
177 fn clear_running(&self) {
178 // Release ordering to ensure all vcpu operations are visible before clearing running
179 self.state.fetch_and(!Self::RUNNING_BIT, Ordering::Release);
180 }
181
182 fn is_debug_interrupted(&self) -> bool {
183 false
184 }
185
186 fn set_dropped(&self) {
187 // Release ordering to ensure all VM cleanup operations are visible
188 // to any thread that checks dropped() via Acquire
189 self.dropped.store(true, Ordering::Release);
190 }
191}
192
193#[cfg(kvm)]
194impl InterruptHandle for LinuxInterruptHandle {
195 fn kill(&self) -> bool {
196 // Release ordering ensures that any writes before kill() are visible to the vcpu thread
197 // when it checks is_cancelled() with Acquire ordering
198 self.state.fetch_or(Self::CANCEL_BIT, Ordering::Release);
199
200 // Send signals to interrupt the vcpu if it's currently running
201 self.send_signal()
202 }
203
204 fn dropped(&self) -> bool {
205 // Acquire ordering to synchronize with the Release in set_dropped()
206 // This ensures we see all VM cleanup operations that happened before drop
207 self.dropped.load(Ordering::Acquire)
208 }
209}
210
211#[derive(Debug)]
212pub(super) struct MultiLaneInterruptHandle {
213 handles: Vec<std::sync::Arc<dyn InterruptHandle>>,
214}
215
216impl MultiLaneInterruptHandle {
217 pub(super) fn new(handles: Vec<std::sync::Arc<dyn InterruptHandle>>) -> Self {
218 Self { handles }
219 }
220}
221
222impl InterruptHandle for MultiLaneInterruptHandle {
223 fn kill(&self) -> bool {
224 let mut interrupted = false;
225 for handle in &self.handles {
226 interrupted = handle.kill() || interrupted;
227 }
228 interrupted
229 }
230
231 fn dropped(&self) -> bool {
232 self.handles.iter().all(|handle| handle.dropped())
233 }
234}