Nothing to see here, move along meow
1use core::ptr;
2
3use lancer_user::show;
4use lancer_user::syscall;
5
6use crate::virtqueue::{QUEUE_SIZE, Virtqueue};
7
8const PCI_CAP_SLOT: u64 = 2;
9
10const VIRTIO_PCI_CAP_COMMON_CFG: u8 = 1;
11const VIRTIO_PCI_CAP_NOTIFY_CFG: u8 = 2;
12const VIRTIO_PCI_CAP_ISR_CFG: u8 = 3;
13const VIRTIO_PCI_CAP_DEVICE_CFG: u8 = 4;
14
15const VIRTIO_F_VERSION_1: u64 = 1 << 32;
16const VIRTIO_F_ACCESS_PLATFORM: u64 = 1 << 33;
17const VIRTIO_NET_F_MAC: u64 = 1 << 5;
18
19const STATUS_ACKNOWLEDGE: u8 = 1;
20const STATUS_DRIVER: u8 = 2;
21const STATUS_FEATURES_OK: u8 = 8;
22const STATUS_DRIVER_OK: u8 = 4;
23const STATUS_FAILED: u8 = 128;
24
25#[derive(Default)]
26struct VirtioPciCaps {
27 common_cfg_offset: u32,
28 common_cfg_bar: u8,
29 notify_offset: u32,
30 notify_bar: u8,
31 notify_off_multiplier: u32,
32 isr_offset: u32,
33 isr_bar: u8,
34 device_cfg_offset: u32,
35 device_cfg_bar: u8,
36}
37
38#[allow(dead_code)]
39pub struct VirtioNetDevice {
40 mmio_base: usize,
41 common_cfg: usize,
42 queue_doorbells: [usize; 2],
43 isr_base: usize,
44 device_cfg: usize,
45 pub rx_queue: Virtqueue,
46 pub tx_queue: Virtqueue,
47 pub mac: [u8; 6],
48 pub dma_virt_base: usize,
49 pub rx_buf_frame_phys: [u64; RX_BUF_FRAME_COUNT],
50 pub rx_buf_virt_base: usize,
51 pub tx_buf_frame_phys: [u64; TX_BUF_FRAME_COUNT],
52 pub tx_next_buf: usize,
53 pub rx_desc_to_buf: [u16; QUEUE_SIZE as usize],
54}
55
56const MMIO_BASE_VADDR: u64 = 0x4000_0000;
57const DMA_BASE_VADDR: u64 = 0x4010_0000;
58const DMA_FRAME_COUNT: u64 = 16;
59
60const RX_DESC_FRAME: usize = 0;
61const TX_DESC_FRAME: usize = 1;
62const RX_BUF_FRAME_START: usize = 2;
63const RX_BUF_FRAME_COUNT: usize = 8;
64pub const TX_BUF_FRAME_START: usize = 10;
65pub const TX_BUF_FRAME_COUNT: usize = (DMA_FRAME_COUNT as usize) - TX_BUF_FRAME_START;
66pub const TX_BUF_SIZE: usize = 2048;
67pub const TX_BUF_COUNT: usize = (TX_BUF_FRAME_COUNT * 4096) / TX_BUF_SIZE;
68
69pub const RX_BUF_SIZE: usize = 2048;
70pub const RX_BUF_COUNT: usize = (RX_BUF_FRAME_COUNT * 4096) / RX_BUF_SIZE;
71
72fn read_mmio_u8(base: usize, offset: u32) -> u8 {
73 unsafe { ptr::read_volatile((base + offset as usize) as *const u8) }
74}
75
76fn write_mmio_u8(base: usize, offset: u32, val: u8) {
77 unsafe { ptr::write_volatile((base + offset as usize) as *mut u8, val) }
78}
79
80fn read_mmio_u16(base: usize, offset: u32) -> u16 {
81 unsafe { ptr::read_volatile((base + offset as usize) as *const u16) }
82}
83
84fn write_mmio_u16(base: usize, offset: u32, val: u16) {
85 unsafe { ptr::write_volatile((base + offset as usize) as *mut u16, val) }
86}
87
88fn read_mmio_u32(base: usize, offset: u32) -> u32 {
89 unsafe { ptr::read_volatile((base + offset as usize) as *const u32) }
90}
91
92fn write_mmio_u32(base: usize, offset: u32, val: u32) {
93 unsafe { ptr::write_volatile((base + offset as usize) as *mut u32, val) }
94}
95
96fn write_mmio_u64(base: usize, offset: u32, val: u64) {
97 unsafe { ptr::write_volatile((base + offset as usize) as *mut u64, val) }
98}
99
100fn discover_caps() -> VirtioPciCaps {
101 let status_cmd = syscall::pci_config_read32(PCI_CAP_SLOT, 0x04);
102 if status_cmd < 0 {
103 show!(virtio, warn, "caps config_read failed");
104 return VirtioPciCaps::default();
105 }
106 let status = (status_cmd as u32 >> 16) & 0xFFFF;
107 if status & 0x10 == 0 {
108 show!(virtio, warn, "caps no cap list bit");
109 return VirtioPciCaps::default();
110 }
111
112 let cap_ptr_word = syscall::pci_config_read32(PCI_CAP_SLOT, 0x34);
113 if cap_ptr_word < 0 {
114 show!(virtio, warn, "caps cap_ptr read fail");
115 return VirtioPciCaps::default();
116 }
117 let mut caps = VirtioPciCaps::default();
118 let mut ptr = (cap_ptr_word as u32) & 0xFC;
119
120 core::iter::from_fn(|| match ptr {
121 0 => None,
122 offset => {
123 let header = syscall::pci_config_read32(PCI_CAP_SLOT, offset as u64);
124 if header < 0 {
125 return None;
126 }
127 let cap_id = (header as u32) & 0xFF;
128 let next = ((header as u32) >> 8) & 0xFC;
129 let current = offset;
130 ptr = next;
131 Some((current, cap_id, header as u32))
132 }
133 })
134 .take(48)
135 .filter(|&(_, cap_id, _)| cap_id == 0x09)
136 .for_each(|(offset, _, header)| {
137 let cfg_type = (header >> 24) & 0xFF;
138
139 let word1 = syscall::pci_config_read32(PCI_CAP_SLOT, (offset + 4) as u64);
140 if word1 < 0 {
141 return;
142 }
143 let bar = (word1 as u32) & 0xFF;
144
145 let cap_offset = syscall::pci_config_read32(PCI_CAP_SLOT, (offset + 8) as u64);
146 if cap_offset < 0 {
147 return;
148 }
149
150 match cfg_type as u8 {
151 VIRTIO_PCI_CAP_COMMON_CFG => {
152 caps.common_cfg_offset = cap_offset as u32;
153 caps.common_cfg_bar = bar as u8;
154 }
155 VIRTIO_PCI_CAP_NOTIFY_CFG => {
156 caps.notify_offset = cap_offset as u32;
157 caps.notify_bar = bar as u8;
158 let mult = syscall::pci_config_read32(PCI_CAP_SLOT, (offset + 16) as u64);
159 caps.notify_off_multiplier = match mult >= 0 {
160 true => mult as u32,
161 false => 0,
162 };
163 }
164 VIRTIO_PCI_CAP_ISR_CFG => {
165 caps.isr_offset = cap_offset as u32;
166 caps.isr_bar = bar as u8;
167 }
168 VIRTIO_PCI_CAP_DEVICE_CFG => {
169 caps.device_cfg_offset = cap_offset as u32;
170 caps.device_cfg_bar = bar as u8;
171 }
172 _ => {}
173 }
174 });
175
176 caps
177}
178
179fn allocate_dma_region() -> (usize, u64) {
180 let iova = syscall::dma_alloc(PCI_CAP_SLOT, DMA_FRAME_COUNT, DMA_BASE_VADDR);
181 if iova < 0 {
182 show!(virtio, error, "dma_alloc failed");
183 syscall::exit();
184 }
185
186 (DMA_BASE_VADDR as usize, iova as u64)
187}
188
189fn map_unique_bars(caps: &VirtioPciCaps) -> [Option<usize>; 6] {
190 let mut bar_vaddrs: [Option<usize>; 6] = [None; 6];
191 let bar_indices = [
192 caps.common_cfg_bar,
193 caps.notify_bar,
194 caps.isr_bar,
195 caps.device_cfg_bar,
196 ];
197
198 bar_indices.iter().enumerate().for_each(|(slot, &bar_idx)| {
199 if bar_idx < 6 && bar_vaddrs[bar_idx as usize].is_none() {
200 let vaddr = MMIO_BASE_VADDR + (slot as u64) * 0x10000;
201 let ret = syscall::pci_bar_map(PCI_CAP_SLOT, bar_idx as u64, vaddr);
202 match ret >= 0 {
203 true => {
204 bar_vaddrs[bar_idx as usize] = Some(vaddr as usize);
205 }
206 false => {
207 show!(virtio, error, "pci_bar_map failed");
208 syscall::exit();
209 }
210 }
211 }
212 });
213
214 bar_vaddrs
215}
216
217pub fn init() -> VirtioNetDevice {
218 let caps = discover_caps();
219 let bar_vaddrs = map_unique_bars(&caps);
220
221 let bar_base = |bar_idx: u8| -> usize {
222 match bar_vaddrs[bar_idx as usize] {
223 Some(v) => v,
224 None => {
225 show!(virtio, error, "bar not mapped");
226 syscall::exit();
227 }
228 }
229 };
230
231 show!(virtio, "bars mapped");
232
233 let mmio_base = bar_base(caps.common_cfg_bar);
234 let common_cfg = bar_base(caps.common_cfg_bar) + caps.common_cfg_offset as usize;
235 let notify_base = bar_base(caps.notify_bar) + caps.notify_offset as usize;
236 let isr_base = bar_base(caps.isr_bar) + caps.isr_offset as usize;
237 let device_cfg = bar_base(caps.device_cfg_bar) + caps.device_cfg_offset as usize;
238
239 write_mmio_u8(common_cfg, 20, 0);
240
241 let reset_ok = core::iter::from_fn(|| {
242 let status = read_mmio_u8(common_cfg, 20);
243 match status {
244 0 => None,
245 _ => Some(()),
246 }
247 })
248 .take(100_000)
249 .count();
250
251 let final_status = read_mmio_u8(common_cfg, 20);
252 if final_status != 0 {
253 show!(virtio, error, "device did not reset");
254 let _ = reset_ok;
255 syscall::exit();
256 }
257
258 write_mmio_u8(common_cfg, 20, STATUS_ACKNOWLEDGE);
259 write_mmio_u8(common_cfg, 20, STATUS_ACKNOWLEDGE | STATUS_DRIVER);
260
261 write_mmio_u32(common_cfg, 0, 0);
262 let features_lo = read_mmio_u32(common_cfg, 4) as u64;
263 write_mmio_u32(common_cfg, 0, 1);
264 let features_hi = read_mmio_u32(common_cfg, 4) as u64;
265 let device_features = features_lo | (features_hi << 32);
266
267 if device_features & VIRTIO_F_VERSION_1 == 0 {
268 show!(virtio, error, "device does not support virtio_f_version_1");
269 write_mmio_u8(common_cfg, 20, STATUS_FAILED);
270 syscall::exit();
271 }
272
273 let mut accepted = VIRTIO_F_VERSION_1 | VIRTIO_F_ACCESS_PLATFORM;
274 if device_features & VIRTIO_NET_F_MAC != 0 {
275 accepted |= VIRTIO_NET_F_MAC;
276 }
277
278 write_mmio_u32(common_cfg, 8, 0);
279 write_mmio_u32(common_cfg, 12, accepted as u32);
280 write_mmio_u32(common_cfg, 8, 1);
281 write_mmio_u32(common_cfg, 12, (accepted >> 32) as u32);
282
283 write_mmio_u16(common_cfg, 16, 0xFFFF);
284
285 let status_before = read_mmio_u8(common_cfg, 20);
286 write_mmio_u8(common_cfg, 20, status_before | STATUS_FEATURES_OK);
287
288 let status_after = read_mmio_u8(common_cfg, 20);
289 if status_after & STATUS_FEATURES_OK == 0 {
290 show!(virtio, error, "features_ok not accepted by device");
291 write_mmio_u8(common_cfg, 20, STATUS_FAILED);
292 syscall::exit();
293 }
294
295 let mac = match device_features & VIRTIO_NET_F_MAC != 0 {
296 true => {
297 let mut m = [0u8; 6];
298 (0..6).for_each(|i| {
299 m[i] = read_mmio_u8(device_cfg, i as u32);
300 });
301 m
302 }
303 false => [0x52, 0x54, 0x00, 0x12, 0x34, 0x56],
304 };
305
306 show!(virtio, "features negotiated, mac read");
307
308 let (dma_virt_base, dma_iova_base) = allocate_dma_region();
309
310 let frame_phys = |frame_idx: usize| -> u64 { dma_iova_base + (frame_idx as u64) * 4096 };
311
312 let rx_desc_virt = dma_virt_base + RX_DESC_FRAME * 4096;
313 let rx_desc_phys = frame_phys(RX_DESC_FRAME);
314
315 let desc_table_size = 16 * QUEUE_SIZE as usize;
316 let avail_ring_size = 6 + 2 * QUEUE_SIZE as usize;
317
318 let rx_avail_virt = rx_desc_virt + desc_table_size;
319 let rx_avail_phys = rx_desc_phys + desc_table_size as u64;
320 let rx_used_virt = (rx_avail_virt + avail_ring_size + 3) & !3;
321 let rx_used_phys = (rx_avail_phys + avail_ring_size as u64 + 3) & !3;
322
323 let tx_desc_virt = dma_virt_base + TX_DESC_FRAME * 4096;
324 let tx_desc_phys = frame_phys(TX_DESC_FRAME);
325 let tx_avail_virt = tx_desc_virt + desc_table_size;
326 let tx_avail_phys = tx_desc_phys + desc_table_size as u64;
327 let tx_used_virt = (tx_avail_virt + avail_ring_size + 3) & !3;
328 let tx_used_phys = (tx_avail_phys + avail_ring_size as u64 + 3) & !3;
329
330 let rx_actual_size = query_queue_size(common_cfg, 0);
331 let tx_actual_size = query_queue_size(common_cfg, 1);
332
333 let rx_queue = Virtqueue::new(
334 rx_desc_virt,
335 rx_avail_virt,
336 rx_used_virt,
337 rx_desc_phys,
338 rx_avail_phys,
339 rx_used_phys,
340 rx_actual_size,
341 );
342
343 let tx_queue = Virtqueue::new(
344 tx_desc_virt,
345 tx_avail_virt,
346 tx_used_virt,
347 tx_desc_phys,
348 tx_avail_phys,
349 tx_used_phys,
350 tx_actual_size,
351 );
352
353 setup_queue(common_cfg, 0, &rx_queue, rx_actual_size);
354 setup_queue(common_cfg, 1, &tx_queue, tx_actual_size);
355
356 let queue_doorbells = [0u16, 1u16].map(|q| {
357 write_mmio_u16(common_cfg, 22, q);
358 let qno = read_mmio_u16(common_cfg, 30) as u32;
359 let offset = match caps.notify_off_multiplier {
360 0 => 0u32,
361 mult => qno.saturating_mul(mult),
362 };
363 notify_base + offset as usize
364 });
365
366 let mut rx_buf_frame_phys = [0u64; RX_BUF_FRAME_COUNT];
367 (0..RX_BUF_FRAME_COUNT).for_each(|i| {
368 rx_buf_frame_phys[i] = frame_phys(RX_BUF_FRAME_START + i);
369 });
370 let rx_buf_virt_base = dma_virt_base + RX_BUF_FRAME_START * 4096;
371
372 let mut tx_buf_frame_phys = [0u64; TX_BUF_FRAME_COUNT];
373 (0..TX_BUF_FRAME_COUNT).for_each(|i| {
374 tx_buf_frame_phys[i] = frame_phys(TX_BUF_FRAME_START + i);
375 });
376
377 let mut dev = VirtioNetDevice {
378 mmio_base,
379 common_cfg,
380 queue_doorbells,
381 isr_base,
382 device_cfg,
383 rx_queue,
384 tx_queue,
385 mac,
386 dma_virt_base,
387 rx_buf_frame_phys,
388 rx_buf_virt_base,
389 tx_buf_frame_phys,
390 tx_next_buf: 0,
391 rx_desc_to_buf: [0u16; QUEUE_SIZE as usize],
392 };
393
394 (0..RX_BUF_COUNT).for_each(|i| {
395 let buf_phys = dev.rx_buf_phys(i);
396 if let Some(desc_idx) = dev
397 .rx_queue
398 .add_buf_single(buf_phys, RX_BUF_SIZE as u32, true)
399 {
400 dev.rx_desc_to_buf[desc_idx as usize] = i as u16;
401 }
402 });
403
404 let status_now = read_mmio_u8(common_cfg, 20);
405 write_mmio_u8(common_cfg, 20, status_now | STATUS_DRIVER_OK);
406
407 show!(virtio, "device ready");
408
409 dev.notify_queue(0);
410
411 dev
412}
413
414fn query_queue_size(common_cfg: usize, queue_idx: u16) -> u16 {
415 write_mmio_u16(common_cfg, 22, queue_idx);
416 let max_size = read_mmio_u16(common_cfg, 24);
417 QUEUE_SIZE.min(max_size)
418}
419
420fn setup_queue(common_cfg: usize, queue_idx: u16, vq: &Virtqueue, actual_size: u16) {
421 write_mmio_u16(common_cfg, 22, queue_idx);
422 write_mmio_u16(common_cfg, 24, actual_size);
423
424 write_mmio_u16(common_cfg, 26, 0);
425
426 write_mmio_u64(common_cfg, 32, vq.desc_phys);
427 write_mmio_u64(common_cfg, 40, vq.avail_phys);
428 write_mmio_u64(common_cfg, 48, vq.used_phys);
429
430 write_mmio_u16(common_cfg, 28, 1);
431}
432
433impl VirtioNetDevice {
434 pub fn notify_queue(&self, queue_idx: u16) {
435 let addr = self.queue_doorbells[queue_idx as usize];
436 unsafe { ptr::write_volatile(addr as *mut u16, queue_idx) }
437 }
438
439 #[allow(dead_code)]
440 pub fn read_isr(&self) -> u8 {
441 read_mmio_u8(self.isr_base, 0)
442 }
443
444 pub fn rx_buf_phys(&self, buf_idx: usize) -> u64 {
445 let byte_offset = buf_idx * RX_BUF_SIZE;
446 let frame_idx = byte_offset / 4096;
447 let intra_frame = (byte_offset % 4096) as u64;
448 self.rx_buf_frame_phys[frame_idx] + intra_frame
449 }
450
451 pub fn tx_buf_phys_for(&self, buf_idx: usize) -> u64 {
452 let byte_offset = buf_idx * TX_BUF_SIZE;
453 let frame_idx = byte_offset / 4096;
454 let intra_frame = (byte_offset % 4096) as u64;
455 self.tx_buf_frame_phys[frame_idx] + intra_frame
456 }
457
458 pub fn tx_buf_virt_for(&self, buf_idx: usize) -> usize {
459 self.dma_virt_base + TX_BUF_FRAME_START * 4096 + buf_idx * TX_BUF_SIZE
460 }
461
462 pub fn acquire_tx_buf(&mut self) -> Option<(usize, u64, usize)> {
463 match self.tx_queue.num_free() {
464 0 => None,
465 _ => {
466 let idx = self.tx_next_buf;
467 self.tx_next_buf = (idx + 1) % TX_BUF_COUNT;
468 Some((idx, self.tx_buf_phys_for(idx), self.tx_buf_virt_for(idx)))
469 }
470 }
471 }
472}