Nothing to see here, move along meow
1#![no_std]
2#![no_main]
3
4mod cmd;
5mod dma;
6mod queue;
7mod regs;
8
9use lancer_user::show;
10use lancer_user::syscall;
11
12const PCI_CAP_SLOT: u64 = 2;
13const IRQ_CAP_SLOT: u64 = 3;
14const NOTIF_CAP_SLOT: u64 = 4;
15const BLOCK_RING_BASE_SLOT: u64 = 64;
16const BLOCK_RING_FRAME_COUNT: u64 = 16;
17const CLIENT_NOTIF_SLOT: u64 = 6;
18
19const BAR0_VADDR: usize = 0x4000_0000;
20
21const ADMIN_SQ_VADDR: u64 = 0x4010_0000;
22const ADMIN_CQ_VADDR: u64 = 0x4011_0000;
23const IDENTIFY_BUF_VADDR: u64 = 0x4012_0000;
24const PRP_LIST_VADDR: u64 = 0x4013_0000;
25
26const IO_QUEUE_BASE_VADDR: u64 = 0x4020_0000;
27const IO_QUEUE_STRIDE: u64 = 0x2000;
28
29const DATA_BUF_VADDR: u64 = IO_QUEUE_BASE_VADDR + (MAX_IO_QUEUES as u64) * IO_QUEUE_STRIDE;
30const DATA_BUF_PAGES: u64 = 16;
31const MAX_PRP_LIST_ENTRIES: usize = 4096 / core::mem::size_of::<u64>();
32const _: () = assert!(DATA_BUF_PAGES as usize <= MAX_PRP_LIST_ENTRIES + 1);
33
34const BLOCK_RING_VADDR: u64 = 0x4080_0000;
35const RING_DATA_OFFSET: usize = 8192;
36const RING_DATA_PAGES: usize = 14;
37
38const ADMIN_QUEUE_DEPTH: u16 = 64;
39const IO_QUEUE_DEPTH: u16 = 64;
40
41const MAX_IO_QUEUES: usize = 32;
42
43struct QueuePair {
44 sq: queue::SubmissionQueue,
45 cq: queue::CompletionQueue,
46 #[allow(dead_code)]
47 sq_dma: dma::DmaRegion,
48 #[allow(dead_code)]
49 cq_dma: dma::DmaRegion,
50 next_cid: u16,
51 qid: u16,
52}
53
54impl QueuePair {
55 fn next_cid(&mut self) -> u16 {
56 let cid = self.next_cid;
57 self.next_cid = self.next_cid.wrapping_add(1);
58 cid
59 }
60}
61
62struct NvmeController {
63 bar0: usize,
64 dstrd: u32,
65 max_queue_depth: u16,
66 admin_sq: queue::SubmissionQueue,
67 admin_cq: queue::CompletionQueue,
68 #[allow(dead_code)]
69 admin_sq_dma: dma::DmaRegion,
70 #[allow(dead_code)]
71 admin_cq_dma: dma::DmaRegion,
72 identify_dma: dma::DmaRegion,
73 io_queues: [Option<QueuePair>; MAX_IO_QUEUES],
74 active_io_queues: usize,
75 nsid: u32,
76 sector_size: u32,
77 total_sectors: u64,
78 mdts_pages: u32,
79 next_admin_cid: u16,
80}
81
82const NONE_QP: Option<QueuePair> = None;
83
84impl NvmeController {
85 fn next_admin_cid(&mut self) -> u16 {
86 let cid = self.next_admin_cid;
87 self.next_admin_cid = self.next_admin_cid.wrapping_add(1);
88 cid
89 }
90
91 fn sq_doorbell_offset(&self, qid: u16) -> u32 {
92 0x1000 + (2 * qid as u32) * (4 << self.dstrd)
93 }
94
95 fn cq_doorbell_offset(&self, qid: u16) -> u32 {
96 0x1000 + (2 * qid as u32 + 1) * (4 << self.dstrd)
97 }
98
99 fn submit_admin_cmd(
100 &mut self,
101 entry: cmd::SubmissionEntry,
102 ) -> Result<cmd::CompletionEntry, u16> {
103 let expected_cid = entry.cid();
104 match self
105 .admin_sq
106 .submit(self.bar0, self.sq_doorbell_offset(0), &entry)
107 {
108 true => {}
109 false => return Err(0xFFFD),
110 }
111
112 let bar0 = self.bar0;
113 let cq_db = self.cq_doorbell_offset(0);
114 let mut polls = 0u32;
115 loop {
116 match self.admin_cq.poll(bar0, cq_db) {
117 Some(cqe) => {
118 self.admin_sq
119 .update_head(queue::CompletionQueue::sq_head_from(&cqe));
120 if cqe.cid() != expected_cid {
121 return Err(0xFFFC);
122 }
123 let status = cqe.status();
124 return match status {
125 0 => Ok(cqe),
126 _ => Err(status),
127 };
128 }
129 None => {
130 polls += 1;
131 if polls > 1_000_000 {
132 return Err(0xFFFF);
133 }
134 core::hint::spin_loop();
135 }
136 }
137 }
138 }
139
140 fn submit_io_cmd(
141 &mut self,
142 queue_idx: usize,
143 entry: cmd::SubmissionEntry,
144 ) -> Result<cmd::CompletionEntry, u16> {
145 let expected_cid = entry.cid();
146 let bar0 = self.bar0;
147 let dstrd = self.dstrd;
148
149 let qid = match &self.io_queues[queue_idx] {
150 Some(qp) => qp.qid,
151 None => return Err(0xFFFE),
152 };
153 let sq_db = 0x1000 + (2 * qid as u32) * (4 << dstrd);
154 let cq_db = 0x1000 + (2 * qid as u32 + 1) * (4 << dstrd);
155
156 let qp = self.io_queues[queue_idx].as_mut().unwrap();
157
158 match qp.sq.submit(bar0, sq_db, &entry) {
159 true => {}
160 false => return Err(0xFFFD),
161 }
162
163 let mut polls = 0u32;
164 loop {
165 match qp.cq.poll(bar0, cq_db) {
166 Some(cqe) => {
167 qp.sq
168 .update_head(queue::CompletionQueue::sq_head_from(&cqe));
169 if cqe.cid() != expected_cid {
170 return Err(0xFFFC);
171 }
172 let status = cqe.status();
173 return match status {
174 0 => Ok(cqe),
175 _ => Err(status),
176 };
177 }
178 None => {
179 polls += 1;
180 if polls > 10_000_000 {
181 return Err(0xFFFF);
182 }
183 core::hint::spin_loop();
184 }
185 }
186 }
187 }
188
189 fn create_io_queue_pair(&mut self, index: usize) {
190 let qid = (index + 1) as u16;
191 let sq_vaddr = IO_QUEUE_BASE_VADDR + (index as u64) * IO_QUEUE_STRIDE;
192 let cq_vaddr = sq_vaddr + 0x1000;
193
194 let cq_dma = dma::alloc(PCI_CAP_SLOT, 1, cq_vaddr);
195 let sq_dma = dma::alloc(PCI_CAP_SLOT, 1, sq_vaddr);
196
197 unsafe {
198 core::ptr::write_bytes(cq_dma.vaddr as *mut u8, 0, 4096);
199 core::ptr::write_bytes(sq_dma.vaddr as *mut u8, 0, 4096);
200 }
201
202 let depth = IO_QUEUE_DEPTH.min(self.max_queue_depth);
203
204 let cid = self.next_admin_cid();
205 let create_cq = cmd::create_io_cq(cid, qid, depth - 1, cq_dma.iova, 0);
206 match self.submit_admin_cmd(create_cq) {
207 Ok(_) => show!(nvme, "io cq {} created", qid),
208 Err(status) => {
209 show!(nvme, error, "create io cq failed, status {:#06x}", status);
210 syscall::exit();
211 }
212 }
213
214 let cid = self.next_admin_cid();
215 let create_sq = cmd::create_io_sq(cid, qid, depth - 1, sq_dma.iova, qid);
216 match self.submit_admin_cmd(create_sq) {
217 Ok(_) => show!(nvme, "io sq {} created", qid),
218 Err(status) => {
219 show!(nvme, error, "create io sq failed, status {:#06x}", status);
220 syscall::exit();
221 }
222 }
223
224 self.io_queues[index] = Some(QueuePair {
225 sq: queue::SubmissionQueue::new(sq_dma.vaddr, depth),
226 cq: queue::CompletionQueue::new(cq_dma.vaddr, depth),
227 sq_dma,
228 cq_dma,
229 next_cid: 0,
230 qid,
231 });
232 self.active_io_queues = self.active_io_queues.max(index + 1);
233 }
234
235 fn check_mdts(&self, block_count: u16) -> Result<(), u16> {
236 let transfer_bytes = block_count as u64 * self.sector_size as u64;
237 let max_bytes = self.mdts_pages as u64 * 4096;
238 match transfer_bytes <= max_bytes {
239 true => Ok(()),
240 false => Err(0xFFFB),
241 }
242 }
243
244 fn execute_io_flush(&mut self) -> Result<cmd::CompletionEntry, u16> {
245 let cid = match &mut self.io_queues[0] {
246 Some(qp) => qp.next_cid(),
247 None => return Err(0xFFFE),
248 };
249 let entry = cmd::nvm_flush(cid, self.nsid);
250 self.submit_io_cmd(0, entry)
251 }
252
253 fn poll_io_completions(&mut self) {
254 let bar0 = self.bar0;
255 let dstrd = self.dstrd;
256
257 (0..self.active_io_queues).for_each(|i| {
258 if let Some(ref mut qp) = self.io_queues[i] {
259 let cq_db = 0x1000 + (2 * qp.qid as u32 + 1) * (4 << dstrd);
260 let mut drained = false;
261 while let Some(cqe) = qp.cq.advance() {
262 qp.sq
263 .update_head(queue::CompletionQueue::sq_head_from(&cqe));
264 drained = true;
265 }
266 if drained {
267 qp.cq.flush_head(bar0, cq_db);
268 }
269 }
270 });
271 }
272
273 fn execute_io(
274 &mut self,
275 is_write: bool,
276 lba: u64,
277 block_count: u16,
278 data_dma: &dma::DmaRegion,
279 byte_offset: usize,
280 prp_list: &dma::DmaRegion,
281 ) -> Result<cmd::CompletionEntry, u16> {
282 self.check_mdts(block_count)?;
283 let transfer_bytes = block_count as usize * self.sector_size as usize;
284 let (prp1, prp2) = build_prp(data_dma.iova, byte_offset, transfer_bytes, prp_list);
285 let cid = match &mut self.io_queues[0] {
286 Some(qp) => qp.next_cid(),
287 None => return Err(0xFFFE),
288 };
289 let entry = match is_write {
290 true => cmd::nvm_write(cid, self.nsid, lba, block_count, prp1, prp2),
291 false => cmd::nvm_read(cid, self.nsid, lba, block_count, prp1, prp2),
292 };
293 self.submit_io_cmd(0, entry)
294 }
295}
296
297fn init_controller() -> NvmeController {
298 let ret = syscall::pci_bar_map(PCI_CAP_SLOT, 0, BAR0_VADDR as u64);
299 if ret < 0 {
300 show!(nvme, error, "pci_bar_map failed");
301 syscall::exit();
302 }
303
304 let bar0 = BAR0_VADDR;
305
306 let cap = regs::read64(bar0, regs::CAP);
307 let mqes = (cap & 0xFFFF) as u16;
308 let dstrd = ((cap >> 32) & 0xF) as u32;
309
310 regs::write32(bar0, regs::CC, 0);
311
312 let mut polls = 0u32;
313 loop {
314 let csts = regs::read32(bar0, regs::CSTS);
315 if csts & regs::CSTS_RDY == 0 {
316 break;
317 }
318 polls += 1;
319 if polls > 10_000_000 {
320 show!(nvme, error, "controller failed to disable");
321 syscall::exit();
322 }
323 core::hint::spin_loop();
324 }
325 let admin_depth = (mqes + 1).min(ADMIN_QUEUE_DEPTH);
326 let admin_sq_dma = dma::alloc(PCI_CAP_SLOT, 1, ADMIN_SQ_VADDR);
327 let admin_cq_dma = dma::alloc(PCI_CAP_SLOT, 1, ADMIN_CQ_VADDR);
328 let identify_dma = dma::alloc(PCI_CAP_SLOT, 1, IDENTIFY_BUF_VADDR);
329
330 unsafe {
331 core::ptr::write_bytes(admin_sq_dma.vaddr as *mut u8, 0, 4096);
332 core::ptr::write_bytes(admin_cq_dma.vaddr as *mut u8, 0, 4096);
333 }
334
335 let admin_sq = queue::SubmissionQueue::new(admin_sq_dma.vaddr, admin_depth);
336 let admin_cq = queue::CompletionQueue::new(admin_cq_dma.vaddr, admin_depth);
337
338 let aqa = ((admin_depth as u32 - 1) << 16) | (admin_depth as u32 - 1);
339 regs::write32(bar0, regs::AQA, aqa);
340 regs::write64(bar0, regs::ASQ, admin_sq_dma.iova);
341 regs::write64(bar0, regs::ACQ, admin_cq_dma.iova);
342
343 let cc = (6u32 << regs::CC_IOSQES_SHIFT) | (4u32 << regs::CC_IOCQES_SHIFT) | regs::CC_EN;
344 regs::write32(bar0, regs::CC, cc);
345
346 polls = 0;
347 loop {
348 let csts = regs::read32(bar0, regs::CSTS);
349 if csts & regs::CSTS_RDY != 0 {
350 break;
351 }
352 if csts & regs::CSTS_CFS != 0 {
353 show!(nvme, error, "controller fatal status during enable");
354 syscall::exit();
355 }
356 polls += 1;
357 if polls > 10_000_000 {
358 show!(nvme, error, "controller failed to enable");
359 syscall::exit();
360 }
361 core::hint::spin_loop();
362 }
363
364 NvmeController {
365 bar0,
366 dstrd,
367 max_queue_depth: admin_depth,
368 admin_sq,
369 admin_cq,
370 admin_sq_dma,
371 admin_cq_dma,
372 identify_dma,
373 io_queues: [NONE_QP; MAX_IO_QUEUES],
374 active_io_queues: 0,
375 nsid: 1,
376 sector_size: 512,
377 total_sectors: 0,
378 mdts_pages: 0,
379 next_admin_cid: 0,
380 }
381}
382
383fn identify_controller(ctrl: &mut NvmeController) {
384 let cid = ctrl.next_admin_cid();
385 let entry = cmd::identify_controller(cid, ctrl.identify_dma.iova);
386
387 match ctrl.submit_admin_cmd(entry) {
388 Ok(_) => {
389 let buf = ctrl.identify_dma.vaddr as *const u8;
390
391 let mdts_raw = unsafe { *buf.add(77) };
392 ctrl.mdts_pages = match mdts_raw {
393 0 => 256,
394 n => 1u32 << n,
395 };
396
397 let mut model = [0u8; 40];
398 unsafe {
399 core::ptr::copy_nonoverlapping(buf.add(24), model.as_mut_ptr(), 40);
400 }
401 let model_len = model
402 .iter()
403 .rposition(|&b| b != b' ' && b != 0)
404 .map_or(0, |p| p + 1);
405
406 match core::str::from_utf8(&model[..model_len]) {
407 Ok(s) => show!(nvme, "model {}", s),
408 Err(_) => show!(nvme, "model <invalid utf-8>"),
409 }
410 }
411 Err(status) => {
412 show!(
413 nvme,
414 error,
415 "identify controller failed, status {:#06x}",
416 status
417 );
418 syscall::exit();
419 }
420 }
421}
422
423fn identify_namespace(ctrl: &mut NvmeController) {
424 let cid = ctrl.next_admin_cid();
425 let entry = cmd::identify_namespace(cid, ctrl.nsid, ctrl.identify_dma.iova);
426
427 match ctrl.submit_admin_cmd(entry) {
428 Ok(_) => {
429 let buf = ctrl.identify_dma.vaddr as *const u8;
430
431 let nsze = unsafe { core::ptr::read_unaligned(buf as *const u64) };
432 ctrl.total_sectors = nsze;
433
434 let flbas = unsafe { *buf.add(26) };
435 let lba_fmt_idx = (flbas & 0x0F) as usize;
436 let lbaf_base = 128 + lba_fmt_idx * 4;
437 let lbaf = unsafe { core::ptr::read_unaligned(buf.add(lbaf_base) as *const u32) };
438 let lba_ds = (lbaf >> 16) & 0xFF;
439 ctrl.sector_size = 1u32 << lba_ds;
440
441 let mb = (ctrl.total_sectors * ctrl.sector_size as u64) / (1024 * 1024);
442 show!(
443 nvme,
444 "ns1 {} sectors {} byte sectors {}MB",
445 ctrl.total_sectors,
446 ctrl.sector_size,
447 mb
448 );
449 }
450 Err(status) => {
451 show!(
452 nvme,
453 error,
454 "identify namespace failed, status {:#06x}",
455 status
456 );
457 syscall::exit();
458 }
459 }
460}
461
462fn alloc_data_buffers() -> dma::DmaRegion {
463 dma::alloc(PCI_CAP_SLOT, DATA_BUF_PAGES, DATA_BUF_VADDR)
464}
465
466fn alloc_prp_list() -> dma::DmaRegion {
467 dma::alloc(PCI_CAP_SLOT, 1, PRP_LIST_VADDR)
468}
469
470fn build_prp(
471 base_iova: u64,
472 byte_offset: usize,
473 transfer_bytes: usize,
474 prp_list: &dma::DmaRegion,
475) -> (u64, u64) {
476 let prp1 = base_iova + byte_offset as u64;
477 let offset_in_page = (prp1 & 0xFFF) as usize;
478 let first_page_bytes = 4096 - offset_in_page;
479 let pages = match transfer_bytes <= first_page_bytes {
480 true => 1,
481 false => 1 + (transfer_bytes - first_page_bytes).div_ceil(4096),
482 };
483 let next_page_base = (prp1 & !0xFFF) + 4096;
484 match pages {
485 0 | 1 => (prp1, 0),
486 2 => (prp1, next_page_base),
487 _ => {
488 let list_ptr = prp_list.vaddr as *mut u64;
489 let entries = pages - 1;
490 assert!(entries <= MAX_PRP_LIST_ENTRIES);
491 (0..entries).for_each(|i| {
492 unsafe {
493 core::ptr::write_volatile(list_ptr.add(i), next_page_base + (i as u64) * 4096)
494 };
495 });
496 (prp1, prp_list.iova)
497 }
498 }
499}
500
501fn selftest(ctrl: &mut NvmeController, data_dma: &dma::DmaRegion, prp_list: &dma::DmaRegion) {
502 let test_pattern: [u8; 16] = [
503 0xDE, 0xAD, 0xBE, 0xEF, 0xCA, 0xFE, 0xBA, 0xBE, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
504 0x08,
505 ];
506
507 let buf = data_dma.vaddr as *mut u8;
508 (0..ctrl.sector_size as usize)
509 .step_by(test_pattern.len())
510 .for_each(|off| {
511 let len = test_pattern.len().min(ctrl.sector_size as usize - off);
512 unsafe {
513 core::ptr::copy_nonoverlapping(test_pattern.as_ptr(), buf.add(off), len);
514 }
515 });
516
517 match ctrl.check_mdts(1) {
518 Ok(()) => {}
519 Err(_) => {
520 show!(nvme, warn, "selftest mdts too small for 1 block");
521 return;
522 }
523 }
524
525 let cid = match &mut ctrl.io_queues[0] {
526 Some(qp) => qp.next_cid(),
527 None => {
528 show!(nvme, warn, "selftest no io queue");
529 return;
530 }
531 };
532 let test_lba = ctrl.total_sectors.saturating_sub(64);
533 let write_cmd = cmd::nvm_write(cid, ctrl.nsid, test_lba, 1, data_dma.iova, 0);
534 match ctrl.submit_io_cmd(0, write_cmd) {
535 Ok(_) => show!(nvme, "selftest write lba 0 ok"),
536 Err(status) => {
537 show!(nvme, warn, "selftest write failed, status {:#06x}", status);
538 return;
539 }
540 }
541
542 unsafe { core::ptr::write_bytes(buf, 0, ctrl.sector_size as usize) };
543
544 let cid = match &mut ctrl.io_queues[0] {
545 Some(qp) => qp.next_cid(),
546 None => return,
547 };
548 let read_cmd = cmd::nvm_read(cid, ctrl.nsid, test_lba, 1, data_dma.iova, 0);
549 match ctrl.submit_io_cmd(0, read_cmd) {
550 Ok(_) => {
551 let readback = unsafe { core::slice::from_raw_parts(buf, test_pattern.len()) };
552 let ok = readback
553 .iter()
554 .zip(test_pattern.iter())
555 .all(|(a, b)| *a == *b);
556 match ok {
557 true => show!(nvme, "selftest read-back verified ok"),
558 false => show!(nvme, warn, "selftest data mismatch"),
559 }
560 }
561 Err(status) => {
562 show!(nvme, warn, "selftest read failed, status {:#06x}", status);
563 }
564 }
565
566 let multi_base = ctrl.total_sectors.saturating_sub(64);
567 selftest_multi(ctrl, data_dma, prp_list, 2, multi_base + 16);
568 selftest_multi(ctrl, data_dma, prp_list, 4, multi_base + 32);
569}
570
571fn selftest_multi(
572 ctrl: &mut NvmeController,
573 data_dma: &dma::DmaRegion,
574 prp_list: &dma::DmaRegion,
575 pages: usize,
576 test_lba: u64,
577) {
578 let transfer_bytes = pages * 4096;
579 let block_count = (transfer_bytes / ctrl.sector_size as usize) as u16;
580
581 match ctrl.check_mdts(block_count) {
582 Ok(()) => {}
583 Err(_) => {
584 show!(
585 nvme,
586 warn,
587 "selftest mdts too small for {} page test, skipping",
588 pages
589 );
590 return;
591 }
592 }
593
594 let pattern: [u8; 16] = [
595 0xA5, 0x5A, 0xF0, 0x0F, 0xDE, 0xAD, 0xBE, 0xEF, 0xCA, 0xFE, 0xBA, 0xBE, 0x12, 0x34, 0x56,
596 0x78,
597 ];
598
599 let buf = data_dma.vaddr as *mut u8;
600 (0..transfer_bytes).step_by(pattern.len()).for_each(|off| {
601 let len = pattern.len().min(transfer_bytes - off);
602 unsafe { core::ptr::copy_nonoverlapping(pattern.as_ptr(), buf.add(off), len) };
603 });
604
605 match ctrl.execute_io(true, test_lba, block_count, data_dma, 0, prp_list) {
606 Ok(_) => {}
607 Err(status) => {
608 show!(
609 nvme,
610 warn,
611 "selftest {} page write failed, status {:#06x}",
612 pages,
613 status
614 );
615 return;
616 }
617 }
618
619 unsafe { core::ptr::write_bytes(buf, 0, transfer_bytes) };
620
621 match ctrl.execute_io(false, test_lba, block_count, data_dma, 0, prp_list) {
622 Ok(_) => {
623 let readback = unsafe { core::slice::from_raw_parts(buf, transfer_bytes) };
624 let ok = readback
625 .chunks(pattern.len())
626 .all(|chunk| chunk.iter().zip(pattern.iter()).all(|(a, b)| *a == *b));
627 match ok {
628 true => show!(nvme, "selftest {} page read-back verified ok", pages),
629 false => show!(nvme, warn, "selftest {} page data mismatch", pages),
630 }
631 }
632 Err(status) => {
633 show!(
634 nvme,
635 warn,
636 "selftest {} page read failed, status {:#06x}",
637 pages,
638 status
639 );
640 }
641 }
642}
643
644fn pop_request(
645 ring: &lancer_core::packet_ring::PacketRingReader,
646) -> Option<lancer_core::block_io::BlockRequest> {
647 let mut buf = [0u8; 32];
648 let len = ring.try_pop(&mut buf)?;
649 match len >= lancer_core::block_io::BlockRequest::SIZE {
650 true => lancer_core::block_io::BlockRequest::from_bytes(&buf),
651 false => None,
652 }
653}
654
655fn process_block_request(
656 ctrl: &mut NvmeController,
657 data_dma: &dma::DmaRegion,
658 prp_list: &dma::DmaRegion,
659 req: &lancer_core::block_io::BlockRequest,
660 data_area: usize,
661) -> lancer_core::block_io::BlockResponse {
662 use lancer_core::block_io::{BlockOpType, BlockResponse};
663
664 let transfer_bytes = req.count as usize * ctrl.sector_size as usize;
665 let data_area_size = RING_DATA_PAGES * 4096;
666 let dma_buf_size = DATA_BUF_PAGES as usize * 4096;
667
668 let end = (req.buf_offset as usize).saturating_add(transfer_bytes);
669 let lba_end = req.lba.saturating_add(req.count as u64);
670 let invalid = req.count == 0
671 || end > data_area_size
672 || transfer_bytes > dma_buf_size
673 || lba_end > ctrl.total_sectors;
674
675 if let Some(BlockOpType::Flush) = req.op_type() {
676 return match ctrl.execute_io_flush() {
677 Ok(_) => BlockResponse::flush_complete(req.tag, 0),
678 Err(status) => BlockResponse::flush_complete(req.tag, status),
679 };
680 }
681
682 match (invalid, req.op_type()) {
683 (true, Some(BlockOpType::Read)) => BlockResponse::read_complete(req.tag, 0xFFFA),
684 (true, _) => BlockResponse::write_complete(req.tag, 0xFFFA),
685 (false, Some(BlockOpType::Write)) => {
686 unsafe {
687 core::ptr::copy_nonoverlapping(
688 (data_area + req.buf_offset as usize) as *const u8,
689 data_dma.vaddr as *mut u8,
690 transfer_bytes,
691 );
692 }
693 match ctrl.execute_io(true, req.lba, req.count, data_dma, 0, prp_list) {
694 Ok(_) => BlockResponse::write_complete(req.tag, 0),
695 Err(status) => BlockResponse::write_complete(req.tag, status),
696 }
697 }
698 (false, Some(BlockOpType::Read)) => {
699 match ctrl.execute_io(false, req.lba, req.count, data_dma, 0, prp_list) {
700 Ok(_) => {
701 unsafe {
702 core::ptr::copy_nonoverlapping(
703 data_dma.vaddr as *const u8,
704 (data_area + req.buf_offset as usize) as *mut u8,
705 transfer_bytes,
706 );
707 }
708 BlockResponse::read_complete(req.tag, 0)
709 }
710 Err(status) => BlockResponse::read_complete(req.tag, status),
711 }
712 }
713 (false, _) => BlockResponse::write_complete(req.tag, 0xFFF9),
714 }
715}
716
717fn event_loop_standalone(ctrl: &mut NvmeController) -> ! {
718 loop {
719 let (status, _bits) = syscall::notify_wait(NOTIF_CAP_SLOT);
720 if status < 0 {
721 continue;
722 }
723 ctrl.poll_io_completions();
724 syscall::irq_ack(IRQ_CAP_SLOT);
725 }
726}
727
728const PUSH_RETRY_LIMIT: u32 = 100_000;
729
730fn push_response(
731 ring: &lancer_core::packet_ring::PacketRingWriter,
732 resp: &lancer_core::block_io::BlockResponse,
733) {
734 let data = resp.as_bytes();
735 (0..PUSH_RETRY_LIMIT)
736 .take_while(|_| !ring.try_push(data))
737 .for_each(|_| core::hint::spin_loop());
738}
739
740fn event_loop_ipc(
741 ctrl: &mut NvmeController,
742 data_dma: &dma::DmaRegion,
743 prp_list: &dma::DmaRegion,
744 request_ring: &lancer_core::packet_ring::PacketRingReader,
745 response_ring: &lancer_core::packet_ring::PacketRingWriter,
746 data_area: usize,
747) -> ! {
748 loop {
749 let (status, _bits) = syscall::notify_wait(NOTIF_CAP_SLOT);
750 if status < 0 {
751 continue;
752 }
753
754 ctrl.poll_io_completions();
755
756 let responded = core::iter::from_fn(|| pop_request(request_ring))
757 .take(IO_QUEUE_DEPTH as usize)
758 .fold(false, |_, req| {
759 let resp = process_block_request(ctrl, data_dma, prp_list, &req, data_area);
760 push_response(response_ring, &resp);
761 true
762 });
763
764 if responded {
765 syscall::notify_signal(CLIENT_NOTIF_SLOT, 1);
766 }
767
768 syscall::irq_ack(IRQ_CAP_SLOT);
769 }
770}
771
772#[unsafe(no_mangle)]
773pub extern "C" fn lancer_main() -> ! {
774 show!(nvme, "driver starting");
775
776 let mut ctrl = init_controller();
777 identify_controller(&mut ctrl);
778 identify_namespace(&mut ctrl);
779 ctrl.create_io_queue_pair(0);
780
781 let data_dma = alloc_data_buffers();
782 let prp_list_dma = alloc_prp_list();
783 selftest(&mut ctrl, &data_dma, &prp_list_dma);
784
785 show!(nvme, "ready, entering event loop");
786 syscall::irq_ack(IRQ_CAP_SLOT);
787
788 let ring_ok = (0..BLOCK_RING_FRAME_COUNT)
789 .all(|i| syscall::frame_map(BLOCK_RING_BASE_SLOT + i, BLOCK_RING_VADDR + i * 4096, 1) >= 0);
790 match ring_ok {
791 true => {
792 show!(nvme, "block ring mapped, ipc active");
793 syscall::notify_signal(CLIENT_NOTIF_SLOT, 1);
794 let ring_base = BLOCK_RING_VADDR as *mut u8;
795 let response_ring = unsafe {
796 lancer_core::packet_ring::PacketRingWriter::init(ring_base.add(4096), 4096, 16)
797 };
798 let request_ring =
799 unsafe { lancer_core::packet_ring::PacketRingReader::attach(ring_base, 4096) };
800 let data_area = BLOCK_RING_VADDR as usize + RING_DATA_OFFSET;
801 event_loop_ipc(
802 &mut ctrl,
803 &data_dma,
804 &prp_list_dma,
805 &request_ring,
806 &response_ring,
807 data_area,
808 )
809 }
810 false => event_loop_standalone(&mut ctrl),
811 }
812}