diff --git a/CMakeLists.txt b/CMakeLists.txt index 8dd6ecd..6acb0b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -217,7 +217,7 @@ add_custom_target(extra-img COMMAND ${QEMU_IMG} create -f raw VirtioDisk.img 128M COMMAND ${MKFS_EXT2} VirtioDisk.img COMMAND ${QEMU_IMG} create -f raw SataDisk.img 128M - COMMAND ${MKFS_FAT} -F12 SataDisk.img + COMMAND ${MKFS_EXT2} SataDisk.img COMMAND ${QEMU_IMG} create -f raw NVMeDisk.img 256M COMMAND ${MKFS_EXT2} NVMeDisk.img COMMENT "Creating extra disk images" diff --git a/drivers/ACPI.c b/drivers/ACPI.c index 9c780c9..2ea9c1c 100644 --- a/drivers/ACPI.c +++ b/drivers/ACPI.c @@ -2,6 +2,7 @@ #include "Console.h" #include "Io.h" #include "MemOps.h" +#include "NVMe.h" #include "Scheduler.h" #include "StringOps.h" #include "VMem.h" @@ -153,6 +154,10 @@ void ACPIResetProcedure() { PrintKernel("ACPI: Stopping all processes and services...\n"); KillAllProcess("SHUTDOWN"); PrintKernelSuccess("ACPI: All processes and services stopped\n"); + + PrintKernel("ACPI: Stopping NVMe driver...\n"); + NVMe_Shutdown(); + PrintKernelSuccess("ACPI: NVMe driver stopped\n"); } void ACPIShutdown(void) { diff --git a/drivers/ACPI.h b/drivers/ACPI.h index 24b814f..8e7d683 100644 --- a/drivers/ACPI.h +++ b/drivers/ACPI.h @@ -83,4 +83,5 @@ typedef struct { bool ACPIInit(void); void ACPIShutdown(void); void ACPIReboot(void); -void* AcpiFindTable(const char* signature); \ No newline at end of file +void* AcpiFindTable(const char* signature); +void ACPIResetProcedure(void); \ No newline at end of file diff --git a/drivers/TSC.c b/drivers/TSC.c index 842d97d..c0fb351 100644 --- a/drivers/TSC.c +++ b/drivers/TSC.c @@ -33,6 +33,11 @@ void TSCInit(void) { PrintKernelF("TSC: Calibrated frequency: %llu Hz\n", tsc_freq_hz); } +uint64_t GetTimeInMs(void) { + if (!tsc_calibrated) return 0; + return (rdtsc() * 1000) / tsc_freq_hz; +} + void delay_us(uint32_t microseconds) { if (!tsc_calibrated) return; diff --git a/drivers/TSC.h b/drivers/TSC.h index a8ec14b..0688503 100644 --- a/drivers/TSC.h +++ b/drivers/TSC.h @@ -3,6 +3,7 @@ void TSCInit(void); +uint64_t GetTimeInMs(void); void delay_us(uint32_t microseconds); void delay(uint32_t milliseconds); void delay_s(uint32_t seconds); diff --git a/drivers/storage/NVMe.c b/drivers/storage/NVMe.c index 5550e3a..a3e52db 100644 --- a/drivers/storage/NVMe.c +++ b/drivers/storage/NVMe.c @@ -32,8 +32,10 @@ static void NVMe_WriteReg64(uint32_t offset, uint64_t value) { } static int NVMe_WaitReady(int ready) { - int timeout = 5000; - while (timeout-- > 0) { + uint64_t timeout_ms = 5000; + uint64_t start_time = GetTimeInMs(); + + while ((GetTimeInMs() - start_time) < timeout_ms) { uint32_t csts = NVMe_ReadReg32(NVME_CSTS); if (ready && (csts & NVME_CSTS_RDY)) return 0; if (!ready && !(csts & NVME_CSTS_RDY)) return 0; @@ -45,91 +47,86 @@ static int NVMe_WaitReady(int ready) { static int NVMe_SubmitAdminCommand(NVMeSubmissionEntry* cmd) { NVMeController* ctrl = &g_nvme_controller; - // Copy command to submission queue - FastMemcpy(&ctrl->admin_sq[ctrl->admin_sq_tail], cmd, sizeof(NVMeSubmissionEntry)); + uint64_t irq_flags = rust_spinlock_lock_irqsave(ctrl->lock); - // Ensure the SQE is visible to the device before ringing the doorbell + FastMemcpy(&ctrl->admin_sq[ctrl->admin_sq_tail], cmd, sizeof(NVMeSubmissionEntry)); __asm__ volatile("mfence" ::: "memory"); - // Update tail pointer ctrl->admin_sq_tail = (ctrl->admin_sq_tail + 1) % NVME_ADMIN_QUEUE_SIZE; - // Ring doorbell for Admin SQ (QID 0) uint32_t db_shift = 2 + ctrl->dstrd; uint32_t asq_db_off = 0x1000 + ((0 * 2 + 0) << db_shift); - NVMe_WriteReg32(asq_db_off, ctrl->admin_sq_tail); // Admin SQ doorbell + NVMe_WriteReg32(asq_db_off, ctrl->admin_sq_tail); - // Wait for completion - int timeout = 5000; - while (timeout-- > 0) { + rust_spinlock_unlock_irqrestore(ctrl->lock, irq_flags); + + uint64_t timeout_ms = 5000; + uint64_t start_time = GetTimeInMs(); + + while ((GetTimeInMs() - start_time) < timeout_ms) { NVMeCompletionEntry* cqe = &ctrl->admin_cq[ctrl->admin_cq_head]; uint16_t status = cqe->status; if ((status & 1) == ctrl->admin_cq_phase) { - // Command completed ctrl->admin_cq_head = (ctrl->admin_cq_head + 1) % NVME_ADMIN_QUEUE_SIZE; if (ctrl->admin_cq_head == 0) { ctrl->admin_cq_phase = !ctrl->admin_cq_phase; } - // Ring completion doorbell for Admin CQ (QID 0) uint32_t acq_db_off = 0x1000 + ((0 * 2 + 1) << db_shift); - NVMe_WriteReg32(acq_db_off, ctrl->admin_cq_head); // Admin CQ doorbell + NVMe_WriteReg32(acq_db_off, ctrl->admin_cq_head); - return (status >> 1) & 0x7FF; // Return status code + return (status >> 1) & 0x7FF; } delay_us(100); } - return -1; // Timeout + return -1; } static int NVMe_SubmitIOCommand(NVMeSubmissionEntry* cmd) { NVMeController* ctrl = &g_nvme_controller; - // Copy command to I/O submission queue - FastMemcpy(&ctrl->io_sq[ctrl->io_sq_tail], cmd, sizeof(NVMeSubmissionEntry)); + uint64_t irq_flags = rust_spinlock_lock_irqsave(ctrl->lock); - // Ensure the SQE is visible to the device before ringing the doorbell + FastMemcpy(&ctrl->io_sq[ctrl->io_sq_tail], cmd, sizeof(NVMeSubmissionEntry)); __asm__ volatile("mfence" ::: "memory"); - // Update tail pointer ctrl->io_sq_tail = (ctrl->io_sq_tail + 1) % NVME_IO_QUEUE_SIZE; - // Ring doorbell (I/O queue 1) uint32_t db_shift = 2 + ctrl->dstrd; uint32_t iosq_db_off = 0x1000 + ((1 * 2 + 0) << db_shift); - NVMe_WriteReg32(iosq_db_off, ctrl->io_sq_tail); // I/O SQ doorbell + NVMe_WriteReg32(iosq_db_off, ctrl->io_sq_tail); - // Wait for completion - int timeout = 5000; - while (timeout-- > 0) { + rust_spinlock_unlock_irqrestore(ctrl->lock, irq_flags); + + uint64_t timeout_ms = 5000; + uint64_t start_time = GetTimeInMs(); + + while ((GetTimeInMs() - start_time) < timeout_ms) { NVMeCompletionEntry* cqe = &ctrl->io_cq[ctrl->io_cq_head]; uint16_t status = cqe->status; if ((status & 1) == ctrl->io_cq_phase) { - // Command completed ctrl->io_cq_head = (ctrl->io_cq_head + 1) % NVME_IO_QUEUE_SIZE; if (ctrl->io_cq_head == 0) { ctrl->io_cq_phase = !ctrl->io_cq_phase; } - // Ring completion doorbell (I/O queue 1) uint32_t iocq_db_off = 0x1000 + ((1 * 2 + 1) << db_shift); - NVMe_WriteReg32(iocq_db_off, ctrl->io_cq_head); // I/O CQ doorbell + NVMe_WriteReg32(iocq_db_off, ctrl->io_cq_head); - return (status >> 1) & 0x7FF; // Return status code + return (status >> 1) & 0x7FF; } delay_us(50); } - return -1; // Timeout + return -1; } static int NVMe_CreateIOQueues(void) { NVMeController* ctrl = &g_nvme_controller; - // Allocate I/O queues ctrl->io_sq = (NVMeSubmissionEntry*)VMemAlloc(NVME_IO_QUEUE_SIZE * sizeof(NVMeSubmissionEntry)); ctrl->io_cq = (NVMeCompletionEntry*)VMemAlloc(NVME_IO_QUEUE_SIZE * sizeof(NVMeCompletionEntry)); @@ -147,24 +144,22 @@ static int NVMe_CreateIOQueues(void) { ctrl->io_cq_head = 0; ctrl->io_cq_phase = 1; - // Create I/O Completion Queue NVMeSubmissionEntry cmd = {0}; - cmd.cdw0 = 0x05 | ((uint32_t)(++ctrl->next_cid) << 16); // CREATE_IO_CQ opcode + cmd.cdw0 = 0x05 | ((uint32_t)(++ctrl->next_cid) << 16); cmd.prp1 = ctrl->io_cq_phys; - cmd.cdw10 = ((NVME_IO_QUEUE_SIZE - 1) << 16) | 1; // Queue size and ID - cmd.cdw11 = 1; // Physically contiguous + cmd.cdw10 = ((NVME_IO_QUEUE_SIZE - 1) << 16) | 1; + cmd.cdw11 = 1; if (NVMe_SubmitAdminCommand(&cmd) != 0) { PrintKernel("NVMe: Failed to create I/O completion queue\n"); return -1; } - // Create I/O Submission Queue FastMemset(&cmd, 0, sizeof(cmd)); - cmd.cdw0 = 0x01 | ((uint32_t)(++ctrl->next_cid) << 16); // CREATE_IO_SQ opcode + cmd.cdw0 = 0x01 | ((uint32_t)(++ctrl->next_cid) << 16); cmd.prp1 = ctrl->io_sq_phys; - cmd.cdw10 = ((NVME_IO_QUEUE_SIZE - 1) << 16) | 1; // Queue size and ID - cmd.cdw11 = (1 << 16) | 1; // CQ ID and physically contiguous + cmd.cdw10 = ((NVME_IO_QUEUE_SIZE - 1) << 16) | 1; + cmd.cdw11 = (1 << 16) | 1; if (NVMe_SubmitAdminCommand(&cmd) != 0) { PrintKernel("NVMe: Failed to create I/O submission queue\n"); @@ -175,116 +170,143 @@ static int NVMe_CreateIOQueues(void) { } static uint64_t NVMe_GetNamespaceSize(void) { - // Allocate buffer for identify data uint8_t* identify_data = (uint8_t*)KernelMemoryAlloc(4096); if (!identify_data) return 0; FastMemset(identify_data, 0, 4096); uint64_t identify_phys = VMemGetPhysAddr((uint64_t)identify_data); - // Send IDENTIFY command for namespace 1 NVMeSubmissionEntry cmd = {0}; cmd.cdw0 = NVME_ADMIN_IDENTIFY | ((uint32_t)(++g_nvme_controller.next_cid) << 16); - cmd.nsid = 1; // Namespace 1 + cmd.nsid = 1; cmd.prp1 = identify_phys; - cmd.cdw10 = 0; // Identify namespace + cmd.cdw10 = 0; int result = NVMe_SubmitAdminCommand(&cmd); if (result != 0) { KernelFree(identify_data); - return 0x1000000; // Fallback size + return 0; } - // Get namespace size from identify data (bytes 0-7) uint64_t nsze = *(uint64_t*)identify_data; - KernelFree(identify_data); - - return nsze ? nsze : 0x1000000; // Return size or fallback + return nsze; +} + +static int NVMe_SetupPrpList(uint64_t buffer_phys, uint32_t total_bytes) { + NVMeController* ctrl = &g_nvme_controller; + uint32_t page_size = 4096; + uint32_t offset_in_first_page = buffer_phys & (page_size - 1); + uint32_t bytes_in_first_page = page_size - offset_in_first_page; + + int num_prp_entries = 0; + if (total_bytes <= bytes_in_first_page) { + return 0; // No PRP list needed + } else { + uint32_t bytes_after_first_page = total_bytes - bytes_in_first_page; + num_prp_entries = (bytes_after_first_page + page_size - 1) / page_size; + } + + if (num_prp_entries > PRP_LIST_ENTRIES) { + return -1; // Too large for our PRP list + } + + uint64_t current_phys = buffer_phys + bytes_in_first_page; + for (int i = 0; i < num_prp_entries; ++i) { + ctrl->prp_list[i] = current_phys; + current_phys += page_size; + } + + return num_prp_entries; } int NVMe_ReadSectors(uint64_t lba, uint16_t count, void* buffer) { if (!g_nvme_controller.initialized) return -1; - // Split into page-safe chunks so PRP1 is sufficient and we never cross a 4KiB page. - const uint32_t sector_size = 512; - uint8_t* buf = (uint8_t*)buffer; - uint16_t remaining = count; - - while (remaining > 0) { - uint64_t phys = VMemGetPhysAddr((uint64_t)buf); - uint32_t page_off = (uint32_t)(phys & 0xFFFULL); - uint32_t bytes_left_in_page = 4096U - page_off; - uint16_t max_sectors_in_page = (uint16_t)(bytes_left_in_page / sector_size); - if (max_sectors_in_page == 0) max_sectors_in_page = 1; // Safety: at least one sector - - // NVMe without PRP2/PRP list supports only one page via PRP1 here, cap at 8 sectors (4KiB) - uint16_t sectors_this = remaining < max_sectors_in_page ? remaining : max_sectors_in_page; - if (sectors_this > 8) sectors_this = 8; - - NVMeSubmissionEntry cmd = (NVMeSubmissionEntry){0}; - cmd.cdw0 = NVME_CMD_READ | ((uint32_t)(++g_nvme_controller.next_cid) << 16); - cmd.nsid = 1; // Namespace 1 - cmd.prp1 = phys; - cmd.cdw10 = (uint32_t)(lba & 0xFFFFFFFFULL); - cmd.cdw11 = (uint32_t)((lba >> 32) & 0xFFFFFFFFULL); - cmd.cdw12 = (uint32_t)(sectors_this - 1); // 0-based count - - int st = NVMe_SubmitIOCommand(&cmd); - if (st != 0) return st ? st : -1; - - // Advance - lba += sectors_this; - buf += (uint32_t)sectors_this * sector_size; - remaining -= sectors_this; + uint32_t total_bytes = count * 512; + uint64_t buffer_phys = VMemGetPhysAddr((uint64_t)buffer); + + NVMeSubmissionEntry cmd = {0}; + cmd.cdw0 = NVME_CMD_READ | ((uint32_t)(++g_nvme_controller.next_cid) << 16); + cmd.nsid = 1; + cmd.cdw10 = (uint32_t)(lba & 0xFFFFFFFFULL); + cmd.cdw11 = (uint32_t)((lba >> 32) & 0xFFFFFFFFULL); + cmd.cdw12 = (uint32_t)(count - 1); + + int num_prp_entries = NVMe_SetupPrpList(buffer_phys, total_bytes); + if (num_prp_entries < 0) { + return -1; // Error } - return 0; + cmd.prp1 = buffer_phys; + if (num_prp_entries > 0) { + cmd.prp2 = g_nvme_controller.prp_list_phys; + } + + return NVMe_SubmitIOCommand(&cmd); } static int NVMe_Flush(void) { - NVMeSubmissionEntry cmd = (NVMeSubmissionEntry){0}; + NVMeSubmissionEntry cmd = {0}; cmd.cdw0 = NVME_CMD_FLUSH | ((uint32_t)(++g_nvme_controller.next_cid) << 16); - cmd.nsid = 1; // Namespace 1 + cmd.nsid = 1; return NVMe_SubmitIOCommand(&cmd); } int NVMe_WriteSectors(uint64_t lba, uint16_t count, const void* buffer) { if (!g_nvme_controller.initialized) return -1; - const uint32_t sector_size = 512; - const uint8_t* buf = (const uint8_t*)buffer; - uint16_t remaining = count; - - while (remaining > 0) { - uint64_t phys = VMemGetPhysAddr((uint64_t)buf); - uint32_t page_off = (uint32_t)(phys & 0xFFFULL); - uint32_t bytes_left_in_page = 4096U - page_off; - uint16_t max_sectors_in_page = (uint16_t)(bytes_left_in_page / sector_size); - if (max_sectors_in_page == 0) max_sectors_in_page = 1; - - uint16_t sectors_this = remaining < max_sectors_in_page ? remaining : max_sectors_in_page; - if (sectors_this > 8) sectors_this = 8; - - NVMeSubmissionEntry cmd = (NVMeSubmissionEntry){0}; - cmd.cdw0 = NVME_CMD_WRITE | ((uint32_t)(++g_nvme_controller.next_cid) << 16); - cmd.nsid = 1; // Namespace 1 - cmd.prp1 = phys; - cmd.cdw10 = (uint32_t)(lba & 0xFFFFFFFFULL); - cmd.cdw11 = (uint32_t)((lba >> 32) & 0xFFFFFFFFULL); - cmd.cdw12 = (uint32_t)(sectors_this - 1); // 0-based count - - int st = NVMe_SubmitIOCommand(&cmd); - if (st != 0) return st ? st : -1; - - lba += sectors_this; - buf += (uint32_t)sectors_this * sector_size; - remaining -= sectors_this; + uint32_t total_bytes = count * 512; + uint64_t buffer_phys = VMemGetPhysAddr((uint64_t)buffer); + + NVMeSubmissionEntry cmd = {0}; + cmd.cdw0 = NVME_CMD_WRITE | ((uint32_t)(++g_nvme_controller.next_cid) << 16); + cmd.nsid = 1; + cmd.cdw10 = (uint32_t)(lba & 0xFFFFFFFFULL); + cmd.cdw11 = (uint32_t)((lba >> 32) & 0xFFFFFFFFULL); + cmd.cdw12 = (uint32_t)(count - 1); + + int num_prp_entries = NVMe_SetupPrpList(buffer_phys, total_bytes); + if (num_prp_entries < 0) { + return -1; // Error + } + + cmd.prp1 = buffer_phys; + if (num_prp_entries > 0) { + cmd.prp2 = g_nvme_controller.prp_list_phys; } - // Ensure the device flushes its volatile write cache so subsequent reads see consistent data - int flush_status = NVMe_Flush(); - return flush_status == 0 ? 0 : flush_status; + int st = NVMe_SubmitIOCommand(&cmd); + if (st != 0) return st; + + return NVMe_Flush(); +} + +void NVMe_Shutdown(void) { + if (!g_nvme_controller.initialized) return; + + PrintKernel("NVMe: Shutting down NVMe controller...\n"); + + NVMe_WriteReg32(NVME_CC, 0); + NVMe_WaitReady(0); + + if (g_nvme_controller.admin_sq) VMemFree(g_nvme_controller.admin_sq, NVME_ADMIN_QUEUE_SIZE * sizeof(NVMeSubmissionEntry)); + if (g_nvme_controller.admin_cq) VMemFree(g_nvme_controller.admin_cq, NVME_ADMIN_QUEUE_SIZE * sizeof(NVMeCompletionEntry)); + if (g_nvme_controller.io_sq) VMemFree(g_nvme_controller.io_sq, NVME_IO_QUEUE_SIZE * sizeof(NVMeSubmissionEntry)); + if (g_nvme_controller.io_cq) VMemFree(g_nvme_controller.io_cq, NVME_IO_QUEUE_SIZE * sizeof(NVMeCompletionEntry)); + if (g_nvme_controller.prp_list) VMemFree(g_nvme_controller.prp_list, PRP_LIST_ENTRIES * sizeof(uint64_t)); + + if (g_nvme_controller.mmio_base) { + VMemUnmap((uint64_t)g_nvme_controller.mmio_base, g_nvme_controller.mmio_size); + VMemFree((void*)g_nvme_controller.mmio_base, g_nvme_controller.mmio_size); + } + + if (g_nvme_controller.lock) { + rust_spinlock_free(g_nvme_controller.lock); + } + + FastMemset(&g_nvme_controller, 0, sizeof(NVMeController)); + PrintKernel("NVMe: Shutdown complete.\n"); } static int NVMe_ReadBlocksWrapper(struct BlockDevice* device, uint64_t start_lba, uint32_t count, void* buffer) { @@ -300,7 +322,6 @@ static int NVMe_WriteBlocksWrapper(struct BlockDevice* device, uint64_t start_lb int NVMe_Init(void) { PrintKernel("NVMe: Initializing NVMe driver...\n"); - // Find NVMe controller PciDevice pci_dev; if (PciFindByClass(NVME_CLASS_CODE, NVME_SUBCLASS, NVME_PROG_IF, &pci_dev) != 0) { PrintKernel("NVMe: No NVMe controller found\n"); @@ -308,19 +329,24 @@ int NVMe_Init(void) { } g_nvme_controller.pci_device = pci_dev; - - // Enable PCI device + g_nvme_controller.lock = rust_spinlock_new(); + + g_nvme_controller.prp_list = (uint64_t*)VMemAlloc(PRP_LIST_ENTRIES * sizeof(uint64_t)); + if (!g_nvme_controller.prp_list) { + PrintKernel("NVMe: Failed to allocate PRP list\n"); + NVMe_Shutdown(); + return -1; + } + g_nvme_controller.prp_list_phys = VMemGetPhysAddr((uint64_t)g_nvme_controller.prp_list); + uint16_t cmd = PciReadConfig16(pci_dev.bus, pci_dev.device, pci_dev.function, PCI_COMMAND_REG); cmd |= PCI_CMD_MEM_SPACE_EN | PCI_CMD_BUS_MASTER_EN; PciWriteConfig16(pci_dev.bus, pci_dev.device, pci_dev.function, PCI_COMMAND_REG, cmd); - // Map MMIO (NVMe typically uses a 64-bit BAR at BAR0/1) uint32_t bar0_val = PciConfigReadDWord(pci_dev.bus, pci_dev.device, pci_dev.function, PCI_BAR0_REG); uint64_t mmio_phys = 0; - bool is_mem_bar = ((bar0_val & 0x1) == 0); // bit0==0 => memory BAR - bool is_64bit_bar = ((bar0_val & 0x06) == 0x04); - if (is_mem_bar) { - if (is_64bit_bar) { + if ((bar0_val & 0x1) == 0) { + if ((bar0_val & 0x06) == 0x04) { uint32_t bar1_val = PciConfigReadDWord(pci_dev.bus, pci_dev.device, pci_dev.function, PCI_BAR0_REG + 4); mmio_phys = ((uint64_t)bar1_val << 32) | (uint64_t)(bar0_val & ~0xFULL); } else { @@ -332,56 +358,49 @@ int NVMe_Init(void) { return -1; } - // Align physical address and size to page boundaries uint64_t mmio_phys_aligned = mmio_phys & ~0xFFFULL; uint64_t mmio_offset = mmio_phys - mmio_phys_aligned; g_nvme_controller.mmio_size = GetPCIMMIOSize(&pci_dev, bar0_val); uint64_t mmio_size_aligned = ((g_nvme_controller.mmio_size + mmio_offset + 0xFFFULL) & ~0xFFFULL); - // Allocate virtual space volatile uint8_t* mmio_base_raw = (volatile uint8_t*)VMemAlloc(mmio_size_aligned); if (!mmio_base_raw) { PrintKernel("NVMe: Failed to allocate virtual space\n"); return -1; } - // Unmap RAM pages if (VMemUnmap((uint64_t)mmio_base_raw, mmio_size_aligned) != VMEM_SUCCESS) { PrintKernel("NVMe: Failed to unmap RAM pages\n"); VMemFree((void*)mmio_base_raw, mmio_size_aligned); return -1; } - // Map MMIO uint64_t map_flags = PAGE_WRITABLE | PAGE_NOCACHE; if (VMemMapMMIO((uint64_t)mmio_base_raw, mmio_phys_aligned, mmio_size_aligned, map_flags) != VMEM_SUCCESS) { PrintKernel("NVMe: Failed to map MMIO\n"); return -1; } - // Adjust base pointer to account for offset g_nvme_controller.mmio_base = mmio_base_raw + mmio_offset; g_nvme_controller.mmio_size = mmio_size_aligned; __asm__ volatile("mfence" ::: "memory"); - // Read capabilities (for doorbell stride) uint64_t cap = NVMe_ReadReg64(NVME_CAP); g_nvme_controller.dstrd = (uint8_t)((cap >> 32) & 0xF); - // Reset controller NVMe_WriteReg32(NVME_CC, 0); if (NVMe_WaitReady(0) != 0) { PrintKernel("NVMe: Controller reset timeout\n"); return -1; } - // Allocate admin queues g_nvme_controller.admin_sq = (NVMeSubmissionEntry*)VMemAlloc(NVME_ADMIN_QUEUE_SIZE * sizeof(NVMeSubmissionEntry)); g_nvme_controller.admin_cq = (NVMeCompletionEntry*)VMemAlloc(NVME_ADMIN_QUEUE_SIZE * sizeof(NVMeCompletionEntry)); if (!g_nvme_controller.admin_sq || !g_nvme_controller.admin_cq) { PrintKernel("NVMe: Failed to allocate admin queues\n"); + NVMe_Shutdown(); return -1; } @@ -395,37 +414,40 @@ int NVMe_Init(void) { g_nvme_controller.admin_cq_phase = 1; g_nvme_controller.next_cid = 0; - // Configure admin queues NVMe_WriteReg32(NVME_AQA, ((NVME_ADMIN_QUEUE_SIZE - 1) << 16) | (NVME_ADMIN_QUEUE_SIZE - 1)); NVMe_WriteReg64(NVME_ASQ, g_nvme_controller.admin_sq_phys); NVMe_WriteReg64(NVME_ACQ, g_nvme_controller.admin_cq_phys); - // Enable controller uint32_t cc = NVME_CC_EN | NVME_CC_CSS_NVM | NVME_CC_MPS_4K | NVME_CC_AMS_RR | NVME_CC_SHN_NONE | NVME_CC_IOSQES_64 | NVME_CC_IOCQES_16; NVMe_WriteReg32(NVME_CC, cc); if (NVMe_WaitReady(1) != 0) { PrintKernel("NVMe: Controller enable timeout\n"); + NVMe_Shutdown(); return -1; } - // Create I/O queues if (NVMe_CreateIOQueues() != 0) { PrintKernel("NVMe: Failed to create I/O queues\n"); + NVMe_Shutdown(); return -1; } - // Get namespace size g_nvme_controller.namespace_size = NVMe_GetNamespaceSize(); + if (g_nvme_controller.namespace_size == 0) { + PrintKernel("NVMe: Failed to get namespace size\n"); + NVMe_Shutdown(); + return -1; + } + g_nvme_controller.initialized = 1; - // Register block device char dev_name[16]; GenerateDriveNameInto(DEVICE_TYPE_NVME, dev_name); BlockDevice* nvme_device = BlockDeviceRegister( DEVICE_TYPE_NVME, - 512, // Block size + 512, g_nvme_controller.namespace_size, dev_name, &g_nvme_controller, @@ -440,5 +462,6 @@ int NVMe_Init(void) { } PrintKernel("NVMe: Failed to register block device\n"); + NVMe_Shutdown(); return -1; } \ No newline at end of file diff --git a/drivers/storage/NVMe.h b/drivers/storage/NVMe.h index 378e7d6..07c6e1a 100644 --- a/drivers/storage/NVMe.h +++ b/drivers/storage/NVMe.h @@ -3,6 +3,7 @@ #include "stdint.h" #include "PCI/PCI.h" +#include "kernel/atomic/SpinlockRust.h" // NVMe PCI Class/Subclass #define NVME_CLASS_CODE 0x01 @@ -42,6 +43,8 @@ #define NVME_ADMIN_QUEUE_SIZE 64 #define NVME_IO_QUEUE_SIZE 256 +#define PRP_LIST_ENTRIES 512 + // NVMe Submission Queue Entry typedef struct { uint32_t cdw0; // Command Dword 0 @@ -73,6 +76,9 @@ typedef struct { PciDevice pci_device; volatile uint8_t* mmio_base; uint64_t mmio_size; + RustSpinLock* lock; + uint64_t* prp_list; + uint64_t prp_list_phys; // Doorbell parameters uint8_t dstrd; // CAP.DSTRD value (stride as 2^n of 4-byte units) @@ -102,6 +108,7 @@ typedef struct { // Function prototypes int NVMe_Init(void); +void NVMe_Shutdown(void); int NVMe_ReadSectors(uint64_t lba, uint16_t count, void* buffer); int NVMe_WriteSectors(uint64_t lba, uint16_t count, const void* buffer); diff --git a/kernel/core/Panic.c b/kernel/core/Panic.c index d74ab37..6a73011 100644 --- a/kernel/core/Panic.c +++ b/kernel/core/Panic.c @@ -1,6 +1,7 @@ #include "Panic.h" #include "Io.h" // The ONLY necessary includes for display are now Console and Serial +#include "ACPI.h" #include "../../mm/KernelHeap.h" #include "../../mm/PMem.h" #include "Console.h" @@ -78,8 +79,16 @@ void __attribute__((noreturn)) KernelPanicHandler(const char* message, uint64_t VBEShowPanic(); #endif PrintKernel("[FATAL] - [KERNEL PANIC] - "); - if (message) PrintKernel(message); + if (message) { + PrintKernel(message); + PrintNewline(); + } else PrintKernel("No message provided."); + + PrintKernel("Starting reset procedure...\n"); + ACPIResetProcedure(); + PrintKernelSuccess("Reset procedure complete\n"); + PrintKernel("\n"); if (ctx) { char hex[20]; @@ -141,7 +150,7 @@ void __attribute__((noreturn)) KernelPanicHandler(const char* message, uint64_t PrintKernelInt(stats.fragmentation_score); PrintKernel("% fragmented\n"); PrintVMemStats(); - PrintHeapStats(); + // PrintHeapStats(); PrintKernelAt("SYSTEM HALTED.", 50, 45); ConsoleSetColor(VGA_COLOR_DEFAULT); diff --git a/mm/KernelHeap.h b/mm/KernelHeap.h index 874f168..45f18af 100644 --- a/mm/KernelHeap.h +++ b/mm/KernelHeap.h @@ -7,7 +7,6 @@ #include "stdint.h" #include "stddef.h" -#include "Console.h" #include "VMem.h" #include "APIC.h" #include "Magazine.h"