diff --git a/src/aarch64/mmu.rs b/src/aarch64/mmu.rs index a380131..50aac0c 100644 --- a/src/aarch64/mmu.rs +++ b/src/aarch64/mmu.rs @@ -49,8 +49,54 @@ pub const STACK_START_ADDR: usize = !KERNEL_VIRTUAL_MEM_SPACE & (!0xF); pub mod physical_mapping; -type VirtAddr = usize; -type PhysAddr = usize; +pub type VirtAddr = usize; +pub type PhysAddr = usize; + +#[derive(Clone, Copy)] +pub struct TableEntry { + value: u64, +} + +impl TableEntry { + pub fn invalid() -> Self { + Self { value: 0 } + } + + fn table_descriptor(addr: PhysAddr) -> Self { + Self { + value: (addr as u64 & 0x0000_FFFF_FFFF_F000) | TABLE, + } + } + + fn block_descriptor(physical_address: usize, additional_flags: u64) -> Self { + Self { + value: (physical_address as u64 & 0x0000_FFFF_FFFF_F000) + | BLOCK + | ACCESS_FLAG + | INNER_SHAREABILITY + | additional_flags, + } + } + + fn page_descriptor(physical_address: usize, additional_flags: u64) -> Self { + Self { + value: (physical_address as u64 & 0x0000_FFFF_FFFF_F000) + | PAGE + | ACCESS_FLAG + | INNER_SHAREABILITY + | additional_flags, + } + } + + fn is_invalid(self) -> bool { + self.value & 0b11 == 0 + } + + #[inline] + fn address(self) -> PhysAddr { + self.value as usize & 0x0000_FFFF_FFFF_F000 + } +} pub enum PhysSource { Any, @@ -58,12 +104,12 @@ pub enum PhysSource { } #[repr(align(4096))] -pub struct PageTable([u64; TABLE_ENTRY_COUNT]); +pub struct PageTable([TableEntry; TABLE_ENTRY_COUNT]); #[no_mangle] -pub static mut TRANSLATIONTABLE_TTBR0: PageTable = PageTable([0; 512]); +pub static mut TRANSLATIONTABLE_TTBR0: PageTable = PageTable([TableEntry { value: 0 }; 512]); #[no_mangle] -pub static mut TRANSLATIONTABLE_TTBR1: PageTable = PageTable([0; 512]); +pub static mut TRANSLATIONTABLE_TTBR1: PageTable = PageTable([TableEntry { value: 0 }; 512]); /// Allocate a memory block of `size` starting at `virtual_address`. pub fn allocate_memory( @@ -102,7 +148,7 @@ fn map_range_explicit( ) -> Result<(), NovaError> { let mut remaining = size_bytes; - while virt % LEVEL2_BLOCK_SIZE != 0 { + while !virt.is_multiple_of(LEVEL2_BLOCK_SIZE) && remaining > 0 { map_page(virt, phys, base, flags)?; (virt, _) = virt.overflowing_add(GRANULARITY); phys += GRANULARITY; @@ -192,11 +238,11 @@ pub fn map_page( let table_ptr = navigate_table(base_table_ptr, &offsets, true)?; let table = unsafe { &mut *table_ptr }; - if table.0[l3_off] & 0b11 > 0 { + if !table.0[l3_off].is_invalid() { return Err(NovaError::Paging); } - table.0[l3_off] = create_page_descriptor_entry(physical_address, additional_flags); + table.0[l3_off] = TableEntry::page_descriptor(physical_address, additional_flags); Ok(()) } @@ -234,11 +280,11 @@ pub fn map_l2_block( let table = unsafe { &mut *table_ptr }; // Verify virtual address is available. - if table.0[l2_off] & 0b11 != 0 { + if !table.0[l2_off].is_invalid() { return Err(NovaError::Paging); } - let new_entry = create_block_descriptor_entry(physical_address, additional_flags); + let new_entry = TableEntry::block_descriptor(physical_address, additional_flags); table.0[l2_off] = new_entry; @@ -278,26 +324,6 @@ pub fn reserve_range( Ok(start_physical_address) } -fn create_block_descriptor_entry(physical_address: usize, additional_flags: u64) -> u64 { - (physical_address as u64 & 0x0000_FFFF_FFFF_F000) - | BLOCK - | ACCESS_FLAG - | INNER_SHAREABILITY - | additional_flags -} - -fn create_page_descriptor_entry(physical_address: usize, additional_flags: u64) -> u64 { - (physical_address as u64 & 0x0000_FFFF_FFFF_F000) - | PAGE - | ACCESS_FLAG - | INNER_SHAREABILITY - | additional_flags -} - -fn create_table_descriptor_entry(addr: usize) -> u64 { - (addr as u64 & 0x0000_FFFF_FFFF_F000) | TABLE -} - fn virtual_address_to_table_offset(virtual_addr: usize) -> (usize, usize, usize) { let absolute_page_off = (virtual_addr & !KERNEL_VIRTUAL_MEM_SPACE) / GRANULARITY; let l3_off = absolute_page_off % TABLE_ENTRY_COUNT; @@ -329,14 +355,14 @@ fn next_table( create_missing: bool, ) -> Result<*mut PageTable, NovaError> { let table = unsafe { &mut *table_ptr }; - match table.0[offset] & 0b11 { + match table.0[offset].value & 0b11 { 0 => { if !create_missing { return Err(NovaError::Paging); } let new_phys_page_table_address = reserve_page(); - table.0[offset] = create_table_descriptor_entry(new_phys_page_table_address); + table.0[offset] = TableEntry::table_descriptor(new_phys_page_table_address); map_page( phys_table_to_kernel_space(new_phys_page_table_address), new_phys_page_table_address, @@ -344,26 +370,29 @@ fn next_table( NORMAL_MEM | WRITABLE | PXN | UXN, )?; - Ok(entry_table_addr(table.0[offset]) as *mut PageTable) + Ok(resolve_table_addr(table.0[offset].address()) as *mut PageTable) } 1 => Err(NovaError::Paging), - 3 => Ok(entry_table_addr(table.0[offset]) as *mut PageTable), + 3 => Ok(resolve_table_addr(table.0[offset].address()) as *mut PageTable), _ => unreachable!(), } } -/// Extracts the physical address out of an table entry. +/// Converts a physical table address and returns the corresponding virtual address depending on EL. +/// +/// - `== EL0` -> panic +/// - `== EL1` -> 0xFFFFFF82XXXXXXXX +/// - `>= EL2` -> physical address #[inline] -fn entry_phys(entry: u64) -> PhysAddr { - entry as usize & 0x0000_FFFF_FFFF_F000 -} +fn resolve_table_addr(physical_address: PhysAddr) -> VirtAddr { + let current_el = get_current_el(); -#[inline] -fn entry_table_addr(entry: u64) -> VirtAddr { - if get_current_el() == 1 { - phys_table_to_kernel_space(entry_phys(entry)) + if current_el >= 2 { + physical_address + } else if get_current_el() == 1 { + phys_table_to_kernel_space(physical_address) } else { - entry_phys(entry) + panic!("Access to table entries is forbidden in EL0.") } } @@ -372,16 +401,3 @@ fn entry_table_addr(entry: u64) -> VirtAddr { fn phys_table_to_kernel_space(entry: usize) -> VirtAddr { entry | TRANSLATION_TABLE_BASE_ADDR } - -fn page_address_to_physical_address(mut virtual_address: VirtAddr) -> PhysAddr { - let root_table = if virtual_address & KERNEL_VIRTUAL_MEM_SPACE > 0 { - &raw mut TRANSLATIONTABLE_TTBR1 - } else { - &raw mut TRANSLATIONTABLE_TTBR0 - }; - virtual_address &= !KERNEL_VIRTUAL_MEM_SPACE; - let (l1_off, l2_off, l3_off) = virtual_address_to_table_offset(virtual_address); - let offsets = [l1_off, l2_off]; - let table = unsafe { &*navigate_table(root_table, &offsets, false).unwrap() }; - entry_phys(table.0[l3_off]) -} diff --git a/src/aarch64/mmu/physical_mapping.rs b/src/aarch64/mmu/physical_mapping.rs index 4dba6f0..641d20b 100644 --- a/src/aarch64/mmu/physical_mapping.rs +++ b/src/aarch64/mmu/physical_mapping.rs @@ -1,13 +1,19 @@ use crate::aarch64::mmu::{PhysAddr, GRANULARITY, L2_BLOCK_BITMAP_WORDS, MAX_PAGE_COUNT}; use nova_error::NovaError; -static mut PAGING_BITMAP: [u64; MAX_PAGE_COUNT / 64] = [0; MAX_PAGE_COUNT / 64]; +struct PagingMap { + bitmap: [u64; MAX_PAGE_COUNT / 64], +} + +static mut PAGING_BITMAP: PagingMap = PagingMap { + bitmap: [0; MAX_PAGE_COUNT / 64], +}; pub fn reserve_page() -> PhysAddr { if let Some(address) = find_unallocated_page() { let page = address / GRANULARITY; let word_index = page / 64; - unsafe { PAGING_BITMAP[word_index] |= 1 << (page % 64) }; + unsafe { PAGING_BITMAP.bitmap[word_index] |= 1 << (page % 64) }; return address; } panic!("Out of Memory!"); @@ -17,18 +23,18 @@ pub fn reserve_page_explicit(physical_address: usize) -> Result 0 { + if unsafe { PAGING_BITMAP.bitmap[word_index] } & (1 << (page % 64)) > 0 { return Err(NovaError::Paging); } - unsafe { PAGING_BITMAP[word_index] |= 1 << (page % 64) }; + unsafe { PAGING_BITMAP.bitmap[word_index] |= 1 << (page % 64) }; Ok(physical_address) } pub fn reserve_block() -> usize { if let Some(start) = find_contiguous_free_bitmap_words(L2_BLOCK_BITMAP_WORDS) { for j in 0..L2_BLOCK_BITMAP_WORDS { - unsafe { PAGING_BITMAP[start + j] = u64::MAX }; + unsafe { PAGING_BITMAP.bitmap[start + j] = u64::MAX }; } return start * 64 * GRANULARITY; } @@ -40,21 +46,21 @@ pub fn reserve_block_explicit(physical_address: usize) -> Result<(), NovaError> let page = physical_address / GRANULARITY; for i in 0..L2_BLOCK_BITMAP_WORDS { unsafe { - if PAGING_BITMAP[(page / 64) + i] != 0 { + if PAGING_BITMAP.bitmap[(page / 64) + i] != 0 { return Err(NovaError::Paging); } }; } for i in 0..L2_BLOCK_BITMAP_WORDS { unsafe { - PAGING_BITMAP[(page / 64) + i] = u64::MAX; + PAGING_BITMAP.bitmap[(page / 64) + i] = u64::MAX; }; } Ok(()) } fn find_unallocated_page() -> Option { - for (i, entry) in unsafe { PAGING_BITMAP }.iter().enumerate() { + for (i, entry) in unsafe { PAGING_BITMAP.bitmap }.iter().enumerate() { if *entry != u64::MAX { for offset in 0..64 { if entry >> offset & 0b1 == 0 { @@ -70,7 +76,7 @@ fn find_contiguous_free_bitmap_words(required_words: usize) -> Option { let mut run_start = 0; let mut run_len = 0; - for (i, entry) in unsafe { PAGING_BITMAP }.iter().enumerate() { + for (i, entry) in unsafe { PAGING_BITMAP.bitmap }.iter().enumerate() { if *entry == 0 { if run_len == 0 { run_start = i; diff --git a/src/configuration.rs b/src/configuration.rs index 949eede..08cdc10 100644 --- a/src/configuration.rs +++ b/src/configuration.rs @@ -31,99 +31,4 @@ const AS: u64 = 0b1 << 36; // configure an ASID size of 16 bits #[no_mangle] pub static TCR_EL1_CONF: u64 = IPS | TG0 | TG1 | T0SZ | T1SZ | SH0 | SH1 | AS; -pub mod mmu { - use crate::{ - aarch64::mmu::{ - alloc_block_l2_explicit, allocate_memory, map_l2_block, map_page, reserve_range, - PhysSource, DEVICE_MEM, EL0_ACCESSIBLE, GRANULARITY, KERNEL_VIRTUAL_MEM_SPACE, - LEVEL1_BLOCK_SIZE, LEVEL2_BLOCK_SIZE, NORMAL_MEM, PXN, READ_ONLY, STACK_START_ADDR, - TRANSLATIONTABLE_TTBR0, UXN, WRITABLE, - }, - PERIPHERAL_BASE, - }; - - #[no_mangle] - static EL1_STACK_TOP: usize = STACK_START_ADDR | KERNEL_VIRTUAL_MEM_SPACE; - const EL1_STACK_SIZE: usize = LEVEL2_BLOCK_SIZE * 2; - #[no_mangle] - static EL0_STACK_TOP: usize = STACK_START_ADDR; - const EL0_STACK_SIZE: usize = LEVEL2_BLOCK_SIZE * 2; - extern "C" { - static __text_end: u64; - static __share_end: u64; - static __kernel_end: u64; - } - - pub fn initialize_mmu_translation_tables() { - let text_end = unsafe { &__text_end } as *const _ as usize; - let shared_segment_end = unsafe { &__share_end } as *const _ as usize; - let kernel_end = unsafe { &__kernel_end } as *const _ as usize; - - reserve_range(0x0, kernel_end).unwrap(); - - for addr in (0..text_end).step_by(GRANULARITY) { - map_page( - addr, - addr, - core::ptr::addr_of_mut!(TRANSLATIONTABLE_TTBR0), - EL0_ACCESSIBLE | READ_ONLY | NORMAL_MEM, - ) - .unwrap(); - } - - for addr in (text_end..shared_segment_end).step_by(GRANULARITY) { - map_page( - addr, - addr, - core::ptr::addr_of_mut!(TRANSLATIONTABLE_TTBR0), - EL0_ACCESSIBLE | WRITABLE | NORMAL_MEM, - ) - .unwrap(); - } - - for addr in (shared_segment_end..kernel_end).step_by(LEVEL2_BLOCK_SIZE) { - map_l2_block( - addr, - addr, - core::ptr::addr_of_mut!(TRANSLATIONTABLE_TTBR0), - WRITABLE | UXN | NORMAL_MEM, - ) - .unwrap(); - } - - for addr in (PERIPHERAL_BASE..LEVEL1_BLOCK_SIZE).step_by(LEVEL2_BLOCK_SIZE) { - alloc_block_l2_explicit( - addr, - addr, - core::ptr::addr_of_mut!(TRANSLATIONTABLE_TTBR0), - EL0_ACCESSIBLE | WRITABLE | UXN | PXN | DEVICE_MEM, - ) - .unwrap(); - } - - // Frame Buffer memory range - allocate_memory( - 0x3c100000, - 1080 * 1920 * 4, - PhysSource::Explicit(0x3c100000), - NORMAL_MEM | PXN | UXN | WRITABLE | EL0_ACCESSIBLE, - ) - .unwrap(); - - allocate_memory( - EL1_STACK_TOP - EL1_STACK_SIZE + 0x10, - EL1_STACK_SIZE, - PhysSource::Any, - WRITABLE | NORMAL_MEM, - ) - .unwrap(); - - allocate_memory( - EL0_STACK_TOP - EL0_STACK_SIZE + 0x10, - EL0_STACK_SIZE, - PhysSource::Any, - WRITABLE | EL0_ACCESSIBLE | NORMAL_MEM, - ) - .unwrap(); - } -} +pub mod memory_mapping; diff --git a/src/configuration/memory_mapping.rs b/src/configuration/memory_mapping.rs new file mode 100644 index 0000000..e33e659 --- /dev/null +++ b/src/configuration/memory_mapping.rs @@ -0,0 +1,113 @@ +use crate::{ + aarch64::mmu::{ + alloc_block_l2_explicit, allocate_memory, map_l2_block, map_page, + physical_mapping::reserve_page, reserve_range, PhysAddr, PhysSource, VirtAddr, DEVICE_MEM, + EL0_ACCESSIBLE, GRANULARITY, KERNEL_VIRTUAL_MEM_SPACE, LEVEL1_BLOCK_SIZE, + LEVEL2_BLOCK_SIZE, NORMAL_MEM, PXN, READ_ONLY, STACK_START_ADDR, TRANSLATIONTABLE_TTBR0, + UXN, WRITABLE, + }, + PERIPHERAL_BASE, +}; + +#[no_mangle] +static EL1_STACK_TOP: usize = STACK_START_ADDR | KERNEL_VIRTUAL_MEM_SPACE; +const EL1_STACK_SIZE: usize = LEVEL2_BLOCK_SIZE * 2; +#[no_mangle] +static EL0_STACK_TOP: usize = STACK_START_ADDR; +const EL0_STACK_SIZE: usize = LEVEL2_BLOCK_SIZE * 2; + +pub const MAILBOX_VIRTUAL_ADDRESS: VirtAddr = 0xFFFF_FF81_FFFF_E000; +pub static mut MAILBOX_PHYSICAL_ADDRESS: Option = None; + +extern "C" { + static __text_end: u64; + static __share_end: u64; + static __kernel_end: u64; +} + +pub fn initialize_mmu_translation_tables() { + let text_end = unsafe { &__text_end } as *const _ as usize; + let shared_segment_end = unsafe { &__share_end } as *const _ as usize; + let kernel_end = unsafe { &__kernel_end } as *const _ as usize; + + reserve_range(0x0, kernel_end).unwrap(); + + for addr in (0..text_end).step_by(GRANULARITY) { + map_page( + addr, + addr, + core::ptr::addr_of_mut!(TRANSLATIONTABLE_TTBR0), + EL0_ACCESSIBLE | READ_ONLY | NORMAL_MEM, + ) + .unwrap(); + } + + for addr in (text_end..shared_segment_end).step_by(GRANULARITY) { + map_page( + addr, + addr, + core::ptr::addr_of_mut!(TRANSLATIONTABLE_TTBR0), + EL0_ACCESSIBLE | WRITABLE | NORMAL_MEM, + ) + .unwrap(); + } + + for addr in (shared_segment_end..kernel_end).step_by(LEVEL2_BLOCK_SIZE) { + map_l2_block( + addr, + addr, + core::ptr::addr_of_mut!(TRANSLATIONTABLE_TTBR0), + WRITABLE | UXN | NORMAL_MEM, + ) + .unwrap(); + } + + for addr in (PERIPHERAL_BASE..LEVEL1_BLOCK_SIZE).step_by(LEVEL2_BLOCK_SIZE) { + alloc_block_l2_explicit( + addr, + addr, + core::ptr::addr_of_mut!(TRANSLATIONTABLE_TTBR0), + EL0_ACCESSIBLE | WRITABLE | UXN | PXN | DEVICE_MEM, + ) + .unwrap(); + } + + // Frame Buffer memory range + allocate_memory( + 0x3c100000, + 1080 * 1920 * 4, + PhysSource::Explicit(0x3c100000), + NORMAL_MEM | PXN | UXN | WRITABLE | EL0_ACCESSIBLE, + ) + .unwrap(); + + // Allocate EL1 stack + allocate_memory( + EL1_STACK_TOP - EL1_STACK_SIZE + 0x10, + EL1_STACK_SIZE, + PhysSource::Any, + WRITABLE | NORMAL_MEM, + ) + .unwrap(); + + // Allocate EL0 stack + allocate_memory( + EL0_STACK_TOP - EL0_STACK_SIZE + 0x10, + EL0_STACK_SIZE, + PhysSource::Any, + WRITABLE | EL0_ACCESSIBLE | NORMAL_MEM, + ) + .unwrap(); + + { + let addr = reserve_page(); + unsafe { MAILBOX_PHYSICAL_ADDRESS = Some(addr) }; + allocate_memory( + MAILBOX_VIRTUAL_ADDRESS, + GRANULARITY, + PhysSource::Explicit(addr), + WRITABLE | NORMAL_MEM, + ) + .unwrap(); + } +} diff --git a/src/interrupt_handlers.rs b/src/interrupt_handlers.rs index bcba2af..f70cda2 100644 --- a/src/interrupt_handlers.rs +++ b/src/interrupt_handlers.rs @@ -3,12 +3,9 @@ use core::arch::asm; use alloc::vec::Vec; use crate::{ - aarch64::{ - mmu::{allocate_memory, physical_mapping::reserve_page}, - registers::{ - daif::{mask_all, unmask_irq}, - read_elr_el1, read_esr_el1, read_exception_source_el, - }, + aarch64::registers::{ + daif::{mask_all, unmask_irq}, + read_elr_el1, read_esr_el1, read_exception_source_el, }, get_current_el, peripherals::{ @@ -159,14 +156,14 @@ unsafe extern "C" fn rust_synchronous_interrupt_imm_lower_aarch64(frame: &mut Tr println!("Returning to kernel main..."); set_return_to_kernel_main(); - return 0; + 0 } fn handle_svc(frame: &mut TrapFrame) -> usize { match frame.x8 { 67 => { let response = mailbox::read_soc_temp([0]).unwrap(); - response[0] as usize + response[1] as usize } _ => 0, } diff --git a/src/lib.rs b/src/lib.rs index 8f7d910..9d8cf5c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ use crate::{ }, interrupt_handlers::initialize_interrupt_handler, logger::DefaultLogger, + pi3::timer::sleep_s, }; static PERIPHERAL_BASE: usize = 0x3F00_0000; @@ -43,6 +44,7 @@ pub unsafe fn init_kernel_heap() { fn panic(_panic: &PanicInfo) -> ! { loop { println!("Panic: {}", _panic.message()); + sleep_s(1); } } diff --git a/src/main.rs b/src/main.rs index 8dd7119..8faefc5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ #![allow(clippy::missing_safety_doc)] use core::{ arch::{asm, global_asm}, - ptr::{read_volatile, write_volatile}, + ptr::write_volatile, }; extern crate alloc; @@ -12,7 +12,7 @@ extern crate alloc; use alloc::vec::Vec; use nova::{ aarch64::registers::{daif, read_id_aa64mmfr0_el1}, - configuration::mmu::initialize_mmu_translation_tables, + configuration::memory_mapping::initialize_mmu_translation_tables, framebuffer::{FrameBuffer, BLUE, GREEN, RED}, get_current_el, interrupt_handlers::{enable_irq_source, IRQSource}, @@ -64,11 +64,9 @@ pub extern "C" fn main() -> ! { println!("Hello World!"); println!("Exception level: {}", get_current_el()); - unsafe { - initialize_mmu_translation_tables(); - configure_mmu_el1(); - println!("MMU initialized..."); - }; + initialize_mmu_translation_tables(); + unsafe { configure_mmu_el1() }; + println!("MMU initialized..."); println!("Register: AA64MMFR0_EL1: {:064b}", read_id_aa64mmfr0_el1()); println!("Moving El2->EL1"); diff --git a/src/pi3/mailbox.rs b/src/pi3/mailbox.rs index c586684..97cbc88 100644 --- a/src/pi3/mailbox.rs +++ b/src/pi3/mailbox.rs @@ -1,4 +1,9 @@ -use crate::{read_address, write_address}; +use core::slice; + +use crate::{ + aarch64::mmu::GRANULARITY, configuration::memory_mapping::MAILBOX_PHYSICAL_ADDRESS, + configuration::memory_mapping::MAILBOX_VIRTUAL_ADDRESS, read_address, write_address, +}; use nova_error::NovaError; const MBOX_BASE: u32 = 0x3F00_0000 + 0xB880; @@ -31,8 +36,9 @@ macro_rules! mailbox_command { pub fn $name( request_data: [u32; $request_len / 4], ) -> Result<[u32; $response_len / 4], NovaError> { - let mut mailbox = - [0u32; (HEADER_LENGTH + max!($request_len, $response_len) + FOOTER_LENGTH) / 4]; + let mailbox = unsafe { + slice::from_raw_parts_mut(MAILBOX_VIRTUAL_ADDRESS as *mut u32, GRANULARITY / 4) + }; mailbox[0] = (HEADER_LENGTH + max!($request_len, $response_len) + FOOTER_LENGTH) as u32; // Total length in Bytes mailbox[1] = 0; // Request mailbox[2] = $tag; // Command Tag @@ -42,9 +48,9 @@ macro_rules! mailbox_command { mailbox[5..(5 + ($request_len / 4))].copy_from_slice(&request_data); mailbox[(5 + ($request_len / 4))..].fill(0); - let addr = core::ptr::addr_of!(mailbox[0]) as u32; + //let addr = core::ptr::addr_of!(mailbox[0]) as u32; - write_mailbox(8, addr); + write_mailbox(8, unsafe { MAILBOX_PHYSICAL_ADDRESS.unwrap() } as u32); let _ = read_mailbox(8);