From 661fe06d9daee555a39c16a558c0722ea6bc84be Mon Sep 17 00:00:00 2001
From: bunnei <bunneidev@gmail.com>
Date: Sat, 29 Oct 2022 16:08:33 -0700
Subject: [PATCH] core: hle: kernel: k_page_table: Implement IPC memory
methods.
---
src/core/hle/kernel/k_page_table.cpp | 812 ++++++++++++++++++++++++++-
src/core/hle/kernel/k_page_table.h | 100 ++++
src/core/hle/kernel/svc_results.h | 1 +
3 files changed, 910 insertions(+), 3 deletions(-)
diff --git a/src/core/hle/kernel/k_page_table.cpp b/src/core/hle/kernel/k_page_table.cpp
index 0f1bab067..2635d8148 100644
--- a/src/core/hle/kernel/k_page_table.cpp
+++ b/src/core/hle/kernel/k_page_table.cpp
@@ -24,6 +24,65 @@ namespace Kernel {
namespace {
+class KScopedLightLockPair {
+ YUZU_NON_COPYABLE(KScopedLightLockPair);
+ YUZU_NON_MOVEABLE(KScopedLightLockPair);
+
+private:
+ KLightLock* m_lower;
+ KLightLock* m_upper;
+
+public:
+ KScopedLightLockPair(KLightLock& lhs, KLightLock& rhs) {
+ // Ensure our locks are in a consistent order.
+ if (std::addressof(lhs) <= std::addressof(rhs)) {
+ m_lower = std::addressof(lhs);
+ m_upper = std::addressof(rhs);
+ } else {
+ m_lower = std::addressof(rhs);
+ m_upper = std::addressof(lhs);
+ }
+
+ // Acquire both locks.
+ m_lower->Lock();
+ if (m_lower != m_upper) {
+ m_upper->Lock();
+ }
+ }
+
+ ~KScopedLightLockPair() {
+ // Unlock the upper lock.
+ if (m_upper != nullptr && m_upper != m_lower) {
+ m_upper->Unlock();
+ }
+
+ // Unlock the lower lock.
+ if (m_lower != nullptr) {
+ m_lower->Unlock();
+ }
+ }
+
+public:
+ // Utility.
+ void TryUnlockHalf(KLightLock& lock) {
+ // Only allow unlocking if the lock is half the pair.
+ if (m_lower != m_upper) {
+ // We want to be sure the lock is one we own.
+ if (m_lower == std::addressof(lock)) {
+ lock.Unlock();
+ m_lower = nullptr;
+ } else if (m_upper == std::addressof(lock)) {
+ lock.Unlock();
+ m_upper = nullptr;
+ }
+ }
+ }
+};
+
+} // namespace
+
+namespace {
+
using namespace Common::Literals;
constexpr size_t GetAddressSpaceWidthFromType(FileSys::ProgramAddressSpaceType as_type) {
@@ -676,7 +735,8 @@ bool KPageTable::IsValidPageGroup(const KPageGroup& pg_ll, VAddr addr, size_t nu
Result KPageTable::UnmapProcessMemory(VAddr dst_addr, size_t size, KPageTable& src_page_table,
VAddr src_addr) {
- KScopedLightLock lk(m_general_lock);
+ // Acquire the table locks.
+ KScopedLightLockPair lk(src_page_table.m_general_lock, m_general_lock);
const size_t num_pages{size / PageSize};
@@ -712,6 +772,723 @@ Result KPageTable::UnmapProcessMemory(VAddr dst_addr, size_t size, KPageTable& s
R_SUCCEED();
}
+Result KPageTable::SetupForIpcClient(PageLinkedList* page_list, size_t* out_blocks_needed,
+ VAddr address, size_t size, KMemoryPermission test_perm,
+ KMemoryState dst_state) {
+ // Validate pre-conditions.
+ ASSERT(this->IsLockedByCurrentThread());
+ ASSERT(test_perm == KMemoryPermission::UserReadWrite ||
+ test_perm == KMemoryPermission::UserRead);
+
+ // Check that the address is in range.
+ R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory);
+
+ // Get the source permission.
+ const auto src_perm = static_cast<KMemoryPermission>(
+ (test_perm == KMemoryPermission::UserReadWrite)
+ ? KMemoryPermission::KernelReadWrite | KMemoryPermission::NotMapped
+ : KMemoryPermission::UserRead);
+
+ // Get aligned extents.
+ const VAddr aligned_src_start = Common::AlignDown((address), PageSize);
+ const VAddr aligned_src_end = Common::AlignUp((address) + size, PageSize);
+ const VAddr mapping_src_start = Common::AlignUp((address), PageSize);
+ const VAddr mapping_src_end = Common::AlignDown((address) + size, PageSize);
+
+ const auto aligned_src_last = (aligned_src_end)-1;
+ const auto mapping_src_last = (mapping_src_end)-1;
+
+ // Get the test state and attribute mask.
+ KMemoryState test_state;
+ KMemoryAttribute test_attr_mask;
+ switch (dst_state) {
+ case KMemoryState::Ipc:
+ test_state = KMemoryState::FlagCanUseIpc;
+ test_attr_mask =
+ KMemoryAttribute::Uncached | KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked;
+ break;
+ case KMemoryState::NonSecureIpc:
+ test_state = KMemoryState::FlagCanUseNonSecureIpc;
+ test_attr_mask = KMemoryAttribute::Uncached | KMemoryAttribute::Locked;
+ break;
+ case KMemoryState::NonDeviceIpc:
+ test_state = KMemoryState::FlagCanUseNonDeviceIpc;
+ test_attr_mask = KMemoryAttribute::Uncached | KMemoryAttribute::Locked;
+ break;
+ default:
+ R_THROW(ResultInvalidCombination);
+ }
+
+ // Ensure that on failure, we roll back appropriately.
+ size_t mapped_size = 0;
+ ON_RESULT_FAILURE {
+ if (mapped_size > 0) {
+ this->CleanupForIpcClientOnServerSetupFailure(page_list, mapping_src_start, mapped_size,
+ src_perm);
+ }
+ };
+
+ size_t blocks_needed = 0;
+
+ // Iterate, mapping as needed.
+ KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(aligned_src_start);
+ while (true) {
+ const KMemoryInfo info = it->GetMemoryInfo();
+
+ // Validate the current block.
+ R_TRY(this->CheckMemoryState(info, test_state, test_state, test_perm, test_perm,
+ test_attr_mask, KMemoryAttribute::None));
+
+ if (mapping_src_start < mapping_src_end && (mapping_src_start) < info.GetEndAddress() &&
+ info.GetAddress() < (mapping_src_end)) {
+ const auto cur_start =
+ info.GetAddress() >= (mapping_src_start) ? info.GetAddress() : (mapping_src_start);
+ const auto cur_end = mapping_src_last >= info.GetLastAddress() ? info.GetEndAddress()
+ : (mapping_src_end);
+ const size_t cur_size = cur_end - cur_start;
+
+ if (info.GetAddress() < (mapping_src_start)) {
+ ++blocks_needed;
+ }
+ if (mapping_src_last < info.GetLastAddress()) {
+ ++blocks_needed;
+ }
+
+ // Set the permissions on the block, if we need to.
+ if ((info.GetPermission() & KMemoryPermission::IpcLockChangeMask) != src_perm) {
+ R_TRY(Operate(cur_start, cur_size / PageSize, src_perm,
+ OperationType::ChangePermissions));
+ }
+
+ // Note that we mapped this part.
+ mapped_size += cur_size;
+ }
+
+ // If the block is at the end, we're done.
+ if (aligned_src_last <= info.GetLastAddress()) {
+ break;
+ }
+
+ // Advance.
+ ++it;
+ ASSERT(it != m_memory_block_manager.end());
+ }
+
+ if (out_blocks_needed != nullptr) {
+ ASSERT(blocks_needed <= KMemoryBlockManagerUpdateAllocator::MaxBlocks);
+ *out_blocks_needed = blocks_needed;
+ }
+
+ R_SUCCEED();
+}
+
+Result KPageTable::SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_addr,
+ KMemoryPermission test_perm, KMemoryState dst_state,
+ KPageTable& src_page_table, bool send) {
+ ASSERT(this->IsLockedByCurrentThread());
+ ASSERT(src_page_table.IsLockedByCurrentThread());
+
+ // Check that we can theoretically map.
+ const VAddr region_start = m_alias_region_start;
+ const size_t region_size = m_alias_region_end - m_alias_region_start;
+ R_UNLESS(size < region_size, ResultOutOfAddressSpace);
+
+ // Get aligned source extents.
+ const VAddr src_start = src_addr;
+ const VAddr src_end = src_addr + size;
+ const VAddr aligned_src_start = Common::AlignDown((src_start), PageSize);
+ const VAddr aligned_src_end = Common::AlignUp((src_start) + size, PageSize);
+ const VAddr mapping_src_start = Common::AlignUp((src_start), PageSize);
+ const VAddr mapping_src_end = Common::AlignDown((src_start) + size, PageSize);
+ const size_t aligned_src_size = aligned_src_end - aligned_src_start;
+ const size_t mapping_src_size =
+ (mapping_src_start < mapping_src_end) ? (mapping_src_end - mapping_src_start) : 0;
+
+ // Select a random address to map at.
+ VAddr dst_addr =
+ this->FindFreeArea(region_start, region_size / PageSize, aligned_src_size / PageSize,
+ PageSize, 0, this->GetNumGuardPages());
+
+ R_UNLESS(dst_addr != 0, ResultOutOfAddressSpace);
+
+ // Check that we can perform the operation we're about to perform.
+ ASSERT(this->CanContain(dst_addr, aligned_src_size, dst_state));
+
+ // Create an update allocator.
+ Result allocator_result;
+ KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result),
+ m_memory_block_slab_manager);
+ R_TRY(allocator_result);
+
+ // We're going to perform an update, so create a helper.
+ KScopedPageTableUpdater updater(this);
+
+ // Reserve space for any partial pages we allocate.
+ const size_t unmapped_size = aligned_src_size - mapping_src_size;
+ KScopedResourceReservation memory_reservation(m_resource_limit,
+ LimitableResource::PhysicalMemory, unmapped_size);
+ R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached);
+
+ // Ensure that we manage page references correctly.
+ PAddr start_partial_page = 0;
+ PAddr end_partial_page = 0;
+ VAddr cur_mapped_addr = dst_addr;
+
+ // If the partial pages are mapped, an extra reference will have been opened. Otherwise, they'll
+ // free on scope exit.
+ SCOPE_EXIT({
+ if (start_partial_page != 0) {
+ m_system.Kernel().MemoryManager().Close(start_partial_page, 1);
+ }
+ if (end_partial_page != 0) {
+ m_system.Kernel().MemoryManager().Close(end_partial_page, 1);
+ }
+ });
+
+ ON_RESULT_FAILURE {
+ if (cur_mapped_addr != dst_addr) {
+ ASSERT(Operate(dst_addr, (cur_mapped_addr - dst_addr) / PageSize,
+ KMemoryPermission::None, OperationType::Unmap)
+ .IsSuccess());
+ }
+ };
+
+ // Allocate the start page as needed.
+ if (aligned_src_start < mapping_src_start) {
+ start_partial_page =
+ m_system.Kernel().MemoryManager().AllocateAndOpenContinuous(1, 1, m_allocate_option);
+ R_UNLESS(start_partial_page != 0, ResultOutOfMemory);
+ }
+
+ // Allocate the end page as needed.
+ if (mapping_src_end < aligned_src_end &&
+ (aligned_src_start < mapping_src_end || aligned_src_start == mapping_src_start)) {
+ end_partial_page =
+ m_system.Kernel().MemoryManager().AllocateAndOpenContinuous(1, 1, m_allocate_option);
+ R_UNLESS(end_partial_page != 0, ResultOutOfMemory);
+ }
+
+ // Get the implementation.
+ auto& src_impl = src_page_table.PageTableImpl();
+
+ // Get the fill value for partial pages.
+ const auto fill_val = m_ipc_fill_value;
+
+ // Begin traversal.
+ Common::PageTable::TraversalContext context;
+ Common::PageTable::TraversalEntry next_entry;
+ bool traverse_valid = src_impl.BeginTraversal(next_entry, context, aligned_src_start);
+ ASSERT(traverse_valid);
+
+ // Prepare tracking variables.
+ PAddr cur_block_addr = next_entry.phys_addr;
+ size_t cur_block_size =
+ next_entry.block_size - ((cur_block_addr) & (next_entry.block_size - 1));
+ size_t tot_block_size = cur_block_size;
+
+ // Map the start page, if we have one.
+ if (start_partial_page != 0) {
+ // Ensure the page holds correct data.
+ const VAddr start_partial_virt =
+ GetHeapVirtualAddress(m_system.Kernel().MemoryLayout(), start_partial_page);
+ if (send) {
+ const size_t partial_offset = src_start - aligned_src_start;
+ size_t copy_size, clear_size;
+ if (src_end < mapping_src_start) {
+ copy_size = size;
+ clear_size = mapping_src_start - src_end;
+ } else {
+ copy_size = mapping_src_start - src_start;
+ clear_size = 0;
+ }
+
+ std::memset(m_system.Memory().GetPointer<void>(start_partial_virt), fill_val,
+ partial_offset);
+ std::memcpy(
+ m_system.Memory().GetPointer<void>(start_partial_virt + partial_offset),
+ m_system.Memory().GetPointer<void>(
+ GetHeapVirtualAddress(m_system.Kernel().MemoryLayout(), cur_block_addr) +
+ partial_offset),
+ copy_size);
+ if (clear_size > 0) {
+ std::memset(m_system.Memory().GetPointer<void>(start_partial_virt + partial_offset +
+ copy_size),
+ fill_val, clear_size);
+ }
+ } else {
+ std::memset(m_system.Memory().GetPointer<void>(start_partial_virt), fill_val, PageSize);
+ }
+
+ // Map the page.
+ R_TRY(Operate(cur_mapped_addr, 1, test_perm, OperationType::Map, start_partial_page));
+
+ // Update tracking extents.
+ cur_mapped_addr += PageSize;
+ cur_block_addr += PageSize;
+ cur_block_size -= PageSize;
+
+ // If the block's size was one page, we may need to continue traversal.
+ if (cur_block_size == 0 && aligned_src_size > PageSize) {
+ traverse_valid = src_impl.ContinueTraversal(next_entry, context);
+ ASSERT(traverse_valid);
+
+ cur_block_addr = next_entry.phys_addr;
+ cur_block_size = next_entry.block_size;
+ tot_block_size += next_entry.block_size;
+ }
+ }
+
+ // Map the remaining pages.
+ while (aligned_src_start + tot_block_size < mapping_src_end) {
+ // Continue the traversal.
+ traverse_valid = src_impl.ContinueTraversal(next_entry, context);
+ ASSERT(traverse_valid);
+
+ // Process the block.
+ if (next_entry.phys_addr != cur_block_addr + cur_block_size) {
+ // Map the block we've been processing so far.
+ R_TRY(Operate(cur_mapped_addr, cur_block_size / PageSize, test_perm, OperationType::Map,
+ cur_block_addr));
+
+ // Update tracking extents.
+ cur_mapped_addr += cur_block_size;
+ cur_block_addr = next_entry.phys_addr;
+ cur_block_size = next_entry.block_size;
+ } else {
+ cur_block_size += next_entry.block_size;
+ }
+ tot_block_size += next_entry.block_size;
+ }
+
+ // Handle the last direct-mapped page.
+ if (const VAddr mapped_block_end = aligned_src_start + tot_block_size - cur_block_size;
+ mapped_block_end < mapping_src_end) {
+ const size_t last_block_size = mapping_src_end - mapped_block_end;
+
+ // Map the last block.
+ R_TRY(Operate(cur_mapped_addr, last_block_size / PageSize, test_perm, OperationType::Map,
+ cur_block_addr));
+
+ // Update tracking extents.
+ cur_mapped_addr += last_block_size;
+ cur_block_addr += last_block_size;
+ if (mapped_block_end + cur_block_size < aligned_src_end &&
+ cur_block_size == last_block_size) {
+ traverse_valid = src_impl.ContinueTraversal(next_entry, context);
+ ASSERT(traverse_valid);
+
+ cur_block_addr = next_entry.phys_addr;
+ }
+ }
+
+ // Map the end page, if we have one.
+ if (end_partial_page != 0) {
+ // Ensure the page holds correct data.
+ const VAddr end_partial_virt =
+ GetHeapVirtualAddress(m_system.Kernel().MemoryLayout(), end_partial_page);
+ if (send) {
+ const size_t copy_size = src_end - mapping_src_end;
+ std::memcpy(m_system.Memory().GetPointer<void>(end_partial_virt),
+ m_system.Memory().GetPointer<void>(GetHeapVirtualAddress(
+ m_system.Kernel().MemoryLayout(), cur_block_addr)),
+ copy_size);
+ std::memset(m_system.Memory().GetPointer<void>(end_partial_virt + copy_size), fill_val,
+ PageSize - copy_size);
+ } else {
+ std::memset(m_system.Memory().GetPointer<void>(end_partial_virt), fill_val, PageSize);
+ }
+
+ // Map the page.
+ R_TRY(Operate(cur_mapped_addr, 1, test_perm, OperationType::Map, end_partial_page));
+ }
+
+ // Update memory blocks to reflect our changes
+ m_memory_block_manager.Update(std::addressof(allocator), dst_addr, aligned_src_size / PageSize,
+ dst_state, test_perm, KMemoryAttribute::None,
+ KMemoryBlockDisableMergeAttribute::Normal,
+ KMemoryBlockDisableMergeAttribute::None);
+
+ // Set the output address.
+ *out_addr = dst_addr + (src_start - aligned_src_start);
+
+ // We succeeded.
+ memory_reservation.Commit();
+ R_SUCCEED();
+}
+
+Result KPageTable::SetupForIpc(VAddr* out_dst_addr, size_t size, VAddr src_addr,
+ KPageTable& src_page_table, KMemoryPermission test_perm,
+ KMemoryState dst_state, bool send) {
+ // For convenience, alias this.
+ KPageTable& dst_page_table = *this;
+
+ // Acquire the table locks.
+ KScopedLightLockPair lk(src_page_table.m_general_lock, dst_page_table.m_general_lock);
+
+ // We're going to perform an update, so create a helper.
+ KScopedPageTableUpdater updater(std::addressof(src_page_table));
+
+ // Perform client setup.
+ size_t num_allocator_blocks;
+ R_TRY(src_page_table.SetupForIpcClient(updater.GetPageList(),
+ std::addressof(num_allocator_blocks), src_addr, size,
+ test_perm, dst_state));
+
+ // Create an update allocator.
+ Result allocator_result;
+ KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result),
+ src_page_table.m_memory_block_slab_manager,
+ num_allocator_blocks);
+ R_TRY(allocator_result);
+
+ // Get the mapped extents.
+ const VAddr src_map_start = Common::AlignUp((src_addr), PageSize);
+ const VAddr src_map_end = Common::AlignDown((src_addr) + size, PageSize);
+ const size_t src_map_size = src_map_end - src_map_start;
+
+ // Ensure that we clean up appropriately if we fail after this.
+ const auto src_perm = static_cast<KMemoryPermission>(
+ (test_perm == KMemoryPermission::UserReadWrite)
+ ? KMemoryPermission::KernelReadWrite | KMemoryPermission::NotMapped
+ : KMemoryPermission::UserRead);
+ ON_RESULT_FAILURE {
+ if (src_map_end > src_map_start) {
+ src_page_table.CleanupForIpcClientOnServerSetupFailure(
+ updater.GetPageList(), src_map_start, src_map_size, src_perm);
+ }
+ };
+
+ // Perform server setup.
+ R_TRY(dst_page_table.SetupForIpcServer(out_dst_addr, size, src_addr, test_perm, dst_state,
+ src_page_table, send));
+
+ // If anything was mapped, ipc-lock the pages.
+ if (src_map_start < src_map_end) {
+ // Get the source permission.
+ src_page_table.m_memory_block_manager.UpdateLock(std::addressof(allocator), src_map_start,
+ (src_map_end - src_map_start) / PageSize,
+ &KMemoryBlock::LockForIpc, src_perm);
+ }
+
+ R_SUCCEED();
+}
+
+Result KPageTable::CleanupForIpcServer(VAddr address, size_t size, KMemoryState dst_state) {
+ // Validate the address.
+ R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory);
+
+ // Lock the table.
+ KScopedLightLock lk(m_general_lock);
+
+ // Validate the memory state.
+ size_t num_allocator_blocks;
+ R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size,
+ KMemoryState::All, dst_state, KMemoryPermission::UserRead,
+ KMemoryPermission::UserRead, KMemoryAttribute::All,
+ KMemoryAttribute::None));
+
+ // Create an update allocator.
+ Result allocator_result;
+ KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result),
+ m_memory_block_slab_manager, num_allocator_blocks);
+ R_TRY(allocator_result);
+
+ // We're going to perform an update, so create a helper.
+ KScopedPageTableUpdater updater(this);
+
+ // Get aligned extents.
+ const VAddr aligned_start = Common::AlignDown((address), PageSize);
+ const VAddr aligned_end = Common::AlignUp((address) + size, PageSize);
+ const size_t aligned_size = aligned_end - aligned_start;
+ const size_t aligned_num_pages = aligned_size / PageSize;
+
+ // Unmap the pages.
+ R_TRY(Operate(aligned_start, aligned_num_pages, KMemoryPermission::None, OperationType::Unmap));
+
+ // Update memory blocks.
+ m_memory_block_manager.Update(std::addressof(allocator), aligned_start, aligned_num_pages,
+ KMemoryState::None, KMemoryPermission::None,
+ KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None,
+ KMemoryBlockDisableMergeAttribute::Normal);
+
+ // Release from the resource limit as relevant.
+ const VAddr mapping_start = Common::AlignUp((address), PageSize);
+ const VAddr mapping_end = Common::AlignDown((address) + size, PageSize);
+ const size_t mapping_size = (mapping_start < mapping_end) ? mapping_end - mapping_start : 0;
+ m_resource_limit->Release(LimitableResource::PhysicalMemory, aligned_size - mapping_size);
+
+ R_SUCCEED();
+}
+
+Result KPageTable::CleanupForIpcClient(VAddr address, size_t size, KMemoryState dst_state) {
+ // Validate the address.
+ R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory);
+
+ // Get aligned source extents.
+ const VAddr mapping_start = Common::AlignUp((address), PageSize);
+ const VAddr mapping_end = Common::AlignDown((address) + size, PageSize);
+ const VAddr mapping_last = mapping_end - 1;
+ const size_t mapping_size = (mapping_start < mapping_end) ? (mapping_end - mapping_start) : 0;
+
+ // If nothing was mapped, we're actually done immediately.
+ R_SUCCEED_IF(mapping_size == 0);
+
+ // Get the test state and attribute mask.
+ KMemoryState test_state;
+ KMemoryAttribute test_attr_mask;
+ switch (dst_state) {
+ case KMemoryState::Ipc:
+ test_state = KMemoryState::FlagCanUseIpc;
+ test_attr_mask =
+ KMemoryAttribute::Uncached | KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked;
+ break;
+ case KMemoryState::NonSecureIpc:
+ test_state = KMemoryState::FlagCanUseNonSecureIpc;
+ test_attr_mask = KMemoryAttribute::Uncached | KMemoryAttribute::Locked;
+ break;
+ case KMemoryState::NonDeviceIpc:
+ test_state = KMemoryState::FlagCanUseNonDeviceIpc;
+ test_attr_mask = KMemoryAttribute::Uncached | KMemoryAttribute::Locked;
+ break;
+ default:
+ R_THROW(ResultInvalidCombination);
+ }
+
+ // Lock the table.
+ // NOTE: Nintendo does this *after* creating the updater below, but this does not follow
+ // convention elsewhere in KPageTable.
+ KScopedLightLock lk(m_general_lock);
+
+ // We're going to perform an update, so create a helper.
+ KScopedPageTableUpdater updater(this);
+
+ // Ensure that on failure, we roll back appropriately.
+ size_t mapped_size = 0;
+ ON_RESULT_FAILURE {
+ if (mapped_size > 0) {
+ // Determine where the mapping ends.
+ const auto mapped_end = (mapping_start) + mapped_size;
+ const auto mapped_last = mapped_end - 1;
+
+ // Get current and next iterators.
+ KMemoryBlockManager::const_iterator start_it =
+ m_memory_block_manager.FindIterator(mapping_start);
+ KMemoryBlockManager::const_iterator next_it = start_it;
+ ++next_it;
+
+ // Get the current block info.
+ KMemoryInfo cur_info = start_it->GetMemoryInfo();
+
+ // Create tracking variables.
+ VAddr cur_address = cur_info.GetAddress();
+ size_t cur_size = cur_info.GetSize();
+ bool cur_perm_eq = cur_info.GetPermission() == cur_info.GetOriginalPermission();
+ bool cur_needs_set_perm = !cur_perm_eq && cur_info.GetIpcLockCount() == 1;
+ bool first =
+ cur_info.GetIpcDisableMergeCount() == 1 &&
+ (cur_info.GetDisableMergeAttribute() & KMemoryBlockDisableMergeAttribute::Locked) ==
+ KMemoryBlockDisableMergeAttribute::None;
+
+ while (((cur_address) + cur_size - 1) < mapped_last) {
+ // Check that we have a next block.
+ ASSERT(next_it != m_memory_block_manager.end());
+
+ // Get the next info.
+ const KMemoryInfo next_info = next_it->GetMemoryInfo();
+
+ // Check if we can consolidate the next block's permission set with the current one.
+
+ const bool next_perm_eq =
+ next_info.GetPermission() == next_info.GetOriginalPermission();
+ const bool next_needs_set_perm = !next_perm_eq && next_info.GetIpcLockCount() == 1;
+ if (cur_perm_eq == next_perm_eq && cur_needs_set_perm == next_needs_set_perm &&
+ cur_info.GetOriginalPermission() == next_info.GetOriginalPermission()) {
+ // We can consolidate the reprotection for the current and next block into a
+ // single call.
+ cur_size += next_info.GetSize();
+ } else {
+ // We have to operate on the current block.
+ if ((cur_needs_set_perm || first) && !cur_perm_eq) {
+ ASSERT(Operate(cur_address, cur_size / PageSize, cur_info.GetPermission(),
+ OperationType::ChangePermissions)
+ .IsSuccess());
+ }
+
+ // Advance.
+ cur_address = next_info.GetAddress();
+ cur_size = next_info.GetSize();
+ first = false;
+ }
+
+ // Advance.
+ cur_info = next_info;
+ cur_perm_eq = next_perm_eq;
+ cur_needs_set_perm = next_needs_set_perm;
+ ++next_it;
+ }
+
+ // Process the last block.
+ if ((first || cur_needs_set_perm) && !cur_perm_eq) {
+ ASSERT(Operate(cur_address, cur_size / PageSize, cur_info.GetPermission(),
+ OperationType::ChangePermissions)
+ .IsSuccess());
+ }
+ }
+ };
+
+ // Iterate, reprotecting as needed.
+ {
+ // Get current and next iterators.
+ KMemoryBlockManager::const_iterator start_it =
+ m_memory_block_manager.FindIterator(mapping_start);
+ KMemoryBlockManager::const_iterator next_it = start_it;
+ ++next_it;
+
+ // Validate the current block.
+ KMemoryInfo cur_info = start_it->GetMemoryInfo();
+ ASSERT(this->CheckMemoryState(cur_info, test_state, test_state, KMemoryPermission::None,
+ KMemoryPermission::None,
+ test_attr_mask | KMemoryAttribute::IpcLocked,
+ KMemoryAttribute::IpcLocked)
+ .IsSuccess());
+
+ // Create tracking variables.
+ VAddr cur_address = cur_info.GetAddress();
+ size_t cur_size = cur_info.GetSize();
+ bool cur_perm_eq = cur_info.GetPermission() == cur_info.GetOriginalPermission();
+ bool cur_needs_set_perm = !cur_perm_eq && cur_info.GetIpcLockCount() == 1;
+ bool first =
+ cur_info.GetIpcDisableMergeCount() == 1 &&
+ (cur_info.GetDisableMergeAttribute() & KMemoryBlockDisableMergeAttribute::Locked) ==
+ KMemoryBlockDisableMergeAttribute::None;
+
+ while ((cur_address + cur_size - 1) < mapping_last) {
+ // Check that we have a next block.
+ ASSERT(next_it != m_memory_block_manager.end());
+
+ // Get the next info.
+ const KMemoryInfo next_info = next_it->GetMemoryInfo();
+
+ // Validate the next block.
+ ASSERT(this->CheckMemoryState(next_info, test_state, test_state,
+ KMemoryPermission::None, KMemoryPermission::None,
+ test_attr_mask | KMemoryAttribute::IpcLocked,
+ KMemoryAttribute::IpcLocked)
+ .IsSuccess());
+
+ // Check if we can consolidate the next block's permission set with the current one.
+ const bool next_perm_eq =
+ next_info.GetPermission() == next_info.GetOriginalPermission();
+ const bool next_needs_set_perm = !next_perm_eq && next_info.GetIpcLockCount() == 1;
+ if (cur_perm_eq == next_perm_eq && cur_needs_set_perm == next_needs_set_perm &&
+ cur_info.GetOriginalPermission() == next_info.GetOriginalPermission()) {
+ // We can consolidate the reprotection for the current and next block into a single
+ // call.
+ cur_size += next_info.GetSize();
+ } else {
+ // We have to operate on the current block.
+ if ((cur_needs_set_perm || first) && !cur_perm_eq) {
+ R_TRY(Operate(cur_address, cur_size / PageSize,
+ cur_needs_set_perm ? cur_info.GetOriginalPermission()
+ : cur_info.GetPermission(),
+ OperationType::ChangePermissions));
+ }
+
+ // Mark that we mapped the block.
+ mapped_size += cur_size;
+
+ // Advance.
+ cur_address = next_info.GetAddress();
+ cur_size = next_info.GetSize();
+ first = false;
+ }
+
+ // Advance.
+ cur_info = next_info;
+ cur_perm_eq = next_perm_eq;
+ cur_needs_set_perm = next_needs_set_perm;
+ ++next_it;
+ }
+
+ // Process the last block.
+ const auto lock_count =
+ cur_info.GetIpcLockCount() +
+ (next_it != m_memory_block_manager.end()
+ ? (next_it->GetIpcDisableMergeCount() - next_it->GetIpcLockCount())
+ : 0);
+ if ((first || cur_needs_set_perm || (lock_count == 1)) && !cur_perm_eq) {
+ R_TRY(Operate(cur_address, cur_size / PageSize,
+ cur_needs_set_perm ? cur_info.GetOriginalPermission()
+ : cur_info.GetPermission(),
+ OperationType::ChangePermissions));
+ }
+ }
+
+ // Create an update allocator.
+ // NOTE: Guaranteed zero blocks needed here.
+ Result allocator_result;
+ KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result),
+ m_memory_block_slab_manager, 0);
+ R_TRY(allocator_result);
+
+ // Unlock the pages.
+ m_memory_block_manager.UpdateLock(std::addressof(allocator), mapping_start,
+ mapping_size / PageSize, &KMemoryBlock::UnlockForIpc,
+ KMemoryPermission::None);
+
+ R_SUCCEED();
+}
+
+void KPageTable::CleanupForIpcClientOnServerSetupFailure([[maybe_unused]] PageLinkedList* page_list,
+ VAddr address, size_t size,
+ KMemoryPermission prot_perm) {
+ ASSERT(this->IsLockedByCurrentThread());
+ ASSERT(Common::IsAligned(address, PageSize));
+ ASSERT(Common::IsAligned(size, PageSize));
+
+ // Get the mapped extents.
+ const VAddr src_map_start = address;
+ const VAddr src_map_end = address + size;
+ const VAddr src_map_last = src_map_end - 1;
+
+ // This function is only invoked when there's something to do.
+ ASSERT(src_map_end > src_map_start);
+
+ // Iterate over blocks, fixing permissions.
+ KMemoryBlockManager::const_iterator it = m_memory_block_manager.FindIterator(address);
+ while (true) {
+ const KMemoryInfo info = it->GetMemoryInfo();
+
+ const auto cur_start =
+ info.GetAddress() >= src_map_start ? info.GetAddress() : src_map_start;
+ const auto cur_end =
+ src_map_last <= info.GetLastAddress() ? src_map_end : info.GetEndAddress();
+
+ // If we can, fix the protections on the block.
+ if ((info.GetIpcLockCount() == 0 &&
+ (info.GetPermission() & KMemoryPermission::IpcLockChangeMask) != prot_perm) ||
+ (info.GetIpcLockCount() != 0 &&
+ (info.GetOriginalPermission() & KMemoryPermission::IpcLockChangeMask) != prot_perm)) {
+ // Check if we actually need to fix the protections on the block.
+ if (cur_end == src_map_end || info.GetAddress() <= src_map_start ||
+ (info.GetPermission() & KMemoryPermission::IpcLockChangeMask) != prot_perm) {
+ ASSERT(Operate(cur_start, (cur_end - cur_start) / PageSize, info.GetPermission(),
+ OperationType::ChangePermissions)
+ .IsSuccess());
+ }
+ }
+
+ // If we're past the end of the region, we're done.
+ if (src_map_last <= info.GetLastAddress()) {
+ break;
+ }
+
+ // Advance.
+ ++it;
+ ASSERT(it != m_memory_block_manager.end());
+ }
+}
+
void KPageTable::HACK_OpenPages(PAddr phys_addr, size_t num_pages) {
m_system.Kernel().MemoryManager().OpenFirst(phys_addr, num_pages);
}
@@ -858,7 +1635,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) {
R_TRY(allocator_result);
// We're going to perform an update, so create a helper.
- // KScopedPageTableUpdater updater(this);
+ KScopedPageTableUpdater updater(this);
// Prepare to iterate over the memory.
auto pg_it = pg.Nodes().begin();
@@ -1074,7 +1851,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) {
R_TRY(allocator_result);
// We're going to perform an update, so create a helper.
- // KScopedPageTableUpdater updater(this);
+ KScopedPageTableUpdater updater(this);
// Separate the mapping.
R_TRY(Operate(map_start_address, (map_last_address + 1 - map_start_address) / PageSize,
@@ -1935,6 +2712,24 @@ Result KPageTable::UnlockForDeviceAddressSpace(VAddr address, size_t size) {
R_SUCCEED();
}
+Result KPageTable::LockForIpcUserBuffer(PAddr* out, VAddr address, size_t size) {
+ R_RETURN(this->LockMemoryAndOpen(
+ nullptr, out, address, size, KMemoryState::FlagCanIpcUserBuffer,
+ KMemoryState::FlagCanIpcUserBuffer, KMemoryPermission::All,
+ KMemoryPermission::UserReadWrite, KMemoryAttribute::All, KMemoryAttribute::None,
+ static_cast<KMemoryPermission>(KMemoryPermission::NotMapped |
+ KMemoryPermission::KernelReadWrite),
+ KMemoryAttribute::Locked));
+}
+
+Result KPageTable::UnlockForIpcUserBuffer(VAddr address, size_t size) {
+ R_RETURN(this->UnlockMemory(address, size, KMemoryState::FlagCanIpcUserBuffer,
+ KMemoryState::FlagCanIpcUserBuffer, KMemoryPermission::None,
+ KMemoryPermission::None, KMemoryAttribute::All,
+ KMemoryAttribute::Locked, KMemoryPermission::UserReadWrite,
+ KMemoryAttribute::Locked, nullptr));
+}
+
Result KPageTable::LockForCodeMemory(KPageGroup* out, VAddr addr, size_t size) {
R_RETURN(this->LockMemoryAndOpen(
out, nullptr, addr, size, KMemoryState::FlagCanCodeMemory, KMemoryState::FlagCanCodeMemory,
@@ -2038,6 +2833,17 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm,
R_SUCCEED();
}
+void KPageTable::FinalizeUpdate(PageLinkedList* page_list) {
+ while (page_list->Peek()) {
+ [[maybe_unused]] auto page = page_list->Pop();
+
+ // TODO(bunnei): Free pages once they are allocated in guest memory
+ // ASSERT(this->GetPageTableManager().IsInPageTableHeap(page));
+ // ASSERT(this->GetPageTableManager().GetRefCount(page) == 0);
+ // this->GetPageTableManager().Free(page);
+ }
+}
+
VAddr KPageTable::GetRegionAddress(KMemoryState state) const {
switch (state) {
case KMemoryState::Free:
diff --git a/src/core/hle/kernel/k_page_table.h b/src/core/hle/kernel/k_page_table.h
index 753e07c94..950850291 100644
--- a/src/core/hle/kernel/k_page_table.h
+++ b/src/core/hle/kernel/k_page_table.h
@@ -16,6 +16,7 @@
#include "core/hle/kernel/k_memory_layout.h"
#include "core/hle/kernel/k_memory_manager.h"
#include "core/hle/result.h"
+#include "core/memory.h"
namespace Core {
class System;
@@ -83,6 +84,14 @@ public:
Result UnlockForDeviceAddressSpace(VAddr addr, size_t size);
+ Result LockForIpcUserBuffer(PAddr* out, VAddr address, size_t size);
+ Result UnlockForIpcUserBuffer(VAddr address, size_t size);
+
+ Result SetupForIpc(VAddr* out_dst_addr, size_t size, VAddr src_addr, KPageTable& src_page_table,
+ KMemoryPermission test_perm, KMemoryState dst_state, bool send);
+ Result CleanupForIpcServer(VAddr address, size_t size, KMemoryState dst_state);
+ Result CleanupForIpcClient(VAddr address, size_t size, KMemoryState dst_state);
+
Result LockForCodeMemory(KPageGroup* out, VAddr addr, size_t size);
Result UnlockForCodeMemory(VAddr addr, size_t size, const KPageGroup& pg);
Result MakeAndOpenPageGroup(KPageGroup* out, VAddr address, size_t num_pages,
@@ -100,6 +109,45 @@ public:
bool CanContain(VAddr addr, size_t size, KMemoryState state) const;
+protected:
+ struct PageLinkedList {
+ private:
+ struct Node {
+ Node* m_next;
+ std::array<u8, PageSize - sizeof(Node*)> m_buffer;
+ };
+
+ public:
+ constexpr PageLinkedList() = default;
+
+ void Push(Node* n) {
+ ASSERT(Common::IsAligned(reinterpret_cast<uintptr_t>(n), PageSize));
+ n->m_next = m_root;
+ m_root = n;
+ }
+
+ void Push(Core::Memory::Memory& memory, VAddr addr) {
+ this->Push(memory.GetPointer<Node>(addr));
+ }
+
+ Node* Peek() const {
+ return m_root;
+ }
+
+ Node* Pop() {
+ Node* const r = m_root;
+
+ m_root = r->m_next;
+ r->m_next = nullptr;
+
+ return r;
+ }
+
+ private:
+ Node* m_root{};
+ };
+ static_assert(std::is_trivially_destructible<PageLinkedList>::value);
+
private:
enum class OperationType : u32 {
Map = 0,
@@ -128,6 +176,7 @@ private:
OperationType operation);
Result Operate(VAddr addr, size_t num_pages, KMemoryPermission perm, OperationType operation,
PAddr map_addr = 0);
+ void FinalizeUpdate(PageLinkedList* page_list);
VAddr GetRegionAddress(KMemoryState state) const;
size_t GetRegionSize(KMemoryState state) const;
@@ -204,6 +253,14 @@ private:
return *out != 0;
}
+ Result SetupForIpcClient(PageLinkedList* page_list, size_t* out_blocks_needed, VAddr address,
+ size_t size, KMemoryPermission test_perm, KMemoryState dst_state);
+ Result SetupForIpcServer(VAddr* out_addr, size_t size, VAddr src_addr,
+ KMemoryPermission test_perm, KMemoryState dst_state,
+ KPageTable& src_page_table, bool send);
+ void CleanupForIpcClientOnServerSetupFailure(PageLinkedList* page_list, VAddr address,
+ size_t size, KMemoryPermission prot_perm);
+
// HACK: These will be removed once we automatically manage page reference counts.
void HACK_OpenPages(PAddr phys_addr, size_t num_pages);
void HACK_ClosePages(VAddr virt_addr, size_t num_pages);
@@ -325,6 +382,31 @@ public:
addr + size - 1 <= m_address_space_end - 1;
}
+public:
+ static VAddr GetLinearMappedVirtualAddress(const KMemoryLayout& layout, PAddr addr) {
+ return layout.GetLinearVirtualAddress(addr);
+ }
+
+ static PAddr GetLinearMappedPhysicalAddress(const KMemoryLayout& layout, VAddr addr) {
+ return layout.GetLinearPhysicalAddress(addr);
+ }
+
+ static VAddr GetHeapVirtualAddress(const KMemoryLayout& layout, PAddr addr) {
+ return GetLinearMappedVirtualAddress(layout, addr);
+ }
+
+ static PAddr GetHeapPhysicalAddress(const KMemoryLayout& layout, VAddr addr) {
+ return GetLinearMappedPhysicalAddress(layout, addr);
+ }
+
+ static VAddr GetPageTableVirtualAddress(const KMemoryLayout& layout, PAddr addr) {
+ return GetLinearMappedVirtualAddress(layout, addr);
+ }
+
+ static PAddr GetPageTablePhysicalAddress(const KMemoryLayout& layout, VAddr addr) {
+ return GetLinearMappedPhysicalAddress(layout, addr);
+ }
+
private:
constexpr bool IsKernel() const {
return m_is_kernel;
@@ -339,6 +421,24 @@ private:
(addr + num_pages * PageSize - 1 <= m_address_space_end - 1);
}
+private:
+ class KScopedPageTableUpdater {
+ private:
+ KPageTable* m_pt{};
+ PageLinkedList m_ll;
+
+ public:
+ explicit KScopedPageTableUpdater(KPageTable* pt) : m_pt(pt) {}
+ explicit KScopedPageTableUpdater(KPageTable& pt) : KScopedPageTableUpdater(&pt) {}
+ ~KScopedPageTableUpdater() {
+ m_pt->FinalizeUpdate(this->GetPageList());
+ }
+
+ PageLinkedList* GetPageList() {
+ return &m_ll;
+ }
+ };
+
private:
VAddr m_address_space_start{};
VAddr m_address_space_end{};
diff --git a/src/core/hle/kernel/svc_results.h b/src/core/hle/kernel/svc_results.h
index f27cade33..b7ca53085 100644
--- a/src/core/hle/kernel/svc_results.h
+++ b/src/core/hle/kernel/svc_results.h
@@ -37,6 +37,7 @@ constexpr Result ResultInvalidState{ErrorModule::Kernel, 125};
constexpr Result ResultReservedUsed{ErrorModule::Kernel, 126};
constexpr Result ResultPortClosed{ErrorModule::Kernel, 131};
constexpr Result ResultLimitReached{ErrorModule::Kernel, 132};
+constexpr Result ResultOutOfAddressSpace{ErrorModule::Kernel, 259};
constexpr Result ResultInvalidId{ErrorModule::Kernel, 519};
} // namespace Kernel