diff options
Diffstat (limited to 'src/core/hle/service/nvdrv/devices')
10 files changed, 200 insertions, 52 deletions
diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp index 51c40f620..122c1d5e1 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp @@ -3,9 +3,11 @@ // SPDX-License-Identifier: GPL-3.0-or-later Licensed under GPLv3 // or any later version Refer to the license.txt file included. +#include <bit> #include <cstdlib> #include <cstring> +#include <fmt/format.h> #include "common/assert.h" #include "common/logging/log.h" #include "common/scope_exit.h" @@ -22,8 +24,19 @@ namespace Service::Nvidia::Devices { nvhost_ctrl::nvhost_ctrl(Core::System& system_, EventInterface& events_interface_, NvCore::Container& core_) : nvdevice{system_}, events_interface{events_interface_}, core{core_}, - syncpoint_manager{core_.GetSyncpointManager()} {} -nvhost_ctrl::~nvhost_ctrl() = default; + syncpoint_manager{core_.GetSyncpointManager()} { + events_interface.RegisterForSignal(this); +} + +nvhost_ctrl::~nvhost_ctrl() { + events_interface.UnregisterForSignal(this); + for (auto& event : events) { + if (!event.registered) { + continue; + } + events_interface.FreeEvent(event.kevent); + } +} NvResult nvhost_ctrl::Ioctl1(DeviceFD fd, Ioctl command, const std::vector<u8>& input, std::vector<u8>& output) { @@ -87,7 +100,7 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector SCOPE_EXIT({ std::memcpy(output.data(), ¶ms, sizeof(params)); if (must_unmark_fail) { - events_interface.fails[event_id] = 0; + events[event_id].fails = 0; } }); @@ -116,12 +129,12 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector auto& gpu = system.GPU(); const u32 target_value = params.fence.value; - auto lock = events_interface.Lock(); + auto lock = NvEventsLock(); u32 slot = [&]() { if (is_allocation) { params.value.raw = 0; - return events_interface.FindFreeEvent(fence_id); + return FindFreeNvEvent(fence_id); } else { return params.value.raw; } @@ -130,7 +143,7 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector must_unmark_fail = true; const auto check_failing = [&]() { - if (events_interface.fails[slot] > 2) { + if (events[slot].fails > 2) { { auto lk = system.StallProcesses(); gpu.WaitFence(fence_id, target_value); @@ -142,6 +155,10 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector return false; }; + if (slot >= MaxNvEvents) { + return NvResult::BadParameter; + } + if (params.timeout == 0) { if (check_failing()) { return NvResult::Success; @@ -149,17 +166,13 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector return NvResult::Timeout; } - if (slot >= MaxNvEvents) { - return NvResult::BadParameter; - } - - auto* event = events_interface.events[slot]; + auto& event = events[slot]; - if (!event) { + if (!event.registered) { return NvResult::BadParameter; } - if (events_interface.IsBeingUsed(slot)) { + if (event.IsBeingUsed()) { return NvResult::BadParameter; } @@ -169,9 +182,9 @@ NvResult nvhost_ctrl::IocCtrlEventWait(const std::vector<u8>& input, std::vector params.value.raw = 0; - events_interface.status[slot].store(EventState::Waiting, std::memory_order_release); - events_interface.assigned_syncpt[slot] = fence_id; - events_interface.assigned_value[slot] = target_value; + event.status.store(EventState::Waiting, std::memory_order_release); + event.assigned_syncpt = fence_id; + event.assigned_value = target_value; if (is_allocation) { params.value.syncpoint_id_for_allocation.Assign(static_cast<u16>(fence_id)); params.value.event_allocated.Assign(1); @@ -189,15 +202,17 @@ NvResult nvhost_ctrl::FreeEvent(u32 slot) { return NvResult::BadParameter; } - if (!events_interface.registered[slot]) { + auto& event = events[slot]; + + if (!event.registered) { return NvResult::Success; } - if (events_interface.IsBeingUsed(slot)) { + if (event.IsBeingUsed()) { return NvResult::Busy; } - events_interface.Free(slot); + FreeNvEvent(slot); return NvResult::Success; } @@ -210,15 +225,15 @@ NvResult nvhost_ctrl::IocCtrlEventRegister(const std::vector<u8>& input, std::ve return NvResult::BadParameter; } - auto lock = events_interface.Lock(); + auto lock = NvEventsLock(); - if (events_interface.registered[event_id]) { + if (events[event_id].registered) { const auto result = FreeEvent(event_id); if (result != NvResult::Success) { return result; } } - events_interface.Create(event_id); + CreateNvEvent(event_id); return NvResult::Success; } @@ -229,7 +244,7 @@ NvResult nvhost_ctrl::IocCtrlEventUnregister(const std::vector<u8>& input, const u32 event_id = params.user_event_id & 0x00FF; LOG_DEBUG(Service_NVDRV, " called, user_event_id: {:X}", event_id); - auto lock = events_interface.Lock(); + auto lock = NvEventsLock(); return FreeEvent(event_id); } @@ -244,44 +259,121 @@ NvResult nvhost_ctrl::IocCtrlClearEventWait(const std::vector<u8>& input, std::v return NvResult::BadParameter; } - auto lock = events_interface.Lock(); + auto lock = NvEventsLock(); - if (events_interface.status[event_id].exchange( - EventState::Cancelling, std::memory_order_acq_rel) == EventState::Waiting) { - system.GPU().CancelSyncptInterrupt(events_interface.assigned_syncpt[event_id], - events_interface.assigned_value[event_id]); - syncpoint_manager.RefreshSyncpoint(events_interface.assigned_syncpt[event_id]); + auto& event = events[event_id]; + if (event.status.exchange(EventState::Cancelling, std::memory_order_acq_rel) == + EventState::Waiting) { + system.GPU().CancelSyncptInterrupt(event.assigned_syncpt, event.assigned_value); + syncpoint_manager.RefreshSyncpoint(event.assigned_syncpt); } - events_interface.fails[event_id]++; - events_interface.status[event_id].store(EventState::Cancelled, std::memory_order_release); - events_interface.events[event_id]->GetWritableEvent().Clear(); + event.fails++; + event.status.store(EventState::Cancelled, std::memory_order_release); + event.kevent->GetWritableEvent().Clear(); return NvResult::Success; } Kernel::KEvent* nvhost_ctrl::QueryEvent(u32 event_id) { - const auto event = SyncpointEventValue{.raw = event_id}; + const auto desired_event = SyncpointEventValue{.raw = event_id}; - const bool allocated = event.event_allocated.Value() != 0; - const u32 slot{allocated ? event.partial_slot.Value() : static_cast<u32>(event.slot)}; + const bool allocated = desired_event.event_allocated.Value() != 0; + const u32 slot{allocated ? desired_event.partial_slot.Value() + : static_cast<u32>(desired_event.slot)}; if (slot >= MaxNvEvents) { ASSERT(false); return nullptr; } - const u32 syncpoint_id{allocated ? event.syncpoint_id_for_allocation.Value() - : event.syncpoint_id.Value()}; + const u32 syncpoint_id{allocated ? desired_event.syncpoint_id_for_allocation.Value() + : desired_event.syncpoint_id.Value()}; - auto lock = events_interface.Lock(); + auto lock = NvEventsLock(); - if (events_interface.registered[slot] && - events_interface.assigned_syncpt[slot] == syncpoint_id) { - ASSERT(events_interface.events[slot]); - return events_interface.events[slot]; + auto& event = events[slot]; + if (event.registered && event.assigned_syncpt == syncpoint_id) { + ASSERT(event.kevent); + return event.kevent; } // Is this possible in hardware? ASSERT_MSG(false, "Slot:{}, SyncpointID:{}, requested", slot, syncpoint_id); return nullptr; } +std::unique_lock<std::mutex> nvhost_ctrl::NvEventsLock() { + return std::unique_lock<std::mutex>(events_mutex); +} + +void nvhost_ctrl::CreateNvEvent(u32 event_id) { + auto& event = events[event_id]; + ASSERT(!event.kevent); + ASSERT(!event.registered); + ASSERT(!event.IsBeingUsed()); + event.kevent = events_interface.CreateEvent(fmt::format("NVCTRL::NvEvent_{}", event_id)); + event.status = EventState::Available; + event.registered = true; + const u64 mask = 1ULL << event_id; + event.fails = 0; + events_mask |= mask; + event.assigned_syncpt = 0; +} + +void nvhost_ctrl::FreeNvEvent(u32 event_id) { + auto& event = events[event_id]; + ASSERT(event.kevent); + ASSERT(event.registered); + ASSERT(!event.IsBeingUsed()); + events_interface.FreeEvent(event.kevent); + event.kevent = nullptr; + event.status = EventState::Available; + event.registered = false; + const u64 mask = ~(1ULL << event_id); + events_mask &= mask; +} + +u32 nvhost_ctrl::FindFreeNvEvent(u32 syncpoint_id) { + u32 slot{MaxNvEvents}; + u32 free_slot{MaxNvEvents}; + for (u32 i = 0; i < MaxNvEvents; i++) { + auto& event = events[i]; + if (event.registered) { + if (!event.IsBeingUsed()) { + slot = i; + if (event.assigned_syncpt == syncpoint_id) { + return slot; + } + } + } else if (free_slot == MaxNvEvents) { + free_slot = i; + } + } + if (free_slot < MaxNvEvents) { + CreateNvEvent(free_slot); + return free_slot; + } + + if (slot < MaxNvEvents) { + return slot; + } + + LOG_CRITICAL(Service_NVDRV, "Failed to allocate an event"); + return 0; +} + +void nvhost_ctrl::SignalNvEvent(u32 syncpoint_id, u32 value) { + const u32 max = MaxNvEvents - std::countl_zero(events_mask); + const u32 min = std::countr_zero(events_mask); + for (u32 i = min; i < max; i++) { + auto& event = events[i]; + if (event.assigned_syncpt != syncpoint_id || event.assigned_value != value) { + continue; + } + if (event.status.exchange(EventState::Signalling, std::memory_order_acq_rel) == + EventState::Waiting) { + event.kevent->GetWritableEvent().Signal(); + } + event.status.store(EventState::Signalled, std::memory_order_release); + } +} + } // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h index 9fd46ea5f..f2fc5d047 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.h @@ -53,7 +53,49 @@ public: }; static_assert(sizeof(SyncpointEventValue) == sizeof(u32)); + void SignalNvEvent(u32 syncpoint_id, u32 value); + private: + struct InternalEvent { + // Mask representing registered events + + // Each kernel event associated to an NV event + Kernel::KEvent* kevent{}; + // The status of the current NVEvent + std::atomic<EventState> status{}; + + // Tells the NVEvent that it has failed. + u32 fails{}; + // When an NVEvent is waiting on GPU interrupt, this is the sync_point + // associated with it. + u32 assigned_syncpt{}; + // This is the value of the GPU interrupt for which the NVEvent is waiting + // for. + u32 assigned_value{}; + + // Tells if an NVEvent is registered or not + bool registered{}; + + bool IsBeingUsed() { + const auto current_status = status.load(std::memory_order_acquire); + return current_status == EventState::Waiting || + current_status == EventState::Cancelling || + current_status == EventState::Signalling; + } + }; + + std::unique_lock<std::mutex> NvEventsLock(); + + void CreateNvEvent(u32 event_id); + + void FreeNvEvent(u32 event_id); + + u32 FindFreeNvEvent(u32 syncpoint_id); + + std::array<InternalEvent, MaxNvEvents> events{}; + std::mutex events_mutex; + u64 events_mask{}; + struct IocSyncptReadParams { u32_le id{}; u32_le value{}; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index e353408eb..ced57dfe6 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -13,10 +13,13 @@ namespace Service::Nvidia::Devices { nvhost_ctrl_gpu::nvhost_ctrl_gpu(Core::System& system_, EventInterface& events_interface_) : nvdevice{system_}, events_interface{events_interface_} { - error_notifier_event = events_interface.CreateNonCtrlEvent("CtrlGpuErrorNotifier"); - unknown_event = events_interface.CreateNonCtrlEvent("CtrlGpuUknownEvent"); + error_notifier_event = events_interface.CreateEvent("CtrlGpuErrorNotifier"); + unknown_event = events_interface.CreateEvent("CtrlGpuUknownEvent"); +} +nvhost_ctrl_gpu::~nvhost_ctrl_gpu() { + events_interface.FreeEvent(error_notifier_event); + events_interface.FreeEvent(unknown_event); } -nvhost_ctrl_gpu::~nvhost_ctrl_gpu() = default; NvResult nvhost_ctrl_gpu::Ioctl1(DeviceFD fd, Ioctl command, const std::vector<u8>& input, std::vector<u8>& output) { diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index e7921ade2..cb54ee5a4 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -30,13 +30,17 @@ nvhost_gpu::nvhost_gpu(Core::System& system_, EventInterface& events_interface_, channel_fence.id = syncpoint_manager.AllocateSyncpoint(); channel_fence.value = system_.GPU().GetSyncpointValue(channel_fence.id); sm_exception_breakpoint_int_report_event = - events_interface.CreateNonCtrlEvent("GpuChannelSMExceptionBreakpointInt"); + events_interface.CreateEvent("GpuChannelSMExceptionBreakpointInt"); sm_exception_breakpoint_pause_report_event = - events_interface.CreateNonCtrlEvent("GpuChannelSMExceptionBreakpointPause"); - error_notifier_event = events_interface.CreateNonCtrlEvent("GpuChannelErrorNotifier"); + events_interface.CreateEvent("GpuChannelSMExceptionBreakpointPause"); + error_notifier_event = events_interface.CreateEvent("GpuChannelErrorNotifier"); } -nvhost_gpu::~nvhost_gpu() = default; +nvhost_gpu::~nvhost_gpu() { + events_interface.FreeEvent(sm_exception_breakpoint_int_report_event); + events_interface.FreeEvent(sm_exception_breakpoint_pause_report_event); + events_interface.FreeEvent(error_notifier_event); +} NvResult nvhost_gpu::Ioctl1(DeviceFD fd, Ioctl command, const std::vector<u8>& input, std::vector<u8>& output) { diff --git a/src/core/hle/service/nvdrv/devices/nvhost_nvdec.cpp b/src/core/hle/service/nvdrv/devices/nvhost_nvdec.cpp index aa1a00832..00947ea19 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_nvdec.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_nvdec.cpp @@ -10,6 +10,8 @@ namespace Service::Nvidia::Devices { +u32 nvhost_nvdec::next_id{}; + nvhost_nvdec::nvhost_nvdec(Core::System& system_, NvCore::Container& core) : nvhost_nvdec_common{system_, core} {} nvhost_nvdec::~nvhost_nvdec() = default; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_nvdec.h b/src/core/hle/service/nvdrv/devices/nvhost_nvdec.h index fef4b3216..3261ce1d4 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_nvdec.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_nvdec.h @@ -24,7 +24,7 @@ public: void OnClose(DeviceFD fd) override; private: - u32 next_id{}; + static u32 next_id; }; } // namespace Service::Nvidia::Devices diff --git a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp index e76c9e5ed..77e6a1cd6 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.cpp @@ -45,6 +45,8 @@ std::size_t WriteVectors(std::vector<u8>& dst, const std::vector<T>& src, std::s } } // Anonymous namespace +std::unordered_map<DeviceFD, u32> nvhost_nvdec_common::fd_to_id{}; + nvhost_nvdec_common::nvhost_nvdec_common(Core::System& system_, NvCore::Container& core_) : nvdevice{system_}, core{core_}, syncpoint_manager{core.GetSyncpointManager()}, nvmap{core.GetNvMapFile()} {} diff --git a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.h b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.h index 74231d5c5..53029af6a 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_nvdec_common.h @@ -115,7 +115,7 @@ protected: Kernel::KEvent* QueryEvent(u32 event_id) override; - std::unordered_map<DeviceFD, u32> fd_to_id{}; + static std::unordered_map<DeviceFD, u32> fd_to_id; s32_le nvmap_fd{}; u32_le submit_timeout{}; NvCore::Container& core; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_vic.cpp b/src/core/hle/service/nvdrv/devices/nvhost_vic.cpp index 358e89aa8..c89ff6b27 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_vic.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_vic.cpp @@ -8,6 +8,9 @@ #include "video_core/renderer_base.h" namespace Service::Nvidia::Devices { + +u32 nvhost_vic::next_id{}; + nvhost_vic::nvhost_vic(Core::System& system_, NvCore::Container& core) : nvhost_nvdec_common{system_, core} {} diff --git a/src/core/hle/service/nvdrv/devices/nvhost_vic.h b/src/core/hle/service/nvdrv/devices/nvhost_vic.h index 252b1e6f2..59e23b41e 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_vic.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_vic.h @@ -23,6 +23,6 @@ public: void OnClose(DeviceFD fd) override; private: - u32 next_id{}; + static u32 next_id; }; } // namespace Service::Nvidia::Devices |