diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index bf12d69a5d..9540008c52 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -13,6 +13,7 @@ #include "core/hle/service/nvdrv/nvdrv.h" #include "core/memory.h" #include "video_core/control/channel_state.h" +#include "video_core/control/scheduler.h" #include "video_core/engines/puller.h" #include "video_core/gpu.h" #include "video_core/host1x/host1x.h" @@ -33,6 +34,7 @@ nvhost_gpu::nvhost_gpu(Core::System& system_, EventInterface& events_interface_, syncpoint_manager{core_.GetSyncpointManager()}, nvmap{core.GetNvMapFile()}, channel_state{system.GPU().AllocateChannel()} { channel_syncpoint = syncpoint_manager.AllocateSyncpoint(false); + channel_state->syncpoint_id = channel_syncpoint; sm_exception_breakpoint_int_report_event = events_interface.CreateEvent("GpuChannelSMExceptionBreakpointInt"); sm_exception_breakpoint_pause_report_event = @@ -157,6 +159,9 @@ NvResult nvhost_gpu::SetErrorNotifier(IoctlSetErrorNotifier& params) { NvResult nvhost_gpu::SetChannelPriority(IoctlChannelSetPriority& params) { channel_priority = params.priority; + if (channel_state->initialized) { + system.GPU().Scheduler().ChangePriority(channel_state->bind_id, channel_priority); + } LOG_DEBUG(Service_NVDRV, "(STUBBED) called, priority={:X}", channel_priority); return NvResult::Success; } @@ -314,6 +319,7 @@ NvResult nvhost_gpu::GetWaitbase(IoctlGetWaitbase& params) { NvResult nvhost_gpu::ChannelSetTimeout(IoctlChannelSetTimeout& params) { LOG_INFO(Service_NVDRV, "called, timeout=0x{:X}", params.timeout); + channel_state->timeout = params.timeout; return NvResult::Success; } @@ -321,6 +327,7 @@ NvResult nvhost_gpu::ChannelSetTimeslice(IoctlSetTimeslice& params) { LOG_INFO(Service_NVDRV, "called, timeslice=0x{:X}", params.timeslice); channel_timeslice = params.timeslice; + channel_state->timeslice = params.timeslice; return NvResult::Success; } diff --git a/src/video_core/control/channel_state.h b/src/video_core/control/channel_state.h index 3a7b9872c1..97b225391e 100644 --- a/src/video_core/control/channel_state.h +++ b/src/video_core/control/channel_state.h @@ -45,6 +45,12 @@ struct ChannelState { void BindRasterizer(VideoCore::RasterizerInterface* rasterizer); s32 bind_id = -1; + /// Scheduling info + u32 syncpoint_id = 0xFFFF; + u32 priority = 0; + u32 timeslice = 0; + u32 timeout = 0; + /// 3D engine std::unique_ptr maxwell_3d; /// 2D engine diff --git a/src/video_core/control/scheduler.cpp b/src/video_core/control/scheduler.cpp index 3165f9bf36..9ee2a6fb25 100644 --- a/src/video_core/control/scheduler.cpp +++ b/src/video_core/control/scheduler.cpp @@ -1,112 +1,243 @@ // SPDX-FileCopyrightText: 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include +#include +#include #include +#include +#include +#include #include "common/assert.h" -#include "video_core/control/channel_state.h" +#include "common/fiber.h" #include "video_core/control/scheduler.h" +#include "video_core/dma_pusher.h" #include "video_core/gpu.h" namespace Tegra::Control { -Scheduler::Scheduler(GPU& gpu_) : gpu{gpu_} {} + +struct GPFifoContext { + bool is_active; + bool is_running; + std::shared_ptr context; + std::deque pending_work; + std::mutex guard; + s32 bind_id; + std::shared_ptr info; + size_t yield_count; + size_t scheduled_count; +}; + +struct Scheduler::SchedulerImpl { + // Fifos + std::map, std::greater> schedule_priority_queue; + std::unordered_map channel_gpfifo_ids; + std::deque gpfifos; + std::deque free_fifos; + + // Scheduling + std::mutex scheduling_guard; + std::shared_ptr master_control; + bool must_reschedule{}; + GPFifoContext* current_fifo{}; +}; + +Scheduler::Scheduler(GPU& gpu_) : gpu{gpu_} { + impl = std::make_unique(); +} Scheduler::~Scheduler() = default; void Scheduler::Init() { - master_control = Common::Fiber::ThreadToFiber(); + impl->master_control = Common::Fiber::ThreadToFiber(); } void Scheduler::Resume() { - bool nothing_pending; - do { - nothing_pending = true; - current_fifo = nullptr; - { - std::unique_lock lk(scheduling_guard); - size_t num_iters = gpfifos.size(); - for (size_t i = 0; i < num_iters; i++) { - size_t current_id = (current_fifo_rotation_id + i) % gpfifos.size(); - auto& fifo = gpfifos[current_id]; - if (!fifo.is_active) { - continue; - } - std::scoped_lock lk2(fifo.guard); - if (!fifo.pending_work.empty() || fifo.working.load(std::memory_order_acquire)) { - current_fifo = &fifo; - current_fifo_rotation_id = current_id; - nothing_pending = false; - break; - } - } + while (UpdateHighestPriorityChannel()) { + impl->current_fifo->scheduled_count++; + Common::Fiber::YieldTo(impl->master_control, *impl->current_fifo->context); + } +} + +bool Scheduler::UpdateHighestPriorityChannel() { + std::scoped_lock lk(impl->scheduling_guard); + + // Clear needs to schedule state. + impl->must_reschedule = false; + + // By default, we don't have a channel to schedule. + impl->current_fifo = nullptr; + + // Check each level to see if we can schedule. + for (auto& level : impl->schedule_priority_queue) { + if (ScheduleLevel(level.second)) { + return true; } - if (current_fifo) { - Common::Fiber::YieldTo(master_control, *current_fifo->context); - current_fifo = nullptr; + } + + // Nothing to schedule. + return false; +} + +bool Scheduler::ScheduleLevel(std::list& queue) { + bool found_anything = false; + size_t min_schedule_count = std::numeric_limits::max(); + for (auto id : queue) { + auto& fifo = impl->gpfifos[id]; + std::scoped_lock lk(fifo.guard); + + // With no pending work and nothing running, this channel can't be scheduled. + if (fifo.pending_work.empty() && !fifo.is_running) { + continue; } - } while (!nothing_pending); + // Prioritize channels at current priority which have been run the least. + if (fifo.scheduled_count > min_schedule_count) { + continue; + } + + // Try not to select the same channel we just yielded from. + if (fifo.scheduled_count < fifo.yield_count) { + fifo.scheduled_count++; + continue; + } + + // Update best selection. + min_schedule_count = fifo.scheduled_count; + impl->current_fifo = &fifo; + found_anything = true; + } + return found_anything; +} + +void Scheduler::ChangePriority(s32 channel_id, u32 new_priority) { + std::scoped_lock lk(impl->scheduling_guard); + // Ensure we are tracking this channel. + auto fifo_it = impl->channel_gpfifo_ids.find(channel_id); + if (fifo_it == impl->channel_gpfifo_ids.end()) { + return; + } + + // Get the fifo and update its priority. + const size_t fifo_id = fifo_it->second; + auto& fifo = impl->gpfifos[fifo_id]; + const auto old_priority = std::exchange(fifo.info->priority, new_priority); + + // Create the new level if needed. + impl->schedule_priority_queue.try_emplace(new_priority); + + // Remove the old level and add to the new level. + impl->schedule_priority_queue[new_priority].push_back(fifo_id); + impl->schedule_priority_queue[old_priority].remove_if( + [fifo_id](size_t id) { return id == fifo_id; }); } void Scheduler::Yield() { - ASSERT(current_fifo != nullptr); - Common::Fiber::YieldTo(current_fifo->context, *master_control); - gpu.BindChannel(current_fifo->bind_id); + ASSERT(impl->current_fifo != nullptr); + + // Set yield count higher + impl->current_fifo->yield_count = impl->current_fifo->scheduled_count + 1; + Common::Fiber::YieldTo(impl->current_fifo->context, *impl->master_control); + gpu.BindChannel(impl->current_fifo->bind_id); +} + +void Scheduler::CheckStatus() { + { + std::unique_lock lk(impl->scheduling_guard); + // If no reschedule is needed, don't transfer control + if (!impl->must_reschedule) { + return; + } + } + // Transfer control to the scheduler + Common::Fiber::YieldTo(impl->current_fifo->context, *impl->master_control); + gpu.BindChannel(impl->current_fifo->bind_id); } void Scheduler::Push(s32 channel, CommandList&& entries) { - std::unique_lock lk(scheduling_guard); - auto it = channel_gpfifo_ids.find(channel); - ASSERT(it != channel_gpfifo_ids.end()); + std::scoped_lock lk(impl->scheduling_guard); + // Get and ensure we have this channel. + auto it = impl->channel_gpfifo_ids.find(channel); + ASSERT(it != impl->channel_gpfifo_ids.end()); auto gpfifo_id = it->second; - auto& fifo = gpfifos[gpfifo_id]; + auto& fifo = impl->gpfifos[gpfifo_id]; + // Add the new new work to the channel. { std::scoped_lock lk2(fifo.guard); fifo.pending_work.emplace_back(std::move(entries)); } + + // If the current running FIFO is null or the one being pushed to then + // just return + if (impl->current_fifo == nullptr || impl->current_fifo == &fifo) { + return; + } + + // If the current fifo has higher or equal priority to the current fifo then return + if (impl->current_fifo->info->priority >= fifo.info->priority) { + return; + } + // Mark scheduler update as required. + impl->must_reschedule = true; } void Scheduler::ChannelLoop(size_t gpfifo_id, s32 channel_id) { - gpu.BindChannel(channel_id); - auto& fifo = gpfifos[gpfifo_id]; - while (true) { - auto* channel_state = channels[channel_id].get(); - fifo.guard.lock(); - while (!fifo.pending_work.empty()) { - { - - fifo.working.store(true, std::memory_order_release); - CommandList&& entries = std::move(fifo.pending_work.front()); - channel_state->dma_pusher->Push(std::move(entries)); - fifo.pending_work.pop_front(); - } - fifo.guard.unlock(); - channel_state->dma_pusher->DispatchCalls(); - fifo.guard.lock(); + auto& fifo = impl->gpfifos[gpfifo_id]; + auto* channel_state = fifo.info.get(); + const auto SendToPuller = [&] { + std::scoped_lock lk(fifo.guard); + if (fifo.pending_work.empty()) { + // Stop if no work available. + fifo.is_running = false; + return false; } - fifo.working.store(false, std::memory_order_relaxed); - fifo.guard.unlock(); - Common::Fiber::YieldTo(fifo.context, *master_control); + // Otherwise, send work to puller and mark as running. + CommandList&& entries = std::move(fifo.pending_work.front()); + channel_state->dma_pusher->Push(std::move(entries)); + fifo.pending_work.pop_front(); + fifo.is_running = true; + // Succeed. + return true; + }; + // Inform the GPU about the current channel. + gpu.BindChannel(channel_id); + while (true) { + while (SendToPuller()) { + // Execute. + channel_state->dma_pusher->DispatchCalls(); + // Reschedule. + CheckStatus(); + } + // Return to host execution when all work is completed. + Common::Fiber::YieldTo(fifo.context, *impl->master_control); + // Inform the GPU about the current channel. gpu.BindChannel(channel_id); } } void Scheduler::DeclareChannel(std::shared_ptr new_channel) { s32 channel = new_channel->bind_id; - std::unique_lock lk(scheduling_guard); - channels.emplace(channel, new_channel); + std::unique_lock lk(impl->scheduling_guard); + size_t new_fifo_id; - if (!free_fifos.empty()) { - new_fifo_id = free_fifos.front(); - free_fifos.pop_front(); + if (!impl->free_fifos.empty()) { + new_fifo_id = impl->free_fifos.front(); + impl->free_fifos.pop_front(); } else { - new_fifo_id = gpfifos.size(); - gpfifos.emplace_back(); + new_fifo_id = impl->gpfifos.size(); + impl->gpfifos.emplace_back(); } - auto& new_fifo = gpfifos[new_fifo_id]; - channel_gpfifo_ids[channel] = new_fifo_id; + auto& new_fifo = impl->gpfifos[new_fifo_id]; + impl->channel_gpfifo_ids[channel] = new_fifo_id; new_fifo.is_active = true; new_fifo.bind_id = channel; new_fifo.pending_work.clear(); + new_fifo.info = new_channel; + new_fifo.scheduled_count = 0; + new_fifo.yield_count = 0; + new_fifo.is_running = false; + impl->schedule_priority_queue.try_emplace(new_channel->priority); + impl->schedule_priority_queue[new_channel->priority].push_back(new_fifo_id); std::function callback = std::bind(&Scheduler::ChannelLoop, this, new_fifo_id, channel); new_fifo.context = std::make_shared(std::move(callback)); } diff --git a/src/video_core/control/scheduler.h b/src/video_core/control/scheduler.h index c6f374f66a..e9af75d9ce 100644 --- a/src/video_core/control/scheduler.h +++ b/src/video_core/control/scheduler.h @@ -3,13 +3,11 @@ #pragma once -#include -#include +#include #include -#include -#include -#include "common/fiber.h" +#include "common/common_types.h" +#include "video_core/control/channel_state.h" #include "video_core/dma_pusher.h" namespace Tegra { @@ -35,26 +33,18 @@ public: void DeclareChannel(std::shared_ptr new_channel); + void ChangePriority(s32 channel_id, u32 new_priority); + private: void ChannelLoop(size_t gpfifo_id, s32 channel_id); + bool ScheduleLevel(std::list& queue); + void CheckStatus(); + bool UpdateHighestPriorityChannel(); + + struct SchedulerImpl; + std::unique_ptr impl; - std::unordered_map> channels; - std::unordered_map channel_gpfifo_ids; - std::mutex scheduling_guard; - std::shared_ptr master_control; - struct GPFifoContext { - bool is_active; - std::shared_ptr context; - std::deque pending_work; - std::atomic working{}; - std::mutex guard; - s32 bind_id; - }; - std::deque gpfifos; - std::deque free_fifos; GPU& gpu; - size_t current_fifo_rotation_id{}; - GPFifoContext* current_fifo{}; }; } // namespace Control