diff options
Diffstat (limited to 'dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c')
| -rwxr-xr-x | dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c | 145 |
1 files changed, 88 insertions, 57 deletions
diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c index 4e4ed05..8ee897f 100755 --- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c +++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c @@ -1,11 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note /* * - * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -16,11 +17,8 @@ * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * - * SPDX-License-Identifier: GPL-2.0 - * */ - /* * Register-based HW access backend specific APIs */ @@ -33,16 +31,19 @@ #include <tl/mali_kbase_tracepoints.h> #include <mali_kbase_hwcnt_context.h> #include <mali_kbase_reset_gpu.h> +#include <mali_kbase_kinstr_jm.h> #include <backend/gpu/mali_kbase_cache_policy_backend.h> -#include <backend/gpu/mali_kbase_device_internal.h> +#include <device/mali_kbase_device.h> #include <backend/gpu/mali_kbase_jm_internal.h> #include <backend/gpu/mali_kbase_pm_internal.h> /* Return whether the specified ringbuffer is empty. HW access lock must be - * held */ + * held + */ #define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) /* Return number of atoms currently in the specified ringbuffer. HW access lock - * must be held */ + * must be held + */ #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) static void kbase_gpu_release_atom(struct kbase_device *kbdev, @@ -253,6 +254,8 @@ static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, int kbase_backend_slot_free(struct kbase_device *kbdev, int js) { + lockdep_assert_held(&kbdev->hwaccess_lock); + if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != KBASE_RESET_GPU_NOT_PENDING) { /* The GPU is being reset - so prevent submission */ @@ -278,15 +281,19 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev, break; case KBASE_ATOM_GPU_RB_SUBMITTED: + kbase_kinstr_jm_atom_hw_release(katom); /* Inform power management at start/finish of atom so it can * update its GPU utilisation metrics. Mark atom as not - * submitted beforehand. */ + * submitted beforehand. + */ katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; kbase_pm_metrics_update(kbdev, end_timestamp); + /* Inform platform at start/finish of atom */ + kbasep_platform_event_atom_complete(katom); + if (katom->core_req & BASE_JD_REQ_PERMON) kbase_pm_release_gpu_cycle_counter_nolock(kbdev); - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, katom, &kbdev->gpu_props.props.raw_props.js_features @@ -296,6 +303,8 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev, &kbdev->gpu_props.props.raw_props.js_features [katom->slot_nr]); + /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ + case KBASE_ATOM_GPU_RB_READY: /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ @@ -540,7 +549,8 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev, kbdev); /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV * should ensure that we are not already transitiong, and that - * there are no atoms currently on the GPU. */ + * there are no atoms currently on the GPU. + */ WARN_ON(kbdev->protected_mode_transition); WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); /* If hwcnt is disabled, it means we didn't clean up correctly @@ -566,19 +576,15 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, /* We couldn't disable atomically, so kick off a worker */ if (!kbdev->protected_mode_hwcnt_disabled) { -#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE - queue_work(system_wq, - &kbdev->protected_mode_hwcnt_disable_work); -#else - queue_work(system_highpri_wq, + kbase_hwcnt_context_queue_work( + kbdev->hwcnt_gpu_ctx, &kbdev->protected_mode_hwcnt_disable_work); -#endif return -EAGAIN; } - /* Once reaching this point GPU must be - * switched to protected mode or hwcnt - * re-enabled. */ + /* Once reaching this point GPU must be switched to protected + * mode or hwcnt re-enabled. + */ if (kbase_pm_protected_entry_override_enable(kbdev)) return -EAGAIN; @@ -618,7 +624,7 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, KBASE_PM_CORE_L2) || kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2) || - kbase_is_gpu_lost(kbdev)) { + kbase_is_gpu_removed(kbdev)) { /* * The L2 is still powered, wait for all * the users to finish with it before doing @@ -718,7 +724,8 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev, kbdev); /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV * should ensure that we are not already transitiong, and that - * there are no atoms currently on the GPU. */ + * there are no atoms currently on the GPU. + */ WARN_ON(kbdev->protected_mode_transition); WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); @@ -764,8 +771,8 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); /* Only return if head atom or previous atom - * already removed - as atoms must be returned - * in order */ + * already removed - as atoms must be returned in order + */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); @@ -811,7 +818,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) lockdep_assert_held(&kbdev->hwaccess_lock); #ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbase_reset_gpu_is_active(kbdev) || kbase_is_gpu_lost(kbdev)) + if (kbase_reset_gpu_is_active(kbdev) || + kbase_is_gpu_removed(kbdev)) #else if (kbase_reset_gpu_is_active(kbdev)) #endif @@ -843,7 +851,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) break; katom[idx]->gpu_rb_state = - KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; + KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; /* ***TRANSITION TO HIGHER STATE*** */ /* fallthrough */ @@ -907,12 +915,14 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); /* Set EVENT_DONE so this atom will be - completed, not unpulled. */ + * completed, not unpulled. + */ katom[idx]->event_code = BASE_JD_EVENT_DONE; /* Only return if head atom or previous * atom already removed - as atoms must - * be returned in order. */ + * be returned in order. + */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); @@ -943,7 +953,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) if (idx == 1) { /* Only submit if head atom or previous - * atom already submitted */ + * atom already submitted + */ if ((katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED && katom[0]->gpu_rb_state != @@ -959,7 +970,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) } /* If inter-slot serialization in use then don't - * submit atom if any other slots are in use */ + * submit atom if any other slots are in use + */ if ((kbdev->serialize_jobs & KBASE_SERIALIZE_INTER_SLOT) && other_slots_busy(kbdev, js)) @@ -971,31 +983,37 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) break; #endif /* Check if this job needs the cycle counter - * enabled before submission */ + * enabled before submission + */ if (katom[idx]->core_req & BASE_JD_REQ_PERMON) kbase_pm_request_gpu_cycle_counter_l2_is_on( kbdev); kbase_job_hw_submit(kbdev, katom[idx], js); katom[idx]->gpu_rb_state = - KBASE_ATOM_GPU_RB_SUBMITTED; + KBASE_ATOM_GPU_RB_SUBMITTED; + + /* ***TRANSITION TO HIGHER STATE*** */ + /* fallthrough */ + case KBASE_ATOM_GPU_RB_SUBMITTED: /* Inform power management at start/finish of * atom so it can update its GPU utilisation - * metrics. */ + * metrics. + */ kbase_pm_metrics_update(kbdev, &katom[idx]->start_timestamp); - /* ***TRANSITION TO HIGHER STATE*** */ - /* fallthrough */ - case KBASE_ATOM_GPU_RB_SUBMITTED: - /* Atom submitted to HW, nothing else to do */ + /* Inform platform at start/finish of atom */ + kbasep_platform_event_atom_submit(katom[idx]); + break; case KBASE_ATOM_GPU_RB_RETURN_TO_JS: /* Only return if head atom or previous atom * already removed - as atoms must be returned - * in order */ + * in order + */ if (idx == 0 || katom[0]->gpu_rb_state == KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { kbase_gpu_dequeue_atom(kbdev, js, NULL); @@ -1013,7 +1031,7 @@ void kbase_backend_run_atom(struct kbase_device *kbdev, struct kbase_jd_atom *katom) { lockdep_assert_held(&kbdev->hwaccess_lock); - dev_dbg(kbdev->dev, "Backend running atom %p\n", (void *)katom); + dev_dbg(kbdev->dev, "Backend running atom %pK\n", (void *)katom); kbase_gpu_enqueue_atom(kbdev, katom); kbase_backend_slot_update(kbdev); @@ -1074,7 +1092,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, struct kbase_context *kctx = katom->kctx; dev_dbg(kbdev->dev, - "Atom %p completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n", + "Atom %pK completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n", (void *)katom, completion_code, job_tail, js); lockdep_assert_held(&kbdev->hwaccess_lock); @@ -1098,7 +1116,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not * flushed. To prevent future evictions causing possible memory * corruption we need to flush the cache manually before any - * affected memory gets reused. */ + * affected memory gets reused. + */ katom->need_cache_flush_cores_retained = true; } @@ -1181,7 +1200,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) { /* Can not dequeue this atom yet - will be - * dequeued when atom at idx0 completes */ + * dequeued when atom at idx0 completes + */ katom_idx1->event_code = BASE_JD_EVENT_STOPPED; kbase_gpu_mark_atom_for_return(kbdev, katom_idx1); @@ -1194,7 +1214,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, if (job_tail != 0 && job_tail != katom->jc) { /* Some of the job has been executed */ dev_dbg(kbdev->dev, - "Update job chain address of atom %p to resume from 0x%llx\n", + "Update job chain address of atom %pK to resume from 0x%llx\n", (void *)katom, job_tail); katom->jc = job_tail; @@ -1214,7 +1234,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, * - Schedule out the parent context if necessary, and schedule a new * one in. */ -#ifdef CONFIG_GPU_TRACEPOINTS +#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) { /* The atom in the HEAD */ struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, @@ -1255,7 +1275,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, if (katom) { dev_dbg(kbdev->dev, - "Cross-slot dependency %p has become runnable.\n", + "Cross-slot dependency %pK has become runnable.\n", (void *)katom); /* Check if there are lower priority jobs to soft stop */ @@ -1268,7 +1288,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, kbase_pm_update_state(kbdev); /* Job completion may have unblocked other atoms. Try to update all job - * slots */ + * slots + */ kbase_backend_slot_update(kbdev); } @@ -1319,7 +1340,8 @@ void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; /* As the atom was not removed, increment the * index so that we read the correct atom in the - * next iteration. */ + * next iteration. + */ atom_idx++; continue; } @@ -1422,7 +1444,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, katom_idx0_valid = (katom_idx0 == katom); /* If idx0 is to be removed and idx1 is on the same context, * then idx1 must also be removed otherwise the atoms might be - * returned out of order */ + * returned out of order + */ if (katom_idx1) katom_idx1_valid = (katom_idx1 == katom) || (katom_idx0_valid && @@ -1469,7 +1492,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT)) == 0) { /* idx0 has already completed - stop - * idx1 if needed*/ + * idx1 if needed + */ if (katom_idx1_valid) { kbase_gpu_stop_atom(kbdev, js, katom_idx1, @@ -1478,7 +1502,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, } } else { /* idx1 is in NEXT registers - attempt - * to remove */ + * to remove + */ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), @@ -1493,7 +1518,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, JS_HEAD_NEXT_HI)) != 0) { /* idx1 removed successfully, - * will be handled in IRQ */ + * will be handled in IRQ + */ kbase_gpu_remove_atom(kbdev, katom_idx1, action, true); @@ -1507,7 +1533,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ret = true; } else if (katom_idx1_valid) { /* idx0 has already completed, - * stop idx1 if needed */ + * stop idx1 if needed + */ kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); @@ -1526,7 +1553,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, * flow was also interrupted, and this function * might not enter disjoint state e.g. if we * don't actually do a hard stop on the head - * atom */ + * atom + */ kbase_gpu_stop_atom(kbdev, js, katom_idx0, action); ret = true; @@ -1554,7 +1582,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, ret = true; } else { /* idx1 is in NEXT registers - attempt to - * remove */ + * remove + */ kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), JS_COMMAND_NOP); @@ -1564,13 +1593,15 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI)) != 0) { /* idx1 removed successfully, will be - * handled in IRQ once idx0 completes */ + * handled in IRQ once idx0 completes + */ kbase_gpu_remove_atom(kbdev, katom_idx1, action, false); } else { /* idx0 has already completed - stop - * idx1 */ + * idx1 + */ kbase_gpu_stop_atom(kbdev, js, katom_idx1, action); @@ -1644,7 +1675,7 @@ void kbase_gpu_dump_slots(struct kbase_device *kbdev) if (katom) dev_info(kbdev->dev, - " js%d idx%d : katom=%p gpu_rb_state=%d\n", + " js%d idx%d : katom=%pK gpu_rb_state=%d\n", js, idx, katom, katom->gpu_rb_state); else dev_info(kbdev->dev, " js%d idx%d : empty\n", |