diff options
| author | binqi zhang <binqi.zhang@amlogic.com> | 2021-08-18 17:26:46 +0800 |
|---|---|---|
| committer | Liang Ji <liang.ji@amlogic.com> | 2021-08-26 19:29:07 +0800 |
| commit | aa33406fe610ecd15f40b4c6ece1777fb2110917 (patch) | |
| tree | 1d5e2be0a27c7a894eb77f0ef2f5ff54796091bf | |
| parent | 4b60f6d4f1254a7c18d5f30e88a8fd629ba4e633 (diff) | |
| download | mali-driver-android-tv-deadpool-4.9-android12.tar.gz | |
gpu: fix r32p1 build error on user version [3/14]android-tv-13.0.0_r0.1android-tv-12.0.0_r0.1android-tv-deadpool-4.9-android12
PD#SWPL-56978
Problem:
mali driver is not official for AndroidS
Solution:
update to r32p1 beta
Verify:
full verification tests on ohm
Change-Id: Ie689970b5a2e6aef6f03adbcc210e4f3d61c23fa
Signed-off-by: binqi zhang <binqi.zhang@amlogic.com>
| -rw-r--r-- | 0001-update-r32p1-gpu-driver.patch | 94611 | ||||
| -rw-r--r-- | bifrost/Makefile | 4 | ||||
| -rw-r--r-- | dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h | 2 | ||||
| -rw-r--r-- | dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c | 8 | ||||
| -rw-r--r-- | dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h | 3 |
5 files changed, 8 insertions, 94620 deletions
diff --git a/0001-update-r32p1-gpu-driver.patch b/0001-update-r32p1-gpu-driver.patch deleted file mode 100644 index ca10415..0000000 --- a/0001-update-r32p1-gpu-driver.patch +++ /dev/null @@ -1,94611 +0,0 @@ -From 39f791b03c5e534884f13ecfb9f86306fe7324dd Mon Sep 17 00:00:00 2001 -From: binqi zhang <binqi.zhang@amlogic.com> -Date: Thu, 12 Aug 2021 16:34:29 +0800 -Subject: [PATCH] update r32p1 gpu driver - -Change-Id: Ic8672b57c42e98ed97c65b48925280fbb38de0ee ---- - .../ABI/testing/sysfs-device-mali | 293 + - .../devicetree/bindings/arm/mali-midgard.txt | 19 +- - .../bindings/arm/memory_group_manager.txt | 7 +- - .../bindings/arm/priority_control_manager.txt | 48 + - .../arm/protected_memory_allocator.txt | 7 +- - .../devicetree/bindings/power/mali-opp.txt | 9 +- - .../Documentation/dma-buf-test-exporter.txt | 10 +- - dvalin/kernel/Mconfig | 31 +- - dvalin/kernel/build.bp | 65 +- - dvalin/kernel/drivers/base/arm/Kbuild | 34 + - dvalin/kernel/drivers/base/arm/Kconfig | 64 + - dvalin/kernel/drivers/base/arm/Makefile | 98 + - dvalin/kernel/drivers/base/arm/Mconfig | 64 + - .../base/{ => arm}/dma_buf_lock/src/Kbuild | 9 +- - .../base/{ => arm}/dma_buf_lock/src/Makefile | 17 +- - .../{ => arm}/dma_buf_lock/src/dma_buf_lock.c | 132 +- - .../{ => arm}/dma_buf_lock/src/dma_buf_lock.h | 7 +- - .../base/arm/dma_buf_test_exporter/Kbuild | 23 + - .../base/arm/dma_buf_test_exporter/build.bp | 36 + - .../dma-buf-test-exporter.c | 106 +- - .../memory_group_manager}/Kbuild | 11 +- - .../base/arm/memory_group_manager/build.bp | 36 + - .../memory_group_manager.c | 14 +- - .../arm/protected_memory_allocator/Kbuild | 23 + - .../arm/protected_memory_allocator/build.bp | 36 + - .../protected_memory_allocator.c | 551 ++ - .../base/dma_buf_test_exporter/Kconfig | 26 - - .../base/dma_buf_test_exporter/Makefile | 36 - - .../base/dma_buf_test_exporter/build.bp | 26 - - .../base/memory_group_manager/Makefile | 35 - - .../base/memory_group_manager/build.bp | 22 - - .../base/protected_memory_allocator/Makefile | 35 - - .../base/protected_memory_allocator/build.bp | 26 - - .../protected_memory_allocator.c | 308 - - dvalin/kernel/drivers/gpu/arm/Kbuild | 8 +- - dvalin/kernel/drivers/gpu/arm/Kconfig | 8 +- - .../Kbuild => gpu/arm/Makefile} | 9 +- - dvalin/kernel/drivers/gpu/arm/midgard/Kbuild | 367 +- - dvalin/kernel/drivers/gpu/arm/midgard/Kconfig | 357 +- - .../kernel/drivers/gpu/arm/midgard/Makefile | 201 +- - dvalin/kernel/drivers/gpu/arm/midgard/Mconfig | 288 +- - .../drivers/gpu/arm/midgard/arbiter/Kbuild | 11 +- - .../arm/midgard/arbiter/mali_kbase_arbif.c | 209 +- - .../arm/midgard/arbiter/mali_kbase_arbif.h | 44 +- - .../midgard/arbiter/mali_kbase_arbiter_defs.h | 34 +- - .../arbiter/mali_kbase_arbiter_interface.h | 70 +- - .../midgard/arbiter/mali_kbase_arbiter_pm.c | 614 +- - .../midgard/arbiter/mali_kbase_arbiter_pm.h | 87 +- - .../gpu/arm/midgard/backend/gpu/Kbuild | 70 +- - .../backend/gpu/mali_kbase_backend_config.h | 7 +- - .../gpu/mali_kbase_cache_policy_backend.c | 9 +- - .../gpu/mali_kbase_cache_policy_backend.h | 10 +- - .../gpu/mali_kbase_clk_rate_trace_mgr.c | 325 ++ - .../gpu/mali_kbase_clk_rate_trace_mgr.h | 154 + - .../gpu/mali_kbase_debug_job_fault_backend.c | 11 +- - .../midgard/backend/gpu/mali_kbase_devfreq.c | 207 +- - .../midgard/backend/gpu/mali_kbase_devfreq.h | 23 +- - .../backend/gpu/mali_kbase_device_hw.c | 388 -- - .../backend/gpu/mali_kbase_device_internal.h | 127 - - .../backend/gpu/mali_kbase_gpuprops_backend.c | 75 +- - .../backend/gpu/mali_kbase_instr_backend.c | 130 +- - .../backend/gpu/mali_kbase_instr_defs.h | 18 +- - .../backend/gpu/mali_kbase_instr_internal.h | 9 +- - .../backend/gpu/mali_kbase_irq_internal.h | 7 +- - .../backend/gpu/mali_kbase_irq_linux.c | 37 +- - .../midgard/backend/gpu/mali_kbase_jm_as.c | 14 +- - .../midgard/backend/gpu/mali_kbase_jm_defs.h | 20 +- - .../midgard/backend/gpu/mali_kbase_jm_hw.c | 236 +- - .../backend/gpu/mali_kbase_jm_internal.h | 17 +- - .../midgard/backend/gpu/mali_kbase_jm_rb.c | 145 +- - .../midgard/backend/gpu/mali_kbase_jm_rb.h | 8 +- - .../backend/gpu/mali_kbase_js_backend.c | 61 +- - .../backend/gpu/mali_kbase_js_internal.h | 8 +- - .../backend/gpu/mali_kbase_l2_mmu_config.c | 47 +- - .../backend/gpu/mali_kbase_l2_mmu_config.h | 25 +- - .../backend/gpu/mali_kbase_pm_always_on.c | 13 +- - .../backend/gpu/mali_kbase_pm_always_on.h | 9 +- - .../backend/gpu/mali_kbase_pm_backend.c | 243 +- - .../midgard/backend/gpu/mali_kbase_pm_ca.c | 44 +- - .../midgard/backend/gpu/mali_kbase_pm_ca.h | 7 +- - .../backend/gpu/mali_kbase_pm_ca_devfreq.h | 7 +- - .../backend/gpu/mali_kbase_pm_coarse_demand.c | 13 +- - .../backend/gpu/mali_kbase_pm_coarse_demand.h | 9 +- - .../midgard/backend/gpu/mali_kbase_pm_defs.h | 244 +- - .../backend/gpu/mali_kbase_pm_driver.c | 907 ++- - .../backend/gpu/mali_kbase_pm_internal.h | 143 +- - .../backend/gpu/mali_kbase_pm_l2_states.h | 20 +- - .../backend/gpu/mali_kbase_pm_mcu_states.h | 63 + - .../backend/gpu/mali_kbase_pm_metrics.c | 271 +- - .../backend/gpu/mali_kbase_pm_policy.c | 204 +- - .../backend/gpu/mali_kbase_pm_policy.h | 7 +- - .../backend/gpu/mali_kbase_pm_shader_states.h | 44 +- - .../arm/midgard/backend/gpu/mali_kbase_time.c | 57 +- - .../kernel/drivers/gpu/arm/midgard/build.bp | 203 +- - .../arm/midgard/context/Kbuild} | 19 +- - .../context/backend/mali_kbase_context_csf.c | 201 + - .../context/backend/mali_kbase_context_jm.c | 138 +- - .../arm/midgard/context/mali_kbase_context.c | 205 +- - .../arm/midgard/context/mali_kbase_context.h | 35 +- - .../context/mali_kbase_context_internal.h | 18 +- - .../kernel/drivers/gpu/arm/midgard/csf/Kbuild | 47 + - .../ipa_control/Kbuild} | 11 +- - .../ipa_control/mali_kbase_csf_ipa_control.c | 925 +++ - .../ipa_control/mali_kbase_csf_ipa_control.h | 244 + - .../gpu/arm/midgard/csf/mali_kbase_csf.c | 3069 ++++++++++ - .../gpu/arm/midgard/csf/mali_kbase_csf.h | 564 ++ - .../csf/mali_kbase_csf_cpu_queue_debugfs.c | 191 + - .../csf/mali_kbase_csf_cpu_queue_debugfs.h | 90 + - .../midgard/csf/mali_kbase_csf_csg_debugfs.c | 591 ++ - .../midgard/csf/mali_kbase_csf_csg_debugfs.h | 47 + - .../gpu/arm/midgard/csf/mali_kbase_csf_defs.h | 1254 ++++ - .../arm/midgard/csf/mali_kbase_csf_firmware.c | 2337 ++++++++ - .../arm/midgard/csf/mali_kbase_csf_firmware.h | 811 +++ - .../midgard/csf/mali_kbase_csf_firmware_cfg.c | 327 ++ - .../midgard/csf/mali_kbase_csf_firmware_cfg.h | 74 + - .../csf/mali_kbase_csf_firmware_no_mali.c | 1389 +++++ - .../csf/mali_kbase_csf_heap_context_alloc.c | 195 + - .../csf/mali_kbase_csf_heap_context_alloc.h | 75 + - .../gpu/arm/midgard/csf/mali_kbase_csf_kcpu.c | 2258 ++++++++ - .../gpu/arm/midgard/csf/mali_kbase_csf_kcpu.h | 356 ++ - .../midgard/csf/mali_kbase_csf_kcpu_debugfs.c | 197 + - .../midgard/csf/mali_kbase_csf_kcpu_debugfs.h | 37 + - .../csf/mali_kbase_csf_protected_memory.c | 119 + - .../csf/mali_kbase_csf_protected_memory.h | 71 + - .../midgard/csf/mali_kbase_csf_reset_gpu.c | 629 ++ - .../midgard/csf/mali_kbase_csf_scheduler.c | 5063 +++++++++++++++++ - .../midgard/csf/mali_kbase_csf_scheduler.h | 494 ++ - .../midgard/csf/mali_kbase_csf_tiler_heap.c | 611 ++ - .../midgard/csf/mali_kbase_csf_tiler_heap.h | 115 + - .../csf/mali_kbase_csf_tiler_heap_debugfs.c | 106 + - .../csf/mali_kbase_csf_tiler_heap_debugfs.h | 37 + - .../csf/mali_kbase_csf_tiler_heap_def.h | 114 + - .../arm/midgard/csf/mali_kbase_csf_timeout.c | 178 + - .../arm/midgard/csf/mali_kbase_csf_timeout.h | 66 + - .../midgard/csf/mali_kbase_csf_tl_reader.c | 534 ++ - .../midgard/csf/mali_kbase_csf_tl_reader.h | 185 + - .../midgard/csf/mali_kbase_csf_trace_buffer.c | 688 +++ - .../midgard/csf/mali_kbase_csf_trace_buffer.h | 182 + - .../drivers/gpu/arm/midgard/debug/Kbuild | 27 + - .../mali_kbase_debug_ktrace_codes_csf.h | 278 + - .../mali_kbase_debug_ktrace_codes_jm.h | 10 +- - .../backend/mali_kbase_debug_ktrace_csf.c | 193 + - .../backend/mali_kbase_debug_ktrace_csf.h | 203 + - .../mali_kbase_debug_ktrace_defs_csf.h | 116 + - .../backend/mali_kbase_debug_ktrace_defs_jm.h | 100 +- - .../backend/mali_kbase_debug_ktrace_jm.c | 50 +- - .../backend/mali_kbase_debug_ktrace_jm.h | 118 +- - .../mali_kbase_debug_linux_ktrace_csf.h | 241 + - .../mali_kbase_debug_linux_ktrace_jm.h | 52 +- - .../midgard/debug/mali_kbase_debug_ktrace.c | 55 +- - .../midgard/debug/mali_kbase_debug_ktrace.h | 20 +- - .../debug/mali_kbase_debug_ktrace_codes.h | 24 +- - .../debug/mali_kbase_debug_ktrace_defs.h | 83 +- - .../debug/mali_kbase_debug_ktrace_internal.h | 7 +- - .../debug/mali_kbase_debug_linux_ktrace.h | 40 +- - .../{tests/kutf/Makefile => device/Kbuild} | 30 +- - .../device/backend/mali_kbase_device_csf.c | 464 ++ - .../device/backend/mali_kbase_device_hw_csf.c | 163 + - .../device/backend/mali_kbase_device_hw_jm.c | 98 + - .../device/backend/mali_kbase_device_jm.c | 181 +- - .../arm/midgard/device/mali_kbase_device.c | 233 +- - .../arm/midgard/device/mali_kbase_device.h | 126 +- - .../arm/midgard/device/mali_kbase_device_hw.c | 182 + - .../device/mali_kbase_device_internal.h | 24 +- - .../{tests/kutf/Kconfig => gpu/Kbuild} | 19 +- - .../gpu/backend/mali_kbase_gpu_fault_csf.c | 104 + - .../gpu/backend/mali_kbase_gpu_fault_jm.c | 13 +- - .../gpu/arm/midgard/gpu/mali_kbase_gpu.c | 8 +- - .../arm/midgard/gpu/mali_kbase_gpu_fault.h | 23 +- - .../arm/midgard/gpu/mali_kbase_gpu_regmap.h | 416 +- - .../kernel/drivers/gpu/arm/midgard/ipa/Kbuild | 25 +- - .../mali_kbase_ipa_counter_common_csf.c | 457 ++ - .../mali_kbase_ipa_counter_common_csf.h | 159 + - .../mali_kbase_ipa_counter_common_jm.c} | 20 +- - .../mali_kbase_ipa_counter_common_jm.h} | 40 +- - .../ipa/backend/mali_kbase_ipa_counter_csf.c | 171 + - .../mali_kbase_ipa_counter_jm.c} | 113 +- - .../gpu/arm/midgard/ipa/mali_kbase_ipa.c | 287 +- - .../gpu/arm/midgard/ipa/mali_kbase_ipa.h | 90 +- - .../arm/midgard/ipa/mali_kbase_ipa_debugfs.c | 12 +- - .../arm/midgard/ipa/mali_kbase_ipa_debugfs.h | 12 +- - .../arm/midgard/ipa/mali_kbase_ipa_simple.c | 53 +- - .../arm/midgard/ipa/mali_kbase_ipa_simple.h | 7 +- - .../gpu/arm/midgard/jm/mali_kbase_jm_defs.h | 76 +- - .../gpu/arm/midgard/jm/mali_kbase_jm_js.h | 89 +- - .../gpu/arm/midgard/jm/mali_kbase_js_defs.h | 411 +- - .../arm/midgard/mali_base_hwconfig_features.h | 62 +- - .../arm/midgard/mali_base_hwconfig_issues.h | 62 +- - .../drivers/gpu/arm/midgard/mali_kbase.h | 165 +- - .../arm/midgard/mali_kbase_as_fault_debugfs.c | 11 +- - .../arm/midgard/mali_kbase_as_fault_debugfs.h | 9 +- - .../drivers/gpu/arm/midgard/mali_kbase_bits.h | 16 +- - .../gpu/arm/midgard/mali_kbase_cache_policy.c | 12 +- - .../gpu/arm/midgard/mali_kbase_cache_policy.h | 11 +- - .../drivers/gpu/arm/midgard/mali_kbase_caps.h | 61 + - .../gpu/arm/midgard/mali_kbase_ccswe.c | 100 + - .../gpu/arm/midgard/mali_kbase_ccswe.h | 96 + - .../gpu/arm/midgard/mali_kbase_config.c | 68 +- - .../gpu/arm/midgard/mali_kbase_config.h | 294 +- - .../arm/midgard/mali_kbase_config_defaults.h | 26 +- - .../gpu/arm/midgard/mali_kbase_core_linux.c | 1969 +++++-- - .../arm/midgard/mali_kbase_cs_experimental.h | 20 +- - .../gpu/arm/midgard/mali_kbase_ctx_sched.c | 84 +- - .../gpu/arm/midgard/mali_kbase_ctx_sched.h | 44 +- - .../gpu/arm/midgard/mali_kbase_debug.c | 9 +- - .../gpu/arm/midgard/mali_kbase_debug.h | 79 +- - .../arm/midgard/mali_kbase_debug_job_fault.c | 36 +- - .../arm/midgard/mali_kbase_debug_job_fault.h | 10 +- - .../arm/midgard/mali_kbase_debug_mem_view.c | 26 +- - .../arm/midgard/mali_kbase_debug_mem_view.h | 7 +- - .../arm/midgard/mali_kbase_debugfs_helper.c | 104 +- - .../arm/midgard/mali_kbase_debugfs_helper.h | 53 +- - .../drivers/gpu/arm/midgard/mali_kbase_defs.h | 573 +- - .../arm/midgard/mali_kbase_disjoint_events.c | 7 +- - .../gpu/arm/midgard/mali_kbase_dma_fence.c | 59 +- - .../gpu/arm/midgard/mali_kbase_dma_fence.h | 26 +- - .../gpu/arm/midgard/mali_kbase_dummy_job_wa.c | 15 +- - .../gpu/arm/midgard/mali_kbase_dummy_job_wa.h | 36 +- - .../gpu/arm/midgard/mali_kbase_dvfs_debugfs.c | 98 + - .../gpu/arm/midgard/mali_kbase_dvfs_debugfs.h | 35 + - .../gpu/arm/midgard/mali_kbase_event.c | 30 +- - .../gpu/arm/midgard/mali_kbase_fence.c | 73 +- - .../gpu/arm/midgard/mali_kbase_fence.h | 17 +- - .../gpu/arm/midgard/mali_kbase_fence_defs.h | 15 +- - .../gpu/arm/midgard/mali_kbase_fence_ops.c | 83 + - .../gpu/arm/midgard/mali_kbase_gator.h | 7 +- - .../midgard/mali_kbase_gpu_memory_debugfs.c | 22 +- - .../midgard/mali_kbase_gpu_memory_debugfs.h | 26 +- - .../gpu/arm/midgard/mali_kbase_gpuprops.c | 315 +- - .../gpu/arm/midgard/mali_kbase_gpuprops.h | 67 +- - .../arm/midgard/mali_kbase_gpuprops_types.h | 89 +- - .../drivers/gpu/arm/midgard/mali_kbase_gwt.c | 11 +- - .../drivers/gpu/arm/midgard/mali_kbase_gwt.h | 9 +- - .../drivers/gpu/arm/midgard/mali_kbase_hw.c | 230 +- - .../drivers/gpu/arm/midgard/mali_kbase_hw.h | 23 +- - .../arm/midgard/mali_kbase_hwaccess_backend.h | 8 +- - .../arm/midgard/mali_kbase_hwaccess_defs.h | 15 +- - .../midgard/mali_kbase_hwaccess_gpuprops.h | 39 +- - .../arm/midgard/mali_kbase_hwaccess_instr.h | 22 +- - .../gpu/arm/midgard/mali_kbase_hwaccess_jm.h | 12 +- - .../gpu/arm/midgard/mali_kbase_hwaccess_pm.h | 75 +- - .../arm/midgard/mali_kbase_hwaccess_time.h | 30 +- - .../gpu/arm/midgard/mali_kbase_hwcnt.c | 103 +- - .../midgard/mali_kbase_hwcnt_accumulator.h | 7 +- - .../arm/midgard/mali_kbase_hwcnt_backend.h | 102 +- - .../midgard/mali_kbase_hwcnt_backend_csf.c | 1864 ++++++ - .../midgard/mali_kbase_hwcnt_backend_csf.h | 162 + - .../midgard/mali_kbase_hwcnt_backend_csf_if.h | 311 + - .../mali_kbase_hwcnt_backend_csf_if_fw.c | 787 +++ - .../mali_kbase_hwcnt_backend_csf_if_fw.h | 50 + - .../midgard/mali_kbase_hwcnt_backend_gpu.c | 510 -- - .../arm/midgard/mali_kbase_hwcnt_backend_jm.c | 793 +++ - ...nd_gpu.h => mali_kbase_hwcnt_backend_jm.h} | 23 +- - .../arm/midgard/mali_kbase_hwcnt_context.h | 46 +- - .../gpu/arm/midgard/mali_kbase_hwcnt_gpu.c | 760 +-- - .../gpu/arm/midgard/mali_kbase_hwcnt_gpu.h | 314 +- - .../gpu/arm/midgard/mali_kbase_hwcnt_legacy.c | 11 +- - .../gpu/arm/midgard/mali_kbase_hwcnt_legacy.h | 7 +- - .../gpu/arm/midgard/mali_kbase_hwcnt_types.c | 129 +- - .../gpu/arm/midgard/mali_kbase_hwcnt_types.h | 125 +- - .../midgard/mali_kbase_hwcnt_virtualizer.c | 26 +- - .../midgard/mali_kbase_hwcnt_virtualizer.h | 23 +- - .../drivers/gpu/arm/midgard/mali_kbase_jd.c | 331 +- - .../gpu/arm/midgard/mali_kbase_jd_debugfs.c | 56 +- - .../gpu/arm/midgard/mali_kbase_jd_debugfs.h | 12 +- - .../drivers/gpu/arm/midgard/mali_kbase_jm.c | 16 +- - .../drivers/gpu/arm/midgard/mali_kbase_jm.h | 12 +- - .../drivers/gpu/arm/midgard/mali_kbase_js.c | 422 +- - .../drivers/gpu/arm/midgard/mali_kbase_js.h | 12 +- - .../gpu/arm/midgard/mali_kbase_js_ctx_attr.c | 39 +- - .../gpu/arm/midgard/mali_kbase_js_ctx_attr.h | 47 +- - .../gpu/arm/midgard/mali_kbase_kinstr_jm.c | 894 +++ - .../gpu/arm/midgard/mali_kbase_kinstr_jm.h | 275 + - .../gpu/arm/midgard/mali_kbase_linux.h | 12 +- - .../drivers/gpu/arm/midgard/mali_kbase_mem.c | 1160 ++-- - .../drivers/gpu/arm/midgard/mali_kbase_mem.h | 518 +- - .../gpu/arm/midgard/mali_kbase_mem_linux.c | 683 ++- - .../gpu/arm/midgard/mali_kbase_mem_linux.h | 36 +- - .../gpu/arm/midgard/mali_kbase_mem_lowlevel.h | 13 +- - .../gpu/arm/midgard/mali_kbase_mem_pool.c | 44 +- - .../arm/midgard/mali_kbase_mem_pool_debugfs.c | 7 +- - .../arm/midgard/mali_kbase_mem_pool_debugfs.h | 7 +- - .../arm/midgard/mali_kbase_mem_pool_group.c | 7 +- - .../arm/midgard/mali_kbase_mem_pool_group.h | 7 +- - .../midgard/mali_kbase_mem_profile_debugfs.c | 26 +- - .../midgard/mali_kbase_mem_profile_debugfs.h | 23 +- - .../mali_kbase_mem_profile_debugfs_buf_size.h | 8 +- - .../arm/midgard/mali_kbase_mipe_gen_header.h | 40 +- - .../gpu/arm/midgard/mali_kbase_mipe_proto.h | 7 +- - .../gpu/arm/midgard/mali_kbase_native_mgm.c | 7 +- - .../gpu/arm/midgard/mali_kbase_native_mgm.h | 9 +- - .../arm/midgard/mali_kbase_platform_fake.c | 15 +- - .../drivers/gpu/arm/midgard/mali_kbase_pm.c | 55 +- - .../drivers/gpu/arm/midgard/mali_kbase_pm.h | 34 +- - .../midgard/mali_kbase_regs_history_debugfs.c | 138 +- - .../midgard/mali_kbase_regs_history_debugfs.h | 40 +- - .../gpu/arm/midgard/mali_kbase_reset_gpu.h | 162 +- - .../drivers/gpu/arm/midgard/mali_kbase_smc.c | 9 +- - .../drivers/gpu/arm/midgard/mali_kbase_smc.h | 11 +- - .../gpu/arm/midgard/mali_kbase_softjobs.c | 167 +- - .../gpu/arm/midgard/mali_kbase_strings.c | 10 +- - .../gpu/arm/midgard/mali_kbase_strings.h | 7 +- - .../drivers/gpu/arm/midgard/mali_kbase_sync.h | 28 +- - .../gpu/arm/midgard/mali_kbase_sync_android.c | 67 +- - .../gpu/arm/midgard/mali_kbase_sync_common.c | 11 +- - .../gpu/arm/midgard/mali_kbase_sync_file.c | 36 +- - .../arm/midgard/mali_kbase_trace_gpu_mem.c | 221 + - .../arm/midgard/mali_kbase_trace_gpu_mem.h | 100 + - .../gpu/arm/midgard/mali_kbase_utility.h | 11 +- - .../gpu/arm/midgard/mali_kbase_vinstr.c | 311 +- - .../gpu/arm/midgard/mali_kbase_vinstr.h | 7 +- - .../gpu/arm/midgard/mali_linux_trace.h | 47 +- - .../drivers/gpu/arm/midgard/mali_malisw.h | 60 +- - ...gpu.h => mali_power_gpu_frequency_trace.c} | 21 +- - .../midgard/mali_power_gpu_frequency_trace.h | 68 + - .../Kconfig => gpu/arm/midgard/mmu/Kbuild} | 22 +- - .../midgard/mmu/backend/mali_kbase_mmu_csf.c | 565 ++ - .../midgard/mmu/backend/mali_kbase_mmu_jm.c | 90 +- - .../gpu/arm/midgard/mmu/mali_kbase_mmu.c | 220 +- - .../gpu/arm/midgard/mmu/mali_kbase_mmu.h | 45 +- - .../gpu/arm/midgard/mmu/mali_kbase_mmu_hw.h | 12 +- - .../midgard/mmu/mali_kbase_mmu_hw_direct.c | 67 +- - .../arm/midgard/mmu/mali_kbase_mmu_internal.h | 49 +- - .../midgard/mmu/mali_kbase_mmu_mode_aarch64.c | 25 +- - .../midgard/mmu/mali_kbase_mmu_mode_lpae.c | 215 - - .../drivers/gpu/arm/midgard/platform/Kconfig | 10 +- - .../arm/midgard/platform/devicetree/Kbuild | 7 +- - .../midgard/platform/devicetree/Kbuild.rej | 17 + - .../devicetree/mali_kbase_clk_rate_trace.c | 105 + - .../devicetree/mali_kbase_config_devicetree.c | 39 +- - .../devicetree/mali_kbase_config_platform.c | 43 + - .../devicetree/mali_kbase_config_platform.h | 54 +- - .../mali_kbase_config_platform.h.rej | 42 + - .../devicetree/mali_kbase_runtime_pm.c | 26 +- - .../gpu/arm/midgard/platform/vexpress/Kbuild | 11 +- - .../vexpress/mali_kbase_config_platform.h | 7 +- - .../vexpress/mali_kbase_config_vexpress.c | 22 +- - .../midgard/platform/vexpress_1xv7_a57/Kbuild | 11 +- - .../mali_kbase_config_platform.h | 7 +- - .../mali_kbase_config_vexpress.c | 20 +- - .../platform/vexpress_6xvirtex7_10mhz/Kbuild | 13 +- - .../mali_kbase_config_platform.h | 7 +- - .../mali_kbase_config_vexpress.c | 22 +- - .../gpu/arm/midgard/protected_mode_switcher.h | 31 +- - .../drivers/gpu/arm/midgard/tests/Kbuild | 17 +- - .../drivers/gpu/arm/midgard/tests/Kconfig | 46 +- - .../drivers/gpu/arm/midgard/tests/Mconfig | 81 +- - .../drivers/gpu/arm/midgard/tests/build.bp | 40 + - .../midgard/tests/include/kutf/kutf_helpers.h | 15 +- - .../tests/include/kutf/kutf_helpers_user.h | 25 +- - .../arm/midgard/tests/include/kutf/kutf_mem.h | 7 +- - .../tests/include/kutf/kutf_resultset.h | 7 +- - .../midgard/tests/include/kutf/kutf_suite.h | 29 +- - .../midgard/tests/include/kutf/kutf_utils.h | 7 +- - .../drivers/gpu/arm/midgard/tests/kutf/Kbuild | 21 +- - .../gpu/arm/midgard/tests/kutf/build.bp | 24 +- - .../gpu/arm/midgard/tests/kutf/kutf_helpers.c | 13 +- - .../midgard/tests/kutf/kutf_helpers_user.c | 28 +- - .../gpu/arm/midgard/tests/kutf/kutf_mem.c | 7 +- - .../arm/midgard/tests/kutf/kutf_resultset.c | 7 +- - .../gpu/arm/midgard/tests/kutf/kutf_suite.c | 23 +- - .../gpu/arm/midgard/tests/kutf/kutf_utils.c | 7 +- - .../mali_kutf_clk_rate_trace/kernel/Kbuild | 25 + - .../mali_kutf_clk_rate_trace/kernel/build.bp | 43 + - .../kernel/mali_kutf_clk_rate_trace_test.c | 957 ++++ - .../mali_kutf_clk_rate_trace_test.h | 151 + - .../midgard/tests/mali_kutf_irq_test/Kbuild | 13 +- - .../midgard/tests/mali_kutf_irq_test/Kconfig | 29 - - .../midgard/tests/mali_kutf_irq_test/Makefile | 51 - - .../midgard/tests/mali_kutf_irq_test/build.bp | 21 +- - .../mali_kutf_irq_test_main.c | 15 +- - .../arm/midgard/thirdparty}/Kbuild | 9 +- - .../arm/midgard/thirdparty/mali_kbase_mmap.c | 77 +- - .../kernel/drivers/gpu/arm/midgard/tl/Kbuild | 32 + - .../tl/backend/mali_kbase_timeline_csf.c | 171 + - .../tl/backend/mali_kbase_timeline_jm.c | 23 +- - .../gpu/arm/midgard/tl/mali_kbase_timeline.c | 142 +- - .../gpu/arm/midgard/tl/mali_kbase_timeline.h | 57 +- - .../arm/midgard/tl/mali_kbase_timeline_io.c | 177 +- - .../arm/midgard/tl/mali_kbase_timeline_priv.h | 26 +- - .../arm/midgard/tl/mali_kbase_tl_serialize.h | 7 +- - .../gpu/arm/midgard/tl/mali_kbase_tlstream.c | 43 +- - .../gpu/arm/midgard/tl/mali_kbase_tlstream.h | 22 +- - .../arm/midgard/tl/mali_kbase_tracepoints.c | 585 +- - .../arm/midgard/tl/mali_kbase_tracepoints.h | 1159 +++- - .../include/linux/dma-buf-test-exporter.h | 8 +- - .../include/linux/memory_group_manager.h | 7 +- - .../include/linux/priority_control_manager.h | 77 + - .../linux/protected_memory_allocator.h | 7 +- - .../include/linux/protected_mode_switcher.h | 7 +- - .../arm/midgard/csf/mali_base_csf_kernel.h | 765 +++ - .../csf/mali_gpu_csf_control_registers.h | 32 + - .../arm/midgard/csf/mali_gpu_csf_registers.h | 1488 +++++ - .../arm/midgard/csf/mali_kbase_csf_ioctl.h | 433 ++ - .../gpu/backend/mali_kbase_gpu_regmap_csf.h | 335 ++ - .../gpu/backend/mali_kbase_gpu_regmap_jm.h | 47 +- - .../midgard/gpu/mali_kbase_gpu_coherency.h | 13 +- - .../gpu/arm/midgard/gpu/mali_kbase_gpu_id.h | 64 +- - .../arm/midgard/gpu/mali_kbase_gpu_regmap.h | 434 ++ - .../gpu/arm/midgard/jm/mali_base_jm_kernel.h | 285 +- - .../gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h | 107 +- - .../uapi}/gpu/arm/midgard/mali_base_kernel.h | 461 +- - .../gpu/arm/midgard/mali_base_mem_priv.h | 23 +- - .../gpu/arm/midgard/mali_kbase_hwcnt_reader.h | 76 +- - .../uapi}/gpu/arm/midgard/mali_kbase_ioctl.h | 212 +- - .../arm/midgard/mali_kbase_kinstr_jm_reader.h | 69 + - .../uapi}/gpu/arm/midgard/mali_uk.h | 46 +- - 407 files changed, 58537 insertions(+), 10409 deletions(-) - create mode 100644 dvalin/kernel/Documentation/ABI/testing/sysfs-device-mali - create mode 100644 dvalin/kernel/Documentation/devicetree/bindings/arm/priority_control_manager.txt - create mode 100644 dvalin/kernel/drivers/base/arm/Kbuild - create mode 100644 dvalin/kernel/drivers/base/arm/Kconfig - create mode 100644 dvalin/kernel/drivers/base/arm/Makefile - create mode 100644 dvalin/kernel/drivers/base/arm/Mconfig - rename dvalin/kernel/drivers/base/{ => arm}/dma_buf_lock/src/Kbuild (78%) - rename dvalin/kernel/drivers/base/{ => arm}/dma_buf_lock/src/Makefile (71%) - rename dvalin/kernel/drivers/base/{ => arm}/dma_buf_lock/src/dma_buf_lock.c (90%) - rename dvalin/kernel/drivers/base/{ => arm}/dma_buf_lock/src/dma_buf_lock.h (88%) - create mode 100644 dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/Kbuild - create mode 100644 dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/build.bp - rename dvalin/kernel/drivers/base/{ => arm}/dma_buf_test_exporter/dma-buf-test-exporter.c (89%) - rename dvalin/kernel/drivers/base/{dma_buf_test_exporter => arm/memory_group_manager}/Kbuild (74%) - create mode 100644 dvalin/kernel/drivers/base/arm/memory_group_manager/build.bp - rename dvalin/kernel/drivers/base/{ => arm}/memory_group_manager/memory_group_manager.c (98%) - create mode 100644 dvalin/kernel/drivers/base/arm/protected_memory_allocator/Kbuild - create mode 100644 dvalin/kernel/drivers/base/arm/protected_memory_allocator/build.bp - create mode 100644 dvalin/kernel/drivers/base/arm/protected_memory_allocator/protected_memory_allocator.c - delete mode 100644 dvalin/kernel/drivers/base/dma_buf_test_exporter/Kconfig - delete mode 100644 dvalin/kernel/drivers/base/dma_buf_test_exporter/Makefile - delete mode 100644 dvalin/kernel/drivers/base/dma_buf_test_exporter/build.bp - delete mode 100644 dvalin/kernel/drivers/base/memory_group_manager/Makefile - delete mode 100644 dvalin/kernel/drivers/base/memory_group_manager/build.bp - delete mode 100644 dvalin/kernel/drivers/base/protected_memory_allocator/Makefile - delete mode 100644 dvalin/kernel/drivers/base/protected_memory_allocator/build.bp - delete mode 100644 dvalin/kernel/drivers/base/protected_memory_allocator/protected_memory_allocator.c - rename dvalin/kernel/drivers/{base/protected_memory_allocator/Kbuild => gpu/arm/Makefile} (77%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_clk_rate_trace_mgr.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_clk_rate_trace_mgr.h - delete mode 100755 dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c - delete mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_mcu_states.h - rename dvalin/kernel/drivers/{base/memory_group_manager/Kconfig => gpu/arm/midgard/context/Kbuild} (63%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/context/backend/mali_kbase_context_csf.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/Kbuild - rename dvalin/kernel/drivers/gpu/arm/midgard/{Makefile.kbase => csf/ipa_control/Kbuild} (75%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/mali_kbase_csf_ipa_control.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/mali_kbase_csf_ipa_control.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_cpu_queue_debugfs.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_cpu_queue_debugfs.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_csg_debugfs.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_csg_debugfs.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_defs.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_cfg.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_cfg.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_no_mali.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_heap_context_alloc.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_heap_context_alloc.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu_debugfs.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu_debugfs.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_protected_memory.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_protected_memory.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_reset_gpu.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_scheduler.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_scheduler.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_debugfs.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_debugfs.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_def.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_timeout.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_timeout.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tl_reader.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tl_reader.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_trace_buffer.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_trace_buffer.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/debug/Kbuild - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_codes_csf.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_csf.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_csf.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_defs_csf.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_linux_ktrace_csf.h - rename dvalin/kernel/drivers/gpu/arm/midgard/{tests/kutf/Makefile => device/Kbuild} (56%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_csf.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_hw_csf.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_hw_jm.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device_hw.c - rename dvalin/kernel/drivers/gpu/arm/midgard/{tests/kutf/Kconfig => gpu/Kbuild} (65%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_fault_csf.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_csf.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_csf.h - rename dvalin/kernel/drivers/gpu/arm/midgard/ipa/{mali_kbase_ipa_vinstr_common.c => backend/mali_kbase_ipa_counter_common_jm.c} (95%) - rename dvalin/kernel/drivers/gpu/arm/midgard/ipa/{mali_kbase_ipa_vinstr_common.h => backend/mali_kbase_ipa_counter_common_jm.h} (85%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_csf.c - rename dvalin/kernel/drivers/gpu/arm/midgard/ipa/{mali_kbase_ipa_vinstr_g7x.c => backend/mali_kbase_ipa_counter_jm.c} (83%) - mode change 100644 => 100755 dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_caps.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ccswe.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ccswe.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dvfs_debugfs.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dvfs_debugfs.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence_ops.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if_fw.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if_fw.h - delete mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_gpu.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_jm.c - rename dvalin/kernel/drivers/gpu/arm/midgard/{mali_kbase_hwcnt_backend_gpu.h => mali_kbase_hwcnt_backend_jm.h} (75%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_kinstr_jm.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_kinstr_jm.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_trace_gpu_mem.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_trace_gpu_mem.h - rename dvalin/kernel/drivers/gpu/arm/midgard/{gpu/mali_kbase_gpu.h => mali_power_gpu_frequency_trace.c} (67%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mali_power_gpu_frequency_trace.h - rename dvalin/kernel/drivers/{base/protected_memory_allocator/Kconfig => gpu/arm/midgard/mmu/Kbuild} (61%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mmu/backend/mali_kbase_mmu_csf.c - delete mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_mode_lpae.c - mode change 100644 => 100755 dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/Kbuild - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/Kbuild.rej - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_clk_rate_trace.c - mode change 100644 => 100755 dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.c - mode change 100644 => 100755 dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h.rej - mode change 100644 => 100755 dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/tests/build.bp - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/Kbuild - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/build.bp - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h - delete mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig - delete mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile - rename dvalin/kernel/drivers/{base/memory_group_manager => gpu/arm/midgard/thirdparty}/Kbuild (82%) - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/tl/Kbuild - create mode 100644 dvalin/kernel/drivers/gpu/arm/midgard/tl/backend/mali_kbase_timeline_csf.c - create mode 100644 dvalin/kernel/include/linux/priority_control_manager.h - create mode 100644 dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_base_csf_kernel.h - create mode 100644 dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_gpu_csf_control_registers.h - create mode 100644 dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h - create mode 100644 dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_kbase_csf_ioctl.h - create mode 100644 dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_csf.h - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_jm.h (89%) - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/gpu/mali_kbase_gpu_coherency.h (75%) - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/gpu/mali_kbase_gpu_id.h (74%) - create mode 100644 dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_regmap.h - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/jm/mali_base_jm_kernel.h (75%) - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h (57%) - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/mali_base_kernel.h (72%) - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/mali_base_mem_priv.h (80%) - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/mali_kbase_hwcnt_reader.h (52%) - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/mali_kbase_ioctl.h (83%) - create mode 100644 dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_kinstr_jm_reader.h - rename dvalin/kernel/{drivers => include/uapi}/gpu/arm/midgard/mali_uk.h (69%) - -diff --git a/dvalin/kernel/Documentation/ABI/testing/sysfs-device-mali b/dvalin/kernel/Documentation/ABI/testing/sysfs-device-mali -new file mode 100644 -index 0000000..99f8ae5 ---- /dev/null -+++ b/dvalin/kernel/Documentation/ABI/testing/sysfs-device-mali -@@ -0,0 +1,293 @@ -+/* -+ * -+ * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation) and any use by you of this program is subject to the terms -+ * of such GNU licence. -+ * -+ * A copy of the licence is included with the program) and can also be obtained -+ * from Free Software Foundation) Inc.) 51 Franklin Street) Fifth Floor) -+ * Boston) MA 02110-1301) USA. -+ * -+ */ -+ -+What: /sys/class/misc/mali%u/device/core_mask -+Description: -+ This attribute is used to restrict the number of shader cores -+ available in this instance, is useful for debugging purposes. -+ Reading this attribute provides us mask of all cores available. -+ Writing to it will set the current core mask. Doesn't -+ allow disabling all the cores present in this instance. -+ -+What: /sys/class/misc/mali%u/device/debug_command -+Description: -+ This attribute is used to issue debug commands that supported -+ by the driver. On reading it provides the list of debug commands -+ that are supported, and writing back one of those commands will -+ enable that debug option. -+ -+What: /sys/class/misc/mali%u/device/dvfs_period -+Description: -+ This is used to set the DVFS sampling period to be used by the -+ driver, On reading it provides the current DVFS sampling period, -+ on writing a value we set the DVFS sampling period. -+ -+What: /sys/class/misc/mali%u/device/dummy_job_wa_info -+Description: -+ This attribute is available only with platform device that -+ supports a Job Manager based GPU that requires a GPU workaround -+ to execute the dummy fragment job on all shader cores to -+ workaround a hang issue. -+ -+ Its a readonly attribute and on reading gives details on the -+ options used with the dummy workaround. -+ -+What: /sys/class/misc/mali%u/device/fw_timeout -+Description: -+ This attribute is available only with mali platform -+ device-driver that supports a CSF GPU. This attribute is -+ used to set the duration value in milliseconds for the -+ waiting timeout used for a GPU status change request being -+ acknowledged by the FW. -+ -+What: /sys/class/misc/mali%u/device/gpuinfo -+Description: -+ This attribute provides description of the present Mali GPU. -+ Its a read only attribute provides details like GPU family, the -+ number of cores, the hardware version and the raw product id. -+ -+What: /sys/class/misc/mali%u/device/idle_hysteresis_time -+Description: -+ This attribute is available only with mali platform -+ device-driver that supports a CSF GPU. This attribute is -+ used to set the duration value in milliseconds for the -+ configuring hysteresis field for determining GPU idle detection. -+ -+What: /sys/class/misc/mali%u/device/js_ctx_scheduling_mode -+Description: -+ This attribute is available only with platform device that -+ supports a Job Manager based GPU. This attribute is used to set -+ context scheduling priority for a job slot. -+ -+ On Reading it provides the currently set job slot context -+ priority. -+ -+ Writing 0 to this attribute sets it to the mode were -+ higher priority atoms will be scheduled first, regardless of -+ the context they belong to. Newly-runnable higher priority atoms -+ can preempt lower priority atoms currently running on the GPU, -+ even if they belong to a different context. -+ -+ Writing 1 to this attribute set it to the mode were the -+ highest-priority atom will be chosen from each context in turn -+ using a round-robin algorithm, so priority only has an effect -+ within the context an atom belongs to. Newly-runnable higher -+ priority atoms can preempt the lower priority atoms currently -+ running on the GPU, but only if they belong to the same context. -+ -+What: /sys/class/misc/mali%u/device/js_scheduling_period -+Description: -+ This attribute is available only with platform device that -+ supports a Job Manager based GPU. Used to set the job scheduler -+ tick period in nano-seconds. The Job Scheduler determines the -+ jobs that are run on the GPU, and for how long, Job Scheduler -+ makes decisions at a regular time interval determined by value -+ in js_scheduling_period. -+ -+What: /sys/class/misc/mali%u/device/js_softstop_always -+Description: -+ This attribute is available only with platform device that -+ supports a Job Manager based GPU. Soft-stops are disabled when -+ only a single context is present, this attribute is used to -+ enable soft-stop when only a single context is present can be -+ used for debug and unit-testing purposes. -+ -+What: /sys/class/misc/mali%u/device/js_timeouts -+Description: -+ This attribute is available only with platform device that -+ supports a Job Manager based GPU. It used to set the soft stop -+ and hard stop times for the job scheduler. -+ -+ Writing value 0 causes no change, or -1 to restore the -+ default timeout. -+ -+ The format used to set js_timeouts is -+ "<soft_stop_ms> <soft_stop_ms_cl> <hard_stop_ms_ss> -+ <hard_stop_ms_cl> <hard_stop_ms_dumping> <reset_ms_ss> -+ <reset_ms_cl> <reset_ms_dumping>" -+ -+ -+What: /sys/class/misc/mali%u/device/lp_mem_pool_max_size -+Description: -+ This attribute is used to set the maximum number of large pages -+ memory pools that the driver can contain. Large pages are of -+ size 2MB. On read it displays all the max size of all memory -+ pools and can be used to modify each individual pools as well. -+ -+What: /sys/class/misc/mali%u/device/lp_mem_pool_size -+Description: -+ This attribute is used to set the number of large memory pages -+ which should be populated, changing this value may cause -+ existing pages to be removed from the pool, or new pages to be -+ created and then added to the pool. On read it will provide -+ pool size for all available pools and we can modify individual -+ pool. -+ -+What: /sys/class/misc/mali%u/device/mem_pool_max_size -+Description: -+ This attribute is used to set the maximum number of small pages -+ for memory pools that the driver can contain. Here small pages -+ are of size 4KB. On read it will display the max size for all -+ available pools and allows us to set max size of -+ individual pools. -+ -+What: /sys/class/misc/mali%u/device/mem_pool_size -+Description: -+ This attribute is used to set the number of small memory pages -+ which should be populated, changing this value may cause -+ existing pages to be removed from the pool, or new pages to -+ be created and then added to the pool. On read it will provide -+ pool size for all available pools and we can modify individual -+ pool. -+ -+What: /sys/class/misc/mali%u/device/device/mempool/ctx_default_max_size -+Description: -+ This attribute is used to set maximum memory pool size for -+ all the memory pool so that the maximum amount of free memory -+ that each pool can hold is identical. -+ -+What: /sys/class/misc/mali%u/device/device/mempool/lp_max_size -+Description: -+ This attribute is used to set the maximum number of large pages -+ for all memory pools that the driver can contain. -+ Large pages are of size 2MB. -+ -+What: /sys/class/misc/mali%u/device/device/mempool/max_size -+Description: -+ This attribute is used to set the maximum number of small pages -+ for all the memory pools that the driver can contain. -+ Here small pages are of size 4KB. -+ -+What: /sys/class/misc/mali%u/device/pm_poweroff -+Description: -+ This attribute contains the current values, represented as the -+ following space-separated integers: -+ • PM_GPU_POWEROFF_TICK_NS. -+ • PM_POWEROFF_TICK_SHADER. -+ • PM_POWEROFF_TICK_GPU. -+ -+ Example: -+ echo 100000 4 4 > /sys/class/misc/mali0/device/pm_poweroff -+ -+ Sets the following new values: 100,000ns tick, four ticks -+ for shader power down, and four ticks for GPU power down. -+ -+What: /sys/class/misc/mali%u/device/power_policy -+Description: -+ This attribute is used to find the current power policy been -+ used, reading will list the power policies available and -+ enclosed in square bracket is the current one been selected. -+ -+ Example: -+ cat /sys/class/misc/mali0/device/power_policy -+ [demand] coarse_demand always_on -+ -+ To switch to a different policy at runtime write the valid entry -+ name back to the attribute. -+ -+ Example: -+ echo "coarse_demand" > /sys/class/misc/mali0/device/power_policy -+ -+What: /sys/class/misc/mali%u/device/progress_timeout -+Description: -+ This attribute is available only with mali platform -+ device-driver that supports a CSF GPU. This attribute -+ is used to set the progress timeout value and read the current -+ progress timeout value. -+ -+ Progress timeout value is the maximum number of GPU cycles -+ without forward progress to allow to elapse before terminating a -+ GPU command queue group. -+ -+What: /sys/class/misc/mali%u/device/reset_timeout -+Description: -+ This attribute is used to set the number of milliseconds to -+ wait for the soft stop to complete for the GPU jobs before -+ proceeding with the GPU reset. -+ -+What: /sys/class/misc/mali%u/device/soft_job_timeout -+Description: -+ This attribute is available only with platform device that -+ supports a Job Manager based GPU. It used to set the timeout -+ value for waiting for any soft event to complete. -+ -+What: /sys/class/misc/mali%u/device/scheduling/serialize_jobs -+Description: -+ This attribute is available only with platform device that -+ supports a Job Manager based GPU. -+ -+ Various options available under this are: -+ • none - for disabling serialization. -+ • intra-slot - Serialize atoms within a slot, only one -+ atom per job slot. -+ • inter-slot - Serialize atoms between slots, only one -+ job slot running at any time. -+ • full - it a combination of both inter and intra slot, -+ so only one atom and one job slot running -+ at any time. -+ • full-reset - full serialization and Reset the GPU after -+ each atom completion -+ -+ These options are useful for debugging and investigating -+ failures and gpu hangs to narrow down atoms that could cause -+ troubles. -+ -+What: /sys/class/misc/mali%u/device/firmware_config/Compute iterator count/* -+Description: -+ This attribute is available only with mali platform -+ device-driver that supports a CSF GPU. Its a read-only attribute -+ which indicates the maximum number of Compute iterators -+ supported by the GPU. -+ -+What: /sys/class/misc/mali%u/device/firmware_config/CSHWIF count/* -+Description: -+ This attribute is available only with mali platform -+ device-driver that supports a CSF GPU. Its a read-only -+ attribute which indicates the maximum number of CSHWIFs -+ supported by the GPU. -+ -+What: /sys/class/misc/mali%u/device/firmware_config/Fragment iterator count/* -+Description: -+ This attribute is available only with mali platform -+ device-driver that supports a CSF GPU. Its a read-only -+ attribute which indicates the maximum number of -+ Fragment iterators supported by the GPU. -+ -+What: /sys/class/misc/mali%u/device/firmware_config/Scoreboard set count/* -+Description: -+ This attribute is available only with mali platform -+ device-driver that supports a CSF GPU. Its a read-only -+ attribute which indicates the maximum number of -+ Scoreboard set supported by the GPU. -+ -+What: /sys/class/misc/mali%u/device/firmware_config/Tiler iterator count/* -+Description: -+ This attribute is available only with mali platform -+ device-driver that supports a CSF GPU. Its a read-only -+ attribute which indicates the maximum number of Tiler iterators -+ supported by the GPU. -+ -+What: /sys/class/misc/mali%u/device/firmware_config/Log verbosity/* -+Description: -+ This attribute is available only with mali platform -+ device-driver that supports a CSF GPU. -+ -+ Used to enable firmware logs, logging levels valid values -+ are indicated using 'min and 'max' attribute values -+ values that are read-only. -+ -+ Log level can be set using the 'cur' read, write attribute, -+ we can use a valid log level value from min and max range values -+ and set a valid desired log level for firmware logs. -diff --git a/dvalin/kernel/Documentation/devicetree/bindings/arm/mali-midgard.txt b/dvalin/kernel/Documentation/devicetree/bindings/arm/mali-midgard.txt -index dd8f733..a74d569 100644 ---- a/dvalin/kernel/Documentation/devicetree/bindings/arm/mali-midgard.txt -+++ b/dvalin/kernel/Documentation/devicetree/bindings/arm/mali-midgard.txt -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2013-2020 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2013-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,8 +16,6 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - * ARM Mali Midgard / Bifrost devices -@@ -46,12 +45,12 @@ Documentation/devicetree/bindings/regulator/regulator.txt for details. - This is optional. - - operating-points-v2 : Refer to Documentation/devicetree/bindings/power/mali-opp.txt - for details. --- quirks_jm : Used to write to the JM_CONFIG register or equivalent. -+- quirks_gpu : Used to write to the JM_CONFIG or CSF_CONFIG register. - Should be used with care. Options passed here are used to override - certain default behavior. Note: This will override 'idvs-group-size' - field in devicetree and module param 'corestack_driver_control', -- therefore if 'quirks_jm' is used then 'idvs-group-size' and -- 'corestack_driver_control' value should be incorporated into 'quirks_jm'. -+ therefore if 'quirks_gpu' is used then 'idvs-group-size' and -+ 'corestack_driver_control' value should be incorporated into 'quirks_gpu'. - - quirks_sc : Used to write to the SHADER_CONFIG register. - Should be used with care. Options passed here are used to override - certain default behavior. -@@ -64,8 +63,8 @@ for details. - - power_model : Sets the power model parameters. Defined power models include: - "mali-simple-power-model", "mali-g51-power-model", "mali-g52-power-model", - "mali-g52_r1-power-model", "mali-g71-power-model", "mali-g72-power-model", -- "mali-g76-power-model", "mali-g77-power-model", "mali-tnax-power-model" -- and "mali-tbex-power-model". -+ "mali-g76-power-model", "mali-g77-power-model", "mali-tnax-power-model", -+ "mali-tbex-power-model" and "mali-tbax-power-model". - - mali-simple-power-model: this model derives the GPU power usage based - on the GPU voltage scaled by the system temperature. Note: it was - designed for the Juno platform, and may not be suitable for others. -@@ -98,6 +97,8 @@ for details. - are used at different points so care should be taken to configure - both power models in the device tree (specifically dynamic-coefficient, - static-coefficient and scale) to best match the platform. -+- power_policy : Sets the GPU power policy at probe time. Available options are -+ "coarse_demand" and "always_on". If not set, then "coarse_demand" is used. - - system-coherency : Sets the coherency protocol to be used for coherent - accesses made from the GPU. - If not set then no coherency is used. -diff --git a/dvalin/kernel/Documentation/devicetree/bindings/arm/memory_group_manager.txt b/dvalin/kernel/Documentation/devicetree/bindings/arm/memory_group_manager.txt -index fda8f00..634973f 100644 ---- a/dvalin/kernel/Documentation/devicetree/bindings/arm/memory_group_manager.txt -+++ b/dvalin/kernel/Documentation/devicetree/bindings/arm/memory_group_manager.txt -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,8 +16,6 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - * Arm memory group manager for Mali GPU device drivers -diff --git a/dvalin/kernel/Documentation/devicetree/bindings/arm/priority_control_manager.txt b/dvalin/kernel/Documentation/devicetree/bindings/arm/priority_control_manager.txt -new file mode 100644 -index 0000000..c7dd14f ---- /dev/null -+++ b/dvalin/kernel/Documentation/devicetree/bindings/arm/priority_control_manager.txt -@@ -0,0 +1,48 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+* Arm priority control manager for Mali GPU device drivers -+ -+Required properties: -+ -+- compatible: Must be "arm,priority-control-manager" -+ -+An example node: -+ -+ gpu_priority_control_manager: priority-control-manager { -+ compatible = "arm,priority-control-manager"; -+ }; -+ -+It must be referenced by the GPU as well, see priority-control-manager: -+ -+ gpu: gpu@0x6e000000 { -+ compatible = "arm,mali-midgard"; -+ reg = <0x0 0x6e000000 0x0 0x200000>; -+ interrupts = <0 168 4>, <0 168 4>, <0 168 4>; -+ interrupt-names = "JOB", "MMU", "GPU"; -+ clocks = <&scpi_dvfs 2>; -+ clock-names = "clk_mali"; -+ system-coherency = <31>; -+ priority-control-manager = <&gpu_priority_control_manager>; -+ operating-points = < -+ /* KHz uV */ -+ 50000 820000 -+ >; -+ }; -diff --git a/dvalin/kernel/Documentation/devicetree/bindings/arm/protected_memory_allocator.txt b/dvalin/kernel/Documentation/devicetree/bindings/arm/protected_memory_allocator.txt -index f054348..89a3cc7 100644 ---- a/dvalin/kernel/Documentation/devicetree/bindings/arm/protected_memory_allocator.txt -+++ b/dvalin/kernel/Documentation/devicetree/bindings/arm/protected_memory_allocator.txt -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,8 +16,6 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - * Arm protected memory allocator for Mali GPU device drivers -diff --git a/dvalin/kernel/Documentation/devicetree/bindings/power/mali-opp.txt b/dvalin/kernel/Documentation/devicetree/bindings/power/mali-opp.txt -index 49ed773..b9c0743 100644 ---- a/dvalin/kernel/Documentation/devicetree/bindings/power/mali-opp.txt -+++ b/dvalin/kernel/Documentation/devicetree/bindings/power/mali-opp.txt -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2017, 2019 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2017, 2019-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,8 +16,6 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - * ARM Mali Midgard OPP -@@ -54,7 +53,7 @@ Optional properties: - - - opp-core-count: Number of cores to use for this OPP. If this is present then - the driver will build a core mask using the available core mask provided by -- the GPU hardware. -+ the GPU hardware. An opp-core-count value of 0 is not permitted. - - If neither this nor opp-core-mask are present then all shader cores will be - used for this OPP. -diff --git a/dvalin/kernel/Documentation/dma-buf-test-exporter.txt b/dvalin/kernel/Documentation/dma-buf-test-exporter.txt -index 8d8cbc9..b01020c 100644 ---- a/dvalin/kernel/Documentation/dma-buf-test-exporter.txt -+++ b/dvalin/kernel/Documentation/dma-buf-test-exporter.txt -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012-2013 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2013, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,10 +16,7 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # --# -- - - ===================== - dma-buf-test-exporter -@@ -42,5 +40,3 @@ It supports being compiled as a module both in-tree and out-of-tree. - - See include/linux/dma-buf-test-exporter.h for the ioctl interface. - See Documentation/dma-buf-sharing.txt for details on dma_buf. -- -- -diff --git a/dvalin/kernel/Mconfig b/dvalin/kernel/Mconfig -index e451591..217715c 100644 ---- a/dvalin/kernel/Mconfig -+++ b/dvalin/kernel/Mconfig -@@ -1,27 +1,26 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# --# (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2017-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. - # --# A copy of the licence is included with the program, and can also be obtained --# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, --# Boston, MA 02110-1301, USA. -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. - # - # - --source "kernel/drivers/gpu/arm/midgard/Mconfig" --source "kernel/drivers/gpu/arm/midgard/arbitration/Mconfig" -+menu "Kernel menu" - --config DMA_BUF_SYNC_IOCTL_SUPPORTED -- bool "Kernel DMA buffers support DMA_BUF_IOCTL_SYNC" -- depends on BACKEND_KERNEL -- default y -+source "kernel/drivers/base/arm/Mconfig" -+source "kernel/drivers/gpu/arm/midgard/Mconfig" - --config BUILD_CSF_ONLY_MODULE -- bool "Build CSF GPU specific kernel modules" -- depends on BUILD_KERNEL_MODULES && GPU_HAS_CSF -- default y -+endmenu -diff --git a/dvalin/kernel/build.bp b/dvalin/kernel/build.bp -index 2bc725f..c97da2c 100644 ---- a/dvalin/kernel/build.bp -+++ b/dvalin/kernel/build.bp -@@ -1,15 +1,21 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2016-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * -- * A copy of the licence is included with the program, and can also be obtained -- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- * Boston, MA 02110-1301, USA. -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -@@ -25,6 +31,7 @@ bob_install_group { - - bob_defaults { - name: "kernel_defaults", -+ build_by_default: false, - enabled: false, - exclude_srcs: [ - "**/*.mod.c", -@@ -33,6 +40,7 @@ bob_defaults { - "include", - ], - build_kernel_modules: { -+ build_by_default: true, - enabled: true, - kernel_dir: "{{.kernel_dir}}", - kernel_cross_compile: "{{.kernel_compiler}}", -@@ -42,6 +50,8 @@ bob_defaults { - kernel_ld: "{{.kernel_ld}}", - }, - install_group: "IG_kernel_modules", -+ add_to_alias: ["kernel"], -+ owner: "{{.android_module_owner}}", - cflags: [ - "-Wall", - ], -@@ -54,49 +64,12 @@ bob_defaults { - "optional", - ], - }, -- kbuild_options: [ -- // Start of CS experimental features definitions. -- // If there is nothing below, definition should be added as follows: -- // "MALI_EXPERIMENTAL_FEATURE={{.experimental_feature}}" -- // experimental_feature above comes from Mconfig in -- // <ddk_root>/product/base/ -- // However, in Mconfig, experimental_feature should be looked up (for -- // similar explanation to this one) as ALLCAPS, i.e. -- // EXPERIMENTAL_FEATURE. -- // -- // IMPORTANT: MALI_CS_EXPERIMENTAL should NEVER be defined below as it -- // is an umbrella feature that would be open for inappropriate use -- // (catch-all for experimental CS code without separating it into -- // different features). -- "MALI_JIT_PRESSURE_LIMIT={{.jit_pressure_limit}}", -- "MALI_INCREMENTAL_RENDERING={{.incremental_rendering}}", -- ], --} -- --bob_defaults { -- name: "kutf_includes", -- local_include_dirs: [ -- "drivers/gpu/arm/midgard/tests/include", -- ], --} -- --bob_defaults { -- name: "kernel_test_includes", -- defaults: ["kutf_includes"], -- local_include_dirs: [ -- "drivers/gpu/arm", -- "drivers/gpu/arm/midgard", -- "drivers/gpu/arm/midgard/backend/gpu", -- "drivers/gpu/arm/midgard/debug", -- "drivers/gpu/arm/midgard/debug/backend", -- ], - } - -+// Alias for all kernel modules. `kernel_defaults` uses `add_to_alias` to -+// ensure any modules using that are included in this alias; any -+// bob_kernel_modules not using those defaults must explicitly use -+// `add_to_alias` or be listed here. - bob_alias { - name: "kernel", -- srcs: [ -- "dma-buf-test-exporter", -- "memory_group_manager", -- "mali_kbase", -- ], - } -diff --git a/dvalin/kernel/drivers/base/arm/Kbuild b/dvalin/kernel/drivers/base/arm/Kbuild -new file mode 100644 -index 0000000..b0fbf93 ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/Kbuild -@@ -0,0 +1,34 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+# -+# ccflags -+# -+ccflags-y += -I$(src)/../../../include -+ -+subdir-ccflags-y += $(ccflags-y) -+ -+# -+# Kernel modules -+# -+obj-$(CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER) += dma_buf_test_exporter/ -+obj-$(CONFIG_MALI_MEMORY_GROUP_MANAGER) += memory_group_manager/ -+obj-$(CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR) += protected_memory_allocator/ -+ -diff --git a/dvalin/kernel/drivers/base/arm/Kconfig b/dvalin/kernel/drivers/base/arm/Kconfig -new file mode 100644 -index 0000000..75d5434 ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/Kconfig -@@ -0,0 +1,64 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+menuconfig MALI_BASE_MODULES -+ bool "Mali Base extra modules" -+ default n -+ help -+ Enable this option to build support for a Arm Mali base modules. -+ Those modules provide extra features or debug interfaces and, -+ are optional for the use of the Mali GPU modules. -+ -+config DMA_SHARED_BUFFER_TEST_EXPORTER -+ bool "Build dma-buf framework test exporter module" -+ depends on MALI_BASE_MODULES && DMA_SHARED_BUFFER -+ default y -+ help -+ This option will build the dma-buf framework test exporter module. -+ Usable to help test importers. -+ -+ Modules: -+ - dma-buf-test-exporter.ko -+ -+config MALI_MEMORY_GROUP_MANAGER -+ bool "Build Mali Memory Group Manager module" -+ depends on MALI_BASE_MODULES -+ default y -+ help -+ This option will build the memory group manager module. -+ This is an example implementation for allocation and release of pages -+ for memory pools managed by Mali GPU device drivers. -+ -+ Modules: -+ - memory_group_manager.ko -+ -+config MALI_PROTECTED_MEMORY_ALLOCATOR -+ bool "Build Mali Protected Memory Allocator module" -+ depends on MALI_BASE_MODULES && MALI_CSF_SUPPORT -+ default y -+ help -+ This option will build the protected memory allocator module. -+ This is an example implementation for allocation and release of pages -+ of secure memory intended to be used by the firmware -+ of Mali GPU device drivers. -+ -+ Modules: -+ - protected_memory_allocator.ko -+ -diff --git a/dvalin/kernel/drivers/base/arm/Makefile b/dvalin/kernel/drivers/base/arm/Makefile -new file mode 100644 -index 0000000..0bd6ab5 ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/Makefile -@@ -0,0 +1,98 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+# -+# Paths -+# -+KERNEL_SRC ?= /lib/modules/$(shell uname -r)/build -+KDIR ?= $(KERNEL_SRC) -+ -+ifeq ($(KDIR),) -+ $(error Must specify KDIR to point to the kernel to target)) -+endif -+ -+vars := -+# -+# Default configuration values -+# -+CONFIG_MALI_BASE_MODULES ?= n -+ -+ifeq ($(CONFIG_MALI_BASE_MODULES),y) -+ CONFIG_MALI_CSF_SUPPORT ?= n -+ -+ ifneq ($(CONFIG_DMA_SHARED_BUFFER),n) -+ CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER ?= y -+ else -+ # Prevent misuse when CONFIG_DMA_SHARED_BUFFER=n -+ CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER = n -+ endif -+ -+ CONFIG_MALI_MEMORY_GROUP_MANAGER ?= y -+ -+ ifneq ($(CONFIG_MALI_CSF_SUPPORT), n) -+ CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR ?= y -+ endif -+ -+else -+ # Prevent misuse when CONFIG_MALI_BASE_MODULES=n -+ CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER = n -+ CONFIG_MALI_MEMORY_GROUP_MANAGER = n -+ CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR = n -+ -+endif -+ -+CONFIGS := \ -+ CONFIG_MALI_BASE_MODULES \ -+ CONFIG_MALI_CSF_SUPPORT \ -+ CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER \ -+ CONFIG_MALI_MEMORY_GROUP_MANAGER \ -+ CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR -+ -+ -+# -+# MAKE_ARGS to pass the custom CONFIGs on out-of-tree build -+# -+# Generate the list of CONFIGs and values. -+# $(value config) is the name of the CONFIG option. -+# $(value $(value config)) is its value (y, m). -+# When the CONFIG is not set to y or m, it defaults to n. -+MAKE_ARGS := $(foreach config,$(CONFIGS), \ -+ $(if $(filter y m,$(value $(value config))), \ -+ $(value config)=$(value $(value config)), \ -+ $(value config)=n)) -+ -+# -+# EXTRA_CFLAGS to define the custom CONFIGs on out-of-tree build -+# -+# Generate the list of CONFIGs defines with values from CONFIGS. -+# $(value config) is the name of the CONFIG option. -+# When set to y or m, the CONFIG gets defined to 1. -+EXTRA_CFLAGS := $(foreach config,$(CONFIGS), \ -+ $(if $(filter y m,$(value $(value config))), \ -+ -D$(value config)=1)) -+ -+all: -+ $(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) EXTRA_CFLAGS="$(EXTRA_CFLAGS)" KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules -+ -+modules_install: -+ $(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) modules_install -+ -+clean: -+ $(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) clean -diff --git a/dvalin/kernel/drivers/base/arm/Mconfig b/dvalin/kernel/drivers/base/arm/Mconfig -new file mode 100644 -index 0000000..d21a72e ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/Mconfig -@@ -0,0 +1,64 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+menuconfig MALI_BASE_MODULES -+ bool "Mali Base extra modules" -+ default y if BACKEND_KERNEL -+ help -+ Enable this option to build support for a Arm Mali base modules. -+ Those modules provide extra features or debug interfaces and, -+ are optional for the use of the Mali GPU modules. -+ -+config DMA_SHARED_BUFFER_TEST_EXPORTER -+ bool "Build dma-buf framework test exporter module" -+ depends on MALI_BASE_MODULES -+ default y -+ help -+ This option will build the dma-buf framework test exporter module. -+ Usable to help test importers. -+ -+ Modules: -+ - dma-buf-test-exporter.ko -+ -+config MALI_MEMORY_GROUP_MANAGER -+ bool "Build Mali Memory Group Manager module" -+ depends on MALI_BASE_MODULES -+ default y -+ help -+ This option will build the memory group manager module. -+ This is an example implementation for allocation and release of pages -+ for memory pools managed by Mali GPU device drivers. -+ -+ Modules: -+ - memory_group_manager.ko -+ -+config MALI_PROTECTED_MEMORY_ALLOCATOR -+ bool "Build Mali Protected Memory Allocator module" -+ depends on MALI_BASE_MODULES && GPU_HAS_CSF -+ default y -+ help -+ This option will build the protected memory allocator module. -+ This is an example implementation for allocation and release of pages -+ of secure memory intended to be used by the firmware -+ of Mali GPU device drivers. -+ -+ Modules: -+ - protected_memory_allocator.ko -+ -diff --git a/dvalin/kernel/drivers/base/dma_buf_lock/src/Kbuild b/dvalin/kernel/drivers/base/arm/dma_buf_lock/src/Kbuild -similarity index 78% -rename from dvalin/kernel/drivers/base/dma_buf_lock/src/Kbuild -rename to dvalin/kernel/drivers/base/arm/dma_buf_lock/src/Kbuild -index ddf1bb5..c7ae332 100644 ---- a/dvalin/kernel/drivers/base/dma_buf_lock/src/Kbuild -+++ b/dvalin/kernel/drivers/base/arm/dma_buf_lock/src/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,10 +16,8 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - --ifneq ($(CONFIG_DMA_SHARED_BUFFER),) -+ifeq ($(CONFIG_DMA_SHARED_BUFFER), y) - obj-m := dma_buf_lock.o - endif -diff --git a/dvalin/kernel/drivers/base/dma_buf_lock/src/Makefile b/dvalin/kernel/drivers/base/arm/dma_buf_lock/src/Makefile -similarity index 71% -rename from dvalin/kernel/drivers/base/dma_buf_lock/src/Makefile -rename to dvalin/kernel/drivers/base/arm/dma_buf_lock/src/Makefile -index 3b10406..451d2d7 100644 ---- a/dvalin/kernel/drivers/base/dma_buf_lock/src/Makefile -+++ b/dvalin/kernel/drivers/base/arm/dma_buf_lock/src/Makefile -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,8 +16,6 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - # linux build system bootstrap for out-of-tree module -@@ -24,15 +23,17 @@ - # default to building for the host - ARCH ?= $(shell uname -m) - --ifeq ($(KDIR),) --$(error Must specify KDIR to point to the kernel to target)) --endif -+# Handle Android Common Kernel source naming -+KERNEL_SRC ?= /lib/modules/$(shell uname -r)/build -+KDIR ?= $(KERNEL_SRC) - - all: dma_buf_lock - - dma_buf_lock: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include" -+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../../include" - - clean: - $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean - -+modules_install: -+ $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) modules_install -diff --git a/dvalin/kernel/drivers/base/dma_buf_lock/src/dma_buf_lock.c b/dvalin/kernel/drivers/base/arm/dma_buf_lock/src/dma_buf_lock.c -similarity index 90% -rename from dvalin/kernel/drivers/base/dma_buf_lock/src/dma_buf_lock.c -rename to dvalin/kernel/drivers/base/arm/dma_buf_lock/src/dma_buf_lock.c -index 529ce71..f5ab1ed 100644 ---- a/dvalin/kernel/drivers/base/dma_buf_lock/src/dma_buf_lock.c -+++ b/dvalin/kernel/drivers/base/arm/dma_buf_lock/src/dma_buf_lock.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2013, 2017-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2014, 2017-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/version.h> -@@ -29,7 +28,11 @@ - #include <linux/device.h> - #include <linux/slab.h> - #include <linux/atomic.h> -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - #include <linux/reservation.h> -+#else -+#include <linux/dma-resv.h> -+#endif - #include <linux/dma-buf.h> - #include <linux/wait.h> - #include <linux/sched.h> -@@ -37,7 +40,7 @@ - #include <linux/anon_inodes.h> - #include <linux/file.h> - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - - #include <linux/fence.h> - -@@ -60,7 +63,7 @@ - - #include <linux/dma-fence.h> - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) -+#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) - #define dma_fence_get_status(a) (dma_fence_is_signaled(a) ? \ - (a)->status ?: 1 \ - : 0) -@@ -101,7 +104,7 @@ static struct file_operations dma_buf_lock_fops = - - typedef struct dma_buf_lock_resource - { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence fence; - #else - struct dma_fence fence; -@@ -127,7 +130,7 @@ typedef struct dma_buf_lock_resource - * @node: List head for linking this callback to the lock resource - */ - struct dma_buf_lock_fence_cb { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence_cb fence_cb; - struct fence *fence; - #else -@@ -151,7 +154,7 @@ static void dma_buf_lock_dounlock(struct kref *ref); - static DEFINE_SPINLOCK(dma_buf_lock_fence_lock); - - static const char * --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - dma_buf_lock_fence_get_driver_name(struct fence *fence) - #else - dma_buf_lock_fence_get_driver_name(struct dma_fence *fence) -@@ -161,7 +164,7 @@ dma_buf_lock_fence_get_driver_name(struct dma_fence *fence) - } - - static const char * --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - dma_buf_lock_fence_get_timeline_name(struct fence *fence) - #else - dma_buf_lock_fence_get_timeline_name(struct dma_fence *fence) -@@ -171,7 +174,7 @@ dma_buf_lock_fence_get_timeline_name(struct dma_fence *fence) - } - - static bool --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - dma_buf_lock_fence_enable_signaling(struct fence *fence) - #else - dma_buf_lock_fence_enable_signaling(struct dma_fence *fence) -@@ -180,7 +183,7 @@ dma_buf_lock_fence_enable_signaling(struct dma_fence *fence) - return true; - } - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - const struct fence_ops dma_buf_lock_fence_ops = { - .wait = fence_default_wait, - #else -@@ -235,7 +238,7 @@ dma_buf_lock_fence_work(struct work_struct *pwork) - } - - static void --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - dma_buf_lock_fence_callback(struct fence *fence, struct fence_cb *cb) - #else - dma_buf_lock_fence_callback(struct dma_fence *fence, struct dma_fence_cb *cb) -@@ -256,14 +259,13 @@ dma_buf_lock_fence_callback(struct dma_fence *fence, struct dma_fence_cb *cb) - atomic_set(&resource->locked, 1); - wake_up(&resource->wait); - -- if (resource->exclusive) { -+ if (resource->exclusive) - /* Warn if the work was already queued */ - WARN_ON(!schedule_work(&resource->work)); -- } - } - } - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - static int - dma_buf_lock_fence_add_callback(dma_buf_lock_resource *resource, - struct fence *fence, -@@ -317,12 +319,19 @@ dma_buf_lock_fence_add_callback(dma_buf_lock_resource *resource, - return err; - } - -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - static int - dma_buf_lock_add_fence_reservation_callback(dma_buf_lock_resource *resource, - struct reservation_object *resv, - bool exclusive) -+#else -+static int -+dma_buf_lock_add_fence_reservation_callback(dma_buf_lock_resource *resource, -+ struct dma_resv *resv, -+ bool exclusive) -+#endif - { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *excl_fence = NULL; - struct fence **shared_fences = NULL; - #else -@@ -332,7 +341,12 @@ dma_buf_lock_add_fence_reservation_callback(dma_buf_lock_resource *resource, - unsigned int shared_count = 0; - int err, i; - -- err = reservation_object_get_fences_rcu(resv, -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) -+ err = reservation_object_get_fences_rcu( -+#else -+ err = dma_resv_get_fences_rcu( -+#endif -+ resv, - &excl_fence, - &shared_count, - &shared_fences); -@@ -393,7 +407,11 @@ static int - dma_buf_lock_acquire_fence_reservation(dma_buf_lock_resource *resource, - struct ww_acquire_ctx *ctx) - { -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - struct reservation_object *content_resv = NULL; -+#else -+ struct dma_resv *content_resv = NULL; -+#endif - unsigned int content_resv_idx = 0; - unsigned int r; - int err = 0; -@@ -472,21 +490,16 @@ static unsigned int dma_buf_lock_handle_poll(struct file *file, - #if DMA_BUF_LOCK_DEBUG - printk("dma_buf_lock_handle_poll\n"); - #endif -- if (1 == atomic_read(&resource->locked)) -- { -+ if (atomic_read(&resource->locked) == 1) { - /* Resources have been locked */ - ret = POLLIN | POLLRDNORM; - if (resource->exclusive) -- { - ret |= POLLOUT | POLLWRNORM; -- } - } - else - { -- if (!poll_does_not_wait(wait)) -- { -+ if (!poll_does_not_wait(wait)) - poll_wait(file, &resource->wait, wait); -- } - } - #if DMA_BUF_LOCK_DEBUG - printk("dma_buf_lock_handle_poll : return %i\n", ret); -@@ -525,29 +538,19 @@ static int dma_buf_lock_dolock(dma_buf_lock_k_request *request) - int i; - int ret; - -- if (NULL == request->list_of_dma_buf_fds) -- { -+ if (request->list_of_dma_buf_fds == NULL) - return -EINVAL; -- } - if (request->count <= 0) -- { - return -EINVAL; -- } - if (request->count > DMA_BUF_LOCK_BUF_MAX) -- { - return -EINVAL; -- } - if (request->exclusive != DMA_BUF_LOCK_NONEXCLUSIVE && - request->exclusive != DMA_BUF_LOCK_EXCLUSIVE) -- { - return -EINVAL; -- } - - resource = kzalloc(sizeof(dma_buf_lock_resource), GFP_KERNEL); -- if (NULL == resource) -- { -+ if (resource == NULL) - return -ENOMEM; -- } - - atomic_set(&resource->locked, 0); - kref_init(&resource->refcount); -@@ -559,8 +562,7 @@ static int dma_buf_lock_dolock(dma_buf_lock_k_request *request) - size = request->count * sizeof(int); - resource->list_of_dma_buf_fds = kmalloc(size, GFP_KERNEL); - -- if (NULL == resource->list_of_dma_buf_fds) -- { -+ if (resource->list_of_dma_buf_fds == NULL) { - kfree(resource); - return -ENOMEM; - } -@@ -569,8 +571,7 @@ static int dma_buf_lock_dolock(dma_buf_lock_k_request *request) - size = sizeof(struct dma_buf *) * request->count; - resource->dma_bufs = kmalloc(size, GFP_KERNEL); - -- if (NULL == resource->dma_bufs) -- { -+ if (resource->dma_bufs == NULL) { - kfree(resource->list_of_dma_buf_fds); - kfree(resource); - return -ENOMEM; -@@ -578,8 +579,9 @@ static int dma_buf_lock_dolock(dma_buf_lock_k_request *request) - - /* Copy requested list of dma_buf_fds from user space */ - size = request->count * sizeof(int); -- if (0 != copy_from_user(resource->list_of_dma_buf_fds, (void __user *)request->list_of_dma_buf_fds, size)) -- { -+ if (copy_from_user(resource->list_of_dma_buf_fds, -+ (void __user *)request->list_of_dma_buf_fds, -+ size) != 0) { - kfree(resource->list_of_dma_buf_fds); - kfree(resource->dma_bufs); - kfree(resource); -@@ -587,9 +589,7 @@ static int dma_buf_lock_dolock(dma_buf_lock_k_request *request) - } - #if DMA_BUF_LOCK_DEBUG - for (i = 0; i < request->count; i++) -- { - printk("dma_buf %i = %X\n", i, resource->list_of_dma_buf_fds[i]); -- } - #endif - - /* Initialize the fence associated with dma_buf_lock resource */ -@@ -620,8 +620,7 @@ static int dma_buf_lock_dolock(dma_buf_lock_k_request *request) - } - - /*Check the reservation object associated with dma_buf */ -- if (NULL == resource->dma_bufs[i]->resv) -- { -+ if (resource->dma_bufs[i]->resv == NULL) { - mutex_lock(&dma_buf_lock_mutex); - kref_put(&resource->refcount, dma_buf_lock_dounlock); - mutex_unlock(&dma_buf_lock_mutex); -@@ -680,10 +679,18 @@ static int dma_buf_lock_dolock(dma_buf_lock_k_request *request) - kref_get(&resource->refcount); - - for (i = 0; i < request->count; i++) { -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - struct reservation_object *resv = resource->dma_bufs[i]->resv; -- -+#else -+ struct dma_resv *resv = resource->dma_bufs[i]->resv; -+#endif - if (!test_bit(i, &resource->exclusive)) { -+ -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - ret = reservation_object_reserve_shared(resv); -+#else -+ ret = dma_resv_reserve_shared(resv, 0); -+#endif - if (ret) { - #if DMA_BUF_LOCK_DEBUG - printk(KERN_DEBUG "dma_buf_lock_dolock : Error %d reserving space for shared fence.\n", ret); -@@ -701,7 +708,11 @@ static int dma_buf_lock_dolock(dma_buf_lock_k_request *request) - break; - } - -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - reservation_object_add_shared_fence(resv, &resource->fence); -+#else -+ dma_resv_add_shared_fence(resv, &resource->fence); -+#endif - } else { - ret = dma_buf_lock_add_fence_reservation_callback(resource, - resv, -@@ -713,7 +724,11 @@ static int dma_buf_lock_dolock(dma_buf_lock_k_request *request) - break; - } - -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - reservation_object_add_excl_fence(resv, &resource->fence); -+#else -+ dma_resv_add_excl_fence(resv, &resource->fence); -+#endif - } - } - -@@ -783,27 +798,21 @@ static int __init dma_buf_lock_init(void) - #endif - err = alloc_chrdev_region(&dma_buf_lock_dev, 0, 1, dma_buf_lock_dev_name); - -- if (0 == err) -- { -+ if (err == 0) { - cdev_init(&dma_buf_lock_cdev, &dma_buf_lock_fops); - - err = cdev_add(&dma_buf_lock_cdev, dma_buf_lock_dev, 1); - -- if (0 == err) -- { -+ if (err == 0) { - dma_buf_lock_class = class_create(THIS_MODULE, dma_buf_lock_dev_name); - if (IS_ERR(dma_buf_lock_class)) -- { - err = PTR_ERR(dma_buf_lock_class); -- } - else - { - struct device *mdev; - mdev = device_create(dma_buf_lock_class, NULL, dma_buf_lock_dev, NULL, dma_buf_lock_dev_name); - if (!IS_ERR(mdev)) -- { - return 0; -- } - - err = PTR_ERR(mdev); - class_destroy(dma_buf_lock_class); -@@ -836,7 +845,7 @@ static void __exit dma_buf_lock_exit(void) - } - else - { -- dma_buf_lock_resource *resource = list_entry(dma_buf_lock_resource_list.next, -+ dma_buf_lock_resource *resource = list_entry(dma_buf_lock_resource_list.next, - dma_buf_lock_resource, link); - kref_put(&resource->refcount, dma_buf_lock_dounlock); - mutex_unlock(&dma_buf_lock_mutex); -@@ -862,26 +871,17 @@ static int dma_buf_lock_ioctl(struct inode *inode, struct file *filp, unsigned i - int size = _IOC_SIZE(cmd); - - if (_IOC_TYPE(cmd) != DMA_BUF_LOCK_IOC_MAGIC) -- { - return -ENOTTY; -- -- } - if ((_IOC_NR(cmd) < DMA_BUF_LOCK_IOC_MINNR) || (_IOC_NR(cmd) > DMA_BUF_LOCK_IOC_MAXNR)) -- { - return -ENOTTY; -- } - - switch (cmd) - { - case DMA_BUF_LOCK_FUNC_LOCK_ASYNC: - if (size != sizeof(dma_buf_lock_k_request)) -- { - return -ENOTTY; -- } - if (copy_from_user(&request, (void __user *)arg, size)) -- { - return -EFAULT; -- } - #if DMA_BUF_LOCK_DEBUG - printk("DMA_BUF_LOCK_FUNC_LOCK_ASYNC - %i\n", request.count); - #endif -diff --git a/dvalin/kernel/drivers/base/dma_buf_lock/src/dma_buf_lock.h b/dvalin/kernel/drivers/base/arm/dma_buf_lock/src/dma_buf_lock.h -similarity index 88% -rename from dvalin/kernel/drivers/base/dma_buf_lock/src/dma_buf_lock.h -rename to dvalin/kernel/drivers/base/arm/dma_buf_lock/src/dma_buf_lock.h -index f2ae575..104af1f 100644 ---- a/dvalin/kernel/drivers/base/dma_buf_lock/src/dma_buf_lock.h -+++ b/dvalin/kernel/drivers/base/arm/dma_buf_lock/src/dma_buf_lock.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _DMA_BUF_LOCK_H -diff --git a/dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/Kbuild b/dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/Kbuild -new file mode 100644 -index 0000000..0e20cb4 ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/Kbuild -@@ -0,0 +1,23 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+ifeq ($(CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER), y) -+obj-m += dma-buf-test-exporter.o -+endif -diff --git a/dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/build.bp b/dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/build.bp -new file mode 100644 -index 0000000..a49fb81 ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/build.bp -@@ -0,0 +1,36 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+bob_kernel_module { -+ name: "dma-buf-test-exporter", -+ defaults: [ -+ "kernel_defaults" -+ ], -+ srcs: [ -+ "Kbuild", -+ "dma-buf-test-exporter.c", -+ ], -+ enabled: false, -+ dma_shared_buffer_test_exporter: { -+ kbuild_options: ["CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER=y"], -+ enabled: true, -+ }, -+} -diff --git a/dvalin/kernel/drivers/base/dma_buf_test_exporter/dma-buf-test-exporter.c b/dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.c -similarity index 89% -rename from dvalin/kernel/drivers/base/dma_buf_test_exporter/dma-buf-test-exporter.c -rename to dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.c -index 3eb34c0..ccf73cc 100644 ---- a/dvalin/kernel/drivers/base/dma_buf_test_exporter/dma-buf-test-exporter.c -+++ b/dvalin/kernel/drivers/base/arm/dma_buf_test_exporter/dma-buf-test-exporter.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/dma-buf-test-exporter.h> -@@ -31,18 +30,16 @@ - #include <linux/atomic.h> - #include <linux/mm.h> - #include <linux/highmem.h> --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) -+#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE) - #include <linux/dma-attrs.h> - #endif - #include <linux/dma-mapping.h> --#endif - - /* Maximum size allowed in a single DMA_BUF_TE_ALLOC call */ - #define DMA_BUF_TE_ALLOC_MAX_SIZE ((8ull << 30) >> PAGE_SHIFT) /* 8 GB */ - - /* Since kernel version 5.0 CONFIG_ARCH_NO_SG_CHAIN replaced CONFIG_ARCH_HAS_SG_CHAIN */ --#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0) -+#if KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE - #if (!defined(ARCH_HAS_SG_CHAIN) && !defined(CONFIG_ARCH_HAS_SG_CHAIN)) - #define NO_SG_CHAIN - #endif -@@ -77,7 +74,7 @@ struct dma_buf_te_attachment { - - static struct miscdevice te_device; - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0)) -+#if (KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE) - static int dma_buf_te_attach(struct dma_buf *buf, struct device *dev, struct dma_buf_attachment *attachment) - #else - static int dma_buf_te_attach(struct dma_buf *buf, struct dma_buf_attachment *attachment) -@@ -206,30 +203,29 @@ static void dma_buf_te_release(struct dma_buf *buf) - /* no need for locking */ - - if (alloc->contiguous) { --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) -+#if (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE) - dma_free_attrs(te_device.this_device, - alloc->nr_pages * PAGE_SIZE, - alloc->contig_cpu_addr, - alloc->contig_dma_addr, - DMA_ATTR_WRITE_COMBINE); -- --#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) -+#else - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); - dma_free_attrs(te_device.this_device, - alloc->nr_pages * PAGE_SIZE, - alloc->contig_cpu_addr, alloc->contig_dma_addr, &attrs); --#else -- dma_free_writecombine(te_device.this_device, -- alloc->nr_pages * PAGE_SIZE, -- alloc->contig_cpu_addr, alloc->contig_dma_addr); - #endif - } else { - for (i = 0; i < alloc->nr_pages; i++) - __free_page(alloc->pages[i]); - } -+#if (KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE) -+ kvfree(alloc->pages); -+#else - kfree(alloc->pages); -+#endif - kfree(alloc); - } - -@@ -316,7 +312,7 @@ static void dma_buf_te_mmap_close(struct vm_area_struct *vma) - mutex_unlock(&dma_buf->lock); - } - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - static int dma_buf_te_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) - #elif KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE - static int dma_buf_te_mmap_fault(struct vm_fault *vmf) -@@ -328,7 +324,7 @@ static vm_fault_t dma_buf_te_mmap_fault(struct vm_fault *vmf) - struct dma_buf *dmabuf; - struct page *pageptr; - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - dmabuf = vma->vm_private_data; - #else - dmabuf = vmf->vma->vm_private_data; -@@ -362,11 +358,7 @@ static int dma_buf_te_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) - if (alloc->fail_mmap) - return -ENOMEM; - --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) - vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; --#else -- vma->vm_flags |= VM_RESERVED | VM_IO | VM_DONTEXPAND; --#endif - vma->vm_ops = &dma_buf_te_vm_ops; - vma->vm_private_data = dmabuf; - -@@ -378,7 +370,7 @@ static int dma_buf_te_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) - return 0; - } - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) -+#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE - static void *dma_buf_te_kmap_atomic(struct dma_buf *buf, unsigned long page_num) - { - /* IGNORE */ -@@ -419,19 +411,19 @@ static struct dma_buf_ops dma_buf_te_ops = { - .mmap = dma_buf_te_mmap, - .begin_cpu_access = dma_buf_te_begin_cpu_access, - .end_cpu_access = dma_buf_te_end_cpu_access, --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 12, 0) -+#if KERNEL_VERSION(4, 12, 0) > LINUX_VERSION_CODE - .kmap = dma_buf_te_kmap, - .kunmap = dma_buf_te_kunmap, - - /* nop handlers for mandatory functions we ignore */ - .kmap_atomic = dma_buf_te_kmap_atomic - #else --#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0) -+#if KERNEL_VERSION(5, 6, 0) > LINUX_VERSION_CODE - .map = dma_buf_te_kmap, - .unmap = dma_buf_te_kunmap, - #endif - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 19, 0) -+#if KERNEL_VERSION(4, 19, 0) > LINUX_VERSION_CODE - /* nop handlers for mandatory functions we ignore */ - .map_atomic = dma_buf_te_kmap_atomic - #endif -@@ -481,7 +473,8 @@ static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, - /* Whilst it is possible to allocate larger buffer, we won't be able to - * map it during actual usage (mmap() still succeeds). We fail here so - * userspace code can deal with it early than having driver failure -- * later on. */ -+ * later on. -+ */ - if (max_nr_pages > SG_MAX_SINGLE_ALLOC) - max_nr_pages = SG_MAX_SINGLE_ALLOC; - #endif /* NO_SG_CHAIN */ -@@ -493,7 +486,7 @@ static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, - } - - alloc = kzalloc(sizeof(struct dma_buf_te_alloc), GFP_KERNEL); -- if (NULL == alloc) { -+ if (alloc == NULL) { - dev_err(te_device.this_device, "%s: couldn't alloc object", __func__); - goto no_alloc_object; - } -@@ -501,7 +494,12 @@ static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, - alloc->nr_pages = alloc_req.size; - alloc->contiguous = contiguous; - -+#if (KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE) -+ alloc->pages = kvzalloc(sizeof(struct page *) * alloc->nr_pages, GFP_KERNEL); -+#else - alloc->pages = kzalloc(sizeof(struct page *) * alloc->nr_pages, GFP_KERNEL); -+#endif -+ - if (!alloc->pages) { - dev_err(te_device.this_device, - "%s: couldn't alloc %zu page structures", -@@ -512,14 +510,13 @@ static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, - if (contiguous) { - dma_addr_t dma_aux; - --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) -+#if (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE) - alloc->contig_cpu_addr = dma_alloc_attrs(te_device.this_device, - alloc->nr_pages * PAGE_SIZE, - &alloc->contig_dma_addr, - GFP_KERNEL | __GFP_ZERO, - DMA_ATTR_WRITE_COMBINE); -- --#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) -+#else - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); -@@ -527,11 +524,6 @@ static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, - alloc->nr_pages * PAGE_SIZE, - &alloc->contig_dma_addr, - GFP_KERNEL | __GFP_ZERO, &attrs); --#else -- alloc->contig_cpu_addr = dma_alloc_writecombine(te_device.this_device, -- alloc->nr_pages * PAGE_SIZE, -- &alloc->contig_dma_addr, -- GFP_KERNEL | __GFP_ZERO); - #endif - if (!alloc->contig_cpu_addr) { - dev_err(te_device.this_device, "%s: couldn't alloc contiguous buffer %zu pages", -@@ -546,7 +538,7 @@ static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, - } else { - for (i = 0; i < alloc->nr_pages; i++) { - alloc->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO); -- if (NULL == alloc->pages[i]) { -+ if (alloc->pages[i] == NULL) { - dev_err(te_device.this_device, "%s: couldn't alloc page", __func__); - goto no_page; - } -@@ -554,13 +546,10 @@ static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, - } - - /* alloc ready, let's export it */ --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 1, 0)) - { - struct dma_buf_export_info export_info = { - .exp_name = "dma_buf_te", --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 2, 0)) - .owner = THIS_MODULE, --#endif - .ops = &dma_buf_te_ops, - .size = alloc->nr_pages << PAGE_SHIFT, - .flags = O_CLOEXEC | O_RDWR, -@@ -569,13 +558,6 @@ static int do_dma_buf_te_ioctl_alloc(struct dma_buf_te_ioctl_alloc __user *buf, - - dma_buf = dma_buf_export(&export_info); - } --#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0)) -- dma_buf = dma_buf_export(alloc, &dma_buf_te_ops, -- alloc->nr_pages << PAGE_SHIFT, O_CLOEXEC|O_RDWR, NULL); --#else -- dma_buf = dma_buf_export(alloc, &dma_buf_te_ops, -- alloc->nr_pages << PAGE_SHIFT, O_CLOEXEC|O_RDWR); --#endif - - if (IS_ERR_OR_NULL(dma_buf)) { - dev_err(te_device.this_device, "%s: couldn't export dma_buf", __func__); -@@ -598,32 +580,30 @@ no_export: - /* i still valid */ - no_page: - if (contiguous) { -- --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 8, 0)) -+#if (KERNEL_VERSION(4, 8, 0) <= LINUX_VERSION_CODE) - dma_free_attrs(te_device.this_device, - alloc->nr_pages * PAGE_SIZE, - alloc->contig_cpu_addr, - alloc->contig_dma_addr, - DMA_ATTR_WRITE_COMBINE); -- --#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) -+#else - DEFINE_DMA_ATTRS(attrs); - - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); - dma_free_attrs(te_device.this_device, - alloc->nr_pages * PAGE_SIZE, - alloc->contig_cpu_addr, alloc->contig_dma_addr, &attrs); --#else -- dma_free_writecombine(te_device.this_device, -- alloc->nr_pages * PAGE_SIZE, -- alloc->contig_cpu_addr, alloc->contig_dma_addr); - #endif - } else { - while (i-- > 0) - __free_page(alloc->pages[i]); - } - free_page_struct: -+#if (KERNEL_VERSION(4, 12, 0) <= LINUX_VERSION_CODE) -+ kvfree(alloc->pages); -+#else - kfree(alloc->pages); -+#endif - free_alloc_object: - kfree(alloc); - no_alloc_object: -@@ -727,17 +707,17 @@ static u32 dma_te_buf_fill(struct dma_buf *dma_buf, unsigned int value) - } - - ret = dma_buf_begin_cpu_access(dma_buf, --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -- 0, dma_buf->size, -+#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE -+ 0, dma_buf->size, - #endif -- DMA_BIDIRECTIONAL); -+ DMA_BIDIRECTIONAL); - if (ret) - goto no_cpu_access; - - for_each_sg(sgt->sgl, sg, sgt->nents, count) { - for (i = 0; i < sg_dma_len(sg); i = i + PAGE_SIZE) { - void *addr = NULL; --#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) -+#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE - addr = dma_buf_te_kmap(dma_buf, i >> PAGE_SHIFT); - #else - addr = dma_buf_kmap(dma_buf, i >> PAGE_SHIFT); -@@ -747,7 +727,7 @@ static u32 dma_te_buf_fill(struct dma_buf *dma_buf, unsigned int value) - goto no_kmap; - } - memset(addr, value, PAGE_SIZE); --#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) -+#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE - dma_buf_te_kunmap(dma_buf, i >> PAGE_SHIFT, addr); - #else - dma_buf_kunmap(dma_buf, i >> PAGE_SHIFT, addr); -@@ -758,10 +738,10 @@ static u32 dma_te_buf_fill(struct dma_buf *dma_buf, unsigned int value) - - no_kmap: - dma_buf_end_cpu_access(dma_buf, --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -- 0, dma_buf->size, -+#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE -+ 0, dma_buf->size, - #endif -- DMA_BIDIRECTIONAL); -+ DMA_BIDIRECTIONAL); - no_cpu_access: - dma_buf_unmap_attachment(attachment, sgt, DMA_BIDIRECTIONAL); - no_import: -diff --git a/dvalin/kernel/drivers/base/dma_buf_test_exporter/Kbuild b/dvalin/kernel/drivers/base/arm/memory_group_manager/Kbuild -similarity index 74% -rename from dvalin/kernel/drivers/base/dma_buf_test_exporter/Kbuild -rename to dvalin/kernel/drivers/base/arm/memory_group_manager/Kbuild -index c382b79..99ce311 100644 ---- a/dvalin/kernel/drivers/base/dma_buf_test_exporter/Kbuild -+++ b/dvalin/kernel/drivers/base/arm/memory_group_manager/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,10 +16,8 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - --ifneq ($(CONFIG_DMA_SHARED_BUFFER),) --obj-$(CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER) += dma-buf-test-exporter.o -+ifeq ($(CONFIG_MALI_MEMORY_GROUP_MANAGER), y) -+obj-m := memory_group_manager.o - endif -diff --git a/dvalin/kernel/drivers/base/arm/memory_group_manager/build.bp b/dvalin/kernel/drivers/base/arm/memory_group_manager/build.bp -new file mode 100644 -index 0000000..23db183 ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/memory_group_manager/build.bp -@@ -0,0 +1,36 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+bob_kernel_module { -+ name: "memory_group_manager", -+ defaults: [ -+ "kernel_defaults" -+ ], -+ srcs: [ -+ "Kbuild", -+ "memory_group_manager.c", -+ ], -+ enabled: false, -+ mali_memory_group_manager: { -+ kbuild_options: ["CONFIG_MALI_MEMORY_GROUP_MANAGER=y"], -+ enabled: true, -+ }, -+} -diff --git a/dvalin/kernel/drivers/base/memory_group_manager/memory_group_manager.c b/dvalin/kernel/drivers/base/arm/memory_group_manager/memory_group_manager.c -similarity index 98% -rename from dvalin/kernel/drivers/base/memory_group_manager/memory_group_manager.c -rename to dvalin/kernel/drivers/base/arm/memory_group_manager/memory_group_manager.c -index 44f848a..a70fe70 100644 ---- a/dvalin/kernel/drivers/base/memory_group_manager/memory_group_manager.c -+++ b/dvalin/kernel/drivers/base/arm/memory_group_manager/memory_group_manager.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/fs.h> -@@ -26,7 +25,7 @@ - #include <linux/platform_device.h> - #include <linux/version.h> - #include <linux/module.h> --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - #include <linux/debugfs.h> - #endif - #include <linux/mm.h> -@@ -92,12 +91,12 @@ struct mgm_group { - struct mgm_groups { - struct mgm_group groups[MEMORY_GROUP_MANAGER_NR_GROUPS]; - struct device *dev; --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - struct dentry *mgm_debugfs_root; - #endif - }; - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - - static int mgm_size_get(void *data, u64 *val) - { -@@ -475,7 +474,6 @@ static struct platform_driver memory_group_manager_driver = { - .remove = memory_group_manager_remove, - .driver = { - .name = "physical-memory-group-manager", -- .owner = THIS_MODULE, - .of_match_table = of_match_ptr(memory_group_manager_dt_ids), - /* - * Prevent the mgm_dev from being unbound and freed, as other's -diff --git a/dvalin/kernel/drivers/base/arm/protected_memory_allocator/Kbuild b/dvalin/kernel/drivers/base/arm/protected_memory_allocator/Kbuild -new file mode 100644 -index 0000000..25295a9 ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/protected_memory_allocator/Kbuild -@@ -0,0 +1,23 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+ifeq ($(CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR), y) -+obj-m := protected_memory_allocator.o -+endif -diff --git a/dvalin/kernel/drivers/base/arm/protected_memory_allocator/build.bp b/dvalin/kernel/drivers/base/arm/protected_memory_allocator/build.bp -new file mode 100644 -index 0000000..4c56154 ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/protected_memory_allocator/build.bp -@@ -0,0 +1,36 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+bob_kernel_module { -+ name: "protected_memory_allocator", -+ defaults: [ -+ "kernel_defaults" -+ ], -+ srcs: [ -+ "Kbuild", -+ "protected_memory_allocator.c", -+ ], -+ enabled: false, -+ mali_protected_memory_allocator: { -+ kbuild_options: ["CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR=y"], -+ enabled: true, -+ }, -+} -diff --git a/dvalin/kernel/drivers/base/arm/protected_memory_allocator/protected_memory_allocator.c b/dvalin/kernel/drivers/base/arm/protected_memory_allocator/protected_memory_allocator.c -new file mode 100644 -index 0000000..6684210 ---- /dev/null -+++ b/dvalin/kernel/drivers/base/arm/protected_memory_allocator/protected_memory_allocator.c -@@ -0,0 +1,551 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <linux/version.h> -+#include <linux/of.h> -+#include <linux/of_reserved_mem.h> -+#include <linux/platform_device.h> -+#include <linux/module.h> -+#include <linux/slab.h> -+#include <linux/mm.h> -+#include <linux/io.h> -+#include <linux/protected_memory_allocator.h> -+ -+/* Size of a bitfield element in bytes */ -+#define BITFIELD_ELEM_SIZE sizeof(u64) -+ -+/* We can track whether or not 64 pages are currently allocated in a u64 */ -+#define PAGES_PER_BITFIELD_ELEM (BITFIELD_ELEM_SIZE * BITS_PER_BYTE) -+ -+/* Order 6 (ie, 64) corresponds to the number of pages held in a bitfield */ -+#define ORDER_OF_PAGES_PER_BITFIELD_ELEM 6 -+ -+/** -+ * struct simple_pma_device - Simple implementation of a protected memory -+ * allocator device -+ * -+ * @pma_dev: Protected memory allocator device pointer -+ * @dev: Device pointer -+ * @alloc_pages_bitfield_arr: Status of all the physical memory pages within the -+ * protected memory region, one bit per page -+ * @rmem_base: Base address of the reserved memory region -+ * @rmem_size: Size of the reserved memory region, in pages -+ * @num_free_pages: Number of free pages in the memory region -+ * @rmem_lock: Lock to serialize the allocation and freeing of -+ * physical pages from the protected memory region -+ */ -+struct simple_pma_device { -+ struct protected_memory_allocator_device pma_dev; -+ struct device *dev; -+ u64 *allocated_pages_bitfield_arr; -+ phys_addr_t rmem_base; -+ size_t rmem_size; -+ size_t num_free_pages; -+ spinlock_t rmem_lock; -+}; -+ -+/** -+ * Number of elements in array 'allocated_pages_bitfield_arr'. If the number of -+ * pages required does not divide exactly by PAGES_PER_BITFIELD_ELEM, adds an -+ * extra page for the remainder. -+ */ -+#define ALLOC_PAGES_BITFIELD_ARR_SIZE(num_pages) \ -+ ((PAGES_PER_BITFIELD_ELEM * (0 != (num_pages % PAGES_PER_BITFIELD_ELEM)) + \ -+ num_pages) / PAGES_PER_BITFIELD_ELEM) -+ -+/** -+ * Allocate a power-of-two number of pages, N, where -+ * 0 <= N <= ORDER_OF_PAGES_PER_BITFIELD_ELEM - 1. ie, Up to 32 pages. The routine -+ * fills-in a pma structure and sets the appropriate bits in the allocated-pages -+ * bitfield array but assumes the caller has already determined that these are -+ * already clear. -+ * -+ * This routine always works within only a single allocated-pages bitfield element. -+ * It can be thought of as the 'small-granularity' allocator. -+ */ -+static void small_granularity_alloc(struct simple_pma_device *const epma_dev, -+ size_t alloc_bitfield_idx, size_t start_bit, -+ size_t order, -+ struct protected_memory_allocation *pma) -+{ -+ size_t i; -+ size_t page_idx; -+ u64 *bitfield; -+ size_t alloc_pages_bitfield_size; -+ -+ if (WARN_ON(!epma_dev) || -+ WARN_ON(!pma)) -+ return; -+ -+ WARN(epma_dev->rmem_size == 0, "%s: rmem_size is 0", __func__); -+ alloc_pages_bitfield_size = ALLOC_PAGES_BITFIELD_ARR_SIZE(epma_dev->rmem_size); -+ -+ WARN(alloc_bitfield_idx >= alloc_pages_bitfield_size, -+ "%s: idx>bf_size: %zu %zu", __FUNCTION__, -+ alloc_bitfield_idx, alloc_pages_bitfield_size); -+ -+ WARN((start_bit + (1 << order)) > PAGES_PER_BITFIELD_ELEM, -+ "%s: start=%zu order=%zu ppbe=%zu", -+ __FUNCTION__, start_bit, order, PAGES_PER_BITFIELD_ELEM); -+ -+ bitfield = &epma_dev->allocated_pages_bitfield_arr[alloc_bitfield_idx]; -+ -+ for (i = 0; i < (1 << order); i++) { -+ /* Check the pages represented by this bit are actually free */ -+ WARN (*bitfield & (1ULL << (start_bit + i)), -+ "in %s: page not free: %zu %zu %.16llx %zu\n", -+ __FUNCTION__, i, order, *bitfield, alloc_pages_bitfield_size); -+ -+ /* Mark the pages as now allocated */ -+ *bitfield |= (1ULL << (start_bit + i)); -+ } -+ -+ /* Compute the page index */ -+ page_idx = (alloc_bitfield_idx * PAGES_PER_BITFIELD_ELEM) + start_bit; -+ -+ /* Fill-in the allocation struct for the caller */ -+ pma->pa = epma_dev->rmem_base + (page_idx << PAGE_SHIFT); -+ pma->order = order; -+} -+ -+/** -+ * Allocate a power-of-two number of pages, N, where -+ * N >= ORDER_OF_PAGES_PER_BITFIELD_ELEM. ie, 64 pages or more. The routine fills-in -+ * a pma structure and sets the appropriate bits in the allocated-pages bitfield array -+ * but assumes the caller has already determined that these are already clear. -+ * -+ * Unlike small_granularity_alloc, this routine can work with multiple 64-page groups, -+ * ie multiple elements from the allocated-pages bitfield array. However, it always -+ * works with complete sets of these 64-page groups. It can therefore be thought of -+ * as the 'large-granularity' allocator. -+ */ -+static void large_granularity_alloc(struct simple_pma_device *const epma_dev, -+ size_t start_alloc_bitfield_idx, -+ size_t order, -+ struct protected_memory_allocation *pma) -+{ -+ size_t i; -+ size_t num_pages_to_alloc = (size_t)1 << order; -+ size_t num_bitfield_elements_needed = num_pages_to_alloc / PAGES_PER_BITFIELD_ELEM; -+ size_t start_page_idx = start_alloc_bitfield_idx * PAGES_PER_BITFIELD_ELEM; -+ -+ if (WARN_ON(!epma_dev) || -+ WARN_ON(!pma)) -+ return; -+ -+ /* -+ * Are there anough bitfield array elements (groups of 64 pages) -+ * between the start element and the end of the bitfield array -+ * to fulfill the request? -+ */ -+ WARN((start_alloc_bitfield_idx + order) >= ALLOC_PAGES_BITFIELD_ARR_SIZE(epma_dev->rmem_size), -+ "%s: start=%zu order=%zu ms=%zu", -+ __FUNCTION__, start_alloc_bitfield_idx, order, epma_dev->rmem_size); -+ -+ for (i = 0; i < num_bitfield_elements_needed; i++) { -+ u64 *bitfield = &epma_dev->allocated_pages_bitfield_arr[start_alloc_bitfield_idx + i]; -+ -+ /* We expect all pages that relate to this bitfield element to be free */ -+ WARN((*bitfield != 0), -+ "in %s: pages not free: i=%zu o=%zu bf=%.16llx\n", -+ __FUNCTION__, i, order, *bitfield); -+ -+ /* Mark all the pages for this element as not free */ -+ *bitfield = ~0ULL; -+ } -+ -+ /* Fill-in the allocation struct for the caller */ -+ pma->pa = epma_dev->rmem_base + (start_page_idx << PAGE_SHIFT); -+ pma->order = order; -+} -+ -+static struct protected_memory_allocation *simple_pma_alloc_page( -+ struct protected_memory_allocator_device *pma_dev, unsigned int order) -+{ -+ struct simple_pma_device *const epma_dev = -+ container_of(pma_dev, struct simple_pma_device, pma_dev); -+ struct protected_memory_allocation *pma; -+ size_t num_pages_to_alloc; -+ -+ u64 *bitfields = epma_dev->allocated_pages_bitfield_arr; -+ size_t i; -+ size_t bit; -+ size_t count; -+ -+ dev_dbg(epma_dev->dev, "%s(pma_dev=%px, order=%u\n", -+ __func__, (void *)pma_dev, order); -+ -+ /* This is an example function that follows an extremely simple logic -+ * and is very likely to fail to allocate memory if put under stress. -+ * -+ * The simple_pma_device maintains an array of u64s, with one bit used -+ * to track the status of each page. -+ * -+ * In order to create a memory allocation, the allocator looks for an -+ * adjacent group of cleared bits. This does leave the algorithm open -+ * to fragmentation issues, but is deemed sufficient for now. -+ * If successful, the allocator shall mark all the pages as allocated -+ * and increment the offset accordingly. -+ * -+ * Allocations of 64 pages or more (order 6) can be allocated only with -+ * 64-page alignment, in order to keep the algorithm as simple as -+ * possible. ie, starting from bit 0 of any 64-bit page-allocation -+ * bitfield. For this, the large-granularity allocator is utilised. -+ * -+ * Allocations of lower-order can only be allocated entirely within the -+ * same group of 64 pages, with the small-ganularity allocator (ie -+ * always from the same 64-bit page-allocation bitfield) - again, to -+ * keep things as simple as possible, but flexible to meet -+ * current needs. -+ */ -+ -+ num_pages_to_alloc = (size_t)1 << order; -+ -+ pma = devm_kzalloc(epma_dev->dev, sizeof(*pma), GFP_KERNEL); -+ if (!pma) { -+ dev_err(epma_dev->dev, "Failed to alloc pma struct"); -+ return NULL; -+ } -+ -+ spin_lock(&epma_dev->rmem_lock); -+ -+ if (epma_dev->num_free_pages < num_pages_to_alloc) { -+ dev_err(epma_dev->dev, "not enough free pages\n"); -+ devm_kfree(epma_dev->dev, pma); -+ spin_unlock(&epma_dev->rmem_lock); -+ return NULL; -+ } -+ -+ /* -+ * For order 0-5 (ie, 1 to 32 pages) we always allocate within the same set of 64 pages -+ * Currently, most allocations will be very small (1 page), so the more likely path -+ * here is order < ORDER_OF_PAGES_PER_BITFIELD_ELEM. -+ */ -+ if (likely(order < ORDER_OF_PAGES_PER_BITFIELD_ELEM)) { -+ size_t alloc_pages_bitmap_size = ALLOC_PAGES_BITFIELD_ARR_SIZE(epma_dev->rmem_size); -+ -+ for (i = 0; i < alloc_pages_bitmap_size; i++) { -+ count = 0; -+ -+ for (bit = 0; bit < PAGES_PER_BITFIELD_ELEM; bit++) { -+ if (0 == (bitfields[i] & (1ULL << bit))) { -+ if ((count + 1) >= num_pages_to_alloc) { -+ /* -+ * We've found enough free, consecutive pages with which to -+ * make an allocation -+ */ -+ small_granularity_alloc( -+ epma_dev, i, -+ bit - count, order, -+ pma); -+ -+ epma_dev->num_free_pages -= -+ num_pages_to_alloc; -+ -+ spin_unlock( -+ &epma_dev->rmem_lock); -+ return pma; -+ } -+ -+ /* So far so good, but we need more set bits yet */ -+ count++; -+ } else { -+ /* -+ * We found an allocated page, so nothing we've seen so far can be used. -+ * Keep looking. -+ */ -+ count = 0; -+ } -+ } -+ } -+ } else { -+ /** -+ * For allocations of order ORDER_OF_PAGES_PER_BITFIELD_ELEM and above (>= 64 pages), we know -+ * we'll only get allocations for whole groups of 64 pages, which hugely simplifies the task. -+ */ -+ size_t alloc_pages_bitmap_size = ALLOC_PAGES_BITFIELD_ARR_SIZE(epma_dev->rmem_size); -+ -+ /* How many 64-bit bitfield elements will be needed for the allocation? */ -+ size_t num_bitfield_elements_needed = num_pages_to_alloc / PAGES_PER_BITFIELD_ELEM; -+ -+ count = 0; -+ -+ for (i = 0; i < alloc_pages_bitmap_size; i++) { -+ /* Are all the pages free for the i'th u64 bitfield element? */ -+ if (bitfields[i] == 0) { -+ count += PAGES_PER_BITFIELD_ELEM; -+ -+ if (count >= (1 << order)) { -+ size_t start_idx = (i + 1) - num_bitfield_elements_needed; -+ -+ large_granularity_alloc(epma_dev, -+ start_idx, -+ order, pma); -+ -+ epma_dev->num_free_pages -= 1 << order; -+ spin_unlock(&epma_dev->rmem_lock); -+ return pma; -+ } -+ } -+ else -+ { -+ count = 0; -+ } -+ } -+ } -+ -+ spin_unlock(&epma_dev->rmem_lock); -+ devm_kfree(epma_dev->dev, pma); -+ -+ dev_err(epma_dev->dev, "not enough contiguous pages (need %zu), total free pages left %zu\n", -+ num_pages_to_alloc, epma_dev->num_free_pages); -+ return NULL; -+} -+ -+static phys_addr_t simple_pma_get_phys_addr( -+ struct protected_memory_allocator_device *pma_dev, -+ struct protected_memory_allocation *pma) -+{ -+ struct simple_pma_device *const epma_dev = -+ container_of(pma_dev, struct simple_pma_device, pma_dev); -+ -+ dev_dbg(epma_dev->dev, "%s(pma_dev=%px, pma=%px, pa=%llx\n", -+ __func__, (void *)pma_dev, (void *)pma, -+ (unsigned long long)pma->pa); -+ -+ return pma->pa; -+} -+ -+static void simple_pma_free_page( -+ struct protected_memory_allocator_device *pma_dev, -+ struct protected_memory_allocation *pma) -+{ -+ struct simple_pma_device *const epma_dev = -+ container_of(pma_dev, struct simple_pma_device, pma_dev); -+ size_t num_pages_in_allocation; -+ size_t offset; -+ size_t i; -+ size_t bitfield_idx; -+ size_t bitfield_start_bit; -+ size_t page_num; -+ u64 *bitfield; -+ size_t alloc_pages_bitmap_size; -+ size_t num_bitfield_elems_used_by_alloc; -+ -+ WARN_ON(pma == NULL); -+ -+ dev_dbg(epma_dev->dev, "%s(pma_dev=%px, pma=%px, pa=%llx\n", -+ __func__, (void *)pma_dev, (void *)pma, -+ (unsigned long long)pma->pa); -+ -+ WARN_ON(pma->pa < epma_dev->rmem_base); -+ -+ /* This is an example function that follows an extremely simple logic -+ * and is vulnerable to abuse. -+ */ -+ offset = (pma->pa - epma_dev->rmem_base); -+ num_pages_in_allocation = (size_t)1 << pma->order; -+ -+ /* The number of bitfield elements used by the allocation */ -+ num_bitfield_elems_used_by_alloc = num_pages_in_allocation / PAGES_PER_BITFIELD_ELEM; -+ -+ /* The page number of the first page of the allocation, relative to rmem_base */ -+ page_num = offset >> PAGE_SHIFT; -+ -+ /* Which u64 bitfield refers to this page? */ -+ bitfield_idx = page_num / PAGES_PER_BITFIELD_ELEM; -+ -+ alloc_pages_bitmap_size = ALLOC_PAGES_BITFIELD_ARR_SIZE(epma_dev->rmem_size); -+ -+ /* Is the allocation within expected bounds? */ -+ WARN_ON((bitfield_idx + num_bitfield_elems_used_by_alloc) >= alloc_pages_bitmap_size); -+ -+ spin_lock(&epma_dev->rmem_lock); -+ -+ if (pma->order < ORDER_OF_PAGES_PER_BITFIELD_ELEM) { -+ bitfield = &epma_dev->allocated_pages_bitfield_arr[bitfield_idx]; -+ -+ /* Which bit within that u64 bitfield is the lsb covering this allocation? */ -+ bitfield_start_bit = page_num % PAGES_PER_BITFIELD_ELEM; -+ -+ /* Clear the bits for the pages we're now freeing */ -+ *bitfield &= ~(((1ULL << num_pages_in_allocation) - 1) << bitfield_start_bit); -+ } -+ else { -+ WARN(page_num % PAGES_PER_BITFIELD_ELEM, -+ "%s: Expecting allocs of order >= %d to be %zu-page aligned\n", -+ __FUNCTION__, ORDER_OF_PAGES_PER_BITFIELD_ELEM, PAGES_PER_BITFIELD_ELEM); -+ -+ for (i = 0; i < num_bitfield_elems_used_by_alloc; i++) { -+ bitfield = &epma_dev->allocated_pages_bitfield_arr[bitfield_idx + i]; -+ -+ /* We expect all bits to be set (all pages allocated) */ -+ WARN((*bitfield != ~0), -+ "%s: alloc being freed is not fully allocated: of=%zu np=%zu bf=%.16llx\n", -+ __FUNCTION__, offset, num_pages_in_allocation, *bitfield); -+ -+ /* -+ * Now clear all the bits in the bitfield element to mark all the pages -+ * it refers to as free. -+ */ -+ *bitfield = 0ULL; -+ } -+ } -+ -+ epma_dev->num_free_pages += num_pages_in_allocation; -+ spin_unlock(&epma_dev->rmem_lock); -+ devm_kfree(epma_dev->dev, pma); -+} -+ -+static int protected_memory_allocator_probe(struct platform_device *pdev) -+{ -+ struct simple_pma_device *epma_dev; -+ struct device_node *np; -+ phys_addr_t rmem_base; -+ size_t rmem_size; -+ size_t alloc_bitmap_pages_arr_size; -+#if (KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE) -+ struct reserved_mem *rmem; -+#endif -+ -+ np = pdev->dev.of_node; -+ -+ if (!np) { -+ dev_err(&pdev->dev, "device node pointer not set\n"); -+ return -ENODEV; -+ } -+ -+ np = of_parse_phandle(np, "memory-region", 0); -+ if (!np) { -+ dev_err(&pdev->dev, "memory-region node not set\n"); -+ return -ENODEV; -+ } -+ -+#if (KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE) -+ rmem = of_reserved_mem_lookup(np); -+ if (rmem) { -+ rmem_base = rmem->base; -+ rmem_size = rmem->size >> PAGE_SHIFT; -+ } else -+#endif -+ { -+ of_node_put(np); -+ dev_err(&pdev->dev, "could not read reserved memory-region\n"); -+ return -ENODEV; -+ } -+ -+ of_node_put(np); -+ epma_dev = devm_kzalloc(&pdev->dev, sizeof(*epma_dev), GFP_KERNEL); -+ if (!epma_dev) -+ return -ENOMEM; -+ -+ epma_dev->pma_dev.ops.pma_alloc_page = simple_pma_alloc_page; -+ epma_dev->pma_dev.ops.pma_get_phys_addr = simple_pma_get_phys_addr; -+ epma_dev->pma_dev.ops.pma_free_page = simple_pma_free_page; -+ epma_dev->pma_dev.owner = THIS_MODULE; -+ epma_dev->dev = &pdev->dev; -+ epma_dev->rmem_base = rmem_base; -+ epma_dev->rmem_size = rmem_size; -+ epma_dev->num_free_pages = rmem_size; -+ spin_lock_init(&epma_dev->rmem_lock); -+ -+ alloc_bitmap_pages_arr_size = ALLOC_PAGES_BITFIELD_ARR_SIZE(epma_dev->rmem_size); -+ -+ epma_dev->allocated_pages_bitfield_arr = devm_kzalloc(&pdev->dev, -+ alloc_bitmap_pages_arr_size * BITFIELD_ELEM_SIZE, GFP_KERNEL); -+ -+ if (!epma_dev->allocated_pages_bitfield_arr) { -+ dev_err(&pdev->dev, "failed to allocate resources\n"); -+ devm_kfree(&pdev->dev, epma_dev); -+ return -ENOMEM; -+ } -+ -+ if (epma_dev->rmem_size % PAGES_PER_BITFIELD_ELEM) { -+ size_t extra_pages = -+ alloc_bitmap_pages_arr_size * PAGES_PER_BITFIELD_ELEM - -+ epma_dev->rmem_size; -+ size_t last_bitfield_index = alloc_bitmap_pages_arr_size - 1; -+ -+ /* Mark the extra pages (that lie outside the reserved range) as -+ * always in use. -+ */ -+ epma_dev->allocated_pages_bitfield_arr[last_bitfield_index] = -+ ((1ULL << extra_pages) - 1) << -+ (PAGES_PER_BITFIELD_ELEM - extra_pages); -+ } -+ -+ platform_set_drvdata(pdev, &epma_dev->pma_dev); -+ dev_info(&pdev->dev, -+ "Protected memory allocator probed successfully\n"); -+ dev_info(&pdev->dev, "Protected memory region: base=%llx num pages=%zu\n", -+ (unsigned long long)rmem_base, rmem_size); -+ -+ return 0; -+} -+ -+static int protected_memory_allocator_remove(struct platform_device *pdev) -+{ -+ struct protected_memory_allocator_device *pma_dev = -+ platform_get_drvdata(pdev); -+ struct simple_pma_device *epma_dev; -+ struct device *dev; -+ -+ if (!pma_dev) -+ return -EINVAL; -+ -+ epma_dev = container_of(pma_dev, struct simple_pma_device, pma_dev); -+ dev = epma_dev->dev; -+ -+ if (epma_dev->num_free_pages < epma_dev->rmem_size) { -+ dev_warn(&pdev->dev, "Leaking %zu pages of protected memory\n", -+ epma_dev->rmem_size - epma_dev->num_free_pages); -+ } -+ -+ platform_set_drvdata(pdev, NULL); -+ devm_kfree(dev, epma_dev->allocated_pages_bitfield_arr); -+ devm_kfree(dev, epma_dev); -+ -+ dev_info(&pdev->dev, -+ "Protected memory allocator removed successfully\n"); -+ -+ return 0; -+} -+ -+static const struct of_device_id protected_memory_allocator_dt_ids[] = { -+ { .compatible = "arm,protected-memory-allocator" }, -+ { /* sentinel */ } -+}; -+MODULE_DEVICE_TABLE(of, protected_memory_allocator_dt_ids); -+ -+static struct platform_driver protected_memory_allocator_driver = { -+ .probe = protected_memory_allocator_probe, -+ .remove = protected_memory_allocator_remove, -+ .driver = { -+ .name = "simple_protected_memory_allocator", -+ .of_match_table = of_match_ptr(protected_memory_allocator_dt_ids), -+ } -+}; -+ -+module_platform_driver(protected_memory_allocator_driver); -+ -+MODULE_LICENSE("GPL"); -+MODULE_AUTHOR("ARM Ltd."); -+MODULE_VERSION("1.0"); -diff --git a/dvalin/kernel/drivers/base/dma_buf_test_exporter/Kconfig b/dvalin/kernel/drivers/base/dma_buf_test_exporter/Kconfig -deleted file mode 100644 -index 66ca1bc..0000000 ---- a/dvalin/kernel/drivers/base/dma_buf_test_exporter/Kconfig -+++ /dev/null -@@ -1,26 +0,0 @@ --# --# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. --# --# This program is free software and is provided to you under the terms of the --# GNU General Public License version 2 as published by the Free Software --# Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. --# --# This program is distributed in the hope that it will be useful, --# but WITHOUT ANY WARRANTY; without even the implied warranty of --# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --# GNU General Public License for more details. --# --# You should have received a copy of the GNU General Public License --# along with this program; if not, you can access it online at --# http://www.gnu.org/licenses/gpl-2.0.html. --# --# SPDX-License-Identifier: GPL-2.0 --# --# -- --config DMA_SHARED_BUFFER_TEST_EXPORTER -- tristate "Test exporter for the dma-buf framework" -- depends on DMA_SHARED_BUFFER -- help -- This option enables the test exporter usable to help test importerts. -diff --git a/dvalin/kernel/drivers/base/dma_buf_test_exporter/Makefile b/dvalin/kernel/drivers/base/dma_buf_test_exporter/Makefile -deleted file mode 100644 -index 528582c..0000000 ---- a/dvalin/kernel/drivers/base/dma_buf_test_exporter/Makefile -+++ /dev/null -@@ -1,36 +0,0 @@ --# --# (C) COPYRIGHT 2011-2013 ARM Limited. All rights reserved. --# --# This program is free software and is provided to you under the terms of the --# GNU General Public License version 2 as published by the Free Software --# Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. --# --# This program is distributed in the hope that it will be useful, --# but WITHOUT ANY WARRANTY; without even the implied warranty of --# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --# GNU General Public License for more details. --# --# You should have received a copy of the GNU General Public License --# along with this program; if not, you can access it online at --# http://www.gnu.org/licenses/gpl-2.0.html. --# --# SPDX-License-Identifier: GPL-2.0 --# --# -- --# linux build system bootstrap for out-of-tree module -- --# default to building for the host --ARCH ?= $(shell uname -m) -- --ifeq ($(KDIR),) --$(error Must specify KDIR to point to the kernel to target)) --endif -- --all: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../include" CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER=m -- --clean: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean -- -diff --git a/dvalin/kernel/drivers/base/dma_buf_test_exporter/build.bp b/dvalin/kernel/drivers/base/dma_buf_test_exporter/build.bp -deleted file mode 100644 -index 7b0bd5d..0000000 ---- a/dvalin/kernel/drivers/base/dma_buf_test_exporter/build.bp -+++ /dev/null -@@ -1,26 +0,0 @@ --/* -- * -- * (C) COPYRIGHT 2017, 2020 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * A copy of the licence is included with the program, and can also be obtained -- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- * Boston, MA 02110-1301, USA. -- * -- */ -- --bob_kernel_module { -- name: "dma-buf-test-exporter", -- srcs: [ -- "Kbuild", -- "dma-buf-test-exporter.c", -- ], -- kbuild_options: [ -- "CONFIG_DMA_SHARED_BUFFER_TEST_EXPORTER=m", -- ], -- defaults: ["kernel_defaults"], --} -diff --git a/dvalin/kernel/drivers/base/memory_group_manager/Makefile b/dvalin/kernel/drivers/base/memory_group_manager/Makefile -deleted file mode 100644 -index a5bceae..0000000 ---- a/dvalin/kernel/drivers/base/memory_group_manager/Makefile -+++ /dev/null -@@ -1,35 +0,0 @@ --# --# (C) COPYRIGHT 2019 ARM Limited. All rights reserved. --# --# This program is free software and is provided to you under the terms of the --# GNU General Public License version 2 as published by the Free Software --# Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. --# --# This program is distributed in the hope that it will be useful, --# but WITHOUT ANY WARRANTY; without even the implied warranty of --# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --# GNU General Public License for more details. --# --# You should have received a copy of the GNU General Public License --# along with this program; if not, you can access it online at --# http://www.gnu.org/licenses/gpl-2.0.html. --# --# SPDX-License-Identifier: GPL-2.0 --# --# -- --# linux build system bootstrap for out-of-tree module -- --# default to building for the host --ARCH ?= $(shell uname -m) -- --ifeq ($(KDIR),) --$(error Must specify KDIR to point to the kernel to target)) --endif -- --all: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../include" modules CONFIG_MALI_MEMORY_GROUP_MANAGER=m -- --clean: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean -diff --git a/dvalin/kernel/drivers/base/memory_group_manager/build.bp b/dvalin/kernel/drivers/base/memory_group_manager/build.bp -deleted file mode 100644 -index 04dbfd3..0000000 ---- a/dvalin/kernel/drivers/base/memory_group_manager/build.bp -+++ /dev/null -@@ -1,22 +0,0 @@ --/* -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * A copy of the licence is included with the program, and can also be obtained -- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- * Boston, MA 02110-1301, USA. -- */ -- --bob_kernel_module { -- name: "memory_group_manager", -- srcs: [ -- "Kbuild", -- "memory_group_manager.c", -- ], -- kbuild_options: ["CONFIG_MALI_MEMORY_GROUP_MANAGER=m"], -- defaults: ["kernel_defaults"], --} -diff --git a/dvalin/kernel/drivers/base/protected_memory_allocator/Makefile b/dvalin/kernel/drivers/base/protected_memory_allocator/Makefile -deleted file mode 100644 -index 17b2600..0000000 ---- a/dvalin/kernel/drivers/base/protected_memory_allocator/Makefile -+++ /dev/null -@@ -1,35 +0,0 @@ --# --# (C) COPYRIGHT 2019 ARM Limited. All rights reserved. --# --# This program is free software and is provided to you under the terms of the --# GNU General Public License version 2 as published by the Free Software --# Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. --# --# This program is distributed in the hope that it will be useful, --# but WITHOUT ANY WARRANTY; without even the implied warranty of --# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --# GNU General Public License for more details. --# --# You should have received a copy of the GNU General Public License --# along with this program; if not, you can access it online at --# http://www.gnu.org/licenses/gpl-2.0.html. --# --# SPDX-License-Identifier: GPL-2.0 --# --# -- --# linux build system bootstrap for out-of-tree module -- --# default to building for the host --ARCH ?= $(shell uname -m) -- --ifeq ($(KDIR),) --$(error Must specify KDIR to point to the kernel to target)) --endif -- --all: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../include" modules CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR=m -- --clean: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean -diff --git a/dvalin/kernel/drivers/base/protected_memory_allocator/build.bp b/dvalin/kernel/drivers/base/protected_memory_allocator/build.bp -deleted file mode 100644 -index 165b17b..0000000 ---- a/dvalin/kernel/drivers/base/protected_memory_allocator/build.bp -+++ /dev/null -@@ -1,26 +0,0 @@ --/* -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * A copy of the licence is included with the program, and can also be obtained -- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- * Boston, MA 02110-1301, USA. -- */ -- --bob_kernel_module { -- name: "protected_memory_allocator", -- srcs: [ -- "Kbuild", -- "protected_memory_allocator.c", -- ], -- kbuild_options: ["CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR=m"], -- defaults: ["kernel_defaults"], -- enabled: false, -- build_csf_only_module: { -- enabled: true, -- }, --} -diff --git a/dvalin/kernel/drivers/base/protected_memory_allocator/protected_memory_allocator.c b/dvalin/kernel/drivers/base/protected_memory_allocator/protected_memory_allocator.c -deleted file mode 100644 -index bb0b1dd..0000000 ---- a/dvalin/kernel/drivers/base/protected_memory_allocator/protected_memory_allocator.c -+++ /dev/null -@@ -1,308 +0,0 @@ --/* -- * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- */ -- --#include <linux/version.h> --#include <linux/of.h> --#include <linux/of_reserved_mem.h> --#include <linux/platform_device.h> --#include <linux/module.h> --#include <linux/slab.h> --#include <linux/mm.h> --#include <linux/io.h> --#include <linux/protected_memory_allocator.h> -- --/** -- * struct simple_pma_device - Simple implementation of a protected memory -- * allocator device -- * -- * @pma_dev: Protected memory allocator device pointer -- * @dev: Device pointer -- * @alloc_pages: Status of all the physical memory pages within the -- * protected memory region; true for allocated pages -- * @rmem_base: Base address of the reserved memory region -- * @rmem_size: Size of the reserved memory region, in pages -- * @free_pa_offset: Offset of the lowest physical address within the protected -- * memory region that is currently associated with a free page -- * @num_free_pages: Number of free pages in the memory region -- */ --struct simple_pma_device { -- struct protected_memory_allocator_device pma_dev; -- struct device *dev; -- bool *alloc_pages; -- phys_addr_t rmem_base; -- size_t rmem_size; -- size_t free_pa_offset; -- size_t num_free_pages; --}; -- --static struct protected_memory_allocation *simple_pma_alloc_page( -- struct protected_memory_allocator_device *pma_dev, unsigned int order) --{ -- struct simple_pma_device *const epma_dev = -- container_of(pma_dev, struct simple_pma_device, pma_dev); -- struct protected_memory_allocation *pma; -- size_t num_pages; -- size_t i; -- -- dev_dbg(epma_dev->dev, "%s(pma_dev=%px, order=%u\n", -- __func__, (void *)pma_dev, order); -- -- /* This is an example function that follows an extremely simple logic -- * and is very likely to fail to allocate memory if put under stress. -- * -- * The simple_pma_device maintains an array of booleans to track -- * the status of every page and an offset to the free page to use -- * for the next allocation. The offset starts from 0 and can only grow, -- * and be reset when the end of the memory region is reached. -- * -- * In order to create a memory allocation, the allocator simply looks -- * at the offset and verifies whether there are enough free pages -- * after it to accommodate the allocation request. If successful, -- * the allocator shall mark all the pages as allocated and increment -- * the offset accordingly. -- * -- * The allocator does not look for any other free pages inside the -- * memory region, even if plenty of free memory is available. -- * Free memory pages are counted and the offset is ignored if the -- * memory region is fully allocated. -- */ -- -- /* The only candidate for allocation is the sub-region starting -- * from the free_pa_offset. Verify that enough contiguous pages -- * are available and that they are all free. -- */ -- num_pages = (size_t)1 << order; -- -- if (epma_dev->num_free_pages < num_pages) -- dev_err(epma_dev->dev, "not enough free pages\n"); -- -- if (epma_dev->free_pa_offset + num_pages > epma_dev->rmem_size) { -- dev_err(epma_dev->dev, "not enough contiguous pages\n"); -- return NULL; -- } -- -- for (i = 0; i < num_pages; i++) -- if (epma_dev->alloc_pages[epma_dev->free_pa_offset + i]) -- break; -- -- if (i < num_pages) { -- dev_err(epma_dev->dev, "free pages are not contiguous\n"); -- return NULL; -- } -- -- /* Memory allocation is successful. Mark pages as allocated. -- * Update the free_pa_offset if free pages are still available: -- * increment the free_pa_offset accordingly, and then making sure -- * that it points at the next free page, potentially wrapping over -- * the end of the memory region. -- */ -- pma = devm_kzalloc(epma_dev->dev, sizeof(*pma), GFP_KERNEL); -- if (!pma) -- return NULL; -- -- pma->pa = epma_dev->rmem_base + (epma_dev->free_pa_offset << PAGE_SHIFT); -- pma->order = order; -- -- for (i = 0; i < num_pages; i++) -- epma_dev->alloc_pages[epma_dev->free_pa_offset + i] = true; -- -- epma_dev->num_free_pages -= num_pages; -- -- if (epma_dev->num_free_pages) { -- epma_dev->free_pa_offset += num_pages; -- i = 0; -- while (epma_dev->alloc_pages[epma_dev->free_pa_offset + i]) { -- epma_dev->free_pa_offset++; -- if (epma_dev->free_pa_offset > epma_dev->rmem_size) -- epma_dev->free_pa_offset = 0; -- } -- } -- -- return pma; --} -- --static phys_addr_t simple_pma_get_phys_addr( -- struct protected_memory_allocator_device *pma_dev, -- struct protected_memory_allocation *pma) --{ -- struct simple_pma_device *const epma_dev = -- container_of(pma_dev, struct simple_pma_device, pma_dev); -- -- dev_dbg(epma_dev->dev, "%s(pma_dev=%px, pma=%px, pa=%llx\n", -- __func__, (void *)pma_dev, (void *)pma, pma->pa); -- -- return pma->pa; --} -- --static void simple_pma_free_page( -- struct protected_memory_allocator_device *pma_dev, -- struct protected_memory_allocation *pma) --{ -- struct simple_pma_device *const epma_dev = -- container_of(pma_dev, struct simple_pma_device, pma_dev); -- size_t num_pages; -- size_t offset; -- size_t i; -- -- dev_dbg(epma_dev->dev, "%s(pma_dev=%px, pma=%px, pa=%llx\n", -- __func__, (void *)pma_dev, (void *)pma, pma->pa); -- -- /* This is an example function that follows an extremely simple logic -- * and is vulnerable to abuse. For instance, double frees won't be -- * detected. -- * -- * If memory is full, must update the free_pa_offset that is currently -- * pointing at an allocated page. -- * -- * Increase the number of free pages and mark them as free. -- */ -- offset = (pma->pa - epma_dev->rmem_base) >> PAGE_SHIFT; -- num_pages = (size_t)1 << pma->order; -- -- if (epma_dev->num_free_pages == 0) -- epma_dev->free_pa_offset = offset; -- -- epma_dev->num_free_pages += num_pages; -- for (i = 0; i < num_pages; i++) -- epma_dev->alloc_pages[offset + i] = false; -- -- devm_kfree(epma_dev->dev, pma); --} -- --static int protected_memory_allocator_probe(struct platform_device *pdev) --{ -- struct simple_pma_device *epma_dev; -- struct device_node *np; -- phys_addr_t rmem_base; -- size_t rmem_size; --#if (KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE) -- struct reserved_mem *rmem; --#endif -- -- np = pdev->dev.of_node; -- -- if (!np) { -- dev_err(&pdev->dev, "device node pointer not set\n"); -- return -ENODEV; -- } -- -- np = of_parse_phandle(np, "memory-region", 0); -- if (!np) { -- dev_err(&pdev->dev, "memory-region node not set\n"); -- return -ENODEV; -- } -- --#if (KERNEL_VERSION(4, 15, 0) <= LINUX_VERSION_CODE) -- rmem = of_reserved_mem_lookup(np); -- if (rmem) { -- rmem_base = rmem->base; -- rmem_size = rmem->size >> PAGE_SHIFT; -- } else --#endif -- { -- of_node_put(np); -- dev_err(&pdev->dev, "could not read reserved memory-region\n"); -- return -ENODEV; -- } -- -- of_node_put(np); -- epma_dev = devm_kzalloc(&pdev->dev, sizeof(*epma_dev), GFP_KERNEL); -- if (!epma_dev) -- return -ENOMEM; -- -- epma_dev->pma_dev.ops.pma_alloc_page = simple_pma_alloc_page; -- epma_dev->pma_dev.ops.pma_get_phys_addr = simple_pma_get_phys_addr; -- epma_dev->pma_dev.ops.pma_free_page = simple_pma_free_page; -- epma_dev->pma_dev.owner = THIS_MODULE; -- epma_dev->dev = &pdev->dev; -- epma_dev->rmem_base = rmem_base; -- epma_dev->rmem_size = rmem_size; -- epma_dev->free_pa_offset = 0; -- epma_dev->num_free_pages = rmem_size; -- -- epma_dev->alloc_pages = devm_kzalloc(&pdev->dev, -- sizeof(bool) * epma_dev->rmem_size, GFP_KERNEL); -- -- if (!epma_dev->alloc_pages) { -- dev_err(&pdev->dev, "failed to allocate resources\n"); -- devm_kfree(&pdev->dev, epma_dev); -- return -ENOMEM; -- } -- -- platform_set_drvdata(pdev, &epma_dev->pma_dev); -- dev_info(&pdev->dev, -- "Protected memory allocator probed successfully\n"); -- dev_info(&pdev->dev, "Protected memory region: base=%llx num pages=%zu\n", -- rmem_base, rmem_size); -- -- return 0; --} -- --static int protected_memory_allocator_remove(struct platform_device *pdev) --{ -- struct protected_memory_allocator_device *pma_dev = -- platform_get_drvdata(pdev); -- struct simple_pma_device *epma_dev; -- struct device *dev; -- -- if (!pma_dev) -- return -EINVAL; -- -- epma_dev = container_of(pma_dev, struct simple_pma_device, pma_dev); -- dev = epma_dev->dev; -- -- if (epma_dev->num_free_pages < epma_dev->rmem_size) { -- dev_warn(&pdev->dev, "Leaking %zu pages of protected memory\n", -- epma_dev->rmem_size - epma_dev->num_free_pages); -- } -- -- platform_set_drvdata(pdev, NULL); -- devm_kfree(dev, epma_dev->alloc_pages); -- devm_kfree(dev, epma_dev); -- -- dev_info(&pdev->dev, -- "Protected memory allocator removed successfully\n"); -- -- return 0; --} -- --static const struct of_device_id protected_memory_allocator_dt_ids[] = { -- { .compatible = "arm,protected-memory-allocator" }, -- { /* sentinel */ } --}; --MODULE_DEVICE_TABLE(of, protected_memory_allocator_dt_ids); -- --static struct platform_driver protected_memory_allocator_driver = { -- .probe = protected_memory_allocator_probe, -- .remove = protected_memory_allocator_remove, -- .driver = { -- .name = "simple_protected_memory_allocator", -- .owner = THIS_MODULE, -- .of_match_table = of_match_ptr(protected_memory_allocator_dt_ids), -- } --}; -- --module_platform_driver(protected_memory_allocator_driver); -- --MODULE_LICENSE("GPL"); --MODULE_AUTHOR("ARM Ltd."); --MODULE_VERSION("1.0"); -diff --git a/dvalin/kernel/drivers/gpu/arm/Kbuild b/dvalin/kernel/drivers/gpu/arm/Kbuild -index 1a6fa3c..52ea5fb 100644 ---- a/dvalin/kernel/drivers/gpu/arm/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,9 +16,6 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # --# -- - - obj-$(CONFIG_MALI_MIDGARD) += midgard/ -diff --git a/dvalin/kernel/drivers/gpu/arm/Kconfig b/dvalin/kernel/drivers/gpu/arm/Kconfig -index 693b86f..2da8c98 100644 ---- a/dvalin/kernel/drivers/gpu/arm/Kconfig -+++ b/dvalin/kernel/drivers/gpu/arm/Kconfig -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012, 2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,10 +16,7 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # --# -- - - menu "ARM GPU Configuration" - source "drivers/gpu/arm/midgard/Kconfig" -diff --git a/dvalin/kernel/drivers/base/protected_memory_allocator/Kbuild b/dvalin/kernel/drivers/gpu/arm/Makefile -similarity index 77% -rename from dvalin/kernel/drivers/base/protected_memory_allocator/Kbuild -rename to dvalin/kernel/drivers/gpu/arm/Makefile -index 241aeb9..ea9ecc7 100644 ---- a/dvalin/kernel/drivers/base/protected_memory_allocator/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/Makefile -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,8 +16,6 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - --obj-$(CONFIG_MALI_PROTECTED_MEMORY_ALLOCATOR) := protected_memory_allocator.o -\ No newline at end of file -+include midgard/Makefile -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/Kbuild -index fa52548..d3c4ee1 100755 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,202 +16,240 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # -+ -+# make $(src) as absolute path if it is not already, by prefixing $(srctree) -+# This is to prevent any build issue due to wrong path. -+src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src)) -+ - # -+# Prevent misuse when Kernel configurations are not present by default -+# in out-of-tree builds -+# -+ifneq ($(CONFIG_ANDROID),n) -+ifeq ($(CONFIG_GPU_TRACEPOINTS),n) -+ $(error CONFIG_GPU_TRACEPOINTS must be set in Kernel configuration) -+endif -+endif - -+ifeq ($(CONFIG_DMA_SHARED_BUFFER),n) -+ $(error CONFIG_DMA_SHARED_BUFFER must be set in Kernel configuration) -+endif - --# Driver version string which is returned to userspace via an ioctl --MALI_RELEASE_NAME ?= "r25p0-01rel0" -+ifeq ($(CONFIG_PM_DEVFREQ),n) -+ $(error CONFIG_PM_DEVFREQ must be set in Kernel configuration) -+endif - --# Paths required for build --# make $(src) as absolute path if it isn't already, by prefixing $(srctree) --src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src)) --KBASE_PATH = $(src) --KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy --UMP_PATH = $(src)/../../../base -+ifeq ($(CONFIG_DEVFREQ_THERMAL),n) -+ $(error CONFIG_DEVFREQ_THERMAL must be set in Kernel configuration) -+endif -+ -+ifeq ($(CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND),n) -+ $(error CONFIG_DEVFREQ_GOV_SIMPLE_ONDEMAND must be set in Kernel configuration) -+endif -+ -+ifeq ($(CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS), y) -+ ifneq ($(CONFIG_DEBUG_FS), y) -+ $(error CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS depends on CONFIG_DEBUG_FS to be set in Kernel configuration) -+ endif -+endif -+ -+ifeq ($(CONFIG_MALI_FENCE_DEBUG), y) -+ ifneq ($(CONFIG_SYNC), y) -+ ifneq ($(CONFIG_SYNC_FILE), y) -+ $(error CONFIG_MALI_FENCE_DEBUG depends on CONFIG_SYNC || CONFIG_SYNC_FILE to be set in Kernel configuration) -+ endif -+ endif -+endif - -+# -+# Configurations -+# -+ -+# Driver version string which is returned to userspace via an ioctl -+MALI_RELEASE_NAME ?= '"r32p1-01bet0"' - # Set up defaults if not defined by build system --MALI_CUSTOMER_RELEASE ?= 1 --MALI_USE_CSF ?= 0 --MALI_UNIT_TEST ?= 0 --MALI_KERNEL_TEST_API ?= 0 -+ifeq ($(CONFIG_MALI_DEBUG), y) -+ MALI_UNIT_TEST = 1 -+ MALI_CUSTOMER_RELEASE ?= 0 -+else -+ MALI_UNIT_TEST ?= 0 -+ MALI_CUSTOMER_RELEASE ?= 1 -+endif - MALI_COVERAGE ?= 0 -+ - CONFIG_MALI_PLATFORM_NAME ?= "devicetree" -+ -+# Kconfig passes in the name with quotes for in-tree builds - remove them. -+MALI_PLATFORM_DIR := $(shell echo $(CONFIG_MALI_PLATFORM_NAME)) -+ -+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) -+ MALI_JIT_PRESSURE_LIMIT_BASE = 0 -+ MALI_USE_CSF = 1 -+else -+ MALI_JIT_PRESSURE_LIMIT_BASE ?= 1 -+ MALI_USE_CSF ?= 0 -+endif -+ -+ifneq ($(CONFIG_MALI_KUTF), n) -+ MALI_KERNEL_TEST_API ?= 1 -+else -+ MALI_KERNEL_TEST_API ?= 0 -+endif -+ - # Experimental features (corresponding -D definition should be appended to --# DEFINES below, e.g. for MALI_EXPERIMENTAL_FEATURE, -+# ccflags-y below, e.g. for MALI_EXPERIMENTAL_FEATURE, - # -DMALI_EXPERIMENTAL_FEATURE=$(MALI_EXPERIMENTAL_FEATURE) should be appended) - # - # Experimental features must default to disabled, e.g.: - # MALI_EXPERIMENTAL_FEATURE ?= 0 --MALI_JIT_PRESSURE_LIMIT ?= 0 - MALI_INCREMENTAL_RENDERING ?= 0 - --# Set up our defines, which will be passed to gcc --DEFINES = \ -- -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ -- -DMALI_USE_CSF=$(MALI_USE_CSF) \ -- -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ -- -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ -- -DMALI_COVERAGE=$(MALI_COVERAGE) \ -- -DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \ -- -DMALI_JIT_PRESSURE_LIMIT=$(MALI_JIT_PRESSURE_LIMIT) \ -- -DMALI_INCREMENTAL_RENDERING=$(MALI_INCREMENTAL_RENDERING) -+# -+# ccflags -+# -+ccflags-y = \ -+ -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ -+ -DMALI_USE_CSF=$(MALI_USE_CSF) \ -+ -DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \ -+ -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ -+ -DMALI_COVERAGE=$(MALI_COVERAGE) \ -+ -DMALI_RELEASE_NAME=$(MALI_RELEASE_NAME) \ -+ -DMALI_JIT_PRESSURE_LIMIT_BASE=$(MALI_JIT_PRESSURE_LIMIT_BASE) \ -+ -DMALI_INCREMENTAL_RENDERING=$(MALI_INCREMENTAL_RENDERING) \ -+ -DMALI_KBASE_BUILD \ -+ -DMALI_PLATFORM_DIR=$(MALI_PLATFORM_DIR) -+ - - ifeq ($(KBUILD_EXTMOD),) - # in-tree --DEFINES +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) -+ ccflags-y +=-DMALI_KBASE_PLATFORM_PATH=../../$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) - else - # out-of-tree --DEFINES +=-DMALI_KBASE_PLATFORM_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) --endif -- --DEFINES += -I$(srctree)/drivers/staging/android --#meson graphics start --ldflags-y += --strip-debug --#meson graphics end -- --DEFINES += -DMALI_KBASE_BUILD -- --# Use our defines when compiling --ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux --subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux -- --SRC := \ -- context/mali_kbase_context.c \ -- debug/mali_kbase_debug_ktrace.c \ -- device/mali_kbase_device.c \ -- mali_kbase_cache_policy.c \ -- mali_kbase_mem.c \ -- mali_kbase_mem_pool_group.c \ -- mali_kbase_native_mgm.c \ -- mali_kbase_ctx_sched.c \ -- mali_kbase_jm.c \ -- mali_kbase_gpuprops.c \ -- mali_kbase_pm.c \ -- mali_kbase_config.c \ -- mali_kbase_vinstr.c \ -- mali_kbase_hwcnt.c \ -- mali_kbase_hwcnt_backend_gpu.c \ -- mali_kbase_hwcnt_gpu.c \ -- mali_kbase_hwcnt_legacy.c \ -- mali_kbase_hwcnt_types.c \ -- mali_kbase_hwcnt_virtualizer.c \ -- mali_kbase_softjobs.c \ -- mali_kbase_hw.c \ -- mali_kbase_debug.c \ -- mali_kbase_gpu_memory_debugfs.c \ -- mali_kbase_mem_linux.c \ -- mali_kbase_core_linux.c \ -- mali_kbase_mem_profile_debugfs.c \ -- mmu/mali_kbase_mmu.c \ -- mmu/mali_kbase_mmu_hw_direct.c \ -- mmu/mali_kbase_mmu_mode_lpae.c \ -- mmu/mali_kbase_mmu_mode_aarch64.c \ -- mali_kbase_disjoint_events.c \ -- mali_kbase_debug_mem_view.c \ -- mali_kbase_smc.c \ -- mali_kbase_mem_pool.c \ -- mali_kbase_mem_pool_debugfs.c \ -- mali_kbase_debugfs_helper.c \ -- mali_kbase_strings.c \ -- mali_kbase_as_fault_debugfs.c \ -- mali_kbase_regs_history_debugfs.c \ -- thirdparty/mali_kbase_mmap.c \ -- tl/mali_kbase_timeline.c \ -- tl/mali_kbase_timeline_io.c \ -- tl/mali_kbase_tlstream.c \ -- tl/mali_kbase_tracepoints.c \ -- gpu/mali_kbase_gpu.c -- --ifeq ($(MALI_USE_CSF),1) -- SRC += \ -- debug/backend/mali_kbase_debug_ktrace_csf.c \ -- device/backend/mali_kbase_device_csf.c \ -- gpu/backend/mali_kbase_gpu_fault_csf.c \ -- tl/backend/mali_kbase_timeline_csf.c \ -- mmu/backend/mali_kbase_mmu_csf.c \ -- context/backend/mali_kbase_context_csf.c --else -- SRC += \ -- mali_kbase_dummy_job_wa.c \ -- mali_kbase_debug_job_fault.c \ -- mali_kbase_event.c \ -- mali_kbase_jd.c \ -- mali_kbase_jd_debugfs.c \ -- mali_kbase_js.c \ -- mali_kbase_js_ctx_attr.c \ -- debug/backend/mali_kbase_debug_ktrace_jm.c \ -- device/backend/mali_kbase_device_jm.c \ -- gpu/backend/mali_kbase_gpu_fault_jm.c \ -- tl/backend/mali_kbase_timeline_jm.c \ -- mmu/backend/mali_kbase_mmu_jm.c \ -- context/backend/mali_kbase_context_jm.c -+ ccflags-y +=-DMALI_KBASE_PLATFORM_PATH=$(src)/platform/$(CONFIG_MALI_PLATFORM_NAME) - endif - --ifeq ($(CONFIG_MALI_CINSTR_GWT),y) -- SRC += mali_kbase_gwt.c --endif -+ccflags-y += \ -+ -I$(srctree)/include/linux \ -+ -I$(srctree)/drivers/staging/android \ -+ -I$(src) \ -+ -I$(src)/platform/$(MALI_PLATFORM_DIR) \ -+ -I$(src)/../../../base \ -+ -I$(src)/../../../../include - --ifeq ($(MALI_UNIT_TEST),1) -- SRC += tl/mali_kbase_timeline_test.c --endif -+subdir-ccflags-y += $(ccflags-y) - --ifeq ($(MALI_CUSTOMER_RELEASE),0) -- SRC += mali_kbase_regs_dump_debugfs.c --endif -+# -+# Kernel Modules -+# -+obj-$(CONFIG_MALI_MIDGARD) += mali_kbase.o -+obj-$(CONFIG_MALI_ARBITRATION) += arbitration/ -+obj-$(CONFIG_MALI_KUTF) += tests/ -+ -+mali_kbase-y := \ -+ mali_kbase_cache_policy.o \ -+ mali_kbase_ccswe.o \ -+ mali_kbase_mem.o \ -+ mali_kbase_mem_pool_group.o \ -+ mali_kbase_native_mgm.o \ -+ mali_kbase_ctx_sched.o \ -+ mali_kbase_gpuprops.o \ -+ mali_kbase_pm.o \ -+ mali_kbase_config.o \ -+ mali_kbase_vinstr.o \ -+ mali_kbase_hwcnt.o \ -+ mali_kbase_hwcnt_gpu.o \ -+ mali_kbase_hwcnt_legacy.o \ -+ mali_kbase_hwcnt_types.o \ -+ mali_kbase_hwcnt_virtualizer.o \ -+ mali_kbase_softjobs.o \ -+ mali_kbase_hw.o \ -+ mali_kbase_debug.o \ -+ mali_kbase_gpu_memory_debugfs.o \ -+ mali_kbase_mem_linux.o \ -+ mali_kbase_core_linux.o \ -+ mali_kbase_mem_profile_debugfs.o \ -+ mali_kbase_disjoint_events.o \ -+ mali_kbase_debug_mem_view.o \ -+ mali_kbase_smc.o \ -+ mali_kbase_mem_pool.o \ -+ mali_kbase_mem_pool_debugfs.o \ -+ mali_kbase_debugfs_helper.o \ -+ mali_kbase_strings.o \ -+ mali_kbase_as_fault_debugfs.o \ -+ mali_kbase_regs_history_debugfs.o \ -+ mali_kbase_dvfs_debugfs.o \ -+ mali_power_gpu_frequency_trace.o \ -+ mali_kbase_trace_gpu_mem.o -+ -+mali_kbase-$(CONFIG_MALI_CINSTR_GWT) += mali_kbase_gwt.o - -+mali_kbase-$(CONFIG_SYNC) += \ -+ mali_kbase_sync_android.o \ -+ mali_kbase_sync_common.o - --ccflags-y += -I$(KBASE_PATH) -I$(KBASE_PATH)/debug \ -- -I$(KBASE_PATH)/debug/backend -+mali_kbase-$(CONFIG_SYNC_FILE) += \ -+ mali_kbase_fence_ops.o \ -+ mali_kbase_sync_file.o \ -+ mali_kbase_sync_common.o -+ -+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) -+ mali_kbase-y += \ -+ mali_kbase_hwcnt_backend_csf.o \ -+ mali_kbase_hwcnt_backend_csf_if_fw.o -+else -+ mali_kbase-y += \ -+ mali_kbase_jm.o \ -+ mali_kbase_hwcnt_backend_jm.o \ -+ mali_kbase_dummy_job_wa.o \ -+ mali_kbase_debug_job_fault.o \ -+ mali_kbase_event.o \ -+ mali_kbase_jd.o \ -+ mali_kbase_jd_debugfs.o \ -+ mali_kbase_js.o \ -+ mali_kbase_js_ctx_attr.o \ -+ mali_kbase_kinstr_jm.o -+ -+ mali_kbase-$(CONFIG_MALI_DMA_FENCE) += \ -+ mali_kbase_fence_ops.o \ -+ mali_kbase_dma_fence.o \ -+ mali_kbase_fence.o -+ -+ mali_kbase-$(CONFIG_SYNC_FILE) += \ -+ mali_kbase_fence_ops.o \ -+ mali_kbase_fence.o -+endif - --# Tell the Linux build system from which .o file to create the kernel module --obj-$(CONFIG_MALI_MIDGARD) += mali_kbase.o - --# Tell the Linux build system to enable building of our .c files --mali_kbase-y := $(SRC:.c=.o) -+INCLUDE_SUBDIR = \ -+ $(src)/context/Kbuild \ -+ $(src)/debug/Kbuild \ -+ $(src)/device/Kbuild \ -+ $(src)/backend/gpu/Kbuild \ -+ $(src)/mmu/Kbuild \ -+ $(src)/tl/Kbuild \ -+ $(src)/gpu/Kbuild \ -+ $(src)/thirdparty/Kbuild \ -+ $(src)/platform/$(MALI_PLATFORM_DIR)/Kbuild - --# Kconfig passes in the name with quotes for in-tree builds - remove them. --platform_name := $(shell echo $(CONFIG_MALI_PLATFORM_NAME)) --MALI_PLATFORM_DIR := platform/$(platform_name) --ccflags-y += -I$(src)/$(MALI_PLATFORM_DIR) --include $(src)/$(MALI_PLATFORM_DIR)/Kbuild -+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) -+ INCLUDE_SUBDIR += $(src)/csf/Kbuild -+endif - --ifeq ($(CONFIG_MALI_DEVFREQ),y) -- ifeq ($(CONFIG_DEVFREQ_THERMAL),y) -- include $(src)/ipa/Kbuild -- endif -+ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y) -+ INCLUDE_SUBDIR += $(src)/arbiter/Kbuild - endif - --ifeq ($(MALI_USE_CSF),1) -- include $(src)/csf/Kbuild --else --# empty -+ifeq ($(CONFIG_MALI_DEVFREQ),y) -+ ifeq ($(CONFIG_DEVFREQ_THERMAL),y) -+ INCLUDE_SUBDIR += $(src)/ipa/Kbuild -+ endif - endif - --ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y) -- include $(src)/arbiter/Kbuild -+ifeq ($(KBUILD_EXTMOD),) -+# in-tree -+ -include $(INCLUDE_SUBDIR) - else --# empty -+# out-of-tree -+ include $(INCLUDE_SUBDIR) - endif -- --mali_kbase-$(CONFIG_MALI_DMA_FENCE) += \ -- mali_kbase_dma_fence.o \ -- mali_kbase_fence.o --mali_kbase-$(CONFIG_SYNC) += \ -- mali_kbase_sync_android.o \ -- mali_kbase_sync_common.o --mali_kbase-$(CONFIG_SYNC_FILE) += \ -- mali_kbase_sync_file.o \ -- mali_kbase_sync_common.o \ -- mali_kbase_fence.o -- --include $(src)/backend/gpu/Kbuild --mali_kbase-y += $(BACKEND:.c=.o) -- -- --ccflags-y += -I$(src)/backend/gpu --subdir-ccflags-y += -I$(src)/backend/gpu -- --# For kutf and mali_kutf_irq_latency_test --obj-$(CONFIG_MALI_KUTF) += tests/ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/Kconfig b/dvalin/kernel/drivers/gpu/arm/midgard/Kconfig -index ca59dbb..5541383 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/Kconfig -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/Kconfig -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,15 +16,14 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # --# -- - - menuconfig MALI_MIDGARD - tristate "Mali Midgard series support" - select GPU_TRACEPOINTS if ANDROID - select DMA_SHARED_BUFFER -+ select PM_DEVFREQ -+ select DEVFREQ_THERMAL - default n - help - Enable this option to build support for a ARM Mali Midgard GPU. -@@ -31,13 +31,43 @@ menuconfig MALI_MIDGARD - To compile this driver as a module, choose M here: - this will generate a single module, called mali_kbase. - --config MALI_GATOR_SUPPORT -- bool "Enable Streamline tracing support" -+if MALI_MIDGARD -+ -+config MALI_PLATFORM_NAME - depends on MALI_MIDGARD -+ string "Platform name" -+ default "devicetree" -+ help -+ Enter the name of the desired platform configuration directory to -+ include in the build. 'platform/$(MALI_PLATFORM_NAME)/Makefile' must -+ exist. -+ -+config MALI_REAL_HW -+ depends on MALI_MIDGARD -+ def_bool !MALI_NO_MALI -+ -+menu "Platform specific options" -+source "drivers/gpu/arm/midgard/platform/Kconfig" -+endmenu -+ -+config MALI_CSF_SUPPORT -+ bool "Enable Mali CSF based GPU support" -+ depends on MALI_MIDGARD=m -+ default n -+ help -+ Enables support for CSF based GPUs. -+ -+config MALI_DEVFREQ -+ bool "Enable devfreq support for Mali" -+ depends on MALI_MIDGARD && PM_DEVFREQ -+ select DEVFREQ_GOV_SIMPLE_ONDEMAND - default y - help -- Enables kbase tracing used by the Arm Streamline Performance Analyzer. -- The tracepoints are used to derive GPU activity charts in Streamline. -+ Support devfreq for Mali. -+ -+ Using the devfreq framework and, by default, the simple on-demand -+ governor, the frequency of Mali will be dynamically selected from the -+ available OPPs. - - config MALI_MIDGARD_DVFS - bool "Enable legacy DVFS" -@@ -46,28 +76,25 @@ config MALI_MIDGARD_DVFS - help - Choose this option to enable legacy DVFS in the Mali Midgard DDK. - -+config MALI_GATOR_SUPPORT -+ bool "Enable Streamline tracing support" -+ depends on MALI_MIDGARD -+ default y -+ help -+ Enables kbase tracing used by the Arm Streamline Performance Analyzer. -+ The tracepoints are used to derive GPU activity charts in Streamline. -+ - config MALI_MIDGARD_ENABLE_TRACE - bool "Enable kbase tracing" - depends on MALI_MIDGARD - default y if MALI_DEBUG - default n - help -- Enables tracing in kbase. Trace log available through -+ Enables tracing in kbase. Trace log available through - the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled - --config MALI_DEVFREQ -- bool "devfreq support for Mali" -- depends on MALI_MIDGARD && PM_DEVFREQ -- default y -- help -- Support devfreq for Mali. -- -- Using the devfreq framework and, by default, the simpleondemand -- governor, the frequency of Mali will be dynamically selected from the -- available OPPs. -- - config MALI_DMA_FENCE -- bool "DMA_BUF fence support for Mali" -+ bool "Enable DMA_BUF fence support for Mali" - depends on MALI_MIDGARD - default n - help -@@ -76,18 +103,9 @@ config MALI_DMA_FENCE - This option should only be enabled if the Linux Kernel has built in - support for DMA_BUF fences. - --config MALI_PLATFORM_NAME -- depends on MALI_MIDGARD -- string "Platform name" -- default "devicetree" -- help -- Enter the name of the desired platform configuration directory to -- include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must -- exist. -- - config MALI_ARBITER_SUPPORT - bool "Enable arbiter support for Mali" -- depends on MALI_MIDGARD -+ depends on MALI_MIDGARD && !MALI_CSF_SUPPORT - default n - help - Enable support for the arbiter interface in the driver. -@@ -96,18 +114,64 @@ config MALI_ARBITER_SUPPORT - - If unsure, say N. - --# MALI_EXPERT configuration options -+config MALI_DMA_BUF_MAP_ON_DEMAND -+ bool "Enable map imported dma-bufs on demand" -+ depends on MALI_MIDGARD -+ default n -+ help -+ This option caused kbase to set up the GPU mapping of imported -+ dma-buf when needed to run atoms. This is the legacy behavior. -+ -+ This is intended for testing and the option will get removed in the -+ future. -+ -+config MALI_DMA_BUF_LEGACY_COMPAT -+ bool "Enable legacy compatibility cache flush on dma-buf map" -+ depends on MALI_MIDGARD && !MALI_DMA_BUF_MAP_ON_DEMAND -+ default n -+ help -+ This option enables compatibility with legacy dma-buf mapping -+ behavior, then the dma-buf is mapped on import, by adding cache -+ maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping, -+ including a cache flush. -+ -+ This option might work-around issues related to missing cache -+ flushes in other drivers. This only has an effect for clients using -+ UK 11.18 or older. For later UK versions it is not possible. - - menuconfig MALI_EXPERT - depends on MALI_MIDGARD - bool "Enable Expert Settings" - default n - help -- Enabling this option and modifying the default settings may produce a driver with performance or -- other limitations. -+ Enabling this option and modifying the default settings may produce -+ a driver with performance or other limitations. -+ -+if MALI_EXPERT -+ -+config MALI_2MB_ALLOC -+ bool "Attempt to allocate 2MB pages" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ Rather than allocating all GPU memory page-by-page, attempt to -+ allocate 2MB pages from the kernel. This reduces TLB pressure and -+ helps to prevent memory fragmentation. -+ -+ If in doubt, say N -+ -+config MALI_MEMORY_FULLY_BACKED -+ bool "Enable memory fully physically-backed" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ This option enables full physical backing of all virtual -+ memory allocations in the kernel. Notice that this build -+ option only affects allocations of grow-on-GPU-page-fault -+ memory. - - config MALI_CORESTACK -- bool "Support controlling power to the GPU core stack" -+ bool "Enable support of GPU core stack power control" - depends on MALI_MIDGARD && MALI_EXPERT - default n - help -@@ -119,15 +183,48 @@ config MALI_CORESTACK - - If unsure, say N. - -+comment "Platform options" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ -+config MALI_NO_MALI -+ bool "Enable No Mali" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ This can be used to test the driver in a simulated environment -+ whereby the hardware is not physically present. If the hardware is physically -+ present it will not be used. This can be used to test the majority of the -+ driver without needing actual hardware or for software benchmarking. -+ All calls to the simulated hardware will complete immediately as if the hardware -+ completed the task. -+ -+config MALI_ERROR_INJECT -+ bool "Enable No Mali error injection" -+ depends on MALI_MIDGARD && MALI_EXPERT && MALI_NO_MALI -+ default n -+ help -+ Enables insertion of errors to test module failure and recovery mechanisms. -+ -+config MALI_GEM5_BUILD -+ bool "Enable build of Mali kernel driver for GEM5" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ This option is to do a Mali GEM5 build. -+ If unsure, say N. -+ -+comment "Debug options" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ - config MALI_DEBUG -- bool "Debug build" -+ bool "Enable debug build" - depends on MALI_MIDGARD && MALI_EXPERT - default n - help - Select this option for increased checking and reporting of errors. - - config MALI_FENCE_DEBUG -- bool "Debug sync fence usage" -+ bool "Enable debug sync fence usage" - depends on MALI_MIDGARD && MALI_EXPERT && (SYNC || SYNC_FILE) - default y if MALI_DEBUG - help -@@ -143,28 +240,6 @@ config MALI_FENCE_DEBUG - The timeout can be changed at runtime through the js_soft_timeout - device attribute, where the timeout is specified in milliseconds. - --config MALI_NO_MALI -- bool "No Mali" -- depends on MALI_MIDGARD && MALI_EXPERT -- default n -- help -- This can be used to test the driver in a simulated environment -- whereby the hardware is not physically present. If the hardware is physically -- present it will not be used. This can be used to test the majority of the -- driver without needing actual hardware or for software benchmarking. -- All calls to the simulated hardware will complete immediately as if the hardware -- completed the task. -- --config MALI_REAL_HW -- def_bool !MALI_NO_MALI -- --config MALI_ERROR_INJECT -- bool "Error injection" -- depends on MALI_MIDGARD && MALI_EXPERT && MALI_NO_MALI -- default n -- help -- Enables insertion of errors to test module failure and recovery mechanisms. -- - config MALI_SYSTEM_TRACE - bool "Enable system event tracing support" - depends on MALI_MIDGARD && MALI_EXPERT -@@ -176,63 +251,93 @@ config MALI_SYSTEM_TRACE - minimal overhead when not in use. Enable only if you know what - you are doing. - --config MALI_2MB_ALLOC -- bool "Attempt to allocate 2MB pages" -+comment "Instrumentation options" - depends on MALI_MIDGARD && MALI_EXPERT -- default n -- help -- Rather than allocating all GPU memory page-by-page, attempt to -- allocate 2MB pages from the kernel. This reduces TLB pressure and -- helps to prevent memory fragmentation. - -- If in doubt, say N -+choice -+ prompt "Select Performance counters set" -+ default MALI_PRFCNT_SET_PRIMARY -+ depends on MALI_MIDGARD && MALI_EXPERT - --config MALI_PWRSOFT_765 -- bool "PWRSOFT-765 ticket" -+config MALI_PRFCNT_SET_PRIMARY -+ bool "Primary" - depends on MALI_MIDGARD && MALI_EXPERT -- default n - help -- PWRSOFT-765 fixes devfreq cooling devices issues. The fix was merged -- in kernel v4.10, however if backported into the kernel then this -- option must be manually selected. -+ Select this option to use primary set of performance counters. - -- If using kernel >= v4.10 then say N, otherwise if devfreq cooling -- changes have been backported say Y to avoid compilation errors. -+config MALI_PRFCNT_SET_SECONDARY -+ bool "Secondary" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ help -+ Select this option to use secondary set of performance counters. Kernel -+ features that depend on an access to the primary set of counters may -+ become unavailable. Enabling this option will prevent power management -+ from working optimally and may cause instrumentation tools to return -+ bogus results. - --config MALI_MEMORY_FULLY_BACKED -- bool "Memory fully physically-backed" -+ If unsure, use MALI_PRFCNT_SET_PRIMARY. -+ -+config MALI_PRFCNT_SET_TERTIARY -+ bool "Tertiary" - depends on MALI_MIDGARD && MALI_EXPERT -+ help -+ Select this option to use tertiary set of performance counters. Kernel -+ features that depend on an access to the primary set of counters may -+ become unavailable. Enabling this option will prevent power management -+ from working optimally and may cause instrumentation tools to return -+ bogus results. -+ -+ If unsure, use MALI_PRFCNT_SET_PRIMARY. -+ -+endchoice -+ -+config MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS -+ bool "Enable runtime selection of performance counters set via debugfs" -+ depends on MALI_MIDGARD && MALI_EXPERT && DEBUG_FS - default n - help -- This option enables full physical backing of all virtual -- memory allocations in the kernel. Notice that this build -- option only affects allocations of grow-on-GPU-page-fault -- memory. -+ Select this option to make the secondary set of performance counters -+ available at runtime via debugfs. Kernel features that depend on an -+ access to the primary set of counters may become unavailable. - --config MALI_DMA_BUF_MAP_ON_DEMAND -- bool "Map imported dma-bufs on demand" -- depends on MALI_MIDGARD -+ If no runtime debugfs option is set, the build time counter set -+ choice will be used. -+ -+ This feature is unsupported and unstable, and may break at any time. -+ Enabling this option will prevent power management from working -+ optimally and may cause instrumentation tools to return bogus results. -+ -+ No validation is done on the debugfs input. Invalid input could cause -+ performance counter errors. Valid inputs are the values accepted by -+ the SET_SELECT bits of the PRFCNT_CONFIG register as defined in the -+ architecture specification. -+ -+ If unsure, say N. -+ -+config MALI_JOB_DUMP -+ bool "Enable system level support needed for job dumping" -+ depends on MALI_MIDGARD && MALI_EXPERT - default n - help -- This option caused kbase to set up the GPU mapping of imported -- dma-buf when needed to run atoms. This is the legacy behaviour. -+ Choose this option to enable system level support needed for -+ job dumping. This is typically used for instrumentation but has -+ minimal overhead when not in use. Enable only if you know what -+ you are doing. - -- This is intended for testing and the option will get removed in the -- future. -+comment "Workarounds" -+ depends on MALI_MIDGARD && MALI_EXPERT - --config MALI_DMA_BUF_LEGACY_COMPAT -- bool "Enable legacy compatibility cache flush on dma-buf map" -- depends on MALI_MIDGARD && !MALI_DMA_BUF_MAP_ON_DEMAND -+config MALI_PWRSOFT_765 -+ bool "Enable workaround for PWRSOFT-765" -+ depends on MALI_MIDGARD && MALI_EXPERT - default n - help -- This option enables compatibility with legacy dma-buf mapping -- behavior, then the dma-buf is mapped on import, by adding cache -- maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping, -- including a cache flush. -+ PWRSOFT-765 fixes devfreq cooling devices issues. The fix was merged -+ in kernel v4.10, however if backported into the kernel then this -+ option must be manually selected. - -- This option might work-around issues related to missing cache -- flushes in other drivers. This only has an effect for clients using -- UK 11.18 or older. For later UK versions it is not possible. -+ If using kernel >= v4.10 then say N, otherwise if devfreq cooling -+ changes have been backported say Y to avoid compilation errors. - - config MALI_HW_ERRATA_1485982_NOT_AFFECTED - bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336" -@@ -252,58 +357,22 @@ config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE - default n - help - This option uses an alternative workaround for GPU2017-1336. Lowering -- the GPU clock to a, platform specific, known good frequeuncy before -+ the GPU clock to a, platform specific, known good frequency before - powering down the L2 cache. The clock can be specified in the device - tree using the property, opp-mali-errata-1485982. Otherwise the - slowest clock will be selected. - --config MALI_GEM5_BUILD -- bool "Enable build of Mali kernel driver for GEM5" -- depends on MALI_MIDGARD -- default n -- help -- This option is to do a Mali GEM5 build. -- If unsure, say N. -- --# Instrumentation options. -+endif - --config MALI_JOB_DUMP -- bool "Enable system level support needed for job dumping" -- depends on MALI_MIDGARD && MALI_EXPERT -- default n -- help -- Choose this option to enable system level support needed for -- job dumping. This is typically used for instrumentation but has -- minimal overhead when not in use. Enable only if you know what -- you are doing. -- --config MALI_PRFCNT_SET_SECONDARY -- bool "Use secondary set of performance counters" -- depends on MALI_MIDGARD && MALI_EXPERT -- default n -- help -- Select this option to use secondary set of performance counters. Kernel -- features that depend on an access to the primary set of counters may -- become unavailable. Enabling this option will prevent power management -- from working optimally and may cause instrumentation tools to return -- bogus results. -- -- If unsure, say N. -- --config MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS -- bool "Use secondary set of performance counters" -- depends on MALI_MIDGARD && MALI_EXPERT && !MALI_PRFCNT_SET_SECONDARY && DEBUG_FS -+config MALI_ARBITRATION -+ bool "Enable Virtualization reference code" -+ depends on MALI_MIDGARD - default n - help -- Select this option to make the secondary set of performance counters -- available at runtime via debugfs. Kernel features that depend on an -- access to the primary set of counters may become unavailable. -- -- This feature is unsupported and unstable, and may break at any time. -- Enabling this option will prevent power management from working -- optimally and may cause instrumentation tools to return bogus results. -- -+ Enables the build of several reference modules used in the reference -+ virtualization setup for Mali - If unsure, say N. - --source "drivers/gpu/arm/midgard/platform/Kconfig" - source "drivers/gpu/arm/midgard/tests/Kconfig" -+ -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/Makefile b/dvalin/kernel/drivers/gpu/arm/midgard/Makefile -index 53a1209..4384e80 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/Makefile -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/Makefile -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,24 +16,200 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # -+ -+KERNEL_SRC ?= /lib/modules/$(shell uname -r)/build -+KDIR ?= $(KERNEL_SRC) -+ -+ifeq ($(KDIR),) -+ $(error Must specify KDIR to point to the kernel to target)) -+endif -+ -+# -+# Default configuration values -+# -+# Dependency resolution is done through statements as Kconfig -+# is not supported for out-of-tree builds. - # - -+CONFIG_MALI_MIDGARD ?= m -+ifeq ($(CONFIG_MALI_MIDGARD),m) -+ CONFIG_MALI_GATOR_SUPPORT ?= y -+ CONFIG_MALI_ARBITRATION ?= n -+ CONFIG_MALI_PARTITION_MANAGER ?= n -+ -+ ifneq ($(CONFIG_MALI_NO_MALI),y) -+ # Prevent misuse when CONFIG_MALI_NO_MALI=y -+ CONFIG_MALI_REAL_HW ?= y -+ endif -+ -+ ifeq ($(CONFIG_MALI_MIDGARD_DVFS),y) -+ # Prevent misuse when CONFIG_MALI_MIDGARD_DVFS=y -+ CONFIG_MALI_DEVFREQ ?= n -+ else -+ CONFIG_MALI_DEVFREQ ?= y -+ endif -+ -+ ifeq ($(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND), y) -+ # Prevent misuse when CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND=y -+ CONFIG_MALI_DMA_BUF_LEGACY_COMPAT = n -+ endif -+ -+ ifeq ($(CONFIG_BSP_HAS_HYPERVISOR),y) -+ ifneq ($(CONFIG_MALI_ARBITRATION), n) -+ CONFIG_MALI_XEN ?= m -+ endif -+ endif -+ -+ # -+ # Expert/Debug/Test released configurations -+ # -+ ifeq ($(CONFIG_MALI_EXPERT), y) -+ ifeq ($(CONFIG_MALI_NO_MALI), y) -+ CONFIG_MALI_REAL_HW = n -+ else -+ # Prevent misuse when CONFIG_MALI_NO_MALI=n -+ CONFIG_MALI_REAL_HW = y -+ CONFIG_MALI_ERROR_INJECT = n -+ endif -+ -+ ifeq ($(CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED), y) -+ # Prevent misuse when CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED=y -+ CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE = n -+ endif - --KDIR ?= /lib/modules/$(shell uname -r)/build -+ ifeq ($(CONFIG_MALI_DEBUG), y) -+ CONFIG_MALI_MIDGARD_ENABLE_TRACE ?= y -+ CONFIG_MALI_SYSTEM_TRACE ?= y - --BUSLOG_PATH_RELATIVE = $(CURDIR)/../../../.. --KBASE_PATH_RELATIVE = $(CURDIR) -+ ifeq ($(CONFIG_SYNC), y) -+ CONFIG_MALI_FENCE_DEBUG ?= y -+ else -+ ifeq ($(CONFIG_SYNC_FILE), y) -+ CONFIG_MALI_FENCE_DEBUG ?= y -+ else -+ CONFIG_MALI_FENCE_DEBUG = n -+ endif -+ endif -+ else -+ # Prevent misuse when CONFIG_MALI_DEBUG=n -+ CONFIG_MALI_MIDGARD_ENABLE_TRACE = n -+ CONFIG_MALI_SYSTEM_TRACE = n -+ CONFIG_MALI_FENCE_DEBUG = n -+ endif -+ else -+ # Prevent misuse when CONFIG_MALI_EXPERT=n -+ CONFIG_MALI_CORESTACK = n -+ CONFIG_MALI_2MB_ALLOC = n -+ CONFIG_MALI_PWRSOFT_765 = n -+ CONFIG_MALI_MEMORY_FULLY_BACKED = n -+ CONFIG_MALI_JOB_DUMP = n -+ CONFIG_MALI_NO_MALI = n -+ CONFIG_MALI_REAL_HW = y -+ CONFIG_MALI_ERROR_INJECT = n -+ CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED = n -+ CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE = n -+ CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS = n -+ CONFIG_MALI_DEBUG = n -+ CONFIG_MALI_MIDGARD_ENABLE_TRACE = n -+ CONFIG_MALI_SYSTEM_TRACE = n -+ CONFIG_MALI_FENCE_DEBUG = n -+ endif - --ifeq ($(CONFIG_MALI_BUSLOG),y) --#Add bus logger symbols --EXTRA_SYMBOLS += $(BUSLOG_PATH_RELATIVE)/drivers/base/bus_logger/Module.symvers -+ ifeq ($(CONFIG_MALI_DEBUG), y) -+ CONFIG_MALI_KUTF ?= y -+ ifeq ($(CONFIG_MALI_KUTF), y) -+ CONFIG_MALI_KUTF_IRQ_TEST ?= y -+ CONFIG_MALI_KUTF_CLK_RATE_TRACE ?= y -+ else -+ # Prevent misuse when CONFIG_MALI_KUTF=n -+ CONFIG_MALI_KUTF_IRQ_TEST = n -+ CONFIG_MALI_KUTF_CLK_RATE_TRACE = n -+ endif -+ else -+ # Prevent misuse when CONFIG_MALI_DEBUG=n -+ CONFIG_MALI_KUTF = n -+ CONFIG_MALI_KUTF_IRQ_TEST = n -+ CONFIG_MALI_KUTF_CLK_RATE_TRACE = n -+ endif -+else -+ # Prevent misuse when CONFIG_MALI_MIDGARD=n -+ CONFIG_MALI_ARBITRATION = n -+ CONFIG_MALI_KUTF = n -+ CONFIG_MALI_KUTF_IRQ_TEST = n -+ CONFIG_MALI_KUTF_CLK_RATE_TRACE = n - endif - --# we get the symbols from modules using KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions -+# All Mali CONFIG should be listed here -+CONFIGS := \ -+ CONFIG_MALI_MIDGARD \ -+ CONFIG_MALI_CSF_SUPPORT \ -+ CONFIG_MALI_GATOR_SUPPORT \ -+ CONFIG_MALI_DMA_FENCE \ -+ CONFIG_MALI_ARBITER_SUPPORT \ -+ CONFIG_MALI_ARBITRATION \ -+ CONFIG_MALI_PARTITION_MANAGER \ -+ CONFIG_MALI_REAL_HW \ -+ CONFIG_MALI_GEM5_BUILD \ -+ CONFIG_MALI_DEVFREQ \ -+ CONFIG_MALI_MIDGARD_DVFS \ -+ CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND \ -+ CONFIG_MALI_DMA_BUF_LEGACY_COMPAT \ -+ CONFIG_MALI_EXPERT \ -+ CONFIG_MALI_CORESTACK \ -+ CONFIG_MALI_2MB_ALLOC \ -+ CONFIG_MALI_PWRSOFT_765 \ -+ CONFIG_MALI_MEMORY_FULLY_BACKED \ -+ CONFIG_MALI_JOB_DUMP \ -+ CONFIG_MALI_NO_MALI \ -+ CONFIG_MALI_ERROR_INJECT \ -+ CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED \ -+ CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE \ -+ CONFIG_MALI_PRFCNT_SET_PRIMARY \ -+ CONFIG_MALI_PRFCNT_SET_SECONDARY \ -+ CONFIG_MALI_PRFCNT_SET_TERTIARY \ -+ CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS \ -+ CONFIG_MALI_DEBUG \ -+ CONFIG_MALI_MIDGARD_ENABLE_TRACE \ -+ CONFIG_MALI_SYSTEM_TRACE \ -+ CONFIG_MALI_FENCE_DEBUG \ -+ CONFIG_MALI_KUTF \ -+ CONFIG_MALI_KUTF_IRQ_TEST \ -+ CONFIG_MALI_KUTF_CLK_RATE_TRACE \ -+ CONFIG_MALI_XEN -+ -+ -+# -+# MAKE_ARGS to pass the custom CONFIGs on out-of-tree build -+# -+# Generate the list of CONFIGs and values. -+# $(value config) is the name of the CONFIG option. -+# $(value $(value config)) is its value (y, m). -+# When the CONFIG is not set to y or m, it defaults to n. -+MAKE_ARGS := $(foreach config,$(CONFIGS), \ -+ $(if $(filter y m,$(value $(value config))), \ -+ $(value config)=$(value $(value config)), \ -+ $(value config)=n)) -+ -+# -+# EXTRA_CFLAGS to define the custom CONFIGs on out-of-tree build -+# -+# Generate the list of CONFIGs defines with values from CONFIGS. -+# $(value config) is the name of the CONFIG option. -+# When set to y or m, the CONFIG gets defined to 1. -+EXTRA_CFLAGS := $(foreach config,$(CONFIGS), \ -+ $(if $(filter y m,$(value $(value config))), \ -+ -D$(value config)=1)) -+ -+# -+# KBUILD_EXTRA_SYMBOLS to prevent warnings about unknown functions -+# -+ - all: -- $(MAKE) -C $(KDIR) M=$(CURDIR) EXTRA_CFLAGS="-I$(CURDIR)/../../../../include -I$(CURDIR)/../../../../tests/include $(SCONS_CFLAGS)" $(SCONS_CONFIGS) KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules -+ $(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) EXTRA_CFLAGS="$(EXTRA_CFLAGS)" KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" modules -+ -+modules_install: -+ $(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) modules_install - - clean: -- $(MAKE) -C $(KDIR) M=$(CURDIR) clean -+ $(MAKE) -C $(KDIR) M=$(CURDIR) $(MAKE_ARGS) clean -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/Mconfig b/dvalin/kernel/drivers/gpu/arm/midgard/Mconfig -index b137793..d71a113 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/Mconfig -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/Mconfig -@@ -1,17 +1,22 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # --# A copy of the licence is included with the program, and can also be obtained --# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, --# Boston, MA 02110-1301, USA. -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. - # - # -- - - menuconfig MALI_MIDGARD - bool "Mali Midgard series support" -@@ -22,13 +27,44 @@ menuconfig MALI_MIDGARD - To compile this driver as a module, choose M here: - this will generate a single module, called mali_kbase. - --config MALI_GATOR_SUPPORT -- bool "Enable Streamline tracing support" -- depends on MALI_MIDGARD && !BACKEND_USER -+config MALI_PLATFORM_NAME -+ depends on MALI_MIDGARD -+ string "Platform name" -+ default "hisilicon" if PLATFORM_HIKEY960 -+ default "hisilicon" if PLATFORM_HIKEY970 -+ default "devicetree" -+ help -+ Enter the name of the desired platform configuration directory to -+ include in the build. 'platform/$(MALI_PLATFORM_NAME)/Makefile' must -+ exist. -+ -+ When PLATFORM_CUSTOM is set, this needs to be set manually to -+ pick up the desired platform files. -+ -+config MALI_REAL_HW -+ bool -+ depends on MALI_MIDGARD - default y -+ default n if NO_MALI -+ -+config MALI_CSF_SUPPORT -+ bool "Enable Mali CSF based GPU support" -+ depends on MALI_MIDGARD -+ default y if GPU_HAS_CSF - help -- Enables kbase tracing used by the Arm Streamline Performance Analyzer. -- The tracepoints are used to derive GPU activity charts in Streamline. -+ Enables support for CSF based GPUs. -+ -+config MALI_DEVFREQ -+ bool "Enable devfreq support for Mali" -+ depends on MALI_MIDGARD -+ default y if PLATFORM_JUNO -+ default y if PLATFORM_CUSTOM -+ help -+ Support devfreq for Mali. -+ -+ Using the devfreq framework and, by default, the simple on-demand -+ governor, the frequency of Mali will be dynamically selected from the -+ available OPPs. - - config MALI_MIDGARD_DVFS - bool "Enable legacy DVFS" -@@ -37,29 +73,25 @@ config MALI_MIDGARD_DVFS - help - Choose this option to enable legacy DVFS in the Mali Midgard DDK. - -+config MALI_GATOR_SUPPORT -+ bool "Enable Streamline tracing support" -+ depends on MALI_MIDGARD && !BACKEND_USER -+ default y -+ help -+ Enables kbase tracing used by the Arm Streamline Performance Analyzer. -+ The tracepoints are used to derive GPU activity charts in Streamline. -+ - config MALI_MIDGARD_ENABLE_TRACE - bool "Enable kbase tracing" - depends on MALI_MIDGARD - default y if MALI_DEBUG - default n - help -- Enables tracing in kbase. Trace log available through -+ Enables tracing in kbase. Trace log available through - the "mali_trace" debugfs file, when the CONFIG_DEBUG_FS is enabled - --config MALI_DEVFREQ -- bool "devfreq support for Mali" -- depends on MALI_MIDGARD -- default y if PLATFORM_JUNO -- default y if PLATFORM_CUSTOM -- help -- Support devfreq for Mali. -- -- Using the devfreq framework and, by default, the simpleondemand -- governor, the frequency of Mali will be dynamically selected from the -- available OPPs. -- - config MALI_DMA_FENCE -- bool "DMA_BUF fence support for Mali" -+ bool "Enable DMA_BUF fence support for Mali" - depends on MALI_MIDGARD - default n - help -@@ -68,23 +100,9 @@ config MALI_DMA_FENCE - This option should only be enabled if the Linux Kernel has built in - support for DMA_BUF fences. - --config MALI_PLATFORM_NAME -- depends on MALI_MIDGARD -- string "Platform name" -- default "hisilicon" if PLATFORM_HIKEY960 -- default "hisilicon" if PLATFORM_HIKEY970 -- default "devicetree" -- help -- Enter the name of the desired platform configuration directory to -- include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must -- exist. -- -- When PLATFORM_CUSTOM is set, this needs to be set manually to -- pick up the desired platform files. -- - config MALI_ARBITER_SUPPORT - bool "Enable arbiter support for Mali" -- depends on MALI_MIDGARD -+ depends on MALI_MIDGARD && !MALI_CSF_SUPPORT - default n - help - Enable support for the arbiter interface in the driver. -@@ -93,62 +111,89 @@ config MALI_ARBITER_SUPPORT - - If unsure, say N. - --# MALI_EXPERT configuration options -+config DMA_BUF_SYNC_IOCTL_SUPPORTED -+ bool "Enable Kernel DMA buffers support DMA_BUF_IOCTL_SYNC" -+ depends on MALI_MIDGARD && BACKEND_KERNEL -+ default y -+ -+config MALI_DMA_BUF_MAP_ON_DEMAND -+ bool "Enable map imported dma-bufs on demand" -+ depends on MALI_MIDGARD -+ default n -+ default y if !DMA_BUF_SYNC_IOCTL_SUPPORTED -+ help -+ This option caused kbase to set up the GPU mapping of imported -+ dma-buf when needed to run atoms. This is the legacy behavior. -+ -+ This is intended for testing and the option will get removed in the -+ future. -+ -+config MALI_DMA_BUF_LEGACY_COMPAT -+ bool "Enable legacy compatibility cache flush on dma-buf map" -+ depends on MALI_MIDGARD && !MALI_DMA_BUF_MAP_ON_DEMAND -+ default n -+ help -+ This option enables compatibility with legacy dma-buf mapping -+ behavior, then the dma-buf is mapped on import, by adding cache -+ maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping, -+ including a cache flush. -+ -+ This option might work-around issues related to missing cache -+ flushes in other drivers. This only has an effect for clients using -+ UK 11.18 or older. For later UK versions it is not possible. - - menuconfig MALI_EXPERT - depends on MALI_MIDGARD - bool "Enable Expert Settings" - default y - help -- Enabling this option and modifying the default settings may produce a driver with performance or -- other limitations. -+ Enabling this option and modifying the default settings may produce -+ a driver with performance or other limitations. - --config MALI_CORESTACK -- bool "Support controlling power to the GPU core stack" -+config MALI_2MB_ALLOC -+ bool "Attempt to allocate 2MB pages" - depends on MALI_MIDGARD && MALI_EXPERT - default n - help -- Enabling this feature on supported GPUs will let the driver powering -- on/off the GPU core stack independently without involving the Power -- Domain Controller. This should only be enabled on platforms which -- integration of the PDC to the Mali GPU is known to be problematic. -- This feature is currently only supported on t-Six and t-HEx GPUs. -+ Rather than allocating all GPU memory page-by-page, attempt to -+ allocate 2MB pages from the kernel. This reduces TLB pressure and -+ helps to prevent memory fragmentation. - -- If unsure, say N. -+ If in doubt, say N - --config MALI_DEBUG -- bool "Debug build" -+config MALI_MEMORY_FULLY_BACKED -+ bool "Enable memory fully physically-backed" - depends on MALI_MIDGARD && MALI_EXPERT -- default y if DEBUG - default n - help -- Select this option for increased checking and reporting of errors. -+ This option enables full physical backing of all virtual -+ memory allocations in the kernel. Notice that this build -+ option only affects allocations of grow-on-GPU-page-fault -+ memory. - --config MALI_FENCE_DEBUG -- bool "Debug sync fence usage" -+config MALI_CORESTACK -+ bool "Enable support of GPU core stack power control" - depends on MALI_MIDGARD && MALI_EXPERT -- default y if MALI_DEBUG -+ default n - help -- Select this option to enable additional checking and reporting on the -- use of sync fences in the Mali driver. -- -- This will add a 3s timeout to all sync fence waits in the Mali -- driver, so that when work for Mali has been waiting on a sync fence -- for a long time a debug message will be printed, detailing what fence -- is causing the block, and which dependent Mali atoms are blocked as a -- result of this. -+ Enabling this feature on supported GPUs will let the driver powering -+ on/off the GPU core stack independently without involving the Power -+ Domain Controller. This should only be enabled on platforms which -+ integration of the PDC to the Mali GPU is known to be problematic. -+ This feature is currently only supported on t-Six and t-HEx GPUs. - -- The timeout can be changed at runtime through the js_soft_timeout -- device attribute, where the timeout is specified in milliseconds. -+ If unsure, say N. - - choice - prompt "Error injection level" -+ depends on MALI_MIDGARD && MALI_EXPERT - default MALI_ERROR_INJECT_NONE - help - Enables insertion of errors to test module failure and recovery mechanisms. - - config MALI_ERROR_INJECT_NONE - bool "disabled" -+ depends on MALI_MIDGARD && MALI_EXPERT - help - Error injection is disabled. - -@@ -168,14 +213,49 @@ endchoice - - config MALI_ERROR_INJECT_ON - string -+ depends on MALI_MIDGARD && MALI_EXPERT - default "0" if MALI_ERROR_INJECT_NONE - default "1" if MALI_ERROR_INJECT_TRACK_LIST - default "2" if MALI_ERROR_INJECT_RANDOM - - config MALI_ERROR_INJECT - bool -+ depends on MALI_MIDGARD && MALI_EXPERT - default y if !MALI_ERROR_INJECT_NONE - -+config MALI_GEM5_BUILD -+ bool "Enable build of Mali kernel driver for GEM5" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default n -+ help -+ This option is to do a Mali GEM5 build. -+ If unsure, say N. -+ -+config MALI_DEBUG -+ bool "Enable debug build" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default y if DEBUG -+ default n -+ help -+ Select this option for increased checking and reporting of errors. -+ -+config MALI_FENCE_DEBUG -+ bool "Enable debug sync fence usage" -+ depends on MALI_MIDGARD && MALI_EXPERT -+ default y if MALI_DEBUG -+ help -+ Select this option to enable additional checking and reporting on the -+ use of sync fences in the Mali driver. -+ -+ This will add a 3s timeout to all sync fence waits in the Mali -+ driver, so that when work for Mali has been waiting on a sync fence -+ for a long time a debug message will be printed, detailing what fence -+ is causing the block, and which dependent Mali atoms are blocked as a -+ result of this. -+ -+ The timeout can be changed at runtime through the js_soft_timeout -+ device attribute, where the timeout is specified in milliseconds. -+ - config MALI_SYSTEM_TRACE - bool "Enable system event tracing support" - depends on MALI_MIDGARD && MALI_EXPERT -@@ -187,56 +267,35 @@ config MALI_SYSTEM_TRACE - minimal overhead when not in use. Enable only if you know what - you are doing. - --config MALI_2MB_ALLOC -- bool "Attempt to allocate 2MB pages" -- depends on MALI_MIDGARD && MALI_EXPERT -- default n -- help -- Rather than allocating all GPU memory page-by-page, attempt to -- allocate 2MB pages from the kernel. This reduces TLB pressure and -- helps to prevent memory fragmentation. -+# Instrumentation options. - -- If in doubt, say N -+# config MALI_PRFCNT_SET_PRIMARY exists in the Kernel Kconfig but is configured using CINSTR_PRIMARY_HWC in Mconfig. -+# config MALI_PRFCNT_SET_SECONDARY exists in the Kernel Kconfig but is configured using CINSTR_SECONDARY_HWC in Mconfig. -+# config MALI_PRFCNT_SET_TERTIARY exists in the Kernel Kconfig but is configured using CINSTR_TERTIARY_HWC in Mconfig. -+# config MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS exists in the Kernel Kconfig but is configured using CINSTR_HWC_SET_SELECT_VIA_DEBUG_FS in Mconfig. - --config MALI_PWRSOFT_765 -- bool "PWRSOFT-765 ticket" -+config MALI_JOB_DUMP -+ bool "Enable system level support needed for job dumping" - depends on MALI_MIDGARD && MALI_EXPERT - default n - help -- PWRSOFT-765 fixes devfreq cooling devices issues. However, they are -- not merged in mainline kernel yet. So this define helps to guard those -- parts of the code. -- --config MALI_MEMORY_FULLY_BACKED -- bool "Memory fully physically-backed" -- default n -- help -- This option enables full backing of all virtual memory allocations -- for the kernel. This only affects grow-on-GPU-page-fault memory. -+ Choose this option to enable system level support needed for -+ job dumping. This is typically used for instrumentation but has -+ minimal overhead when not in use. Enable only if you know what -+ you are doing. - --config MALI_DMA_BUF_MAP_ON_DEMAND -- bool "Map imported dma-bufs on demand" -- depends on MALI_MIDGARD -+config MALI_PWRSOFT_765 -+ bool "Enable workaround for PWRSOFT-765" -+ depends on MALI_MIDGARD && MALI_EXPERT - default n -- default y if !DMA_BUF_SYNC_IOCTL_SUPPORTED - help -- This option caused kbase to set up the GPU mapping of imported -- dma-buf when needed to run atoms. This is the legacy behaviour. -+ PWRSOFT-765 fixes devfreq cooling devices issues. The fix was merged -+ in kernel v4.10, however if backported into the kernel then this -+ option must be manually selected. - --config MALI_DMA_BUF_LEGACY_COMPAT -- bool "Enable legacy compatibility cache flush on dma-buf map" -- depends on MALI_MIDGARD && !MALI_DMA_BUF_MAP_ON_DEMAND -- default n -- help -- This option enables compatibility with legacy dma-buf mapping -- behavior, then the dma-buf is mapped on import, by adding cache -- maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping, -- including a cache flush. -+ If using kernel >= v4.10 then say N, otherwise if devfreq cooling -+ changes have been backported say Y to avoid compilation errors. - --config MALI_REAL_HW -- bool -- default y -- default n if NO_MALI - - config MALI_HW_ERRATA_1485982_NOT_AFFECTED - bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336" -@@ -262,17 +321,6 @@ config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE - tree using the property, opp-mali-errata-1485982. Otherwise the - slowest clock will be selected. - --config MALI_GEM5_BUILD -- bool "Enable build of Mali kernel driver for GEM5" -- depends on MALI_MIDGARD -- default n -- help -- This option is to do a Mali GEM5 build. -- If unsure, say N. -- --# Instrumentation options. -- --# config MALI_JOB_DUMP exists in the Kernel Kconfig but is configured using CINSTR_JOB_DUMP in Mconfig. --# config MALI_PRFCNT_SET_SECONDARY exists in the Kernel Kconfig but is configured using CINSTR_SECONDARY_HWC in Mconfig. - -+source "kernel/drivers/gpu/arm/midgard/arbitration/Mconfig" - source "kernel/drivers/gpu/arm/midgard/tests/Mconfig" -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/Kbuild -index 98e47be..5203281 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,10 +16,8 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - mali_kbase-y += \ -- arbiter/mali_kbase_arbif.o \ -- arbiter/mali_kbase_arbiter_pm.o -+ arbiter/mali_kbase_arbif.o \ -+ arbiter/mali_kbase_arbiter_pm.o -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbif.c b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbif.c -index d193cb9..64e11ce 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbif.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbif.c -@@ -1,13 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -- -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -18,13 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -- * @file mali_kbase_arbif.c -- * Mali arbiter interface APIs to share GPU between Virtual Machines -+ * DOC: Mali arbiter interface APIs to share GPU between Virtual Machines - */ - - #include <mali_kbase.h> -@@ -34,32 +30,155 @@ - #include <linux/of_platform.h> - #include "mali_kbase_arbiter_interface.h" - -+/* Arbiter interface version against which was implemented this module */ -+#define MALI_REQUIRED_KBASE_ARBITER_INTERFACE_VERSION 5 -+#if MALI_REQUIRED_KBASE_ARBITER_INTERFACE_VERSION != \ -+ MALI_KBASE_ARBITER_INTERFACE_VERSION -+#error "Unsupported Mali Arbiter interface version." -+#endif -+ -+static void on_max_config(struct device *dev, uint32_t max_l2_slices, -+ uint32_t max_core_mask) -+{ -+ struct kbase_device *kbdev; -+ -+ if (!dev) { -+ pr_err("%s(): dev is NULL", __func__); -+ return; -+ } -+ -+ kbdev = dev_get_drvdata(dev); -+ if (!kbdev) { -+ dev_err(dev, "%s(): kbdev is NULL", __func__); -+ return; -+ } -+ -+ if (!max_l2_slices || !max_core_mask) { -+ dev_dbg(dev, -+ "%s(): max_config ignored as one of the fields is zero", -+ __func__); -+ return; -+ } -+ -+ /* set the max config info in the kbase device */ -+ kbase_arbiter_set_max_config(kbdev, max_l2_slices, max_core_mask); -+} -+ -+/** -+ * on_update_freq() - Updates GPU clock frequency -+ * @dev: arbiter interface device handle -+ * @freq: GPU clock frequency value reported from arbiter -+ * -+ * call back function to update GPU clock frequency with -+ * new value from arbiter -+ */ -+static void on_update_freq(struct device *dev, uint32_t freq) -+{ -+ struct kbase_device *kbdev; -+ -+ if (!dev) { -+ pr_err("%s(): dev is NULL", __func__); -+ return; -+ } -+ -+ kbdev = dev_get_drvdata(dev); -+ if (!kbdev) { -+ dev_err(dev, "%s(): kbdev is NULL", __func__); -+ return; -+ } -+ -+ kbase_arbiter_pm_update_gpu_freq(&kbdev->arb.arb_freq, freq); -+} -+ -+/** -+ * on_gpu_stop() - sends KBASE_VM_GPU_STOP_EVT event on VM stop -+ * @dev: arbiter interface device handle -+ * -+ * call back function to signal a GPU STOP event from arbiter interface -+ */ - static void on_gpu_stop(struct device *dev) - { -- struct kbase_device *kbdev = dev_get_drvdata(dev); -+ struct kbase_device *kbdev; -+ -+ if (!dev) { -+ pr_err("%s(): dev is NULL", __func__); -+ return; -+ } - -- KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED(kbdev, kbdev); -+ kbdev = dev_get_drvdata(dev); -+ if (!kbdev) { -+ dev_err(dev, "%s(): kbdev is NULL", __func__); -+ return; -+ } -+ -+ KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED(kbdev, kbdev); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT); - } - -+/** -+ * on_gpu_granted() - sends KBASE_VM_GPU_GRANTED_EVT event on GPU granted -+ * @dev: arbiter interface device handle -+ * -+ * call back function to signal a GPU GRANT event from arbiter interface -+ */ - static void on_gpu_granted(struct device *dev) - { -- struct kbase_device *kbdev = dev_get_drvdata(dev); -+ struct kbase_device *kbdev; -+ -+ if (!dev) { -+ pr_err("%s(): dev is NULL", __func__); -+ return; -+ } -+ -+ kbdev = dev_get_drvdata(dev); -+ if (!kbdev) { -+ dev_err(dev, "%s(): kbdev is NULL", __func__); -+ return; -+ } - -- KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED(kbdev, kbdev); -+ KBASE_TLSTREAM_TL_ARBITER_GRANTED(kbdev, kbdev); - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT); - } - -+/** -+ * on_gpu_lost() - sends KBASE_VM_GPU_LOST_EVT event on GPU granted -+ * @dev: arbiter interface device handle -+ * -+ * call back function to signal a GPU LOST event from arbiter interface -+ */ - static void on_gpu_lost(struct device *dev) - { -- struct kbase_device *kbdev = dev_get_drvdata(dev); -+ struct kbase_device *kbdev; -+ -+ if (!dev) { -+ pr_err("%s(): dev is NULL", __func__); -+ return; -+ } -+ -+ kbdev = dev_get_drvdata(dev); -+ if (!kbdev) { -+ dev_err(dev, "%s(): kbdev is NULL", __func__); -+ return; -+ } - - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT); - } - -+/** -+ * kbase_arbif_init() - Kbase Arbiter interface initialisation. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Initialise Kbase Arbiter interface and assign callback functions. -+ * -+ * Return: -+ * * 0 - the interface was initialized or was not specified -+ * * in the device tree. -+ * * -EFAULT - the interface was specified but failed to initialize. -+ * * -EPROBE_DEFER - module dependencies are not yet available. -+ */ - int kbase_arbif_init(struct kbase_device *kbdev) - { --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - struct arbiter_if_arb_vm_ops ops; - struct arbiter_if_dev *arb_if; - struct device_node *arbiter_if_node; -@@ -100,17 +219,26 @@ int kbase_arbif_init(struct kbase_device *kbdev) - ops.arb_vm_gpu_stop = on_gpu_stop; - ops.arb_vm_gpu_granted = on_gpu_granted; - ops.arb_vm_gpu_lost = on_gpu_lost; -+ ops.arb_vm_max_config = on_max_config; -+ ops.arb_vm_update_freq = on_update_freq; -+ -+ kbdev->arb.arb_freq.arb_freq = 0; -+ kbdev->arb.arb_freq.freq_updated = false; -+ mutex_init(&kbdev->arb.arb_freq.arb_freq_lock); - - /* register kbase arbiter_if callbacks */ - if (arb_if->vm_ops.vm_arb_register_dev) { - err = arb_if->vm_ops.vm_arb_register_dev(arb_if, - kbdev->dev, &ops); - if (err) { -- dev_err(kbdev->dev, "Arbiter registration failed.\n"); -+ dev_err(&pdev->dev, "Failed to register with arbiter\n"); - module_put(pdev->dev.driver->owner); -+ if (err != -EPROBE_DEFER) -+ err = -EFAULT; - return err; - } - } -+ - #else /* CONFIG_OF */ - dev_dbg(kbdev->dev, "No arbiter without Device Tree support\n"); - kbdev->arb.arb_dev = NULL; -@@ -119,6 +247,12 @@ int kbase_arbif_init(struct kbase_device *kbdev) - return 0; - } - -+/** -+ * kbase_arbif_destroy() - De-init Kbase arbiter interface -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * De-initialise Kbase arbiter interface -+ */ - void kbase_arbif_destroy(struct kbase_device *kbdev) - { - struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; -@@ -133,27 +267,64 @@ void kbase_arbif_destroy(struct kbase_device *kbdev) - kbdev->arb.arb_dev = NULL; - } - -+/** -+ * kbase_arbif_get_max_config() - Request max config info -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * call back function from arb interface to arbiter requesting max config info -+ */ -+void kbase_arbif_get_max_config(struct kbase_device *kbdev) -+{ -+ struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; -+ -+ if (arb_if && arb_if->vm_ops.vm_arb_get_max_config) { -+ dev_dbg(kbdev->dev, "%s\n", __func__); -+ arb_if->vm_ops.vm_arb_get_max_config(arb_if); -+ } -+} -+ -+/** -+ * kbase_arbif_gpu_request() - Request GPU from -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * call back function from arb interface to arbiter requesting GPU for VM -+ */ - void kbase_arbif_gpu_request(struct kbase_device *kbdev) - { - struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; - - if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) { - dev_dbg(kbdev->dev, "%s\n", __func__); -+ KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev); - arb_if->vm_ops.vm_arb_gpu_request(arb_if); - } - } - -+/** -+ * kbase_arbif_gpu_stopped() - send GPU stopped message to the arbiter -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @gpu_required: GPU request flag -+ * -+ */ - void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required) - { - struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; - - if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) { - dev_dbg(kbdev->dev, "%s\n", __func__); -- KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED(kbdev, kbdev); -+ KBASE_TLSTREAM_TL_ARBITER_STOPPED(kbdev, kbdev); -+ if (gpu_required) -+ KBASE_TLSTREAM_TL_ARBITER_REQUESTED(kbdev, kbdev); - arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required); - } - } - -+/** -+ * kbase_arbif_gpu_active() - Sends a GPU_ACTIVE message to the Arbiter -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Informs the arbiter VM is active -+ */ - void kbase_arbif_gpu_active(struct kbase_device *kbdev) - { - struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; -@@ -164,6 +335,12 @@ void kbase_arbif_gpu_active(struct kbase_device *kbdev) - } - } - -+/** -+ * kbase_arbif_gpu_idle() - Inform the arbiter that the VM has gone idle -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Informs the arbiter VM is idle -+ */ - void kbase_arbif_gpu_idle(struct kbase_device *kbdev) - { - struct arbiter_if_dev *arb_if = kbdev->arb.arb_if; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbif.h b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbif.h -index e7e9de7..701ffd4 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbif.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbif.h -@@ -1,28 +1,7 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ -- --/* -- * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software -@@ -38,12 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * - */ - - /** -- * @file -- * Mali arbiter interface APIs to share GPU between Virtual Machines -+ * DOC: Mali arbiter interface APIs to share GPU between Virtual Machines - */ - - #ifndef _MALI_KBASE_ARBIF_H_ -@@ -80,8 +57,11 @@ enum kbase_arbif_evt { - * Initialize the arbiter interface and also determines - * if Arbiter functionality is required. - * -- * Return: 0 if the Arbiter interface was successfully initialized or the -- * Arbiter was not required. -+ * Return: -+ * * 0 - the interface was initialized or was not specified -+ * * in the device tree. -+ * * -EFAULT - the interface was specified but failed to initialize. -+ * * -EPROBE_DEFER - module dependencies are not yet available. - */ - int kbase_arbif_init(struct kbase_device *kbdev); - -@@ -94,6 +74,14 @@ int kbase_arbif_init(struct kbase_device *kbdev); - */ - void kbase_arbif_destroy(struct kbase_device *kbdev); - -+/** -+ * kbase_arbif_get_max_config() - Request max config info -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * call back function from arb interface to arbiter requesting max config info -+ */ -+void kbase_arbif_get_max_config(struct kbase_device *kbdev); -+ - /** - * kbase_arbif_gpu_request() - Send GPU request message to the arbiter - * @kbdev: The kbase device structure for the device (must be a valid pointer) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_defs.h -index 1f53cbf..570a82a 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_defs.h -@@ -1,28 +1,7 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ -- --/* -- * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software -@@ -38,7 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * - */ - - /** -@@ -66,7 +44,8 @@ - * @vm_resume_work: Work item for vm_arb_wq to resume current work on GPU - * @vm_arb_starting: Work queue resume in progress - * @vm_arb_stopping: Work queue suspend in progress -- * @vm_arb_users_waiting: Count of users waiting for GPU -+ * @interrupts_installed: Flag set when interrupts are installed -+ * @vm_request_timer: Timer to monitor GPU request - */ - struct kbase_arbiter_vm_state { - struct kbase_device *kbdev; -@@ -78,7 +57,8 @@ struct kbase_arbiter_vm_state { - struct work_struct vm_resume_work; - bool vm_arb_starting; - bool vm_arb_stopping; -- int vm_arb_users_waiting; -+ bool interrupts_installed; -+ struct hrtimer vm_request_timer; - }; - - /** -@@ -86,10 +66,12 @@ struct kbase_arbiter_vm_state { - * allocated from the probe method of Mali driver - * @arb_if: Pointer to the arbiter interface device - * @arb_dev: Pointer to the arbiter device -+ * @arb_freq: GPU clock frequency retrieved from arbiter. - */ - struct kbase_arbiter_device { - struct arbiter_if_dev *arb_if; - struct device *arb_dev; -+ struct kbase_arbiter_freq arb_freq; - }; - - #endif /* _MALI_KBASE_ARBITER_DEFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_interface.h b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_interface.h -index 5d5d8a7..c0137f7 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_interface.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_interface.h -@@ -1,28 +1,7 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ -- --/* -- * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software -@@ -38,7 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * - */ - - /** -@@ -50,7 +28,7 @@ - #define _MALI_KBASE_ARBITER_INTERFACE_H_ - - /** -- * @brief Mali arbiter interface version -+ * Mali arbiter interface version - * - * This specifies the current version of the configuration interface. Whenever - * the arbiter interface changes, so that integration effort is required, the -@@ -61,8 +39,15 @@ - * 1 - Added the Mali arbiter configuration interface. - * 2 - Strip out reference code from header - * 3 - Removed DVFS utilization interface (DVFS moved to arbiter side) -+ * 4 - Added max_config support -+ * 5 - Added GPU clock frequency reporting support from arbiter - */ --#define MALI_KBASE_ARBITER_INTERFACE_VERSION 3 -+#define MALI_KBASE_ARBITER_INTERFACE_VERSION 5 -+ -+/** -+ * NO_FREQ is used in case platform doesn't support reporting frequency -+ */ -+#define NO_FREQ 0 - - struct arbiter_if_dev; - -@@ -108,6 +93,27 @@ struct arbiter_if_arb_vm_ops { - * If successful, will respond with a vm_arb_gpu_stopped message. - */ - void (*arb_vm_gpu_lost)(struct device *dev); -+ -+ /** -+ * arb_vm_max_config() - Send max config info to the VM -+ * @dev: The arbif kernel module device. -+ * @max_l2_slices: The maximum number of L2 slices. -+ * @max_core_mask: The largest core mask. -+ * -+ * Informs KBase the maximum resources that can be allocated to the -+ * partition in use. -+ */ -+ void (*arb_vm_max_config)(struct device *dev, uint32_t max_l2_slices, -+ uint32_t max_core_mask); -+ -+ /** -+ * arb_vm_update_freq() - GPU clock frequency has been updated -+ * @dev: The arbif kernel module device. -+ * @freq: GPU clock frequency value reported from arbiter -+ * -+ * Informs KBase that the GPU clock frequency has been updated. -+ */ -+ void (*arb_vm_update_freq)(struct device *dev, uint32_t freq); - }; - - /** -@@ -126,6 +132,11 @@ struct arbiter_if_vm_arb_ops { - * @dev: The device structure to supply in the callbacks. - * @ops: The callbacks that the device driver supports - * (none are optional). -+ * -+ * Return: -+ * * 0 - successful. -+ * * -EINVAL - invalid argument. -+ * * -EPROBE_DEFER - module dependencies are not yet available. - */ - int (*vm_arb_register_dev)(struct arbiter_if_dev *arbif_dev, - struct device *dev, struct arbiter_if_arb_vm_ops *ops); -@@ -136,6 +147,13 @@ struct arbiter_if_vm_arb_ops { - */ - void (*vm_arb_unregister_dev)(struct arbiter_if_dev *arbif_dev); - -+ /** -+ * vm_arb_gpu_get_max_config() - Request the max config from the -+ * Arbiter. -+ * @arbif_dev: The arbiter interface we want to issue the request. -+ */ -+ void (*vm_arb_get_max_config)(struct arbiter_if_dev *arbif_dev); -+ - /** - * vm_arb_gpu_request() - Ask the arbiter interface for GPU access. - * @arbif_dev: The arbiter interface we want to issue the request. -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_pm.c b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_pm.c -index 6c35e16..5c75686 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_pm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_pm.c -@@ -1,13 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -- -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -18,27 +17,49 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -- * @file mali_kbase_arbiter_pm.c -+ * @file - * Mali arbiter power manager state machine and APIs - */ - - #include <mali_kbase.h> - #include <mali_kbase_pm.h> - #include <mali_kbase_hwaccess_jm.h> --#include <mali_kbase_irq_internal.h> -+#include <backend/gpu/mali_kbase_irq_internal.h> - #include <mali_kbase_hwcnt_context.h> --#include <mali_kbase_pm_internal.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> - #include <tl/mali_kbase_tracepoints.h> -+#include <mali_kbase_gpuprops.h> -+ -+/* A dmesg warning will occur if the GPU is not granted -+ * after the following time (in milliseconds) has ellapsed. -+ */ -+#define GPU_REQUEST_TIMEOUT 1000 -+#define KHZ_TO_HZ 1000 -+ -+#define MAX_L2_SLICES_MASK 0xFF -+ -+/* Maximum time in ms, before deferring probe incase -+ * GPU_GRANTED message is not received -+ */ -+static int gpu_req_timeout = 1; -+module_param(gpu_req_timeout, int, 0644); -+MODULE_PARM_DESC(gpu_req_timeout, -+ "On a virtualized platform, if the GPU is not granted within this time(ms) kbase will defer the probe"); - - static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev); - static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld( - struct kbase_device *kbdev); - -+/** -+ * kbase_arbiter_pm_vm_state_str() - Helper function to get string -+ * for kbase VM state.(debug) -+ * @state: kbase VM state -+ * -+ * Return: string representation of Kbase_vm_state -+ */ - static inline const char *kbase_arbiter_pm_vm_state_str( - enum kbase_vm_state state) - { -@@ -73,6 +94,13 @@ static inline const char *kbase_arbiter_pm_vm_state_str( - } - } - -+/** -+ * kbase_arbiter_pm_vm_event_str() - Helper function to get string -+ * for kbase VM event.(debug) -+ * @evt: kbase VM state -+ * -+ * Return: String representation of Kbase_arbif_event -+ */ - static inline const char *kbase_arbiter_pm_vm_event_str( - enum kbase_arbif_evt evt) - { -@@ -99,6 +127,13 @@ static inline const char *kbase_arbiter_pm_vm_event_str( - } - } - -+/** -+ * kbase_arbiter_pm_vm_set_state() - Sets new kbase_arbiter_vm_state -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @new_state: kbase VM new state -+ * -+ * This function sets the new state for the VM -+ */ - static void kbase_arbiter_pm_vm_set_state(struct kbase_device *kbdev, - enum kbase_vm_state new_state) - { -@@ -107,11 +142,22 @@ static void kbase_arbiter_pm_vm_set_state(struct kbase_device *kbdev, - dev_dbg(kbdev->dev, "VM set_state %s -> %s", - kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state), - kbase_arbiter_pm_vm_state_str(new_state)); -+ - lockdep_assert_held(&arb_vm_state->vm_state_lock); - arb_vm_state->vm_state = new_state; -+ if (new_state != KBASE_VM_STATE_INITIALIZING_WITH_GPU && -+ new_state != KBASE_VM_STATE_INITIALIZING) -+ KBASE_KTRACE_ADD(kbdev, ARB_VM_STATE, NULL, new_state); - wake_up(&arb_vm_state->vm_state_wait); - } - -+/** -+ * kbase_arbiter_pm_suspend_wq() - suspend work queue of the driver. -+ * @data: work queue -+ * -+ * Suspends work queue of the driver, when VM is in SUSPEND_PENDING or -+ * STOPPING_IDLE or STOPPING_ACTIVE state -+ */ - static void kbase_arbiter_pm_suspend_wq(struct work_struct *data) - { - struct kbase_arbiter_vm_state *arb_vm_state = container_of(data, -@@ -136,6 +182,13 @@ static void kbase_arbiter_pm_suspend_wq(struct work_struct *data) - dev_dbg(kbdev->dev, "<%s\n", __func__); - } - -+/** -+ * kbase_arbiter_pm_resume_wq() -Kbase resume work queue. -+ * @data: work item -+ * -+ * Resume work queue of the driver when VM is in STARTING state, -+ * else if its in STOPPING_ACTIVE will request a stop event. -+ */ - static void kbase_arbiter_pm_resume_wq(struct work_struct *data) - { - struct kbase_arbiter_vm_state *arb_vm_state = container_of(data, -@@ -157,9 +210,74 @@ static void kbase_arbiter_pm_resume_wq(struct work_struct *data) - } - arb_vm_state->vm_arb_starting = false; - mutex_unlock(&arb_vm_state->vm_state_lock); -+ KBASE_TLSTREAM_TL_ARBITER_STARTED(kbdev, kbdev); - dev_dbg(kbdev->dev, "<%s\n", __func__); - } - -+/** -+ * request_timer_callback() - Issue warning on request timer expiration -+ * @timer: Request hr timer data -+ * -+ * Called when the Arbiter takes too long to grant the GPU after a -+ * request has been made. Issues a warning in dmesg. -+ * -+ * Return: Always returns HRTIMER_NORESTART -+ */ -+static enum hrtimer_restart request_timer_callback(struct hrtimer *timer) -+{ -+ struct kbase_arbiter_vm_state *arb_vm_state = container_of(timer, -+ struct kbase_arbiter_vm_state, vm_request_timer); -+ -+ KBASE_DEBUG_ASSERT(arb_vm_state); -+ KBASE_DEBUG_ASSERT(arb_vm_state->kbdev); -+ -+ dev_warn(arb_vm_state->kbdev->dev, -+ "Still waiting for GPU to be granted from Arbiter after %d ms\n", -+ GPU_REQUEST_TIMEOUT); -+ return HRTIMER_NORESTART; -+} -+ -+/** -+ * start_request_timer() - Start a timer after requesting GPU -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Start a timer to track when kbase is waiting for the GPU from the -+ * Arbiter. If the timer expires before GPU is granted, a warning in -+ * dmesg will be issued. -+ */ -+static void start_request_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -+ -+ hrtimer_start(&arb_vm_state->vm_request_timer, -+ HR_TIMER_DELAY_MSEC(GPU_REQUEST_TIMEOUT), -+ HRTIMER_MODE_REL); -+} -+ -+/** -+ * cancel_request_timer() - Stop the request timer -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Stops the request timer once GPU has been granted. Safe to call -+ * even if timer is no longer running. -+ */ -+static void cancel_request_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -+ -+ hrtimer_cancel(&arb_vm_state->vm_request_timer); -+} -+ -+/** -+ * kbase_arbiter_pm_early_init() - Initialize arbiter for VM -+ * Paravirtualized use. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Initialize the arbiter and other required resources during the runtime -+ * and request the GPU for the VM for the first time. -+ * -+ * Return: 0 if success, or a Linux error code -+ */ - int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) - { - int err; -@@ -179,29 +297,49 @@ int kbase_arbiter_pm_early_init(struct kbase_device *kbdev) - WQ_HIGHPRI); - if (!arb_vm_state->vm_arb_wq) { - dev_err(kbdev->dev, "Failed to allocate vm_arb workqueue\n"); -+ kfree(arb_vm_state); - return -ENOMEM; - } - INIT_WORK(&arb_vm_state->vm_suspend_work, kbase_arbiter_pm_suspend_wq); - INIT_WORK(&arb_vm_state->vm_resume_work, kbase_arbiter_pm_resume_wq); - arb_vm_state->vm_arb_starting = false; -- arb_vm_state->vm_arb_users_waiting = 0; -+ atomic_set(&kbdev->pm.gpu_users_waiting, 0); -+ hrtimer_init(&arb_vm_state->vm_request_timer, CLOCK_MONOTONIC, -+ HRTIMER_MODE_REL); -+ arb_vm_state->vm_request_timer.function = -+ request_timer_callback; - kbdev->pm.arb_vm_state = arb_vm_state; - - err = kbase_arbif_init(kbdev); - if (err) { -+ dev_err(kbdev->dev, "Failed to initialise arbif module\n"); - goto arbif_init_fail; - } -+ - if (kbdev->arb.arb_if) { - kbase_arbif_gpu_request(kbdev); - dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n"); -- wait_event(arb_vm_state->vm_state_wait, -+ err = wait_event_timeout(arb_vm_state->vm_state_wait, - arb_vm_state->vm_state == -- KBASE_VM_STATE_INITIALIZING_WITH_GPU); -+ KBASE_VM_STATE_INITIALIZING_WITH_GPU, -+ msecs_to_jiffies(gpu_req_timeout)); -+ -+ if (!err) { -+ dev_dbg(kbdev->dev, -+ "Kbase probe Deferred after waiting %d ms to receive GPU_GRANT\n", -+ gpu_req_timeout); -+ err = -EPROBE_DEFER; -+ goto arbif_eprobe_defer; -+ } -+ - dev_dbg(kbdev->dev, - "Waiting for initial GPU assignment - done\n"); - } - return 0; - -+arbif_eprobe_defer: -+ kbase_arbiter_pm_early_term(kbdev); -+ return err; - arbif_init_fail: - destroy_workqueue(arb_vm_state->vm_arb_wq); - kfree(arb_vm_state); -@@ -209,35 +347,72 @@ arbif_init_fail: - return err; - } - -+/** -+ * kbase_arbiter_pm_early_term() - Shutdown arbiter and free resources -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Clean up all the resources -+ */ - void kbase_arbiter_pm_early_term(struct kbase_device *kbdev) - { - struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - -+ cancel_request_timer(kbdev); - mutex_lock(&arb_vm_state->vm_state_lock); -- if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) -+ if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED) { -+ kbase_pm_set_gpu_lost(kbdev, false); - kbase_arbif_gpu_stopped(kbdev, false); -- -+ } - mutex_unlock(&arb_vm_state->vm_state_lock); -- kbase_arbif_destroy(kbdev); - destroy_workqueue(arb_vm_state->vm_arb_wq); -+ kbase_arbif_destroy(kbdev); - arb_vm_state->vm_arb_wq = NULL; - kfree(kbdev->pm.arb_vm_state); - kbdev->pm.arb_vm_state = NULL; - } - -+/** -+ * kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Releases interrupts and set the interrupt flag to false -+ */ - void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev) - { - struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - - mutex_lock(&arb_vm_state->vm_state_lock); -- if (!kbdev->arb.arb_if || -- arb_vm_state->vm_state > -- KBASE_VM_STATE_STOPPED_GPU_REQUESTED) -+ if (arb_vm_state->interrupts_installed == true) { -+ arb_vm_state->interrupts_installed = false; - kbase_release_interrupts(kbdev); -+ } -+ mutex_unlock(&arb_vm_state->vm_state_lock); -+} - -+/** -+ * kbase_arbiter_pm_install_interrupts() - Install the GPU interrupts -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Install interrupts and set the interrupt_install flag to true. -+ */ -+int kbase_arbiter_pm_install_interrupts(struct kbase_device *kbdev) -+{ -+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -+ int err; -+ -+ mutex_lock(&arb_vm_state->vm_state_lock); -+ arb_vm_state->interrupts_installed = true; -+ err = kbase_install_interrupts(kbdev); - mutex_unlock(&arb_vm_state->vm_state_lock); -+ return err; - } - -+/** -+ * kbase_arbiter_pm_vm_stopped() - Handle stop state for the VM -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Handles a stop state for the VM -+ */ - void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) - { - bool request_gpu = false; -@@ -245,14 +420,19 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) - - lockdep_assert_held(&arb_vm_state->vm_state_lock); - -- if (arb_vm_state->vm_arb_users_waiting > 0 && -+ if (atomic_read(&kbdev->pm.gpu_users_waiting) > 0 && - arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE) - kbase_arbiter_pm_vm_set_state(kbdev, - KBASE_VM_STATE_STOPPING_ACTIVE); - - dev_dbg(kbdev->dev, "%s %s\n", __func__, - kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); -- kbase_release_interrupts(kbdev); -+ -+ if (arb_vm_state->interrupts_installed) { -+ arb_vm_state->interrupts_installed = false; -+ kbase_release_interrupts(kbdev); -+ } -+ - switch (arb_vm_state->vm_state) { - case KBASE_VM_STATE_STOPPING_ACTIVE: - request_gpu = true; -@@ -271,14 +451,95 @@ void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev) - break; - } - -+ kbase_pm_set_gpu_lost(kbdev, false); - kbase_arbif_gpu_stopped(kbdev, request_gpu); -+ if (request_gpu) -+ start_request_timer(kbdev); -+} -+ -+void kbase_arbiter_set_max_config(struct kbase_device *kbdev, -+ uint32_t max_l2_slices, -+ uint32_t max_core_mask) -+{ -+ struct kbase_arbiter_vm_state *arb_vm_state; -+ struct max_config_props max_config; -+ -+ if (!kbdev) -+ return; -+ -+ /* Mask the max_l2_slices as it is stored as 8 bits into kbase */ -+ max_config.l2_slices = max_l2_slices & MAX_L2_SLICES_MASK; -+ max_config.core_mask = max_core_mask; -+ arb_vm_state = kbdev->pm.arb_vm_state; -+ -+ mutex_lock(&arb_vm_state->vm_state_lock); -+ /* Just set the max_props in kbase during initialization. */ -+ if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING) -+ kbase_gpuprops_set_max_config(kbdev, &max_config); -+ else -+ dev_dbg(kbdev->dev, "Unexpected max_config on VM state %s", -+ kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); -+ -+ mutex_unlock(&arb_vm_state->vm_state_lock); -+} -+ -+int kbase_arbiter_pm_gpu_assigned(struct kbase_device *kbdev) -+{ -+ struct kbase_arbiter_vm_state *arb_vm_state; -+ int result = -EINVAL; -+ -+ if (!kbdev) -+ return result; -+ -+ /* First check the GPU_LOST state */ -+ kbase_pm_lock(kbdev); -+ if (kbase_pm_is_gpu_lost(kbdev)) { -+ kbase_pm_unlock(kbdev); -+ return 0; -+ } -+ kbase_pm_unlock(kbdev); -+ -+ /* Then the arbitration state machine */ -+ arb_vm_state = kbdev->pm.arb_vm_state; -+ -+ mutex_lock(&arb_vm_state->vm_state_lock); -+ switch (arb_vm_state->vm_state) { -+ case KBASE_VM_STATE_INITIALIZING: -+ case KBASE_VM_STATE_SUSPENDED: -+ case KBASE_VM_STATE_STOPPED: -+ case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: -+ case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: -+ result = 0; -+ break; -+ default: -+ result = 1; -+ break; -+ } -+ mutex_unlock(&arb_vm_state->vm_state_lock); -+ -+ return result; - } - -+/** -+ * kbase_arbiter_pm_vm_gpu_start() - Handles the start state of the VM -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Handles the start state of the VM -+ */ - static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) - { - struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -+ bool freq_updated = false; - - lockdep_assert_held(&arb_vm_state->vm_state_lock); -+ mutex_lock(&kbdev->arb.arb_freq.arb_freq_lock); -+ if (kbdev->arb.arb_freq.freq_updated) { -+ kbdev->arb.arb_freq.freq_updated = false; -+ freq_updated = true; -+ } -+ mutex_unlock(&kbdev->arb.arb_freq.arb_freq_lock); -+ -+ cancel_request_timer(kbdev); - switch (arb_vm_state->vm_state) { - case KBASE_VM_STATE_INITIALIZING: - kbase_arbiter_pm_vm_set_state(kbdev, -@@ -286,22 +547,43 @@ static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev) - break; - case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: - kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING); -+ arb_vm_state->interrupts_installed = true; - kbase_install_interrupts(kbdev); -+ /* -+ * GPU GRANTED received while in stop can be a result of a -+ * repartitioning. -+ */ -+ kbase_gpuprops_req_curr_config_update(kbdev); -+ /* curr_config will be updated while resuming the PM. */ - queue_work(arb_vm_state->vm_arb_wq, - &arb_vm_state->vm_resume_work); - break; - case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: -+ kbase_pm_set_gpu_lost(kbdev, false); - kbase_arbif_gpu_stopped(kbdev, false); - kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); - break; - default: -- dev_warn(kbdev->dev, -- "GPU_GRANTED when not expected - state %s\n", -- kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); -+ /* -+ * GPU_GRANTED can be received when there is a frequency update -+ * Only show a warning if received in an unexpected state -+ * without a frequency update -+ */ -+ if (!freq_updated) -+ dev_warn(kbdev->dev, -+ "GPU_GRANTED when not expected - state %s\n", -+ kbase_arbiter_pm_vm_state_str( -+ arb_vm_state->vm_state)); - break; - } - } - -+/** -+ * kbase_arbiter_pm_vm_gpu_stop() - Handles the stop state of the VM -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Handles the start state of the VM -+ */ - static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev) - { - struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -@@ -344,9 +626,16 @@ static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev) - } - } - -+/** -+ * kbase_gpu_lost() - Kbase signals GPU is lost on a lost event signal -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * On GPU lost event signals GPU_LOST to the aribiter -+ */ - static void kbase_gpu_lost(struct kbase_device *kbdev) - { - struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -+ bool handle_gpu_lost = false; - - lockdep_assert_held(&arb_vm_state->vm_state_lock); - -@@ -357,33 +646,47 @@ static void kbase_gpu_lost(struct kbase_device *kbdev) - dev_warn(kbdev->dev, "GPU lost in state %s", - kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state)); - kbase_arbiter_pm_vm_gpu_stop(kbdev); -- mutex_unlock(&arb_vm_state->vm_state_lock); -- kbase_pm_handle_gpu_lost(kbdev); -- mutex_lock(&arb_vm_state->vm_state_lock); -+ handle_gpu_lost = true; - break; - case KBASE_VM_STATE_STOPPING_IDLE: - case KBASE_VM_STATE_STOPPING_ACTIVE: - case KBASE_VM_STATE_SUSPEND_PENDING: -- dev_info(kbdev->dev, "GPU lost while stopping"); -- mutex_unlock(&arb_vm_state->vm_state_lock); -- kbase_pm_handle_gpu_lost(kbdev); -- mutex_lock(&arb_vm_state->vm_state_lock); -+ dev_dbg(kbdev->dev, "GPU lost while stopping"); -+ handle_gpu_lost = true; - break; - case KBASE_VM_STATE_SUSPENDED: - case KBASE_VM_STATE_STOPPED: - case KBASE_VM_STATE_STOPPED_GPU_REQUESTED: -- dev_info(kbdev->dev, "GPU lost while already stopped"); -+ dev_dbg(kbdev->dev, "GPU lost while already stopped"); - break; - case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: -- dev_info(kbdev->dev, "GPU lost while waiting to suspend"); -+ dev_dbg(kbdev->dev, "GPU lost while waiting to suspend"); - kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED); - break; - default: - break; - } -- -+ if (handle_gpu_lost) { -+ /* Releasing the VM state lock here is safe because -+ * we are guaranteed to be in either STOPPING_IDLE, -+ * STOPPING_ACTIVE or SUSPEND_PENDING at this point. -+ * The only transitions that are valid from here are to -+ * STOPPED, STOPPED_GPU_REQUESTED or SUSPENDED which can -+ * only happen at the completion of the GPU lost handling. -+ */ -+ mutex_unlock(&arb_vm_state->vm_state_lock); -+ kbase_pm_handle_gpu_lost(kbdev); -+ mutex_lock(&arb_vm_state->vm_state_lock); -+ } - } - -+/** -+ * kbase_arbiter_pm_vm_os_suspend_ready_state() - checks if VM is ready -+ * to be moved to suspended state. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Return: True if its ready to be suspended else False. -+ */ - static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state( - struct kbase_device *kbdev) - { -@@ -398,6 +701,14 @@ static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state( - } - } - -+/** -+ * kbase_arbiter_pm_vm_os_prepare_suspend() - Prepare OS to be in suspend state -+ * until it receives the grant message from arbiter -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Prepares OS to be in suspend state until it receives GRANT message -+ * from Arbiter asynchronously. -+ */ - static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) - { - struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -@@ -463,6 +774,14 @@ static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev) - } - } - -+/** -+ * kbase_arbiter_pm_vm_os_resume() - Resume OS function once it receives -+ * a grant message from arbiter -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Resume OS function once it receives GRANT message -+ * from Arbiter asynchronously. -+ */ - static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev) - { - struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -@@ -475,6 +794,7 @@ static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev) - kbase_arbiter_pm_vm_set_state(kbdev, - KBASE_VM_STATE_STOPPED_GPU_REQUESTED); - kbase_arbif_gpu_request(kbdev); -+ start_request_timer(kbdev); - - /* Release lock and block resume OS function until we have - * asynchronously received the GRANT message from the Arbiter and -@@ -486,6 +806,14 @@ static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev) - mutex_lock(&arb_vm_state->vm_state_lock); - } - -+/** -+ * kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @evt: VM event -+ * -+ * The state machine function. Receives events and transitions states -+ * according the event received and the current state -+ */ - void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, - enum kbase_arbif_evt evt) - { -@@ -497,7 +825,9 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, - mutex_lock(&arb_vm_state->vm_state_lock); - dev_dbg(kbdev->dev, "%s %s\n", __func__, - kbase_arbiter_pm_vm_event_str(evt)); -- -+ if (arb_vm_state->vm_state != KBASE_VM_STATE_INITIALIZING_WITH_GPU && -+ arb_vm_state->vm_state != KBASE_VM_STATE_INITIALIZING) -+ KBASE_KTRACE_ADD(kbdev, ARB_VM_EVT, NULL, evt); - switch (evt) { - case KBASE_VM_GPU_GRANTED_EVT: - kbase_arbiter_pm_vm_gpu_start(kbdev); -@@ -506,7 +836,7 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, - kbase_arbiter_pm_vm_gpu_stop(kbdev); - break; - case KBASE_VM_GPU_LOST_EVT: -- dev_info(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!"); -+ dev_dbg(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!"); - kbase_gpu_lost(kbdev); - break; - case KBASE_VM_OS_SUSPEND_EVENT: -@@ -530,8 +860,6 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, - case KBASE_VM_REF_EVENT: - switch (arb_vm_state->vm_state) { - case KBASE_VM_STATE_STARTING: -- KBASE_TLSTREAM_TL_EVENT_ARB_STARTED(kbdev, kbdev); -- /* FALL THROUGH */ - case KBASE_VM_STATE_IDLE: - kbase_arbiter_pm_vm_set_state(kbdev, - KBASE_VM_STATE_ACTIVE); -@@ -547,15 +875,21 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, - break; - - case KBASE_VM_GPU_INITIALIZED_EVT: -- lockdep_assert_held(&kbdev->pm.lock); -- if (kbdev->pm.active_count > 0) { -- kbase_arbiter_pm_vm_set_state(kbdev, -- KBASE_VM_STATE_ACTIVE); -- kbase_arbif_gpu_active(kbdev); -- } else { -- kbase_arbiter_pm_vm_set_state(kbdev, -- KBASE_VM_STATE_IDLE); -- kbase_arbif_gpu_idle(kbdev); -+ switch (arb_vm_state->vm_state) { -+ case KBASE_VM_STATE_INITIALIZING_WITH_GPU: -+ lockdep_assert_held(&kbdev->pm.lock); -+ if (kbdev->pm.active_count > 0) { -+ kbase_arbiter_pm_vm_set_state(kbdev, -+ KBASE_VM_STATE_ACTIVE); -+ kbase_arbif_gpu_active(kbdev); -+ } else { -+ kbase_arbiter_pm_vm_set_state(kbdev, -+ KBASE_VM_STATE_IDLE); -+ kbase_arbif_gpu_idle(kbdev); -+ } -+ break; -+ default: -+ break; - } - break; - -@@ -566,6 +900,14 @@ void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev, - mutex_unlock(&arb_vm_state->vm_state_lock); - } - -+KBASE_EXPORT_TEST_API(kbase_arbiter_pm_vm_event); -+ -+/** -+ * kbase_arbiter_pm_vm_wait_gpu_assignment() - VM wait for a GPU assignment. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * VM waits for a GPU assignment. -+ */ - static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev) - { - struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -@@ -577,6 +919,12 @@ static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev) - dev_dbg(kbdev->dev, "Waiting for GPU assignment - done\n"); - } - -+/** -+ * kbase_arbiter_pm_vm_gpu_assigned_lockheld() - Check if VM holds VM state lock -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Checks if the virtual machine holds VM state lock. -+ */ - static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld( - struct kbase_device *kbdev) - { -@@ -587,11 +935,25 @@ static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld( - arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE); - } - -+/** -+ * kbase_arbiter_pm_ctx_active_handle_suspend() - Handle suspend operation for -+ * arbitration mode -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @suspend_handler: The handler code for how to handle a suspend -+ * that might occur -+ * -+ * This function handles a suspend event from the driver, -+ * communicating with the arbiter and waiting synchronously for the GPU -+ * to be granted again depending on the VM state. -+ * -+ * Return: 0 on success else 1 suspend handler isn not possible. -+ */ - int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, - enum kbase_pm_suspend_handler suspend_handler) - { - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; -+ int res = 0; - - if (kbdev->arb.arb_if) { - mutex_lock(&arb_vm_state->vm_state_lock); -@@ -606,30 +968,41 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, - kbase_arbiter_pm_vm_set_state(kbdev, - KBASE_VM_STATE_STOPPED_GPU_REQUESTED); - kbase_arbif_gpu_request(kbdev); -+ start_request_timer(kbdev); - } else if (arb_vm_state->vm_state == - KBASE_VM_STATE_INITIALIZING_WITH_GPU) - break; - - if (suspend_handler != - KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) { -- if (suspend_handler == -- KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED -- || -- kbdev->pm.active_count > 0) -- break; - -- mutex_unlock(&arb_vm_state->vm_state_lock); -- mutex_unlock(&kbdev->pm.lock); -- mutex_unlock(&js_devdata->runpool_mutex); -- return 1; -- } -+ /* In case of GPU lost, even if -+ * active_count > 0, we no longer have GPU -+ * access -+ */ -+ if (kbase_pm_is_gpu_lost(kbdev)) -+ res = 1; - -- if (arb_vm_state->vm_state == -- KBASE_VM_STATE_INITIALIZING_WITH_GPU) -+ switch (suspend_handler) { -+ case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE: -+ res = 1; -+ break; -+ case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: -+ if (kbdev->pm.active_count == 0) -+ res = 1; -+ break; -+ case KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED: -+ break; -+ default: -+ WARN(1, "Unknown suspend_handler\n"); -+ res = 1; -+ break; -+ } - break; -+ } - - /* Need to synchronously wait for GPU assignment */ -- arb_vm_state->vm_arb_users_waiting++; -+ atomic_inc(&kbdev->pm.gpu_users_waiting); - mutex_unlock(&arb_vm_state->vm_state_lock); - mutex_unlock(&kbdev->pm.lock); - mutex_unlock(&js_devdata->runpool_mutex); -@@ -637,9 +1010,128 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, - mutex_lock(&js_devdata->runpool_mutex); - mutex_lock(&kbdev->pm.lock); - mutex_lock(&arb_vm_state->vm_state_lock); -- arb_vm_state->vm_arb_users_waiting--; -+ atomic_dec(&kbdev->pm.gpu_users_waiting); - } - mutex_unlock(&arb_vm_state->vm_state_lock); - } -- return 0; -+ return res; -+} -+ -+/** -+ * kbase_arbiter_pm_update_gpu_freq() - Updates GPU clock frequency received -+ * from arbiter. -+ * @arb_freq - Pointer to struchture holding GPU clock frequenecy data -+ * @freq - New frequency value in KHz -+ */ -+void kbase_arbiter_pm_update_gpu_freq(struct kbase_arbiter_freq *arb_freq, -+ uint32_t freq) -+{ -+ struct kbase_gpu_clk_notifier_data ndata; -+ -+ mutex_lock(&arb_freq->arb_freq_lock); -+ if (arb_freq->arb_freq != freq) { -+ ndata.new_rate = freq * KHZ_TO_HZ; -+ ndata.old_rate = arb_freq->arb_freq * KHZ_TO_HZ; -+ ndata.gpu_clk_handle = arb_freq; -+ arb_freq->arb_freq = freq; -+ arb_freq->freq_updated = true; -+ if (arb_freq->nb) -+ arb_freq->nb->notifier_call(arb_freq->nb, -+ POST_RATE_CHANGE, &ndata); -+ } -+ -+ mutex_unlock(&arb_freq->arb_freq_lock); -+} -+ -+/** -+ * enumerate_arb_gpu_clk() - Enumerate a GPU clock on the given index -+ * @kbdev - kbase_device pointer -+ * @index - GPU clock index -+ * -+ * Returns pointer to structure holding GPU clock frequency data reported from -+ * arbiter, only index 0 is valid. -+ */ -+static void *enumerate_arb_gpu_clk(struct kbase_device *kbdev, -+ unsigned int index) -+{ -+ if (index == 0) -+ return &kbdev->arb.arb_freq; -+ return NULL; -+} -+ -+/** -+ * get_arb_gpu_clk_rate() - Get the current rate of GPU clock frequency value -+ * @kbdev - kbase_device pointer -+ * @index - GPU clock index -+ * -+ * Returns the GPU clock frequency value saved when gpu is granted from arbiter -+ */ -+static unsigned long get_arb_gpu_clk_rate(struct kbase_device *kbdev, -+ void *gpu_clk_handle) -+{ -+ uint32_t freq; -+ struct kbase_arbiter_freq *arb_dev_freq = -+ (struct kbase_arbiter_freq *) gpu_clk_handle; -+ -+ mutex_lock(&arb_dev_freq->arb_freq_lock); -+ /* Convert from KHz to Hz */ -+ freq = arb_dev_freq->arb_freq * KHZ_TO_HZ; -+ mutex_unlock(&arb_dev_freq->arb_freq_lock); -+ return freq; -+} -+ -+/** -+ * arb_gpu_clk_notifier_register() - Register a clock rate change notifier. -+ * @kbdev - kbase_device pointer -+ * @gpu_clk_handle - Handle unique to the enumerated GPU clock -+ * @nb - notifier block containing the callback function pointer -+ * -+ * Returns 0 on success, negative error code otherwise. -+ * -+ * This function registers a callback function that is invoked whenever the -+ * frequency of the clock corresponding to @gpu_clk_handle changes. -+ */ -+static int arb_gpu_clk_notifier_register(struct kbase_device *kbdev, -+ void *gpu_clk_handle, struct notifier_block *nb) -+{ -+ int ret = 0; -+ struct kbase_arbiter_freq *arb_dev_freq = -+ (struct kbase_arbiter_freq *)gpu_clk_handle; -+ -+ if (!arb_dev_freq->nb) -+ arb_dev_freq->nb = nb; -+ else -+ ret = -EBUSY; -+ -+ return ret; -+} -+ -+/** -+ * gpu_clk_notifier_unregister() - Unregister clock rate change notifier -+ * @kbdev - kbase_device pointer -+ * @gpu_clk_handle - Handle unique to the enumerated GPU clock -+ * @nb - notifier block containing the callback function pointer -+ * -+ * This function pointer is used to unregister a callback function that -+ * was previously registered to get notified of a frequency change of the -+ * clock corresponding to @gpu_clk_handle. -+ */ -+static void arb_gpu_clk_notifier_unregister(struct kbase_device *kbdev, -+ void *gpu_clk_handle, struct notifier_block *nb) -+{ -+ struct kbase_arbiter_freq *arb_dev_freq = -+ (struct kbase_arbiter_freq *)gpu_clk_handle; -+ if (arb_dev_freq->nb == nb) { -+ arb_dev_freq->nb = NULL; -+ } else { -+ dev_err(kbdev->dev, "%s - notifier did not match\n", -+ __func__); -+ } - } -+ -+struct kbase_clk_rate_trace_op_conf arb_clk_rate_trace_ops = { -+ .get_gpu_clk_rate = get_arb_gpu_clk_rate, -+ .enumerate_gpu_clk = enumerate_arb_gpu_clk, -+ .gpu_clk_notifier_register = arb_gpu_clk_notifier_register, -+ .gpu_clk_notifier_unregister = arb_gpu_clk_notifier_unregister -+}; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_pm.h b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_pm.h -index 3c49eb1..1f570bb 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_pm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/arbiter/mali_kbase_arbiter_pm.h -@@ -1,28 +1,7 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ -- --/* -- * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software -@@ -38,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -@@ -116,10 +93,18 @@ void kbase_arbiter_pm_early_term(struct kbase_device *kbdev); - * kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts - * @kbdev: The kbase device structure for the device (must be a valid pointer) - * -- * Releases interrupts if needed (GPU is available) otherwise does nothing -+ * Releases interrupts and set the interrupt flag to false - */ - void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev); - -+/** -+ * kbase_arbiter_pm_install_interrupts() - Install the GPU interrupts -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Install interrupts and set the interrupt_install flag to true. -+ */ -+int kbase_arbiter_pm_install_interrupts(struct kbase_device *kbdev); -+ - /** - * kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine - * @kbdev: The kbase device structure for the device (must be a valid pointer) -@@ -156,4 +141,54 @@ int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev, - */ - void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev); - -+/** -+ * kbase_arbiter_set_max_config() - Set the max config data in kbase device. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer). -+ * @max_l2_slices: The maximum number of L2 slices. -+ * @max_core_mask: The largest core mask. -+ * -+ * This function handles a stop event for the VM. -+ * It will update the VM state and forward the stop event to the driver. -+ */ -+void kbase_arbiter_set_max_config(struct kbase_device *kbdev, -+ uint32_t max_l2_slices, -+ uint32_t max_core_mask); -+ -+/** -+ * kbase_arbiter_pm_gpu_assigned() - Determine if this VM has access to the GPU -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Return: 0 if the VM does not have access, 1 if it does, and a negative number -+ * if an error occurred -+ */ -+int kbase_arbiter_pm_gpu_assigned(struct kbase_device *kbdev); -+ -+extern struct kbase_clk_rate_trace_op_conf arb_clk_rate_trace_ops; -+ -+/** -+ * struct kbase_arbiter_freq - Holding the GPU clock frequency data retrieved -+ * from arbiter -+ * @arb_freq: GPU clock frequency value -+ * @arb_freq_lock: Mutex protecting access to arbfreq value -+ * @nb: Notifier block to receive rate change callbacks -+ * @freq_updated: Flag to indicate whether a frequency changed has just been -+ * communicated to avoid "GPU_GRANTED when not expected" warning -+ */ -+struct kbase_arbiter_freq { -+ uint32_t arb_freq; -+ struct mutex arb_freq_lock; -+ struct notifier_block *nb; -+ bool freq_updated; -+}; -+ -+/** -+ * kbase_arbiter_pm_update_gpu_freq() - Update GPU frequency -+ * @arb_freq: Pointer to GPU clock frequency data -+ * @freq: The new frequency -+ * -+ * Updates the GPU frequency and triggers any notifications -+ */ -+void kbase_arbiter_pm_update_gpu_freq(struct kbase_arbiter_freq *arb_freq, -+ uint32_t freq); -+ - #endif /*_MALI_KBASE_ARBITER_PM_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/Kbuild -index 2449e80..5dbcff3 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,51 +16,34 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - --BACKEND += \ -- backend/gpu/mali_kbase_cache_policy_backend.c \ -- backend/gpu/mali_kbase_device_hw.c \ -- backend/gpu/mali_kbase_gpuprops_backend.c \ -- backend/gpu/mali_kbase_irq_linux.c \ -- backend/gpu/mali_kbase_instr_backend.c \ -- backend/gpu/mali_kbase_js_backend.c \ -- backend/gpu/mali_kbase_pm_backend.c \ -- backend/gpu/mali_kbase_pm_driver.c \ -- backend/gpu/mali_kbase_pm_metrics.c \ -- backend/gpu/mali_kbase_pm_ca.c \ -- backend/gpu/mali_kbase_pm_always_on.c \ -- backend/gpu/mali_kbase_pm_coarse_demand.c \ -- backend/gpu/mali_kbase_pm_policy.c \ -- backend/gpu/mali_kbase_time.c \ -- backend/gpu/mali_kbase_l2_mmu_config.c -+mali_kbase-y += \ -+ backend/gpu/mali_kbase_cache_policy_backend.o \ -+ backend/gpu/mali_kbase_gpuprops_backend.o \ -+ backend/gpu/mali_kbase_irq_linux.o \ -+ backend/gpu/mali_kbase_js_backend.o \ -+ backend/gpu/mali_kbase_pm_backend.o \ -+ backend/gpu/mali_kbase_pm_driver.o \ -+ backend/gpu/mali_kbase_pm_metrics.o \ -+ backend/gpu/mali_kbase_pm_ca.o \ -+ backend/gpu/mali_kbase_pm_always_on.o \ -+ backend/gpu/mali_kbase_pm_coarse_demand.o \ -+ backend/gpu/mali_kbase_pm_policy.o \ -+ backend/gpu/mali_kbase_time.o \ -+ backend/gpu/mali_kbase_l2_mmu_config.o \ -+ backend/gpu/mali_kbase_clk_rate_trace_mgr.o - --ifeq ($(MALI_USE_CSF),1) --# empty --else -- BACKEND += \ -- backend/gpu/mali_kbase_jm_as.c \ -- backend/gpu/mali_kbase_debug_job_fault_backend.c \ -- backend/gpu/mali_kbase_jm_hw.c \ -- backend/gpu/mali_kbase_jm_rb.c -+ifeq ($(MALI_USE_CSF),0) -+ mali_kbase-y += \ -+ backend/gpu/mali_kbase_instr_backend.o \ -+ backend/gpu/mali_kbase_jm_as.o \ -+ backend/gpu/mali_kbase_debug_job_fault_backend.o \ -+ backend/gpu/mali_kbase_jm_hw.o \ -+ backend/gpu/mali_kbase_jm_rb.o - endif - --ifeq ($(MALI_CUSTOMER_RELEASE),0) --BACKEND += \ -- backend/gpu/mali_kbase_pm_always_on_demand.c --endif - --ifeq ($(CONFIG_MALI_DEVFREQ),y) --BACKEND += \ -- backend/gpu/mali_kbase_devfreq.c --endif -+mali_kbase-$(CONFIG_MALI_DEVFREQ) += \ -+ backend/gpu/mali_kbase_devfreq.o - --ifeq ($(CONFIG_MALI_NO_MALI),y) -- # Dummy model -- BACKEND += backend/gpu/mali_kbase_model_dummy.c -- BACKEND += backend/gpu/mali_kbase_model_linux.c -- # HW error simulation -- BACKEND += backend/gpu/mali_kbase_model_error_generator.c --endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h -index 4a61f96..6924fdb 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_backend_config.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c -index 7378bfd..e542ccf 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2015-2016,2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2016, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "backend/gpu/mali_kbase_cache_policy_backend.h" --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - - void kbase_cache_set_coherency_mode(struct kbase_device *kbdev, - u32 mode) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h -index f78ada7..278125a 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_cache_policy_backend.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2015-2016 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2016, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,16 +17,13 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - #ifndef _KBASE_CACHE_POLICY_BACKEND_H_ - #define _KBASE_CACHE_POLICY_BACKEND_H_ - - #include "mali_kbase.h" --#include "mali_base_kernel.h" -+#include <uapi/gpu/arm/midgard/mali_base_kernel.h> - - /** - * kbase_cache_set_coherency_mode() - Sets the system coherency mode -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_clk_rate_trace_mgr.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_clk_rate_trace_mgr.c -new file mode 100644 -index 0000000..6ad0f58 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_clk_rate_trace_mgr.c -@@ -0,0 +1,325 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * Implementation of the GPU clock rate trace manager. -+ */ -+ -+#include <mali_kbase.h> -+#include <mali_kbase_config_defaults.h> -+#include <linux/clk.h> -+#include <asm/div64.h> -+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" -+ -+#ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY -+#include <trace/events/power_gpu_frequency.h> -+#else -+#include "mali_power_gpu_frequency_trace.h" -+#endif -+ -+#ifndef CLK_RATE_TRACE_OPS -+#define CLK_RATE_TRACE_OPS (NULL) -+#endif -+ -+/** -+ * get_clk_rate_trace_callbacks() - Returns pointer to clk trace ops. -+ * @kbdev: Pointer to kbase device, used to check if arbitration is enabled -+ * when compiled with arbiter support. -+ * Return: Pointer to clk trace ops if supported or NULL. -+ */ -+static struct kbase_clk_rate_trace_op_conf * -+get_clk_rate_trace_callbacks(struct kbase_device *kbdev __maybe_unused) -+{ -+ /* base case */ -+ struct kbase_clk_rate_trace_op_conf *callbacks = -+ (struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS; -+#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) -+ const void *arbiter_if_node; -+ -+ if (WARN_ON(!kbdev) || WARN_ON(!kbdev->dev)) -+ return callbacks; -+ -+ arbiter_if_node = -+ of_get_property(kbdev->dev->of_node, "arbiter_if", NULL); -+ /* Arbitration enabled, override the callback pointer.*/ -+ if (arbiter_if_node) -+ callbacks = &arb_clk_rate_trace_ops; -+ else -+ dev_dbg(kbdev->dev, -+ "Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n"); -+ -+#endif -+ -+ return callbacks; -+} -+ -+static int gpu_clk_rate_change_notifier(struct notifier_block *nb, -+ unsigned long event, void *data) -+{ -+ struct kbase_gpu_clk_notifier_data *ndata = data; -+ struct kbase_clk_data *clk_data = -+ container_of(nb, struct kbase_clk_data, clk_rate_change_nb); -+ struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm; -+ unsigned long flags; -+ -+ if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle)) -+ return NOTIFY_BAD; -+ -+ spin_lock_irqsave(&clk_rtm->lock, flags); -+ if (event == POST_RATE_CHANGE) { -+ if (!clk_rtm->gpu_idle && -+ (clk_data->clock_val != ndata->new_rate)) { -+ kbase_clk_rate_trace_manager_notify_all( -+ clk_rtm, clk_data->index, ndata->new_rate); -+ } -+ -+ clk_data->clock_val = ndata->new_rate; -+ } -+ spin_unlock_irqrestore(&clk_rtm->lock, flags); -+ -+ return NOTIFY_DONE; -+} -+ -+static int gpu_clk_data_init(struct kbase_device *kbdev, -+ void *gpu_clk_handle, unsigned int index) -+{ -+ struct kbase_clk_rate_trace_op_conf *callbacks; -+ struct kbase_clk_data *clk_data; -+ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; -+ int ret = 0; -+ -+ callbacks = get_clk_rate_trace_callbacks(kbdev); -+ -+ if (WARN_ON(!callbacks) || -+ WARN_ON(!gpu_clk_handle) || -+ WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS)) -+ return -EINVAL; -+ -+ clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL); -+ if (!clk_data) { -+ dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index); -+ return -ENOMEM; -+ } -+ -+ clk_data->index = (u8)index; -+ clk_data->gpu_clk_handle = gpu_clk_handle; -+ /* Store the initial value of clock */ -+ clk_data->clock_val = -+ callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle); -+ -+ { -+ /* At the initialization time, GPU is powered off. */ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&clk_rtm->lock, flags); -+ kbase_clk_rate_trace_manager_notify_all( -+ clk_rtm, clk_data->index, 0); -+ spin_unlock_irqrestore(&clk_rtm->lock, flags); -+ } -+ -+ clk_data->clk_rtm = clk_rtm; -+ clk_rtm->clks[index] = clk_data; -+ -+ clk_data->clk_rate_change_nb.notifier_call = -+ gpu_clk_rate_change_notifier; -+ -+ if (callbacks->gpu_clk_notifier_register) -+ ret = callbacks->gpu_clk_notifier_register(kbdev, -+ gpu_clk_handle, &clk_data->clk_rate_change_nb); -+ if (ret) { -+ dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index); -+ kfree(clk_data); -+ } -+ -+ return ret; -+} -+ -+int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev) -+{ -+ struct kbase_clk_rate_trace_op_conf *callbacks; -+ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; -+ unsigned int i; -+ int ret = 0; -+ -+ callbacks = get_clk_rate_trace_callbacks(kbdev); -+ -+ spin_lock_init(&clk_rtm->lock); -+ INIT_LIST_HEAD(&clk_rtm->listeners); -+ -+ /* Return early if no callbacks provided for clock rate tracing */ -+ if (!callbacks) { -+ WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL); -+ return 0; -+ } -+ -+ clk_rtm->gpu_idle = true; -+ -+ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { -+ void *gpu_clk_handle = -+ callbacks->enumerate_gpu_clk(kbdev, i); -+ -+ if (!gpu_clk_handle) -+ break; -+ -+ ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i); -+ if (ret) -+ goto error; -+ } -+ -+ /* Activate clock rate trace manager if at least one GPU clock was -+ * enumerated. -+ */ -+ if (i) { -+ WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks); -+ } else { -+ dev_info(kbdev->dev, "No clock(s) available for rate tracing"); -+ WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL); -+ } -+ -+ return 0; -+ -+error: -+ while (i--) { -+ clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister( -+ kbdev, clk_rtm->clks[i]->gpu_clk_handle, -+ &clk_rtm->clks[i]->clk_rate_change_nb); -+ kfree(clk_rtm->clks[i]); -+ } -+ -+ return ret; -+} -+ -+void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev) -+{ -+ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; -+ unsigned int i; -+ -+ WARN_ON(!list_empty(&clk_rtm->listeners)); -+ -+ if (!clk_rtm->clk_rate_trace_ops) -+ return; -+ -+ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { -+ if (!clk_rtm->clks[i]) -+ break; -+ -+ if (clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister) -+ clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister -+ (kbdev, clk_rtm->clks[i]->gpu_clk_handle, -+ &clk_rtm->clks[i]->clk_rate_change_nb); -+ kfree(clk_rtm->clks[i]); -+ } -+ -+ WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL); -+} -+ -+void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev) -+{ -+ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; -+ unsigned int i; -+ unsigned long flags; -+ -+ if (!clk_rtm->clk_rate_trace_ops) -+ return; -+ -+ spin_lock_irqsave(&clk_rtm->lock, flags); -+ -+ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { -+ struct kbase_clk_data *clk_data = clk_rtm->clks[i]; -+ -+ if (!clk_data) -+ break; -+ -+ if (unlikely(!clk_data->clock_val)) -+ continue; -+ -+ kbase_clk_rate_trace_manager_notify_all( -+ clk_rtm, clk_data->index, clk_data->clock_val); -+ } -+ -+ clk_rtm->gpu_idle = false; -+ spin_unlock_irqrestore(&clk_rtm->lock, flags); -+} -+ -+void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev) -+{ -+ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; -+ unsigned int i; -+ unsigned long flags; -+ -+ if (!clk_rtm->clk_rate_trace_ops) -+ return; -+ -+ spin_lock_irqsave(&clk_rtm->lock, flags); -+ -+ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { -+ struct kbase_clk_data *clk_data = clk_rtm->clks[i]; -+ -+ if (!clk_data) -+ break; -+ -+ if (unlikely(!clk_data->clock_val)) -+ continue; -+ -+ kbase_clk_rate_trace_manager_notify_all( -+ clk_rtm, clk_data->index, 0); -+ } -+ -+ clk_rtm->gpu_idle = true; -+ spin_unlock_irqrestore(&clk_rtm->lock, flags); -+} -+ -+void kbase_clk_rate_trace_manager_notify_all( -+ struct kbase_clk_rate_trace_manager *clk_rtm, -+ u32 clk_index, -+ unsigned long new_rate) -+{ -+ struct kbase_clk_rate_listener *pos; -+ struct kbase_device *kbdev; -+ -+ lockdep_assert_held(&clk_rtm->lock); -+ -+ kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm); -+ -+ dev_dbg(kbdev->dev, "%s - GPU clock %u rate changed to %lu, pid: %d", -+ __func__, clk_index, new_rate, current->pid); -+ -+ /* Raise standard `power/gpu_frequency` ftrace event */ -+ { -+ unsigned long new_rate_khz = new_rate; -+ -+#if BITS_PER_LONG == 64 -+ do_div(new_rate_khz, 1000); -+#elif BITS_PER_LONG == 32 -+ new_rate_khz /= 1000; -+#else -+#error "unsigned long division is not supported for this architecture" -+#endif -+ -+ trace_gpu_frequency(new_rate_khz, clk_index); -+ } -+ -+ /* Notify the listeners. */ -+ list_for_each_entry(pos, &clk_rtm->listeners, node) { -+ pos->notify(pos, clk_index, new_rate); -+ } -+} -+KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_clk_rate_trace_mgr.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_clk_rate_trace_mgr.h -new file mode 100644 -index 0000000..f7ec9d1 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_clk_rate_trace_mgr.h -@@ -0,0 +1,154 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CLK_RATE_TRACE_MGR_ -+#define _KBASE_CLK_RATE_TRACE_MGR_ -+ -+/* The index of top clock domain in kbase_clk_rate_trace_manager:clks. */ -+#define KBASE_CLOCK_DOMAIN_TOP (0) -+ -+/* The index of shader-cores clock domain in -+ * kbase_clk_rate_trace_manager:clks. -+ */ -+#define KBASE_CLOCK_DOMAIN_SHADER_CORES (1) -+ -+/** -+ * struct kbase_clk_data - Data stored per enumerated GPU clock. -+ * -+ * @clk_rtm: Pointer to clock rate trace manager object. -+ * @gpu_clk_handle: Handle unique to the enumerated GPU clock. -+ * @plat_private: Private data for the platform to store into -+ * @clk_rate_change_nb: notifier block containing the pointer to callback -+ * function that is invoked whenever the rate of -+ * enumerated GPU clock changes. -+ * @clock_val: Current rate of the enumerated GPU clock. -+ * @index: Index at which the GPU clock was enumerated. -+ */ -+struct kbase_clk_data { -+ struct kbase_clk_rate_trace_manager *clk_rtm; -+ void *gpu_clk_handle; -+ void *plat_private; -+ struct notifier_block clk_rate_change_nb; -+ unsigned long clock_val; -+ u8 index; -+}; -+ -+/** -+ * kbase_clk_rate_trace_manager_init - Initialize GPU clock rate trace manager. -+ * -+ * @kbdev: Device pointer -+ * -+ * Return: 0 if success, or an error code on failure. -+ */ -+int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_clk_rate_trace_manager_term - Terminate GPU clock rate trace manager. -+ * -+ * @kbdev: Device pointer -+ */ -+void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_clk_rate_trace_manager_gpu_active - Inform GPU clock rate trace -+ * manager of GPU becoming active. -+ * -+ * @kbdev: Device pointer -+ */ -+void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev); -+ -+/** -+ * kbase_clk_rate_trace_manager_gpu_idle - Inform GPU clock rate trace -+ * manager of GPU becoming idle. -+ * @kbdev: Device pointer -+ */ -+void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev); -+ -+/** -+ * kbase_clk_rate_trace_manager_subscribe_no_lock() - Add freq change listener. -+ * -+ * @clk_rtm: Clock rate manager instance. -+ * @listener: Listener handle -+ * -+ * kbase_clk_rate_trace_manager:lock must be held by the caller. -+ */ -+static inline void kbase_clk_rate_trace_manager_subscribe_no_lock( -+ struct kbase_clk_rate_trace_manager *clk_rtm, -+ struct kbase_clk_rate_listener *listener) -+{ -+ lockdep_assert_held(&clk_rtm->lock); -+ list_add(&listener->node, &clk_rtm->listeners); -+} -+ -+/** -+ * kbase_clk_rate_trace_manager_subscribe() - Add freq change listener. -+ * -+ * @clk_rtm: Clock rate manager instance. -+ * @listener: Listener handle -+ */ -+static inline void kbase_clk_rate_trace_manager_subscribe( -+ struct kbase_clk_rate_trace_manager *clk_rtm, -+ struct kbase_clk_rate_listener *listener) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&clk_rtm->lock, flags); -+ kbase_clk_rate_trace_manager_subscribe_no_lock( -+ clk_rtm, listener); -+ spin_unlock_irqrestore(&clk_rtm->lock, flags); -+} -+ -+/** -+ * kbase_clk_rate_trace_manager_unsubscribe() - Remove freq change listener. -+ * -+ * @clk_rtm: Clock rate manager instance. -+ * @listener: Listener handle -+ */ -+static inline void kbase_clk_rate_trace_manager_unsubscribe( -+ struct kbase_clk_rate_trace_manager *clk_rtm, -+ struct kbase_clk_rate_listener *listener) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&clk_rtm->lock, flags); -+ list_del(&listener->node); -+ spin_unlock_irqrestore(&clk_rtm->lock, flags); -+} -+ -+/** -+ * kbase_clk_rate_trace_manager_notify_all() - Notify all clock \ -+ * rate listeners. -+ * -+ * @clk_rtm: Clock rate manager instance. -+ * @clock_index: Clock index. -+ * @new_rate: New clock frequency(Hz) -+ * -+ * kbase_clk_rate_trace_manager:lock must be locked. -+ * This function is exported to be used by clock rate trace test -+ * portal. -+ */ -+void kbase_clk_rate_trace_manager_notify_all( -+ struct kbase_clk_rate_trace_manager *clk_rtm, -+ u32 clock_index, -+ unsigned long new_rate); -+ -+#endif /* _KBASE_CLK_RATE_TRACE_MGR_ */ -+ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c -index b05844e..e121b41 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_debug_job_fault_backend.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2015,2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2015, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,13 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include "mali_kbase_debug_job_fault.h" - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - - /*GPU_CONTROL_REG(r)*/ - static int gpu_control_reg_snapshot[] = { -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c -index 2806f05..8c31499 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> -@@ -27,40 +26,53 @@ - #include <linux/of.h> - #include <linux/clk.h> - #include <linux/devfreq.h> --#ifdef CONFIG_DEVFREQ_THERMAL -+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL) - #include <linux/devfreq_cooling.h> - #endif - - #include <linux/version.h> --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0) - #include <linux/pm_opp.h> --#else /* Linux >= 3.13 */ --/* In 3.13 the OPP include header file, types, and functions were all -- * renamed. Use the old filename for the include, and define the new names to -- * the old, when an old kernel is detected. -- */ --#include <linux/opp.h> --#define dev_pm_opp opp --#define dev_pm_opp_get_voltage opp_get_voltage --#define dev_pm_opp_get_opp_count opp_get_opp_count --#define dev_pm_opp_find_freq_ceil opp_find_freq_ceil --#define dev_pm_opp_find_freq_floor opp_find_freq_floor --#endif /* Linux >= 3.13 */ - - /** -- * opp_translate - Translate nominal OPP frequency from devicetree into real -- * frequency and core mask -- * @kbdev: Device pointer -- * @freq: Nominal frequency -- * @core_mask: Pointer to u64 to store core mask to -- * @freqs: Pointer to array of frequencies -- * @volts: Pointer to array of voltages -+ * get_voltage() - Get the voltage value corresponding to the nominal frequency -+ * used by devfreq. -+ * @kbdev: Device pointer -+ * @freq: Nominal frequency in Hz passed by devfreq. -+ * -+ * This function will be called only when the opp table which is compatible with -+ * "operating-points-v2-mali", is not present in the devicetree for GPU device. - * -- * This function will only perform translation if an operating-points-v2-mali -- * table is present in devicetree. If one is not present then it will return an -- * untranslated frequency and all cores enabled. -+ * Return: Voltage value in milli volts, 0 in case of error. - */ --static void opp_translate(struct kbase_device *kbdev, unsigned long freq, -+static unsigned long get_voltage(struct kbase_device *kbdev, unsigned long freq) -+{ -+ struct dev_pm_opp *opp; -+ unsigned long voltage = 0; -+ -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE -+ rcu_read_lock(); -+#endif -+ -+ opp = dev_pm_opp_find_freq_exact(kbdev->dev, freq, true); -+ -+ if (IS_ERR_OR_NULL(opp)) -+ dev_err(kbdev->dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); -+ else { -+ voltage = dev_pm_opp_get_voltage(opp); -+#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE -+ dev_pm_opp_put(opp); -+#endif -+ } -+ -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE -+ rcu_read_unlock(); -+#endif -+ -+ /* Return the voltage in milli volts */ -+ return voltage / 1000; -+} -+ -+void kbase_devfreq_opp_translate(struct kbase_device *kbdev, unsigned long freq, - u64 *core_mask, unsigned long *freqs, unsigned long *volts) - { - unsigned int i; -@@ -82,12 +94,17 @@ static void opp_translate(struct kbase_device *kbdev, unsigned long freq, - } - - /* If failed to find OPP, return all cores enabled -- * and nominal frequency -+ * and nominal frequency and the corresponding voltage. - */ - if (i == kbdev->num_opps) { -+ unsigned long voltage = get_voltage(kbdev, freq); -+ - *core_mask = kbdev->gpu_props.props.raw_props.shader_present; -- for (i = 0; i < kbdev->nr_clocks; i++) -+ -+ for (i = 0; i < kbdev->nr_clocks; i++) { - freqs[i] = freq; -+ volts[i] = voltage; -+ } - } - } - -@@ -104,18 +121,18 @@ kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) - - nominal_freq = *target_freq; - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_lock(); - #endif - opp = devfreq_recommended_opp(dev, &nominal_freq, flags); --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_unlock(); - #endif - if (IS_ERR_OR_NULL(opp)) { - dev_err(dev, "Failed to get opp (%ld)\n", PTR_ERR(opp)); - return PTR_ERR(opp); - } --#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE - dev_pm_opp_put(opp); - #endif - -@@ -127,9 +144,10 @@ kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) - return 0; - } - -- opp_translate(kbdev, nominal_freq, &core_mask, freqs, volts); -+ kbase_devfreq_opp_translate(kbdev, nominal_freq, &core_mask, -+ freqs, volts); - --#ifdef CONFIG_REGULATOR -+#if IS_ENABLED(CONFIG_REGULATOR) - /* Regulators and clocks work in pairs: every clock has a regulator, - * and we never expect to have more regulators than clocks. - * -@@ -177,7 +195,7 @@ kbase_devfreq_target(struct device *dev, unsigned long *target_freq, u32 flags) - } - } - --#ifdef CONFIG_REGULATOR -+#if IS_ENABLED(CONFIG_REGULATOR) - for (i = 0; i < kbdev->nr_clocks; i++) { - if (kbdev->regulators[i] && - kbdev->current_voltages[i] != volts[i] && -@@ -238,6 +256,10 @@ kbase_devfreq_status(struct device *dev, struct devfreq_dev_status *stat) - stat->current_frequency = kbdev->current_nominal_freq; - stat->private_data = NULL; - -+#if MALI_USE_CSF && defined CONFIG_DEVFREQ_THERMAL -+ kbase_ipa_reset_data(kbdev); -+#endif -+ - return 0; - } - -@@ -249,11 +271,11 @@ static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, - unsigned long freq; - struct dev_pm_opp *opp; - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_lock(); - #endif - count = dev_pm_opp_get_opp_count(kbdev->dev); --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_unlock(); - #endif - if (count < 0) -@@ -264,20 +286,20 @@ static int kbase_devfreq_init_freq_table(struct kbase_device *kbdev, - if (!dp->freq_table) - return -ENOMEM; - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_lock(); - #endif - for (i = 0, freq = ULONG_MAX; i < count; i++, freq--) { - opp = dev_pm_opp_find_freq_floor(kbdev->dev, &freq); - if (IS_ERR(opp)) - break; --#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE - dev_pm_opp_put(opp); --#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) */ -+#endif /* KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE */ - - dp->freq_table[i] = freq; - } --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0) -+#if KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE - rcu_read_unlock(); - #endif - -@@ -309,18 +331,21 @@ static void kbase_devfreq_term_freq_table(struct kbase_device *kbdev) - struct devfreq_dev_profile *dp = &kbdev->devfreq_profile; - - kfree(dp->freq_table); -+ dp->freq_table = NULL; - } - - static void kbase_devfreq_term_core_mask_table(struct kbase_device *kbdev) - { - kfree(kbdev->devfreq_table); -+ kbdev->devfreq_table = NULL; - } - - static void kbase_devfreq_exit(struct device *dev) - { - struct kbase_device *kbdev = dev_get_drvdata(dev); - -- kbase_devfreq_term_freq_table(kbdev); -+ if (kbdev) -+ kbase_devfreq_term_freq_table(kbdev); - } - - static void kbasep_devfreq_read_suspend_clock(struct kbase_device *kbdev, -@@ -359,7 +384,7 @@ static void kbasep_devfreq_read_suspend_clock(struct kbase_device *kbdev, - - static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) - { --#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF) -+#ifndef CONFIG_OF - /* OPP table initialization requires at least the capability to get - * regulators and clocks from the device tree, as well as parsing - * arrays of unsigned integer values. -@@ -392,7 +417,7 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) - u64 core_mask, opp_freq, - real_freqs[BASE_MAX_NR_CLOCKS_REGULATORS]; - int err; --#ifdef CONFIG_REGULATOR -+#if IS_ENABLED(CONFIG_REGULATOR) - u32 opp_volts[BASE_MAX_NR_CLOCKS_REGULATORS]; - #endif - -@@ -420,7 +445,7 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) - err); - continue; - } --#ifdef CONFIG_REGULATOR -+#if IS_ENABLED(CONFIG_REGULATOR) - err = of_property_read_u32_array(node, - "opp-microvolt", opp_volts, kbdev->nr_regulators); - if (err < 0) { -@@ -474,7 +499,7 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) - kbdev->devfreq_table[i].real_freqs[j] = - real_freqs[j]; - } --#ifdef CONFIG_REGULATOR -+#if IS_ENABLED(CONFIG_REGULATOR) - if (kbdev->nr_regulators > 0) { - int j; - -@@ -493,11 +518,9 @@ static int kbase_devfreq_init_core_mask_table(struct kbase_device *kbdev) - kbdev->num_opps = i; - - return 0; --#endif /* KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE */ -+#endif /* CONFIG_OF */ - } - --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) -- - static const char *kbase_devfreq_req_type_name(enum kbase_devfreq_work_type type) - { - const char *p; -@@ -554,27 +577,26 @@ static void kbase_devfreq_suspend_resume_worker(struct work_struct *work) - } - } - --#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) */ -- - void kbase_devfreq_enqueue_work(struct kbase_device *kbdev, - enum kbase_devfreq_work_type work_type) - { --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) - unsigned long flags; - - WARN_ON(work_type == DEVFREQ_WORK_NONE); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- kbdev->devfreq_queue.req_type = work_type; -- queue_work(kbdev->devfreq_queue.workq, &kbdev->devfreq_queue.work); -+ /* Skip enqueuing a work if workqueue has already been terminated. */ -+ if (likely(kbdev->devfreq_queue.workq)) { -+ kbdev->devfreq_queue.req_type = work_type; -+ queue_work(kbdev->devfreq_queue.workq, -+ &kbdev->devfreq_queue.work); -+ } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - dev_dbg(kbdev->dev, "Enqueuing devfreq req: %s\n", - kbase_devfreq_req_type_name(work_type)); --#endif - } - - static int kbase_devfreq_work_init(struct kbase_device *kbdev) - { --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) - kbdev->devfreq_queue.req_type = DEVFREQ_WORK_NONE; - kbdev->devfreq_queue.acted_type = DEVFREQ_WORK_RESUME; - -@@ -584,17 +606,23 @@ static int kbase_devfreq_work_init(struct kbase_device *kbdev) - - INIT_WORK(&kbdev->devfreq_queue.work, - kbase_devfreq_suspend_resume_worker); --#endif - return 0; - } - - static void kbase_devfreq_work_term(struct kbase_device *kbdev) - { --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) -- destroy_workqueue(kbdev->devfreq_queue.workq); --#endif -+ unsigned long flags; -+ struct workqueue_struct *workq; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ workq = kbdev->devfreq_queue.workq; -+ kbdev->devfreq_queue.workq = NULL; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ destroy_workqueue(workq); - } - -+ - int kbase_devfreq_init(struct kbase_device *kbdev) - { - struct devfreq_dev_profile *dp; -@@ -631,19 +659,11 @@ int kbase_devfreq_init(struct kbase_device *kbdev) - /* Record the maximum frequency possible */ - kbdev->gpu_props.props.core_props.gpu_freq_khz_max = - dp->freq_table[0] / 1000; -- }; -- -- err = kbase_devfreq_init_core_mask_table(kbdev); -- if (err) { -- kbase_devfreq_term_freq_table(kbdev); -- return err; - } - -- /* Initialise devfreq suspend/resume workqueue */ -- err = kbase_devfreq_work_init(kbdev); -+ err = kbase_devfreq_init_core_mask_table(kbdev); - if (err) { - kbase_devfreq_term_freq_table(kbdev); -- dev_err(kbdev->dev, "Devfreq initialization failed"); - return err; - } - -@@ -651,13 +671,27 @@ int kbase_devfreq_init(struct kbase_device *kbdev) - "simple_ondemand", NULL); - if (IS_ERR(kbdev->devfreq)) { - err = PTR_ERR(kbdev->devfreq); -- kbase_devfreq_work_term(kbdev); -+ kbdev->devfreq = NULL; -+ kbase_devfreq_term_core_mask_table(kbdev); - kbase_devfreq_term_freq_table(kbdev); -+ dev_err(kbdev->dev, "Fail to add devfreq device(%d)\n", err); -+ return err; -+ } -+ -+ /* Initialize devfreq suspend/resume workqueue */ -+ err = kbase_devfreq_work_init(kbdev); -+ if (err) { -+ if (devfreq_remove_device(kbdev->devfreq)) -+ dev_err(kbdev->dev, "Fail to rm devfreq\n"); -+ kbdev->devfreq = NULL; -+ kbase_devfreq_term_core_mask_table(kbdev); -+ dev_err(kbdev->dev, "Fail to init devfreq workqueue\n"); - return err; - } - - /* devfreq_add_device only copies a few of kbdev->dev's fields, so -- * set drvdata explicitly so IPA models can access kbdev. */ -+ * set drvdata explicitly so IPA models can access kbdev. -+ */ - dev_set_drvdata(&kbdev->devfreq->dev, kbdev); - - err = devfreq_register_opp_notifier(kbdev->dev, kbdev->devfreq); -@@ -667,11 +701,11 @@ int kbase_devfreq_init(struct kbase_device *kbdev) - goto opp_notifier_failed; - } - --#ifdef CONFIG_DEVFREQ_THERMAL -+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL) - err = kbase_ipa_init(kbdev); - if (err) { - dev_err(kbdev->dev, "IPA initialization failed\n"); -- goto cooling_failed; -+ goto ipa_init_failed; - } - - kbdev->devfreq_cooling = of_devfreq_cooling_register_power( -@@ -683,23 +717,28 @@ int kbase_devfreq_init(struct kbase_device *kbdev) - dev_err(kbdev->dev, - "Failed to register cooling device (%d)\n", - err); -- goto cooling_failed; -+ goto cooling_reg_failed; - } - #endif - - return 0; - --#ifdef CONFIG_DEVFREQ_THERMAL --cooling_failed: -+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL) -+cooling_reg_failed: -+ kbase_ipa_term(kbdev); -+ipa_init_failed: - devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); - #endif /* CONFIG_DEVFREQ_THERMAL */ -+ - opp_notifier_failed: -+ kbase_devfreq_work_term(kbdev); -+ - if (devfreq_remove_device(kbdev->devfreq)) - dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); -- else -- kbdev->devfreq = NULL; - -- kbase_devfreq_work_term(kbdev); -+ kbdev->devfreq = NULL; -+ -+ kbase_devfreq_term_core_mask_table(kbdev); - - return err; - } -@@ -710,7 +749,7 @@ void kbase_devfreq_term(struct kbase_device *kbdev) - - dev_dbg(kbdev->dev, "Term Mali devfreq\n"); - --#ifdef CONFIG_DEVFREQ_THERMAL -+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL) - if (kbdev->devfreq_cooling) - devfreq_cooling_unregister(kbdev->devfreq_cooling); - -@@ -719,6 +758,8 @@ void kbase_devfreq_term(struct kbase_device *kbdev) - - devfreq_unregister_opp_notifier(kbdev->dev, kbdev->devfreq); - -+ kbase_devfreq_work_term(kbdev); -+ - err = devfreq_remove_device(kbdev->devfreq); - if (err) - dev_err(kbdev->dev, "Failed to terminate devfreq (%d)\n", err); -@@ -726,6 +767,4 @@ void kbase_devfreq_term(struct kbase_device *kbdev) - kbdev->devfreq = NULL; - - kbase_devfreq_term_core_mask_table(kbdev); -- -- kbase_devfreq_work_term(kbdev); - } -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h -index 8c976b2..901827e 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_devfreq.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _BASE_DEVFREQ_H_ -@@ -43,4 +42,20 @@ void kbase_devfreq_force_freq(struct kbase_device *kbdev, unsigned long freq); - void kbase_devfreq_enqueue_work(struct kbase_device *kbdev, - enum kbase_devfreq_work_type work_type); - -+/** -+ * kbase_devfreq_opp_translate - Translate nominal OPP frequency from devicetree -+ * into real frequency & voltage pair, along with -+ * core mask -+ * @kbdev: Device pointer -+ * @freq: Nominal frequency -+ * @core_mask: Pointer to u64 to store core mask to -+ * @freqs: Pointer to array of frequencies -+ * @volts: Pointer to array of voltages -+ * -+ * This function will only perform translation if an operating-points-v2-mali -+ * table is present in devicetree. If one is not present then it will return an -+ * untranslated frequency (and corresponding voltage) and all cores enabled. -+ */ -+void kbase_devfreq_opp_translate(struct kbase_device *kbdev, unsigned long freq, -+ u64 *core_mask, unsigned long *freqs, unsigned long *volts); - #endif /* _BASE_DEVFREQ_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c -deleted file mode 100755 -index 5943e4e..0000000 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_hw.c -+++ /dev/null -@@ -1,388 +0,0 @@ --/* -- * -- * (C) COPYRIGHT 2014-2016, 2018-2020 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- */ -- -- --/* -- * -- */ --#include <mali_kbase.h> --#include <gpu/mali_kbase_gpu_fault.h> --#include <backend/gpu/mali_kbase_instr_internal.h> --#include <backend/gpu/mali_kbase_pm_internal.h> --#include <backend/gpu/mali_kbase_device_internal.h> --#include <mali_kbase_reset_gpu.h> --#include <mmu/mali_kbase_mmu.h> -- --#if !defined(CONFIG_MALI_NO_MALI) -- --#ifdef CONFIG_DEBUG_FS -- --int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size) --{ -- struct kbase_io_access *old_buf; -- struct kbase_io_access *new_buf; -- unsigned long flags; -- -- if (!new_size) -- goto out_err; /* The new size must not be 0 */ -- -- new_buf = vmalloc(new_size * sizeof(*h->buf)); -- if (!new_buf) -- goto out_err; -- -- spin_lock_irqsave(&h->lock, flags); -- -- old_buf = h->buf; -- -- /* Note: we won't bother with copying the old data over. The dumping -- * logic wouldn't work properly as it relies on 'count' both as a -- * counter and as an index to the buffer which would have changed with -- * the new array. This is a corner case that we don't need to support. -- */ -- h->count = 0; -- h->size = new_size; -- h->buf = new_buf; -- -- spin_unlock_irqrestore(&h->lock, flags); -- -- vfree(old_buf); -- -- return 0; -- --out_err: -- return -1; --} -- -- --int kbase_io_history_init(struct kbase_io_history *h, u16 n) --{ -- h->enabled = false; -- spin_lock_init(&h->lock); -- h->count = 0; -- h->size = 0; -- h->buf = NULL; -- if (kbase_io_history_resize(h, n)) -- return -1; -- -- return 0; --} -- -- --void kbase_io_history_term(struct kbase_io_history *h) --{ -- vfree(h->buf); -- h->buf = NULL; --} -- -- --/* kbase_io_history_add - add new entry to the register access history -- * -- * @h: Pointer to the history data structure -- * @addr: Register address -- * @value: The value that is either read from or written to the register -- * @write: 1 if it's a register write, 0 if it's a read -- */ --static void kbase_io_history_add(struct kbase_io_history *h, -- void __iomem const *addr, u32 value, u8 write) --{ -- struct kbase_io_access *io; -- unsigned long flags; -- -- spin_lock_irqsave(&h->lock, flags); -- -- io = &h->buf[h->count % h->size]; -- io->addr = (uintptr_t)addr | write; -- io->value = value; -- ++h->count; -- /* If count overflows, move the index by the buffer size so the entire -- * buffer will still be dumped later */ -- if (unlikely(!h->count)) -- h->count = h->size; -- -- spin_unlock_irqrestore(&h->lock, flags); --} -- -- --void kbase_io_history_dump(struct kbase_device *kbdev) --{ -- struct kbase_io_history *const h = &kbdev->io_history; -- u16 i; -- size_t iters; -- unsigned long flags; -- -- if (!unlikely(h->enabled)) -- return; -- -- spin_lock_irqsave(&h->lock, flags); -- -- dev_err(kbdev->dev, "Register IO History:"); -- iters = (h->size > h->count) ? h->count : h->size; -- dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters, -- h->count); -- for (i = 0; i < iters; ++i) { -- struct kbase_io_access *io = -- &h->buf[(h->count - iters + i) % h->size]; -- char const access = (io->addr & 1) ? 'w' : 'r'; -- -- dev_err(kbdev->dev, "%6i: %c: reg 0x%016lx val %08x\n", i, -- access, (unsigned long)(io->addr & ~0x1), io->value); -- } -- -- spin_unlock_irqrestore(&h->lock, flags); --} -- -- --#endif /* CONFIG_DEBUG_FS */ -- -- --void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value) --{ -- KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); -- KBASE_DEBUG_ASSERT(kbdev->dev != NULL); -- -- writel(value, kbdev->reg + offset); -- --#ifdef CONFIG_DEBUG_FS -- if (unlikely(kbdev->io_history.enabled)) -- kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, -- value, 1); --#endif /* CONFIG_DEBUG_FS */ -- dev_dbg(kbdev->dev, "w: reg %08x val %08x", offset, value); --} -- --KBASE_EXPORT_TEST_API(kbase_reg_write); -- --u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset) --{ -- u32 val; -- KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); -- KBASE_DEBUG_ASSERT(kbdev->dev != NULL); -- -- val = readl(kbdev->reg + offset); -- --#ifdef CONFIG_DEBUG_FS -- if (unlikely(kbdev->io_history.enabled)) -- kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, -- val, 0); --#endif /* CONFIG_DEBUG_FS */ -- dev_dbg(kbdev->dev, "r: reg %08x val %08x", offset, val); -- -- return val; --} -- --KBASE_EXPORT_TEST_API(kbase_reg_read); -- --bool kbase_is_gpu_lost(struct kbase_device *kbdev) --{ -- u32 val; -- -- val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); -- -- return val == 0; --} --#endif /* !defined(CONFIG_MALI_NO_MALI) */ -- --/** -- * kbase_report_gpu_fault - Report a GPU fault. -- * @kbdev: Kbase device pointer -- * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS -- * was also set -- * -- * This function is called from the interrupt handler when a GPU fault occurs. -- * It reports the details of the fault using dev_warn(). -- */ --static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) --{ -- u32 status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS)); -- u64 address = (u64) kbase_reg_read(kbdev, -- GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; -- -- address |= kbase_reg_read(kbdev, -- GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); -- -- meson_gpu_fault ++; -- dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", -- status, -- kbase_gpu_exception_name(status & 0xFF), -- address); -- if (multiple) -- dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n"); --} -- --static bool kbase_gpu_fault_interrupt(struct kbase_device *kbdev, int multiple) --{ -- kbase_report_gpu_fault(kbdev, multiple); -- return false; --} -- --void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev) --{ -- u32 irq_mask; -- -- lockdep_assert_held(&kbdev->hwaccess_lock); -- -- if (kbdev->cache_clean_in_progress) { -- /* If this is called while another clean is in progress, we -- * can't rely on the current one to flush any new changes in -- * the cache. Instead, trigger another cache clean immediately -- * after this one finishes. -- */ -- kbdev->cache_clean_queued = true; -- return; -- } -- -- /* Enable interrupt */ -- irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); -- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), -- irq_mask | CLEAN_CACHES_COMPLETED); -- -- KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); -- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -- GPU_COMMAND_CLEAN_INV_CACHES); -- -- kbdev->cache_clean_in_progress = true; --} -- --void kbase_gpu_start_cache_clean(struct kbase_device *kbdev) --{ -- unsigned long flags; -- -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- kbase_gpu_start_cache_clean_nolock(kbdev); -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); --} -- --void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev) --{ -- lockdep_assert_held(&kbdev->hwaccess_lock); -- -- kbdev->cache_clean_queued = false; -- kbdev->cache_clean_in_progress = false; -- wake_up(&kbdev->cache_clean_wait); --} -- --static void kbase_clean_caches_done(struct kbase_device *kbdev) --{ -- u32 irq_mask; -- unsigned long flags; -- -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- -- if (kbdev->cache_clean_queued) { -- kbdev->cache_clean_queued = false; -- -- KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); -- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -- GPU_COMMAND_CLEAN_INV_CACHES); -- } else { -- /* Disable interrupt */ -- irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); -- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), -- irq_mask & ~CLEAN_CACHES_COMPLETED); -- -- kbase_gpu_cache_clean_wait_complete(kbdev); -- } -- -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); --} -- --static inline bool get_cache_clean_flag(struct kbase_device *kbdev) --{ -- bool cache_clean_in_progress; -- unsigned long flags; -- -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- cache_clean_in_progress = kbdev->cache_clean_in_progress; -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -- -- return cache_clean_in_progress; --} -- --void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev) --{ -- while (get_cache_clean_flag(kbdev)) { -- wait_event_interruptible(kbdev->cache_clean_wait, -- !kbdev->cache_clean_in_progress); -- } --} -- --int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev, -- unsigned int wait_timeout_ms) --{ -- long remaining = msecs_to_jiffies(wait_timeout_ms); -- -- while (remaining && get_cache_clean_flag(kbdev)) { -- remaining = wait_event_timeout(kbdev->cache_clean_wait, -- !kbdev->cache_clean_in_progress, -- remaining); -- } -- -- return (remaining ? 0 : -ETIMEDOUT); --} -- --void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) --{ -- bool clear_gpu_fault = false; -- -- KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); -- if (val & GPU_FAULT) -- clear_gpu_fault = kbase_gpu_fault_interrupt(kbdev, -- val & MULTIPLE_GPU_FAULTS); -- -- if (val & RESET_COMPLETED) -- kbase_pm_reset_done(kbdev); -- -- if (val & PRFCNT_SAMPLE_COMPLETED) -- kbase_instr_hwcnt_sample_done(kbdev); -- -- KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val); -- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val); -- -- /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must -- * be called after the IRQ has been cleared. This is because it might -- * trigger further power transitions and we don't want to miss the -- * interrupt raised to notify us that these further transitions have -- * finished. The same applies to kbase_clean_caches_done() - if another -- * clean was queued, it might trigger another clean, which might -- * generate another interrupt which shouldn't be missed. -- */ -- -- if (val & CLEAN_CACHES_COMPLETED) -- kbase_clean_caches_done(kbdev); -- -- if (val & POWER_CHANGED_ALL) { -- kbase_pm_power_changed(kbdev); -- } else if (val & CLEAN_CACHES_COMPLETED) { -- /* If cache line evict messages can be lost when shader cores -- * power down then we need to flush the L2 cache before powering -- * down cores. When the flush completes, the shaders' state -- * machine needs to be re-invoked to proceed with powering down -- * cores. -- */ -- if (kbdev->pm.backend.l2_always_on || -- kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) -- kbase_pm_power_changed(kbdev); -- } -- -- -- KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); --} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h -deleted file mode 100644 -index 2e1d011..0000000 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_device_internal.h -+++ /dev/null -@@ -1,127 +0,0 @@ --/* -- * -- * (C) COPYRIGHT 2014,2019-2020 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- */ -- -- -- --/* -- * Backend-specific HW access device APIs -- */ -- --#ifndef _KBASE_DEVICE_INTERNAL_H_ --#define _KBASE_DEVICE_INTERNAL_H_ -- --/** -- * kbase_reg_write - write to GPU register -- * @kbdev: Kbase device pointer -- * @offset: Offset of register -- * @value: Value to write -- * -- * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). -- */ --void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value); -- --/** -- * kbase_reg_read - read from GPU register -- * @kbdev: Kbase device pointer -- * @offset: Offset of register -- * -- * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). -- * -- * Return: Value in desired register -- */ --u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset); -- --/** -- * kbase_is_gpu_lost() - Has the GPU been lost. -- * @kbdev: Kbase device pointer -- * -- * This function will return true if the GPU has been lost. -- * When this happens register reads will be zero. A zero GPU_ID is -- * invalid so this is used to detect GPU_LOST -- * -- * Return: True if GPU LOST -- */ --bool kbase_is_gpu_lost(struct kbase_device *kbdev); -- --/** -- * kbase_gpu_start_cache_clean - Start a cache clean -- * @kbdev: Kbase device -- * -- * Issue a cache clean and invalidate command to hardware. This function will -- * take hwaccess_lock. -- */ --void kbase_gpu_start_cache_clean(struct kbase_device *kbdev); -- --/** -- * kbase_gpu_start_cache_clean_nolock - Start a cache clean -- * @kbdev: Kbase device -- * -- * Issue a cache clean and invalidate command to hardware. hwaccess_lock -- * must be held by the caller. -- */ --void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev); -- --/** -- * kbase_gpu_wait_cache_clean - Wait for cache cleaning to finish -- * @kbdev: Kbase device -- * -- * This function will take hwaccess_lock, and may sleep. -- */ --void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev); -- --/** -- * kbase_gpu_wait_cache_clean_timeout - Wait for certain time for cache -- * cleaning to finish -- * @kbdev: Kbase device -- * @wait_timeout_ms: Time, in milli seconds, to wait for cache clean to complete. -- * -- * This function will take hwaccess_lock, and may sleep. This is supposed to be -- * called from paths (like GPU reset) where an indefinite wait for the completion -- * of cache clean operation can cause deadlock, as the operation may never -- * complete. -- * -- * Return: 0 if successful or a negative error code on failure. -- */ --int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev, -- unsigned int wait_timeout_ms); -- --/** -- * kbase_gpu_cache_clean_wait_complete - Called after the cache cleaning is -- * finished. Would also be called after -- * the GPU reset. -- * @kbdev: Kbase device -- * -- * Caller must hold the hwaccess_lock. -- */ --void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev); -- --/** -- * kbase_gpu_interrupt - GPU interrupt handler -- * @kbdev: Kbase device pointer -- * @val: The value of the GPU IRQ status register which triggered the call -- * -- * This function is called from the interrupt handler when a GPU irq is to be -- * handled. -- */ --void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); -- --#endif /* _KBASE_DEVICE_INTERNAL_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c -index 352afa1..11088db 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_gpuprops_backend.c -@@ -1,12 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -17,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -26,7 +24,7 @@ - */ - - #include <mali_kbase.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include <backend/gpu/mali_kbase_pm_internal.h> - #include <mali_kbase_hwaccess_gpuprops.h> - -@@ -41,8 +39,19 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, - - registers.l2_features = kbase_reg_read(kbdev, - GPU_CONTROL_REG(L2_FEATURES)); -+ registers.core_features = 0; -+#if !MALI_USE_CSF -+ /* TGOx */ - registers.core_features = kbase_reg_read(kbdev, - GPU_CONTROL_REG(CORE_FEATURES)); -+#else /* !MALI_USE_CSF */ -+ if (((registers.gpu_id & GPU_ID2_PRODUCT_MODEL) == -+ GPU_ID2_PRODUCT_TGRX) || -+ ((registers.gpu_id & GPU_ID2_PRODUCT_MODEL) == -+ GPU_ID2_PRODUCT_TVAX)) -+ registers.core_features = -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(CORE_FEATURES)); -+#endif /* MALI_USE_CSF */ - registers.tiler_features = kbase_reg_read(kbdev, - GPU_CONTROL_REG(TILER_FEATURES)); - registers.mem_features = kbase_reg_read(kbdev, -@@ -51,12 +60,20 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, - GPU_CONTROL_REG(MMU_FEATURES)); - registers.as_present = kbase_reg_read(kbdev, - GPU_CONTROL_REG(AS_PRESENT)); -+#if !MALI_USE_CSF - registers.js_present = kbase_reg_read(kbdev, - GPU_CONTROL_REG(JS_PRESENT)); -+#else /* !MALI_USE_CSF */ -+ registers.js_present = 0; -+#endif /* !MALI_USE_CSF */ - - for (i = 0; i < GPU_MAX_JOB_SLOTS; i++) -+#if !MALI_USE_CSF - registers.js_features[i] = kbase_reg_read(kbdev, - GPU_CONTROL_REG(JS_FEATURES_REG(i))); -+#else /* !MALI_USE_CSF */ -+ registers.js_features[i] = 0; -+#endif /* !MALI_USE_CSF */ - - for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++) - registers.texture_features[i] = kbase_reg_read(kbdev, -@@ -93,13 +110,49 @@ int kbase_backend_gpuprops_get(struct kbase_device *kbdev, - registers.stack_present_hi = kbase_reg_read(kbdev, - GPU_CONTROL_REG(STACK_PRESENT_HI)); - -- if (!kbase_is_gpu_lost(kbdev)) { -+ if (registers.gpu_id >= GPU_ID2_PRODUCT_MAKE(11, 8, 5, 2)) { -+ registers.gpu_features_lo = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FEATURES_LO)); -+ registers.gpu_features_hi = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FEATURES_HI)); -+ } else { -+ registers.gpu_features_lo = 0; -+ registers.gpu_features_hi = 0; -+ } -+ -+ if (!kbase_is_gpu_removed(kbdev)) { - *regdump = registers; - return 0; - } else - return -EIO; - } - -+int kbase_backend_gpuprops_get_curr_config(struct kbase_device *kbdev, -+ struct kbase_current_config_regdump *curr_config_regdump) -+{ -+ if (WARN_ON(!kbdev) || WARN_ON(!curr_config_regdump)) -+ return -EINVAL; -+ -+ curr_config_regdump->mem_features = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(MEM_FEATURES)); -+ -+ curr_config_regdump->shader_present_lo = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(SHADER_PRESENT_LO)); -+ curr_config_regdump->shader_present_hi = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(SHADER_PRESENT_HI)); -+ -+ curr_config_regdump->l2_present_lo = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(L2_PRESENT_LO)); -+ curr_config_regdump->l2_present_hi = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(L2_PRESENT_HI)); -+ -+ if (WARN_ON(kbase_is_gpu_removed(kbdev))) -+ return -EIO; -+ -+ return 0; -+ -+} -+ - int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, - struct kbase_gpuprops_regdump *regdump) - { -@@ -112,7 +165,7 @@ int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev, - coherency_features = kbase_reg_read(kbdev, - GPU_CONTROL_REG(COHERENCY_FEATURES)); - -- if (kbase_is_gpu_lost(kbdev)) -+ if (kbase_is_gpu_removed(kbdev)) - return -EIO; - - regdump->coherency_features = coherency_features; -@@ -135,11 +188,15 @@ int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev, - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) { - u32 l2_features = kbase_reg_read(kbdev, - GPU_CONTROL_REG(L2_FEATURES)); -+ u32 l2_config = -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_CONFIG)); -+ - -- if (kbase_is_gpu_lost(kbdev)) -+ if (kbase_is_gpu_removed(kbdev)) - return -EIO; - - regdump->l2_features = l2_features; -+ regdump->l2_config = l2_config; - } - - return 0; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c -index 8b320c7..d7edf30 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_backend.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * GPU backend instrumentation APIs. - */ -@@ -29,7 +26,7 @@ - #include <mali_kbase.h> - #include <gpu/mali_kbase_gpu_regmap.h> - #include <mali_kbase_hwaccess_instr.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include <backend/gpu/mali_kbase_instr_internal.h> - - -@@ -71,12 +68,12 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, - - /* Configure */ - prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT; --#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS -- if (kbdev->hwcnt.backend.use_secondary_override) -+#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS -+ prfcnt_config |= kbdev->hwcnt.backend.override_counter_set -+ << PRFCNT_CONFIG_SETSELECT_SHIFT; - #else -- if (enable->use_secondary) -+ prfcnt_config |= enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT; - #endif -- prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT; - - kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), - prfcnt_config | PRFCNT_CONFIG_MODE_OFF); -@@ -87,7 +84,7 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, - enable->dump_buffer >> 32); - - kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), -- enable->jm_bm); -+ enable->fe_bm); - - kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), - enable->shader_bm); -@@ -110,7 +107,7 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev, - - err = 0; - -- dev_dbg(kbdev->dev, "HW counters dumping set-up for context %p", kctx); -+ dev_dbg(kbdev->dev, "HW counters dumping set-up for context %pK", kctx); - return err; - out_err: - return err; -@@ -170,11 +167,10 @@ int kbase_instr_hwcnt_disable_internal(struct kbase_context *kctx) - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); - -- dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", -+ dev_dbg(kbdev->dev, "HW counters dumping disabled for context %pK", - kctx); - - err = 0; -- - out: - return err; - } -@@ -194,7 +190,8 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) - - if (kbdev->hwcnt.backend.state != KBASE_INSTR_STATE_IDLE) { - /* HW counters are disabled or another dump is ongoing, or we're -- * resetting */ -+ * resetting -+ */ - goto unlock; - } - -@@ -204,7 +201,6 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) - */ - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING; - -- - /* Reconfigure the dump address */ - kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), - kbdev->hwcnt.addr & 0xFFFFFFFF); -@@ -218,14 +214,13 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx) - kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), - GPU_COMMAND_PRFCNT_SAMPLE); - -- dev_dbg(kbdev->dev, "HW counters dumping done for context %p", kctx); -+ dev_dbg(kbdev->dev, "HW counters dumping done for context %pK", kctx); - - err = 0; - - unlock: - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - -- - return err; - } - KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump); -@@ -254,40 +249,6 @@ bool kbase_instr_hwcnt_dump_complete(struct kbase_context *kctx, - } - KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_dump_complete); - --void kbasep_cache_clean_worker(struct work_struct *data) --{ -- struct kbase_device *kbdev; -- unsigned long flags, pm_flags; -- -- kbdev = container_of(data, struct kbase_device, -- hwcnt.backend.cache_clean_work); -- -- spin_lock_irqsave(&kbdev->hwaccess_lock, pm_flags); -- spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -- -- /* Clean and invalidate the caches so we're sure the mmu tables for the -- * dump buffer is valid. -- */ -- KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == -- KBASE_INSTR_STATE_REQUEST_CLEAN); -- kbase_gpu_start_cache_clean_nolock(kbdev); -- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, pm_flags); -- -- kbase_gpu_wait_cache_clean(kbdev); -- -- spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -- KBASE_DEBUG_ASSERT(kbdev->hwcnt.backend.state == -- KBASE_INSTR_STATE_REQUEST_CLEAN); -- /* All finished and idle */ -- kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; -- kbdev->hwcnt.backend.triggered = 1; -- wake_up(&kbdev->hwcnt.backend.wait); -- -- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); --} -- -- - void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) - { - unsigned long flags; -@@ -298,20 +259,10 @@ void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev) - kbdev->hwcnt.backend.triggered = 1; - wake_up(&kbdev->hwcnt.backend.wait); - } else if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_DUMPING) { -- if (kbdev->mmu_mode->flags & KBASE_MMU_MODE_HAS_NON_CACHEABLE) { -- /* All finished and idle */ -- kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; -- kbdev->hwcnt.backend.triggered = 1; -- wake_up(&kbdev->hwcnt.backend.wait); -- } else { -- int ret; -- /* Always clean and invalidate the cache after a successful dump -- */ -- kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_REQUEST_CLEAN; -- ret = queue_work(kbdev->hwcnt.backend.cache_clean_wq, -- &kbdev->hwcnt.backend.cache_clean_work); -- KBASE_DEBUG_ASSERT(ret); -- } -+ /* All finished and idle */ -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; -+ kbdev->hwcnt.backend.triggered = 1; -+ wake_up(&kbdev->hwcnt.backend.wait); - } - - spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -@@ -353,7 +304,8 @@ int kbase_instr_hwcnt_clear(struct kbase_context *kctx) - spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - - /* Check it's the context previously set up and we're not already -- * dumping */ -+ * dumping -+ */ - if (kbdev->hwcnt.kctx != kctx || kbdev->hwcnt.backend.state != - KBASE_INSTR_STATE_IDLE) - goto out; -@@ -373,39 +325,45 @@ KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_clear); - - int kbase_instr_backend_init(struct kbase_device *kbdev) - { -- int ret = 0; -+ spin_lock_init(&kbdev->hwcnt.lock); - - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DISABLED; - - init_waitqueue_head(&kbdev->hwcnt.backend.wait); -- INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work, -- kbasep_cache_clean_worker); -- - - kbdev->hwcnt.backend.triggered = 0; - --#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS -- kbdev->hwcnt.backend.use_secondary_override = false; -+#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS -+/* Use the build time option for the override default. */ -+#if defined(CONFIG_MALI_PRFCNT_SET_SECONDARY) -+ kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_SET_SECONDARY; -+#elif defined(CONFIG_MALI_PRFCNT_SET_TERTIARY) -+ kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_SET_TERTIARY; -+#else -+ /* Default to primary */ -+ kbdev->hwcnt.backend.override_counter_set = KBASE_HWCNT_SET_PRIMARY; - #endif -- -- kbdev->hwcnt.backend.cache_clean_wq = -- alloc_workqueue("Mali cache cleaning workqueue", 0, 1); -- if (NULL == kbdev->hwcnt.backend.cache_clean_wq) -- ret = -EINVAL; -- -- return ret; -+#endif -+ return 0; - } - - void kbase_instr_backend_term(struct kbase_device *kbdev) - { -- destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq); -+ CSTD_UNUSED(kbdev); - } - --#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS -+#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS - void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev) - { -- debugfs_create_bool("hwcnt_use_secondary", S_IRUGO | S_IWUSR, -- kbdev->mali_debugfs_directory, -- &kbdev->hwcnt.backend.use_secondary_override); -+ /* No validation is done on the debugfs input. Invalid input could cause -+ * performance counter errors. This is acceptable since this is a debug -+ * only feature and users should know what they are doing. -+ * -+ * Valid inputs are the values accepted bythe SET_SELECT bits of the -+ * PRFCNT_CONFIG register as defined in the architecture specification. -+ */ -+ debugfs_create_u8("hwcnt_set_select", S_IRUGO | S_IWUSR, -+ kbdev->mali_debugfs_directory, -+ (u8 *)&kbdev->hwcnt.backend.override_counter_set); - } - #endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h -index 9930968..e356348 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_defs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2016, 2018, 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -27,6 +26,8 @@ - #ifndef _KBASE_INSTR_DEFS_H_ - #define _KBASE_INSTR_DEFS_H_ - -+#include <mali_kbase_hwcnt_gpu.h> -+ - /* - * Instrumentation State Machine States - */ -@@ -37,8 +38,6 @@ enum kbase_instr_state { - KBASE_INSTR_STATE_IDLE, - /* Hardware is currently dumping a frame. */ - KBASE_INSTR_STATE_DUMPING, -- /* We've requested a clean to occur on a workqueue */ -- KBASE_INSTR_STATE_REQUEST_CLEAN, - /* An error has occured during DUMPING (page fault). */ - KBASE_INSTR_STATE_FAULT - }; -@@ -47,14 +46,11 @@ enum kbase_instr_state { - struct kbase_instr_backend { - wait_queue_head_t wait; - int triggered; --#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS -- bool use_secondary_override; -+#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS -+ enum kbase_hwcnt_physical_set override_counter_set; - #endif - - enum kbase_instr_state state; -- struct workqueue_struct *cache_clean_wq; -- struct work_struct cache_clean_work; - }; - - #endif /* _KBASE_INSTR_DEFS_H_ */ -- -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h -index 2254b9f..332cc69 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_instr_internal.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Backend-specific HW access instrumentation APIs - */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h -index ca3c048..2671ce5 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_internal.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2015, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c -index 8696c6a..a29f7ef 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_irq_linux.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,17 +17,15 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include <backend/gpu/mali_kbase_irq_internal.h> - - #include <linux/interrupt.h> - --#if !defined(CONFIG_MALI_NO_MALI) -+#if !IS_ENABLED(CONFIG_MALI_NO_MALI) - - /* GPU IRQ Tags */ - #define JOB_IRQ_TAG 0 -@@ -72,7 +71,12 @@ static irqreturn_t kbase_job_irq_handler(int irq, void *data) - - dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val); - -+#if MALI_USE_CSF -+ /* call the csf interrupt handler */ -+ kbase_csf_interrupt(kbdev, val); -+#else - kbase_job_done(kbdev, val); -+#endif - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -@@ -210,24 +214,25 @@ int kbase_set_custom_irq_handler(struct kbase_device *kbdev, - int result = 0; - irq_handler_t requested_irq_handler = NULL; - -- KBASE_DEBUG_ASSERT((JOB_IRQ_HANDLER <= irq_type) && -- (GPU_IRQ_HANDLER >= irq_type)); -+ KBASE_DEBUG_ASSERT((irq_type >= JOB_IRQ_HANDLER) && -+ (irq_type <= GPU_IRQ_HANDLER)); - - /* Release previous handler */ - if (kbdev->irqs[irq_type].irq) - free_irq(kbdev->irqs[irq_type].irq, kbase_tag(kbdev, irq_type)); - -- requested_irq_handler = (NULL != custom_handler) ? custom_handler : -- kbase_handler_table[irq_type]; -+ requested_irq_handler = (custom_handler != NULL) ? -+ custom_handler : -+ kbase_handler_table[irq_type]; - -- if (0 != request_irq(kbdev->irqs[irq_type].irq, -- requested_irq_handler, -+ if (request_irq(kbdev->irqs[irq_type].irq, requested_irq_handler, - kbdev->irqs[irq_type].flags | IRQF_SHARED, -- dev_name(kbdev->dev), kbase_tag(kbdev, irq_type))) { -+ dev_name(kbdev->dev), -+ kbase_tag(kbdev, irq_type)) != 0) { - result = -EINVAL; - dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", - kbdev->irqs[irq_type].irq, irq_type); --#ifdef CONFIG_SPARSE_IRQ -+#if IS_ENABLED(CONFIG_SPARSE_IRQ) - dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); - #endif /* CONFIG_SPARSE_IRQ */ - } -@@ -456,7 +461,7 @@ int kbase_install_interrupts(struct kbase_device *kbdev) - if (err) { - dev_err(kbdev->dev, "Can't request interrupt %d (index %d)\n", - kbdev->irqs[i].irq, i); --#ifdef CONFIG_SPARSE_IRQ -+#if IS_ENABLED(CONFIG_SPARSE_IRQ) - dev_err(kbdev->dev, "You have CONFIG_SPARSE_IRQ support enabled - is the interrupt number correct for this configuration?\n"); - #endif /* CONFIG_SPARSE_IRQ */ - goto release; -@@ -496,4 +501,4 @@ void kbase_synchronize_irqs(struct kbase_device *kbdev) - - KBASE_EXPORT_TEST_API(kbase_synchronize_irqs); - --#endif /* !defined(CONFIG_MALI_NO_MALI) */ -+#endif /* !IS_ENABLED(CONFIG_MALI_NO_MALI) */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c -index bb4f548..888aa59 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_as.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * Register backend context / address space management - */ -@@ -58,8 +56,10 @@ static void assign_and_activate_kctx_addr_space(struct kbase_device *kbdev, - lockdep_assert_held(&js_devdata->runpool_mutex); - lockdep_assert_held(&kbdev->hwaccess_lock); - -+#if !MALI_USE_CSF - /* Attribute handling */ - kbasep_js_ctx_attr_runpool_retain_ctx(kbdev, kctx); -+#endif - - /* Allow it to run jobs */ - kbasep_js_set_submit_allowed(js_devdata, kctx); -@@ -188,8 +188,8 @@ int kbase_backend_find_and_release_free_address_space( - } - - /* Context was retained while locks were dropped, -- * continue looking for free AS */ -- -+ * continue looking for free AS -+ */ - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&as_js_kctx_info->ctx.jsctx_mutex); - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h -index 7cda61a..e29ace7 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_defs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2016, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * Register-based HW access backend specific definitions - */ -@@ -78,11 +76,12 @@ struct slot_rb { - * The hwaccess_lock (a spinlock) must be held when accessing this structure - */ - struct kbase_backend_data { -+#if !MALI_USE_CSF - struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS]; -- - struct hrtimer scheduling_timer; - - bool timer_running; -+#endif - bool suspend_timer; - - atomic_t reset_gpu; -@@ -92,13 +91,16 @@ struct kbase_backend_data { - /* kbase_prepare_to_reset_gpu has been called */ - #define KBASE_RESET_GPU_PREPARED 1 - /* kbase_reset_gpu has been called - the reset will now definitely happen -- * within the timeout period */ -+ * within the timeout period -+ */ - #define KBASE_RESET_GPU_COMMITTED 2 - /* The GPU reset process is currently occuring (timeout has expired or -- * kbasep_try_reset_gpu_early was called) */ -+ * kbasep_try_reset_gpu_early was called) -+ */ - #define KBASE_RESET_GPU_HAPPENING 3 - /* Reset the GPU silently, used when resetting the GPU as part of normal -- * behavior (e.g. when exiting protected mode). */ -+ * behavior (e.g. when exiting protected mode). -+ */ - #define KBASE_RESET_GPU_SILENT 4 - struct workqueue_struct *reset_workq; - struct work_struct reset_work; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c -index fa6bc83..ae0377f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -33,16 +32,20 @@ - #include <mali_kbase_hwaccess_jm.h> - #include <mali_kbase_reset_gpu.h> - #include <mali_kbase_ctx_sched.h> -+#include <mali_kbase_kinstr_jm.h> - #include <mali_kbase_hwcnt_context.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include <backend/gpu/mali_kbase_irq_internal.h> - #include <backend/gpu/mali_kbase_jm_internal.h> -+#include <mali_kbase_regs_history_debugfs.h> - - static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev); -+static u64 kbasep_apply_limited_core_mask(const struct kbase_device *kbdev, -+ const u64 affinity, const u64 limited_core_mask); - - static u64 kbase_job_write_affinity(struct kbase_device *kbdev, - base_jd_core_req core_req, -- int js) -+ int js, const u64 limited_core_mask) - { - u64 affinity; - -@@ -71,14 +74,21 @@ static u64 kbase_job_write_affinity(struct kbase_device *kbdev, - */ - if (js == 2 && num_core_groups > 1) - affinity &= coherency_info->group[1].core_mask; -- else -+ else if (num_core_groups > 1) - affinity &= coherency_info->group[0].core_mask; -+ else -+ affinity &= kbdev->gpu_props.curr_config.shader_present; - } else { - /* Use all cores */ - affinity = kbdev->pm.backend.shaders_avail & - kbdev->pm.debug_core_mask[js]; - } - -+ if (core_req & BASE_JD_REQ_LIMITED_CORE_MASK) { -+ /* Limiting affinity due to BASE_JD_REQ_LIMITED_CORE_MASK by applying the limited core mask. */ -+ affinity = kbasep_apply_limited_core_mask(kbdev, affinity, limited_core_mask); -+ } -+ - if (unlikely(!affinity)) { - #ifdef CONFIG_MALI_DEBUG - u64 shaders_ready = -@@ -88,6 +98,16 @@ static u64 kbase_job_write_affinity(struct kbase_device *kbdev, - #endif - - affinity = kbdev->pm.backend.shaders_avail; -+ -+ if (core_req & BASE_JD_REQ_LIMITED_CORE_MASK) { -+ /* Limiting affinity again to make sure it only enables shader cores with backed TLS memory. */ -+ affinity = kbasep_apply_limited_core_mask(kbdev, affinity, limited_core_mask); -+ -+#ifdef CONFIG_MALI_DEBUG -+ /* affinity should never be 0 */ -+ WARN_ON(!affinity); -+#endif -+ } - } - - kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO), -@@ -168,7 +188,7 @@ static u64 select_job_chain(struct kbase_jd_atom *katom) - } - - dev_dbg(kctx->kbdev->dev, -- "Selected job chain 0x%llx for end atom %p in state %d\n", -+ "Selected job chain 0x%llx for end atom %pK in state %d\n", - jc, (void *)katom, (int)rp->state); - - katom->jc = jc; -@@ -192,7 +212,7 @@ void kbase_job_hw_submit(struct kbase_device *kbdev, - /* Command register must be available */ - KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx)); - -- dev_dbg(kctx->kbdev->dev, "Write JS_HEAD_NEXT 0x%llx for atom %p\n", -+ dev_dbg(kctx->kbdev->dev, "Write JS_HEAD_NEXT 0x%llx for atom %pK\n", - jc_head, (void *)katom); - - kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), -@@ -200,10 +220,12 @@ void kbase_job_hw_submit(struct kbase_device *kbdev, - kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), - jc_head >> 32); - -- affinity = kbase_job_write_affinity(kbdev, katom->core_req, js); -+ affinity = kbase_job_write_affinity(kbdev, katom->core_req, js, -+ kctx->limited_core_mask); - - /* start MMU, medium priority, cache clean/flush on end, clean/flush on -- * start */ -+ * start -+ */ - cfg = kctx->as_nr; - - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION) && -@@ -255,7 +277,7 @@ void kbase_job_hw_submit(struct kbase_device *kbdev, - katom->start_timestamp = ktime_get(); - - /* GO ! */ -- dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx", -+ dev_dbg(kbdev->dev, "JS: Submitting atom %pK from ctx %pK to js[%d] with head=0x%llx", - katom, kctx, js, jc_head); - - KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, -@@ -277,7 +299,8 @@ void kbase_job_hw_submit(struct kbase_device *kbdev, - katom, - &kbdev->gpu_props.props.raw_props.js_features[js], - "ctx_nr,atom_nr"); --#ifdef CONFIG_GPU_TRACEPOINTS -+ kbase_kinstr_jm_atom_hw_submit(katom); -+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) - if (!kbase_backend_nr_atoms_submitted(kbdev, js)) { - /* If this is the only job on the slot, trace it as starting */ - char js_string[16]; -@@ -328,7 +351,8 @@ static void kbasep_job_slot_update_head_start_timestamp( - /* Only update the timestamp if it's a better estimate - * than what's currently stored. This is because our - * estimate that accounts for the throttle time may be -- * too much of an overestimate */ -+ * too much of an overestimate -+ */ - katom->start_timestamp = end_timestamp; - } - } -@@ -371,9 +395,9 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) - /* treat failed slots as finished slots */ - u32 finished = (done & 0xFFFF) | failed; - -- /* Note: This is inherently unfair, as we always check -- * for lower numbered interrupts before the higher -- * numbered ones.*/ -+ /* Note: This is inherently unfair, as we always check for lower -+ * numbered interrupts before the higher numbered ones. -+ */ - i = ffs(finished) - 1; - KBASE_DEBUG_ASSERT(i >= 0); - -@@ -385,7 +409,8 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) - - if (failed & (1u << i)) { - /* read out the job slot status code if the job -- * slot reported failure */ -+ * slot reported failure -+ */ - completion_code = kbase_reg_read(kbdev, - JOB_SLOT_REG(i, JS_STATUS)); - -@@ -399,7 +424,8 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) - - /* Soft-stopped job - read the value of - * JS<n>_TAIL so that the job chain can -- * be resumed */ -+ * be resumed -+ */ - job_tail = (u64)kbase_reg_read(kbdev, - JOB_SLOT_REG(i, JS_TAIL_LO)) | - ((u64)kbase_reg_read(kbdev, -@@ -408,21 +434,26 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) - } else if (completion_code == - BASE_JD_EVENT_NOT_STARTED) { - /* PRLAM-10673 can cause a TERMINATED -- * job to come back as NOT_STARTED, but -- * the error interrupt helps us detect -- * it */ -+ * job to come back as NOT_STARTED, -+ * but the error interrupt helps us -+ * detect it -+ */ - completion_code = - BASE_JD_EVENT_TERMINATED; - } - - kbase_gpu_irq_evict(kbdev, i, completion_code); - -- /* Some jobs that encounter a BUS FAULT may result in corrupted -- * state causing future jobs to hang. Reset GPU before -- * allowing any other jobs on the slot to continue. */ -+ /* Some jobs that encounter a BUS FAULT may -+ * result in corrupted state causing future -+ * jobs to hang. Reset GPU before allowing -+ * any other jobs on the slot to continue. -+ */ - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3076)) { - if (completion_code == BASE_JD_EVENT_JOB_BUS_FAULT) { -- if (kbase_prepare_to_reset_gpu_locked(kbdev)) -+ if (kbase_prepare_to_reset_gpu_locked( -+ kbdev, -+ RESET_FLAGS_NONE)) - kbase_reset_gpu_locked(kbdev); - } - } -@@ -480,7 +511,8 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done) - - if ((rawstat >> (i + 16)) & 1) { - /* There is a failed job that we've -- * missed - add it back to active */ -+ * missed - add it back to active -+ */ - active |= (1u << i); - } - } -@@ -582,7 +614,8 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, - } - - /* We are about to issue a soft stop, so mark the atom as having -- * been soft stopped */ -+ * been soft stopped -+ */ - target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED; - - /* Mark the point where we issue the soft-stop command */ -@@ -692,12 +725,40 @@ void kbase_backend_jm_kill_running_jobs_from_kctx(struct kbase_context *kctx) - kbase_job_slot_hardstop(kctx, i, NULL); - } - -+/** -+ * kbase_is_existing_atom_submitted_later_than_ready -+ * @ready: sequence number of the ready atom -+ * @existing: sequence number of the existing atom -+ * -+ * Returns true if the existing atom has been submitted later than the -+ * ready atom. It is used to understand if an atom that is ready has been -+ * submitted earlier than the currently running atom, so that the currently -+ * running atom should be preempted to allow the ready atom to run. -+ */ -+static inline bool kbase_is_existing_atom_submitted_later_than_ready(u64 ready, u64 existing) -+{ -+ /* No seq_nr set? */ -+ if (!ready || !existing) -+ return false; -+ -+ /* Efficiently handle the unlikely case of wrapping. -+ * The following code assumes that the delta between the sequence number -+ * of the two atoms is less than INT64_MAX. -+ * In the extremely unlikely case where the delta is higher, the comparison -+ * defaults for no preemption. -+ * The code also assumes that the conversion from unsigned to signed types -+ * works because the signed integers are 2's complement. -+ */ -+ return (s64)(ready - existing) < 0; -+} -+ - void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, - struct kbase_jd_atom *target_katom) - { - struct kbase_device *kbdev; - int js = target_katom->slot_nr; - int priority = target_katom->sched_priority; -+ int seq_nr = target_katom->seq_nr; - int i; - bool stop_sent = false; - -@@ -719,7 +780,8 @@ void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx, - (katom->kctx != kctx)) - continue; - -- if (katom->sched_priority > priority) { -+ if ((katom->sched_priority > priority) || -+ (katom->kctx == kctx && kbase_is_existing_atom_submitted_later_than_ready(seq_nr, katom->seq_nr))) { - if (!stop_sent) - KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITIZED( - kbdev, -@@ -749,7 +811,7 @@ static int softstop_start_rp_nolock( - - if (!(katom->core_req & BASE_JD_REQ_START_RENDERPASS)) { - dev_dbg(kctx->kbdev->dev, -- "Atom %p on job slot is not start RP\n", (void *)katom); -+ "Atom %pK on job slot is not start RP\n", (void *)katom); - return -EPERM; - } - -@@ -762,13 +824,13 @@ static int softstop_start_rp_nolock( - rp->state != KBASE_JD_RP_RETRY)) - return -EINVAL; - -- dev_dbg(kctx->kbdev->dev, "OOM in state %d with region %p\n", -+ dev_dbg(kctx->kbdev->dev, "OOM in state %d with region %pK\n", - (int)rp->state, (void *)reg); - - if (WARN_ON(katom != rp->start_katom)) - return -EINVAL; - -- dev_dbg(kctx->kbdev->dev, "Adding region %p to list %p\n", -+ dev_dbg(kctx->kbdev->dev, "Adding region %pK to list %pK\n", - (void *)reg, (void *)&rp->oom_reg_list); - list_move_tail(®->link, &rp->oom_reg_list); - dev_dbg(kctx->kbdev->dev, "Added region to list\n"); -@@ -813,9 +875,9 @@ void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) - if (timeout != 0) - goto exit; - -- if (kbase_prepare_to_reset_gpu(kbdev)) { -+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) { - dev_err(kbdev->dev, -- "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", -+ "Issuing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n", - ZAP_TIMEOUT); - kbase_reset_gpu(kbdev); - } -@@ -823,7 +885,7 @@ void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx) - /* Wait for the reset to complete */ - kbase_reset_gpu_wait(kbdev); - exit: -- dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx); -+ dev_dbg(kbdev->dev, "Zap: Finished Context %pK", kctx); - - /* Ensure that the signallers of the waitqs have finished */ - mutex_lock(&kctx->jctx.lock); -@@ -884,7 +946,7 @@ KBASE_EXPORT_TEST_API(kbase_job_slot_term); - void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js, - struct kbase_jd_atom *target_katom, u32 sw_flags) - { -- dev_dbg(kbdev->dev, "Soft-stop atom %p with flags 0x%x (s:%d)\n", -+ dev_dbg(kbdev->dev, "Soft-stop atom %pK with flags 0x%x (s:%d)\n", - target_katom, sw_flags, js); - - KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK)); -@@ -988,6 +1050,33 @@ void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, - } - } - -+int kbase_reset_gpu_prevent_and_wait(struct kbase_device *kbdev) -+{ -+ WARN(true, "%s Not implemented for JM GPUs", __func__); -+ return -EINVAL; -+} -+ -+int kbase_reset_gpu_try_prevent(struct kbase_device *kbdev) -+{ -+ WARN(true, "%s Not implemented for JM GPUs", __func__); -+ return -EINVAL; -+} -+ -+void kbase_reset_gpu_allow(struct kbase_device *kbdev) -+{ -+ WARN(true, "%s Not implemented for JM GPUs", __func__); -+} -+ -+void kbase_reset_gpu_assert_prevented(struct kbase_device *kbdev) -+{ -+ WARN(true, "%s Not implemented for JM GPUs", __func__); -+} -+ -+void kbase_reset_gpu_assert_failed_or_prevented(struct kbase_device *kbdev) -+{ -+ WARN(true, "%s Not implemented for JM GPUs", __func__); -+} -+ - static void kbase_debug_dump_registers(struct kbase_device *kbdev) - { - int i; -@@ -1054,13 +1143,15 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) - - /* Make sure the timer has completed - this cannot be done from - * interrupt context, so this cannot be done within -- * kbasep_try_reset_gpu_early. */ -+ * kbasep_try_reset_gpu_early. -+ */ - hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); - - if (kbase_pm_context_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { - /* This would re-activate the GPU. Since it's already idle, -- * there's no need to reset it */ -+ * there's no need to reset it -+ */ - atomic_set(&kbdev->hwaccess.backend.reset_gpu, - KBASE_RESET_GPU_NOT_PENDING); - kbase_disjoint_state_down(kbdev); -@@ -1081,14 +1172,16 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) - kbdev->irq_reset_flush = true; - - /* Disable IRQ to avoid IRQ handlers to kick in after releasing the -- * spinlock; this also clears any outstanding interrupts */ -+ * spinlock; this also clears any outstanding interrupts -+ */ - kbase_pm_disable_interrupts_nolock(kbdev); - - spin_unlock(&kbdev->mmu_mask_change); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* Ensure that any IRQ handlers have finished -- * Must be done without any locks IRQ handlers will take */ -+ * Must be done without any locks IRQ handlers will take -+ */ - kbase_synchronize_irqs(kbdev); - - /* Flush out any in-flight work items */ -@@ -1099,7 +1192,8 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) - - if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TMIX_8463)) { - /* Ensure that L2 is not transitioning when we send the reset -- * command */ -+ * command -+ */ - while (--max_loops && kbase_pm_get_trans_cores(kbdev, - KBASE_PM_CORE_L2)) - ; -@@ -1114,14 +1208,16 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) - /* All slot have been soft-stopped and we've waited - * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we - * assume that anything that is still left on the GPU is stuck there and -- * we'll kill it when we reset the GPU */ -+ * we'll kill it when we reset the GPU -+ */ - - if (!silent) - dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", - RESET_TIMEOUT); - - /* Output the state of some interesting registers to help in the -- * debugging of GPU resets */ -+ * debugging of GPU resets -+ */ - if (!silent) - kbase_debug_dump_registers(kbdev); - -@@ -1160,7 +1256,8 @@ static void kbasep_reset_timeout_worker(struct work_struct *data) - kbase_pm_update_cores_state(kbdev); - - /* Synchronously request and wait for those cores, because if -- * instrumentation is enabled it would need them immediately. */ -+ * instrumentation is enabled it would need them immediately. -+ */ - kbase_pm_wait_for_desired_state(kbdev); - - mutex_unlock(&kbdev->pm.lock); -@@ -1237,7 +1334,8 @@ static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev) - - /* Check that the reset has been committed to (i.e. kbase_reset_gpu has - * been called), and that no other thread beat this thread to starting -- * the reset */ -+ * the reset -+ */ - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, - KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != - KBASE_RESET_GPU_COMMITTED) { -@@ -1261,6 +1359,7 @@ static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) - /** - * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU - * @kbdev: kbase device -+ * @flags: Bitfield indicating impact of reset (see flag defines) - * - * This function just soft-stops all the slots to ensure that as many jobs as - * possible are saved. -@@ -1271,12 +1370,23 @@ static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev) - * false - Another thread is performing a reset, kbase_reset_gpu should - * not be called. - */ --bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) -+bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev, -+ unsigned int flags) - { - int i; - -+ CSTD_UNUSED(flags); - KBASE_DEBUG_ASSERT(kbdev); - -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ if (kbase_pm_is_gpu_lost(kbdev)) { -+ /* GPU access has been removed, reset will be done by -+ * Arbiter instead -+ */ -+ return false; -+ } -+#endif -+ - if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu, - KBASE_RESET_GPU_NOT_PENDING, - KBASE_RESET_GPU_PREPARED) != -@@ -1293,14 +1403,14 @@ bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev) - return true; - } - --bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev) -+bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev, unsigned int flags) - { -- unsigned long flags; -+ unsigned long lock_flags; - bool ret; - -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- ret = kbase_prepare_to_reset_gpu_locked(kbdev); -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, lock_flags); -+ ret = kbase_prepare_to_reset_gpu_locked(kbdev, flags); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, lock_flags); - - return ret; - } -@@ -1321,7 +1431,8 @@ void kbase_reset_gpu(struct kbase_device *kbdev) - KBASE_DEBUG_ASSERT(kbdev); - - /* Note this is an assert/atomic_set because it is a software issue for -- * a race to be occuring here */ -+ * a race to be occurring here -+ */ - KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == - KBASE_RESET_GPU_PREPARED); - atomic_set(&kbdev->hwaccess.backend.reset_gpu, -@@ -1344,7 +1455,8 @@ void kbase_reset_gpu_locked(struct kbase_device *kbdev) - KBASE_DEBUG_ASSERT(kbdev); - - /* Note this is an assert/atomic_set because it is a software issue for -- * a race to be occuring here */ -+ * a race to be occurring here -+ */ - KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) == - KBASE_RESET_GPU_PREPARED); - atomic_set(&kbdev->hwaccess.backend.reset_gpu, -@@ -1419,3 +1531,21 @@ void kbase_reset_gpu_term(struct kbase_device *kbdev) - { - destroy_workqueue(kbdev->hwaccess.backend.reset_workq); - } -+ -+static u64 kbasep_apply_limited_core_mask(const struct kbase_device *kbdev, -+ const u64 affinity, const u64 limited_core_mask) -+{ -+ const u64 result = affinity & limited_core_mask; -+ -+#ifdef CONFIG_MALI_DEBUG -+ dev_dbg(kbdev->dev, -+ "Limiting affinity due to BASE_JD_REQ_LIMITED_CORE_MASK from 0x%lx to 0x%lx (mask is 0x%lx)\n", -+ (unsigned long int)affinity, -+ (unsigned long int)result, -+ (unsigned long int)limited_core_mask); -+#else -+ CSTD_UNUSED(kbdev); -+#endif -+ -+ return result; -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h -index 1419b59..1039e85 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_internal.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Job Manager backend-specific low-level APIs. - */ -@@ -34,7 +31,7 @@ - #include <linux/atomic.h> - - #include <backend/gpu/mali_kbase_jm_rb.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - - /** - * kbase_job_submit_nolock() - Submit a job to a certain job-slot -@@ -62,7 +59,7 @@ void kbase_job_submit_nolock(struct kbase_device *kbdev, - void kbase_job_done_slot(struct kbase_device *kbdev, int s, u32 completion_code, - u64 job_tail, ktime_t *end_timestamp); - --#ifdef CONFIG_GPU_TRACEPOINTS -+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) - static inline char *kbasep_make_job_slot_string(int js, char *js_string, - size_t js_size) - { -@@ -71,11 +68,13 @@ static inline char *kbasep_make_job_slot_string(int js, char *js_string, - } - #endif - -+#if !MALI_USE_CSF - static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js, - struct kbase_context *kctx) - { - return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT)); - } -+#endif - - - /** -@@ -94,6 +93,7 @@ void kbase_job_hw_submit(struct kbase_device *kbdev, - struct kbase_jd_atom *katom, - int js); - -+#if !MALI_USE_CSF - /** - * kbasep_job_slot_soft_or_hard_stop_do_action() - Perform a soft or hard stop - * on the specified atom -@@ -112,6 +112,7 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev, - u32 action, - base_jd_core_req core_reqs, - struct kbase_jd_atom *target_katom); -+#endif /* !MALI_USE_CSF */ - - /** - * kbase_backend_soft_hard_stop_slot() - Soft or hard stop jobs on a given job -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c -index 4e4ed05..8ee897f 100755 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * Register-based HW access backend specific APIs - */ -@@ -33,16 +31,19 @@ - #include <tl/mali_kbase_tracepoints.h> - #include <mali_kbase_hwcnt_context.h> - #include <mali_kbase_reset_gpu.h> -+#include <mali_kbase_kinstr_jm.h> - #include <backend/gpu/mali_kbase_cache_policy_backend.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include <backend/gpu/mali_kbase_jm_internal.h> - #include <backend/gpu/mali_kbase_pm_internal.h> - - /* Return whether the specified ringbuffer is empty. HW access lock must be -- * held */ -+ * held -+ */ - #define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx) - /* Return number of atoms currently in the specified ringbuffer. HW access lock -- * must be held */ -+ * must be held -+ */ - #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx) - - static void kbase_gpu_release_atom(struct kbase_device *kbdev, -@@ -253,6 +254,8 @@ static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev, - - int kbase_backend_slot_free(struct kbase_device *kbdev, int js) - { -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ - if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) != - KBASE_RESET_GPU_NOT_PENDING) { - /* The GPU is being reset - so prevent submission */ -@@ -278,15 +281,19 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev, - break; - - case KBASE_ATOM_GPU_RB_SUBMITTED: -+ kbase_kinstr_jm_atom_hw_release(katom); - /* Inform power management at start/finish of atom so it can - * update its GPU utilisation metrics. Mark atom as not -- * submitted beforehand. */ -+ * submitted beforehand. -+ */ - katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY; - kbase_pm_metrics_update(kbdev, end_timestamp); - -+ /* Inform platform at start/finish of atom */ -+ kbasep_platform_event_atom_complete(katom); -+ - if (katom->core_req & BASE_JD_REQ_PERMON) - kbase_pm_release_gpu_cycle_counter_nolock(kbdev); -- /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - - KBASE_TLSTREAM_TL_NRET_ATOM_LPU(kbdev, katom, - &kbdev->gpu_props.props.raw_props.js_features -@@ -296,6 +303,8 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev, - &kbdev->gpu_props.props.raw_props.js_features - [katom->slot_nr]); - -+ /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ -+ - case KBASE_ATOM_GPU_RB_READY: - /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */ - -@@ -540,7 +549,8 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, - KBASE_TLSTREAM_AUX_PROTECTED_ENTER_START(kbdev, kbdev); - /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV - * should ensure that we are not already transitiong, and that -- * there are no atoms currently on the GPU. */ -+ * there are no atoms currently on the GPU. -+ */ - WARN_ON(kbdev->protected_mode_transition); - WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); - /* If hwcnt is disabled, it means we didn't clean up correctly -@@ -566,19 +576,15 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, - - /* We couldn't disable atomically, so kick off a worker */ - if (!kbdev->protected_mode_hwcnt_disabled) { --#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE -- queue_work(system_wq, -- &kbdev->protected_mode_hwcnt_disable_work); --#else -- queue_work(system_highpri_wq, -+ kbase_hwcnt_context_queue_work( -+ kbdev->hwcnt_gpu_ctx, - &kbdev->protected_mode_hwcnt_disable_work); --#endif - return -EAGAIN; - } - -- /* Once reaching this point GPU must be -- * switched to protected mode or hwcnt -- * re-enabled. */ -+ /* Once reaching this point GPU must be switched to protected -+ * mode or hwcnt re-enabled. -+ */ - - if (kbase_pm_protected_entry_override_enable(kbdev)) - return -EAGAIN; -@@ -618,7 +624,7 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev, - KBASE_PM_CORE_L2) || - kbase_pm_get_trans_cores(kbdev, - KBASE_PM_CORE_L2) || -- kbase_is_gpu_lost(kbdev)) { -+ kbase_is_gpu_removed(kbdev)) { - /* - * The L2 is still powered, wait for all - * the users to finish with it before doing -@@ -718,7 +724,8 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, - KBASE_TLSTREAM_AUX_PROTECTED_LEAVE_START(kbdev, kbdev); - /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV - * should ensure that we are not already transitiong, and that -- * there are no atoms currently on the GPU. */ -+ * there are no atoms currently on the GPU. -+ */ - WARN_ON(kbdev->protected_mode_transition); - WARN_ON(kbase_gpu_atoms_submitted_any(kbdev)); - -@@ -764,8 +771,8 @@ static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev, - katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID; - kbase_gpu_mark_atom_for_return(kbdev, katom[idx]); - /* Only return if head atom or previous atom -- * already removed - as atoms must be returned -- * in order */ -+ * already removed - as atoms must be returned in order -+ */ - if (idx == 0 || katom[0]->gpu_rb_state == - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); -@@ -811,7 +818,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) - lockdep_assert_held(&kbdev->hwaccess_lock); - - #ifdef CONFIG_MALI_ARBITER_SUPPORT -- if (kbase_reset_gpu_is_active(kbdev) || kbase_is_gpu_lost(kbdev)) -+ if (kbase_reset_gpu_is_active(kbdev) || -+ kbase_is_gpu_removed(kbdev)) - #else - if (kbase_reset_gpu_is_active(kbdev)) - #endif -@@ -843,7 +851,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) - break; - - katom[idx]->gpu_rb_state = -- KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; -+ KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV; - - /* ***TRANSITION TO HIGHER STATE*** */ - /* fallthrough */ -@@ -907,12 +915,14 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) - kbase_gpu_mark_atom_for_return(kbdev, - katom[idx]); - /* Set EVENT_DONE so this atom will be -- completed, not unpulled. */ -+ * completed, not unpulled. -+ */ - katom[idx]->event_code = - BASE_JD_EVENT_DONE; - /* Only return if head atom or previous - * atom already removed - as atoms must -- * be returned in order. */ -+ * be returned in order. -+ */ - if (idx == 0 || katom[0]->gpu_rb_state == - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); -@@ -943,7 +953,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) - - if (idx == 1) { - /* Only submit if head atom or previous -- * atom already submitted */ -+ * atom already submitted -+ */ - if ((katom[0]->gpu_rb_state != - KBASE_ATOM_GPU_RB_SUBMITTED && - katom[0]->gpu_rb_state != -@@ -959,7 +970,8 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) - } - - /* If inter-slot serialization in use then don't -- * submit atom if any other slots are in use */ -+ * submit atom if any other slots are in use -+ */ - if ((kbdev->serialize_jobs & - KBASE_SERIALIZE_INTER_SLOT) && - other_slots_busy(kbdev, js)) -@@ -971,31 +983,37 @@ void kbase_backend_slot_update(struct kbase_device *kbdev) - break; - #endif - /* Check if this job needs the cycle counter -- * enabled before submission */ -+ * enabled before submission -+ */ - if (katom[idx]->core_req & BASE_JD_REQ_PERMON) - kbase_pm_request_gpu_cycle_counter_l2_is_on( - kbdev); - - kbase_job_hw_submit(kbdev, katom[idx], js); - katom[idx]->gpu_rb_state = -- KBASE_ATOM_GPU_RB_SUBMITTED; -+ KBASE_ATOM_GPU_RB_SUBMITTED; -+ -+ /* ***TRANSITION TO HIGHER STATE*** */ -+ /* fallthrough */ -+ case KBASE_ATOM_GPU_RB_SUBMITTED: - - /* Inform power management at start/finish of - * atom so it can update its GPU utilisation -- * metrics. */ -+ * metrics. -+ */ - kbase_pm_metrics_update(kbdev, - &katom[idx]->start_timestamp); - -- /* ***TRANSITION TO HIGHER STATE*** */ -- /* fallthrough */ -- case KBASE_ATOM_GPU_RB_SUBMITTED: -- /* Atom submitted to HW, nothing else to do */ -+ /* Inform platform at start/finish of atom */ -+ kbasep_platform_event_atom_submit(katom[idx]); -+ - break; - - case KBASE_ATOM_GPU_RB_RETURN_TO_JS: - /* Only return if head atom or previous atom - * already removed - as atoms must be returned -- * in order */ -+ * in order -+ */ - if (idx == 0 || katom[0]->gpu_rb_state == - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - kbase_gpu_dequeue_atom(kbdev, js, NULL); -@@ -1013,7 +1031,7 @@ void kbase_backend_run_atom(struct kbase_device *kbdev, - struct kbase_jd_atom *katom) - { - lockdep_assert_held(&kbdev->hwaccess_lock); -- dev_dbg(kbdev->dev, "Backend running atom %p\n", (void *)katom); -+ dev_dbg(kbdev->dev, "Backend running atom %pK\n", (void *)katom); - - kbase_gpu_enqueue_atom(kbdev, katom); - kbase_backend_slot_update(kbdev); -@@ -1074,7 +1092,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, - struct kbase_context *kctx = katom->kctx; - - dev_dbg(kbdev->dev, -- "Atom %p completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n", -+ "Atom %pK completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n", - (void *)katom, completion_code, job_tail, js); - - lockdep_assert_held(&kbdev->hwaccess_lock); -@@ -1098,7 +1116,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, - * BASE_JD_REQ_SKIP_CACHE_END is set, the GPU cache is not - * flushed. To prevent future evictions causing possible memory - * corruption we need to flush the cache manually before any -- * affected memory gets reused. */ -+ * affected memory gets reused. -+ */ - katom->need_cache_flush_cores_retained = true; - } - -@@ -1181,7 +1200,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, - katom_idx1->gpu_rb_state != - KBASE_ATOM_GPU_RB_SUBMITTED) { - /* Can not dequeue this atom yet - will be -- * dequeued when atom at idx0 completes */ -+ * dequeued when atom at idx0 completes -+ */ - katom_idx1->event_code = BASE_JD_EVENT_STOPPED; - kbase_gpu_mark_atom_for_return(kbdev, - katom_idx1); -@@ -1194,7 +1214,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, - if (job_tail != 0 && job_tail != katom->jc) { - /* Some of the job has been executed */ - dev_dbg(kbdev->dev, -- "Update job chain address of atom %p to resume from 0x%llx\n", -+ "Update job chain address of atom %pK to resume from 0x%llx\n", - (void *)katom, job_tail); - - katom->jc = job_tail; -@@ -1214,7 +1234,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, - * - Schedule out the parent context if necessary, and schedule a new - * one in. - */ --#ifdef CONFIG_GPU_TRACEPOINTS -+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) - { - /* The atom in the HEAD */ - struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js, -@@ -1255,7 +1275,7 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, - - if (katom) { - dev_dbg(kbdev->dev, -- "Cross-slot dependency %p has become runnable.\n", -+ "Cross-slot dependency %pK has become runnable.\n", - (void *)katom); - - /* Check if there are lower priority jobs to soft stop */ -@@ -1268,7 +1288,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js, - kbase_pm_update_state(kbdev); - - /* Job completion may have unblocked other atoms. Try to update all job -- * slots */ -+ * slots -+ */ - kbase_backend_slot_update(kbdev); - } - -@@ -1319,7 +1340,8 @@ void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp) - katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK; - /* As the atom was not removed, increment the - * index so that we read the correct atom in the -- * next iteration. */ -+ * next iteration. -+ */ - atom_idx++; - continue; - } -@@ -1422,7 +1444,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, - katom_idx0_valid = (katom_idx0 == katom); - /* If idx0 is to be removed and idx1 is on the same context, - * then idx1 must also be removed otherwise the atoms might be -- * returned out of order */ -+ * returned out of order -+ */ - if (katom_idx1) - katom_idx1_valid = (katom_idx1 == katom) || - (katom_idx0_valid && -@@ -1469,7 +1492,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, - if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, - JS_COMMAND_NEXT)) == 0) { - /* idx0 has already completed - stop -- * idx1 if needed*/ -+ * idx1 if needed -+ */ - if (katom_idx1_valid) { - kbase_gpu_stop_atom(kbdev, js, - katom_idx1, -@@ -1478,7 +1502,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, - } - } else { - /* idx1 is in NEXT registers - attempt -- * to remove */ -+ * to remove -+ */ - kbase_reg_write(kbdev, - JOB_SLOT_REG(js, - JS_COMMAND_NEXT), -@@ -1493,7 +1518,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, - JS_HEAD_NEXT_HI)) - != 0) { - /* idx1 removed successfully, -- * will be handled in IRQ */ -+ * will be handled in IRQ -+ */ - kbase_gpu_remove_atom(kbdev, - katom_idx1, - action, true); -@@ -1507,7 +1533,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, - ret = true; - } else if (katom_idx1_valid) { - /* idx0 has already completed, -- * stop idx1 if needed */ -+ * stop idx1 if needed -+ */ - kbase_gpu_stop_atom(kbdev, js, - katom_idx1, - action); -@@ -1526,7 +1553,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, - * flow was also interrupted, and this function - * might not enter disjoint state e.g. if we - * don't actually do a hard stop on the head -- * atom */ -+ * atom -+ */ - kbase_gpu_stop_atom(kbdev, js, katom_idx0, - action); - ret = true; -@@ -1554,7 +1582,8 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, - ret = true; - } else { - /* idx1 is in NEXT registers - attempt to -- * remove */ -+ * remove -+ */ - kbase_reg_write(kbdev, JOB_SLOT_REG(js, - JS_COMMAND_NEXT), - JS_COMMAND_NOP); -@@ -1564,13 +1593,15 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev, - kbase_reg_read(kbdev, JOB_SLOT_REG(js, - JS_HEAD_NEXT_HI)) != 0) { - /* idx1 removed successfully, will be -- * handled in IRQ once idx0 completes */ -+ * handled in IRQ once idx0 completes -+ */ - kbase_gpu_remove_atom(kbdev, katom_idx1, - action, - false); - } else { - /* idx0 has already completed - stop -- * idx1 */ -+ * idx1 -+ */ - kbase_gpu_stop_atom(kbdev, js, - katom_idx1, - action); -@@ -1644,7 +1675,7 @@ void kbase_gpu_dump_slots(struct kbase_device *kbdev) - - if (katom) - dev_info(kbdev->dev, -- " js%d idx%d : katom=%p gpu_rb_state=%d\n", -+ " js%d idx%d : katom=%pK gpu_rb_state=%d\n", - js, idx, katom, katom->gpu_rb_state); - else - dev_info(kbdev->dev, " js%d idx%d : empty\n", -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h -index c3b9f2d..d3ff203 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * Register-based HW access backend specific APIs - */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c -index fcc0437..02d7cdb 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * Register-based HW access backend specific job scheduler APIs - */ -@@ -31,13 +29,14 @@ - #include <backend/gpu/mali_kbase_jm_internal.h> - #include <backend/gpu/mali_kbase_js_internal.h> - -+#if !MALI_USE_CSF - /* - * Hold the runpool_mutex for this - */ - static inline bool timer_callback_should_run(struct kbase_device *kbdev) - { - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; -- s8 nr_running_ctxs; -+ int nr_running_ctxs; - - lockdep_assert_held(&kbdev->js_data.runpool_mutex); - -@@ -47,7 +46,8 @@ static inline bool timer_callback_should_run(struct kbase_device *kbdev) - - /* nr_contexts_pullable is updated with the runpool_mutex. However, the - * locking in the caller gives us a barrier that ensures -- * nr_contexts_pullable is up-to-date for reading */ -+ * nr_contexts_pullable is up-to-date for reading -+ */ - nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable); - - #ifdef CONFIG_MALI_DEBUG -@@ -69,10 +69,10 @@ static inline bool timer_callback_should_run(struct kbase_device *kbdev) - * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE). - */ - { -- s8 nr_compute_ctxs = -+ int nr_compute_ctxs = - kbasep_js_ctx_attr_count_on_runpool(kbdev, - KBASEP_JS_CTX_ATTR_COMPUTE); -- s8 nr_noncompute_ctxs = nr_running_ctxs - -+ int nr_noncompute_ctxs = nr_running_ctxs - - nr_compute_ctxs; - - return (bool) (nr_compute_ctxs >= 2 || -@@ -113,7 +113,8 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) - - if (atom != NULL) { - /* The current version of the model doesn't support -- * Soft-Stop */ -+ * Soft-Stop -+ */ - if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) { - u32 ticks = atom->ticks++; - -@@ -141,7 +142,8 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) - * new soft_stop timeout. This ensures that - * atoms do not miss any of the timeouts due to - * races between this worker and the thread -- * changing the timeouts. */ -+ * changing the timeouts. -+ */ - if (backend->timeouts_updated && - ticks > soft_stop_ticks) - ticks = atom->ticks = soft_stop_ticks; -@@ -171,10 +173,11 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) - * - * Similarly, if it's about to be - * decreased, the last job from another -- * context has already finished, so it's -- * not too bad that we observe the older -- * value and register a disjoint event -- * when we try soft-stopping */ -+ * context has already finished, so -+ * it's not too bad that we observe the -+ * older value and register a disjoint -+ * event when we try soft-stopping -+ */ - if (js_devdata->nr_user_contexts_running - >= disjoint_threshold) - softstop_flags |= -@@ -252,9 +255,9 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) - } - } - if (reset_needed) { -- dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve."); -+ dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issuing GPU soft-reset to resolve."); - -- if (kbase_prepare_to_reset_gpu_locked(kbdev)) -+ if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_NONE)) - kbase_reset_gpu_locked(kbdev); - } - /* the timer is re-issued if there is contexts in the run-pool */ -@@ -270,9 +273,11 @@ static enum hrtimer_restart timer_callback(struct hrtimer *timer) - - return HRTIMER_NORESTART; - } -+#endif /* !MALI_USE_CSF */ - - void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) - { -+#if !MALI_USE_CSF - struct kbasep_js_device_data *js_devdata = &kbdev->js_data; - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - unsigned long flags; -@@ -284,11 +289,12 @@ void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - backend->timer_running = false; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -- /* From now on, return value of timer_callback_should_run() will -- * also cause the timer to not requeue itself. Its return value -- * cannot change, because it depends on variables updated with -- * the runpool_mutex held, which the caller of this must also -- * hold */ -+ /* From now on, return value of timer_callback_should_run() -+ * will also cause the timer to not requeue itself. Its return -+ * value cannot change, because it depends on variables updated -+ * with the runpool_mutex held, which the caller of this must -+ * also hold -+ */ - hrtimer_cancel(&backend->scheduling_timer); - } - -@@ -303,25 +309,36 @@ void kbase_backend_ctx_count_changed(struct kbase_device *kbdev) - - KBASE_KTRACE_ADD_JM(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u); - } -+#else /* !MALI_USE_CSF */ -+ CSTD_UNUSED(kbdev); -+#endif /* !MALI_USE_CSF */ - } - - int kbase_backend_timer_init(struct kbase_device *kbdev) - { -+#if !MALI_USE_CSF - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC, - HRTIMER_MODE_REL); - backend->scheduling_timer.function = timer_callback; - backend->timer_running = false; -+#else /* !MALI_USE_CSF */ -+ CSTD_UNUSED(kbdev); -+#endif /* !MALI_USE_CSF */ - - return 0; - } - - void kbase_backend_timer_term(struct kbase_device *kbdev) - { -+#if !MALI_USE_CSF - struct kbase_backend_data *backend = &kbdev->hwaccess.backend; - - hrtimer_cancel(&backend->scheduling_timer); -+#else /* !MALI_USE_CSF */ -+ CSTD_UNUSED(kbdev); -+#endif /* !MALI_USE_CSF */ - } - - void kbase_backend_timer_suspend(struct kbase_device *kbdev) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h -index 6576e55..5284288 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_internal.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2015, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * Register-based HW access backend specific job scheduler APIs - */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_l2_mmu_config.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_l2_mmu_config.c -index e67d12b..7131546 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_l2_mmu_config.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_l2_mmu_config.c -@@ -1,12 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -17,14 +17,12 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> - #include <mali_kbase_bits.h> - #include <mali_kbase_config_defaults.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include "mali_kbase_l2_mmu_config.h" - - /** -@@ -56,23 +54,34 @@ struct l2_mmu_config_limit { - /* - * Zero represents no limit - * -- * For LBEX TBEX TTRX and TNAX: -+ * For LBEX TBEX TBAX TTRX and TNAX: - * The value represents the number of outstanding reads (6 bits) or writes (5 bits) - * - * For all other GPUS it is a fraction see: mali_kbase_config_defaults.h - */ - static const struct l2_mmu_config_limit limits[] = { -- /* GPU read write */ -- {GPU_ID2_PRODUCT_LBEX, {0, GENMASK(10, 5), 5}, {0, GENMASK(16, 12), 12} }, -- {GPU_ID2_PRODUCT_TBEX, {0, GENMASK(10, 5), 5}, {0, GENMASK(16, 12), 12} }, -- {GPU_ID2_PRODUCT_TTRX, {0, GENMASK(12, 7), 7}, {0, GENMASK(17, 13), 13} }, -- {GPU_ID2_PRODUCT_TNAX, {0, GENMASK(12, 7), 7}, {0, GENMASK(17, 13), 13} }, -- {GPU_ID2_PRODUCT_TGOX, -- {KBASE_3BIT_AID_32, GENMASK(14, 12), 12}, -- {KBASE_3BIT_AID_32, GENMASK(17, 15), 15} }, -- {GPU_ID2_PRODUCT_TNOX, -- {KBASE_3BIT_AID_32, GENMASK(14, 12), 12}, -- {KBASE_3BIT_AID_32, GENMASK(17, 15), 15} }, -+ /* GPU, read, write */ -+ {GPU_ID2_PRODUCT_LBEX, -+ {0, GENMASK(10, 5), 5}, -+ {0, GENMASK(16, 12), 12} }, -+ {GPU_ID2_PRODUCT_TBEX, -+ {0, GENMASK(10, 5), 5}, -+ {0, GENMASK(16, 12), 12} }, -+ {GPU_ID2_PRODUCT_TBAX, -+ {0, GENMASK(10, 5), 5}, -+ {0, GENMASK(16, 12), 12} }, -+ {GPU_ID2_PRODUCT_TTRX, -+ {0, GENMASK(12, 7), 7}, -+ {0, GENMASK(17, 13), 13} }, -+ {GPU_ID2_PRODUCT_TNAX, -+ {0, GENMASK(12, 7), 7}, -+ {0, GENMASK(17, 13), 13} }, -+ {GPU_ID2_PRODUCT_TGOX, -+ {KBASE_3BIT_AID_32, GENMASK(14, 12), 12}, -+ {KBASE_3BIT_AID_32, GENMASK(17, 15), 15} }, -+ {GPU_ID2_PRODUCT_TNOX, -+ {KBASE_3BIT_AID_32, GENMASK(14, 12), 12}, -+ {KBASE_3BIT_AID_32, GENMASK(17, 15), 15} }, - }; - - int kbase_set_mmu_quirks(struct kbase_device *kbdev) -@@ -100,7 +109,7 @@ int kbase_set_mmu_quirks(struct kbase_device *kbdev) - - mmu_config = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG)); - -- if (kbase_is_gpu_lost(kbdev)) -+ if (kbase_is_gpu_removed(kbdev)) - return -EIO; - - mmu_config &= ~(limit.read.mask | limit.write.mask); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_l2_mmu_config.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_l2_mmu_config.h -index 0c779ac..07014ad 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_l2_mmu_config.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_l2_mmu_config.h -@@ -1,31 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ --/* -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c -index e33fe0b..077c234 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2015, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * "Always on" power management policy - */ -@@ -61,7 +58,11 @@ const struct kbase_pm_policy kbase_pm_always_on_policy_ops = { - always_on_term, /* term */ - always_on_shaders_needed, /* shaders_needed */ - always_on_get_core_active, /* get_core_active */ -+ NULL, /* handle_event */ - KBASE_PM_POLICY_ID_ALWAYS_ON, /* id */ -+#if MALI_USE_CSF -+ ALWAYS_ON_PM_SCHED_FLAGS, /* pm_sched_flags */ -+#endif - }; - - KBASE_EXPORT_TEST_API(kbase_pm_always_on_policy_ops); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h -index e7927cf..98d35da 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_always_on.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2015,2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2015, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * "Always on" power management policy - */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c -index 576c9f2..cc791df 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c -@@ -1,11 +1,12 @@ -- /* -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * GPU backend implementation of base kernel power management APIs - */ -@@ -30,14 +28,16 @@ - #include <mali_kbase_config_defaults.h> - - #include <mali_kbase_pm.h> -+#if !MALI_USE_CSF - #include <mali_kbase_hwaccess_jm.h> - #include <backend/gpu/mali_kbase_js_internal.h> - #include <backend/gpu/mali_kbase_jm_internal.h> -+#endif /* !MALI_USE_CSF */ - #include <mali_kbase_hwcnt_context.h> - #include <backend/gpu/mali_kbase_pm_internal.h> - #include <backend/gpu/mali_kbase_devfreq.h> - #include <mali_kbase_dummy_job_wa.h> --#include <mali_kbase_irq_internal.h> -+#include <backend/gpu/mali_kbase_irq_internal.h> - - static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data); - static void kbase_pm_hwcnt_disable_worker(struct work_struct *data); -@@ -106,6 +106,11 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev) - if (callbacks) - callbacks->power_on_callback(kbdev); - -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) -+ dev_err(kbdev->dev, "Attempting to power on while GPU lost\n"); -+#endif -+ - kbdev->pm.backend.gpu_powered = true; - } - -@@ -139,24 +144,35 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev) - - kbdev->pm.backend.ca_cores_enabled = ~0ull; - kbdev->pm.backend.gpu_powered = false; -+ kbdev->pm.backend.gpu_ready = false; - kbdev->pm.suspending = false; - #ifdef CONFIG_MALI_ARBITER_SUPPORT -- kbdev->pm.gpu_lost = false; -+ kbase_pm_set_gpu_lost(kbdev, false); - #endif - #ifdef CONFIG_MALI_DEBUG - kbdev->pm.backend.driver_ready_for_irqs = false; - #endif /* CONFIG_MALI_DEBUG */ - init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait); - -+#if !MALI_USE_CSF - /* Initialise the metrics subsystem */ - ret = kbasep_pm_metrics_init(kbdev); - if (ret) - return ret; -+#else -+ mutex_init(&kbdev->pm.backend.policy_change_lock); -+ kbdev->pm.backend.policy_change_clamp_state_to_off = false; -+ /* Due to dependency on kbase_ipa_control, the metrics subsystem can't -+ * be initialized here. -+ */ -+ CSTD_UNUSED(ret); -+#endif - - init_waitqueue_head(&kbdev->pm.backend.reset_done_wait); - kbdev->pm.backend.reset_done = false; - - init_waitqueue_head(&kbdev->pm.zero_active_count_wait); -+ init_waitqueue_head(&kbdev->pm.resume_wait); - kbdev->pm.active_count = 0; - - spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock); -@@ -213,7 +229,9 @@ pm_state_machine_fail: - kbase_pm_policy_term(kbdev); - kbase_pm_ca_term(kbdev); - workq_fail: -+#if !MALI_USE_CSF - kbasep_pm_metrics_term(kbdev); -+#endif - return -EINVAL; - } - -@@ -222,7 +240,8 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) - lockdep_assert_held(&kbdev->pm.lock); - - /* Turn clocks and interrupts on - no-op if we haven't done a previous -- * kbase_pm_clock_off() */ -+ * kbase_pm_clock_off() -+ */ - kbase_pm_clock_on(kbdev, is_resume); - - if (!is_resume) { -@@ -240,7 +259,8 @@ void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume) - kbase_pm_update_cores_state(kbdev); - - /* NOTE: We don't wait to reach the desired state, since running atoms -- * will wait for that state to be reached anyway */ -+ * will wait for that state to be reached anyway -+ */ - } - - static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) -@@ -251,13 +271,20 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) - struct kbase_pm_backend_data *backend = &pm->backend; - unsigned long flags; - -+#if !MALI_USE_CSF - /* Wait for power transitions to complete. We do this with no locks held - * so that we don't deadlock with any pending workqueues. - */ - kbase_pm_wait_for_desired_state(kbdev); -+#endif - - kbase_pm_lock(kbdev); - -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ if (kbase_pm_is_gpu_lost(kbdev)) -+ backend->poweron_required = false; -+#endif -+ - if (!backend->poweron_required) { - unsigned long flags; - -@@ -278,6 +305,14 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) - kbase_flush_mmu_wqs(kbdev); - kbase_pm_lock(kbdev); - -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ /* poweron_required may have changed while pm lock -+ * was released. -+ */ -+ if (kbase_pm_is_gpu_lost(kbdev)) -+ backend->poweron_required = false; -+#endif -+ - /* Turn off clock now that fault have been handled. We - * dropped locks so poweron_required may have changed - - * power back on if this is the case (effectively only -@@ -296,9 +331,14 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data) - if (backend->poweron_required) { - backend->poweron_required = false; - kbdev->pm.backend.l2_desired = true; -+#if MALI_USE_CSF -+ kbdev->pm.backend.mcu_desired = true; -+#endif - kbase_pm_update_state(kbdev); - kbase_pm_update_cores_state_nolock(kbdev); -+#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -+#endif /* !MALI_USE_CSF */ - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -@@ -451,12 +491,22 @@ static void kbase_pm_hwcnt_disable_worker(struct work_struct *data) - */ - backend->hwcnt_disabled = true; - kbase_pm_update_state(kbdev); -+#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -+#endif /* !MALI_USE_CSF */ - } else { - /* PM state was updated while we were doing the disable, - * so we need to undo the disable we just performed. - */ -+#if MALI_USE_CSF -+ unsigned long lock_flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &lock_flags); -+#endif - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -+#if MALI_USE_CSF -+ kbase_csf_scheduler_spin_unlock(kbdev, lock_flags); -+#endif - } - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -@@ -476,8 +526,12 @@ void kbase_pm_do_poweroff(struct kbase_device *kbdev) - if (kbdev->pm.backend.poweroff_wait_in_progress) - goto unlock_hwaccess; - -+#if MALI_USE_CSF -+ kbdev->pm.backend.mcu_desired = false; -+#else - /* Force all cores off */ - kbdev->pm.backend.shaders_desired = false; -+#endif - kbdev->pm.backend.l2_desired = false; - - kbdev->pm.backend.poweroff_wait_in_progress = true; -@@ -528,20 +582,35 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, - KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev)); - - /* Power up the GPU, don't enable IRQs as we are not ready to receive -- * them. */ -+ * them -+ */ - ret = kbase_pm_init_hw(kbdev, flags); - if (ret) { - kbase_pm_unlock(kbdev); - return ret; - } -- -+#if MALI_USE_CSF -+ kbdev->pm.debug_core_mask = -+ kbdev->gpu_props.props.raw_props.shader_present; -+ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -+ /* Set the initial value for 'shaders_avail'. It would be later -+ * modified only from the MCU state machine, when the shader core -+ * allocation enable mask request has completed. So its value would -+ * indicate the mask of cores that are currently being used by FW for -+ * the allocation of endpoints requested by CSGs. -+ */ -+ kbdev->pm.backend.shaders_avail = kbase_pm_ca_get_core_mask(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -+#else - kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] = - kbdev->pm.debug_core_mask[1] = - kbdev->pm.debug_core_mask[2] = - kbdev->gpu_props.props.raw_props.shader_present; -+#endif - - /* Pretend the GPU is active to prevent a power policy turning the GPU -- * cores off */ -+ * cores off -+ */ - kbdev->pm.active_count = 1; - - spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, -@@ -553,13 +622,27 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, - irq_flags); - - /* We are ready to receive IRQ's now as power policy is set up, so -- * enable them now. */ -+ * enable them now. -+ */ - #ifdef CONFIG_MALI_DEBUG - kbdev->pm.backend.driver_ready_for_irqs = true; - #endif - kbase_pm_enable_interrupts(kbdev); - -+ WARN_ON(!kbdev->pm.backend.gpu_powered); -+ /* GPU has been powered up (by kbase_pm_init_hw) and interrupts have -+ * been enabled, so GPU is ready for use and PM state machine can be -+ * exercised from this point onwards. -+ */ -+ kbdev->pm.backend.gpu_ready = true; -+ - /* Turn on the GPU and any cores needed by the policy */ -+#if MALI_USE_CSF -+ /* Turn on the L2 caches, needed for firmware boot */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -+ kbdev->pm.backend.l2_desired = true; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -+#endif - kbase_pm_do_poweron(kbdev, false); - kbase_pm_unlock(kbdev); - -@@ -573,6 +656,8 @@ void kbase_hwaccess_pm_halt(struct kbase_device *kbdev) - mutex_lock(&kbdev->pm.lock); - kbase_pm_do_poweroff(kbdev); - mutex_unlock(&kbdev->pm.lock); -+ -+ kbase_pm_wait_for_poweroff_complete(kbdev); - } - - KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt); -@@ -587,10 +672,15 @@ void kbase_hwaccess_pm_term(struct kbase_device *kbdev) - - if (kbdev->pm.backend.hwcnt_disabled) { - unsigned long flags; -- -+#if MALI_USE_CSF -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+#else - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+#endif - } - - /* Free any resources the policy allocated */ -@@ -598,8 +688,16 @@ void kbase_hwaccess_pm_term(struct kbase_device *kbdev) - kbase_pm_policy_term(kbdev); - kbase_pm_ca_term(kbdev); - -+#if !MALI_USE_CSF - /* Shut down the metrics subsystem */ - kbasep_pm_metrics_term(kbdev); -+#else -+ if (WARN_ON(mutex_is_locked(&kbdev->pm.backend.policy_change_lock))) { -+ mutex_lock(&kbdev->pm.backend.policy_change_lock); -+ mutex_unlock(&kbdev->pm.backend.policy_change_lock); -+ } -+ mutex_destroy(&kbdev->pm.backend.policy_change_lock); -+#endif - - destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq); - } -@@ -611,11 +709,24 @@ void kbase_pm_power_changed(struct kbase_device *kbdev) - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_pm_update_state(kbdev); - -+#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -+#endif /* !MALI_USE_CSF */ - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } - -+#if MALI_USE_CSF -+void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ kbdev->pm.debug_core_mask = new_core_mask; -+ kbase_pm_update_dynamic_cores_onoff(kbdev); -+} -+KBASE_EXPORT_TEST_API(kbase_pm_set_debug_core_mask); -+#else - void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, - u64 new_core_mask_js0, u64 new_core_mask_js1, - u64 new_core_mask_js2) -@@ -636,6 +747,7 @@ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, - - kbase_pm_update_dynamic_cores_onoff(kbdev); - } -+#endif /* MALI_USE_CSF */ - - void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev) - { -@@ -651,12 +763,15 @@ void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev) - { - /* Force power off the GPU and all cores (regardless of policy), only - * after the PM active count reaches zero (otherwise, we risk turning it -- * off prematurely) */ -+ * off prematurely) -+ */ - kbase_pm_lock(kbdev); - - kbase_pm_do_poweroff(kbdev); - -+#if !MALI_USE_CSF - kbase_backend_timer_suspend(kbdev); -+#endif /* !MALI_USE_CSF */ - - kbase_pm_unlock(kbdev); - -@@ -672,12 +787,19 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) - - kbdev->pm.suspending = false; - #ifdef CONFIG_MALI_ARBITER_SUPPORT -- kbdev->pm.gpu_lost = false; -+ if (kbase_pm_is_gpu_lost(kbdev)) { -+ dev_dbg(kbdev->dev, "%s: GPU lost in progress\n", __func__); -+ kbase_pm_unlock(kbdev); -+ return; -+ } - #endif - kbase_pm_do_poweron(kbdev, true); - -+#if !MALI_USE_CSF - kbase_backend_timer_resume(kbdev); -+#endif /* !MALI_USE_CSF */ - -+ wake_up_all(&kbdev->pm.resume_wait); - kbase_pm_unlock(kbdev); - } - -@@ -685,63 +807,50 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev) - void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev) - { - unsigned long flags; -- struct kbase_pm_backend_data *backend = &kbdev->pm.backend; - ktime_t end_timestamp = ktime_get(); -+ struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state; - -- /* Full GPU reset will have been done by hypervisor, so cancel */ -- atomic_set(&kbdev->hwaccess.backend.reset_gpu, -- KBASE_RESET_GPU_NOT_PENDING); -- hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); -- -- /* GPU is no longer mapped to VM. So no interrupts will be received -- * and Mali registers have been replaced by dummy RAM -- */ -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- spin_lock(&kbdev->mmu_mask_change); -- kbdev->irq_reset_flush = true; -- spin_unlock(&kbdev->mmu_mask_change); -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -- kbase_synchronize_irqs(kbdev); -- kbase_flush_mmu_wqs(kbdev); -- kbdev->irq_reset_flush = false; -+ if (!kbdev->arb.arb_if) -+ return; - -- /* Clear all jobs running on the GPU */ - mutex_lock(&kbdev->pm.lock); -- kbdev->pm.gpu_lost = true; -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- kbdev->protected_mode = false; -- if (!kbdev->pm.backend.protected_entry_transition_override) -- kbase_backend_reset(kbdev, &end_timestamp); -- kbase_pm_metrics_update(kbdev, NULL); -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_lock(&arb_vm_state->vm_state_lock); -+ if (kbdev->pm.backend.gpu_powered && -+ !kbase_pm_is_gpu_lost(kbdev)) { -+ kbase_pm_set_gpu_lost(kbdev, true); -+ -+ /* GPU is no longer mapped to VM. So no interrupts will -+ * be received and Mali registers have been replaced by -+ * dummy RAM -+ */ -+ WARN(!kbase_is_gpu_removed(kbdev), -+ "GPU is still available after GPU lost event\n"); - -- /* Cancel any pending HWC dumps */ -- spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -- kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; -- kbdev->hwcnt.backend.triggered = 1; -- wake_up(&kbdev->hwcnt.backend.wait); -- spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ /* Full GPU reset will have been done by hypervisor, so -+ * cancel -+ */ -+ atomic_set(&kbdev->hwaccess.backend.reset_gpu, -+ KBASE_RESET_GPU_NOT_PENDING); -+ hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer); -+ kbase_synchronize_irqs(kbdev); - -- /* Wait for all threads keeping GPU active to complete */ -- mutex_unlock(&kbdev->pm.lock); -- wait_event(kbdev->pm.zero_active_count_wait, -- kbdev->pm.active_count == 0); -- mutex_lock(&kbdev->pm.lock); -+ /* Clear all jobs running on the GPU */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->protected_mode = false; -+ kbase_backend_reset(kbdev, &end_timestamp); -+ kbase_pm_metrics_update(kbdev, NULL); -+ kbase_pm_update_state(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -- /* Update state to GPU off */ -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- kbdev->pm.backend.shaders_desired = false; -- kbdev->pm.backend.l2_desired = false; -- backend->l2_state = KBASE_L2_OFF; -- backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF; -- kbdev->pm.backend.gpu_powered = false; -- backend->poweroff_wait_in_progress = false; -- KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0); -- wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait); -- kbase_gpu_cache_clean_wait_complete(kbdev); -- backend->poweroff_wait_in_progress = false; -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -- wake_up(&kbdev->pm.backend.poweroff_wait); -+ /* Cancel any pending HWC dumps */ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); -+ kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE; -+ kbdev->hwcnt.backend.triggered = 1; -+ wake_up(&kbdev->hwcnt.backend.wait); -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); -+ } -+ mutex_unlock(&arb_vm_state->vm_state_lock); - mutex_unlock(&kbdev->pm.lock); - } -+ - #endif /* CONFIG_MALI_ARBITER_SUPPORT */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c -index e7eef26..efc620f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2013-2018, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2013-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -27,9 +26,6 @@ - #include <mali_kbase.h> - #include <mali_kbase_pm.h> - #include <backend/gpu/mali_kbase_pm_internal.h> --#ifdef CONFIG_MALI_NO_MALI --#include <backend/gpu/mali_kbase_model_dummy.h> --#endif - #include <mali_kbase_dummy_job_wa.h> - - int kbase_pm_ca_init(struct kbase_device *kbdev) -@@ -59,6 +55,14 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - -+#if MALI_USE_CSF -+ if (!(core_mask & kbdev->pm.debug_core_mask)) { -+ dev_err(kbdev->dev, -+ "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", -+ core_mask, kbdev->pm.debug_core_mask); -+ goto unlock; -+ } -+#else - if (!(core_mask & kbdev->pm.debug_core_mask_all)) { - dev_err(kbdev->dev, "OPP core mask 0x%llX does not intersect with debug mask 0x%llX\n", - core_mask, kbdev->pm.debug_core_mask_all); -@@ -69,6 +73,7 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask) - dev_err(kbdev->dev, "Dynamic core scaling not supported as dummy job WA is enabled"); - goto unlock; - } -+#endif /* MALI_USE_CSF */ - - pm_backend->ca_cores_enabled = core_mask; - -@@ -80,21 +85,32 @@ unlock: - dev_dbg(kbdev->dev, "Devfreq policy : new core mask=%llX\n", - pm_backend->ca_cores_enabled); - } -+KBASE_EXPORT_TEST_API(kbase_devfreq_set_core_mask); - #endif - - u64 kbase_pm_ca_get_core_mask(struct kbase_device *kbdev) - { --#ifdef CONFIG_MALI_DEVFREQ -- struct kbase_pm_backend_data *pm_backend = &kbdev->pm.backend; -+#if MALI_USE_CSF -+ u64 debug_core_mask = kbdev->pm.debug_core_mask; -+#else -+ u64 debug_core_mask = kbdev->pm.debug_core_mask_all; - #endif - - lockdep_assert_held(&kbdev->hwaccess_lock); - - #ifdef CONFIG_MALI_DEVFREQ -- return pm_backend->ca_cores_enabled & kbdev->pm.debug_core_mask_all; -+ /* -+ * Although in the init we let the pm_backend->ca_cores_enabled to be -+ * the max config (it uses the base_gpu_props), at this function we need -+ * to limit it to be a subgroup of the curr config, otherwise the -+ * shaders state machine on the PM does not evolve. -+ */ -+ return kbdev->gpu_props.curr_config.shader_present & -+ kbdev->pm.backend.ca_cores_enabled & -+ debug_core_mask; - #else -- return kbdev->gpu_props.props.raw_props.shader_present & -- kbdev->pm.debug_core_mask_all; -+ return kbdev->gpu_props.curr_config.shader_present & -+ debug_core_mask; - #endif - } - -@@ -104,8 +120,8 @@ u64 kbase_pm_ca_get_instr_core_mask(struct kbase_device *kbdev) - { - lockdep_assert_held(&kbdev->hwaccess_lock); - --#ifdef CONFIG_MALI_NO_MALI -- return (((1ull) << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1); -+#if MALI_USE_CSF -+ return kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); - #else - return kbdev->pm.backend.pm_shaders_core_mask; - #endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h -index 5423e96..8d169c3 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h -index f67ec65..41f3c14 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_ca_devfreq.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c -index 9eef44a..f40b753 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * "Coarse Demand" power management policy - */ -@@ -60,7 +57,11 @@ const struct kbase_pm_policy kbase_pm_coarse_demand_policy_ops = { - coarse_demand_term, /* term */ - coarse_demand_shaders_needed, /* shaders_needed */ - coarse_demand_get_core_active, /* get_core_active */ -+ NULL, /* handle_event */ - KBASE_PM_POLICY_ID_COARSE_DEMAND, /* id */ -+#if MALI_USE_CSF -+ COARSE_ON_DEMAND_PM_SCHED_FLAGS, /* pm_sched_flags */ -+#endif - }; - - KBASE_EXPORT_TEST_API(kbase_pm_coarse_demand_policy_ops); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h -index 304e5d7..5e3f17e 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_coarse_demand.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2015,2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2015, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * "Coarse Demand" power management policy - */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h -index f4bcf3e..1a0572b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_defs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -29,9 +28,6 @@ - - #include "mali_kbase_pm_always_on.h" - #include "mali_kbase_pm_coarse_demand.h" --#if !MALI_CUSTOMER_RELEASE --#include "mali_kbase_pm_always_on_demand.h" --#endif - - /* Forward definition - see mali_kbase.h */ - struct kbase_device; -@@ -62,24 +58,9 @@ enum kbase_pm_core_type { - KBASE_PM_CORE_STACK = STACK_PRESENT_LO - }; - --/** -+/* - * enum kbase_l2_core_state - The states used for the L2 cache & tiler power - * state machine. -- * -- * @KBASE_L2_OFF: The L2 cache and tiler are off -- * @KBASE_L2_PEND_ON: The L2 cache and tiler are powering on -- * @KBASE_L2_RESTORE_CLOCKS: The GPU clock is restored. Conditionally used. -- * @KBASE_L2_ON_HWCNT_ENABLE: The L2 cache and tiler are on, and hwcnt is being -- * enabled -- * @KBASE_L2_ON: The L2 cache and tiler are on, and hwcnt is enabled -- * @KBASE_L2_ON_HWCNT_DISABLE: The L2 cache and tiler are on, and hwcnt is being -- * disabled -- * @KBASE_L2_SLOW_DOWN_CLOCKS: The GPU clock is set to appropriate or lowest -- * clock. Conditionally used. -- * @KBASE_L2_POWER_DOWN: The L2 cache and tiler are about to be powered off -- * @KBASE_L2_PEND_OFF: The L2 cache and tiler are powering off -- * @KBASE_L2_RESET_WAIT: The GPU is resetting, L2 cache and tiler power state -- * are unknown - */ - enum kbase_l2_core_state { - #define KBASEP_L2_STATE(n) KBASE_L2_ ## n, -@@ -87,45 +68,19 @@ enum kbase_l2_core_state { - #undef KBASEP_L2_STATE - }; - --/** -+#if MALI_USE_CSF -+/* -+ * enum kbase_mcu_state - The states used for the MCU state machine. -+ */ -+enum kbase_mcu_state { -+#define KBASEP_MCU_STATE(n) KBASE_MCU_ ## n, -+#include "mali_kbase_pm_mcu_states.h" -+#undef KBASEP_MCU_STATE -+}; -+#endif -+ -+/* - * enum kbase_shader_core_state - The states used for the shaders' state machine. -- * -- * @KBASE_SHADERS_OFF_CORESTACK_OFF: The shaders and core stacks are off -- * @KBASE_SHADERS_OFF_CORESTACK_PEND_ON: The shaders are off, core stacks have -- * been requested to power on and hwcnt -- * is being disabled -- * @KBASE_SHADERS_PEND_ON_CORESTACK_ON: Core stacks are on, shaders have been -- * requested to power on. Or after doing -- * partial shader on/off, checking whether -- * it's the desired state. -- * @KBASE_SHADERS_ON_CORESTACK_ON: The shaders and core stacks are on, and hwcnt -- * already enabled. -- * @KBASE_SHADERS_ON_CORESTACK_ON_RECHECK: The shaders and core stacks -- * are on, hwcnt disabled, and checks -- * to powering down or re-enabling -- * hwcnt. -- * @KBASE_SHADERS_WAIT_OFF_CORESTACK_ON: The shaders have been requested to -- * power off, but they remain on for the -- * duration of the hysteresis timer -- * @KBASE_SHADERS_WAIT_GPU_IDLE: The shaders partial poweroff needs to reach -- * a state where jobs on the GPU are finished -- * including jobs currently running and in the -- * GPU queue because of GPU2017-861 -- * @KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON: The hysteresis timer has expired -- * @KBASE_SHADERS_L2_FLUSHING_CORESTACK_ON: The core stacks are on and the -- * level 2 cache is being flushed. -- * @KBASE_SHADERS_READY_OFF_CORESTACK_ON: The core stacks are on and the shaders -- * are ready to be powered off. -- * @KBASE_SHADERS_PEND_OFF_CORESTACK_ON: The core stacks are on, and the shaders -- * have been requested to power off -- * @KBASE_SHADERS_OFF_CORESTACK_PEND_OFF: The shaders are off, and the core stacks -- * have been requested to power off -- * @KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF: Shaders and corestacks are -- * off, but the tick timer -- * cancellation is still -- * pending. -- * @KBASE_SHADERS_RESET_WAIT: The GPU is resetting, shader and core stack power -- * states are unknown - */ - enum kbase_shader_core_state { - #define KBASEP_SHADER_STATE(n) KBASE_SHADERS_ ## n, -@@ -137,28 +92,40 @@ enum kbase_shader_core_state { - * struct kbasep_pm_metrics - Metrics data collected for use by the power - * management framework. - * -- * @time_busy: number of ns the GPU was busy executing jobs since the -- * @time_period_start timestamp. -- * @time_idle: number of ns since time_period_start the GPU was not executing -- * jobs since the @time_period_start timestamp. -- * @busy_cl: number of ns the GPU was busy executing CL jobs. Note that -- * if two CL jobs were active for 400ns, this value would be updated -- * with 800. -- * @busy_gl: number of ns the GPU was busy executing GL jobs. Note that -- * if two GL jobs were active for 400ns, this value would be updated -- * with 800. -+ * @time_busy: the amount of time the GPU was busy executing jobs since the -+ * @time_period_start timestamp, in units of 256ns. This also includes -+ * time_in_protm, the time spent in protected mode, since it's assumed -+ * the GPU was busy 100% during this period. -+ * @time_idle: the amount of time the GPU was not executing jobs since the -+ * time_period_start timestamp, measured in units of 256ns. -+ * @time_in_protm: The amount of time the GPU has spent in protected mode since -+ * the time_period_start timestamp, measured in units of 256ns. -+ * @busy_cl: the amount of time the GPU was busy executing CL jobs. Note that -+ * if two CL jobs were active for 256ns, this value would be updated -+ * with 2 (2x256ns). -+ * @busy_gl: the amount of time the GPU was busy executing GL jobs. Note that -+ * if two GL jobs were active for 256ns, this value would be updated -+ * with 2 (2x256ns). - */ - struct kbasep_pm_metrics { - u32 time_busy; - u32 time_idle; -+#if MALI_USE_CSF -+ u32 time_in_protm; -+#else - u32 busy_cl[2]; - u32 busy_gl; -+#endif - }; - - /** - * struct kbasep_pm_metrics_state - State required to collect the metrics in - * struct kbasep_pm_metrics - * @time_period_start: time at which busy/idle measurements started -+ * @ipa_control_client: Handle returned on registering DVFS as a -+ * kbase_ipa_control client -+ * @skip_gpu_active_sanity_check: Decide whether to skip GPU_ACTIVE sanity -+ * check in DVFS utilisation calculation - * @gpu_active: true when the GPU is executing jobs. false when - * not. Updated when the job scheduler informs us a job in submitted - * or removed from a GPU slot. -@@ -170,6 +137,7 @@ struct kbasep_pm_metrics { - * @values: The current values of the power management metrics. The - * kbase_pm_get_dvfs_metrics() function is used to compare these - * current values with the saved values from a previous invocation. -+ * @initialized: tracks whether metrics_state has been initialized or not. - * @timer: timer to regularly make DVFS decisions based on the power - * management metrics. - * @timer_active: boolean indicating @timer is running -@@ -178,9 +146,14 @@ struct kbasep_pm_metrics { - */ - struct kbasep_pm_metrics_state { - ktime_t time_period_start; -+#if MALI_USE_CSF -+ void *ipa_control_client; -+ bool skip_gpu_active_sanity_check; -+#else - bool gpu_active; - u32 active_cl_ctx[2]; - u32 active_gl_ctx[3]; -+#endif - spinlock_t lock; - - void *platform_data; -@@ -189,6 +162,7 @@ struct kbasep_pm_metrics_state { - struct kbasep_pm_metrics values; - - #ifdef CONFIG_MALI_MIDGARD_DVFS -+ bool initialized; - struct hrtimer timer; - bool timer_active; - struct kbasep_pm_metrics dvfs_last; -@@ -202,8 +176,12 @@ struct kbasep_pm_metrics_state { - * @work: Work item which cancels the timer - * @timer: Timer for powering off the shader cores - * @configured_interval: Period of GPU poweroff timer -- * @configured_ticks: User-configured number of ticks to wait after the shader -- * power down request is received before turning off the cores -+ * @default_ticks: User-configured number of ticks to wait after the shader -+ * power down request is received before turning off the cores -+ * @configured_ticks: Power-policy configured number of ticks to wait after the -+ * shader power down request is received before turning off -+ * the cores. For simple power policies, this is equivalent -+ * to @default_ticks. - * @remaining_ticks: Number of remaining timer ticks until shaders are powered off - * @cancel_queued: True if the cancellation work item has been queued. This is - * required to ensure that it is not queued twice, e.g. after -@@ -217,6 +195,7 @@ struct kbasep_pm_tick_timer_state { - struct hrtimer timer; - - ktime_t configured_interval; -+ unsigned int default_ticks; - unsigned int configured_ticks; - unsigned int remaining_ticks; - -@@ -227,9 +206,6 @@ struct kbasep_pm_tick_timer_state { - union kbase_pm_policy_data { - struct kbasep_pm_policy_always_on always_on; - struct kbasep_pm_policy_coarse_demand coarse_demand; --#if !MALI_CUSTOMER_RELEASE -- struct kbasep_pm_policy_always_on_demand always_on_demand; --#endif - }; - - /** -@@ -240,7 +216,8 @@ union kbase_pm_policy_data { - * - * @pm_current_policy: The policy that is currently actively controlling the - * power state. -- * @pm_policy_data: Private data for current PM policy -+ * @pm_policy_data: Private data for current PM policy. This is automatically -+ * zeroed when a policy change occurs. - * @reset_done: Flag when a reset is complete - * @reset_done_wait: Wait queue to wait for changes to @reset_done - * @gpu_cycle_counter_requests: The reference count of active gpu cycle counter -@@ -254,6 +231,11 @@ union kbase_pm_policy_data { - * variable should be protected by: both the hwaccess_lock - * spinlock and the pm.lock mutex for writes; or at least - * one of either lock for reads. -+ * @gpu_ready: Indicates whether the GPU is in a state in which it is -+ * safe to perform PM changes. When false, the PM state -+ * machine needs to wait before making changes to the GPU -+ * power policy, DevFreq or core_mask, so as to avoid these -+ * changing while implicit GPU resets are ongoing. - * @pm_shaders_core_mask: Shader PM state synchronised shaders core mask. It - * holds the cores enabled in a hardware counters dump, - * and may differ from @shaders_avail when under different -@@ -294,6 +276,8 @@ union kbase_pm_policy_data { - * @callback_soft_reset: Optional callback to software reset the GPU. See - * &struct kbase_pm_callback_conf - * @ca_cores_enabled: Cores that are currently available -+ * @mcu_state: The current state of the micro-control unit, only applicable -+ * to GPUs that have such a component - * @l2_state: The current state of the L2 cache state machine. See - * &enum kbase_l2_core_state - * @l2_desired: True if the L2 cache should be powered on by the L2 cache state -@@ -303,10 +287,10 @@ union kbase_pm_policy_data { - * @shaders_avail: This is updated by the state machine when it is in a state - * where it can write to the SHADER_PWRON or PWROFF registers - * to have the same set of available cores as specified by -- * @shaders_desired_mask. So it would eventually have the same -- * value as @shaders_desired_mask and would precisely indicate -- * the cores that are currently available. This is internal to -- * shader state machine and should *not* be modified elsewhere. -+ * @shaders_desired_mask. So would precisely indicate the cores -+ * that are currently available. This is internal to shader -+ * state machine of JM GPUs and should *not* be modified -+ * elsewhere. - * @shaders_desired_mask: This is updated by the state machine when it is in - * a state where it can handle changes to the core - * availability (either by DVFS or sysfs). This is -@@ -318,6 +302,16 @@ union kbase_pm_policy_data { - * cores may be different, but there should be transitions in - * progress that will eventually achieve this state (assuming - * that the policy doesn't change its mind in the mean time). -+ * @mcu_desired: True if the micro-control unit should be powered on -+ * @policy_change_clamp_state_to_off: Signaling the backend is in PM policy -+ * change transition, needs the mcu/L2 to be brought back to the -+ * off state and remain in that state until the flag is cleared. -+ * @csf_pm_sched_flags: CSF Dynamic PM control flags in accordance to the -+ * current active PM policy. This field is updated whenever a -+ * new policy is activated. -+ * @policy_change_lock: Used to serialize the policy change calls. In CSF case, -+ * the change of policy may involve the scheduler to -+ * suspend running CSGs and then reconfigure the MCU. - * @in_reset: True if a GPU is resetting and normal power manager operation is - * suspended - * @partial_shaderoff: True if we want to partial power off shader cores, -@@ -373,6 +367,7 @@ struct kbase_pm_backend_data { - wait_queue_head_t gpu_in_desired_state_wait; - - bool gpu_powered; -+ bool gpu_ready; - - u64 pm_shaders_core_mask; - -@@ -406,10 +401,19 @@ struct kbase_pm_backend_data { - - u64 ca_cores_enabled; - -+#if MALI_USE_CSF -+ enum kbase_mcu_state mcu_state; -+#endif - enum kbase_l2_core_state l2_state; - enum kbase_shader_core_state shaders_state; - u64 shaders_avail; - u64 shaders_desired_mask; -+#if MALI_USE_CSF -+ bool mcu_desired; -+ bool policy_change_clamp_state_to_off; -+ unsigned int csf_pm_sched_flags; -+ struct mutex policy_change_lock; -+#endif - bool l2_desired; - bool l2_always_on; - bool shaders_desired; -@@ -433,6 +437,23 @@ struct kbase_pm_backend_data { - struct work_struct gpu_clock_control_work; - }; - -+#if MALI_USE_CSF -+/* CSF PM flag, signaling that the MCU CORE should be kept on */ -+#define CSF_DYNAMIC_PM_CORE_KEEP_ON (1 << 0) -+/* CSF PM flag, signaling no scheduler suspension on idle groups */ -+#define CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE (1 << 1) -+/* CSF PM flag, signaling no scheduler suspension on no runnable groups */ -+#define CSF_DYNAMIC_PM_SCHED_NO_SUSPEND (1 << 2) -+ -+/* The following flags corresponds to existing defined PM policies */ -+#define ALWAYS_ON_PM_SCHED_FLAGS (CSF_DYNAMIC_PM_CORE_KEEP_ON | \ -+ CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE | \ -+ CSF_DYNAMIC_PM_SCHED_NO_SUSPEND) -+#define COARSE_ON_DEMAND_PM_SCHED_FLAGS (0) -+#if !MALI_CUSTOMER_RELEASE -+#define ALWAYS_ON_DEMAND_PM_SCHED_FLAGS (CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE) -+#endif -+#endif - - /* List of policy IDs */ - enum kbase_pm_policy_id { -@@ -443,6 +464,33 @@ enum kbase_pm_policy_id { - KBASE_PM_POLICY_ID_ALWAYS_ON - }; - -+/** -+ * enum kbase_pm_policy_event - PM Policy event ID -+ */ -+enum kbase_pm_policy_event { -+ /** -+ * @KBASE_PM_POLICY_EVENT_IDLE: Indicates that the GPU power state -+ * model has determined that the GPU has gone idle. -+ */ -+ KBASE_PM_POLICY_EVENT_IDLE, -+ /** -+ * @KBASE_PM_POLICY_EVENT_POWER_ON: Indicates that the GPU state model -+ * is preparing to power on the GPU. -+ */ -+ KBASE_PM_POLICY_EVENT_POWER_ON, -+ /** -+ * @KBASE_PM_POLICY_EVENT_TIMER_HIT: Indicates that the GPU became -+ * active while the Shader Tick Timer was holding the GPU in a powered -+ * on state. -+ */ -+ KBASE_PM_POLICY_EVENT_TIMER_HIT, -+ /** -+ * @KBASE_PM_POLICY_EVENT_TIMER_MISS: Indicates that the GPU did not -+ * become active before the Shader Tick Timer timeout occurred. -+ */ -+ KBASE_PM_POLICY_EVENT_TIMER_MISS, -+}; -+ - /** - * struct kbase_pm_policy - Power policy structure. - * -@@ -455,15 +503,22 @@ enum kbase_pm_policy_id { - * @shaders_needed: Function called to find out if shader cores are needed - * @get_core_active: Function called to get the current overall GPU power - * state -+ * @handle_event: Function called when a PM policy event occurs. Should be -+ * set to NULL if the power policy doesn't require any -+ * event notifications. - * @id: Field indicating an ID for this policy. This is not - * necessarily the same as its index in the list returned - * by kbase_pm_list_policies(). - * It is used purely for debugging. -+ * @pm_sched_flags: Policy associated with CSF PM scheduling operational flags. -+ * Pre-defined required flags exist for each of the -+ * ARM released policies, such as 'always_on', 'coarse_demand' -+ * and etc. - */ - struct kbase_pm_policy { - char *name; - -- /** -+ /* - * Function called when the policy is selected - * - * This should initialize the kbdev->pm.pm_policy_data structure. It -@@ -477,7 +532,7 @@ struct kbase_pm_policy { - */ - void (*init)(struct kbase_device *kbdev); - -- /** -+ /* - * Function called when the policy is unselected. - * - * @kbdev: The kbase device structure for the device (must be a -@@ -485,7 +540,7 @@ struct kbase_pm_policy { - */ - void (*term)(struct kbase_device *kbdev); - -- /** -+ /* - * Function called to find out if shader cores are needed - * - * This needs to at least satisfy kbdev->pm.backend.shaders_desired, -@@ -498,7 +553,7 @@ struct kbase_pm_policy { - */ - bool (*shaders_needed)(struct kbase_device *kbdev); - -- /** -+ /* - * Function called to get the current overall GPU power state - * - * This function must meet or exceed the requirements for power -@@ -511,7 +566,26 @@ struct kbase_pm_policy { - */ - bool (*get_core_active)(struct kbase_device *kbdev); - -+ /** -+ * Function called when a power event occurs -+ * -+ * @kbdev: The kbase device structure for the device (must be a -+ * valid pointer) -+ * @event: The id of the power event that has occurred -+ */ -+ void (*handle_event)(struct kbase_device *kbdev, -+ enum kbase_pm_policy_event event); -+ - enum kbase_pm_policy_id id; -+ -+#if MALI_USE_CSF -+ /* Policy associated with CSF PM scheduling operational flags. -+ * There are pre-defined required flags exist for each of the -+ * ARM released policies, such as 'always_on', 'coarse_demand' -+ * and etc. -+ */ -+ unsigned int pm_sched_flags; -+#endif - }; - - #endif /* _KBASE_PM_HWACCESS_DEFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c -index 6b821f7..bcada93 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c -@@ -1,12 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -17,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -32,12 +30,18 @@ - #include <mali_kbase_pm.h> - #include <mali_kbase_config_defaults.h> - #include <mali_kbase_smc.h> -+ -+#if MALI_USE_CSF -+#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> -+#else - #include <mali_kbase_hwaccess_jm.h> -+#endif /* !MALI_USE_CSF */ -+ - #include <mali_kbase_reset_gpu.h> - #include <mali_kbase_ctx_sched.h> - #include <mali_kbase_hwcnt_context.h> - #include <backend/gpu/mali_kbase_cache_policy_backend.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include <backend/gpu/mali_kbase_irq_internal.h> - #include <backend/gpu/mali_kbase_pm_internal.h> - #include <backend/gpu/mali_kbase_l2_mmu_config.h> -@@ -45,6 +49,9 @@ - #ifdef CONFIG_MALI_ARBITER_SUPPORT - #include <arbiter/mali_kbase_arbiter_pm.h> - #endif /* CONFIG_MALI_ARBITER_SUPPORT */ -+#if MALI_USE_CSF -+#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> -+#endif - - #include <linux/of.h> - -@@ -89,6 +96,28 @@ static u64 kbase_pm_get_state( - enum kbase_pm_core_type core_type, - enum kbasep_pm_action action); - -+#if MALI_USE_CSF -+bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (unlikely(!kbdev->csf.firmware_inited)) -+ return false; -+ -+ if (kbdev->csf.scheduler.pm_active_count) -+ return true; -+ -+ /* MCU is supposed to be ON, only when scheduler.pm_active_count is -+ * non zero. But for always_on policy, the MCU needs to be kept on, -+ * unless policy changing transition needs it off. -+ */ -+ -+ return (kbdev->pm.backend.mcu_desired && -+ kbase_pm_no_mcu_core_pwroff(kbdev) && -+ !kbdev->pm.backend.policy_change_clamp_state_to_off); -+} -+#endif -+ - bool kbase_pm_is_l2_desired(struct kbase_device *kbdev) - { - if (kbdev->pm.backend.protected_entry_transition_override) -@@ -102,6 +131,11 @@ bool kbase_pm_is_l2_desired(struct kbase_device *kbdev) - !kbdev->pm.backend.shaders_desired) - return false; - -+#if MALI_USE_CSF -+ if (kbdev->pm.backend.policy_change_clamp_state_to_off) -+ return false; -+#endif -+ - return kbdev->pm.backend.l2_desired; - } - -@@ -210,7 +244,7 @@ static u32 core_type_to_reg(enum kbase_pm_core_type core_type, - return (u32)core_type + (u32)action; - } - --#ifdef CONFIG_ARM64 -+#if IS_ENABLED(CONFIG_ARM64) - static void mali_cci_flush_l2(struct kbase_device *kbdev) - { - const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED; -@@ -233,7 +267,8 @@ static void mali_cci_flush_l2(struct kbase_device *kbdev) - GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)); - - /* Wait for cache flush to complete before continuing, exit on -- * gpu resets or loop expiry. */ -+ * gpu resets or loop expiry. -+ */ - while (((raw & mask) == 0) && --loops) { - raw = kbase_reg_read(kbdev, - GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)); -@@ -372,9 +407,9 @@ u64 kbase_pm_get_present_cores(struct kbase_device *kbdev, - - switch (type) { - case KBASE_PM_CORE_L2: -- return kbdev->gpu_props.props.raw_props.l2_present; -+ return kbdev->gpu_props.curr_config.l2_present; - case KBASE_PM_CORE_SHADER: -- return kbdev->gpu_props.props.raw_props.shader_present; -+ return kbdev->gpu_props.curr_config.shader_present; - case KBASE_PM_CORE_TILER: - return kbdev->gpu_props.props.raw_props.tiler_present; - case KBASE_PM_CORE_STACK: -@@ -468,14 +503,10 @@ static void kbase_pm_trigger_hwcnt_disable(struct kbase_device *kbdev) - */ - if (kbase_hwcnt_context_disable_atomic(kbdev->hwcnt_gpu_ctx)) { - backend->hwcnt_disabled = true; -+ - } else { --#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE -- queue_work(system_wq, -- &backend->hwcnt_disable_work); --#else -- queue_work(system_highpri_wq, -- &backend->hwcnt_disable_work); --#endif -+ kbase_hwcnt_context_queue_work(kbdev->hwcnt_gpu_ctx, -+ &backend->hwcnt_disable_work); - } - } - -@@ -493,7 +524,8 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) - * Skip if size and hash are not given explicitly, - * which means default values are used. - */ -- if ((kbdev->l2_size_override == 0) && (kbdev->l2_hash_override == 0)) -+ if ((kbdev->l2_size_override == 0) && (kbdev->l2_hash_override == 0) && -+ (!kbdev->l2_hash_values_override)) - return; - - val = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_CONFIG)); -@@ -504,13 +536,25 @@ static void kbase_pm_l2_config_override(struct kbase_device *kbdev) - } - - if (kbdev->l2_hash_override) { -+ WARN_ON(kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH)); - val &= ~L2_CONFIG_HASH_MASK; - val |= (kbdev->l2_hash_override << L2_CONFIG_HASH_SHIFT); -+ } else if (kbdev->l2_hash_values_override) { -+ int i; -+ -+ WARN_ON(!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH)); -+ val &= ~L2_CONFIG_ASN_HASH_ENABLE_MASK; -+ val |= (0x1 << L2_CONFIG_ASN_HASH_ENABLE_SHIFT); -+ -+ for (i = 0; i < ASN_HASH_COUNT; i++) { -+ dev_dbg(kbdev->dev, "Program 0x%x to ASN_HASH[%d]\n", -+ kbdev->l2_hash_values[i], i); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(ASN_HASH(i)), -+ kbdev->l2_hash_values[i]); -+ } - } - - dev_dbg(kbdev->dev, "Program 0x%x to L2_CONFIG\n", val); -- -- /* Write L2_CONFIG to override */ - kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_CONFIG), val); - } - -@@ -523,6 +567,278 @@ static void kbase_pm_control_gpu_clock(struct kbase_device *kbdev) - queue_work(system_wq, &backend->gpu_clock_control_work); - } - -+#if MALI_USE_CSF -+static const char *kbase_mcu_state_to_string(enum kbase_mcu_state state) -+{ -+ const char *const strings[] = { -+#define KBASEP_MCU_STATE(n) #n, -+#include "mali_kbase_pm_mcu_states.h" -+#undef KBASEP_MCU_STATE -+ }; -+ if (WARN_ON((size_t)state >= ARRAY_SIZE(strings))) -+ return "Bad MCU state"; -+ else -+ return strings[state]; -+} -+ -+static inline bool kbase_pm_handle_mcu_core_attr_update(struct kbase_device *kbdev) -+{ -+ struct kbase_pm_backend_data *backend = &kbdev->pm.backend; -+ bool timer_update; -+ bool core_mask_update; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ WARN_ON(backend->mcu_state != KBASE_MCU_ON); -+ -+ /* This function is only for cases where the MCU managing Cores, if -+ * the firmware mode is with host control, do nothing here. -+ */ -+ if (unlikely(kbdev->csf.firmware_hctl_core_pwr)) -+ return false; -+ -+ core_mask_update = -+ backend->shaders_avail != backend->shaders_desired_mask; -+ -+ timer_update = kbdev->csf.mcu_core_pwroff_dur_count != -+ kbdev->csf.mcu_core_pwroff_reg_shadow; -+ -+ if (core_mask_update || timer_update) -+ kbase_csf_firmware_update_core_attr(kbdev, timer_update, -+ core_mask_update, backend->shaders_desired_mask); -+ -+ return (core_mask_update || timer_update); -+} -+ -+static int kbase_pm_mcu_update_state(struct kbase_device *kbdev) -+{ -+ struct kbase_pm_backend_data *backend = &kbdev->pm.backend; -+ enum kbase_mcu_state prev_state; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* -+ * Initial load of firmare should have been done to -+ * exercise the MCU state machine. -+ */ -+ if (unlikely(!kbdev->csf.firmware_inited)) { -+ WARN_ON(backend->mcu_state != KBASE_MCU_OFF); -+ return -EIO; -+ } -+ -+ do { -+ u64 shaders_trans = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER); -+ u64 shaders_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER); -+ -+ /* mask off ready from trans in case transitions finished -+ * between the register reads -+ */ -+ shaders_trans &= ~shaders_ready; -+ -+ prev_state = backend->mcu_state; -+ -+ switch (backend->mcu_state) { -+ case KBASE_MCU_OFF: -+ if (kbase_pm_is_mcu_desired(kbdev) && -+ !backend->policy_change_clamp_state_to_off && -+ backend->l2_state == KBASE_L2_ON) { -+ kbase_csf_firmware_trigger_reload(kbdev); -+ backend->mcu_state = KBASE_MCU_PEND_ON_RELOAD; -+ } -+ break; -+ -+ case KBASE_MCU_PEND_ON_RELOAD: -+ if (kbdev->csf.firmware_reloaded) { -+ backend->shaders_desired_mask = -+ kbase_pm_ca_get_core_mask(kbdev); -+ kbase_csf_firmware_global_reinit(kbdev, -+ backend->shaders_desired_mask); -+ backend->mcu_state = -+ KBASE_MCU_ON_GLB_REINIT_PEND; -+ } -+ break; -+ -+ case KBASE_MCU_ON_GLB_REINIT_PEND: -+ if (kbase_csf_firmware_global_reinit_complete(kbdev)) { -+ backend->shaders_avail = -+ backend->shaders_desired_mask; -+ backend->pm_shaders_core_mask = 0; -+ if (kbdev->csf.firmware_hctl_core_pwr) { -+ kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, -+ backend->shaders_avail, ACTION_PWRON); -+ backend->mcu_state = -+ KBASE_MCU_HCTL_SHADERS_PEND_ON; -+ } else -+ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; -+ } -+ break; -+ -+ case KBASE_MCU_HCTL_SHADERS_PEND_ON: -+ if (!shaders_trans && -+ shaders_ready == backend->shaders_avail) { -+ /* Cores now stable, notify MCU the stable mask */ -+ kbase_csf_firmware_update_core_attr(kbdev, -+ false, true, shaders_ready); -+ -+ backend->pm_shaders_core_mask = shaders_ready; -+ backend->mcu_state = -+ KBASE_MCU_HCTL_CORES_NOTIFY_PEND; -+ } -+ break; -+ -+ case KBASE_MCU_HCTL_CORES_NOTIFY_PEND: -+ /* Wait for the acknowledgement */ -+ if (kbase_csf_firmware_core_attr_updated(kbdev)) -+ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; -+ break; -+ -+ case KBASE_MCU_ON_HWCNT_ENABLE: -+ backend->hwcnt_desired = true; -+ if (backend->hwcnt_disabled) { -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ kbase_hwcnt_context_enable( -+ kbdev->hwcnt_gpu_ctx); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ backend->hwcnt_disabled = false; -+ } -+ backend->mcu_state = KBASE_MCU_ON; -+ break; -+ -+ case KBASE_MCU_ON: -+ backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); -+ -+ if (!kbase_pm_is_mcu_desired(kbdev)) -+ backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; -+ else if (kbdev->csf.firmware_hctl_core_pwr) { -+ /* Host control add additional Cores to be active */ -+ if (backend->shaders_desired_mask & ~shaders_ready) { -+ backend->hwcnt_desired = false; -+ if (!backend->hwcnt_disabled) -+ kbase_pm_trigger_hwcnt_disable(kbdev); -+ backend->mcu_state = -+ KBASE_MCU_HCTL_MCU_ON_RECHECK; -+ } -+ } else if (kbase_pm_handle_mcu_core_attr_update(kbdev)) -+ kbdev->pm.backend.mcu_state = -+ KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND; -+ break; -+ -+ case KBASE_MCU_HCTL_MCU_ON_RECHECK: -+ backend->shaders_desired_mask = kbase_pm_ca_get_core_mask(kbdev); -+ -+ if (!backend->hwcnt_disabled) { -+ /* Wait for being disabled */ -+ ; -+ } else if (!kbase_pm_is_mcu_desired(kbdev)) { -+ /* Converging to MCU powering down flow */ -+ backend->mcu_state = KBASE_MCU_ON_HWCNT_DISABLE; -+ } else if (backend->shaders_desired_mask & ~shaders_ready) { -+ /* set cores ready but not available to -+ * meet SHADERS_PEND_ON check pass -+ */ -+ backend->shaders_avail = -+ (backend->shaders_desired_mask | shaders_ready); -+ -+ kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, -+ backend->shaders_avail & ~shaders_ready, -+ ACTION_PWRON); -+ backend->mcu_state = -+ KBASE_MCU_HCTL_SHADERS_PEND_ON; -+ } else { -+ backend->mcu_state = -+ KBASE_MCU_HCTL_SHADERS_PEND_ON; -+ } -+ break; -+ -+ case KBASE_MCU_ON_CORE_ATTR_UPDATE_PEND: -+ if (kbase_csf_firmware_core_attr_updated(kbdev)) { -+ backend->shaders_avail = -+ backend->shaders_desired_mask; -+ backend->mcu_state = KBASE_MCU_ON; -+ } -+ break; -+ -+ case KBASE_MCU_ON_HWCNT_DISABLE: -+ if (kbase_pm_is_mcu_desired(kbdev)) { -+ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; -+ break; -+ } -+ -+ backend->hwcnt_desired = false; -+ if (!backend->hwcnt_disabled) -+ kbase_pm_trigger_hwcnt_disable(kbdev); -+ -+ if (backend->hwcnt_disabled) -+ backend->mcu_state = KBASE_MCU_ON_HALT; -+ break; -+ -+ case KBASE_MCU_ON_HALT: -+ if (!kbase_pm_is_mcu_desired(kbdev)) { -+ kbase_csf_firmware_trigger_mcu_halt(kbdev); -+ backend->mcu_state = KBASE_MCU_ON_PEND_HALT; -+ } else -+ backend->mcu_state = KBASE_MCU_ON_HWCNT_ENABLE; -+ break; -+ -+ case KBASE_MCU_ON_PEND_HALT: -+ if (kbase_csf_firmware_mcu_halted(kbdev)) { -+ if (kbdev->csf.firmware_hctl_core_pwr) -+ backend->mcu_state = -+ KBASE_MCU_HCTL_SHADERS_READY_OFF; -+ else -+ backend->mcu_state = KBASE_MCU_POWER_DOWN; -+ } -+ break; -+ -+ case KBASE_MCU_HCTL_SHADERS_READY_OFF: -+ kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, -+ shaders_ready, ACTION_PWROFF); -+ backend->mcu_state = -+ KBASE_MCU_HCTL_SHADERS_PEND_OFF; -+ break; -+ -+ case KBASE_MCU_HCTL_SHADERS_PEND_OFF: -+ if (!shaders_trans && !shaders_ready) { -+ backend->pm_shaders_core_mask = 0; -+ backend->mcu_state = KBASE_MCU_POWER_DOWN; -+ } -+ break; -+ -+ case KBASE_MCU_POWER_DOWN: -+ kbase_csf_firmware_disable_mcu(kbdev); -+ backend->mcu_state = KBASE_MCU_PEND_OFF; -+ break; -+ -+ case KBASE_MCU_PEND_OFF: -+ /* wait synchronously for the MCU to get disabled */ -+ kbase_csf_firmware_disable_mcu_wait(kbdev); -+ backend->mcu_state = KBASE_MCU_OFF; -+ break; -+ -+ case KBASE_MCU_RESET_WAIT: -+ /* Reset complete */ -+ if (!backend->in_reset) -+ backend->mcu_state = KBASE_MCU_OFF; -+ break; -+ -+ default: -+ WARN(1, "Invalid state in mcu_state: %d", -+ backend->mcu_state); -+ } -+ -+ if (backend->mcu_state != prev_state) -+ dev_dbg(kbdev->dev, "MCU state transition: %s to %s\n", -+ kbase_mcu_state_to_string(prev_state), -+ kbase_mcu_state_to_string(backend->mcu_state)); -+ -+ } while (backend->mcu_state != prev_state); -+ -+ return 0; -+} -+#endif -+ - static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state) - { - const char *const strings[] = { -@@ -539,8 +855,10 @@ static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state) - static int kbase_pm_l2_update_state(struct kbase_device *kbdev) - { - struct kbase_pm_backend_data *backend = &kbdev->pm.backend; -- u64 l2_present = kbdev->gpu_props.props.raw_props.l2_present; -+ u64 l2_present = kbdev->gpu_props.curr_config.l2_present; -+#if !MALI_USE_CSF - u64 tiler_present = kbdev->gpu_props.props.raw_props.tiler_present; -+#endif - enum kbase_l2_core_state prev_state; - - lockdep_assert_held(&kbdev->hwaccess_lock); -@@ -551,35 +869,49 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) - KBASE_PM_CORE_L2); - u64 l2_ready = kbase_pm_get_ready_cores(kbdev, - KBASE_PM_CORE_L2); -+ -+#if !MALI_USE_CSF - u64 tiler_trans = kbase_pm_get_trans_cores(kbdev, - KBASE_PM_CORE_TILER); - u64 tiler_ready = kbase_pm_get_ready_cores(kbdev, - KBASE_PM_CORE_TILER); -+#endif - - /* - * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores - * are vulnerable to corruption if gpu is lost - */ -- if (kbase_is_gpu_lost(kbdev)) -- return -EIO; -+ if (kbase_is_gpu_removed(kbdev) -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ || kbase_pm_is_gpu_lost(kbdev)) { -+#else -+ ) { -+#endif -+ backend->shaders_state = -+ KBASE_SHADERS_OFF_CORESTACK_OFF; -+ backend->l2_state = KBASE_L2_OFF; -+ dev_dbg(kbdev->dev, "GPU lost has occurred - L2 off\n"); -+ break; -+ } - - /* mask off ready from trans in case transitions finished - * between the register reads - */ - l2_trans &= ~l2_ready; -+#if !MALI_USE_CSF - tiler_trans &= ~tiler_ready; -- -+#endif - prev_state = backend->l2_state; - - switch (backend->l2_state) { - case KBASE_L2_OFF: - if (kbase_pm_is_l2_desired(kbdev)) { - /* -- * Set the desired config for L2 before powering -- * it on -+ * Set the desired config for L2 before -+ * powering it on - */ - kbase_pm_l2_config_override(kbdev); -- -+#if !MALI_USE_CSF - /* L2 is required, power on. Powering on the - * tiler will also power the first L2 cache. - */ -@@ -593,14 +925,30 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) - kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, - l2_present & ~1, - ACTION_PWRON); -+#else -+ /* With CSF firmware, Host driver doesn't need to -+ * handle power management with both shader and tiler cores. -+ * The CSF firmware will power up the cores appropriately. -+ * So only power the l2 cache explicitly. -+ */ -+ kbase_pm_invoke(kbdev, KBASE_PM_CORE_L2, -+ l2_present, ACTION_PWRON); -+#endif - backend->l2_state = KBASE_L2_PEND_ON; - } - break; - - case KBASE_L2_PEND_ON: -+#if !MALI_USE_CSF - if (!l2_trans && l2_ready == l2_present && !tiler_trans - && tiler_ready == tiler_present) { -- KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, tiler_ready); -+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, -+ tiler_ready); -+#else -+ if (!l2_trans && l2_ready == l2_present) { -+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL, -+ l2_ready); -+#endif - /* - * Ensure snoops are enabled after L2 is powered - * up. Note that kbase keeps track of the snoop -@@ -658,22 +1006,30 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) - break; - - case KBASE_L2_ON_HWCNT_ENABLE: -+#if !MALI_USE_CSF - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { - kbase_hwcnt_context_enable( - kbdev->hwcnt_gpu_ctx); - backend->hwcnt_disabled = false; - } -+#endif - backend->l2_state = KBASE_L2_ON; - break; - - case KBASE_L2_ON: - if (!kbase_pm_is_l2_desired(kbdev)) { -+#if !MALI_USE_CSF - /* Do not power off L2 until the shaders and - * core stacks are off. - */ - if (backend->shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) - break; -+#else -+ /* Do not power off L2 until the MCU has been stopped */ -+ if (backend->mcu_state != KBASE_MCU_OFF) -+ break; -+#endif - - /* We need to make sure hardware counters are - * disabled before powering down the L2, to -@@ -690,6 +1046,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) - break; - - case KBASE_L2_ON_HWCNT_DISABLE: -+#if !MALI_USE_CSF - /* If the L2 became desired while we were waiting on the - * worker to do the actual hwcnt disable (which might - * happen if some work was submitted immediately after -@@ -719,6 +1076,7 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) - if (!backend->hwcnt_disabled) { - kbase_pm_trigger_hwcnt_disable(kbdev); - } -+#endif - - if (backend->hwcnt_disabled) { - if (kbdev->pm.backend.gpu_clock_slow_down_wa) -@@ -769,9 +1127,11 @@ static int kbase_pm_l2_update_state(struct kbase_device *kbdev) - */ - kbase_gpu_start_cache_clean_nolock( - kbdev); -- -+#if !MALI_USE_CSF - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u); -- -+#else -+ KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_L2, NULL, 0u); -+#endif - backend->l2_state = KBASE_L2_PEND_OFF; - break; - -@@ -877,6 +1237,7 @@ static void shader_poweroff_timer_queue_cancel(struct kbase_device *kbdev) - } - } - -+#if !MALI_USE_CSF - static const char *kbase_shader_core_state_to_string( - enum kbase_shader_core_state state) - { -@@ -898,7 +1259,6 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) - &kbdev->pm.backend.shader_tick_timer; - enum kbase_shader_core_state prev_state; - u64 stacks_avail = 0; -- int err = 0; - - lockdep_assert_held(&kbdev->hwaccess_lock); - -@@ -924,8 +1284,15 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) - * kbase_pm_get_ready_cores and kbase_pm_get_trans_cores - * are vulnerable to corruption if gpu is lost - */ -- if (kbase_is_gpu_lost(kbdev)) { -- err = -EIO; -+ if (kbase_is_gpu_removed(kbdev) -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ || kbase_pm_is_gpu_lost(kbdev)) { -+#else -+ ) { -+#endif -+ backend->shaders_state = -+ KBASE_SHADERS_OFF_CORESTACK_OFF; -+ dev_dbg(kbdev->dev, "GPU lost has occurred - shaders off\n"); - break; - } - -@@ -976,6 +1343,12 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) - kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER, - backend->shaders_avail, ACTION_PWRON); - -+ if (backend->pm_current_policy && -+ backend->pm_current_policy->handle_event) -+ backend->pm_current_policy->handle_event( -+ kbdev, -+ KBASE_PM_POLICY_EVENT_POWER_ON); -+ - backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON; - } - break; -@@ -986,8 +1359,18 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) - backend->pm_shaders_core_mask = shaders_ready; - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { -+#if MALI_USE_CSF -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, -+ &flags); -+#endif - kbase_hwcnt_context_enable( - kbdev->hwcnt_gpu_ctx); -+#if MALI_USE_CSF -+ kbase_csf_scheduler_spin_unlock(kbdev, -+ flags); -+#endif - backend->hwcnt_disabled = false; - } - -@@ -1018,6 +1401,12 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) - /* Wait for being disabled */ - ; - } else if (!backend->shaders_desired) { -+ if (backend->pm_current_policy && -+ backend->pm_current_policy->handle_event) -+ backend->pm_current_policy->handle_event( -+ kbdev, -+ KBASE_PM_POLICY_EVENT_IDLE); -+ - if (kbdev->pm.backend.protected_transition_override || - #ifdef CONFIG_MALI_ARBITER_SUPPORT - kbase_pm_is_suspending(kbdev) || -@@ -1078,9 +1467,21 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) - } - - if (backend->shaders_desired) { -+ if (backend->pm_current_policy && -+ backend->pm_current_policy->handle_event) -+ backend->pm_current_policy->handle_event( -+ kbdev, -+ KBASE_PM_POLICY_EVENT_TIMER_HIT); -+ - stt->remaining_ticks = 0; - backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK; - } else if (stt->remaining_ticks == 0) { -+ if (backend->pm_current_policy && -+ backend->pm_current_policy->handle_event) -+ backend->pm_current_policy->handle_event( -+ kbdev, -+ KBASE_PM_POLICY_EVENT_TIMER_MISS); -+ - backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON; - #ifdef CONFIG_MALI_ARBITER_SUPPORT - } else if (kbase_pm_is_suspending(kbdev) || -@@ -1167,8 +1568,18 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) - backend->pm_shaders_core_mask = 0; - backend->hwcnt_desired = true; - if (backend->hwcnt_disabled) { -+#if MALI_USE_CSF -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, -+ &flags); -+#endif - kbase_hwcnt_context_enable( - kbdev->hwcnt_gpu_ctx); -+#if MALI_USE_CSF -+ kbase_csf_scheduler_spin_unlock(kbdev, -+ flags); -+#endif - backend->hwcnt_disabled = false; - } - backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF_TIMER_PEND_OFF; -@@ -1195,8 +1606,9 @@ static int kbase_pm_shaders_update_state(struct kbase_device *kbdev) - - } while (backend->shaders_state != prev_state); - -- return err; -+ return 0; - } -+#endif - - static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev) - { -@@ -1211,12 +1623,21 @@ static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev) - kbdev->pm.backend.l2_state != KBASE_L2_OFF) - in_desired_state = false; - -+#if !MALI_USE_CSF - if (kbdev->pm.backend.shaders_desired && - kbdev->pm.backend.shaders_state != KBASE_SHADERS_ON_CORESTACK_ON) - in_desired_state = false; - else if (!kbdev->pm.backend.shaders_desired && - kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF) - in_desired_state = false; -+#else -+ if (kbase_pm_is_mcu_desired(kbdev) && -+ kbdev->pm.backend.mcu_state != KBASE_MCU_ON) -+ in_desired_state = false; -+ else if (!kbase_pm_is_mcu_desired(kbdev) && -+ kbdev->pm.backend.mcu_state != KBASE_MCU_OFF) -+ in_desired_state = false; -+#endif - - return in_desired_state; - } -@@ -1280,17 +1701,22 @@ static void kbase_pm_trace_power_state(struct kbase_device *kbdev) - - void kbase_pm_update_state(struct kbase_device *kbdev) - { -+#if !MALI_USE_CSF - enum kbase_shader_core_state prev_shaders_state = - kbdev->pm.backend.shaders_state; -+#else -+ enum kbase_mcu_state prev_mcu_state = kbdev->pm.backend.mcu_state; -+#endif - - lockdep_assert_held(&kbdev->hwaccess_lock); - -- if (!kbdev->pm.backend.gpu_powered) -- return; /* Do nothing if the GPU is off */ -+ if (!kbdev->pm.backend.gpu_ready) -+ return; /* Do nothing if the GPU is not ready */ - - if (kbase_pm_l2_update_state(kbdev)) - return; - -+#if !MALI_USE_CSF - if (kbase_pm_shaders_update_state(kbdev)) - return; - -@@ -1304,9 +1730,20 @@ void kbase_pm_update_state(struct kbase_device *kbdev) - if (kbase_pm_l2_update_state(kbdev)) - return; - } -+#else -+ if (kbase_pm_mcu_update_state(kbdev)) -+ return; -+ -+ if (prev_mcu_state != KBASE_MCU_OFF && -+ kbdev->pm.backend.mcu_state == KBASE_MCU_OFF) { -+ if (kbase_pm_l2_update_state(kbdev)) -+ return; -+ } -+#endif - - if (kbase_pm_is_in_desired_state_nolock(kbdev)) { -- KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, kbdev->pm.backend.shaders_avail); -+ KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, -+ kbdev->pm.backend.shaders_avail); - - kbase_pm_trace_power_state(kbdev); - -@@ -1363,7 +1800,8 @@ int kbase_pm_state_machine_init(struct kbase_device *kbdev) - hrtimer_init(&stt->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - stt->timer.function = shader_tick_timer_callback; - stt->configured_interval = HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS); -- stt->configured_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; -+ stt->default_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER; -+ stt->configured_ticks = stt->default_ticks; - - return 0; - } -@@ -1382,7 +1820,19 @@ void kbase_pm_reset_start_locked(struct kbase_device *kbdev) - - backend->in_reset = true; - backend->l2_state = KBASE_L2_RESET_WAIT; -+#if !MALI_USE_CSF - backend->shaders_state = KBASE_SHADERS_RESET_WAIT; -+#else -+ /* MCU state machine is exercised only after the initial load/boot -+ * of the firmware. -+ */ -+ if (likely(kbdev->csf.firmware_inited)) { -+ backend->mcu_state = KBASE_MCU_RESET_WAIT; -+ kbdev->csf.firmware_reload_needed = true; -+ } else { -+ WARN_ON(backend->mcu_state != KBASE_MCU_OFF); -+ } -+#endif - - /* We're in a reset, so hwcnt will have been synchronously disabled by - * this function's caller as part of the reset process. We therefore -@@ -1422,15 +1872,28 @@ void kbase_pm_reset_complete(struct kbase_device *kbdev) - - /* Timeout for kbase_pm_wait_for_desired_state when wait_event_killable has - * aborted due to a fatal signal. If the time spent waiting has exceeded this -- * threshold then there is most likely a hardware issue. */ --#define PM_TIMEOUT (5*HZ) /* 5s */ -+ * threshold then there is most likely a hardware issue. -+ */ -+#define PM_TIMEOUT_MS (5000) /* 5s */ - - static void kbase_pm_timed_out(struct kbase_device *kbdev) - { -+ unsigned long flags; -+ - dev_err(kbdev->dev, "Power transition timed out unexpectedly\n"); -+#if !MALI_USE_CSF -+ CSTD_UNUSED(flags); - dev_err(kbdev->dev, "Desired state :\n"); - dev_err(kbdev->dev, "\tShader=%016llx\n", - kbdev->pm.backend.shaders_desired ? kbdev->pm.backend.shaders_avail : 0); -+#else -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ dev_err(kbdev->dev, "\tMCU desired = %d\n", -+ kbase_pm_is_mcu_desired(kbdev)); -+ dev_err(kbdev->dev, "\tMCU sw state = %d\n", -+ kbdev->pm.backend.mcu_state); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+#endif - dev_err(kbdev->dev, "Current state :\n"); - dev_err(kbdev->dev, "\tShader=%08x%08x\n", - kbase_reg_read(kbdev, -@@ -1447,6 +1910,10 @@ static void kbase_pm_timed_out(struct kbase_device *kbdev) - GPU_CONTROL_REG(L2_READY_HI)), - kbase_reg_read(kbdev, - GPU_CONTROL_REG(L2_READY_LO))); -+#if MALI_USE_CSF -+ dev_err(kbdev->dev, "\tMCU status = %d\n", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS))); -+#endif - dev_err(kbdev->dev, "Cores transitioning :\n"); - dev_err(kbdev->dev, "\tShader=%08x%08x\n", - kbase_reg_read(kbdev, GPU_CONTROL_REG( -@@ -1465,49 +1932,87 @@ static void kbase_pm_timed_out(struct kbase_device *kbdev) - L2_PWRTRANS_LO))); - - dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n"); -- if (kbase_prepare_to_reset_gpu(kbdev)) -+ if (kbase_prepare_to_reset_gpu(kbdev, -+ RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) - kbase_reset_gpu(kbdev); - } - --void kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) -+int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev) - { - unsigned long flags; - unsigned long timeout; -- int err; -+ long remaining; -+ int err = 0; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_pm_update_state(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -- timeout = jiffies + PM_TIMEOUT; -+#if MALI_USE_CSF -+ timeout = kbase_csf_timeout_in_jiffies(PM_TIMEOUT_MS); -+#else -+ timeout = msecs_to_jiffies(PM_TIMEOUT_MS); -+#endif - - /* Wait for cores */ -- err = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait, -- kbase_pm_is_in_desired_state_with_l2_powered(kbdev)); -+#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE -+ remaining = wait_event_killable_timeout( -+#else -+ remaining = wait_event_timeout( -+#endif -+ kbdev->pm.backend.gpu_in_desired_state_wait, -+ kbase_pm_is_in_desired_state_with_l2_powered(kbdev), timeout); - -- if (err < 0 && time_after(jiffies, timeout)) -+ if (!remaining) { - kbase_pm_timed_out(kbdev); -+ err = -ETIMEDOUT; -+ } else if (remaining < 0) { -+ dev_info( -+ kbdev->dev, -+ "Wait for desired PM state with L2 powered got interrupted"); -+ err = (int)remaining; -+ } -+ -+ return err; - } - --void kbase_pm_wait_for_desired_state(struct kbase_device *kbdev) -+int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev) - { - unsigned long flags; -- unsigned long timeout; -- int err; -+ long remaining; -+#if MALI_USE_CSF -+ long timeout = kbase_csf_timeout_in_jiffies(PM_TIMEOUT_MS); -+#else -+ long timeout = msecs_to_jiffies(PM_TIMEOUT_MS); -+#endif -+ int err = 0; - - /* Let the state machine latch the most recent desired state. */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_pm_update_state(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -- timeout = jiffies + PM_TIMEOUT; -- - /* Wait for cores */ -- err = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait, -- kbase_pm_is_in_desired_state(kbdev)); -+#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE -+ remaining = wait_event_killable_timeout( -+ kbdev->pm.backend.gpu_in_desired_state_wait, -+ kbase_pm_is_in_desired_state(kbdev), timeout); -+#else -+ remaining = wait_event_timeout( -+ kbdev->pm.backend.gpu_in_desired_state_wait, -+ kbase_pm_is_in_desired_state(kbdev), timeout); -+#endif - -- if (err < 0 && time_after(jiffies, timeout)) -+ if (!remaining) { - kbase_pm_timed_out(kbdev); -+ err = -ETIMEDOUT; -+ } else if (remaining < 0) { -+ dev_info(kbdev->dev, -+ "Wait for desired PM state got interrupted"); -+ err = (int)remaining; -+ } -+ -+ return err; - } - KBASE_EXPORT_TEST_API(kbase_pm_wait_for_desired_state); - -@@ -1515,7 +2020,7 @@ void kbase_pm_enable_interrupts(struct kbase_device *kbdev) - { - unsigned long flags; - -- KBASE_DEBUG_ASSERT(NULL != kbdev); -+ KBASE_DEBUG_ASSERT(kbdev != NULL); - /* - * Clear all interrupts, - * and unmask them all. -@@ -1529,14 +2034,19 @@ void kbase_pm_enable_interrupts(struct kbase_device *kbdev) - kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF); - - kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF); -+#if MALI_USE_CSF -+ /* Enable only the Page fault bits part */ -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFF); -+#else - kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF); -+#endif - } - - KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts); - - void kbase_pm_disable_interrupts_nolock(struct kbase_device *kbdev) - { -- KBASE_DEBUG_ASSERT(NULL != kbdev); -+ KBASE_DEBUG_ASSERT(kbdev != NULL); - /* - * Mask all interrupts, - * and clear them all. -@@ -1563,6 +2073,23 @@ void kbase_pm_disable_interrupts(struct kbase_device *kbdev) - - KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts); - -+#if MALI_USE_CSF -+static void update_user_reg_page_mapping(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->pm.lock); -+ -+ if (kbdev->csf.mali_file_inode) { -+ /* This would zap the pte corresponding to the mapping of User -+ * register page for all the Kbase contexts. -+ */ -+ unmap_mapping_range(kbdev->csf.mali_file_inode->i_mapping, -+ BASEP_MEM_CSF_USER_REG_PAGE_HANDLE, -+ PAGE_SIZE, 1); -+ } -+} -+#endif -+ -+ - /* - * pmu layout: - * 0x0000: PMU TAG (RO) (0xCAFECAFE) -@@ -1574,10 +2101,20 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) - bool reset_required = is_resume; - unsigned long flags; - -- KBASE_DEBUG_ASSERT(NULL != kbdev); -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+#if !MALI_USE_CSF - lockdep_assert_held(&kbdev->js_data.runpool_mutex); -+#endif /* !MALI_USE_CSF */ - lockdep_assert_held(&kbdev->pm.lock); - -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ if (WARN_ON(kbase_pm_is_gpu_lost(kbdev))) { -+ dev_err(kbdev->dev, -+ "%s: Cannot power up while GPU lost", __func__); -+ return; -+ } -+#endif -+ - if (kbdev->pm.backend.gpu_powered) { - /* Already turned on */ - if (kbdev->poweroff_pending) -@@ -1602,11 +2139,40 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) - kbdev->pm.backend.gpu_powered = true; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -+#if MALI_USE_CSF -+ /* GPU has been turned on, can switch to actual register page */ -+ update_user_reg_page_mapping(kbdev); -+#endif -+ - if (reset_required) { - /* GPU state was lost, reset GPU to ensure it is in a -- * consistent state */ -+ * consistent state -+ */ - kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS); - } -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ else { -+ if (kbdev->arb.arb_if) { -+ struct kbase_arbiter_vm_state *arb_vm_state = -+ kbdev->pm.arb_vm_state; -+ -+ /* In the case that the GPU has just been granted by -+ * the Arbiter, a reset will have already been done. -+ * However, it is still necessary to initialize the GPU. -+ */ -+ if (arb_vm_state->vm_arb_starting) -+ kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS | -+ PM_NO_RESET); -+ } -+ } -+ /* -+ * This point means that the GPU trasitioned to ON. So there is a chance -+ * that a repartitioning occurred. In this case the current config -+ * should be read again. -+ */ -+ kbase_gpuprops_get_curr_config_props(kbdev, -+ &kbdev->gpu_props.curr_config); -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ - - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -@@ -1628,7 +2194,19 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume) - - /* Turn on the L2 caches */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->pm.backend.gpu_ready = true; - kbdev->pm.backend.l2_desired = true; -+#if MALI_USE_CSF -+ if (reset_required) { -+ /* GPU reset was done after the power on, so send the post -+ * reset event instead. This is okay as GPU power off event -+ * is same as pre GPU reset event. -+ */ -+ kbase_ipa_control_handle_gpu_reset_post(kbdev); -+ } else { -+ kbase_ipa_control_handle_gpu_power_on(kbdev); -+ } -+#endif - kbase_pm_update_state(kbdev); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } -@@ -1639,7 +2217,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) - { - unsigned long flags; - -- KBASE_DEBUG_ASSERT(NULL != kbdev); -+ KBASE_DEBUG_ASSERT(kbdev != NULL); - lockdep_assert_held(&kbdev->pm.lock); - - /* ASSERT that the cores should now be unavailable. No lock needed. */ -@@ -1663,16 +2241,38 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev) - - if (atomic_read(&kbdev->faults_pending)) { - /* Page/bus faults are still being processed. The GPU can not -- * be powered off until they have completed */ -+ * be powered off until they have completed -+ */ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - return false; - } - - kbase_pm_cache_snoop_disable(kbdev); -+#if MALI_USE_CSF -+ kbase_ipa_control_handle_gpu_power_off(kbdev); -+#endif -+ -+ kbdev->pm.backend.gpu_ready = false; - - /* The GPU power may be turned off from this point */ - kbdev->pm.backend.gpu_powered = false; -+ -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ if (kbase_pm_is_gpu_lost(kbdev)) { -+ /* Ensure we unblock any threads that are stuck waiting -+ * for the GPU -+ */ -+ kbase_gpu_cache_clean_wait_complete(kbdev); -+ } -+#endif -+ - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+#if MALI_USE_CSF -+ /* GPU is about to be turned off, switch to dummy page */ -+ update_user_reg_page_mapping(kbdev); -+#endif -+ - #ifdef CONFIG_MALI_ARBITER_SUPPORT - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT); - #endif /* CONFIG_MALI_ARBITER_SUPPORT */ -@@ -1720,19 +2320,23 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer) - struct kbasep_reset_timeout_data *rtdata = - container_of(timer, struct kbasep_reset_timeout_data, timer); - -- rtdata->timed_out = 1; -+ rtdata->timed_out = true; - - /* Set the wait queue to wake up kbase_pm_init_hw even though the reset -- * hasn't completed */ -+ * hasn't completed -+ */ - kbase_pm_reset_done(rtdata->kbdev); - - return HRTIMER_NORESTART; - } - --static int kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id) -+static int kbase_set_gpu_quirks(struct kbase_device *kbdev, const u32 prod_id) - { -- u32 hw_quirks_jm = kbase_reg_read(kbdev, -- GPU_CONTROL_REG(JM_CONFIG)); -+#if MALI_USE_CSF -+ kbdev->hw_quirks_gpu = -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(CSF_CONFIG)); -+#else -+ u32 hw_quirks_gpu = kbase_reg_read(kbdev, GPU_CONTROL_REG(JM_CONFIG)); - - if (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == GPU_ID2_PRODUCT_TMIX) { - /* Only for tMIx */ -@@ -1746,38 +2350,38 @@ static int kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id) - */ - if (coherency_features == - COHERENCY_FEATURE_BIT(COHERENCY_ACE)) { -- hw_quirks_jm |= (COHERENCY_ACE_LITE | -- COHERENCY_ACE) << -- JM_FORCE_COHERENCY_FEATURES_SHIFT; -+ hw_quirks_gpu |= (COHERENCY_ACE_LITE | COHERENCY_ACE) -+ << JM_FORCE_COHERENCY_FEATURES_SHIFT; - } - } - -- if (kbase_is_gpu_lost(kbdev)) -+ if (kbase_is_gpu_removed(kbdev)) - return -EIO; - -- kbdev->hw_quirks_jm = hw_quirks_jm; -+ kbdev->hw_quirks_gpu = hw_quirks_gpu; - -+#endif /* !MALI_USE_CSF */ - if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) { - int default_idvs_group_size = 0xF; -- u32 tmp; -+ u32 group_size = 0; - -- if (of_property_read_u32(kbdev->dev->of_node, -- "idvs-group-size", &tmp)) -- tmp = default_idvs_group_size; -+ if (of_property_read_u32(kbdev->dev->of_node, "idvs-group-size", -+ &group_size)) -+ group_size = default_idvs_group_size; - -- if (tmp > IDVS_GROUP_MAX_SIZE) { -+ if (group_size > IDVS_GROUP_MAX_SIZE) { - dev_err(kbdev->dev, - "idvs-group-size of %d is too large. Maximum value is %d", -- tmp, IDVS_GROUP_MAX_SIZE); -- tmp = default_idvs_group_size; -+ group_size, IDVS_GROUP_MAX_SIZE); -+ group_size = default_idvs_group_size; - } - -- kbdev->hw_quirks_jm |= tmp << IDVS_GROUP_SIZE_SHIFT; -+ kbdev->hw_quirks_gpu |= group_size << IDVS_GROUP_SIZE_SHIFT; - } - - #define MANUAL_POWER_CONTROL ((u32)(1 << 8)) - if (corestack_driver_control) -- kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL; -+ kbdev->hw_quirks_gpu |= MANUAL_POWER_CONTROL; - - return 0; - } -@@ -1787,7 +2391,7 @@ static int kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id) - u32 hw_quirks_sc = kbase_reg_read(kbdev, - GPU_CONTROL_REG(SHADER_CONFIG)); - -- if (kbase_is_gpu_lost(kbdev)) -+ if (kbase_is_gpu_removed(kbdev)) - return -EIO; - - if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */ -@@ -1811,7 +2415,7 @@ static int kbase_set_tiler_quirks(struct kbase_device *kbdev) - u32 hw_quirks_tiler = kbase_reg_read(kbdev, - GPU_CONTROL_REG(TILER_CONFIG)); - -- if (kbase_is_gpu_lost(kbdev)) -+ if (kbase_is_gpu_removed(kbdev)) - return -EIO; - - /* Set tiler clock gate override if required */ -@@ -1831,18 +2435,17 @@ static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev) - GPU_ID_VERSION_PRODUCT_ID_SHIFT; - int error = 0; - -- kbdev->hw_quirks_jm = 0; -+ kbdev->hw_quirks_gpu = 0; - kbdev->hw_quirks_sc = 0; - kbdev->hw_quirks_tiler = 0; - kbdev->hw_quirks_mmu = 0; - -- if (!of_property_read_u32(np, "quirks_jm", -- &kbdev->hw_quirks_jm)) { -+ if (!of_property_read_u32(np, "quirks_gpu", &kbdev->hw_quirks_gpu)) { - dev_info(kbdev->dev, -- "Found quirks_jm = [0x%x] in Devicetree\n", -- kbdev->hw_quirks_jm); -+ "Found quirks_gpu = [0x%x] in Devicetree\n", -+ kbdev->hw_quirks_gpu); - } else { -- error = kbase_set_jm_quirks(kbdev, prod_id); -+ error = kbase_set_gpu_quirks(kbdev, prod_id); - if (error) - return error; - } -@@ -1891,15 +2494,20 @@ static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev) - - kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), - kbdev->hw_quirks_mmu); -+#if MALI_USE_CSF -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(CSF_CONFIG), -+ kbdev->hw_quirks_gpu); -+#else - kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG), -- kbdev->hw_quirks_jm); -+ kbdev->hw_quirks_gpu); -+#endif - } - - void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) - { - if ((kbdev->current_gpu_coherency_mode == COHERENCY_ACE) && - !kbdev->cci_snoop_enabled) { --#ifdef CONFIG_ARM64 -+#if IS_ENABLED(CONFIG_ARM64) - if (kbdev->snoop_enable_smc != 0) - kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0); - #endif /* CONFIG_ARM64 */ -@@ -1911,7 +2519,7 @@ void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev) - void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) - { - if (kbdev->cci_snoop_enabled) { --#ifdef CONFIG_ARM64 -+#if IS_ENABLED(CONFIG_ARM64) - if (kbdev->snoop_disable_smc != 0) { - mali_cci_flush_l2(kbdev); - kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0); -@@ -1922,6 +2530,7 @@ void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev) - } - } - -+#if !MALI_USE_CSF - static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev) - { - unsigned long irq_flags; -@@ -1934,6 +2543,7 @@ static void reenable_protected_mode_hwcnt(struct kbase_device *kbdev) - } - spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); - } -+#endif - - static int kbase_pm_do_reset(struct kbase_device *kbdev) - { -@@ -1960,7 +2570,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) - - /* Initialize a structure for tracking the status of the reset */ - rtdata.kbdev = kbdev; -- rtdata.timed_out = 0; -+ rtdata.timed_out = false; - - /* Create a timer to use as a timeout on the reset */ - hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); -@@ -1972,7 +2582,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) - /* Wait for the RESET_COMPLETED interrupt to be raised */ - kbase_pm_wait_for_reset(kbdev); - -- if (rtdata.timed_out == 0) { -+ if (!rtdata.timed_out) { - /* GPU has been reset */ - hrtimer_cancel(&rtdata.timer); - destroy_hrtimer_on_stack(&rtdata.timer); -@@ -1980,46 +2590,60 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev) - } - - /* No interrupt has been received - check if the RAWSTAT register says -- * the reset has completed */ -+ * the reset has completed -+ */ - if ((kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) & -- RESET_COMPLETED) -- || kbase_is_gpu_lost(kbdev)) { -+ RESET_COMPLETED)) { - /* The interrupt is set in the RAWSTAT; this suggests that the -- * interrupts are not getting to the CPU */ -+ * interrupts are not getting to the CPU -+ */ - dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n"); - /* If interrupts aren't working we can't continue. */ - destroy_hrtimer_on_stack(&rtdata.timer); - return -EINVAL; - } - -+ if (kbase_is_gpu_removed(kbdev)) { -+ dev_dbg(kbdev->dev, "GPU has been removed, reset no longer needed.\n"); -+ destroy_hrtimer_on_stack(&rtdata.timer); -+ return -EINVAL; -+ } -+ - /* The GPU doesn't seem to be responding to the reset so try a hard -- * reset */ -- dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", -- RESET_TIMEOUT); -- KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0); -- kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -- GPU_COMMAND_HARD_RESET); -+ * reset, but only when NOT in arbitration mode. -+ */ -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ if (!kbdev->arb.arb_if) { -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ -+ dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", -+ RESET_TIMEOUT); -+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_HARD_RESET); - -- /* Restart the timer to wait for the hard reset to complete */ -- rtdata.timed_out = 0; -+ /* Restart the timer to wait for the hard reset to complete */ -+ rtdata.timed_out = false; - -- hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), -- HRTIMER_MODE_REL); -+ hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), -+ HRTIMER_MODE_REL); - -- /* Wait for the RESET_COMPLETED interrupt to be raised */ -- kbase_pm_wait_for_reset(kbdev); -+ /* Wait for the RESET_COMPLETED interrupt to be raised */ -+ kbase_pm_wait_for_reset(kbdev); - -- if (rtdata.timed_out == 0) { -- /* GPU has been reset */ -- hrtimer_cancel(&rtdata.timer); -- destroy_hrtimer_on_stack(&rtdata.timer); -- return 0; -- } -+ if (!rtdata.timed_out) { -+ /* GPU has been reset */ -+ hrtimer_cancel(&rtdata.timer); -+ destroy_hrtimer_on_stack(&rtdata.timer); -+ return 0; -+ } - -- destroy_hrtimer_on_stack(&rtdata.timer); -+ destroy_hrtimer_on_stack(&rtdata.timer); - -- dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", -- RESET_TIMEOUT); -+ dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n", -+ RESET_TIMEOUT); -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ } -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ - - return -EINVAL; - } -@@ -2041,9 +2665,9 @@ int kbase_pm_protected_mode_disable(struct kbase_device *const kbdev) - int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) - { - unsigned long irq_flags; -- int err; -+ int err = 0; - -- KBASE_DEBUG_ASSERT(NULL != kbdev); -+ KBASE_DEBUG_ASSERT(kbdev != NULL); - lockdep_assert_held(&kbdev->pm.lock); - - /* Ensure the clock is on before attempting to access the hardware */ -@@ -2055,7 +2679,8 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) - } - - /* Ensure interrupts are off to begin with, this also clears any -- * outstanding interrupts */ -+ * outstanding interrupts -+ */ - kbase_pm_disable_interrupts(kbdev); - /* Ensure cache snoops are disabled before reset. */ - kbase_pm_cache_snoop_disable(kbdev); -@@ -2069,10 +2694,24 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) - spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); - - /* Soft reset the GPU */ -- err = kbdev->protected_ops->protected_mode_disable( -- kbdev->protected_dev); -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ if (!(flags & PM_NO_RESET)) -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ -+ err = kbdev->protected_ops->protected_mode_disable( -+ kbdev->protected_dev); - - spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -+#if MALI_USE_CSF -+ if (kbdev->protected_mode) { -+ unsigned long flags; -+ -+ kbase_ipa_control_protm_exited(kbdev); -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ kbase_hwcnt_backend_csf_protm_exited(&kbdev->hwcnt_gpu_iface); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ } -+#endif - kbdev->protected_mode = false; - spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); - -@@ -2093,7 +2732,8 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) - GPU_STATUS_PROTECTED_MODE_ACTIVE); - - /* If cycle counter was in use re-enable it, enable_irqs will only be -- * false when called from kbase_pm_powerup */ -+ * false when called from kbase_pm_powerup -+ */ - if (kbdev->pm.backend.gpu_cycle_counter_requests && - (flags & PM_ENABLE_IRQS)) { - kbase_pm_enable_interrupts(kbdev); -@@ -2116,12 +2756,14 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags) - kbase_pm_enable_interrupts(kbdev); - - exit: -+#if !MALI_USE_CSF - if (!kbdev->pm.backend.protected_entry_transition_override) { - /* Re-enable GPU hardware counters if we're resetting from - * protected mode. - */ - reenable_protected_mode_hwcnt(kbdev); - } -+#endif - - return err; - } -@@ -2148,12 +2790,21 @@ kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev) - - spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock, - flags); -- - ++kbdev->pm.backend.gpu_cycle_counter_requests; - -- if (1 == kbdev->pm.backend.gpu_cycle_counter_requests) -+ if (kbdev->pm.backend.gpu_cycle_counter_requests == 1) - kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), - GPU_COMMAND_CYCLE_COUNT_START); -+ else { -+ /* This might happen after GPU reset. -+ * Then counter needs to be kicked. -+ */ -+ if (!(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) & -+ GPU_STATUS_CYCLE_COUNT_ACTIVE)) { -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CYCLE_COUNT_START); -+ } -+ } - - spin_unlock_irqrestore( - &kbdev->pm.backend.gpu_cycle_counter_requests_lock, -@@ -2169,6 +2820,8 @@ void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev) - KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests < - INT_MAX); - -+ kbase_pm_wait_for_l2_powered(kbdev); -+ - kbase_pm_request_gpu_cycle_counter_do_request(kbdev); - } - -@@ -2203,7 +2856,7 @@ void kbase_pm_release_gpu_cycle_counter_nolock(struct kbase_device *kbdev) - - --kbdev->pm.backend.gpu_cycle_counter_requests; - -- if (0 == kbdev->pm.backend.gpu_cycle_counter_requests) -+ if (kbdev->pm.backend.gpu_cycle_counter_requests == 0) - kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), - GPU_COMMAND_CYCLE_COUNT_STOP); - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h -index 95f10e0..70d009e 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_internal.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Power management API definitions used internally by GPU backend - */ -@@ -31,7 +28,7 @@ - - #include <mali_kbase_hwaccess_pm.h> - --#include "mali_kbase_pm_ca.h" -+#include "backend/gpu/mali_kbase_pm_ca.h" - #include "mali_kbase_pm_policy.h" - - -@@ -205,6 +202,30 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags); - */ - void kbase_pm_reset_done(struct kbase_device *kbdev); - -+#if MALI_USE_CSF -+/** -+ * kbase_pm_wait_for_desired_state - Wait for the desired power state to be -+ * reached -+ * -+ * Wait for the L2 and MCU state machines to reach the states corresponding -+ * to the values of 'kbase_pm_is_l2_desired' and 'kbase_pm_is_mcu_desired'. -+ * -+ * The usual use-case for this is to ensure that all parts of GPU have been -+ * powered up after performing a GPU Reset. -+ * -+ * Unlike kbase_pm_update_state(), the caller must not hold hwaccess_lock, -+ * because this function will take that lock itself. -+ * -+ * NOTE: This may not wait until the correct state is reached if there is a -+ * power off in progress and kbase_pm_context_active() was called instead of -+ * kbase_csf_scheduler_pm_active(). -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Return: 0 on success, error code on error -+ */ -+int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); -+#else - /** - * kbase_pm_wait_for_desired_state - Wait for the desired power state to be - * reached -@@ -224,15 +245,17 @@ void kbase_pm_reset_done(struct kbase_device *kbdev); - * kbase_pm_wait_for_poweroff_complete() - * - * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Return: 0 on success, error code on error - */ --void kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); -+int kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); -+#endif - - /** - * kbase_pm_wait_for_l2_powered - Wait for the L2 cache to be powered on - * -- * Wait for the L2 to be powered on, and for the L2 and shader state machines to -- * stabilise by reaching the states corresponding to the values of 'l2_desired' -- * and 'shaders_desired'. -+ * Wait for the L2 to be powered on, and for the L2 and the state machines of -+ * its dependent stack components to stabilise. - * - * kbdev->pm.active_count must be non-zero when calling this function. - * -@@ -240,8 +263,10 @@ void kbase_pm_wait_for_desired_state(struct kbase_device *kbdev); - * because this function will take that lock itself. - * - * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * -+ * Return: 0 on success, error code on error - */ --void kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev); -+int kbase_pm_wait_for_l2_powered(struct kbase_device *kbdev); - - /** - * kbase_pm_update_dynamic_cores_onoff - Update the L2 and shader power state -@@ -467,7 +492,8 @@ void kbase_pm_register_access_enable(struct kbase_device *kbdev); - void kbase_pm_register_access_disable(struct kbase_device *kbdev); - - /* NOTE: kbase_pm_is_suspending is in mali_kbase.h, because it is an inline -- * function */ -+ * function -+ */ - - /** - * kbase_pm_metrics_is_active - Check if the power management metrics -@@ -511,8 +537,22 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, - - #ifdef CONFIG_MALI_MIDGARD_DVFS - -+#if MALI_USE_CSF -+/** -+ * kbase_platform_dvfs_event - Report utilisation to DVFS code for CSF GPU -+ * -+ * Function provided by platform specific code when DVFS is enabled to allow -+ * the power management metrics system to report utilisation. -+ * -+ * @kbdev: The kbase device structure for the device (must be a -+ * valid pointer) -+ * @utilisation: The current calculated utilisation by the metrics system. -+ * Return: Returns 0 on failure and non zero on success. -+ */ -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation); -+#else - /** -- * kbase_platform_dvfs_event - Report utilisation to DVFS code -+ * kbase_platform_dvfs_event - Report utilisation to DVFS code for JM GPU - * - * Function provided by platform specific code when DVFS is enabled to allow - * the power management metrics system to report utilisation. -@@ -525,11 +565,12 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, - * group. - * Return: Returns 0 on failure and non zero on success. - */ -- - int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, -- u32 util_gl_share, u32 util_cl_share[2]); -+ u32 util_gl_share, u32 util_cl_share[2]); - #endif - -+#endif /* CONFIG_MALI_MIDGARD_DVFS */ -+ - void kbase_pm_power_changed(struct kbase_device *kbdev); - - /** -@@ -683,6 +724,72 @@ extern bool corestack_driver_control; - */ - bool kbase_pm_is_l2_desired(struct kbase_device *kbdev); - -+#if MALI_USE_CSF -+/** -+ * kbase_pm_is_mcu_desired - Check whether MCU is desired -+ * -+ * @kbdev: Device pointer -+ * -+ * This shall be called to check whether MCU needs to be enabled. -+ * -+ * Return: true if MCU needs to be enabled. -+ */ -+bool kbase_pm_is_mcu_desired(struct kbase_device *kbdev); -+ -+/** -+ * kbase_pm_idle_groups_sched_suspendable - Check whether the scheduler can be -+ * suspended to low power state when all -+ * the CSGs are idle -+ * -+ * @kbdev: Device pointer -+ * -+ * Return: true if allowed to enter the suspended state. -+ */ -+static inline -+bool kbase_pm_idle_groups_sched_suspendable(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ return !(kbdev->pm.backend.csf_pm_sched_flags & -+ CSF_DYNAMIC_PM_SCHED_IGNORE_IDLE); -+} -+ -+/** -+ * kbase_pm_no_runnables_sched_suspendable - Check whether the scheduler can be -+ * suspended to low power state when -+ * there are no runnable CSGs. -+ * -+ * @kbdev: Device pointer -+ * -+ * Return: true if allowed to enter the suspended state. -+ */ -+static inline -+bool kbase_pm_no_runnables_sched_suspendable(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ return !(kbdev->pm.backend.csf_pm_sched_flags & -+ CSF_DYNAMIC_PM_SCHED_NO_SUSPEND); -+} -+ -+/** -+ * kbase_pm_no_mcu_core_pwroff - Check whether the PM is required to keep the -+ * MCU core powered in accordance to the active -+ * power management policy -+ * -+ * @kbdev: Device pointer -+ * -+ * Return: true if the MCU is to retain powered. -+ */ -+static inline bool kbase_pm_no_mcu_core_pwroff(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ return kbdev->pm.backend.csf_pm_sched_flags & -+ CSF_DYNAMIC_PM_CORE_KEEP_ON; -+} -+#endif -+ - /** - * kbase_pm_lock - Lock all necessary mutexes to perform PM actions - * -@@ -692,7 +799,9 @@ bool kbase_pm_is_l2_desired(struct kbase_device *kbdev); - */ - static inline void kbase_pm_lock(struct kbase_device *kbdev) - { -+#if !MALI_USE_CSF - mutex_lock(&kbdev->js_data.runpool_mutex); -+#endif /* !MALI_USE_CSF */ - mutex_lock(&kbdev->pm.lock); - } - -@@ -704,7 +813,9 @@ static inline void kbase_pm_lock(struct kbase_device *kbdev) - static inline void kbase_pm_unlock(struct kbase_device *kbdev) - { - mutex_unlock(&kbdev->pm.lock); -+#if !MALI_USE_CSF - mutex_unlock(&kbdev->js_data.runpool_mutex); -+#endif /* !MALI_USE_CSF */ - } - - #endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_l2_states.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_l2_states.h -index 12cb051..ef72f60 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_l2_states.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_l2_states.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -25,6 +24,19 @@ - * The function-like macro KBASEP_L2_STATE() must be defined before including - * this header file. This header file can be included multiple times in the - * same compilation unit with different definitions of KBASEP_L2_STATE(). -+ * -+ * @OFF: The L2 cache and tiler are off -+ * @PEND_ON: The L2 cache and tiler are powering on -+ * @RESTORE_CLOCKS: The GPU clock is restored. Conditionally used. -+ * @ON_HWCNT_ENABLE: The L2 cache and tiler are on, and hwcnt is being enabled -+ * @ON: The L2 cache and tiler are on, and hwcnt is enabled -+ * @ON_HWCNT_DISABLE: The L2 cache and tiler are on, and hwcnt is being disabled -+ * @SLOW_DOWN_CLOCKS: The GPU clock is set to appropriate or lowest clock. -+ * Conditionally used. -+ * @POWER_DOWN: The L2 cache and tiler are about to be powered off -+ * @PEND_OFF: The L2 cache and tiler are powering off -+ * @RESET_WAIT: The GPU is resetting, L2 cache and tiler power state are -+ * unknown - */ - KBASEP_L2_STATE(OFF) - KBASEP_L2_STATE(PEND_ON) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_mcu_states.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_mcu_states.h -new file mode 100644 -index 0000000..4e99928 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_mcu_states.h -@@ -0,0 +1,63 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * Backend-specific Power Manager MCU state definitions. -+ * The function-like macro KBASEP_MCU_STATE() must be defined before including -+ * this header file. This header file can be included multiple times in the -+ * same compilation unit with different definitions of KBASEP_MCU_STATE(). -+ * -+ * @OFF: The MCU is powered off. -+ * @PEND_ON_RELOAD: The warm boot of MCU or cold boot of MCU (with -+ * firmware reloading) is in progress. -+ * @ON_GLB_REINIT_PEND: The MCU is enabled and Global configuration -+ * requests have been sent to the firmware. -+ * @ON_HWCNT_ENABLE: The Global requests have completed and MCU is now -+ * ready for use and hwcnt is being enabled. -+ * @ON: The MCU is active and hwcnt has been enabled. -+ * @ON_CORE_ATTR_UPDATE_PEND: The MCU is active and mask of enabled shader cores -+ * is being updated. -+ * @ON_HWCNT_DISABLE: The MCU is on and hwcnt is being disabled. -+ * @ON_HALT: The MCU is on and hwcnt has been disabled, MCU -+ * halt would be triggered. -+ * @ON_PEND_HALT: MCU halt in progress, confirmation pending. -+ * @POWER_DOWN: MCU halted operations, pending being disabled. -+ * @PEND_OFF: MCU is being disabled, pending on powering off. -+ * @RESET_WAIT: The GPU is resetting, MCU state is unknown. -+ */ -+KBASEP_MCU_STATE(OFF) -+KBASEP_MCU_STATE(PEND_ON_RELOAD) -+KBASEP_MCU_STATE(ON_GLB_REINIT_PEND) -+KBASEP_MCU_STATE(ON_HWCNT_ENABLE) -+KBASEP_MCU_STATE(ON) -+KBASEP_MCU_STATE(ON_CORE_ATTR_UPDATE_PEND) -+KBASEP_MCU_STATE(ON_HWCNT_DISABLE) -+KBASEP_MCU_STATE(ON_HALT) -+KBASEP_MCU_STATE(ON_PEND_HALT) -+KBASEP_MCU_STATE(POWER_DOWN) -+KBASEP_MCU_STATE(PEND_OFF) -+KBASEP_MCU_STATE(RESET_WAIT) -+/* Additional MCU states with HOST_CONTROL_SHADERS */ -+KBASEP_MCU_STATE(HCTL_SHADERS_PEND_ON) -+KBASEP_MCU_STATE(HCTL_CORES_NOTIFY_PEND) -+KBASEP_MCU_STATE(HCTL_MCU_ON_RECHECK) -+KBASEP_MCU_STATE(HCTL_SHADERS_READY_OFF) -+KBASEP_MCU_STATE(HCTL_SHADERS_PEND_OFF) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c -index de3babe..69e8dd3 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_metrics.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Metrics for power management - */ -@@ -29,22 +26,28 @@ - #include <mali_kbase.h> - #include <mali_kbase_pm.h> - #include <backend/gpu/mali_kbase_pm_internal.h> -+ -+#if MALI_USE_CSF -+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" -+#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> -+#else - #include <backend/gpu/mali_kbase_jm_rb.h> -+#endif /* !MALI_USE_CSF */ -+ - #include <backend/gpu/mali_kbase_pm_defs.h> - #include <mali_linux_trace.h> - --/* When VSync is being hit aim for utilisation between 70-90% */ --#define KBASE_PM_VSYNC_MIN_UTILISATION 70 --#define KBASE_PM_VSYNC_MAX_UTILISATION 90 --/* Otherwise aim for 10-40% */ --#define KBASE_PM_NO_VSYNC_MIN_UTILISATION 10 --#define KBASE_PM_NO_VSYNC_MAX_UTILISATION 40 -- - /* Shift used for kbasep_pm_metrics_data.time_busy/idle - units of (1 << 8) ns - * This gives a maximum period between samples of 2^(32+8)/100 ns = slightly -- * under 11s. Exceeding this will cause overflow */ -+ * under 11s. Exceeding this will cause overflow -+ */ - #define KBASE_PM_TIME_SHIFT 8 - -+#if MALI_USE_CSF -+/* To get the GPU_ACTIVE value in nano seconds unit */ -+#define GPU_ACTIVE_SCALING_FACTOR ((u64)1E9) -+#endif -+ - #ifdef CONFIG_MALI_MIDGARD_DVFS - static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) - { -@@ -71,11 +74,45 @@ static enum hrtimer_restart dvfs_callback(struct hrtimer *timer) - - int kbasep_pm_metrics_init(struct kbase_device *kbdev) - { -- KBASE_DEBUG_ASSERT(kbdev != NULL); -+#if MALI_USE_CSF -+ struct kbase_ipa_control_perf_counter perf_counter; -+ int err; - -+ /* One counter group */ -+ const size_t NUM_PERF_COUNTERS = 1; -+ -+ KBASE_DEBUG_ASSERT(kbdev != NULL); - kbdev->pm.backend.metrics.kbdev = kbdev; -+ kbdev->pm.backend.metrics.time_period_start = ktime_get(); -+ kbdev->pm.backend.metrics.values.time_busy = 0; -+ kbdev->pm.backend.metrics.values.time_idle = 0; -+ kbdev->pm.backend.metrics.values.time_in_protm = 0; -+ -+ perf_counter.scaling_factor = GPU_ACTIVE_SCALING_FACTOR; - -+ /* Normalize values by GPU frequency */ -+ perf_counter.gpu_norm = true; -+ -+ /* We need the GPU_ACTIVE counter, which is in the CSHW group */ -+ perf_counter.type = KBASE_IPA_CORE_TYPE_CSHW; -+ -+ /* We need the GPU_ACTIVE counter */ -+ perf_counter.idx = GPU_ACTIVE_CNT_IDX; -+ -+ err = kbase_ipa_control_register( -+ kbdev, &perf_counter, NUM_PERF_COUNTERS, -+ &kbdev->pm.backend.metrics.ipa_control_client); -+ if (err) { -+ dev_err(kbdev->dev, -+ "Failed to register IPA with kbase_ipa_control: err=%d", -+ err); -+ return -1; -+ } -+#else -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ kbdev->pm.backend.metrics.kbdev = kbdev; - kbdev->pm.backend.metrics.time_period_start = ktime_get(); -+ - kbdev->pm.backend.metrics.gpu_active = false; - kbdev->pm.backend.metrics.active_cl_ctx[0] = 0; - kbdev->pm.backend.metrics.active_cl_ctx[1] = 0; -@@ -89,16 +126,25 @@ int kbasep_pm_metrics_init(struct kbase_device *kbdev) - kbdev->pm.backend.metrics.values.busy_cl[1] = 0; - kbdev->pm.backend.metrics.values.busy_gl = 0; - -+#endif - spin_lock_init(&kbdev->pm.backend.metrics.lock); - - #ifdef CONFIG_MALI_MIDGARD_DVFS - hrtimer_init(&kbdev->pm.backend.metrics.timer, CLOCK_MONOTONIC, - HRTIMER_MODE_REL); - kbdev->pm.backend.metrics.timer.function = dvfs_callback; -- -+ kbdev->pm.backend.metrics.initialized = true; - kbase_pm_metrics_start(kbdev); - #endif /* CONFIG_MALI_MIDGARD_DVFS */ - -+#if MALI_USE_CSF -+ /* The sanity check on the GPU_ACTIVE performance counter -+ * is skipped for Juno platforms that have timing problems. -+ */ -+ kbdev->pm.backend.metrics.skip_gpu_active_sanity_check = -+ of_machine_is_compatible("arm,juno"); -+#endif -+ - return 0; - } - KBASE_EXPORT_TEST_API(kbasep_pm_metrics_init); -@@ -115,7 +161,13 @@ void kbasep_pm_metrics_term(struct kbase_device *kbdev) - spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); - - hrtimer_cancel(&kbdev->pm.backend.metrics.timer); -+ kbdev->pm.backend.metrics.initialized = false; - #endif /* CONFIG_MALI_MIDGARD_DVFS */ -+ -+#if MALI_USE_CSF -+ kbase_ipa_control_unregister( -+ kbdev, kbdev->pm.backend.metrics.ipa_control_client); -+#endif - } - - KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); -@@ -123,8 +175,117 @@ KBASE_EXPORT_TEST_API(kbasep_pm_metrics_term); - /* caller needs to hold kbdev->pm.backend.metrics.lock before calling this - * function - */ -+#if MALI_USE_CSF -+#if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) -+static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev) -+{ -+ int err; -+ u64 gpu_active_counter; -+ u64 protected_time; -+ ktime_t now; -+ -+ lockdep_assert_held(&kbdev->pm.backend.metrics.lock); -+ -+ /* Query IPA_CONTROL for the latest GPU-active and protected-time -+ * info. -+ */ -+ err = kbase_ipa_control_query( -+ kbdev, kbdev->pm.backend.metrics.ipa_control_client, -+ &gpu_active_counter, 1, &protected_time); -+ -+ /* Read the timestamp after reading the GPU_ACTIVE counter value. -+ * This ensures the time gap between the 2 reads is consistent for -+ * a meaningful comparison between the increment of GPU_ACTIVE and -+ * elapsed time. The lock taken inside kbase_ipa_control_query() -+ * function can cause lot of variation. -+ */ -+ now = ktime_get(); -+ -+ if (err) { -+ dev_err(kbdev->dev, -+ "Failed to query the increment of GPU_ACTIVE counter: err=%d", -+ err); -+ } else { -+ u64 diff_ns; -+ s64 diff_ns_signed; -+ u32 ns_time; -+ ktime_t diff = ktime_sub( -+ now, kbdev->pm.backend.metrics.time_period_start); -+ -+ diff_ns_signed = ktime_to_ns(diff); -+ -+ if (diff_ns_signed < 0) -+ return; -+ -+ diff_ns = (u64)diff_ns_signed; -+ -+#if !IS_ENABLED(CONFIG_MALI_NO_MALI) -+ /* The GPU_ACTIVE counter shouldn't clock-up more time than has -+ * actually elapsed - but still some margin needs to be given -+ * when doing the comparison. There could be some drift between -+ * the CPU and GPU clock. -+ * -+ * Can do the check only in a real driver build, as an arbitrary -+ * value for GPU_ACTIVE can be fed into dummy model in no_mali -+ * configuration which may not correspond to the real elapsed -+ * time. -+ */ -+ if (!kbdev->pm.backend.metrics.skip_gpu_active_sanity_check) { -+ /* Use a margin value that is approximately 1% of the time -+ * difference. -+ */ -+ u64 margin_ns = diff_ns >> 6; -+ if (gpu_active_counter > (diff_ns + margin_ns)) { -+ dev_info( -+ kbdev->dev, -+ "GPU activity takes longer than time interval: %llu ns > %llu ns", -+ (unsigned long long)gpu_active_counter, -+ (unsigned long long)diff_ns); -+ } -+ } -+#endif -+ /* Calculate time difference in units of 256ns */ -+ ns_time = (u32)(diff_ns >> KBASE_PM_TIME_SHIFT); -+ -+ /* Add protected_time to gpu_active_counter so that time in -+ * protected mode is included in the apparent GPU active time, -+ * then convert it from units of 1ns to units of 256ns, to -+ * match what JM GPUs use. The assumption is made here that the -+ * GPU is 100% busy while in protected mode, so we should add -+ * this since the GPU can't (and thus won't) update these -+ * counters while it's actually in protected mode. -+ * -+ * Perform the add after dividing each value down, to reduce -+ * the chances of overflows. -+ */ -+ protected_time >>= KBASE_PM_TIME_SHIFT; -+ gpu_active_counter >>= KBASE_PM_TIME_SHIFT; -+ gpu_active_counter += protected_time; -+ -+ /* Ensure the following equations don't go wrong if ns_time is -+ * slightly larger than gpu_active_counter somehow -+ */ -+ gpu_active_counter = MIN(gpu_active_counter, ns_time); -+ -+ kbdev->pm.backend.metrics.values.time_busy += -+ gpu_active_counter; -+ -+ kbdev->pm.backend.metrics.values.time_idle += -+ ns_time - gpu_active_counter; -+ -+ /* Also make time in protected mode available explicitly, -+ * so users of this data have this info, too. -+ */ -+ kbdev->pm.backend.metrics.values.time_in_protm += -+ protected_time; -+ } -+ -+ kbdev->pm.backend.metrics.time_period_start = now; -+} -+#endif /* defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) */ -+#else - static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, -- ktime_t now) -+ ktime_t now) - { - ktime_t diff; - -@@ -149,12 +310,13 @@ static void kbase_pm_get_dvfs_utilisation_calc(struct kbase_device *kbdev, - if (kbdev->pm.backend.metrics.active_gl_ctx[2]) - kbdev->pm.backend.metrics.values.busy_gl += ns_time; - } else { -- kbdev->pm.backend.metrics.values.time_idle += (u32) (ktime_to_ns(diff) -- >> KBASE_PM_TIME_SHIFT); -+ kbdev->pm.backend.metrics.values.time_idle += -+ (u32)(ktime_to_ns(diff) >> KBASE_PM_TIME_SHIFT); - } - - kbdev->pm.backend.metrics.time_period_start = now; - } -+#endif /* MALI_USE_CSF */ - - #if defined(CONFIG_MALI_DEVFREQ) || defined(CONFIG_MALI_MIDGARD_DVFS) - void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, -@@ -165,14 +327,23 @@ void kbase_pm_get_dvfs_metrics(struct kbase_device *kbdev, - unsigned long flags; - - spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -+#if MALI_USE_CSF -+ kbase_pm_get_dvfs_utilisation_calc(kbdev); -+#else - kbase_pm_get_dvfs_utilisation_calc(kbdev, ktime_get()); -+#endif - - memset(diff, 0, sizeof(*diff)); - diff->time_busy = cur->time_busy - last->time_busy; - diff->time_idle = cur->time_idle - last->time_idle; -+ -+#if MALI_USE_CSF -+ diff->time_in_protm = cur->time_in_protm - last->time_in_protm; -+#else - diff->busy_cl[0] = cur->busy_cl[0] - last->busy_cl[0]; - diff->busy_cl[1] = cur->busy_cl[1] - last->busy_cl[1]; - diff->busy_gl = cur->busy_gl - last->busy_gl; -+#endif - - *last = *cur; - -@@ -184,26 +355,42 @@ KBASE_EXPORT_TEST_API(kbase_pm_get_dvfs_metrics); - #ifdef CONFIG_MALI_MIDGARD_DVFS - void kbase_pm_get_dvfs_action(struct kbase_device *kbdev) - { -- int utilisation, util_gl_share; -- int util_cl_share[2]; -- int busy; -+ int utilisation; - struct kbasep_pm_metrics *diff; -+#if !MALI_USE_CSF -+ int busy; -+ int util_gl_share; -+ int util_cl_share[2]; -+#endif - - KBASE_DEBUG_ASSERT(kbdev != NULL); - - diff = &kbdev->pm.backend.metrics.dvfs_diff; - -- kbase_pm_get_dvfs_metrics(kbdev, &kbdev->pm.backend.metrics.dvfs_last, diff); -+ kbase_pm_get_dvfs_metrics(kbdev, &kbdev->pm.backend.metrics.dvfs_last, -+ diff); - - utilisation = (100 * diff->time_busy) / - max(diff->time_busy + diff->time_idle, 1u); - -+#if !MALI_USE_CSF - busy = max(diff->busy_gl + diff->busy_cl[0] + diff->busy_cl[1], 1u); -+ - util_gl_share = (100 * diff->busy_gl) / busy; - util_cl_share[0] = (100 * diff->busy_cl[0]) / busy; - util_cl_share[1] = (100 * diff->busy_cl[1]) / busy; - -- kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, util_cl_share); -+ kbase_platform_dvfs_event(kbdev, utilisation, util_gl_share, -+ util_cl_share); -+#else -+ /* Note that, at present, we don't pass protected-mode time to the -+ * platform here. It's unlikely to be useful, however, as the platform -+ * probably just cares whether the GPU is busy or not; time in -+ * protected mode is already added to busy-time at this point, though, -+ * so we should be good. -+ */ -+ kbase_platform_dvfs_event(kbdev, utilisation); -+#endif - } - - bool kbase_pm_metrics_is_active(struct kbase_device *kbdev) -@@ -224,11 +411,20 @@ KBASE_EXPORT_TEST_API(kbase_pm_metrics_is_active); - void kbase_pm_metrics_start(struct kbase_device *kbdev) - { - unsigned long flags; -+ bool update = true; -+ -+ if (unlikely(!kbdev->pm.backend.metrics.initialized)) -+ return; - - spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -- kbdev->pm.backend.metrics.timer_active = true; -+ if (!kbdev->pm.backend.metrics.timer_active) -+ kbdev->pm.backend.metrics.timer_active = true; -+ else -+ update = false; - spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -- hrtimer_start(&kbdev->pm.backend.metrics.timer, -+ -+ if (update) -+ hrtimer_start(&kbdev->pm.backend.metrics.timer, - HR_TIMER_DELAY_MSEC(kbdev->pm.dvfs_period), - HRTIMER_MODE_REL); - } -@@ -236,16 +432,26 @@ void kbase_pm_metrics_start(struct kbase_device *kbdev) - void kbase_pm_metrics_stop(struct kbase_device *kbdev) - { - unsigned long flags; -+ bool update = true; -+ -+ if (unlikely(!kbdev->pm.backend.metrics.initialized)) -+ return; - - spin_lock_irqsave(&kbdev->pm.backend.metrics.lock, flags); -- kbdev->pm.backend.metrics.timer_active = false; -+ if (kbdev->pm.backend.metrics.timer_active) -+ kbdev->pm.backend.metrics.timer_active = false; -+ else -+ update = false; - spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); -- hrtimer_cancel(&kbdev->pm.backend.metrics.timer); -+ -+ if (update) -+ hrtimer_cancel(&kbdev->pm.backend.metrics.timer); - } - - - #endif /* CONFIG_MALI_MIDGARD_DVFS */ - -+#if !MALI_USE_CSF - /** - * kbase_pm_metrics_active_calc - Update PM active counts based on currently - * running atoms -@@ -270,7 +476,8 @@ static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev) - struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0); - - /* Head atom may have just completed, so if it isn't running -- * then try the next atom */ -+ * then try the next atom -+ */ - if (katom && katom->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) - katom = kbase_gpu_inspect(kbdev, js, 1); - -@@ -309,10 +516,12 @@ void kbase_pm_metrics_update(struct kbase_device *kbdev, ktime_t *timestamp) - timestamp = &now; - } - -- /* Track how long CL and/or GL jobs have been busy for */ -+ /* Track how much of time has been spent busy or idle. For JM GPUs, -+ * this also evaluates how long CL and/or GL jobs have been busy for. -+ */ - kbase_pm_get_dvfs_utilisation_calc(kbdev, *timestamp); - - kbase_pm_metrics_active_calc(kbdev); -- - spin_unlock_irqrestore(&kbdev->pm.backend.metrics.lock, flags); - } -+#endif /* !MALI_USE_CSF */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c -index 17ed21e..cf61ef8 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -28,27 +27,54 @@ - #include <gpu/mali_kbase_gpu_regmap.h> - #include <mali_kbase_pm.h> - #include <backend/gpu/mali_kbase_pm_internal.h> -+#include <mali_kbase_reset_gpu.h> - --static const struct kbase_pm_policy *const all_policy_list[] = { --#ifdef CONFIG_MALI_NO_MALI -- &kbase_pm_always_on_policy_ops, -- &kbase_pm_coarse_demand_policy_ops, --#if !MALI_CUSTOMER_RELEASE -- &kbase_pm_always_on_demand_policy_ops, -+#if MALI_USE_CSF && defined CONFIG_MALI_DEBUG -+#include <csf/mali_kbase_csf_firmware.h> - #endif --#else /* CONFIG_MALI_NO_MALI */ -+ -+#include <linux/of.h> -+ -+static const struct kbase_pm_policy *const all_policy_list[] = { - &kbase_pm_coarse_demand_policy_ops, --#if !MALI_CUSTOMER_RELEASE -- &kbase_pm_always_on_demand_policy_ops, --#endif - &kbase_pm_always_on_policy_ops --#endif /* CONFIG_MALI_NO_MALI */ - }; - - void kbase_pm_policy_init(struct kbase_device *kbdev) - { -- kbdev->pm.backend.pm_current_policy = all_policy_list[0]; -- kbdev->pm.backend.pm_current_policy->init(kbdev); -+ const struct kbase_pm_policy *default_policy = all_policy_list[0]; -+ struct device_node *np = kbdev->dev->of_node; -+ const char *power_policy_name; -+ unsigned long flags; -+ int i; -+ -+ if (of_property_read_string(np, "power_policy", &power_policy_name) == 0) { -+ for (i = 0; i < ARRAY_SIZE(all_policy_list); i++) -+ if (sysfs_streq(all_policy_list[i]->name, power_policy_name)) { -+ default_policy = all_policy_list[i]; -+ break; -+ } -+ } -+ -+#if MALI_USE_CSF && defined(CONFIG_MALI_DEBUG) -+ /* Use always_on policy if module param fw_debug=1 is -+ * passed, to aid firmware debugging. -+ */ -+ if (fw_debug) -+ default_policy = &kbase_pm_always_on_policy_ops; -+#endif -+ -+ default_policy->init(kbdev); -+ -+#if MALI_USE_CSF -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->pm.backend.pm_current_policy = default_policy; -+ kbdev->pm.backend.csf_pm_sched_flags = default_policy->pm_sched_flags; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+#else -+ CSTD_UNUSED(flags); -+ kbdev->pm.backend.pm_current_policy = default_policy; -+#endif - } - - void kbase_pm_policy_term(struct kbase_device *kbdev) -@@ -93,13 +119,17 @@ void kbase_pm_update_active(struct kbase_device *kbdev) - pm->backend.invoke_poweroff_wait_wq_when_l2_off = false; - pm->backend.poweroff_wait_in_progress = false; - pm->backend.l2_desired = true; -+#if MALI_USE_CSF -+ pm->backend.mcu_desired = true; -+#endif - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - kbase_pm_do_poweron(kbdev, false); - } - } else { - /* It is an error for the power policy to power off the GPU -- * when there are contexts active */ -+ * when there are contexts active -+ */ - KBASE_DEBUG_ASSERT(pm->active_count == 0); - - pm->backend.poweron_required = false; -@@ -127,6 +157,16 @@ void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev) - return; - if (kbdev->pm.backend.poweroff_wait_in_progress) - return; -+ -+#if MALI_USE_CSF -+ CSTD_UNUSED(shaders_desired); -+ /* Invoke the MCU state machine to send a request to FW for updating -+ * the mask of shader cores that can be used for allocation of -+ * endpoints requested by CSGs. -+ */ -+ if (kbase_pm_is_mcu_desired(kbdev)) -+ kbase_pm_update_state(kbdev); -+#else - /* In protected transition, don't allow outside shader core request - * affect transition, return directly - */ -@@ -138,6 +178,7 @@ void kbase_pm_update_dynamic_cores_onoff(struct kbase_device *kbdev) - if (shaders_desired && kbase_pm_is_l2_desired(kbdev)) { - kbase_pm_update_state(kbdev); - } -+#endif - } - - void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) -@@ -153,11 +194,20 @@ void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev) - - if (kbdev->pm.backend.protected_transition_override) - /* We are trying to change in/out of protected mode - force all -- * cores off so that the L2 powers down */ -+ * cores off so that the L2 powers down -+ */ - shaders_desired = false; - else - shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev); - -+#if MALI_USE_CSF -+ /* On CSF GPUs, Host driver isn't supposed to do the power management -+ * for shader cores. CSF firmware will power up the cores appropriately -+ * and so from Driver's standpoint 'shaders_desired' flag shall always -+ * remain 0. -+ */ -+ shaders_desired = false; -+#endif - if (kbdev->pm.backend.shaders_desired != shaders_desired) { - KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, kbdev->pm.backend.shaders_desired); - -@@ -197,20 +247,106 @@ const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev) - - KBASE_EXPORT_TEST_API(kbase_pm_get_policy); - -+#if MALI_USE_CSF -+static int policy_change_wait_for_L2_off(struct kbase_device *kbdev) -+{ -+#define WAIT_DURATION_MS (3000) -+ long remaining; -+ long timeout = kbase_csf_timeout_in_jiffies(WAIT_DURATION_MS); -+ int err = 0; -+ -+ /* Wait for L2 becoming off, by which the MCU is also implicitly off -+ * since the L2 state machine would only start its power-down -+ * sequence when the MCU is in off state. The L2 off is required -+ * as the tiler may need to be power cycled for MCU reconfiguration -+ * for host control of shader cores. -+ */ -+#if KERNEL_VERSION(4, 13, 1) <= LINUX_VERSION_CODE -+ remaining = wait_event_killable_timeout( -+ kbdev->pm.backend.gpu_in_desired_state_wait, -+ kbdev->pm.backend.l2_state == KBASE_L2_OFF, timeout); -+#else -+ remaining = wait_event_timeout( -+ kbdev->pm.backend.gpu_in_desired_state_wait, -+ kbdev->pm.backend.l2_state == KBASE_L2_OFF, timeout); -+#endif -+ -+ if (!remaining) { -+ err = -ETIMEDOUT; -+ } else if (remaining < 0) { -+ dev_info(kbdev->dev, -+ "Wait for L2_off got interrupted"); -+ err = (int)remaining; -+ } -+ -+ dev_dbg(kbdev->dev, "%s: err=%d mcu_state=%d, L2_state=%d\n", __func__, -+ err, kbdev->pm.backend.mcu_state, kbdev->pm.backend.l2_state); -+ -+ return err; -+} -+#endif -+ - void kbase_pm_set_policy(struct kbase_device *kbdev, - const struct kbase_pm_policy *new_policy) - { - const struct kbase_pm_policy *old_policy; - unsigned long flags; -+#if MALI_USE_CSF -+ unsigned int new_policy_csf_pm_sched_flags; -+ bool sched_suspend; -+ bool reset_gpu = false; -+#endif - - KBASE_DEBUG_ASSERT(kbdev != NULL); - KBASE_DEBUG_ASSERT(new_policy != NULL); - - KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id); - -+#if MALI_USE_CSF -+ /* Serialize calls on kbase_pm_set_policy() */ -+ mutex_lock(&kbdev->pm.backend.policy_change_lock); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ /* policy_change_clamp_state_to_off, when needed, is set/cleared in -+ * this function, a very limited temporal scope for covering the -+ * change transition. -+ */ -+ WARN_ON(kbdev->pm.backend.policy_change_clamp_state_to_off); -+ new_policy_csf_pm_sched_flags = new_policy->pm_sched_flags; -+ -+ /* Requiring the scheduler PM suspend operation when changes involving -+ * the always_on policy, reflected by the CSF_DYNAMIC_PM_CORE_KEEP_ON -+ * flag bit. -+ */ -+ sched_suspend = kbdev->csf.firmware_inited && -+ (CSF_DYNAMIC_PM_CORE_KEEP_ON & -+ (new_policy_csf_pm_sched_flags | -+ kbdev->pm.backend.csf_pm_sched_flags)); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ if (sched_suspend) -+ kbase_csf_scheduler_pm_suspend(kbdev); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ /* If the current active policy is always_on, one needs to clamp the -+ * MCU/L2 for reaching off-state -+ */ -+ if (sched_suspend) -+ kbdev->pm.backend.policy_change_clamp_state_to_off = -+ CSF_DYNAMIC_PM_CORE_KEEP_ON & kbdev->pm.backend.csf_pm_sched_flags; -+ -+ kbase_pm_update_state(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ if (sched_suspend) -+ reset_gpu = policy_change_wait_for_L2_off(kbdev); -+#endif -+ - /* During a policy change we pretend the GPU is active */ - /* A suspend won't happen here, because we're in a syscall from a -- * userspace thread */ -+ * userspace thread -+ */ - kbase_pm_context_active(kbdev); - - kbase_pm_lock(kbdev); -@@ -225,25 +361,49 @@ void kbase_pm_set_policy(struct kbase_device *kbdev, - if (old_policy->term) - old_policy->term(kbdev); - -+ memset(&kbdev->pm.backend.pm_policy_data, 0, -+ sizeof(union kbase_pm_policy_data)); -+ - KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, new_policy->id); - if (new_policy->init) - new_policy->init(kbdev); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbdev->pm.backend.pm_current_policy = new_policy; -+#if MALI_USE_CSF -+ kbdev->pm.backend.csf_pm_sched_flags = new_policy_csf_pm_sched_flags; -+ /* New policy in place, release the clamping on mcu/L2 off state */ -+ kbdev->pm.backend.policy_change_clamp_state_to_off = false; -+ kbase_pm_update_state(kbdev); -+#endif - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* If any core power state changes were previously attempted, but - * couldn't be made because the policy was changing (current_policy was -- * NULL), then re-try them here. */ -+ * NULL), then re-try them here. -+ */ - kbase_pm_update_active(kbdev); - kbase_pm_update_cores_state(kbdev); - - kbase_pm_unlock(kbdev); - - /* Now the policy change is finished, we release our fake context active -- * reference */ -+ * reference -+ */ - kbase_pm_context_idle(kbdev); -+ -+#if MALI_USE_CSF -+ /* Reverse the suspension done */ -+ if (reset_gpu) { -+ dev_warn(kbdev->dev, "Resorting to GPU reset for policy change\n"); -+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) -+ kbase_reset_gpu(kbdev); -+ kbase_reset_gpu_wait(kbdev); -+ } else if (sched_suspend) -+ kbase_csf_scheduler_pm_resume(kbdev); -+ -+ mutex_unlock(&kbdev->pm.backend.policy_change_lock); -+#endif - } - - KBASE_EXPORT_TEST_API(kbase_pm_set_policy); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h -index f103ef0..e811365 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2015, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2015, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_shader_states.h b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_shader_states.h -index 2bd9e47..8622ef7 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_shader_states.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_shader_states.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -26,6 +25,41 @@ - * including this header file. This header file can be included multiple - * times in the same compilation unit with different definitions of - * KBASEP_SHADER_STATE(). -+ * -+ * @OFF_CORESTACK_OFF: The shaders and core stacks are off -+ * @OFF_CORESTACK_PEND_ON: The shaders are off, core stacks have been -+ * requested to power on and hwcnt is being -+ * disabled -+ * @PEND_ON_CORESTACK_ON: Core stacks are on, shaders have been -+ * requested to power on. Or after doing -+ * partial shader on/off, checking whether -+ * it's the desired state. -+ * @ON_CORESTACK_ON: The shaders and core stacks are on, and -+ * hwcnt already enabled. -+ * @ON_CORESTACK_ON_RECHECK: The shaders and core stacks are on, hwcnt -+ * disabled, and checks to powering down or -+ * re-enabling hwcnt. -+ * @WAIT_OFF_CORESTACK_ON: The shaders have been requested to power -+ * off, but they remain on for the duration -+ * of the hysteresis timer -+ * @WAIT_GPU_IDLE: The shaders partial poweroff needs to -+ * reach a state where jobs on the GPU are -+ * finished including jobs currently running -+ * and in the GPU queue because of -+ * GPU2017-861 -+ * @WAIT_FINISHED_CORESTACK_ON: The hysteresis timer has expired -+ * @L2_FLUSHING_CORESTACK_ON: The core stacks are on and the level 2 -+ * cache is being flushed. -+ * @READY_OFF_CORESTACK_ON: The core stacks are on and the shaders are -+ * ready to be powered off. -+ * @PEND_OFF_CORESTACK_ON: The core stacks are on, and the shaders -+ * have been requested to power off -+ * @OFF_CORESTACK_PEND_OFF: The shaders are off, and the core stacks -+ * have been requested to power off -+ * @OFF_CORESTACK_OFF_TIMER_PEND_OFF: Shaders and corestacks are off, but the -+ * tick timer cancellation is still pending. -+ * @RESET_WAIT: The GPU is resetting, shader and core -+ * stack power states are unknown - */ - KBASEP_SHADER_STATE(OFF_CORESTACK_OFF) - KBASEP_SHADER_STATE(OFF_CORESTACK_PEND_ON) -@@ -33,7 +67,9 @@ KBASEP_SHADER_STATE(PEND_ON_CORESTACK_ON) - KBASEP_SHADER_STATE(ON_CORESTACK_ON) - KBASEP_SHADER_STATE(ON_CORESTACK_ON_RECHECK) - KBASEP_SHADER_STATE(WAIT_OFF_CORESTACK_ON) -+#if !MALI_USE_CSF - KBASEP_SHADER_STATE(WAIT_GPU_IDLE) -+#endif /* !MALI_USE_CSF */ - KBASEP_SHADER_STATE(WAIT_FINISHED_CORESTACK_ON) - KBASEP_SHADER_STATE(L2_FLUSHING_CORESTACK_ON) - KBASEP_SHADER_STATE(READY_OFF_CORESTACK_ON) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c -index cb10518..d10e404 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/backend/gpu/mali_kbase_time.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,22 +17,20 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> - #include <mali_kbase_hwaccess_time.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include <backend/gpu/mali_kbase_pm_internal.h> - --void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, -- u64 *system_time, struct timespec64 *ts) -+void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, -+ u64 *cycle_counter, -+ u64 *system_time, -+ struct timespec64 *ts) - { - u32 hi1, hi2; - -- kbase_pm_request_gpu_cycle_counter(kbdev); -- - if (cycle_counter) { - /* Read hi, lo, hi to ensure a coherent u64 */ - do { -@@ -65,6 +64,46 @@ void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, - #else - ktime_get_raw_ts64(ts); - #endif -+} -+ -+#if !MALI_USE_CSF -+/** -+ * timedwait_cycle_count_active() - Timed wait till CYCLE_COUNT_ACTIVE is active -+ * -+ * @kbdev: Kbase device -+ * -+ * Return: true if CYCLE_COUNT_ACTIVE is active within the timeout. -+ */ -+static bool timedwait_cycle_count_active(struct kbase_device *kbdev) -+{ -+ bool success = false; -+ const unsigned int timeout = 100; -+ const unsigned long remaining = jiffies + msecs_to_jiffies(timeout); - -+ while (time_is_after_jiffies(remaining)) { -+ if ((kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) & -+ GPU_STATUS_CYCLE_COUNT_ACTIVE)) { -+ success = true; -+ break; -+ } -+ } -+ return success; -+} -+#endif -+ -+void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, -+ u64 *system_time, struct timespec64 *ts) -+{ -+#if !MALI_USE_CSF -+ kbase_pm_request_gpu_cycle_counter(kbdev); -+ WARN_ONCE(kbdev->pm.backend.l2_state != KBASE_L2_ON, -+ "L2 not powered up"); -+ WARN_ONCE((!timedwait_cycle_count_active(kbdev)), -+ "Timed out on CYCLE_COUNT_ACTIVE"); -+#endif -+ kbase_backend_get_gpu_time_norequest(kbdev, cycle_counter, system_time, -+ ts); -+#if !MALI_USE_CSF - kbase_pm_release_gpu_cycle_counter(kbdev); -+#endif - } -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/build.bp b/dvalin/kernel/drivers/gpu/arm/midgard/build.bp -index 51aeecd..979e06f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/build.bp -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/build.bp -@@ -1,15 +1,21 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * -- * A copy of the licence is included with the program, and can also be obtained -- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- * Boston, MA 02110-1301, USA. -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -@@ -19,11 +25,17 @@ - * both mali_kbase and the test modules. */ - bob_defaults { - name: "mali_kbase_shared_config_defaults", -+ defaults: [ -+ "kernel_defaults", -+ ], - no_mali: { -- kbuild_options: ["CONFIG_MALI_NO_MALI=y"], -+ kbuild_options: [ -+ "CONFIG_MALI_NO_MALI=y", -+ "CONFIG_MALI_NO_MALI_DEFAULT_GPU={{.gpu}}", -+ ], - }, -- mali_real_hw: { -- kbuild_options: ["CONFIG_MALI_REAL_HW=y"], -+ gpu_has_csf: { -+ kbuild_options: ["CONFIG_MALI_CSF_SUPPORT=y"], - }, - mali_devfreq: { - kbuild_options: ["CONFIG_MALI_DEVFREQ=y"], -@@ -31,8 +43,62 @@ bob_defaults { - mali_midgard_dvfs: { - kbuild_options: ["CONFIG_MALI_MIDGARD_DVFS=y"], - }, -+ mali_gator_support: { -+ kbuild_options: ["CONFIG_MALI_GATOR_SUPPORT=y"], -+ }, -+ mali_midgard_enable_trace: { -+ kbuild_options: ["CONFIG_MALI_MIDGARD_ENABLE_TRACE=y"], -+ }, -+ mali_dma_fence: { -+ kbuild_options: ["CONFIG_MALI_DMA_FENCE=y"], -+ }, -+ mali_arbiter_support: { -+ kbuild_options: ["CONFIG_MALI_ARBITER_SUPPORT=y"], -+ }, -+ mali_dma_buf_map_on_demand: { -+ kbuild_options: ["CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND=y"], -+ }, -+ mali_dma_buf_legacy_compat: { -+ kbuild_options: ["CONFIG_MALI_DMA_BUF_LEGACY_COMPAT=y"], -+ }, -+ mali_2mb_alloc: { -+ kbuild_options: ["CONFIG_MALI_2MB_ALLOC=y"], -+ }, -+ mali_memory_fully_backed: { -+ kbuild_options: ["CONFIG_MALI_MEMORY_FULLY_BACKED=y"], -+ }, -+ mali_corestack: { -+ kbuild_options: ["CONFIG_MALI_CORESTACK=y"], -+ }, -+ mali_real_hw: { -+ kbuild_options: ["CONFIG_MALI_REAL_HW=y"], -+ }, -+ mali_error_inject_none: { -+ kbuild_options: ["CONFIG_MALI_ERROR_INJECT_NONE=y"], -+ }, -+ mali_error_inject_track_list: { -+ kbuild_options: ["CONFIG_MALI_ERROR_INJECT_TRACK_LIST=y"], -+ }, -+ mali_error_inject_random: { -+ kbuild_options: ["CONFIG_MALI_ERROR_INJECT_RANDOM=y"], -+ }, -+ mali_error_inject: { -+ kbuild_options: ["CONFIG_MALI_ERROR_INJECT=y"], -+ }, -+ mali_gem5_build: { -+ kbuild_options: ["CONFIG_MALI_GEM5_BUILD=y"], -+ }, - mali_debug: { -- kbuild_options: ["CONFIG_MALI_DEBUG=y"], -+ kbuild_options: [ -+ "CONFIG_MALI_DEBUG=y", -+ "MALI_KERNEL_TEST_API={{.debug}}", -+ ], -+ }, -+ mali_fence_debug: { -+ kbuild_options: ["CONFIG_MALI_FENCE_DEBUG=y"], -+ }, -+ mali_system_trace: { -+ kbuild_options: ["CONFIG_MALI_SYSTEM_TRACE=y"], - }, - buslog: { - kbuild_options: ["CONFIG_MALI_BUSLOG=y"], -@@ -43,44 +109,60 @@ bob_defaults { - cinstr_gwt: { - kbuild_options: ["CONFIG_MALI_CINSTR_GWT=y"], - }, -- mali_gator_support: { -- kbuild_options: ["CONFIG_MALI_GATOR_SUPPORT=y"], -+ cinstr_primary_hwc: { -+ kbuild_options: ["CONFIG_MALI_PRFCNT_SET_PRIMARY=y"], - }, -- mali_midgard_enable_trace: { -- kbuild_options: ["CONFIG_MALI_MIDGARD_ENABLE_TRACE=y"], -+ cinstr_secondary_hwc: { -+ kbuild_options: ["CONFIG_MALI_PRFCNT_SET_SECONDARY=y"], - }, -- mali_system_trace: { -- kbuild_options: ["CONFIG_MALI_SYSTEM_TRACE=y"], -+ cinstr_tertiary_hwc: { -+ kbuild_options: ["CONFIG_MALI_PRFCNT_SET_TERTIARY=y"], - }, -- mali_pwrsoft_765: { -- kbuild_options: ["CONFIG_MALI_PWRSOFT_765=y"], -+ cinstr_hwc_set_select_via_debug_fs: { -+ kbuild_options: ["CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS=y"], - }, -- mali_memory_fully_backed: { -- kbuild_options: ["CONFIG_MALI_MEMORY_FULLY_BACKED=y"], -+ mali_job_dump: { -+ kbuild_options: ["CONFIG_MALI_JOB_DUMP"], - }, -- mali_dma_buf_map_on_demand: { -- kbuild_options: ["CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND=y"], -- }, -- mali_dma_buf_legacy_compat: { -- kbuild_options: ["CONFIG_MALI_DMA_BUF_LEGACY_COMPAT=y"], -+ mali_pwrsoft_765: { -+ kbuild_options: ["CONFIG_MALI_PWRSOFT_765=y"], - }, -- mali_arbiter_support: { -- kbuild_options: ["CONFIG_MALI_ARBITER_SUPPORT=y"], -+ mali_hw_errata_1485982_not_affected: { -+ kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED=y"], - }, -- mali_gem5_build: { -- kbuild_options: ["CONFIG_MALI_GEM5_BUILD=y"], -+ mali_hw_errata_1485982_use_clock_alternative: { -+ kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE=y"], - }, - kbuild_options: [ -- "MALI_UNIT_TEST={{.unit_test_code}}", -+ "CONFIG_MALI_PLATFORM_NAME={{.mali_platform_name}}", - "MALI_CUSTOMER_RELEASE={{.release}}", -+ "MALI_UNIT_TEST={{.unit_test_code}}", - "MALI_USE_CSF={{.gpu_has_csf}}", -- "MALI_KERNEL_TEST_API={{.debug}}", -+ "MALI_JIT_PRESSURE_LIMIT_BASE={{.jit_pressure_limit_base}}", -+ -+ // Start of CS experimental features definitions. -+ // If there is nothing below, definition should be added as follows: -+ // "MALI_EXPERIMENTAL_FEATURE={{.experimental_feature}}" -+ // experimental_feature above comes from Mconfig in -+ // <ddk_root>/product/base/ -+ // However, in Mconfig, experimental_feature should be looked up (for -+ // similar explanation to this one) as ALLCAPS, i.e. -+ // EXPERIMENTAL_FEATURE. -+ // -+ // IMPORTANT: MALI_CS_EXPERIMENTAL should NEVER be defined below as it -+ // is an umbrella feature that would be open for inappropriate use -+ // (catch-all for experimental CS code without separating it into -+ // different features). -+ "MALI_INCREMENTAL_RENDERING={{.incremental_rendering}}", -+ "GPU_TIMESTAMP_CORRECTION={{.gpu_timestamp_correction}}", - ], -- defaults: ["kernel_defaults"], - } - - bob_kernel_module { - name: "mali_kbase", -+ defaults: [ -+ "mali_kbase_shared_config_defaults", -+ ], - srcs: [ - "*.c", - "*.h", -@@ -90,6 +172,7 @@ bob_kernel_module { - "backend/gpu/Kbuild", - "context/*.c", - "context/*.h", -+ "context/Kbuild", - "ipa/*.c", - "ipa/*.h", - "ipa/Kbuild", -@@ -98,52 +181,23 @@ bob_kernel_module { - "platform/*/*.h", - "platform/*/Kbuild", - "thirdparty/*.c", -+ "thirdparty/Kbuild", - "debug/*.c", - "debug/*.h", -+ "debug/Kbuild", - "device/*.c", - "device/*.h", -+ "device/Kbuild", - "gpu/*.c", - "gpu/*.h", -+ "gpu/Kbuild", - "tl/*.c", - "tl/*.h", -+ "tl/Kbuild", - "mmu/*.c", - "mmu/*.h", -+ "mmu/Kbuild", - ], -- kbuild_options: [ -- "CONFIG_MALI_KUTF=n", -- "CONFIG_MALI_MIDGARD=m", -- "CONFIG_MALI_NO_MALI_DEFAULT_GPU={{.gpu}}", -- "CONFIG_MALI_PLATFORM_NAME={{.mali_platform_name}}", -- ], -- buslog: { -- extra_symbols: [ -- "bus_logger", -- ], -- }, -- mali_corestack: { -- kbuild_options: ["CONFIG_MALI_CORESTACK=y"], -- }, -- mali_error_inject: { -- kbuild_options: ["CONFIG_MALI_ERROR_INJECT=y"], -- }, -- mali_error_inject_random: { -- kbuild_options: ["CONFIG_MALI_ERROR_INJECT_RANDOM=y"], -- }, -- cinstr_secondary_hwc: { -- kbuild_options: ["CONFIG_MALI_PRFCNT_SET_SECONDARY=y"], -- }, -- cinstr_secondary_hwc_via_debug_fs: { -- kbuild_options: ["CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS=y"], -- }, -- mali_2mb_alloc: { -- kbuild_options: ["CONFIG_MALI_2MB_ALLOC=y"], -- }, -- mali_hw_errata_1485982_not_affected: { -- kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_NOT_AFFECTED=y"], -- }, -- mali_hw_errata_1485982_use_clock_alternative: { -- kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE=y"], -- }, - gpu_has_job_manager: { - srcs: [ - "context/backend/*_jm.c", -@@ -155,6 +209,8 @@ bob_kernel_module { - "jm/*.h", - "tl/backend/*_jm.c", - "mmu/backend/*_jm.c", -+ "ipa/backend/*_jm.c", -+ "ipa/backend/*_jm.h", - ], - }, - gpu_has_csf: { -@@ -163,6 +219,9 @@ bob_kernel_module { - "csf/*.c", - "csf/*.h", - "csf/Kbuild", -+ "csf/ipa_control/*.c", -+ "csf/ipa_control/*.h", -+ "csf/ipa_control/Kbuild", - "debug/backend/*_csf.c", - "debug/backend/*_csf.h", - "device/backend/*_csf.c", -@@ -170,6 +229,8 @@ bob_kernel_module { - "gpu/backend/*_csf.h", - "tl/backend/*_csf.c", - "mmu/backend/*_csf.c", -+ "ipa/backend/*_csf.c", -+ "ipa/backend/*_csf.h", - ], - }, - mali_arbiter_support: { -@@ -179,5 +240,13 @@ bob_kernel_module { - "arbiter/Kbuild", - ], - }, -- defaults: ["mali_kbase_shared_config_defaults"], -+ kbuild_options: [ -+ "CONFIG_MALI_MIDGARD=m", -+ "CONFIG_MALI_KUTF=n", -+ ], -+ buslog: { -+ extra_symbols: [ -+ "bus_logger", -+ ], -+ }, - } -diff --git a/dvalin/kernel/drivers/base/memory_group_manager/Kconfig b/dvalin/kernel/drivers/gpu/arm/midgard/context/Kbuild -similarity index 63% -rename from dvalin/kernel/drivers/base/memory_group_manager/Kconfig -rename to dvalin/kernel/drivers/gpu/arm/midgard/context/Kbuild -index da464ec..1d9c00a 100644 ---- a/dvalin/kernel/drivers/base/memory_group_manager/Kconfig -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/context/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2013, 2016-2017, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,14 +16,12 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - -+mali_kbase-y += context/mali_kbase_context.o - --config MALI_MEMORY_GROUP_MANAGER -- tristate "MALI_MEMORY_GROUP_MANAGER" -- help -- This option enables an example implementation of a memory group manager -- for allocation and release of pages for memory pools managed by Mali GPU -- device drivers. -+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) -+ mali_kbase-y += context/backend/mali_kbase_context_csf.o -+else -+ mali_kbase-y += context/backend/mali_kbase_context_jm.o -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/context/backend/mali_kbase_context_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/context/backend/mali_kbase_context_csf.c -new file mode 100644 -index 0000000..1ce806f ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/context/backend/mali_kbase_context_csf.c -@@ -0,0 +1,201 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * Base kernel context APIs for CSF GPUs -+ */ -+ -+#include <context/mali_kbase_context_internal.h> -+#include <gpu/mali_kbase_gpu_regmap.h> -+#include <mali_kbase.h> -+#include <mali_kbase_dma_fence.h> -+#include <mali_kbase_mem_linux.h> -+#include <mali_kbase_mem_pool_group.h> -+#include <mmu/mali_kbase_mmu.h> -+#include <tl/mali_kbase_timeline.h> -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+#include <csf/mali_kbase_csf_csg_debugfs.h> -+#include <csf/mali_kbase_csf_kcpu_debugfs.h> -+#include <csf/mali_kbase_csf_tiler_heap_debugfs.h> -+#include <csf/mali_kbase_csf_cpu_queue_debugfs.h> -+#include <mali_kbase_debug_mem_view.h> -+#include <mali_kbase_mem_pool_debugfs.h> -+ -+void kbase_context_debugfs_init(struct kbase_context *const kctx) -+{ -+ kbase_debug_mem_view_init(kctx); -+ kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx); -+ kbase_jit_debugfs_init(kctx); -+ kbase_csf_queue_group_debugfs_init(kctx); -+ kbase_csf_kcpu_debugfs_init(kctx); -+ kbase_csf_tiler_heap_debugfs_init(kctx); -+ kbase_csf_cpu_queue_debugfs_init(kctx); -+} -+KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); -+ -+void kbase_context_debugfs_term(struct kbase_context *const kctx) -+{ -+ debugfs_remove_recursive(kctx->kctx_dentry); -+} -+KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); -+#else -+void kbase_context_debugfs_init(struct kbase_context *const kctx) -+{ -+ CSTD_UNUSED(kctx); -+} -+KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); -+ -+void kbase_context_debugfs_term(struct kbase_context *const kctx) -+{ -+ CSTD_UNUSED(kctx); -+} -+KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); -+#endif /* CONFIG_DEBUG_FS */ -+ -+static void kbase_context_free(struct kbase_context *kctx) -+{ -+ kbase_timeline_post_kbase_context_destroy(kctx); -+ -+ vfree(kctx); -+} -+ -+static const struct kbase_context_init context_init[] = { -+ { NULL, kbase_context_free, NULL }, -+ { kbase_context_common_init, kbase_context_common_term, -+ "Common context initialization failed" }, -+ { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, -+ "Memory pool group initialization failed" }, -+ { kbase_mem_evictable_init, kbase_mem_evictable_deinit, -+ "Memory evictable initialization failed" }, -+ { kbase_context_mmu_init, kbase_context_mmu_term, -+ "MMU initialization failed" }, -+ { kbase_context_mem_alloc_page, kbase_context_mem_pool_free, -+ "Memory alloc page failed" }, -+ { kbase_region_tracker_init, kbase_region_tracker_term, -+ "Region tracker initialization failed" }, -+ { kbase_sticky_resource_init, kbase_context_sticky_resource_term, -+ "Sticky resource initialization failed" }, -+ { kbase_jit_init, kbase_jit_term, "JIT initialization failed" }, -+ { kbase_csf_ctx_init, kbase_csf_ctx_term, -+ "CSF context initialization failed" }, -+ { kbase_context_add_to_dev_list, kbase_context_remove_from_dev_list, -+ "Adding kctx to device failed" }, -+}; -+ -+static void kbase_context_term_partial( -+ struct kbase_context *kctx, -+ unsigned int i) -+{ -+ while (i-- > 0) { -+ if (context_init[i].term) -+ context_init[i].term(kctx); -+ } -+} -+ -+struct kbase_context *kbase_create_context(struct kbase_device *kbdev, -+ bool is_compat, -+ base_context_create_flags const flags, -+ unsigned long const api_version, -+ struct file *const filp) -+{ -+ struct kbase_context *kctx; -+ unsigned int i = 0; -+ -+ if (WARN_ON(!kbdev)) -+ return NULL; -+ -+ /* Validate flags */ -+ if (WARN_ON(flags != (flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS))) -+ return NULL; -+ -+ /* zero-inited as lot of code assume it's zero'ed out on create */ -+ kctx = vzalloc(sizeof(*kctx)); -+ if (WARN_ON(!kctx)) -+ return NULL; -+ -+ kctx->kbdev = kbdev; -+ kctx->api_version = api_version; -+ kctx->filp = filp; -+ kctx->create_flags = flags; -+ -+ if (is_compat) -+ kbase_ctx_flag_set(kctx, KCTX_COMPAT); -+#if defined(CONFIG_64BIT) -+ else -+ kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); -+#endif /* defined(CONFIG_64BIT) */ -+ -+ for (i = 0; i < ARRAY_SIZE(context_init); i++) { -+ int err = 0; -+ -+ if (context_init[i].init) -+ err = context_init[i].init(kctx); -+ -+ if (err) { -+ dev_err(kbdev->dev, "%s error = %d\n", -+ context_init[i].err_mes, err); -+ -+ /* kctx should be freed by kbase_context_free(). -+ * Otherwise it will result in memory leak. -+ */ -+ WARN_ON(i == 0); -+ -+ kbase_context_term_partial(kctx, i); -+ return NULL; -+ } -+ } -+ -+ return kctx; -+} -+KBASE_EXPORT_SYMBOL(kbase_create_context); -+ -+void kbase_destroy_context(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev; -+ -+ if (WARN_ON(!kctx)) -+ return; -+ -+ kbdev = kctx->kbdev; -+ if (WARN_ON(!kbdev)) -+ return; -+ -+ /* Context termination could happen whilst the system suspend of -+ * the GPU device is ongoing or has completed. It has been seen on -+ * Customer side that a hang could occur if context termination is -+ * not blocked until the resume of GPU device. -+ */ -+ while (kbase_pm_context_active_handle_suspend( -+ kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { -+ dev_info(kbdev->dev, -+ "Suspend in progress when destroying context"); -+ wait_event(kbdev->pm.resume_wait, -+ !kbase_pm_is_suspending(kbdev)); -+ } -+ -+ kbase_mem_pool_group_mark_dying(&kctx->mem_pools); -+ -+ kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); -+ -+ kbase_pm_context_idle(kbdev); -+} -+KBASE_EXPORT_SYMBOL(kbase_destroy_context); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/context/backend/mali_kbase_context_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/context/backend/mali_kbase_context_jm.c -index 2cd2551..8ce81e7 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/context/backend/mali_kbase_context_jm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/context/backend/mali_kbase_context_jm.c -@@ -1,12 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -17,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -30,13 +28,13 @@ - #include <mali_kbase.h> - #include <mali_kbase_ctx_sched.h> - #include <mali_kbase_dma_fence.h> -+#include <mali_kbase_kinstr_jm.h> - #include <mali_kbase_mem_linux.h> - #include <mali_kbase_mem_pool_group.h> - #include <mmu/mali_kbase_mmu.h> - #include <tl/mali_kbase_timeline.h> --#include <tl/mali_kbase_tracepoints.h> - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - #include <mali_kbase_debug_mem_view.h> - #include <mali_kbase_mem_pool_debugfs.h> - -@@ -46,14 +44,12 @@ void kbase_context_debugfs_init(struct kbase_context *const kctx) - kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx); - kbase_jit_debugfs_init(kctx); - kbasep_jd_debugfs_ctx_init(kctx); -- kbase_debug_job_fault_context_init(kctx); - } - KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init); - - void kbase_context_debugfs_term(struct kbase_context *const kctx) - { - debugfs_remove_recursive(kctx->kctx_dentry); -- kbase_debug_job_fault_context_term(kctx); - } - KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); - #else -@@ -70,6 +66,16 @@ void kbase_context_debugfs_term(struct kbase_context *const kctx) - KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term); - #endif /* CONFIG_DEBUG_FS */ - -+static int kbase_context_kbase_kinstr_jm_init(struct kbase_context *kctx) -+{ -+ return kbase_kinstr_jm_init(&kctx->kinstr_jm); -+} -+ -+static void kbase_context_kbase_kinstr_jm_term(struct kbase_context *kctx) -+{ -+ kbase_kinstr_jm_term(kctx->kinstr_jm); -+} -+ - static int kbase_context_kbase_timer_setup(struct kbase_context *kctx) - { - kbase_timer_setup(&kctx->soft_job_timeout, -@@ -98,32 +104,59 @@ static int kbase_context_submit_check(struct kbase_context *kctx) - return 0; - } - -+static void kbase_context_flush_jobs(struct kbase_context *kctx) -+{ -+ kbase_jd_zap_context(kctx); -+ flush_workqueue(kctx->jctx.job_done_wq); -+} -+ -+static void kbase_context_free(struct kbase_context *kctx) -+{ -+ kbase_timeline_post_kbase_context_destroy(kctx); -+ -+ vfree(kctx); -+} -+ - static const struct kbase_context_init context_init[] = { -- {kbase_context_common_init, kbase_context_common_term, NULL}, -- {kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, -- "Memory pool goup initialization failed"}, -- {kbase_mem_evictable_init, kbase_mem_evictable_deinit, -- "Memory evictable initialization failed"}, -- {kbasep_js_kctx_init, kbasep_js_kctx_term, -- "JS kctx initialization failed"}, -- {kbase_jd_init, kbase_jd_exit, -- "JD initialization failed"}, -- {kbase_event_init, kbase_event_cleanup, -- "Event initialization failed"}, -- {kbase_dma_fence_init, kbase_dma_fence_term, -- "DMA fence initialization failed"}, -- {kbase_context_mmu_init, kbase_context_mmu_term, -- "MMU initialization failed"}, -- {kbase_context_mem_alloc_page, kbase_context_mem_pool_free, -- "Memory alloc page failed"}, -- {kbase_region_tracker_init, kbase_region_tracker_term, -- "Region tracker initialization failed"}, -- {kbase_sticky_resource_init, kbase_context_sticky_resource_term, -- "Sticky resource initialization failed"}, -- {kbase_jit_init, kbase_jit_term, -- "JIT initialization failed"}, -- {kbase_context_kbase_timer_setup, NULL, NULL}, -- {kbase_context_submit_check, NULL, NULL}, -+ { NULL, kbase_context_free, NULL }, -+ { kbase_context_common_init, kbase_context_common_term, -+ "Common context initialization failed" }, -+ { kbase_dma_fence_init, kbase_dma_fence_term, -+ "DMA fence initialization failed" }, -+ { kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term, -+ "Memory pool group initialization failed" }, -+ { kbase_mem_evictable_init, kbase_mem_evictable_deinit, -+ "Memory evictable initialization failed" }, -+ { kbase_context_mmu_init, kbase_context_mmu_term, -+ "MMU initialization failed" }, -+ { kbase_context_mem_alloc_page, kbase_context_mem_pool_free, -+ "Memory alloc page failed" }, -+ { kbase_region_tracker_init, kbase_region_tracker_term, -+ "Region tracker initialization failed" }, -+ { kbase_sticky_resource_init, kbase_context_sticky_resource_term, -+ "Sticky resource initialization failed" }, -+ { kbase_jit_init, kbase_jit_term, "JIT initialization failed" }, -+ { kbase_context_kbase_kinstr_jm_init, -+ kbase_context_kbase_kinstr_jm_term, -+ "JM instrumentation initialization failed" }, -+ { kbase_context_kbase_timer_setup, NULL, -+ "Timers initialization failed" }, -+ { kbase_event_init, kbase_event_cleanup, -+ "Event initialization failed" }, -+ { kbasep_js_kctx_init, kbasep_js_kctx_term, -+ "JS kctx initialization failed" }, -+ { kbase_jd_init, kbase_jd_exit, "JD initialization failed" }, -+ { kbase_context_submit_check, NULL, "Enabling job submission failed" }, -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ { kbase_debug_job_fault_context_init, -+ kbase_debug_job_fault_context_term, -+ "Job fault context initialization failed" }, -+#endif -+ { NULL, kbase_context_flush_jobs, NULL }, -+ { kbase_context_add_to_dev_list, kbase_context_remove_from_dev_list, -+ "Adding kctx to device failed" }, -+ { kbasep_platform_context_init, kbasep_platform_context_term, -+ "Platform callback for kctx initialization failed" }, - }; - - static void kbase_context_term_partial( -@@ -167,14 +200,23 @@ struct kbase_context *kbase_create_context(struct kbase_device *kbdev, - #if defined(CONFIG_64BIT) - else - kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA); --#endif /* !defined(CONFIG_64BIT) */ -+#endif /* defined(CONFIG_64BIT) */ - - for (i = 0; i < ARRAY_SIZE(context_init); i++) { -- int err = context_init[i].init(kctx); -+ int err = 0; -+ -+ if (context_init[i].init) -+ err = context_init[i].init(kctx); - - if (err) { - dev_err(kbdev->dev, "%s error = %d\n", - context_init[i].err_mes, err); -+ -+ /* kctx should be freed by kbase_context_free(). -+ * Otherwise it will result in memory leak. -+ */ -+ WARN_ON(i == 0); -+ - kbase_context_term_partial(kctx, i); - return NULL; - } -@@ -195,17 +237,27 @@ void kbase_destroy_context(struct kbase_context *kctx) - if (WARN_ON(!kbdev)) - return; - -- /* Ensure the core is powered up for the destroy process -- * A suspend won't happen here, because we're in a syscall -- * from a userspace thread. -+ /* Context termination could happen whilst the system suspend of -+ * the GPU device is ongoing or has completed. It has been seen on -+ * Customer side that a hang could occur if context termination is -+ * not blocked until the resume of GPU device. - */ -- kbase_pm_context_active(kbdev); -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ atomic_inc(&kbdev->pm.gpu_users_waiting); -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ -+ while (kbase_pm_context_active_handle_suspend( -+ kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { -+ dev_dbg(kbdev->dev, -+ "Suspend in progress when destroying context"); -+ wait_event(kbdev->pm.resume_wait, -+ !kbase_pm_is_suspending(kbdev)); -+ } -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ atomic_dec(&kbdev->pm.gpu_users_waiting); -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ - - kbase_mem_pool_group_mark_dying(&kctx->mem_pools); - -- kbase_jd_zap_context(kctx); -- flush_workqueue(kctx->jctx.job_done_wq); -- - kbase_context_term_partial(kctx, ARRAY_SIZE(context_init)); - - kbase_pm_context_idle(kbdev); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.c b/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.c -index 93fe431..b2e7025 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.c -@@ -1,12 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -17,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -28,17 +26,105 @@ - #include <mali_kbase.h> - #include <gpu/mali_kbase_gpu_regmap.h> - #include <mali_kbase_mem_linux.h> --#include <mali_kbase_dma_fence.h> - #include <mali_kbase_ctx_sched.h> - #include <mali_kbase_mem_pool_group.h> --#include <tl/mali_kbase_tracepoints.h> - #include <tl/mali_kbase_timeline.h> - #include <mmu/mali_kbase_mmu.h> - #include <context/mali_kbase_context_internal.h> - -+/** -+ * find_process_node - Used to traverse the process rb_tree to find if -+ * process exists already in process rb_tree. -+ * -+ * @node: Pointer to root node to start search. -+ * @tgid: Thread group PID to search for. -+ * -+ * Return: Pointer to kbase_process if exists otherwise NULL. -+ */ -+static struct kbase_process *find_process_node(struct rb_node *node, pid_t tgid) -+{ -+ struct kbase_process *kprcs = NULL; -+ -+ /* Check if the kctx creation request is from a existing process.*/ -+ while (node) { -+ struct kbase_process *prcs_node = -+ rb_entry(node, struct kbase_process, kprcs_node); -+ if (prcs_node->tgid == tgid) { -+ kprcs = prcs_node; -+ break; -+ } -+ -+ if (tgid < prcs_node->tgid) -+ node = node->rb_left; -+ else -+ node = node->rb_right; -+ } -+ -+ return kprcs; -+} -+ -+/** -+ * kbase_insert_kctx_to_process - Initialise kbase process context. -+ * -+ * @kctx: Pointer to kbase context. -+ * -+ * Here we initialise per process rb_tree managed by kbase_device. -+ * We maintain a rb_tree of each unique process that gets created. -+ * and Each process maintains a list of kbase context. -+ * This setup is currently used by kernel trace functionality -+ * to trace and visualise gpu memory consumption. -+ * -+ * Return: 0 on success and error number on failure. -+ */ -+static int kbase_insert_kctx_to_process(struct kbase_context *kctx) -+{ -+ struct rb_root *const prcs_root = &kctx->kbdev->process_root; -+ const pid_t tgid = kctx->tgid; -+ struct kbase_process *kprcs = NULL; -+ -+ lockdep_assert_held(&kctx->kbdev->kctx_list_lock); -+ -+ kprcs = find_process_node(prcs_root->rb_node, tgid); -+ -+ /* if the kctx is from new process then create a new kbase_process -+ * and add it to the &kbase_device->rb_tree -+ */ -+ if (!kprcs) { -+ struct rb_node **new = &prcs_root->rb_node, *parent = NULL; -+ -+ kprcs = kzalloc(sizeof(*kprcs), GFP_KERNEL); -+ if (kprcs == NULL) -+ return -ENOMEM; -+ kprcs->tgid = tgid; -+ INIT_LIST_HEAD(&kprcs->kctx_list); -+ kprcs->dma_buf_root = RB_ROOT; -+ kprcs->total_gpu_pages = 0; -+ -+ while (*new) { -+ struct kbase_process *prcs_node; -+ -+ parent = *new; -+ prcs_node = rb_entry(parent, struct kbase_process, -+ kprcs_node); -+ if (tgid < prcs_node->tgid) -+ new = &(*new)->rb_left; -+ else -+ new = &(*new)->rb_right; -+ } -+ rb_link_node(&kprcs->kprcs_node, parent, new); -+ rb_insert_color(&kprcs->kprcs_node, prcs_root); -+ } -+ -+ kctx->kprcs = kprcs; -+ list_add(&kctx->kprcs_link, &kprcs->kctx_list); -+ -+ return 0; -+} -+ - int kbase_context_common_init(struct kbase_context *kctx) - { - const unsigned long cookies_mask = KBASE_COOKIE_MASK; -+ int err = 0; - - /* creating a context is considered a disjoint event */ - kbase_disjoint_event(kctx->kbdev); -@@ -66,30 +152,95 @@ int kbase_context_common_init(struct kbase_context *kctx) - - init_waitqueue_head(&kctx->event_queue); - atomic_set(&kctx->event_count, 0); -+#if !MALI_USE_CSF - atomic_set(&kctx->event_closed, false); -- -- bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG); -- --#ifdef CONFIG_GPU_TRACEPOINTS -+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) - atomic_set(&kctx->jctx.work_id, 0); -+#endif - #endif - -+ bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG); -+ - kctx->id = atomic_add_return(1, &(kctx->kbdev->ctx_num)) - 1; - - mutex_init(&kctx->legacy_hwcnt_lock); - - mutex_lock(&kctx->kbdev->kctx_list_lock); -- list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list); - -- KBASE_TLSTREAM_TL_KBASE_NEW_CTX(kctx->kbdev, kctx->id, -- kctx->kbdev->gpu_props.props.raw_props.gpu_id); -- KBASE_TLSTREAM_TL_NEW_CTX(kctx->kbdev, kctx, kctx->id, -- (u32)(kctx->tgid)); -+ err = kbase_insert_kctx_to_process(kctx); -+ if (err) -+ dev_err(kctx->kbdev->dev, -+ "(err:%d) failed to insert kctx to kbase_process\n", err); -+ - mutex_unlock(&kctx->kbdev->kctx_list_lock); - -+ return err; -+} -+ -+int kbase_context_add_to_dev_list(struct kbase_context *kctx) -+{ -+ if (WARN_ON(!kctx)) -+ return -EINVAL; -+ -+ if (WARN_ON(!kctx->kbdev)) -+ return -EINVAL; -+ -+ mutex_lock(&kctx->kbdev->kctx_list_lock); -+ list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list); -+ mutex_unlock(&kctx->kbdev->kctx_list_lock); -+ -+ kbase_timeline_post_kbase_context_create(kctx); -+ - return 0; - } - -+void kbase_context_remove_from_dev_list(struct kbase_context *kctx) -+{ -+ if (WARN_ON(!kctx)) -+ return; -+ -+ if (WARN_ON(!kctx->kbdev)) -+ return; -+ -+ kbase_timeline_pre_kbase_context_destroy(kctx); -+ -+ mutex_lock(&kctx->kbdev->kctx_list_lock); -+ list_del_init(&kctx->kctx_list_link); -+ mutex_unlock(&kctx->kbdev->kctx_list_lock); -+} -+ -+/** -+ * kbase_remove_kctx_from_process - remove a terminating context from -+ * the process list. -+ * -+ * @kctx: Pointer to kbase context. -+ * -+ * Remove the tracking of context from the list of contexts maintained under -+ * kbase process and if the list if empty then there no outstanding contexts -+ * we can remove the process node as well. -+ */ -+ -+static void kbase_remove_kctx_from_process(struct kbase_context *kctx) -+{ -+ struct kbase_process *kprcs = kctx->kprcs; -+ -+ lockdep_assert_held(&kctx->kbdev->kctx_list_lock); -+ list_del(&kctx->kprcs_link); -+ -+ /* if there are no outstanding contexts in current process node, -+ * we can remove it from the process rb_tree. -+ */ -+ if (list_empty(&kprcs->kctx_list)) { -+ rb_erase(&kprcs->kprcs_node, &kctx->kbdev->process_root); -+ /* Add checks, so that the terminating process Should not -+ * hold any gpu_memory. -+ */ -+ WARN_ON(kprcs->total_gpu_pages); -+ WARN_ON(!RB_EMPTY_ROOT(&kprcs->dma_buf_root)); -+ kfree(kprcs); -+ } -+} -+ - void kbase_context_common_term(struct kbase_context *kctx) - { - unsigned long flags; -@@ -109,24 +260,10 @@ void kbase_context_common_term(struct kbase_context *kctx) - WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0); - - mutex_lock(&kctx->kbdev->kctx_list_lock); -- -- KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kctx->kbdev, kctx->id); -- -- KBASE_TLSTREAM_TL_DEL_CTX(kctx->kbdev, kctx); -- list_del(&kctx->kctx_list_link); -+ kbase_remove_kctx_from_process(kctx); - mutex_unlock(&kctx->kbdev->kctx_list_lock); - - KBASE_KTRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, 0u); -- -- /* Flush the timeline stream, so the user can see the termination -- * tracepoints being fired. -- * The "if" statement below is for optimization. It is safe to call -- * kbase_timeline_streams_flush when timeline is disabled. -- */ -- if (atomic_read(&kctx->kbdev->timeline_flags) != 0) -- kbase_timeline_streams_flush(kctx->kbdev->timeline); -- -- vfree(kctx); - } - - int kbase_context_mem_pool_group_init(struct kbase_context *kctx) -@@ -144,11 +281,9 @@ void kbase_context_mem_pool_group_term(struct kbase_context *kctx) - - int kbase_context_mmu_init(struct kbase_context *kctx) - { -- kbase_mmu_init(kctx->kbdev, -- &kctx->mmu, kctx, -+ return kbase_mmu_init( -+ kctx->kbdev, &kctx->mmu, kctx, - base_context_mmu_group_id_get(kctx->create_flags)); -- -- return 0; - } - - void kbase_context_mmu_term(struct kbase_context *kctx) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.h b/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.h -index e4ed894..a0c51c9 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2017, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,18 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ --/* -- * -- * (C) COPYRIGHT 2011-2017, 2019 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * - */ - - #ifndef _KBASE_CONTEXT_H_ -@@ -117,25 +106,7 @@ static inline bool kbase_ctx_flag(struct kbase_context *kctx, - static inline void kbase_ctx_flag_clear(struct kbase_context *kctx, - enum kbase_context_flags flag) - { --#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE -- /* -- * Earlier kernel versions doesn't have atomic_andnot() or -- * atomic_and(). atomic_clear_mask() was only available on some -- * architectures and removed on arm in v3.13 on arm and arm64. -- * -- * Use a compare-exchange loop to clear the flag on pre 4.3 kernels, -- * when atomic_andnot() becomes available. -- */ -- int old, new; -- -- do { -- old = atomic_read(&kctx->flags); -- new = old & ~flag; -- -- } while (atomic_cmpxchg(&kctx->flags, old, new) != old); --#else - atomic_andnot(flag, &kctx->flags); --#endif - } - - /** -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context_internal.h -index 818cdbe..1cde739 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context_internal.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/context/mali_kbase_context_internal.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,16 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ --/* -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. - */ - - #include <mali_kbase.h> -@@ -58,3 +49,6 @@ int kbase_context_mem_alloc_page(struct kbase_context *kctx); - void kbase_context_mem_pool_free(struct kbase_context *kctx); - - void kbase_context_sticky_resource_term(struct kbase_context *kctx); -+ -+int kbase_context_add_to_dev_list(struct kbase_context *kctx); -+void kbase_context_remove_from_dev_list(struct kbase_context *kctx); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/csf/Kbuild -new file mode 100644 -index 0000000..765e419 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/Kbuild -@@ -0,0 +1,47 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+mali_kbase-y += \ -+ csf/mali_kbase_csf_firmware_cfg.o \ -+ csf/mali_kbase_csf_trace_buffer.o \ -+ csf/mali_kbase_csf.o \ -+ csf/mali_kbase_csf_scheduler.o \ -+ csf/mali_kbase_csf_kcpu.o \ -+ csf/mali_kbase_csf_tiler_heap.o \ -+ csf/mali_kbase_csf_timeout.o \ -+ csf/mali_kbase_csf_tl_reader.o \ -+ csf/mali_kbase_csf_heap_context_alloc.o \ -+ csf/mali_kbase_csf_reset_gpu.o \ -+ csf/mali_kbase_csf_csg_debugfs.o \ -+ csf/mali_kbase_csf_kcpu_debugfs.o \ -+ csf/mali_kbase_csf_protected_memory.o \ -+ csf/mali_kbase_csf_tiler_heap_debugfs.o \ -+ csf/mali_kbase_csf_cpu_queue_debugfs.o -+ -+mali_kbase-$(CONFIG_MALI_REAL_HW) += csf/mali_kbase_csf_firmware.o -+ -+ -+ifeq ($(KBUILD_EXTMOD),) -+# in-tree -+ -include $(src)/csf/ipa_control/Kbuild -+else -+# out-of-tree -+ include $(src)/csf/ipa_control/Kbuild -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/Makefile.kbase b/dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/Kbuild -similarity index 75% -rename from dvalin/kernel/drivers/gpu/arm/midgard/Makefile.kbase -rename to dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/Kbuild -index 6b0f81e..08824b2 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/Makefile.kbase -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2010, 2013, 2018 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,9 +16,7 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # --# -- --EXTRA_CFLAGS += -I$(ROOT) -I$(KBASE_PATH) -I$(KBASE_PATH)/platform_$(PLATFORM) - -+mali_kbase-y += \ -+ csf/ipa_control/mali_kbase_csf_ipa_control.o -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/mali_kbase_csf_ipa_control.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/mali_kbase_csf_ipa_control.c -new file mode 100644 -index 0000000..e23d681 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/mali_kbase_csf_ipa_control.c -@@ -0,0 +1,925 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" -+#include "mali_kbase_csf_ipa_control.h" -+ -+/* -+ * Status flags from the STATUS register of the IPA Control interface. -+ */ -+#define STATUS_COMMAND_ACTIVE ((u32)1 << 0) -+#define STATUS_TIMER_ACTIVE ((u32)1 << 1) -+#define STATUS_AUTO_ACTIVE ((u32)1 << 2) -+#define STATUS_PROTECTED_MODE ((u32)1 << 8) -+#define STATUS_RESET ((u32)1 << 9) -+#define STATUS_TIMER_ENABLED ((u32)1 << 31) -+ -+/* -+ * Commands for the COMMAND register of the IPA Control interface. -+ */ -+#define COMMAND_NOP ((u32)0) -+#define COMMAND_APPLY ((u32)1) -+#define COMMAND_CLEAR ((u32)2) -+#define COMMAND_SAMPLE ((u32)3) -+#define COMMAND_PROTECTED_ACK ((u32)4) -+#define COMMAND_RESET_ACK ((u32)5) -+ -+/** -+ * Default value for the TIMER register of the IPA Control interface, -+ * expressed in milliseconds. -+ * -+ * The chosen value is a trade off between two requirements: the IPA Control -+ * interface should sample counters with a resolution in the order of -+ * milliseconds, while keeping GPU overhead as limited as possible. -+ */ -+#define TIMER_DEFAULT_VALUE_MS ((u32)10) /* 10 milliseconds */ -+ -+/** -+ * Number of timer events per second. -+ */ -+#define TIMER_EVENTS_PER_SECOND ((u32)1000 / TIMER_DEFAULT_VALUE_MS) -+ -+/** -+ * Maximum number of loops polling the GPU before we assume the GPU has hung. -+ */ -+#define IPA_INACTIVE_MAX_LOOPS ((unsigned int)8000000) -+ -+/** -+ * Number of bits used to configure a performance counter in SELECT registers. -+ */ -+#define IPA_CONTROL_SELECT_BITS_PER_CNT ((u64)8) -+ -+/** -+ * Maximum value of a performance counter. -+ */ -+#define MAX_PRFCNT_VALUE (((u64)1 << 48) - 1) -+ -+/** -+ * struct kbase_ipa_control_listener_data - Data for the GPU clock frequency -+ * listener -+ * -+ * @listener: GPU clock frequency listener. -+ * @kbdev: Pointer to kbase device. -+ */ -+struct kbase_ipa_control_listener_data { -+ struct kbase_clk_rate_listener listener; -+ struct kbase_device *kbdev; -+}; -+ -+static u32 timer_value(u32 gpu_rate) -+{ -+ return gpu_rate / TIMER_EVENTS_PER_SECOND; -+} -+ -+static int wait_status(struct kbase_device *kbdev, u32 flags) -+{ -+ unsigned int max_loops = IPA_INACTIVE_MAX_LOOPS; -+ u32 status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS)); -+ -+ /* -+ * Wait for the STATUS register to indicate that flags have been -+ * cleared, in case a transition is pending. -+ */ -+ while (--max_loops && (status & flags)) -+ status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS)); -+ if (max_loops == 0) { -+ dev_err(kbdev->dev, "IPA_CONTROL STATUS register stuck"); -+ return -EBUSY; -+ } -+ -+ return 0; -+} -+ -+static int apply_select_config(struct kbase_device *kbdev, u64 *select) -+{ -+ int ret; -+ -+ u32 select_cshw_lo = (u32)(select[KBASE_IPA_CORE_TYPE_CSHW] & U32_MAX); -+ u32 select_cshw_hi = -+ (u32)((select[KBASE_IPA_CORE_TYPE_CSHW] >> 32) & U32_MAX); -+ u32 select_memsys_lo = -+ (u32)(select[KBASE_IPA_CORE_TYPE_MEMSYS] & U32_MAX); -+ u32 select_memsys_hi = -+ (u32)((select[KBASE_IPA_CORE_TYPE_MEMSYS] >> 32) & U32_MAX); -+ u32 select_tiler_lo = -+ (u32)(select[KBASE_IPA_CORE_TYPE_TILER] & U32_MAX); -+ u32 select_tiler_hi = -+ (u32)((select[KBASE_IPA_CORE_TYPE_TILER] >> 32) & U32_MAX); -+ u32 select_shader_lo = -+ (u32)(select[KBASE_IPA_CORE_TYPE_SHADER] & U32_MAX); -+ u32 select_shader_hi = -+ (u32)((select[KBASE_IPA_CORE_TYPE_SHADER] >> 32) & U32_MAX); -+ -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_LO), select_cshw_lo); -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_HI), select_cshw_hi); -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_MEMSYS_LO), -+ select_memsys_lo); -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_MEMSYS_HI), -+ select_memsys_hi); -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_TILER_LO), -+ select_tiler_lo); -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_TILER_HI), -+ select_tiler_hi); -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_SHADER_LO), -+ select_shader_lo); -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_SHADER_HI), -+ select_shader_hi); -+ -+ ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE); -+ -+ if (!ret) -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), COMMAND_APPLY); -+ -+ return ret; -+} -+ -+static u64 read_value_cnt(struct kbase_device *kbdev, u8 type, int select_idx) -+{ -+ u32 value_lo, value_hi; -+ -+ switch (type) { -+ case KBASE_IPA_CORE_TYPE_CSHW: -+ value_lo = kbase_reg_read( -+ kbdev, IPA_CONTROL_REG(VALUE_CSHW_REG_LO(select_idx))); -+ value_hi = kbase_reg_read( -+ kbdev, IPA_CONTROL_REG(VALUE_CSHW_REG_HI(select_idx))); -+ break; -+ case KBASE_IPA_CORE_TYPE_MEMSYS: -+ value_lo = kbase_reg_read( -+ kbdev, -+ IPA_CONTROL_REG(VALUE_MEMSYS_REG_LO(select_idx))); -+ value_hi = kbase_reg_read( -+ kbdev, -+ IPA_CONTROL_REG(VALUE_MEMSYS_REG_HI(select_idx))); -+ break; -+ case KBASE_IPA_CORE_TYPE_TILER: -+ value_lo = kbase_reg_read( -+ kbdev, IPA_CONTROL_REG(VALUE_TILER_REG_LO(select_idx))); -+ value_hi = kbase_reg_read( -+ kbdev, IPA_CONTROL_REG(VALUE_TILER_REG_HI(select_idx))); -+ break; -+ case KBASE_IPA_CORE_TYPE_SHADER: -+ value_lo = kbase_reg_read( -+ kbdev, -+ IPA_CONTROL_REG(VALUE_SHADER_REG_LO(select_idx))); -+ value_hi = kbase_reg_read( -+ kbdev, -+ IPA_CONTROL_REG(VALUE_SHADER_REG_HI(select_idx))); -+ break; -+ default: -+ WARN(1, "Unknown core type: %u\n", type); -+ value_lo = value_hi = 0; -+ break; -+ } -+ -+ return (((u64)value_hi << 32) | value_lo); -+} -+ -+static void build_select_config(struct kbase_ipa_control *ipa_ctrl, -+ u64 *select_config) -+{ -+ size_t i; -+ -+ for (i = 0; i < KBASE_IPA_CORE_TYPE_NUM; i++) { -+ size_t j; -+ -+ select_config[i] = 0ULL; -+ -+ for (j = 0; j < KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; j++) { -+ struct kbase_ipa_control_prfcnt_config *prfcnt_config = -+ &ipa_ctrl->blocks[i].select[j]; -+ -+ select_config[i] |= -+ ((u64)prfcnt_config->idx -+ << (IPA_CONTROL_SELECT_BITS_PER_CNT * j)); -+ } -+ } -+} -+ -+static inline void calc_prfcnt_delta(struct kbase_device *kbdev, -+ struct kbase_ipa_control_prfcnt *prfcnt, -+ bool gpu_ready) -+{ -+ u64 delta_value, raw_value; -+ -+ if (gpu_ready) -+ raw_value = read_value_cnt(kbdev, (u8)prfcnt->type, -+ prfcnt->select_idx); -+ else -+ raw_value = prfcnt->latest_raw_value; -+ -+ if (raw_value < prfcnt->latest_raw_value) { -+ delta_value = (MAX_PRFCNT_VALUE - prfcnt->latest_raw_value) + -+ raw_value; -+ } else { -+ delta_value = raw_value - prfcnt->latest_raw_value; -+ } -+ -+ delta_value *= prfcnt->scaling_factor; -+ -+ if (!WARN_ON_ONCE(kbdev->csf.ipa_control.cur_gpu_rate == 0)) -+ if (prfcnt->gpu_norm) -+ delta_value /= kbdev->csf.ipa_control.cur_gpu_rate; -+ -+ prfcnt->latest_raw_value = raw_value; -+ -+ /* Accumulate the difference */ -+ prfcnt->accumulated_diff += delta_value; -+} -+ -+/** -+ * kbase_ipa_control_rate_change_notify - GPU frequency change callback -+ * -+ * @listener: Clock frequency change listener. -+ * @clk_index: Index of the clock for which the change has occurred. -+ * @clk_rate_hz: Clock frequency(Hz). -+ * -+ * This callback notifies kbase_ipa_control about GPU frequency changes. -+ * Only top-level clock changes are meaningful. GPU frequency updates -+ * affect all performance counters which require GPU normalization -+ * in every session. -+ */ -+static void -+kbase_ipa_control_rate_change_notify(struct kbase_clk_rate_listener *listener, -+ u32 clk_index, u32 clk_rate_hz) -+{ -+ if ((clk_index == KBASE_CLOCK_DOMAIN_TOP) && (clk_rate_hz != 0)) { -+ size_t i; -+ unsigned long flags; -+ struct kbase_ipa_control_listener_data *listener_data = -+ container_of(listener, -+ struct kbase_ipa_control_listener_data, -+ listener); -+ struct kbase_device *kbdev = listener_data->kbdev; -+ struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ if (!kbdev->pm.backend.gpu_ready) { -+ dev_err(kbdev->dev, -+ "%s: GPU frequency cannot change while GPU is off", -+ __func__); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ return; -+ } -+ -+ /* Interrupts are already disabled and interrupt state is also saved */ -+ spin_lock(&ipa_ctrl->lock); -+ -+ for (i = 0; i < ipa_ctrl->num_active_sessions; i++) { -+ size_t j; -+ struct kbase_ipa_control_session *session = &ipa_ctrl->sessions[i]; -+ -+ for (j = 0; j < session->num_prfcnts; j++) { -+ struct kbase_ipa_control_prfcnt *prfcnt = -+ &session->prfcnts[j]; -+ -+ if (prfcnt->gpu_norm) -+ calc_prfcnt_delta(kbdev, prfcnt, true); -+ } -+ } -+ -+ ipa_ctrl->cur_gpu_rate = clk_rate_hz; -+ -+ /* Update the timer for automatic sampling if active sessions -+ * are present. Counters have already been manually sampled. -+ */ -+ if (ipa_ctrl->num_active_sessions > 0) { -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), -+ timer_value(ipa_ctrl->cur_gpu_rate)); -+ } -+ -+ spin_unlock(&ipa_ctrl->lock); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } -+} -+ -+void kbase_ipa_control_init(struct kbase_device *kbdev) -+{ -+ struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; -+ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; -+ struct kbase_ipa_control_listener_data *listener_data; -+ size_t i, j; -+ -+ for (i = 0; i < KBASE_IPA_CORE_TYPE_NUM; i++) { -+ for (j = 0; j < KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; j++) { -+ ipa_ctrl->blocks[i].select[j].idx = 0; -+ ipa_ctrl->blocks[i].select[j].refcount = 0; -+ } -+ ipa_ctrl->blocks[i].num_available_counters = -+ KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; -+ } -+ -+ spin_lock_init(&ipa_ctrl->lock); -+ ipa_ctrl->num_active_sessions = 0; -+ for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) { -+ ipa_ctrl->sessions[i].active = false; -+ } -+ -+ listener_data = kmalloc(sizeof(struct kbase_ipa_control_listener_data), -+ GFP_KERNEL); -+ if (listener_data) { -+ listener_data->listener.notify = -+ kbase_ipa_control_rate_change_notify; -+ listener_data->kbdev = kbdev; -+ ipa_ctrl->rtm_listener_data = listener_data; -+ } -+ -+ spin_lock(&clk_rtm->lock); -+ if (clk_rtm->clks[KBASE_CLOCK_DOMAIN_TOP]) -+ ipa_ctrl->cur_gpu_rate = -+ clk_rtm->clks[KBASE_CLOCK_DOMAIN_TOP]->clock_val; -+ if (listener_data) -+ kbase_clk_rate_trace_manager_subscribe_no_lock( -+ clk_rtm, &listener_data->listener); -+ spin_unlock(&clk_rtm->lock); -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_control_init); -+ -+void kbase_ipa_control_term(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm; -+ struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; -+ struct kbase_ipa_control_listener_data *listener_data = -+ ipa_ctrl->rtm_listener_data; -+ -+ WARN_ON(ipa_ctrl->num_active_sessions); -+ -+ if (listener_data) -+ kbase_clk_rate_trace_manager_unsubscribe(clk_rtm, &listener_data->listener); -+ kfree(ipa_ctrl->rtm_listener_data); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ if (kbdev->pm.backend.gpu_powered) -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), 0); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_control_term); -+ -+int kbase_ipa_control_register( -+ struct kbase_device *kbdev, -+ const struct kbase_ipa_control_perf_counter *perf_counters, -+ size_t num_counters, void **client) -+{ -+ int ret = 0; -+ size_t i, session_idx, req_counters[KBASE_IPA_CORE_TYPE_NUM]; -+ bool already_configured[KBASE_IPA_CONTROL_MAX_COUNTERS]; -+ bool new_config = false; -+ struct kbase_ipa_control *ipa_ctrl; -+ struct kbase_ipa_control_session *session = NULL; -+ unsigned long flags; -+ -+ if (WARN_ON(kbdev == NULL) || WARN_ON(perf_counters == NULL) || -+ WARN_ON(client == NULL) || -+ WARN_ON(num_counters > KBASE_IPA_CONTROL_MAX_COUNTERS)) { -+ dev_err(kbdev->dev, "%s: wrong input arguments", __func__); -+ return -EINVAL; -+ } -+ -+ kbase_pm_context_active(kbdev); -+ -+ ipa_ctrl = &kbdev->csf.ipa_control; -+ spin_lock_irqsave(&ipa_ctrl->lock, flags); -+ -+ if (ipa_ctrl->num_active_sessions == KBASE_IPA_CONTROL_MAX_SESSIONS) { -+ dev_err(kbdev->dev, "%s: too many sessions", __func__); -+ ret = -EBUSY; -+ goto exit; -+ } -+ -+ for (i = 0; i < KBASE_IPA_CORE_TYPE_NUM; i++) -+ req_counters[i] = 0; -+ -+ /* -+ * Count how many counters would need to be configured in order to -+ * satisfy the request. Requested counters which happen to be already -+ * configured can be skipped. -+ */ -+ for (i = 0; i < num_counters; i++) { -+ size_t j; -+ enum kbase_ipa_core_type type = perf_counters[i].type; -+ u8 idx = perf_counters[i].idx; -+ -+ if ((type >= KBASE_IPA_CORE_TYPE_NUM) || -+ (idx >= KBASE_IPA_CONTROL_CNT_MAX_IDX)) { -+ dev_err(kbdev->dev, -+ "%s: invalid requested type %u and/or index %u", -+ __func__, type, idx); -+ ret = -EINVAL; -+ goto exit; -+ } -+ -+ for (j = 0; j < KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; j++) { -+ struct kbase_ipa_control_prfcnt_config *prfcnt_config = -+ &ipa_ctrl->blocks[type].select[j]; -+ -+ if (prfcnt_config->refcount > 0) { -+ if (prfcnt_config->idx == idx) { -+ already_configured[i] = true; -+ break; -+ } -+ } -+ } -+ -+ if (j == KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS) { -+ already_configured[i] = false; -+ req_counters[type]++; -+ new_config = true; -+ } -+ } -+ -+ for (i = 0; i < KBASE_IPA_CORE_TYPE_NUM; i++) -+ if (req_counters[i] > -+ ipa_ctrl->blocks[i].num_available_counters) { -+ dev_err(kbdev->dev, -+ "%s: more counters (%zu) than available (%zu) have been requested for type %zu", -+ __func__, req_counters[i], -+ ipa_ctrl->blocks[i].num_available_counters, i); -+ ret = -EINVAL; -+ goto exit; -+ } -+ -+ /* -+ * The request has been validated. -+ * Firstly, find an available session and then set up the initial state -+ * of the session and update the configuration of performance counters -+ * in the internal state of kbase_ipa_control. -+ */ -+ for (session_idx = 0; session_idx < KBASE_IPA_CONTROL_MAX_SESSIONS; -+ session_idx++) { -+ session = &ipa_ctrl->sessions[session_idx]; -+ if (!session->active) -+ break; -+ } -+ -+ if (!session) { -+ dev_err(kbdev->dev, "%s: wrong or corrupt session state", -+ __func__); -+ ret = -EBUSY; -+ goto exit; -+ } -+ -+ for (i = 0; i < num_counters; i++) { -+ struct kbase_ipa_control_prfcnt_config *prfcnt_config; -+ size_t j; -+ u8 type = perf_counters[i].type; -+ u8 idx = perf_counters[i].idx; -+ -+ for (j = 0; j < KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; j++) { -+ prfcnt_config = &ipa_ctrl->blocks[type].select[j]; -+ -+ if (already_configured[i]) { -+ if ((prfcnt_config->refcount > 0) && -+ (prfcnt_config->idx == idx)) { -+ break; -+ } -+ } else { -+ if (prfcnt_config->refcount == 0) -+ break; -+ } -+ } -+ -+ if (WARN_ON((prfcnt_config->refcount > 0 && -+ prfcnt_config->idx != idx) || -+ (j == KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS))) { -+ dev_err(kbdev->dev, -+ "%s: invalid internal state: counter already configured or no counter available to configure", -+ __func__); -+ ret = -EBUSY; -+ goto exit; -+ } -+ -+ if (prfcnt_config->refcount == 0) { -+ prfcnt_config->idx = idx; -+ ipa_ctrl->blocks[type].num_available_counters--; -+ } -+ -+ session->prfcnts[i].accumulated_diff = 0; -+ session->prfcnts[i].type = type; -+ session->prfcnts[i].select_idx = j; -+ session->prfcnts[i].scaling_factor = -+ perf_counters[i].scaling_factor; -+ session->prfcnts[i].gpu_norm = perf_counters[i].gpu_norm; -+ -+ /* Reports to this client for GPU time spent in protected mode -+ * should begin from the point of registration. -+ */ -+ session->last_query_time = ktime_get_ns(); -+ -+ /* Initially, no time has been spent in protected mode */ -+ session->protm_time = 0; -+ -+ prfcnt_config->refcount++; -+ } -+ -+ /* -+ * Apply new configuration, if necessary. -+ * As a temporary solution, make sure that the GPU is on -+ * before applying the new configuration. -+ */ -+ if (new_config) { -+ u64 select_config[KBASE_IPA_CORE_TYPE_NUM]; -+ -+ build_select_config(ipa_ctrl, select_config); -+ ret = apply_select_config(kbdev, select_config); -+ if (ret) -+ dev_err(kbdev->dev, -+ "%s: failed to apply SELECT configuration", -+ __func__); -+ } -+ -+ if (!ret) { -+ /* Accumulator registers don't contain any sample if the timer -+ * has not been enabled first. Take a sample manually before -+ * enabling the timer. -+ */ -+ if (ipa_ctrl->num_active_sessions == 0) { -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), -+ COMMAND_SAMPLE); -+ ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE); -+ if (!ret) { -+ kbase_reg_write( -+ kbdev, IPA_CONTROL_REG(TIMER), -+ timer_value(ipa_ctrl->cur_gpu_rate)); -+ } else { -+ dev_err(kbdev->dev, -+ "%s: failed to sample new counters", -+ __func__); -+ } -+ } -+ } -+ -+ if (!ret) { -+ session->num_prfcnts = num_counters; -+ session->active = true; -+ ipa_ctrl->num_active_sessions++; -+ *client = session; -+ -+ /* -+ * Read current raw value to initialize the session. -+ * This is necessary to put the first query in condition -+ * to generate a correct value by calculating the difference -+ * from the beginning of the session. -+ */ -+ for (i = 0; i < session->num_prfcnts; i++) { -+ struct kbase_ipa_control_prfcnt *prfcnt = -+ &session->prfcnts[i]; -+ u64 raw_value = read_value_cnt(kbdev, (u8)prfcnt->type, -+ prfcnt->select_idx); -+ prfcnt->latest_raw_value = raw_value; -+ } -+ } -+ -+exit: -+ spin_unlock_irqrestore(&ipa_ctrl->lock, flags); -+ kbase_pm_context_idle(kbdev); -+ return ret; -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_control_register); -+ -+int kbase_ipa_control_unregister(struct kbase_device *kbdev, const void *client) -+{ -+ struct kbase_ipa_control *ipa_ctrl; -+ struct kbase_ipa_control_session *session; -+ int ret = 0; -+ size_t i; -+ unsigned long flags; -+ bool new_config = false, valid_session = false; -+ -+ if (WARN_ON(kbdev == NULL) || WARN_ON(client == NULL)) { -+ dev_err(kbdev->dev, "%s: wrong input arguments", __func__); -+ return -EINVAL; -+ } -+ -+ kbase_pm_context_active(kbdev); -+ -+ ipa_ctrl = &kbdev->csf.ipa_control; -+ session = (struct kbase_ipa_control_session *)client; -+ -+ spin_lock_irqsave(&ipa_ctrl->lock, flags); -+ -+ for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) { -+ if (session == &ipa_ctrl->sessions[i]) { -+ valid_session = true; -+ break; -+ } -+ } -+ -+ if (!valid_session) { -+ dev_err(kbdev->dev, "%s: invalid session handle", __func__); -+ ret = -EINVAL; -+ goto exit; -+ } -+ -+ if (ipa_ctrl->num_active_sessions == 0) { -+ dev_err(kbdev->dev, "%s: no active sessions found", __func__); -+ ret = -EINVAL; -+ goto exit; -+ } -+ -+ if (!session->active) { -+ dev_err(kbdev->dev, "%s: session is already inactive", -+ __func__); -+ ret = -EINVAL; -+ goto exit; -+ } -+ -+ for (i = 0; i < session->num_prfcnts; i++) { -+ struct kbase_ipa_control_prfcnt_config *prfcnt_config; -+ u8 type = session->prfcnts[i].type; -+ u8 idx = session->prfcnts[i].select_idx; -+ -+ prfcnt_config = &ipa_ctrl->blocks[type].select[idx]; -+ -+ if (!WARN_ON(prfcnt_config->refcount == 0)) { -+ prfcnt_config->refcount--; -+ if (prfcnt_config->refcount == 0) { -+ new_config = true; -+ ipa_ctrl->blocks[type].num_available_counters++; -+ } -+ } -+ } -+ -+ if (new_config) { -+ u64 select_config[KBASE_IPA_CORE_TYPE_NUM]; -+ -+ build_select_config(ipa_ctrl, select_config); -+ ret = apply_select_config(kbdev, select_config); -+ if (ret) -+ dev_err(kbdev->dev, -+ "%s: failed to apply SELECT configuration", -+ __func__); -+ } -+ -+ session->num_prfcnts = 0; -+ session->active = false; -+ ipa_ctrl->num_active_sessions--; -+ -+exit: -+ spin_unlock_irqrestore(&ipa_ctrl->lock, flags); -+ kbase_pm_context_idle(kbdev); -+ return ret; -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_control_unregister); -+ -+int kbase_ipa_control_query(struct kbase_device *kbdev, const void *client, -+ u64 *values, size_t num_values, u64 *protected_time) -+{ -+ struct kbase_ipa_control *ipa_ctrl; -+ struct kbase_ipa_control_session *session; -+ size_t i; -+ unsigned long flags; -+ bool gpu_ready; -+ -+ if (WARN_ON(kbdev == NULL) || WARN_ON(client == NULL) || -+ WARN_ON(values == NULL)) { -+ dev_err(kbdev->dev, "%s: wrong input arguments", __func__); -+ return -EINVAL; -+ } -+ -+ ipa_ctrl = &kbdev->csf.ipa_control; -+ session = (struct kbase_ipa_control_session *)client; -+ -+ if (WARN_ON(num_values < session->num_prfcnts)) { -+ dev_err(kbdev->dev, -+ "%s: not enough space (%zu) to return all counter values (%zu)", -+ __func__, num_values, session->num_prfcnts); -+ return -EINVAL; -+ } -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ gpu_ready = kbdev->pm.backend.gpu_ready; -+ -+ for (i = 0; i < session->num_prfcnts; i++) { -+ struct kbase_ipa_control_prfcnt *prfcnt = &session->prfcnts[i]; -+ -+ calc_prfcnt_delta(kbdev, prfcnt, gpu_ready); -+ /* Return all the accumulated difference */ -+ values[i] = prfcnt->accumulated_diff; -+ prfcnt->accumulated_diff = 0; -+ } -+ -+ if (protected_time) { -+ u64 time_now = ktime_get_ns(); -+ -+ /* This is the amount of protected-mode time spent prior to -+ * the current protm period. -+ */ -+ *protected_time = session->protm_time; -+ -+ if (kbdev->protected_mode) { -+ *protected_time += -+ time_now - MAX(session->last_query_time, -+ ipa_ctrl->protm_start); -+ } -+ session->last_query_time = time_now; -+ session->protm_time = 0; -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ for (i = session->num_prfcnts; i < num_values; i++) -+ values[i] = 0; -+ -+ return 0; -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_control_query); -+ -+void kbase_ipa_control_handle_gpu_power_off(struct kbase_device *kbdev) -+{ -+ struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; -+ size_t session_idx; -+ int ret; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* GPU should still be ready for use when this function gets called */ -+ WARN_ON(!kbdev->pm.backend.gpu_ready); -+ -+ /* Interrupts are already disabled and interrupt state is also saved */ -+ spin_lock(&ipa_ctrl->lock); -+ -+ /* First disable the automatic sampling through TIMER */ -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), 0); -+ ret = wait_status(kbdev, STATUS_TIMER_ENABLED); -+ if (ret) { -+ dev_err(kbdev->dev, -+ "Wait for disabling of IPA control timer failed: %d", -+ ret); -+ } -+ -+ /* Now issue the manual SAMPLE command */ -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), COMMAND_SAMPLE); -+ ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE); -+ if (ret) { -+ dev_err(kbdev->dev, -+ "Wait for the completion of manual sample failed: %d", -+ ret); -+ } -+ -+ for (session_idx = 0; session_idx < ipa_ctrl->num_active_sessions; -+ session_idx++) { -+ struct kbase_ipa_control_session *session = -+ &ipa_ctrl->sessions[session_idx]; -+ size_t i; -+ -+ for (i = 0; i < session->num_prfcnts; i++) { -+ struct kbase_ipa_control_prfcnt *prfcnt = -+ &session->prfcnts[i]; -+ -+ calc_prfcnt_delta(kbdev, prfcnt, true); -+ } -+ } -+ -+ spin_unlock(&ipa_ctrl->lock); -+} -+ -+void kbase_ipa_control_handle_gpu_power_on(struct kbase_device *kbdev) -+{ -+ struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; -+ int ret; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* GPU should have become ready for use when this function gets called */ -+ WARN_ON(!kbdev->pm.backend.gpu_ready); -+ -+ /* Interrupts are already disabled and interrupt state is also saved */ -+ spin_lock(&ipa_ctrl->lock); -+ -+ /* Re-issue the APPLY command, this is actually needed only for CSHW */ -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), COMMAND_APPLY); -+ ret = wait_status(kbdev, STATUS_COMMAND_ACTIVE); -+ if (ret) { -+ dev_err(kbdev->dev, -+ "Wait for the completion of apply command failed: %d", -+ ret); -+ } -+ -+ /* Re-enable the timer for periodic sampling */ -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), -+ timer_value(ipa_ctrl->cur_gpu_rate)); -+ -+ spin_unlock(&ipa_ctrl->lock); -+} -+ -+void kbase_ipa_control_handle_gpu_reset_pre(struct kbase_device *kbdev) -+{ -+ /* A soft reset is treated as a power down */ -+ kbase_ipa_control_handle_gpu_power_off(kbdev); -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_reset_pre); -+ -+void kbase_ipa_control_handle_gpu_reset_post(struct kbase_device *kbdev) -+{ -+ struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; -+ int ret; -+ u32 status; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ /* GPU should have become ready for use when this function gets called */ -+ WARN_ON(!kbdev->pm.backend.gpu_ready); -+ -+ /* Interrupts are already disabled and interrupt state is also saved */ -+ spin_lock(&ipa_ctrl->lock); -+ -+ /* Check the status reset bit is set before acknowledging it */ -+ status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS)); -+ if (status & STATUS_RESET) { -+ /* Acknowledge the reset command */ -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), COMMAND_RESET_ACK); -+ ret = wait_status(kbdev, STATUS_RESET); -+ if (ret) { -+ dev_err(kbdev->dev, -+ "Wait for the reset ack command failed: %d", -+ ret); -+ } -+ } -+ -+ spin_unlock(&ipa_ctrl->lock); -+ -+ kbase_ipa_control_handle_gpu_power_on(kbdev); -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_reset_post); -+ -+#if MALI_UNIT_TEST -+void kbase_ipa_control_rate_change_notify_test(struct kbase_device *kbdev, -+ u32 clk_index, u32 clk_rate_hz) -+{ -+ struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; -+ struct kbase_ipa_control_listener_data *listener_data = -+ ipa_ctrl->rtm_listener_data; -+ -+ kbase_ipa_control_rate_change_notify(&listener_data->listener, -+ clk_index, clk_rate_hz); -+} -+KBASE_EXPORT_TEST_API(kbase_ipa_control_rate_change_notify_test); -+#endif -+ -+void kbase_ipa_control_protm_entered(struct kbase_device *kbdev) -+{ -+ struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ ipa_ctrl->protm_start = ktime_get_ns(); -+} -+ -+void kbase_ipa_control_protm_exited(struct kbase_device *kbdev) -+{ -+ struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control; -+ size_t i; -+ u64 time_now = ktime_get_ns(); -+ u32 status; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ for (i = 0; i < ipa_ctrl->num_active_sessions; i++) { -+ struct kbase_ipa_control_session *session = -+ &ipa_ctrl->sessions[i]; -+ u64 protm_time = time_now - MAX(session->last_query_time, -+ ipa_ctrl->protm_start); -+ -+ session->protm_time += protm_time; -+ } -+ -+ /* Acknowledge the protected_mode bit in the IPA_CONTROL STATUS -+ * register -+ */ -+ status = kbase_reg_read(kbdev, IPA_CONTROL_REG(STATUS)); -+ if (status & STATUS_PROTECTED_MODE) { -+ int ret; -+ -+ /* Acknowledge the protm command */ -+ kbase_reg_write(kbdev, IPA_CONTROL_REG(COMMAND), -+ COMMAND_PROTECTED_ACK); -+ ret = wait_status(kbdev, STATUS_PROTECTED_MODE); -+ if (ret) { -+ dev_err(kbdev->dev, -+ "Wait for the protm ack command failed: %d", -+ ret); -+ } -+ } -+} -+ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/mali_kbase_csf_ipa_control.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/mali_kbase_csf_ipa_control.h -new file mode 100644 -index 0000000..348a52f ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/ipa_control/mali_kbase_csf_ipa_control.h -@@ -0,0 +1,244 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_IPA_CONTROL_H_ -+#define _KBASE_CSF_IPA_CONTROL_H_ -+ -+#include <mali_kbase.h> -+ -+/** -+ * Maximum index accepted to configure an IPA Control performance counter. -+ */ -+#define KBASE_IPA_CONTROL_CNT_MAX_IDX ((u8)64 * 3) -+ -+/** -+ * struct kbase_ipa_control_perf_counter - Performance counter description -+ * -+ * @scaling_factor: Scaling factor by which the counter's value shall be -+ * multiplied. A scaling factor of 1 corresponds to units -+ * of 1 second if values are normalised by GPU frequency. -+ * @gpu_norm: Indicating whether counter values shall be normalized by -+ * GPU frequency. If true, returned values represent -+ * an interval of time expressed in seconds (when the scaling -+ * factor is set to 1). -+ * @type: Type of counter block for performance counter. -+ * @idx: Index of the performance counter inside the block. -+ * It may be dependent on GPU architecture. -+ * It cannot be greater than KBASE_IPA_CONTROL_CNT_MAX_IDX. -+ * -+ * This structure is used by clients of the IPA Control component to describe -+ * a performance counter that they intend to read. The counter is identified -+ * by block and index. In addition to that, the client also specifies how -+ * values shall be represented. Raw values are a number of GPU cycles; -+ * if normalized, they are divided by GPU frequency and become an interval -+ * of time expressed in seconds, since the GPU frequency is given in Hz. -+ * The client may specify a scaling factor to multiply counter values before -+ * they are divided by frequency, in case the unit of time of 1 second is -+ * too low in resolution. For instance: a scaling factor of 1000 implies -+ * that the returned value is a time expressed in milliseconds; a scaling -+ * factor of 1000 * 1000 implies that the returned value is a time expressed -+ * in microseconds. -+ */ -+struct kbase_ipa_control_perf_counter { -+ u64 scaling_factor; -+ bool gpu_norm; -+ enum kbase_ipa_core_type type; -+ u8 idx; -+}; -+ -+/** -+ * kbase_ipa_control_init - Initialize the IPA Control component -+ * -+ * @kbdev: Pointer to Kbase device. -+ */ -+void kbase_ipa_control_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_ipa_control_term - Terminate the IPA Control component -+ * -+ * @kbdev: Pointer to Kbase device. -+ */ -+void kbase_ipa_control_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_ipa_control_register - Register a client to the IPA Control component -+ * -+ * @kbdev: Pointer to Kbase device. -+ * @perf_counters: Array of performance counters the client intends to read. -+ * For each counter the client specifies block, index, -+ * scaling factor and whether it must be normalized by GPU -+ * frequency. -+ * @num_counters: Number of performance counters. It cannot exceed the total -+ * number of counters that exist on the IPA Control interface. -+ * @client: Handle to an opaque structure set by IPA Control if -+ * the registration is successful. This handle identifies -+ * a client's session and shall be provided in its future -+ * queries. -+ * -+ * A client needs to subscribe to the IPA Control component by declaring which -+ * performance counters it intends to read, and specifying a scaling factor -+ * and whether normalization is requested for each performance counter. -+ * The function shall configure the IPA Control interface accordingly and start -+ * a session for the client that made the request. A unique handle is returned -+ * if registration is successful in order to identify the client's session -+ * and be used for future queries. -+ * -+ * Return: 0 on success, negative -errno on error -+ */ -+int kbase_ipa_control_register( -+ struct kbase_device *kbdev, -+ const struct kbase_ipa_control_perf_counter *perf_counters, -+ size_t num_counters, void **client); -+ -+/** -+ * kbase_ipa_control_unregister - Unregister a client from IPA Control -+ * -+ * @kbdev: Pointer to kbase device. -+ * @client: Handle to an opaque structure that identifies the client session -+ * to terminate, as returned by kbase_ipa_control_register. -+ * -+ * Return: 0 on success, negative -errno on error -+ */ -+int kbase_ipa_control_unregister(struct kbase_device *kbdev, -+ const void *client); -+ -+/** -+ * kbase_ipa_control_query - Query performance counters -+ * -+ * @kbdev: Pointer to kbase device. -+ * @client: Handle to an opaque structure that identifies the client -+ * session, as returned by kbase_ipa_control_register. -+ * @values: Array of values queried from performance counters, whose -+ * length depends on the number of counters requested at -+ * the time of registration. Values are scaled and normalized -+ * and represent the difference since the last query. -+ * @num_values: Number of entries in the array of values that has been -+ * passed by the caller. It must be at least equal to the -+ * number of performance counters the client registered itself -+ * to read. -+ * @protected_time: Time spent in protected mode since last query, -+ * expressed in nanoseconds. This pointer may be NULL if the -+ * client doesn't want to know about this. -+ * -+ * A client that has already opened a session by registering itself to read -+ * some performance counters may use this function to query the values of -+ * those counters. The values returned are normalized by GPU frequency if -+ * requested and then multiplied by the scaling factor provided at the time -+ * of registration. Values always represent a difference since the last query. -+ * -+ * Performance counters are not updated while the GPU operates in protected -+ * mode. For this reason, returned values may be unreliable if the GPU has -+ * been in protected mode since the last query. The function returns success -+ * in that case, but it also gives a measure of how much time has been spent -+ * in protected mode. -+ * -+ * Return: 0 on success, negative -errno on error -+ */ -+int kbase_ipa_control_query(struct kbase_device *kbdev, const void *client, -+ u64 *values, size_t num_values, -+ u64 *protected_time); -+ -+/** -+ * kbase_ipa_control_handle_gpu_power_on - Handle the GPU power on event -+ * -+ * @kbdev: Pointer to kbase device. -+ * -+ * This function is called after GPU has been powered and is ready for use. -+ * After the GPU power on, IPA Control component needs to ensure that the -+ * counters start incrementing again. -+ */ -+void kbase_ipa_control_handle_gpu_power_on(struct kbase_device *kbdev); -+ -+/** -+ * kbase_ipa_control_handle_gpu_power_off - Handle the GPU power off event -+ * -+ * @kbdev: Pointer to kbase device. -+ * -+ * This function is called just before the GPU is powered off when it is still -+ * ready for use. -+ * IPA Control component needs to be aware of the GPU power off so that it can -+ * handle the query from Clients appropriately and return meaningful values -+ * to them. -+ */ -+void kbase_ipa_control_handle_gpu_power_off(struct kbase_device *kbdev); -+ -+/** -+ * kbase_ipa_control_handle_gpu_reset_pre - Handle the pre GPU reset event -+ * -+ * @kbdev: Pointer to kbase device. -+ * -+ * This function is called when the GPU is about to be reset. -+ */ -+void kbase_ipa_control_handle_gpu_reset_pre(struct kbase_device *kbdev); -+ -+/** -+ * kbase_ipa_control_handle_gpu_reset_post - Handle the post GPU reset event -+ * -+ * @kbdev: Pointer to kbase device. -+ * -+ * This function is called after the GPU has been reset. -+ */ -+void kbase_ipa_control_handle_gpu_reset_post(struct kbase_device *kbdev); -+ -+#if MALI_UNIT_TEST -+/** -+ * kbase_ipa_control_rate_change_notify_test - Notify GPU rate change -+ * (only for testing) -+ * -+ * @kbdev: Pointer to kbase device. -+ * @clk_index: Index of the clock for which the change has occurred. -+ * @clk_rate_hz: Clock frequency(Hz). -+ * -+ * Notify the IPA Control component about a GPU rate change. -+ */ -+void kbase_ipa_control_rate_change_notify_test(struct kbase_device *kbdev, -+ u32 clk_index, u32 clk_rate_hz); -+#endif /* MALI_UNIT_TEST */ -+ -+/** -+ * kbase_ipa_control_protm_entered - Tell IPA_CONTROL that protected mode -+ * has been entered. -+ * -+ * @kbdev: Pointer to kbase device. -+ * -+ * This function provides a means through which IPA_CONTROL can be informed -+ * that the GPU has entered protected mode. Since the GPU cannot access -+ * performance counters while in this mode, this information is useful as -+ * it implies (a) the values of these registers cannot change, so theres no -+ * point trying to read them, and (b) IPA_CONTROL has a means through which -+ * to record the duration of time the GPU is in protected mode, which can -+ * then be forwarded on to clients, who may wish, for example, to assume -+ * that the GPU was busy 100% of the time while in this mode. -+ */ -+void kbase_ipa_control_protm_entered(struct kbase_device *kbdev); -+ -+/** -+ * kbase_ipa_control_protm_exited - Tell IPA_CONTROL that protected mode -+ * has been exited. -+ * -+ * @kbdev: Pointer to kbase device -+ * -+ * This function provides a means through which IPA_CONTROL can be informed -+ * that the GPU has exited from protected mode. -+ */ -+void kbase_ipa_control_protm_exited(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_CSF_IPA_CONTROL_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf.c -new file mode 100644 -index 0000000..d49e343 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf.c -@@ -0,0 +1,3069 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <gpu/mali_kbase_gpu_fault.h> -+#include <mali_kbase_reset_gpu.h> -+#include "mali_kbase_csf.h" -+#include "backend/gpu/mali_kbase_pm_internal.h" -+#include <linux/export.h> -+#include <linux/priority_control_manager.h> -+#include <linux/shmem_fs.h> -+#include <uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h> -+#include "mali_kbase_csf_tiler_heap.h" -+#include <mmu/mali_kbase_mmu.h> -+#include "mali_kbase_csf_timeout.h" -+#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> -+ -+#define CS_REQ_EXCEPTION_MASK (CS_REQ_FAULT_MASK | CS_REQ_FATAL_MASK) -+#define CS_ACK_EXCEPTION_MASK (CS_ACK_FAULT_MASK | CS_ACK_FATAL_MASK) -+#define POWER_DOWN_LATEST_FLUSH_VALUE ((u32)1) -+ -+/** -+ * struct kbase_csf_event - CSF event callback. -+ * -+ * This structure belongs to the list of events which is part of a Kbase -+ * context, and describes a callback function with a custom parameter to pass -+ * to it when a CSF event is signalled. -+ * -+ * @link: Link to the rest of the list. -+ * @kctx: Pointer to the Kbase context this event belongs to. -+ * @callback: Callback function to call when a CSF event is signalled. -+ * @param: Parameter to pass to the callback function. -+ */ -+struct kbase_csf_event { -+ struct list_head link; -+ struct kbase_context *kctx; -+ kbase_csf_event_callback *callback; -+ void *param; -+}; -+ -+const u8 kbasep_csf_queue_group_priority_to_relative[BASE_QUEUE_GROUP_PRIORITY_COUNT] = { -+ KBASE_QUEUE_GROUP_PRIORITY_HIGH, -+ KBASE_QUEUE_GROUP_PRIORITY_MEDIUM, -+ KBASE_QUEUE_GROUP_PRIORITY_LOW, -+ KBASE_QUEUE_GROUP_PRIORITY_REALTIME -+}; -+const u8 kbasep_csf_relative_to_queue_group_priority[KBASE_QUEUE_GROUP_PRIORITY_COUNT] = { -+ BASE_QUEUE_GROUP_PRIORITY_REALTIME, -+ BASE_QUEUE_GROUP_PRIORITY_HIGH, -+ BASE_QUEUE_GROUP_PRIORITY_MEDIUM, -+ BASE_QUEUE_GROUP_PRIORITY_LOW -+}; -+ -+static void put_user_pages_mmap_handle(struct kbase_context *kctx, -+ struct kbase_queue *queue) -+{ -+ unsigned long cookie_nr; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ if (queue->handle == BASEP_MEM_INVALID_HANDLE) -+ return; -+ -+ cookie_nr = -+ PFN_DOWN(queue->handle - BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); -+ -+ if (!WARN_ON(kctx->csf.user_pages_info[cookie_nr] != queue)) { -+ /* free up cookie */ -+ kctx->csf.user_pages_info[cookie_nr] = NULL; -+ bitmap_set(kctx->csf.cookies, cookie_nr, 1); -+ } -+ -+ queue->handle = BASEP_MEM_INVALID_HANDLE; -+} -+ -+/* Reserve a cookie, to be returned as a handle to userspace for creating -+ * the CPU mapping of the pair of input/output pages and Hw doorbell page. -+ * Will return 0 in case of success otherwise negative on failure. -+ */ -+static int get_user_pages_mmap_handle(struct kbase_context *kctx, -+ struct kbase_queue *queue) -+{ -+ unsigned long cookie, cookie_nr; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ if (bitmap_empty(kctx->csf.cookies, -+ KBASE_CSF_NUM_USER_IO_PAGES_HANDLE)) { -+ dev_err(kctx->kbdev->dev, -+ "No csf cookies available for allocation!"); -+ return -ENOMEM; -+ } -+ -+ /* allocate a cookie */ -+ cookie_nr = find_first_bit(kctx->csf.cookies, -+ KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); -+ if (kctx->csf.user_pages_info[cookie_nr]) { -+ dev_err(kctx->kbdev->dev, -+ "Inconsistent state of csf cookies!"); -+ return -EINVAL; -+ } -+ kctx->csf.user_pages_info[cookie_nr] = queue; -+ bitmap_clear(kctx->csf.cookies, cookie_nr, 1); -+ -+ /* relocate to correct base */ -+ cookie = cookie_nr + PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); -+ cookie <<= PAGE_SHIFT; -+ -+ queue->handle = (u64)cookie; -+ -+ return 0; -+} -+ -+static void gpu_munmap_user_io_pages(struct kbase_context *kctx, -+ struct kbase_va_region *reg) -+{ -+ size_t num_pages = 2; -+ -+ kbase_mmu_teardown_pages(kctx->kbdev, &kctx->kbdev->csf.mcu_mmu, -+ reg->start_pfn, num_pages, MCU_AS_NR); -+ -+ WARN_ON(reg->flags & KBASE_REG_FREE); -+ -+ mutex_lock(&kctx->kbdev->csf.reg_lock); -+ kbase_remove_va_region(reg); -+ mutex_unlock(&kctx->kbdev->csf.reg_lock); -+} -+ -+static void init_user_io_pages(struct kbase_queue *queue) -+{ -+ u32 *input_addr = (u32 *)(queue->user_io_addr); -+ u32 *output_addr = (u32 *)(queue->user_io_addr + PAGE_SIZE); -+ -+ input_addr[CS_INSERT_LO/4] = 0; -+ input_addr[CS_INSERT_HI/4] = 0; -+ -+ input_addr[CS_EXTRACT_INIT_LO/4] = 0; -+ input_addr[CS_EXTRACT_INIT_HI/4] = 0; -+ -+ output_addr[CS_EXTRACT_LO/4] = 0; -+ output_addr[CS_EXTRACT_HI/4] = 0; -+ -+ output_addr[CS_ACTIVE/4] = 0; -+} -+ -+/* Map the input/output pages in the shared interface segment of MCU firmware -+ * address space. -+ */ -+static int gpu_mmap_user_io_pages(struct kbase_device *kbdev, -+ struct tagged_addr *phys, struct kbase_va_region *reg) -+{ -+ unsigned long mem_flags = KBASE_REG_GPU_RD; -+ const size_t num_pages = 2; -+ int ret; -+ -+#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ -+ ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ -+ (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) -+ mem_flags |= -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); -+#else -+ if (kbdev->system_coherency == COHERENCY_NONE) { -+ mem_flags |= -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); -+ } else { -+ mem_flags |= KBASE_REG_SHARE_BOTH | -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); -+ } -+#endif -+ -+ mutex_lock(&kbdev->csf.reg_lock); -+ ret = kbase_add_va_region_rbtree(kbdev, reg, 0, num_pages, 1); -+ reg->flags &= ~KBASE_REG_FREE; -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ if (ret) -+ return ret; -+ -+ /* Map input page */ -+ ret = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, -+ reg->start_pfn, &phys[0], -+ 1, mem_flags, MCU_AS_NR, -+ KBASE_MEM_GROUP_CSF_IO); -+ if (ret) -+ goto bad_insert; -+ -+ /* Map output page, it needs rw access */ -+ mem_flags |= KBASE_REG_GPU_WR; -+ ret = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, -+ reg->start_pfn + 1, &phys[1], -+ 1, mem_flags, MCU_AS_NR, -+ KBASE_MEM_GROUP_CSF_IO); -+ if (ret) -+ goto bad_insert_output_page; -+ -+ return 0; -+ -+bad_insert_output_page: -+ kbase_mmu_teardown_pages(kbdev, &kbdev->csf.mcu_mmu, -+ reg->start_pfn, 1, MCU_AS_NR); -+bad_insert: -+ mutex_lock(&kbdev->csf.reg_lock); -+ kbase_remove_va_region(reg); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ return ret; -+} -+ -+static void kernel_unmap_user_io_pages(struct kbase_context *kctx, -+ struct kbase_queue *queue) -+{ -+ const size_t num_pages = 2; -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ vunmap(queue->user_io_addr); -+ -+ WARN_ON(num_pages > atomic_read(&kctx->permanent_mapped_pages)); -+ atomic_sub(num_pages, &kctx->permanent_mapped_pages); -+ -+ kbase_gpu_vm_unlock(kctx); -+} -+ -+static int kernel_map_user_io_pages(struct kbase_context *kctx, -+ struct kbase_queue *queue) -+{ -+ struct page *page_list[2]; -+ pgprot_t cpu_map_prot; -+ int ret = 0; -+ size_t i; -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ if (ARRAY_SIZE(page_list) > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES - -+ atomic_read(&kctx->permanent_mapped_pages))) { -+ ret = -ENOMEM; -+ goto unlock; -+ } -+ -+ /* The pages are mapped to Userspace also, so use the same mapping -+ * attributes as used inside the CPU page fault handler. -+ */ -+#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ -+ ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ -+ (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) -+ cpu_map_prot = pgprot_device(PAGE_KERNEL); -+#else -+ if (kctx->kbdev->system_coherency == COHERENCY_NONE) -+ cpu_map_prot = pgprot_writecombine(PAGE_KERNEL); -+ else -+ cpu_map_prot = PAGE_KERNEL; -+#endif -+ -+ for (i = 0; i < ARRAY_SIZE(page_list); i++) -+ page_list[i] = as_page(queue->phys[i]); -+ -+ queue->user_io_addr = vmap(page_list, ARRAY_SIZE(page_list), VM_MAP, cpu_map_prot); -+ -+ if (!queue->user_io_addr) -+ ret = -ENOMEM; -+ else -+ atomic_add(ARRAY_SIZE(page_list), &kctx->permanent_mapped_pages); -+ -+unlock: -+ kbase_gpu_vm_unlock(kctx); -+ return ret; -+} -+ -+static void term_queue_group(struct kbase_queue_group *group); -+static void get_queue(struct kbase_queue *queue); -+static void release_queue(struct kbase_queue *queue); -+ -+/** -+ * kbase_csf_free_command_stream_user_pages() - Free the resources allocated -+ * for a queue at the time of bind. -+ * -+ * @kctx: Address of the kbase context within which the queue was created. -+ * @queue: Pointer to the queue to be unlinked. -+ * -+ * This function will free the pair of physical pages allocated for a GPU -+ * command queue, and also release the hardware doorbell page, that were mapped -+ * into the process address space to enable direct submission of commands to -+ * the hardware. Also releases the reference taken on the queue when the mapping -+ * was created. -+ * -+ * This function will be called only when the mapping is being removed and -+ * so the resources for queue will not get freed up until the mapping is -+ * removed even though userspace could have terminated the queue. -+ * Kernel will ensure that the termination of Kbase context would only be -+ * triggered after the mapping is removed. -+ * -+ * If an explicit or implicit unbind was missed by the userspace then the -+ * mapping will persist. On process exit kernel itself will remove the mapping. -+ */ -+static void kbase_csf_free_command_stream_user_pages(struct kbase_context *kctx, -+ struct kbase_queue *queue) -+{ -+ const size_t num_pages = 2; -+ -+ gpu_munmap_user_io_pages(kctx, queue->reg); -+ kernel_unmap_user_io_pages(kctx, queue); -+ -+ kbase_mem_pool_free_pages( -+ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_IO], -+ num_pages, queue->phys, true, false); -+ -+ kfree(queue->reg); -+ queue->reg = NULL; -+ -+ /* If the queue has already been terminated by userspace -+ * then the ref count for queue object will drop to 0 here. -+ */ -+ release_queue(queue); -+} -+ -+int kbase_csf_alloc_command_stream_user_pages(struct kbase_context *kctx, -+ struct kbase_queue *queue) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_va_region *reg; -+ const size_t num_pages = 2; -+ int ret; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ reg = kbase_alloc_free_region(&kctx->kbdev->csf.shared_reg_rbtree, 0, -+ num_pages, KBASE_REG_ZONE_MCU_SHARED); -+ if (!reg) -+ return -ENOMEM; -+ -+ ret = kbase_mem_pool_alloc_pages( -+ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_IO], -+ num_pages, queue->phys, false); -+ -+ if (ret != num_pages) -+ goto phys_alloc_failed; -+ -+ ret = kernel_map_user_io_pages(kctx, queue); -+ if (ret) -+ goto kernel_map_failed; -+ -+ init_user_io_pages(queue); -+ -+ ret = gpu_mmap_user_io_pages(kctx->kbdev, queue->phys, reg); -+ if (ret) -+ goto gpu_mmap_failed; -+ -+ queue->reg = reg; -+ -+ mutex_lock(&kbdev->csf.reg_lock); -+ if (kbdev->csf.db_file_offsets > -+ (U32_MAX - BASEP_QUEUE_NR_MMAP_USER_PAGES + 1)) -+ kbdev->csf.db_file_offsets = 0; -+ -+ queue->db_file_offset = kbdev->csf.db_file_offsets; -+ kbdev->csf.db_file_offsets += BASEP_QUEUE_NR_MMAP_USER_PAGES; -+ -+ WARN(atomic_read(&queue->refcount) != 1, "Incorrect refcounting for queue object\n"); -+ /* This is the second reference taken on the queue object and -+ * would be dropped only when the IO mapping is removed either -+ * explicitly by userspace or implicitly by kernel on process exit. -+ */ -+ get_queue(queue); -+ queue->bind_state = KBASE_CSF_QUEUE_BOUND; -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ return 0; -+ -+gpu_mmap_failed: -+ kernel_unmap_user_io_pages(kctx, queue); -+ -+kernel_map_failed: -+ kbase_mem_pool_free_pages( -+ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_IO], -+ num_pages, queue->phys, false, false); -+ -+phys_alloc_failed: -+ kfree(reg); -+ -+ return -ENOMEM; -+} -+ -+static struct kbase_queue_group *find_queue_group(struct kbase_context *kctx, -+ u8 group_handle) -+{ -+ uint index = group_handle; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ if (index < MAX_QUEUE_GROUP_NUM && kctx->csf.queue_groups[index]) { -+ if (WARN_ON(kctx->csf.queue_groups[index]->handle != index)) -+ return NULL; -+ return kctx->csf.queue_groups[index]; -+ } -+ -+ return NULL; -+} -+ -+int kbase_csf_queue_group_handle_is_valid(struct kbase_context *kctx, -+ u8 group_handle) -+{ -+ struct kbase_queue_group *group; -+ -+ mutex_lock(&kctx->csf.lock); -+ group = find_queue_group(kctx, group_handle); -+ mutex_unlock(&kctx->csf.lock); -+ -+ return group ? 0 : -EINVAL; -+} -+ -+static struct kbase_queue *find_queue(struct kbase_context *kctx, u64 base_addr) -+{ -+ struct kbase_queue *queue; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ list_for_each_entry(queue, &kctx->csf.queue_list, link) { -+ if (base_addr == queue->base_addr) -+ return queue; -+ } -+ -+ return NULL; -+} -+ -+static void get_queue(struct kbase_queue *queue) -+{ -+ WARN_ON(!atomic_inc_not_zero(&queue->refcount)); -+} -+ -+static void release_queue(struct kbase_queue *queue) -+{ -+ lockdep_assert_held(&queue->kctx->csf.lock); -+ -+ WARN_ON(atomic_read(&queue->refcount) <= 0); -+ -+ if (atomic_dec_and_test(&queue->refcount)) { -+ /* The queue can't still be on the per context list. */ -+ WARN_ON(!list_empty(&queue->link)); -+ WARN_ON(queue->group); -+ kfree(queue); -+ } -+} -+ -+static void oom_event_worker(struct work_struct *data); -+static void fatal_event_worker(struct work_struct *data); -+ -+/* Between reg and reg_ex, one and only one must be null */ -+static int csf_queue_register_internal(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_register *reg, -+ struct kbase_ioctl_cs_queue_register_ex *reg_ex) -+{ -+ struct kbase_queue *queue; -+ int ret = 0; -+ struct kbase_va_region *region; -+ u64 queue_addr; -+ size_t queue_size; -+ -+ /* Only one pointer expected, otherwise coding error */ -+ if ((reg == NULL && reg_ex == NULL) || (reg && reg_ex)) { -+ dev_err(kctx->kbdev->dev, -+ "Error, one and only one param-ptr expected!"); -+ return -EINVAL; -+ } -+ -+ /* struct kbase_ioctl_cs_queue_register_ex contains a full -+ * struct kbase_ioctl_cs_queue_register at the start address. So -+ * the pointer can be safely cast to pointing to a -+ * kbase_ioctl_cs_queue_register object. -+ */ -+ if (reg_ex) -+ reg = (struct kbase_ioctl_cs_queue_register *)reg_ex; -+ -+ /* Validate the queue priority */ -+ if (reg->priority > BASE_QUEUE_MAX_PRIORITY) -+ return -EINVAL; -+ -+ queue_addr = reg->buffer_gpu_addr; -+ queue_size = reg->buffer_size >> PAGE_SHIFT; -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ /* Check if queue is already registered */ -+ if (find_queue(kctx, queue_addr) != NULL) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ /* Check if the queue address is valid */ -+ kbase_gpu_vm_lock(kctx); -+ region = kbase_region_tracker_find_region_enclosing_address(kctx, -+ queue_addr); -+ -+ if (kbase_is_region_invalid_or_free(region)) { -+ ret = -ENOENT; -+ goto out_unlock_vm; -+ } -+ -+ if (queue_size > (region->nr_pages - -+ ((queue_addr >> PAGE_SHIFT) - region->start_pfn))) { -+ ret = -EINVAL; -+ goto out_unlock_vm; -+ } -+ -+ /* Check address validity on cs_trace buffer etc. Don't care -+ * if not enabled (i.e. when size is 0). -+ */ -+ if (reg_ex && reg_ex->ex_buffer_size) { -+ int buf_pages = (reg_ex->ex_buffer_size + -+ (1 << PAGE_SHIFT) - 1) >> PAGE_SHIFT; -+ -+ region = kbase_region_tracker_find_region_enclosing_address( -+ kctx, reg_ex->ex_buffer_base); -+ if (kbase_is_region_invalid_or_free(region)) { -+ ret = -ENOENT; -+ goto out_unlock_vm; -+ } -+ -+ if (buf_pages > (region->nr_pages - -+ ((reg_ex->ex_buffer_base >> PAGE_SHIFT) - -+ region->start_pfn))) { -+ ret = -EINVAL; -+ goto out_unlock_vm; -+ } -+ -+ region = kbase_region_tracker_find_region_enclosing_address( -+ kctx, reg_ex->ex_offset_var_addr); -+ if (kbase_is_region_invalid_or_free(region)) { -+ ret = -ENOENT; -+ goto out_unlock_vm; -+ } -+ } -+ -+ queue = kzalloc(sizeof(struct kbase_queue), GFP_KERNEL); -+ -+ if (!queue) { -+ ret = -ENOMEM; -+ goto out_unlock_vm; -+ } -+ -+ queue->kctx = kctx; -+ queue->base_addr = queue_addr; -+ queue->queue_reg = region; -+ queue->size = (queue_size << PAGE_SHIFT); -+ queue->csi_index = KBASEP_IF_NR_INVALID; -+ queue->enabled = false; -+ -+ queue->priority = reg->priority; -+ atomic_set(&queue->refcount, 1); -+ -+ queue->group = NULL; -+ queue->bind_state = KBASE_CSF_QUEUE_UNBOUND; -+ queue->handle = BASEP_MEM_INVALID_HANDLE; -+ queue->doorbell_nr = KBASEP_USER_DB_NR_INVALID; -+ -+ queue->status_wait = 0; -+ queue->sync_ptr = 0; -+ queue->sync_value = 0; -+ -+ queue->sb_status = 0; -+ queue->blocked_reason = CS_STATUS_BLOCKED_REASON_REASON_UNBLOCKED; -+ -+ INIT_LIST_HEAD(&queue->link); -+ INIT_LIST_HEAD(&queue->error.link); -+ INIT_WORK(&queue->oom_event_work, oom_event_worker); -+ INIT_WORK(&queue->fatal_event_work, fatal_event_worker); -+ list_add(&queue->link, &kctx->csf.queue_list); -+ -+ region->flags |= KBASE_REG_NO_USER_FREE; -+ -+ /* Initialize the cs_trace configuration parameters, When buffer_size -+ * is 0, trace is disabled. Here we only update the fields when -+ * enabled, otherwise leave them as default zeros. -+ */ -+ if (reg_ex && reg_ex->ex_buffer_size) { -+ u32 cfg = CS_INSTR_CONFIG_EVENT_SIZE_SET( -+ 0, reg_ex->ex_event_size); -+ cfg = CS_INSTR_CONFIG_EVENT_STATE_SET( -+ cfg, reg_ex->ex_event_state); -+ -+ queue->trace_cfg = cfg; -+ queue->trace_buffer_size = reg_ex->ex_buffer_size; -+ queue->trace_buffer_base = reg_ex->ex_buffer_base; -+ queue->trace_offset_ptr = reg_ex->ex_offset_var_addr; -+ } -+ -+out_unlock_vm: -+ kbase_gpu_vm_unlock(kctx); -+out: -+ mutex_unlock(&kctx->csf.lock); -+ -+ return ret; -+} -+ -+int kbase_csf_queue_register(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_register *reg) -+{ -+ return csf_queue_register_internal(kctx, reg, NULL); -+} -+ -+int kbase_csf_queue_register_ex(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_register_ex *reg) -+{ -+ struct kbase_csf_global_iface const *const iface = -+ &kctx->kbdev->csf.global_iface; -+ u32 const glb_version = iface->version; -+ u32 instr = iface->instr_features; -+ u8 max_size = GLB_INSTR_FEATURES_EVENT_SIZE_MAX_GET(instr); -+ u32 min_buf_size = (1u << reg->ex_event_size) * -+ GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_GET(instr); -+ -+ /* If cs_trace_command not supported, the call fails */ -+ if (glb_version < kbase_csf_interface_version(1, 1, 0)) -+ return -EINVAL; -+ -+ /* Validate the cs_trace configuration parameters */ -+ if (reg->ex_buffer_size && -+ ((reg->ex_event_size > max_size) || -+ (reg->ex_buffer_size & (reg->ex_buffer_size - 1)) || -+ (reg->ex_buffer_size < min_buf_size))) -+ return -EINVAL; -+ -+ return csf_queue_register_internal(kctx, NULL, reg); -+} -+ -+static void unbind_queue(struct kbase_context *kctx, -+ struct kbase_queue *queue); -+ -+void kbase_csf_queue_terminate(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_terminate *term) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_queue *queue; -+ int err; -+ bool reset_prevented = false; -+ -+ err = kbase_reset_gpu_prevent_and_wait(kbdev); -+ if (err) -+ dev_warn( -+ kbdev->dev, -+ "Unsuccessful GPU reset detected when terminating queue (buffer_addr=0x%.16llx), attempting to terminate regardless", -+ term->buffer_gpu_addr); -+ else -+ reset_prevented = true; -+ -+ mutex_lock(&kctx->csf.lock); -+ queue = find_queue(kctx, term->buffer_gpu_addr); -+ -+ if (queue) { -+ unsigned long flags; -+ -+ /* As the GPU queue has been terminated by the -+ * user space, undo the actions that were performed when the -+ * queue was registered i.e. remove the queue from the per -+ * context list & release the initial reference. The subsequent -+ * lookups for the queue in find_queue() would fail. -+ */ -+ list_del_init(&queue->link); -+ -+ /* Stop the CSI to which queue was bound */ -+ unbind_queue(kctx, queue); -+ -+ kbase_gpu_vm_lock(kctx); -+ if (!WARN_ON(!queue->queue_reg)) { -+ /* After this the Userspace would be able to free the -+ * memory for GPU queue. In case the Userspace missed -+ * terminating the queue, the cleanup will happen on -+ * context termination where teardown of region tracker -+ * would free up the GPU queue memory. -+ */ -+ queue->queue_reg->flags &= ~KBASE_REG_NO_USER_FREE; -+ } -+ kbase_gpu_vm_unlock(kctx); -+ -+ spin_lock_irqsave(&kctx->csf.event_lock, flags); -+ dev_dbg(kctx->kbdev->dev, -+ "Remove any pending command queue fatal from context %pK\n", -+ (void *)kctx); -+ list_del_init(&queue->error.link); -+ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); -+ -+ release_queue(queue); -+ } -+ -+ mutex_unlock(&kctx->csf.lock); -+ if (reset_prevented) -+ kbase_reset_gpu_allow(kbdev); -+} -+ -+int kbase_csf_queue_bind(struct kbase_context *kctx, union kbase_ioctl_cs_queue_bind *bind) -+{ -+ struct kbase_queue *queue; -+ struct kbase_queue_group *group; -+ u8 max_streams; -+ int ret = -EINVAL; -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ group = find_queue_group(kctx, bind->in.group_handle); -+ queue = find_queue(kctx, bind->in.buffer_gpu_addr); -+ -+ if (!group || !queue) -+ goto out; -+ -+ /* For the time being, all CSGs have the same number of CSs -+ * so we check CSG 0 for this number -+ */ -+ max_streams = kctx->kbdev->csf.global_iface.groups[0].stream_num; -+ -+ if (bind->in.csi_index >= max_streams) -+ goto out; -+ -+ if (group->run_state == KBASE_CSF_GROUP_TERMINATED) -+ goto out; -+ -+ if (queue->group || group->bound_queues[bind->in.csi_index]) -+ goto out; -+ -+ ret = get_user_pages_mmap_handle(kctx, queue); -+ if (ret) -+ goto out; -+ -+ bind->out.mmap_handle = queue->handle; -+ group->bound_queues[bind->in.csi_index] = queue; -+ queue->group = group; -+ queue->csi_index = bind->in.csi_index; -+ queue->bind_state = KBASE_CSF_QUEUE_BIND_IN_PROGRESS; -+ -+out: -+ mutex_unlock(&kctx->csf.lock); -+ -+ return ret; -+} -+ -+static struct kbase_queue_group *get_bound_queue_group( -+ struct kbase_queue *queue) -+{ -+ struct kbase_context *kctx = queue->kctx; -+ struct kbase_queue_group *group; -+ -+ if (queue->bind_state == KBASE_CSF_QUEUE_UNBOUND) -+ return NULL; -+ -+ if (!queue->group) -+ return NULL; -+ -+ if (queue->csi_index == KBASEP_IF_NR_INVALID) { -+ dev_warn(kctx->kbdev->dev, "CS interface index is incorrect\n"); -+ return NULL; -+ } -+ -+ group = queue->group; -+ -+ if (group->bound_queues[queue->csi_index] != queue) { -+ dev_warn(kctx->kbdev->dev, "Incorrect mapping between queues & queue groups\n"); -+ return NULL; -+ } -+ -+ return group; -+} -+ -+void kbase_csf_ring_csg_doorbell(struct kbase_device *kbdev, int slot) -+{ -+ if (WARN_ON(slot < 0)) -+ return; -+ -+ kbase_csf_ring_csg_slots_doorbell(kbdev, (u32) (1 << slot)); -+} -+ -+void kbase_csf_ring_csg_slots_doorbell(struct kbase_device *kbdev, -+ u32 slot_bitmap) -+{ -+ const struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ const u32 allowed_bitmap = -+ (u32) ((1U << kbdev->csf.global_iface.group_num) - 1); -+ u32 value; -+ -+ if (WARN_ON(slot_bitmap > allowed_bitmap)) -+ return; -+ -+ value = kbase_csf_firmware_global_output(global_iface, GLB_DB_ACK); -+ value ^= slot_bitmap; -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_DB_REQ, value, -+ slot_bitmap); -+ -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+} -+ -+void kbase_csf_ring_cs_user_doorbell(struct kbase_device *kbdev, -+ struct kbase_queue *queue) -+{ -+ mutex_lock(&kbdev->csf.reg_lock); -+ -+ if (queue->doorbell_nr != KBASEP_USER_DB_NR_INVALID) -+ kbase_csf_ring_doorbell(kbdev, queue->doorbell_nr); -+ -+ mutex_unlock(&kbdev->csf.reg_lock); -+} -+ -+void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, -+ int csi_index, int csg_nr, -+ bool ring_csg_doorbell) -+{ -+ struct kbase_csf_cmd_stream_group_info *ginfo; -+ u32 value; -+ -+ if (WARN_ON(csg_nr < 0) || -+ WARN_ON(csg_nr >= kbdev->csf.global_iface.group_num)) -+ return; -+ -+ ginfo = &kbdev->csf.global_iface.groups[csg_nr]; -+ -+ if (WARN_ON(csi_index < 0) || -+ WARN_ON(csi_index >= ginfo->stream_num)) -+ return; -+ -+ value = kbase_csf_firmware_csg_output(ginfo, CSG_DB_ACK); -+ value ^= (1 << csi_index); -+ kbase_csf_firmware_csg_input_mask(ginfo, CSG_DB_REQ, value, -+ 1 << csi_index); -+ -+ if (likely(ring_csg_doorbell)) -+ kbase_csf_ring_csg_doorbell(kbdev, csg_nr); -+} -+ -+int kbase_csf_queue_kick(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_kick *kick) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_queue_group *group; -+ struct kbase_queue *queue; -+ int err = 0; -+ -+ err = kbase_reset_gpu_prevent_and_wait(kbdev); -+ if (err) { -+ dev_warn( -+ kbdev->dev, -+ "Unsuccessful GPU reset detected when kicking queue (buffer_addr=0x%.16llx)", -+ kick->buffer_gpu_addr); -+ return err; -+ } -+ -+ mutex_lock(&kctx->csf.lock); -+ queue = find_queue(kctx, kick->buffer_gpu_addr); -+ if (!queue) -+ err = -EINVAL; -+ -+ if (!err) { -+ group = get_bound_queue_group(queue); -+ if (!group) { -+ dev_err(kctx->kbdev->dev, "queue not bound\n"); -+ err = -EINVAL; -+ } -+ } -+ -+ if (!err) -+ err = kbase_csf_scheduler_queue_start(queue); -+ mutex_unlock(&kctx->csf.lock); -+ kbase_reset_gpu_allow(kbdev); -+ -+ return err; -+} -+ -+static void unbind_stopped_queue(struct kbase_context *kctx, -+ struct kbase_queue *queue) -+{ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ if (queue->bind_state != KBASE_CSF_QUEUE_UNBOUND) { -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kctx->kbdev, &flags); -+ bitmap_clear(queue->group->protm_pending_bitmap, -+ queue->csi_index, 1); -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kctx->kbdev, PROTM_PENDING_CLEAR, -+ queue->group, queue, queue->group->protm_pending_bitmap[0]); -+ queue->group->bound_queues[queue->csi_index] = NULL; -+ queue->group = NULL; -+ kbase_csf_scheduler_spin_unlock(kctx->kbdev, flags); -+ -+ put_user_pages_mmap_handle(kctx, queue); -+ queue->bind_state = KBASE_CSF_QUEUE_UNBOUND; -+ } -+} -+/** -+ * unbind_queue() - Remove the linkage between a GPU command queue and the group -+ * to which it was bound or being bound. -+ * -+ * @kctx: Address of the kbase context within which the queue was created. -+ * @queue: Pointer to the queue to be unlinked. -+ * -+ * This function will also send the stop request to firmware for the CS -+ * if the group to which the GPU command queue was bound is scheduled. -+ * -+ * This function would be called when :- -+ * - queue is being unbound. This would happen when the IO mapping -+ * created on bind is removed explicitly by userspace or the process -+ * is getting exited. -+ * - queue group is being terminated which still has queues bound -+ * to it. This could happen on an explicit terminate request from userspace -+ * or when the kbase context is being terminated. -+ * - queue is being terminated without completing the bind operation. -+ * This could happen if either the queue group is terminated -+ * after the CS_QUEUE_BIND ioctl but before the 2nd part of bind operation -+ * to create the IO mapping is initiated. -+ * - There is a failure in executing the 2nd part of bind operation, inside the -+ * mmap handler, which creates the IO mapping for queue. -+ */ -+ -+static void unbind_queue(struct kbase_context *kctx, struct kbase_queue *queue) -+{ -+ kbase_reset_gpu_assert_failed_or_prevented(kctx->kbdev); -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ if (queue->bind_state != KBASE_CSF_QUEUE_UNBOUND) { -+ if (queue->bind_state == KBASE_CSF_QUEUE_BOUND) -+ kbase_csf_scheduler_queue_stop(queue); -+ -+ unbind_stopped_queue(kctx, queue); -+ } -+} -+ -+void kbase_csf_queue_unbind(struct kbase_queue *queue) -+{ -+ struct kbase_context *kctx = queue->kctx; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ /* As the process itself is exiting, the termination of queue group can -+ * be done which would be much faster than stopping of individual -+ * queues. This would ensure a faster exit for the process especially -+ * in the case where CSI gets stuck. -+ * The CSI STOP request will wait for the in flight work to drain -+ * whereas CSG TERM request would result in an immediate abort or -+ * cancellation of the pending work. -+ */ -+ if (current->flags & PF_EXITING) { -+ struct kbase_queue_group *group = get_bound_queue_group(queue); -+ -+ if (group) -+ term_queue_group(group); -+ -+ WARN_ON(queue->bind_state != KBASE_CSF_QUEUE_UNBOUND); -+ } else { -+ unbind_queue(kctx, queue); -+ } -+ -+ /* Free the resources, if allocated for this queue. */ -+ if (queue->reg) -+ kbase_csf_free_command_stream_user_pages(kctx, queue); -+} -+ -+void kbase_csf_queue_unbind_stopped(struct kbase_queue *queue) -+{ -+ struct kbase_context *kctx = queue->kctx; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ WARN_ON(queue->bind_state == KBASE_CSF_QUEUE_BOUND); -+ unbind_stopped_queue(kctx, queue); -+ -+ /* Free the resources, if allocated for this queue. */ -+ if (queue->reg) -+ kbase_csf_free_command_stream_user_pages(kctx, queue); -+} -+ -+/** -+ * find_free_group_handle() - Find a free handle for a queue group -+ * -+ * @kctx: Address of the kbase context within which the queue group -+ * is to be created. -+ * -+ * Return: a queue group handle on success, or a negative error code on failure. -+ */ -+static int find_free_group_handle(struct kbase_context *const kctx) -+{ -+ /* find the available index in the array of CSGs per this context */ -+ int idx, group_handle = -ENOMEM; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ for (idx = 0; -+ (idx != MAX_QUEUE_GROUP_NUM) && (group_handle < 0); -+ idx++) { -+ if (!kctx->csf.queue_groups[idx]) -+ group_handle = idx; -+ } -+ -+ return group_handle; -+} -+ -+/** -+ * iface_has_enough_streams() - Check that at least one CSG supports -+ * a given number of CS -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @cs_min: Minimum number of CSs required. -+ * -+ * Return: true if at least one CSG supports the given number -+ * of CSs (or more); otherwise false. -+ */ -+static bool iface_has_enough_streams(struct kbase_device *const kbdev, -+ u32 const cs_min) -+{ -+ bool has_enough = false; -+ struct kbase_csf_cmd_stream_group_info *const groups = -+ kbdev->csf.global_iface.groups; -+ const u32 group_num = kbdev->csf.global_iface.group_num; -+ u32 i; -+ -+ for (i = 0; (i < group_num) && !has_enough; i++) { -+ if (groups[i].stream_num >= cs_min) -+ has_enough = true; -+ } -+ -+ return has_enough; -+} -+ -+/** -+ * create_normal_suspend_buffer() - Create normal-mode suspend buffer per -+ * queue group -+ * -+ * @kctx: Pointer to kbase context where the queue group is created at -+ * @s_buf: Pointer to suspend buffer that is attached to queue group -+ * -+ * Return: 0 if suspend buffer is successfully allocated and reflected to GPU -+ * MMU page table. Otherwise -ENOMEM. -+ */ -+static int create_normal_suspend_buffer(struct kbase_context *const kctx, -+ struct kbase_normal_suspend_buffer *s_buf) -+{ -+ struct kbase_va_region *reg = NULL; -+ const unsigned long mem_flags = KBASE_REG_GPU_RD | KBASE_REG_GPU_WR; -+ const size_t nr_pages = -+ PFN_UP(kctx->kbdev->csf.global_iface.groups[0].suspend_size); -+ int err = 0; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ /* Allocate and initialize Region Object */ -+ reg = kbase_alloc_free_region(&kctx->kbdev->csf.shared_reg_rbtree, 0, -+ nr_pages, KBASE_REG_ZONE_MCU_SHARED); -+ -+ if (!reg) -+ return -ENOMEM; -+ -+ s_buf->phy = kcalloc(nr_pages, sizeof(*s_buf->phy), GFP_KERNEL); -+ -+ if (!s_buf->phy) { -+ err = -ENOMEM; -+ goto phy_alloc_failed; -+ } -+ -+ /* Get physical page for a normal suspend buffer */ -+ err = kbase_mem_pool_alloc_pages( -+ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ nr_pages, &s_buf->phy[0], false); -+ -+ if (err < 0) -+ goto phy_pages_alloc_failed; -+ -+ /* Insert Region Object into rbtree and make virtual address available -+ * to map it to physical page -+ */ -+ mutex_lock(&kctx->kbdev->csf.reg_lock); -+ err = kbase_add_va_region_rbtree(kctx->kbdev, reg, 0, nr_pages, 1); -+ reg->flags &= ~KBASE_REG_FREE; -+ mutex_unlock(&kctx->kbdev->csf.reg_lock); -+ -+ if (err) -+ goto add_va_region_failed; -+ -+ /* Update MMU table */ -+ err = kbase_mmu_insert_pages(kctx->kbdev, &kctx->kbdev->csf.mcu_mmu, -+ reg->start_pfn, &s_buf->phy[0], -+ nr_pages, mem_flags, -+ MCU_AS_NR, KBASE_MEM_GROUP_CSF_FW); -+ if (err) -+ goto mmu_insert_failed; -+ -+ s_buf->reg = reg; -+ -+ return 0; -+ -+mmu_insert_failed: -+ mutex_lock(&kctx->kbdev->csf.reg_lock); -+ WARN_ON(kbase_remove_va_region(reg)); -+ mutex_unlock(&kctx->kbdev->csf.reg_lock); -+ -+add_va_region_failed: -+ kbase_mem_pool_free_pages( -+ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], nr_pages, -+ &s_buf->phy[0], false, false); -+ -+phy_pages_alloc_failed: -+ kfree(s_buf->phy); -+phy_alloc_failed: -+ kfree(reg); -+ -+ return err; -+} -+ -+/** -+ * create_protected_suspend_buffer() - Create protected-mode suspend buffer -+ * per queue group -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @s_buf: Pointer to suspend buffer that is attached to queue group -+ * -+ * Return: 0 if suspend buffer is successfully allocated and reflected to GPU -+ * MMU page table. Otherwise -ENOMEM. -+ */ -+static int create_protected_suspend_buffer(struct kbase_device *const kbdev, -+ struct kbase_protected_suspend_buffer *s_buf) -+{ -+ struct kbase_va_region *reg = NULL; -+ struct tagged_addr *phys = NULL; -+ const unsigned long mem_flags = KBASE_REG_GPU_RD | KBASE_REG_GPU_WR; -+ const size_t nr_pages = -+ PFN_UP(kbdev->csf.global_iface.groups[0].suspend_size); -+ int err = 0; -+ -+ /* Allocate and initialize Region Object */ -+ reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, -+ nr_pages, KBASE_REG_ZONE_MCU_SHARED); -+ -+ if (!reg) -+ return -ENOMEM; -+ -+ phys = kcalloc(nr_pages, sizeof(*phys), GFP_KERNEL); -+ if (!phys) { -+ err = -ENOMEM; -+ goto phy_alloc_failed; -+ } -+ -+ s_buf->pma = kbase_csf_protected_memory_alloc(kbdev, phys, -+ nr_pages); -+ if (s_buf->pma == NULL) { -+ err = -ENOMEM; -+ goto pma_alloc_failed; -+ } -+ -+ /* Insert Region Object into rbtree and make virtual address available -+ * to map it to physical page -+ */ -+ mutex_lock(&kbdev->csf.reg_lock); -+ err = kbase_add_va_region_rbtree(kbdev, reg, 0, nr_pages, 1); -+ reg->flags &= ~KBASE_REG_FREE; -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ if (err) -+ goto add_va_region_failed; -+ -+ /* Update MMU table */ -+ err = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, -+ reg->start_pfn, phys, -+ nr_pages, mem_flags, MCU_AS_NR, -+ KBASE_MEM_GROUP_CSF_FW); -+ if (err) -+ goto mmu_insert_failed; -+ -+ s_buf->reg = reg; -+ kfree(phys); -+ return 0; -+ -+mmu_insert_failed: -+ mutex_lock(&kbdev->csf.reg_lock); -+ WARN_ON(kbase_remove_va_region(reg)); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+add_va_region_failed: -+ kbase_csf_protected_memory_free(kbdev, s_buf->pma, nr_pages); -+pma_alloc_failed: -+ kfree(phys); -+phy_alloc_failed: -+ kfree(reg); -+ -+ return err; -+} -+ -+static void timer_event_worker(struct work_struct *data); -+static void protm_event_worker(struct work_struct *data); -+static void term_normal_suspend_buffer(struct kbase_context *const kctx, -+ struct kbase_normal_suspend_buffer *s_buf); -+ -+/** -+ * create_suspend_buffers - Setup normal and protected mode -+ * suspend buffers. -+ * -+ * @kctx: Address of the kbase context within which the queue group -+ * is to be created. -+ * @group: Pointer to GPU command queue group data. -+ * -+ * Return: 0 if suspend buffers are successfully allocated. Otherwise -ENOMEM. -+ */ -+static int create_suspend_buffers(struct kbase_context *const kctx, -+ struct kbase_queue_group * const group) -+{ -+ int err = 0; -+ -+ if (create_normal_suspend_buffer(kctx, &group->normal_suspend_buf)) { -+ dev_err(kctx->kbdev->dev, "Failed to create normal suspend buffer\n"); -+ return -ENOMEM; -+ } -+ -+ if (kctx->kbdev->csf.pma_dev) { -+ err = create_protected_suspend_buffer(kctx->kbdev, -+ &group->protected_suspend_buf); -+ if (err) { -+ term_normal_suspend_buffer(kctx, -+ &group->normal_suspend_buf); -+ dev_err(kctx->kbdev->dev, "Failed to create protected suspend buffer\n"); -+ } -+ } else { -+ group->protected_suspend_buf.reg = NULL; -+ } -+ -+ return err; -+} -+ -+/** -+ * generate_group_uid() - Makes an ID unique to all kernel base devices -+ * and contexts, for a queue group and CSG. -+ * -+ * Return: A unique ID in the form of an unsigned 32-bit integer -+ */ -+static u32 generate_group_uid(void) -+{ -+ /* use first KBase device to store max UID */ -+ struct kbase_device *kbdev = kbase_find_device(-1); -+ u32 uid = 1; -+ -+ if (kbdev) -+ uid = (u32) atomic_inc_return(&kbdev->group_max_uid_in_devices); -+ else -+ WARN(1, "NULL kbase device pointer in group UID generation"); -+ -+ return uid; -+} -+ -+/** -+ * create_queue_group() - Create a queue group -+ * -+ * @kctx: Address of the kbase context within which the queue group -+ * is to be created. -+ * @create: Address of a structure which contains details of the -+ * queue group which is to be created. -+ * -+ * Return: a queue group handle on success, or a negative error code on failure. -+ */ -+static int create_queue_group(struct kbase_context *const kctx, -+ union kbase_ioctl_cs_queue_group_create *const create) -+{ -+ int group_handle = find_free_group_handle(kctx); -+ -+ if (group_handle < 0) { -+ dev_err(kctx->kbdev->dev, -+ "All queue group handles are already in use\n"); -+ } else { -+ struct kbase_queue_group * const group = -+ kmalloc(sizeof(struct kbase_queue_group), -+ GFP_KERNEL); -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ if (!group) { -+ dev_err(kctx->kbdev->dev, "Failed to allocate a queue\n"); -+ group_handle = -ENOMEM; -+ } else { -+ int err = 0; -+ -+ group->kctx = kctx; -+ group->handle = group_handle; -+ group->csg_nr = KBASEP_CSG_NR_INVALID; -+ -+ group->tiler_mask = create->in.tiler_mask; -+ group->fragment_mask = create->in.fragment_mask; -+ group->compute_mask = create->in.compute_mask; -+ -+ group->tiler_max = create->in.tiler_max; -+ group->fragment_max = create->in.fragment_max; -+ group->compute_max = create->in.compute_max; -+ group->priority = kbase_csf_priority_queue_group_priority_to_relative( -+ kbase_csf_priority_check(kctx->kbdev, create->in.priority)); -+ group->doorbell_nr = KBASEP_USER_DB_NR_INVALID; -+ group->faulted = false; -+ -+ group->group_uid = generate_group_uid(); -+ create->out.group_uid = group->group_uid; -+ -+ INIT_LIST_HEAD(&group->link); -+ INIT_LIST_HEAD(&group->link_to_schedule); -+ INIT_LIST_HEAD(&group->error_fatal.link); -+ INIT_LIST_HEAD(&group->error_timeout.link); -+ INIT_LIST_HEAD(&group->error_tiler_oom.link); -+ INIT_WORK(&group->timer_event_work, timer_event_worker); -+ INIT_WORK(&group->protm_event_work, protm_event_worker); -+ bitmap_zero(group->protm_pending_bitmap, -+ MAX_SUPPORTED_STREAMS_PER_GROUP); -+ -+ group->run_state = KBASE_CSF_GROUP_INACTIVE; -+ err = create_suspend_buffers(kctx, group); -+ -+ if (err < 0) { -+ kfree(group); -+ group_handle = err; -+ } else { -+ int j; -+ -+ kctx->csf.queue_groups[group_handle] = group; -+ for (j = 0; j < MAX_SUPPORTED_STREAMS_PER_GROUP; -+ j++) -+ group->bound_queues[j] = NULL; -+ } -+ } -+ } -+ -+ return group_handle; -+} -+ -+int kbase_csf_queue_group_create(struct kbase_context *const kctx, -+ union kbase_ioctl_cs_queue_group_create *const create) -+{ -+ int err = 0; -+ const u32 tiler_count = hweight64(create->in.tiler_mask); -+ const u32 fragment_count = hweight64(create->in.fragment_mask); -+ const u32 compute_count = hweight64(create->in.compute_mask); -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ if ((create->in.tiler_max > tiler_count) || -+ (create->in.fragment_max > fragment_count) || -+ (create->in.compute_max > compute_count)) { -+ dev_err(kctx->kbdev->dev, -+ "Invalid maximum number of endpoints for a queue group\n"); -+ err = -EINVAL; -+ } else if (create->in.priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT) { -+ dev_err(kctx->kbdev->dev, "Invalid queue group priority %u\n", -+ (unsigned int)create->in.priority); -+ err = -EINVAL; -+ } else if (!iface_has_enough_streams(kctx->kbdev, create->in.cs_min)) { -+ dev_err(kctx->kbdev->dev, -+ "No CSG has at least %d CSs\n", -+ create->in.cs_min); -+ err = -EINVAL; -+ } else { -+ /* For the CSG which satisfies the condition for having -+ * the needed number of CSs, check whether it also conforms -+ * with the requirements for at least one of its CSs having -+ * the iterator of the needed type -+ * (note: for CSF v1.0 all CSs in a CSG will have access to -+ * the same iterators) -+ */ -+ const int group_handle = create_queue_group(kctx, create); -+ -+ if (group_handle >= 0) -+ create->out.group_handle = group_handle; -+ else -+ err = group_handle; -+ } -+ -+ mutex_unlock(&kctx->csf.lock); -+ -+ return err; -+} -+ -+/** -+ * term_normal_suspend_buffer() - Free normal-mode suspend buffer of queue group -+ * -+ * @kctx: Pointer to kbase context where queue group belongs to -+ * @s_buf: Pointer to queue group suspend buffer to be freed -+ */ -+static void term_normal_suspend_buffer(struct kbase_context *const kctx, -+ struct kbase_normal_suspend_buffer *s_buf) -+{ -+ const size_t nr_pages = -+ PFN_UP(kctx->kbdev->csf.global_iface.groups[0].suspend_size); -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ WARN_ON(kbase_mmu_teardown_pages( -+ kctx->kbdev, &kctx->kbdev->csf.mcu_mmu, -+ s_buf->reg->start_pfn, nr_pages, MCU_AS_NR)); -+ -+ WARN_ON(s_buf->reg->flags & KBASE_REG_FREE); -+ -+ mutex_lock(&kctx->kbdev->csf.reg_lock); -+ WARN_ON(kbase_remove_va_region(s_buf->reg)); -+ mutex_unlock(&kctx->kbdev->csf.reg_lock); -+ -+ kbase_mem_pool_free_pages( -+ &kctx->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ nr_pages, &s_buf->phy[0], false, false); -+ -+ kfree(s_buf->phy); -+ s_buf->phy = NULL; -+ kfree(s_buf->reg); -+ s_buf->reg = NULL; -+} -+ -+/** -+ * term_protected_suspend_buffer() - Free normal-mode suspend buffer of -+ * queue group -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @s_buf: Pointer to queue group suspend buffer to be freed -+ */ -+static void term_protected_suspend_buffer(struct kbase_device *const kbdev, -+ struct kbase_protected_suspend_buffer *s_buf) -+{ -+ const size_t nr_pages = -+ PFN_UP(kbdev->csf.global_iface.groups[0].suspend_size); -+ -+ WARN_ON(kbase_mmu_teardown_pages( -+ kbdev, &kbdev->csf.mcu_mmu, -+ s_buf->reg->start_pfn, nr_pages, MCU_AS_NR)); -+ -+ WARN_ON(s_buf->reg->flags & KBASE_REG_FREE); -+ -+ mutex_lock(&kbdev->csf.reg_lock); -+ WARN_ON(kbase_remove_va_region(s_buf->reg)); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ kbase_csf_protected_memory_free(kbdev, s_buf->pma, nr_pages); -+ s_buf->pma = NULL; -+ kfree(s_buf->reg); -+ s_buf->reg = NULL; -+} -+ -+void kbase_csf_term_descheduled_queue_group(struct kbase_queue_group *group) -+{ -+ struct kbase_context *kctx = group->kctx; -+ -+ /* Currently each group supports the same number of CS */ -+ u32 max_streams = -+ kctx->kbdev->csf.global_iface.groups[0].stream_num; -+ u32 i; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE && -+ group->run_state != KBASE_CSF_GROUP_FAULT_EVICTED); -+ -+ for (i = 0; i < max_streams; i++) { -+ struct kbase_queue *queue = -+ group->bound_queues[i]; -+ -+ /* The group is already being evicted from the scheduler */ -+ if (queue) -+ unbind_stopped_queue(kctx, queue); -+ } -+ -+ term_normal_suspend_buffer(kctx, &group->normal_suspend_buf); -+ if (kctx->kbdev->csf.pma_dev) -+ term_protected_suspend_buffer(kctx->kbdev, -+ &group->protected_suspend_buf); -+ -+ group->run_state = KBASE_CSF_GROUP_TERMINATED; -+} -+ -+/** -+ * term_queue_group - Terminate a GPU command queue group. -+ * -+ * @group: Pointer to GPU command queue group data. -+ * -+ * Terminates a GPU command queue group. From the userspace perspective the -+ * group will still exist but it can't bind new queues to it. Userspace can -+ * still add work in queues bound to the group but it won't be executed. (This -+ * is because the IO mapping created upon binding such queues is still intact.) -+ */ -+static void term_queue_group(struct kbase_queue_group *group) -+{ -+ struct kbase_context *kctx = group->kctx; -+ -+ kbase_reset_gpu_assert_failed_or_prevented(kctx->kbdev); -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ /* Stop the group and evict it from the scheduler */ -+ kbase_csf_scheduler_group_deschedule(group); -+ -+ if (group->run_state == KBASE_CSF_GROUP_TERMINATED) -+ return; -+ -+ dev_dbg(kctx->kbdev->dev, "group %d terminating", group->handle); -+ -+ kbase_csf_term_descheduled_queue_group(group); -+} -+ -+static void cancel_queue_group_events(struct kbase_queue_group *group) -+{ -+ cancel_work_sync(&group->timer_event_work); -+ cancel_work_sync(&group->protm_event_work); -+} -+ -+void kbase_csf_queue_group_terminate(struct kbase_context *kctx, -+ u8 group_handle) -+{ -+ struct kbase_queue_group *group; -+ int err; -+ bool reset_prevented = false; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ err = kbase_reset_gpu_prevent_and_wait(kbdev); -+ if (err) -+ dev_warn( -+ kbdev->dev, -+ "Unsuccessful GPU reset detected when terminating group %d, attempting to terminate regardless", -+ group_handle); -+ else -+ reset_prevented = true; -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ group = find_queue_group(kctx, group_handle); -+ -+ if (group) { -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kctx->csf.event_lock, flags); -+ -+ dev_dbg(kbdev->dev, -+ "Remove any pending group fatal error from context %pK\n", -+ (void *)group->kctx); -+ -+ list_del_init(&group->error_tiler_oom.link); -+ list_del_init(&group->error_timeout.link); -+ list_del_init(&group->error_fatal.link); -+ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); -+ -+ term_queue_group(group); -+ kctx->csf.queue_groups[group_handle] = NULL; -+ } -+ -+ mutex_unlock(&kctx->csf.lock); -+ if (reset_prevented) -+ kbase_reset_gpu_allow(kbdev); -+ -+ if (!group) -+ return; -+ -+ /* Cancel any pending event callbacks. If one is in progress -+ * then this thread waits synchronously for it to complete (which -+ * is why we must unlock the context first). We already ensured -+ * that no more callbacks can be enqueued by terminating the group. -+ */ -+ cancel_queue_group_events(group); -+ kfree(group); -+} -+ -+int kbase_csf_queue_group_suspend(struct kbase_context *kctx, -+ struct kbase_suspend_copy_buffer *sus_buf, -+ u8 group_handle) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ int err; -+ struct kbase_queue_group *group; -+ -+ err = kbase_reset_gpu_prevent_and_wait(kbdev); -+ if (err) { -+ dev_warn( -+ kbdev->dev, -+ "Unsuccessful GPU reset detected when suspending group %d", -+ group_handle); -+ return err; -+ } -+ mutex_lock(&kctx->csf.lock); -+ -+ group = find_queue_group(kctx, group_handle); -+ if (group) -+ err = kbase_csf_scheduler_group_copy_suspend_buf(group, -+ sus_buf); -+ else -+ err = -EINVAL; -+ -+ mutex_unlock(&kctx->csf.lock); -+ kbase_reset_gpu_allow(kbdev); -+ -+ return err; -+} -+ -+/** -+ * add_error() - Add an error to the list of errors to report to user space -+ * -+ * @kctx: Address of a base context associated with a GPU address space. -+ * @error: Address of the item to be added to the context's pending error list. -+ * @data: Error data to be returned to userspace. -+ * -+ * Does not wake up the event queue blocking a user thread in kbase_poll. This -+ * is to make it more efficient to add multiple errors. -+ * -+ * The added error must not already be on the context's list of errors waiting -+ * to be reported (e.g. because a previous error concerning the same object has -+ * not yet been reported). -+ */ -+static void add_error(struct kbase_context *const kctx, -+ struct kbase_csf_notification *const error, -+ struct base_csf_notification const *const data) -+{ -+ unsigned long flags; -+ -+ if (WARN_ON(!kctx)) -+ return; -+ -+ if (WARN_ON(!error)) -+ return; -+ -+ if (WARN_ON(!data)) -+ return; -+ -+ spin_lock_irqsave(&kctx->csf.event_lock, flags); -+ -+ if (!WARN_ON(!list_empty(&error->link))) { -+ error->data = *data; -+ list_add_tail(&error->link, &kctx->csf.error_list); -+ dev_dbg(kctx->kbdev->dev, -+ "Added error %pK of type %d in context %pK\n", -+ (void *)error, data->type, (void *)kctx); -+ } -+ -+ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); -+} -+ -+void kbase_csf_add_group_fatal_error( -+ struct kbase_queue_group *const group, -+ struct base_gpu_queue_group_error const *const err_payload) -+{ -+ struct base_csf_notification error; -+ -+ if (WARN_ON(!group)) -+ return; -+ -+ if (WARN_ON(!err_payload)) -+ return; -+ -+ error = (struct base_csf_notification) { -+ .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, -+ .payload = { -+ .csg_error = { -+ .handle = group->handle, -+ .error = *err_payload -+ } -+ } -+ }; -+ -+ add_error(group->kctx, &group->error_fatal, &error); -+} -+ -+void kbase_csf_active_queue_groups_reset(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct list_head evicted_groups; -+ struct kbase_queue_group *group; -+ int i; -+ -+ INIT_LIST_HEAD(&evicted_groups); -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ kbase_csf_scheduler_evict_ctx_slots(kbdev, kctx, &evicted_groups); -+ while (!list_empty(&evicted_groups)) { -+ group = list_first_entry(&evicted_groups, -+ struct kbase_queue_group, link); -+ -+ dev_dbg(kbdev->dev, "Context %d_%d active group %d terminated", -+ kctx->tgid, kctx->id, group->handle); -+ kbase_csf_term_descheduled_queue_group(group); -+ list_del_init(&group->link); -+ } -+ -+ /* Acting on the queue groups that are pending to be terminated. */ -+ for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) { -+ group = kctx->csf.queue_groups[i]; -+ if (group && -+ group->run_state == KBASE_CSF_GROUP_FAULT_EVICTED) -+ kbase_csf_term_descheduled_queue_group(group); -+ } -+ -+ mutex_unlock(&kctx->csf.lock); -+} -+ -+int kbase_csf_ctx_init(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ int err = -ENOMEM; -+ -+ INIT_LIST_HEAD(&kctx->csf.event_callback_list); -+ INIT_LIST_HEAD(&kctx->csf.queue_list); -+ INIT_LIST_HEAD(&kctx->csf.link); -+ INIT_LIST_HEAD(&kctx->csf.error_list); -+ -+ spin_lock_init(&kctx->csf.event_lock); -+ kctx->csf.user_reg_vma = NULL; -+ mutex_lock(&kbdev->pm.lock); -+ /* The inode information for /dev/malixx file is not available at the -+ * time of device probe as the inode is created when the device node -+ * is created by udevd (through mknod). -+ */ -+ if (kctx->filp) { -+ if (!kbdev->csf.mali_file_inode) -+ kbdev->csf.mali_file_inode = kctx->filp->f_inode; -+ -+ /* inode is unique for a file */ -+ WARN_ON(kbdev->csf.mali_file_inode != kctx->filp->f_inode); -+ } -+ mutex_unlock(&kbdev->pm.lock); -+ -+ /* Mark all the cookies as 'free' */ -+ bitmap_fill(kctx->csf.cookies, KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); -+ -+ kctx->csf.wq = alloc_workqueue("mali_kbase_csf_wq", -+ WQ_UNBOUND, 1); -+ -+ if (likely(kctx->csf.wq)) { -+ err = kbase_csf_scheduler_context_init(kctx); -+ -+ if (likely(!err)) { -+ err = kbase_csf_kcpu_queue_context_init(kctx); -+ -+ if (likely(!err)) { -+ err = kbase_csf_tiler_heap_context_init(kctx); -+ -+ if (likely(!err)) -+ mutex_init(&kctx->csf.lock); -+ else -+ kbase_csf_kcpu_queue_context_term(kctx); -+ } -+ -+ if (unlikely(err)) -+ kbase_csf_scheduler_context_term(kctx); -+ } -+ -+ if (unlikely(err)) -+ destroy_workqueue(kctx->csf.wq); -+ } -+ -+ return err; -+} -+ -+void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, -+ struct kbase_fault *fault) -+{ -+ int gr; -+ bool reported = false; -+ struct base_gpu_queue_group_error err_payload; -+ int err; -+ struct kbase_device *kbdev; -+ -+ if (WARN_ON(!kctx)) -+ return; -+ -+ if (WARN_ON(!fault)) -+ return; -+ -+ kbdev = kctx->kbdev; -+ err = kbase_reset_gpu_try_prevent(kbdev); -+ /* Regardless of whether reset failed or is currently happening, exit -+ * early -+ */ -+ if (err) -+ return; -+ -+ err_payload = (struct base_gpu_queue_group_error) { -+ .error_type = BASE_GPU_QUEUE_GROUP_ERROR_FATAL, -+ .payload = { -+ .fatal_group = { -+ .sideband = fault->addr, -+ .status = fault->status, -+ } -+ } -+ }; -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ for (gr = 0; gr < MAX_QUEUE_GROUP_NUM; gr++) { -+ struct kbase_queue_group *const group = -+ kctx->csf.queue_groups[gr]; -+ -+ if (group && group->run_state != KBASE_CSF_GROUP_TERMINATED) { -+ term_queue_group(group); -+ kbase_csf_add_group_fatal_error(group, &err_payload); -+ reported = true; -+ } -+ } -+ -+ mutex_unlock(&kctx->csf.lock); -+ -+ if (reported) -+ kbase_event_wakeup(kctx); -+ -+ kbase_reset_gpu_allow(kbdev); -+} -+ -+void kbase_csf_ctx_term(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_as *as = NULL; -+ unsigned long flags; -+ u32 i; -+ int err; -+ bool reset_prevented = false; -+ -+ /* As the kbase context is terminating, its debugfs sub-directory would -+ * have been removed already and so would be the debugfs file created -+ * for queue groups & kcpu queues, hence no need to explicitly remove -+ * those debugfs files. -+ */ -+ kbase_csf_event_wait_remove_all(kctx); -+ -+ /* Wait for a GPU reset if it is happening, prevent it if not happening */ -+ err = kbase_reset_gpu_prevent_and_wait(kbdev); -+ if (err) -+ dev_warn( -+ kbdev->dev, -+ "Unsuccessful GPU reset detected when terminating csf context (%d_%d), attempting to terminate regardless", -+ kctx->tgid, kctx->id); -+ else -+ reset_prevented = true; -+ -+ mutex_lock(&kctx->csf.lock); -+ /* Iterate through the queue groups that were not terminated by -+ * userspace and issue the term request to firmware for them. -+ */ -+ for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) { -+ if (kctx->csf.queue_groups[i]) -+ term_queue_group(kctx->csf.queue_groups[i]); -+ } -+ mutex_unlock(&kctx->csf.lock); -+ -+ if (reset_prevented) -+ kbase_reset_gpu_allow(kbdev); -+ -+ /* Now that all queue groups have been terminated, there can be no -+ * more OoM or timer event interrupts but there can be inflight work -+ * items. Destroying the wq will implicitly flush those work items. -+ */ -+ destroy_workqueue(kctx->csf.wq); -+ -+ /* Wait for the firmware error work item to also finish as it could -+ * be affecting this outgoing context also. -+ */ -+ flush_work(&kctx->kbdev->csf.fw_error_work); -+ -+ /* A work item to handle page_fault/bus_fault/gpu_fault could be -+ * pending for the outgoing context. Flush the workqueue that will -+ * execute that work item. -+ */ -+ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); -+ if (kctx->as_nr != KBASEP_AS_NR_INVALID) -+ as = &kctx->kbdev->as[kctx->as_nr]; -+ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); -+ if (as) -+ flush_workqueue(as->pf_wq); -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ for (i = 0; i < MAX_QUEUE_GROUP_NUM; i++) { -+ kfree(kctx->csf.queue_groups[i]); -+ kctx->csf.queue_groups[i] = NULL; -+ } -+ -+ /* Iterate through the queues that were not terminated by -+ * userspace and do the required cleanup for them. -+ */ -+ while (!list_empty(&kctx->csf.queue_list)) { -+ struct kbase_queue *queue; -+ -+ queue = list_first_entry(&kctx->csf.queue_list, -+ struct kbase_queue, link); -+ -+ /* The reference held when the IO mapping was created on bind -+ * would have been dropped otherwise the termination of Kbase -+ * context itself wouldn't have kicked-in. So there shall be -+ * only one reference left that was taken when queue was -+ * registered. -+ */ -+ if (atomic_read(&queue->refcount) != 1) -+ dev_warn(kctx->kbdev->dev, -+ "Releasing queue with incorrect refcounting!\n"); -+ list_del_init(&queue->link); -+ release_queue(queue); -+ } -+ -+ mutex_unlock(&kctx->csf.lock); -+ -+ kbase_csf_tiler_heap_context_term(kctx); -+ kbase_csf_kcpu_queue_context_term(kctx); -+ kbase_csf_scheduler_context_term(kctx); -+ -+ mutex_destroy(&kctx->csf.lock); -+} -+ -+int kbase_csf_event_wait_add(struct kbase_context *kctx, -+ kbase_csf_event_callback *callback, void *param) -+{ -+ int err = -ENOMEM; -+ struct kbase_csf_event *event = -+ kzalloc(sizeof(struct kbase_csf_event), GFP_KERNEL); -+ -+ if (event) { -+ unsigned long flags; -+ -+ event->kctx = kctx; -+ event->callback = callback; -+ event->param = param; -+ -+ spin_lock_irqsave(&kctx->csf.event_lock, flags); -+ list_add_tail(&event->link, &kctx->csf.event_callback_list); -+ dev_dbg(kctx->kbdev->dev, -+ "Added event handler %pK with param %pK\n", event, -+ event->param); -+ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); -+ -+ err = 0; -+ } -+ -+ return err; -+} -+ -+void kbase_csf_event_wait_remove(struct kbase_context *kctx, -+ kbase_csf_event_callback *callback, void *param) -+{ -+ struct kbase_csf_event *event; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kctx->csf.event_lock, flags); -+ -+ list_for_each_entry(event, &kctx->csf.event_callback_list, link) { -+ if ((event->callback == callback) && (event->param == param)) { -+ list_del(&event->link); -+ dev_dbg(kctx->kbdev->dev, -+ "Removed event handler %pK with param %pK\n", -+ event, event->param); -+ kfree(event); -+ break; -+ } -+ } -+ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); -+} -+ -+bool kbase_csf_read_error(struct kbase_context *kctx, -+ struct base_csf_notification *event_data) -+{ -+ bool got_event = true; -+ struct kbase_csf_notification *error_data = NULL; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kctx->csf.event_lock, flags); -+ -+ if (likely(!list_empty(&kctx->csf.error_list))) { -+ error_data = list_first_entry(&kctx->csf.error_list, -+ struct kbase_csf_notification, link); -+ list_del_init(&error_data->link); -+ *event_data = error_data->data; -+ dev_dbg(kctx->kbdev->dev, "Dequeued error %pK in context %pK\n", -+ (void *)error_data, (void *)kctx); -+ } else { -+ got_event = false; -+ } -+ -+ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); -+ -+ return got_event; -+} -+ -+bool kbase_csf_error_pending(struct kbase_context *kctx) -+{ -+ bool event_pended = false; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kctx->csf.event_lock, flags); -+ event_pended = !list_empty(&kctx->csf.error_list); -+ dev_dbg(kctx->kbdev->dev, "%s error is pending in context %pK\n", -+ event_pended ? "An" : "No", (void *)kctx); -+ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); -+ -+ return event_pended; -+} -+ -+void kbase_csf_event_signal(struct kbase_context *kctx, bool notify_gpu) -+{ -+ struct kbase_csf_event *event, *next_event; -+ unsigned long flags; -+ -+ dev_dbg(kctx->kbdev->dev, -+ "Signal event (%s GPU notify) for context %pK\n", -+ notify_gpu ? "with" : "without", (void *)kctx); -+ -+ /* First increment the signal count and wake up event thread. -+ */ -+ atomic_set(&kctx->event_count, 1); -+ kbase_event_wakeup(kctx); -+ -+ /* Signal the CSF firmware. This is to ensure that pending command -+ * stream synch object wait operations are re-evaluated. -+ * Write to GLB_DOORBELL would suffice as spec says that all pending -+ * synch object wait operations are re-evaluated on a write to any -+ * CS_DOORBELL/GLB_DOORBELL register. -+ */ -+ if (notify_gpu) { -+ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); -+ if (kctx->kbdev->pm.backend.gpu_powered) -+ kbase_csf_ring_doorbell(kctx->kbdev, CSF_KERNEL_DOORBELL_NR); -+ KBASE_KTRACE_ADD(kctx->kbdev, SYNC_UPDATE_EVENT_NOTIFY_GPU, kctx, 0u); -+ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); -+ } -+ -+ /* Now invoke the callbacks registered on backend side. -+ * Allow item removal inside the loop, if requested by the callback. -+ */ -+ spin_lock_irqsave(&kctx->csf.event_lock, flags); -+ -+ list_for_each_entry_safe( -+ event, next_event, &kctx->csf.event_callback_list, link) { -+ enum kbase_csf_event_callback_action action; -+ -+ dev_dbg(kctx->kbdev->dev, -+ "Calling event handler %pK with param %pK\n", -+ (void *)event, event->param); -+ action = event->callback(event->param); -+ if (action == KBASE_CSF_EVENT_CALLBACK_REMOVE) { -+ list_del(&event->link); -+ kfree(event); -+ } -+ } -+ -+ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); -+} -+ -+void kbase_csf_event_wait_remove_all(struct kbase_context *kctx) -+{ -+ struct kbase_csf_event *event, *next_event; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kctx->csf.event_lock, flags); -+ -+ list_for_each_entry_safe( -+ event, next_event, &kctx->csf.event_callback_list, link) { -+ list_del(&event->link); -+ dev_dbg(kctx->kbdev->dev, -+ "Removed event handler %pK with param %pK\n", -+ (void *)event, event->param); -+ kfree(event); -+ } -+ -+ spin_unlock_irqrestore(&kctx->csf.event_lock, flags); -+} -+ -+/** -+ * handle_oom_event - Handle the OoM event generated by the firmware for the -+ * CSI. -+ * -+ * This function will handle the OoM event request from the firmware for the -+ * CS. It will retrieve the address of heap context and heap's -+ * statistics (like number of render passes in-flight) from the CS's kernel -+ * kernel output page and pass them to the tiler heap function to allocate a -+ * new chunk. -+ * It will also update the CS's kernel input page with the address -+ * of a new chunk that was allocated. -+ * -+ * @kctx: Pointer to the kbase context in which the tiler heap was initialized. -+ * @stream: Pointer to the structure containing info provided by the firmware -+ * about the CSI. -+ * -+ * Return: 0 if successfully handled the request, otherwise a negative error -+ * code on failure. -+ */ -+static int handle_oom_event(struct kbase_context *const kctx, -+ struct kbase_csf_cmd_stream_info const *const stream) -+{ -+ u64 gpu_heap_va = -+ kbase_csf_firmware_cs_output(stream, CS_HEAP_ADDRESS_LO) | -+ ((u64)kbase_csf_firmware_cs_output(stream, CS_HEAP_ADDRESS_HI) << 32); -+ const u32 vt_start = -+ kbase_csf_firmware_cs_output(stream, CS_HEAP_VT_START); -+ const u32 vt_end = -+ kbase_csf_firmware_cs_output(stream, CS_HEAP_VT_END); -+ const u32 frag_end = -+ kbase_csf_firmware_cs_output(stream, CS_HEAP_FRAG_END); -+ u32 renderpasses_in_flight; -+ u32 pending_frag_count; -+ u64 new_chunk_ptr; -+ int err; -+ -+ if ((frag_end > vt_end) || (vt_end >= vt_start)) { -+ dev_warn(kctx->kbdev->dev, "Invalid Heap statistics provided by firmware: vt_start %d, vt_end %d, frag_end %d\n", -+ vt_start, vt_end, frag_end); -+ return -EINVAL; -+ } -+ -+ renderpasses_in_flight = vt_start - frag_end; -+ pending_frag_count = vt_end - frag_end; -+ -+ err = kbase_csf_tiler_heap_alloc_new_chunk(kctx, -+ gpu_heap_va, renderpasses_in_flight, pending_frag_count, &new_chunk_ptr); -+ -+ /* It is okay to acknowledge with a NULL chunk (firmware will then wait -+ * for the fragment jobs to complete and release chunks) -+ */ -+ if (err == -EBUSY) -+ new_chunk_ptr = 0; -+ else if (err) -+ return err; -+ -+ kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_START_LO, -+ new_chunk_ptr & 0xFFFFFFFF); -+ kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_START_HI, -+ new_chunk_ptr >> 32); -+ -+ kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_END_LO, -+ new_chunk_ptr & 0xFFFFFFFF); -+ kbase_csf_firmware_cs_input(stream, CS_TILER_HEAP_END_HI, -+ new_chunk_ptr >> 32); -+ -+ return 0; -+} -+ -+/** -+ * report_tiler_oom_error - Report a CSG error due to a tiler heap OOM event -+ * -+ * @group: Pointer to the GPU command queue group that encountered the error -+ */ -+static void report_tiler_oom_error(struct kbase_queue_group *group) -+{ -+ struct base_csf_notification const -+ error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, -+ .payload = { -+ .csg_error = { -+ .handle = group->handle, -+ .error = { -+ .error_type = -+ BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM, -+ } } } }; -+ -+ add_error(group->kctx, &group->error_tiler_oom, &error); -+ kbase_event_wakeup(group->kctx); -+} -+ -+/** -+ * kbase_queue_oom_event - Handle tiler out-of-memory for a GPU command queue. -+ * -+ * @queue: Pointer to queue for which out-of-memory event was received. -+ * -+ * Called with the CSF locked for the affected GPU virtual address space. -+ * Do not call in interrupt context. -+ * -+ * Handles tiler out-of-memory for a GPU command queue and then clears the -+ * notification to allow the firmware to report out-of-memory again in future. -+ * If the out-of-memory condition was successfully handled then this function -+ * rings the relevant doorbell to notify the firmware; otherwise, it terminates -+ * the GPU command queue group to which the queue is bound. See -+ * term_queue_group() for details. -+ */ -+static void kbase_queue_oom_event(struct kbase_queue *const queue) -+{ -+ struct kbase_context *const kctx = queue->kctx; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ struct kbase_queue_group *group; -+ int slot_num, err; -+ struct kbase_csf_cmd_stream_group_info const *ginfo; -+ struct kbase_csf_cmd_stream_info const *stream; -+ int csi_index = queue->csi_index; -+ u32 cs_oom_ack, cs_oom_req; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ group = get_bound_queue_group(queue); -+ if (!group) { -+ dev_warn(kctx->kbdev->dev, "queue not bound\n"); -+ return; -+ } -+ -+ kbase_csf_scheduler_lock(kbdev); -+ -+ slot_num = kbase_csf_scheduler_group_get_slot(group); -+ -+ /* The group could have gone off slot before this work item got -+ * a chance to execute. -+ */ -+ if (slot_num < 0) -+ goto unlock; -+ -+ /* If the bound group is on slot yet the kctx is marked with disabled -+ * on address-space fault, the group is pending to be killed. So skip -+ * the inflight oom operation. -+ */ -+ if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) -+ goto unlock; -+ -+ ginfo = &kbdev->csf.global_iface.groups[slot_num]; -+ stream = &ginfo->streams[csi_index]; -+ cs_oom_ack = kbase_csf_firmware_cs_output(stream, CS_ACK) & -+ CS_ACK_TILER_OOM_MASK; -+ cs_oom_req = kbase_csf_firmware_cs_input_read(stream, CS_REQ) & -+ CS_REQ_TILER_OOM_MASK; -+ -+ /* The group could have already undergone suspend-resume cycle before -+ * this work item got a chance to execute. On CSG resume the CS_ACK -+ * register is set by firmware to reflect the CS_REQ register, which -+ * implies that all events signaled before suspension are implicitly -+ * acknowledged. -+ * A new OoM event is expected to be generated after resume. -+ */ -+ if (cs_oom_ack == cs_oom_req) -+ goto unlock; -+ -+ err = handle_oom_event(kctx, stream); -+ -+ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_oom_ack, -+ CS_REQ_TILER_OOM_MASK); -+ -+ if (err) { -+ dev_warn( -+ kbdev->dev, -+ "Queue group to be terminated, couldn't handle the OoM event\n"); -+ kbase_csf_scheduler_unlock(kbdev); -+ term_queue_group(group); -+ report_tiler_oom_error(group); -+ return; -+ } -+ -+ kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, slot_num, true); -+unlock: -+ kbase_csf_scheduler_unlock(kbdev); -+} -+ -+/** -+ * oom_event_worker - Tiler out-of-memory handler called from a workqueue. -+ * -+ * @data: Pointer to a work_struct embedded in GPU command queue data. -+ * -+ * Handles a tiler out-of-memory condition for a GPU command queue and then -+ * releases a reference that was added to prevent the queue being destroyed -+ * while this work item was pending on a workqueue. -+ */ -+static void oom_event_worker(struct work_struct *data) -+{ -+ struct kbase_queue *queue = -+ container_of(data, struct kbase_queue, oom_event_work); -+ struct kbase_context *kctx = queue->kctx; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ int err = kbase_reset_gpu_try_prevent(kbdev); -+ /* Regardless of whether reset failed or is currently happening, exit -+ * early -+ */ -+ if (err) -+ return; -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ kbase_queue_oom_event(queue); -+ release_queue(queue); -+ -+ mutex_unlock(&kctx->csf.lock); -+ kbase_reset_gpu_allow(kbdev); -+} -+ -+/** -+ * report_group_timeout_error - Report the timeout error for the group to userspace. -+ * -+ * @group: Pointer to the group for which timeout error occurred -+ */ -+static void report_group_timeout_error(struct kbase_queue_group *const group) -+{ -+ struct base_csf_notification const -+ error = { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, -+ .payload = { -+ .csg_error = { -+ .handle = group->handle, -+ .error = { -+ .error_type = -+ BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT, -+ } } } }; -+ -+ dev_warn(group->kctx->kbdev->dev, -+ "Notify the event notification thread, forward progress timeout (%llu cycles)\n", -+ kbase_csf_timeout_get(group->kctx->kbdev)); -+ -+ add_error(group->kctx, &group->error_timeout, &error); -+ kbase_event_wakeup(group->kctx); -+} -+ -+/** -+ * timer_event_worker - Handle the progress timeout error for the group -+ * -+ * @data: Pointer to a work_struct embedded in GPU command queue group data. -+ * -+ * Terminate the CSG and report the error to userspace -+ */ -+static void timer_event_worker(struct work_struct *data) -+{ -+ struct kbase_queue_group *const group = -+ container_of(data, struct kbase_queue_group, timer_event_work); -+ struct kbase_context *const kctx = group->kctx; -+ bool reset_prevented = false; -+ int err = kbase_reset_gpu_prevent_and_wait(kctx->kbdev); -+ -+ if (err) -+ dev_warn( -+ kctx->kbdev->dev, -+ "Unsuccessful GPU reset detected when terminating group %d on progress timeout, attempting to terminate regardless", -+ group->handle); -+ else -+ reset_prevented = true; -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ term_queue_group(group); -+ report_group_timeout_error(group); -+ -+ mutex_unlock(&kctx->csf.lock); -+ if (reset_prevented) -+ kbase_reset_gpu_allow(kctx->kbdev); -+} -+ -+/** -+ * handle_progress_timer_event - Progress timer timeout event handler. -+ * -+ * @group: Pointer to GPU queue group for which the timeout event is received. -+ * -+ * Enqueue a work item to terminate the group and notify the event notification -+ * thread of progress timeout fault for the GPU command queue group. -+ */ -+static void handle_progress_timer_event(struct kbase_queue_group *const group) -+{ -+ queue_work(group->kctx->csf.wq, &group->timer_event_work); -+} -+ -+/** -+ * protm_event_worker - Protected mode switch request event handler -+ * called from a workqueue. -+ * -+ * @data: Pointer to a work_struct embedded in GPU command queue group data. -+ * -+ * Request to switch to protected mode. -+ */ -+static void protm_event_worker(struct work_struct *data) -+{ -+ struct kbase_queue_group *const group = -+ container_of(data, struct kbase_queue_group, protm_event_work); -+ -+ KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_BEGIN, -+ group, 0u); -+ kbase_csf_scheduler_group_protm_enter(group); -+ KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, PROTM_EVENT_WORKER_END, -+ group, 0u); -+} -+ -+static void report_queue_fatal_error(struct kbase_queue *const queue, -+ u32 cs_fatal, u64 cs_fatal_info, -+ u8 group_handle) -+{ -+ struct base_csf_notification error = -+ { .type = BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, -+ .payload = { -+ .csg_error = { -+ .handle = group_handle, -+ .error = { -+ .error_type = -+ BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL, -+ .payload = { -+ .fatal_queue = { -+ .sideband = -+ cs_fatal_info, -+ .status = cs_fatal, -+ .csi_index = -+ queue->csi_index, -+ } } } } } }; -+ -+ add_error(queue->kctx, &queue->error, &error); -+ kbase_event_wakeup(queue->kctx); -+} -+ -+/** -+ * handle_fault_event - Handler for CS fault. -+ * -+ * @queue: Pointer to queue for which fault event was received. -+ * @stream: Pointer to the structure containing info provided by the -+ * firmware about the CSI. -+ * -+ * Prints meaningful CS fault information. -+ * -+ */ -+static void -+handle_fault_event(struct kbase_queue *const queue, -+ struct kbase_csf_cmd_stream_info const *const stream) -+{ -+ const u32 cs_fault = kbase_csf_firmware_cs_output(stream, CS_FAULT); -+ const u64 cs_fault_info = -+ kbase_csf_firmware_cs_output(stream, CS_FAULT_INFO_LO) | -+ ((u64)kbase_csf_firmware_cs_output(stream, CS_FAULT_INFO_HI) -+ << 32); -+ const u8 cs_fault_exception_type = -+ CS_FAULT_EXCEPTION_TYPE_GET(cs_fault); -+ const u32 cs_fault_exception_data = -+ CS_FAULT_EXCEPTION_DATA_GET(cs_fault); -+ const u64 cs_fault_info_exception_data = -+ CS_FAULT_INFO_EXCEPTION_DATA_GET(cs_fault_info); -+ struct kbase_device *const kbdev = queue->kctx->kbdev; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ -+ dev_warn(kbdev->dev, -+ "Ctx %d_%d Group %d CSG %d CSI: %d\n" -+ "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n" -+ "CS_FAULT.EXCEPTION_DATA: 0x%x\n" -+ "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n", -+ queue->kctx->tgid, queue->kctx->id, queue->group->handle, -+ queue->group->csg_nr, queue->csi_index, -+ cs_fault_exception_type, -+ kbase_gpu_exception_name(cs_fault_exception_type), -+ cs_fault_exception_data, cs_fault_info_exception_data); -+ -+ if (cs_fault_exception_type == -+ CS_FAULT_EXCEPTION_TYPE_RESOURCE_EVICTION_TIMEOUT) -+ report_queue_fatal_error(queue, GPU_EXCEPTION_TYPE_SW_FAULT_2, -+ 0, queue->group->handle); -+} -+ -+/** -+ * fatal_event_worker - Handle the fatal error for the GPU queue -+ * -+ * @data: Pointer to a work_struct embedded in GPU command queue. -+ * -+ * Terminate the CSG and report the error to userspace. -+ */ -+static void fatal_event_worker(struct work_struct *const data) -+{ -+ struct kbase_queue *const queue = -+ container_of(data, struct kbase_queue, fatal_event_work); -+ struct kbase_context *const kctx = queue->kctx; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ struct kbase_queue_group *group; -+ u8 group_handle; -+ bool reset_prevented = false; -+ int err = kbase_reset_gpu_prevent_and_wait(kbdev); -+ -+ if (err) -+ dev_warn( -+ kbdev->dev, -+ "Unsuccessful GPU reset detected when terminating group to handle fatal event, attempting to terminate regardless"); -+ else -+ reset_prevented = true; -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ group = get_bound_queue_group(queue); -+ if (!group) { -+ dev_warn(kbdev->dev, "queue not bound when handling fatal event"); -+ goto unlock; -+ } -+ -+ group_handle = group->handle; -+ term_queue_group(group); -+ report_queue_fatal_error(queue, queue->cs_fatal, queue->cs_fatal_info, -+ group_handle); -+ -+unlock: -+ release_queue(queue); -+ mutex_unlock(&kctx->csf.lock); -+ if (reset_prevented) -+ kbase_reset_gpu_allow(kbdev); -+} -+ -+/** -+ * handle_fatal_event - Handler for CS fatal. -+ * -+ * @queue: Pointer to queue for which fatal event was received. -+ * @stream: Pointer to the structure containing info provided by the -+ * firmware about the CSI. -+ * -+ * Prints meaningful CS fatal information. -+ * Enqueue a work item to terminate the group and report the fatal error -+ * to user space. -+ */ -+static void -+handle_fatal_event(struct kbase_queue *const queue, -+ struct kbase_csf_cmd_stream_info const *const stream) -+{ -+ const u32 cs_fatal = kbase_csf_firmware_cs_output(stream, CS_FATAL); -+ const u64 cs_fatal_info = -+ kbase_csf_firmware_cs_output(stream, CS_FATAL_INFO_LO) | -+ ((u64)kbase_csf_firmware_cs_output(stream, CS_FATAL_INFO_HI) -+ << 32); -+ const u32 cs_fatal_exception_type = -+ CS_FATAL_EXCEPTION_TYPE_GET(cs_fatal); -+ const u32 cs_fatal_exception_data = -+ CS_FATAL_EXCEPTION_DATA_GET(cs_fatal); -+ const u64 cs_fatal_info_exception_data = -+ CS_FATAL_INFO_EXCEPTION_DATA_GET(cs_fatal_info); -+ struct kbase_device *const kbdev = queue->kctx->kbdev; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ -+ dev_warn(kbdev->dev, -+ "Ctx %d_%d Group %d CSG %d CSI: %d\n" -+ "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" -+ "CS_FATAL.EXCEPTION_DATA: 0x%x\n" -+ "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n", -+ queue->kctx->tgid, queue->kctx->id, queue->group->handle, -+ queue->group->csg_nr, queue->csi_index, -+ cs_fatal_exception_type, -+ kbase_gpu_exception_name(cs_fatal_exception_type), -+ cs_fatal_exception_data, cs_fatal_info_exception_data); -+ -+ if (cs_fatal_exception_type == -+ CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR) { -+ queue_work(system_wq, &kbdev->csf.fw_error_work); -+ } else { -+ get_queue(queue); -+ queue->cs_fatal = cs_fatal; -+ queue->cs_fatal_info = cs_fatal_info; -+ if (!queue_work(queue->kctx->csf.wq, &queue->fatal_event_work)) -+ release_queue(queue); -+ } -+} -+ -+/** -+ * handle_queue_exception_event - Handler for CS fatal/fault exception events. -+ * -+ * @queue: Pointer to queue for which fatal/fault event was received. -+ * @cs_req: Value of the CS_REQ register from the CS's input page. -+ * @cs_ack: Value of the CS_ACK register from the CS's output page. -+ */ -+static void handle_queue_exception_event(struct kbase_queue *const queue, -+ const u32 cs_req, const u32 cs_ack) -+{ -+ struct kbase_csf_cmd_stream_group_info const *ginfo; -+ struct kbase_csf_cmd_stream_info const *stream; -+ struct kbase_context *const kctx = queue->kctx; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ struct kbase_queue_group *group = queue->group; -+ int csi_index = queue->csi_index; -+ int slot_num = group->csg_nr; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ -+ ginfo = &kbdev->csf.global_iface.groups[slot_num]; -+ stream = &ginfo->streams[csi_index]; -+ -+ if ((cs_ack & CS_ACK_FATAL_MASK) != (cs_req & CS_REQ_FATAL_MASK)) { -+ handle_fatal_event(queue, stream); -+ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, -+ CS_REQ_FATAL_MASK); -+ } -+ -+ if ((cs_ack & CS_ACK_FAULT_MASK) != (cs_req & CS_REQ_FAULT_MASK)) { -+ handle_fault_event(queue, stream); -+ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, cs_ack, -+ CS_REQ_FAULT_MASK); -+ kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, slot_num, true); -+ } -+} -+ -+/** -+ * process_cs_interrupts - Process interrupts for a CS. -+ * -+ * @group: Pointer to GPU command queue group data. -+ * @ginfo: The CSG interface provided by the firmware. -+ * @irqreq: CSG's IRQ request bitmask (one bit per CS). -+ * @irqack: CSG's IRQ acknowledge bitmask (one bit per CS). -+ * -+ * If the interrupt request bitmask differs from the acknowledge bitmask -+ * then the firmware is notifying the host of an event concerning those -+ * CSs indicated by bits whose value differs. The actions required -+ * are then determined by examining which notification flags differ between -+ * the request and acknowledge registers for the individual CS(s). -+ */ -+static void process_cs_interrupts(struct kbase_queue_group *const group, -+ struct kbase_csf_cmd_stream_group_info const *const ginfo, -+ u32 const irqreq, u32 const irqack) -+{ -+ struct kbase_device *const kbdev = group->kctx->kbdev; -+ u32 remaining = irqreq ^ irqack; -+ bool protm_pend = false; -+ const bool group_suspending = -+ !kbase_csf_scheduler_group_events_enabled(kbdev, group); -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ -+ while (remaining != 0) { -+ int const i = ffs(remaining) - 1; -+ struct kbase_queue *const queue = group->bound_queues[i]; -+ -+ remaining &= ~(1 << i); -+ -+ /* The queue pointer can be NULL, but if it isn't NULL then it -+ * cannot disappear since scheduler spinlock is held and before -+ * freeing a bound queue it has to be first unbound which -+ * requires scheduler spinlock. -+ */ -+ if (queue && !WARN_ON(queue->csi_index != i)) { -+ struct kbase_csf_cmd_stream_info const *const stream = -+ &ginfo->streams[i]; -+ u32 const cs_req = kbase_csf_firmware_cs_input_read( -+ stream, CS_REQ); -+ u32 const cs_ack = -+ kbase_csf_firmware_cs_output(stream, CS_ACK); -+ struct workqueue_struct *wq = group->kctx->csf.wq; -+ -+ if ((cs_req & CS_REQ_EXCEPTION_MASK) ^ -+ (cs_ack & CS_ACK_EXCEPTION_MASK)) { -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_FAULT_INTERRUPT, group, queue, cs_req ^ cs_ack); -+ handle_queue_exception_event(queue, cs_req, cs_ack); -+ } -+ -+ /* PROTM_PEND and TILER_OOM can be safely ignored -+ * because they will be raised again if the group -+ * is assigned a CSG slot in future. -+ */ -+ if (group_suspending) { -+ u32 const cs_req_remain = cs_req & ~CS_REQ_EXCEPTION_MASK; -+ u32 const cs_ack_remain = cs_ack & ~CS_ACK_EXCEPTION_MASK; -+ -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_IGNORED_INTERRUPTS_GROUP_SUSPEND, -+ group, queue, cs_req_remain ^ cs_ack_remain); -+ continue; -+ } -+ -+ if (((cs_req & CS_REQ_TILER_OOM_MASK) ^ -+ (cs_ack & CS_ACK_TILER_OOM_MASK))) { -+ get_queue(queue); -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_TILER_OOM_INTERRUPT, group, queue, -+ cs_req ^ cs_ack); -+ if (WARN_ON(!queue_work(wq, &queue->oom_event_work))) { -+ /* The work item shall not have been -+ * already queued, there can be only -+ * one pending OoM event for a -+ * queue. -+ */ -+ release_queue(queue); -+ } -+ } -+ -+ if ((cs_req & CS_REQ_PROTM_PEND_MASK) ^ -+ (cs_ack & CS_ACK_PROTM_PEND_MASK)) { -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_PROTM_PEND_INTERRUPT, group, queue, -+ cs_req ^ cs_ack); -+ -+ dev_dbg(kbdev->dev, -+ "Protected mode entry request for queue on csi %d bound to group-%d on slot %d", -+ queue->csi_index, group->handle, -+ group->csg_nr); -+ -+ bitmap_set(group->protm_pending_bitmap, i, 1); -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, PROTM_PENDING_SET, group, queue, -+ group->protm_pending_bitmap[0]); -+ protm_pend = true; -+ } -+ } -+ } -+ -+ if (protm_pend) -+ queue_work(group->kctx->csf.wq, &group->protm_event_work); -+} -+ -+/** -+ * process_csg_interrupts - Process interrupts for a CSG. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @csg_nr: CSG number. -+ * -+ * Handles interrupts for a CSG and for CSs within it. -+ * -+ * If the CSG's request register value differs from its acknowledge register -+ * then the firmware is notifying the host of an event concerning the whole -+ * group. The actions required are then determined by examining which -+ * notification flags differ between those two register values. -+ * -+ * See process_cs_interrupts() for details of per-stream interrupt handling. -+ */ -+static void process_csg_interrupts(struct kbase_device *const kbdev, -+ int const csg_nr) -+{ -+ struct kbase_csf_cmd_stream_group_info *ginfo; -+ struct kbase_queue_group *group = NULL; -+ u32 req, ack, irqreq, irqack; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ -+ if (WARN_ON(csg_nr >= kbdev->csf.global_iface.group_num)) -+ return; -+ -+ KBASE_KTRACE_ADD(kbdev, CSG_INTERRUPT_PROCESS, NULL, csg_nr); -+ -+ ginfo = &kbdev->csf.global_iface.groups[csg_nr]; -+ req = kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ); -+ ack = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); -+ irqreq = kbase_csf_firmware_csg_output(ginfo, CSG_IRQ_REQ); -+ irqack = kbase_csf_firmware_csg_input_read(ginfo, CSG_IRQ_ACK); -+ -+ /* There may not be any pending CSG/CS interrupts to process */ -+ if ((req == ack) && (irqreq == irqack)) -+ goto out; -+ -+ /* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before -+ * examining the CS_ACK & CS_REQ bits. This would ensure that Host -+ * doesn't misses an interrupt for the CS in the race scenario where -+ * whilst Host is servicing an interrupt for the CS, firmware sends -+ * another interrupt for that CS. -+ */ -+ kbase_csf_firmware_csg_input(ginfo, CSG_IRQ_ACK, irqreq); -+ -+ group = kbase_csf_scheduler_get_group_on_slot(kbdev, csg_nr); -+ -+ /* The group pointer can be NULL here if interrupts for the group -+ * (like SYNC_UPDATE, IDLE notification) were delayed and arrived -+ * just after the suspension of group completed. However if not NULL -+ * then the group pointer cannot disappear even if User tries to -+ * terminate the group whilst this loop is running as scheduler -+ * spinlock is held and for freeing a group that is resident on a CSG -+ * slot scheduler spinlock is required. -+ */ -+ if (!group) -+ goto out; -+ -+ if (WARN_ON(kbase_csf_scheduler_group_get_slot_locked(group) != csg_nr)) -+ goto out; -+ -+ if ((req ^ ack) & CSG_REQ_SYNC_UPDATE_MASK) { -+ kbase_csf_firmware_csg_input_mask(ginfo, -+ CSG_REQ, ack, CSG_REQ_SYNC_UPDATE_MASK); -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SYNC_UPDATE_INTERRUPT, group, req ^ ack); -+ kbase_csf_event_signal_cpu_only(group->kctx); -+ } -+ -+ if ((req ^ ack) & CSG_REQ_IDLE_MASK) { -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, ack, -+ CSG_REQ_IDLE_MASK); -+ -+ set_bit(csg_nr, scheduler->csg_slots_idle_mask); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_IDLE_SET, group, -+ scheduler->csg_slots_idle_mask[0]); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_IDLE_INTERRUPT, group, req ^ ack); -+ dev_dbg(kbdev->dev, "Idle notification received for Group %u on slot %d\n", -+ group->handle, csg_nr); -+ -+ /* Check if the scheduling tick can be advanced */ -+ if (kbase_csf_scheduler_all_csgs_idle(kbdev) && -+ !scheduler->gpu_idle_fw_timer_enabled) { -+ kbase_csf_scheduler_advance_tick_nolock(kbdev); -+ } -+ } -+ -+ if ((req ^ ack) & CSG_REQ_PROGRESS_TIMER_EVENT_MASK) { -+ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, ack, -+ CSG_REQ_PROGRESS_TIMER_EVENT_MASK); -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_PROGRESS_TIMER_INTERRUPT, -+ group, req ^ ack); -+ dev_info(kbdev->dev, -+ "Timeout notification received for group %u of ctx %d_%d on slot %d\n", -+ group->handle, group->kctx->tgid, group->kctx->id, csg_nr); -+ -+ handle_progress_timer_event(group); -+ } -+ -+ process_cs_interrupts(group, ginfo, irqreq, irqack); -+ -+out: -+ /* group may still be NULL here */ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_INTERRUPT_PROCESS_END, group, -+ ((u64)req ^ ack) | (((u64)irqreq ^ irqack) << 32)); -+} -+ -+/** -+ * process_prfcnt_interrupts - Process performance counter interrupts. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @glb_req: Global request register value. -+ * @glb_ack: Global acknowledge register value. -+ * -+ * Handles interrupts issued by the firmware that relate to the performance -+ * counters. For example, on completion of a performance counter sample. It is -+ * expected that the scheduler spinlock is already held on calling this -+ * function. -+ */ -+static void process_prfcnt_interrupts(struct kbase_device *kbdev, u32 glb_req, -+ u32 glb_ack) -+{ -+ const struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); -+ -+ /* Process PRFCNT_SAMPLE interrupt. */ -+ if (kbdev->csf.hwcnt.request_pending && -+ ((glb_req & GLB_REQ_PRFCNT_SAMPLE_MASK) == -+ (glb_ack & GLB_REQ_PRFCNT_SAMPLE_MASK))) { -+ kbdev->csf.hwcnt.request_pending = false; -+ -+ dev_dbg(kbdev->dev, "PRFCNT_SAMPLE done interrupt received."); -+ -+ kbase_hwcnt_backend_csf_on_prfcnt_sample( -+ &kbdev->hwcnt_gpu_iface); -+ } -+ -+ /* Process PRFCNT_ENABLE interrupt. */ -+ if (kbdev->csf.hwcnt.enable_pending && -+ ((glb_req & GLB_REQ_PRFCNT_ENABLE_MASK) == -+ (glb_ack & GLB_REQ_PRFCNT_ENABLE_MASK))) { -+ kbdev->csf.hwcnt.enable_pending = false; -+ -+ dev_dbg(kbdev->dev, -+ "PRFCNT_ENABLE status changed interrupt received."); -+ -+ if (glb_ack & GLB_REQ_PRFCNT_ENABLE_MASK) -+ kbase_hwcnt_backend_csf_on_prfcnt_enable( -+ &kbdev->hwcnt_gpu_iface); -+ else -+ kbase_hwcnt_backend_csf_on_prfcnt_disable( -+ &kbdev->hwcnt_gpu_iface); -+ } -+ -+ /* Process PRFCNT_THRESHOLD interrupt. */ -+ if ((glb_req ^ glb_ack) & GLB_REQ_PRFCNT_THRESHOLD_MASK) { -+ dev_dbg(kbdev->dev, "PRFCNT_THRESHOLD interrupt received."); -+ -+ kbase_hwcnt_backend_csf_on_prfcnt_threshold( -+ &kbdev->hwcnt_gpu_iface); -+ -+ /* Set the GLB_REQ.PRFCNT_THRESHOLD flag back to -+ * the same value as GLB_ACK.PRFCNT_THRESHOLD -+ * flag in order to enable reporting of another -+ * PRFCNT_THRESHOLD event. -+ */ -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_REQ, glb_ack, -+ GLB_REQ_PRFCNT_THRESHOLD_MASK); -+ } -+ -+ /* Process PRFCNT_OVERFLOW interrupt. */ -+ if ((glb_req ^ glb_ack) & GLB_REQ_PRFCNT_OVERFLOW_MASK) { -+ dev_dbg(kbdev->dev, "PRFCNT_OVERFLOW interrupt received."); -+ -+ kbase_hwcnt_backend_csf_on_prfcnt_overflow( -+ &kbdev->hwcnt_gpu_iface); -+ -+ /* Set the GLB_REQ.PRFCNT_OVERFLOW flag back to -+ * the same value as GLB_ACK.PRFCNT_OVERFLOW -+ * flag in order to enable reporting of another -+ * PRFCNT_OVERFLOW event. -+ */ -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_REQ, glb_ack, -+ GLB_REQ_PRFCNT_OVERFLOW_MASK); -+ } -+} -+ -+void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val) -+{ -+ unsigned long flags; -+ u32 remaining = val; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ KBASE_KTRACE_ADD(kbdev, CSF_INTERRUPT, NULL, val); -+ kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), val); -+ -+ if (val & JOB_IRQ_GLOBAL_IF) { -+ const struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ kbdev->csf.interrupt_received = true; -+ remaining &= ~JOB_IRQ_GLOBAL_IF; -+ -+ if (!kbdev->csf.firmware_reloaded) -+ kbase_csf_firmware_reload_completed(kbdev); -+ else if (global_iface->output) { -+ u32 glb_req, glb_ack; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ glb_req = kbase_csf_firmware_global_input_read( -+ global_iface, GLB_REQ); -+ glb_ack = kbase_csf_firmware_global_output( -+ global_iface, GLB_ACK); -+ KBASE_KTRACE_ADD(kbdev, GLB_REQ_ACQ, NULL, glb_req ^ glb_ack); -+ -+ if ((glb_req ^ glb_ack) & GLB_REQ_PROTM_EXIT_MASK) { -+ dev_dbg(kbdev->dev, "Protected mode exit interrupt received"); -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_REQ, glb_ack, -+ GLB_REQ_PROTM_EXIT_MASK); -+ WARN_ON(!kbase_csf_scheduler_protected_mode_in_use(kbdev)); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_EXIT_PROTM, scheduler->active_protm_grp, 0u); -+ scheduler->active_protm_grp = NULL; -+ kbdev->protected_mode = false; -+ kbase_ipa_control_protm_exited(kbdev); -+ kbase_hwcnt_backend_csf_protm_exited( -+ &kbdev->hwcnt_gpu_iface); -+ } -+ -+ /* Handle IDLE Hysteresis notification event */ -+ if ((glb_req ^ glb_ack) & GLB_REQ_IDLE_EVENT_MASK) { -+ int non_idle_offslot_grps; -+ bool can_suspend_on_idle; -+ dev_dbg(kbdev->dev, "Idle-hysteresis event flagged"); -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_REQ, glb_ack, -+ GLB_REQ_IDLE_EVENT_MASK); -+ -+ non_idle_offslot_grps = atomic_read(&scheduler->non_idle_offslot_grps); -+ can_suspend_on_idle = kbase_pm_idle_groups_sched_suspendable(kbdev); -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_CAN_IDLE, NULL, -+ ((u64)(u32)non_idle_offslot_grps) | (((u64)can_suspend_on_idle) << 32)); -+ -+ if (!non_idle_offslot_grps) { -+ if (can_suspend_on_idle) -+ queue_work(system_highpri_wq, -+ &scheduler->gpu_idle_work); -+ } else { -+ /* Advance the scheduling tick to get -+ * the non-idle suspended groups loaded -+ * soon. -+ */ -+ kbase_csf_scheduler_advance_tick_nolock( -+ kbdev); -+ } -+ } -+ -+ process_prfcnt_interrupts(kbdev, glb_req, glb_ack); -+ -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ /* Invoke the MCU state machine as a state transition -+ * might have completed. -+ */ -+ kbase_pm_update_state(kbdev); -+ } -+ -+ if (!remaining) { -+ wake_up_all(&kbdev->csf.event_wait); -+ KBASE_KTRACE_ADD(kbdev, CSF_INTERRUPT_END, NULL, val); -+ return; -+ } -+ } -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ while (remaining != 0) { -+ int const csg_nr = ffs(remaining) - 1; -+ -+ process_csg_interrupts(kbdev, csg_nr); -+ remaining &= ~(1 << csg_nr); -+ } -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ wake_up_all(&kbdev->csf.event_wait); -+ KBASE_KTRACE_ADD(kbdev, CSF_INTERRUPT_END, NULL, val); -+} -+ -+void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev) -+{ -+ if (kbdev->csf.db_filp) { -+ struct page *page = as_page(kbdev->csf.dummy_db_page); -+ -+ kbase_mem_pool_free( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ page, false); -+ -+ fput(kbdev->csf.db_filp); -+ } -+} -+ -+int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev) -+{ -+ struct tagged_addr phys; -+ struct file *filp; -+ int ret; -+ -+ filp = shmem_file_setup("mali csf", MAX_LFS_FILESIZE, VM_NORESERVE); -+ if (IS_ERR(filp)) -+ return PTR_ERR(filp); -+ -+ ret = kbase_mem_pool_alloc_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ 1, &phys, false); -+ -+ if (ret <= 0) { -+ fput(filp); -+ return ret; -+ } -+ -+ kbdev->csf.db_filp = filp; -+ kbdev->csf.dummy_db_page = phys; -+ kbdev->csf.db_file_offsets = 0; -+ -+ return 0; -+} -+ -+void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev) -+{ -+ if (as_phys_addr_t(kbdev->csf.dummy_user_reg_page)) { -+ struct page *page = as_page(kbdev->csf.dummy_user_reg_page); -+ -+ kbase_mem_pool_free( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], page, -+ false); -+ } -+} -+ -+int kbase_csf_setup_dummy_user_reg_page(struct kbase_device *kbdev) -+{ -+ struct tagged_addr phys; -+ struct page *page; -+ u32 *addr; -+ int ret; -+ -+ kbdev->csf.dummy_user_reg_page = as_tagged(0); -+ -+ ret = kbase_mem_pool_alloc_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], 1, &phys, -+ false); -+ -+ if (ret <= 0) -+ return ret; -+ -+ page = as_page(phys); -+ addr = kmap_atomic(page); -+ -+ /* Write a special value for the latest flush register inside the -+ * dummy page -+ */ -+ addr[LATEST_FLUSH / sizeof(u32)] = POWER_DOWN_LATEST_FLUSH_VALUE; -+ -+ kbase_sync_single_for_device(kbdev, kbase_dma_addr(page), sizeof(u32), -+ DMA_BIDIRECTIONAL); -+ kunmap_atomic(addr); -+ -+ kbdev->csf.dummy_user_reg_page = phys; -+ -+ return 0; -+} -+ -+u8 kbase_csf_priority_check(struct kbase_device *kbdev, u8 req_priority) -+{ -+ struct priority_control_manager_device *pcm_device = kbdev->pcm_dev; -+ u8 out_priority = req_priority; -+ -+ if (pcm_device) { -+ req_priority = kbase_csf_priority_queue_group_priority_to_relative(req_priority); -+ out_priority = pcm_device->ops.pcm_scheduler_priority_check(pcm_device, current, req_priority); -+ out_priority = kbase_csf_priority_relative_to_queue_group_priority(out_priority); -+ } -+ -+ return out_priority; -+} -+ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf.h -new file mode 100644 -index 0000000..e3bd436 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf.h -@@ -0,0 +1,564 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_H_ -+#define _KBASE_CSF_H_ -+ -+#include "mali_kbase_csf_kcpu.h" -+#include "mali_kbase_csf_scheduler.h" -+#include "mali_kbase_csf_firmware.h" -+#include "mali_kbase_csf_protected_memory.h" -+ -+/* Indicate invalid CS h/w interface -+ */ -+#define KBASEP_IF_NR_INVALID ((s8)-1) -+ -+/* Indicate invalid CSG number for a GPU command queue group -+ */ -+#define KBASEP_CSG_NR_INVALID ((s8)-1) -+ -+/* Indicate invalid user doorbell number for a GPU command queue -+ */ -+#define KBASEP_USER_DB_NR_INVALID ((s8)-1) -+ -+#define FIRMWARE_PING_INTERVAL_MS (4000) /* 4 seconds */ -+ -+#define FIRMWARE_IDLE_HYSTERESIS_TIME_MS (10) /* Default 10 milliseconds */ -+ -+/** -+ * enum kbase_csf_event_callback_action - return type for CSF event callbacks. -+ * -+ * @KBASE_CSF_EVENT_CALLBACK_FIRST: Never set explicitly. -+ * It doesn't correspond to any action or type of event callback. -+ * -+ * @KBASE_CSF_EVENT_CALLBACK_KEEP: The callback will remain registered. -+ * -+ * @KBASE_CSF_EVENT_CALLBACK_REMOVE: The callback will be removed -+ * immediately upon return. -+ * -+ * @KBASE_CSF_EVENT_CALLBACK_LAST: Never set explicitly. -+ * It doesn't correspond to any action or type of event callback. -+ */ -+enum kbase_csf_event_callback_action { -+ KBASE_CSF_EVENT_CALLBACK_FIRST = 0, -+ KBASE_CSF_EVENT_CALLBACK_KEEP, -+ KBASE_CSF_EVENT_CALLBACK_REMOVE, -+ KBASE_CSF_EVENT_CALLBACK_LAST, -+}; -+ -+/** -+ * kbase_csf_event_callback_action - type for callback functions to be -+ * called upon CSF events. -+ * -+ * This is the type of callback functions that can be registered -+ * for CSF events. These function calls shall be triggered by any call -+ * to kbase_csf_event_signal. -+ * -+ * @param: Generic parameter to pass to the callback function. -+ * -+ * Return: KBASE_CSF_EVENT_CALLBACK_KEEP if the callback should remain -+ * registered, or KBASE_CSF_EVENT_CALLBACK_REMOVE if it should be removed. -+ */ -+typedef enum kbase_csf_event_callback_action kbase_csf_event_callback(void *param); -+ -+/** -+ * kbase_csf_event_wait_add - Add a CSF event callback -+ * -+ * This function adds an event callback to the list of CSF event callbacks -+ * belonging to a given Kbase context, to be triggered when a CSF event is -+ * signalled by kbase_csf_event_signal. -+ * -+ * @kctx: The Kbase context the @callback should be registered to. -+ * @callback: The callback function to register. -+ * @param: Custom parameter to be passed to the @callback function. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_event_wait_add(struct kbase_context *kctx, -+ kbase_csf_event_callback *callback, void *param); -+ -+/** -+ * kbase_csf_event_wait_remove - Remove a CSF event callback -+ * -+ * This function removes an event callback from the list of CSF event callbacks -+ * belonging to a given Kbase context. -+ * -+ * @kctx: The kbase context the @callback should be removed from. -+ * @callback: The callback function to remove. -+ * @param: Custom parameter that would have been passed to the @p callback -+ * function. -+ */ -+void kbase_csf_event_wait_remove(struct kbase_context *kctx, -+ kbase_csf_event_callback *callback, void *param); -+ -+/** -+ * kbase_csf_event_wait_remove_all - Removes all CSF event callbacks -+ * -+ * This function empties the list of CSF event callbacks belonging to a given -+ * Kbase context. -+ * -+ * @kctx: The kbase context for which CSF event callbacks have to be removed. -+ */ -+void kbase_csf_event_wait_remove_all(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_read_error - Read CS fatal error -+ * -+ * This function takes the CS fatal error from context's ordered -+ * error_list, copies its contents to @event_data. -+ * -+ * @kctx: The kbase context to read fatal error from -+ * @event_data: Caller-provided buffer to copy the fatal error to -+ * -+ * Return: true if fatal error is read successfully. -+ */ -+bool kbase_csf_read_error(struct kbase_context *kctx, -+ struct base_csf_notification *event_data); -+ -+/** -+ * kbase_csf_error_pending - Check whether fatal error is pending -+ * -+ * @kctx: The kbase context to check fatal error upon. -+ * -+ * Return: true if fatal error is pending. -+ */ -+bool kbase_csf_error_pending(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_event_signal - Signal a CSF event -+ * -+ * This function triggers all the CSF event callbacks that are registered to -+ * a given Kbase context, and also signals the event handling thread of -+ * userspace driver waiting for the CSF event. -+ * -+ * @kctx: The kbase context whose CSF event callbacks shall be triggered. -+ * @notify_gpu: Flag to indicate if CSF firmware should be notified of the -+ * signaling of event that happened on the Driver side, either -+ * the signal came from userspace or from kcpu queues. -+ */ -+void kbase_csf_event_signal(struct kbase_context *kctx, bool notify_gpu); -+ -+static inline void kbase_csf_event_signal_notify_gpu(struct kbase_context *kctx) -+{ -+ kbase_csf_event_signal(kctx, true); -+} -+ -+static inline void kbase_csf_event_signal_cpu_only(struct kbase_context *kctx) -+{ -+ kbase_csf_event_signal(kctx, false); -+} -+ -+/** -+ * kbase_csf_ctx_init - Initialize the CSF interface for a GPU address space. -+ * -+ * @kctx: Pointer to the kbase context which is being initialized. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+int kbase_csf_ctx_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_ctx_handle_fault - Terminate queue groups & notify fault upon -+ * GPU bus fault, MMU page fault or similar. -+ * -+ * This function terminates all GPU command queue groups in the context and -+ * notifies the event notification thread of the fault. -+ * -+ * @kctx: Pointer to faulty kbase context. -+ * @fault: Pointer to the fault. -+ */ -+void kbase_csf_ctx_handle_fault(struct kbase_context *kctx, -+ struct kbase_fault *fault); -+ -+/** -+ * kbase_csf_ctx_term - Terminate the CSF interface for a GPU address space. -+ * -+ * This function terminates any remaining CSGs and CSs which weren't destroyed -+ * before context termination. -+ * -+ * @kctx: Pointer to the kbase context which is being terminated. -+ */ -+void kbase_csf_ctx_term(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_queue_register - Register a GPU command queue. -+ * -+ * @kctx: Pointer to the kbase context within which the -+ * queue is to be registered. -+ * @reg: Pointer to the structure which contains details of the -+ * queue to be registered within the provided -+ * context. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_queue_register(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_register *reg); -+ -+/** -+ * kbase_csf_queue_register_ex - Register a GPU command queue with -+ * extended format. -+ * -+ * @kctx: Pointer to the kbase context within which the -+ * queue is to be registered. -+ * @reg: Pointer to the structure which contains details of the -+ * queue to be registered within the provided -+ * context, together with the extended parameter fields -+ * for supporting cs trace command. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_queue_register_ex(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_register_ex *reg); -+ -+/** -+ * kbase_csf_queue_terminate - Terminate a GPU command queue. -+ * -+ * @kctx: Pointer to the kbase context within which the -+ * queue is to be terminated. -+ * @term: Pointer to the structure which identifies which -+ * queue is to be terminated. -+ */ -+void kbase_csf_queue_terminate(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_terminate *term); -+ -+/** -+ * kbase_csf_alloc_command_stream_user_pages - Allocate resources for a -+ * GPU command queue. -+ * -+ * This function allocates a pair of User mode input/output pages for a -+ * GPU command queue and maps them in the shared interface segment of MCU -+ * firmware address space. Also reserves a hardware doorbell page for the queue. -+ * -+ * @kctx: Pointer to the kbase context within which the resources -+ * for the queue are being allocated. -+ * @queue: Pointer to the queue for which to allocate resources. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_alloc_command_stream_user_pages(struct kbase_context *kctx, -+ struct kbase_queue *queue); -+ -+/** -+ * kbase_csf_queue_bind - Bind a GPU command queue to a queue group. -+ * -+ * @kctx: The kbase context. -+ * @bind: Pointer to the union which specifies a queue group and a -+ * queue to be bound to that group. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_queue_bind(struct kbase_context *kctx, -+ union kbase_ioctl_cs_queue_bind *bind); -+ -+/** -+ * kbase_csf_queue_unbind - Unbind a GPU command queue from a queue group -+ * to which it has been bound and free -+ * resources allocated for this queue if there -+ * are any. -+ * -+ * @queue: Pointer to queue to be unbound. -+ */ -+void kbase_csf_queue_unbind(struct kbase_queue *queue); -+ -+/** -+ * kbase_csf_queue_unbind_stopped - Unbind a GPU command queue in the case -+ * where it was never started. -+ * @queue: Pointer to queue to be unbound. -+ * -+ * Variant of kbase_csf_queue_unbind() for use on error paths for cleaning up -+ * queues that failed to fully bind. -+ */ -+void kbase_csf_queue_unbind_stopped(struct kbase_queue *queue); -+ -+/** -+ * kbase_csf_queue_kick - Schedule a GPU command queue on the firmware -+ * -+ * @kctx: The kbase context. -+ * @kick: Pointer to the struct which specifies the queue -+ * that needs to be scheduled. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_queue_kick(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_kick *kick); -+ -+/** -+ * kbase_csf_queue_group_handle_is_valid - Find if the given queue group handle -+ * is valid. -+ * -+ * This function is used to determine if the queue group handle is valid. -+ * -+ * @kctx: The kbase context under which the queue group exists. -+ * @group_handle: Handle for the group which uniquely identifies it within -+ * the context with which it was created. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_queue_group_handle_is_valid(struct kbase_context *kctx, -+ u8 group_handle); -+ -+/** -+ * kbase_csf_queue_group_create - Create a GPU command queue group. -+ * -+ * @kctx: Pointer to the kbase context within which the -+ * queue group is to be created. -+ * @create: Pointer to the structure which contains details of the -+ * queue group which is to be created within the -+ * provided kbase context. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_queue_group_create(struct kbase_context *kctx, -+ union kbase_ioctl_cs_queue_group_create *create); -+ -+/** -+ * kbase_csf_queue_group_terminate - Terminate a GPU command queue group. -+ * -+ * @kctx: Pointer to the kbase context within which the -+ * queue group is to be terminated. -+ * @group_handle: Pointer to the structure which identifies the queue -+ * group which is to be terminated. -+ */ -+void kbase_csf_queue_group_terminate(struct kbase_context *kctx, -+ u8 group_handle); -+ -+/** -+ * kbase_csf_term_descheduled_queue_group - Terminate a GPU command queue -+ * group that is not operational -+ * inside the scheduler. -+ * -+ * @group: Pointer to the structure which identifies the queue -+ * group to be terminated. The function assumes that the caller -+ * is sure that the given group is not operational inside the -+ * scheduler. If in doubt, use its alternative: -+ * @ref kbase_csf_queue_group_terminate(). -+ */ -+void kbase_csf_term_descheduled_queue_group(struct kbase_queue_group *group); -+ -+/** -+ * kbase_csf_queue_group_suspend - Suspend a GPU command queue group -+ * -+ * This function is used to suspend a queue group and copy the suspend buffer. -+ * -+ * @kctx: The kbase context for which the queue group is to be -+ * suspended. -+ * @sus_buf: Pointer to the structure which contains details of the -+ * user buffer and its kernel pinned pages. -+ * @group_handle: Handle for the group which uniquely identifies it within -+ * the context within which it was created. -+ * -+ * Return: 0 on success or negative value if failed to suspend -+ * queue group and copy suspend buffer contents. -+ */ -+int kbase_csf_queue_group_suspend(struct kbase_context *kctx, -+ struct kbase_suspend_copy_buffer *sus_buf, u8 group_handle); -+ -+/** -+ * kbase_csf_add_group_fatal_error - Report a fatal group error to userspace -+ * -+ * @group: GPU command queue group. -+ * @err_payload: Error payload to report. -+ */ -+void kbase_csf_add_group_fatal_error( -+ struct kbase_queue_group *const group, -+ struct base_gpu_queue_group_error const *const err_payload); -+ -+/** -+ * kbase_csf_interrupt - Handle interrupts issued by CSF firmware. -+ * -+ * @kbdev: The kbase device to handle an IRQ for -+ * @val: The value of JOB IRQ status register which triggered the interrupt -+ */ -+void kbase_csf_interrupt(struct kbase_device *kbdev, u32 val); -+ -+/** -+ * kbase_csf_doorbell_mapping_init - Initialize the fields that facilitates -+ * the update of userspace mapping of HW -+ * doorbell page. -+ * -+ * The function creates a file and allocates a dummy page to facilitate the -+ * update of userspace mapping to point to the dummy page instead of the real -+ * HW doorbell page after the suspend of queue group. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_doorbell_mapping_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_doorbell_mapping_term - Free the dummy page & close the file used -+ * to update the userspace mapping of HW doorbell page -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_doorbell_mapping_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_setup_dummy_user_reg_page - Setup the dummy page that is accessed -+ * instead of the User register page after -+ * the GPU power down. -+ * -+ * The function allocates a dummy page which is used to replace the User -+ * register page in the userspace mapping after the power down of GPU. -+ * On the power up of GPU, the mapping is updated to point to the real -+ * User register page. The mapping is used to allow access to LATEST_FLUSH -+ * register from userspace. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_setup_dummy_user_reg_page(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_free_dummy_user_reg_page - Free the dummy page that was used -+ * used to replace the User register page -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_free_dummy_user_reg_page(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_ring_csg_doorbell - ring the doorbell for a CSG interface. -+ * -+ * The function kicks a notification on the CSG interface to firmware. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @slot: Index of CSG interface for ringing the door-bell. -+ */ -+void kbase_csf_ring_csg_doorbell(struct kbase_device *kbdev, int slot); -+ -+/** -+ * kbase_csf_ring_csg_slots_doorbell - ring the doorbell for a set of CSG -+ * interfaces. -+ * -+ * The function kicks a notification on a set of CSG interfaces to firmware. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @slot_bitmap: bitmap for the given slots, slot-0 on bit-0, etc. -+ */ -+void kbase_csf_ring_csg_slots_doorbell(struct kbase_device *kbdev, -+ u32 slot_bitmap); -+ -+/** -+ * kbase_csf_ring_cs_kernel_doorbell - ring the kernel doorbell for a CSI -+ * assigned to a GPU queue -+ * -+ * The function sends a doorbell interrupt notification to the firmware for -+ * a CSI assigned to a GPU queue. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @csi_index: ID of the CSI assigned to the GPU queue. -+ * @csg_nr: Index of the CSG slot assigned to the queue -+ * group to which the GPU queue is bound. -+ * @ring_csg_doorbell: Flag to indicate if the CSG doorbell needs to be rung -+ * after updating the CSG_DB_REQ. So if this flag is false -+ * the doorbell interrupt will not be sent to FW. -+ * The flag is supposed be false only when the input page -+ * for bound GPU queues is programmed at the time of -+ * starting/resuming the group on a CSG slot. -+ */ -+void kbase_csf_ring_cs_kernel_doorbell(struct kbase_device *kbdev, -+ int csi_index, int csg_nr, -+ bool ring_csg_doorbell); -+ -+/** -+ * kbase_csf_ring_cs_user_doorbell - ring the user doorbell allocated for a -+ * queue. -+ * -+ * The function kicks a notification to the firmware on the doorbell assigned -+ * to the queue. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @queue: Pointer to the queue for ringing the door-bell. -+ */ -+void kbase_csf_ring_cs_user_doorbell(struct kbase_device *kbdev, -+ struct kbase_queue *queue); -+ -+/** -+ * kbase_csf_active_queue_groups_reset - Reset the state of all active GPU -+ * command queue groups associated with the context. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @kctx: The kbase context. -+ * -+ * This function will iterate through all the active/scheduled GPU command -+ * queue groups associated with the context, deschedule and mark them as -+ * terminated (which will then lead to unbinding of all the queues bound to -+ * them) and also no more work would be allowed to execute for them. -+ * -+ * This is similar to the action taken in response to an unexpected OoM event. -+ */ -+void kbase_csf_active_queue_groups_reset(struct kbase_device *kbdev, -+ struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_priority_check - Check the priority requested -+ * -+ * @kbdev: Device pointer -+ * @req_priority: Requested priority -+ * -+ * This will determine whether the requested priority can be satisfied. -+ * -+ * Return: The same or lower priority than requested. -+ */ -+u8 kbase_csf_priority_check(struct kbase_device *kbdev, u8 req_priority); -+ -+extern const u8 kbasep_csf_queue_group_priority_to_relative[BASE_QUEUE_GROUP_PRIORITY_COUNT]; -+extern const u8 kbasep_csf_relative_to_queue_group_priority[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; -+ -+/** -+ * kbase_csf_priority_relative_to_queue_group_priority - Convert relative to base priority -+ * -+ * @priority: kbase relative priority -+ * -+ * This will convert the monotonically increasing realtive priority to the -+ * fixed base priority list. -+ * -+ * Return: base_queue_group_priority priority. -+ */ -+static inline u8 kbase_csf_priority_relative_to_queue_group_priority(u8 priority) -+{ -+ if (priority >= KBASE_QUEUE_GROUP_PRIORITY_COUNT) -+ priority = KBASE_QUEUE_GROUP_PRIORITY_LOW; -+ return kbasep_csf_relative_to_queue_group_priority[priority]; -+} -+ -+/** -+ * kbase_csf_priority_queue_group_priority_to_relative - Convert base priority to relative -+ * -+ * @priority: base_queue_group_priority priority -+ * -+ * This will convert the fixed base priority list to monotonically increasing realtive priority. -+ * -+ * Return: kbase relative priority. -+ */ -+static inline u8 kbase_csf_priority_queue_group_priority_to_relative(u8 priority) -+{ -+ /* Apply low priority in case of invalid priority */ -+ if (priority >= BASE_QUEUE_GROUP_PRIORITY_COUNT) -+ priority = BASE_QUEUE_GROUP_PRIORITY_LOW; -+ return kbasep_csf_queue_group_priority_to_relative[priority]; -+} -+ -+ -+#endif /* _KBASE_CSF_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_cpu_queue_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_cpu_queue_debugfs.c -new file mode 100644 -index 0000000..26637bf ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_cpu_queue_debugfs.c -@@ -0,0 +1,191 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_csf_cpu_queue_debugfs.h" -+#include <mali_kbase.h> -+#include <linux/seq_file.h> -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ -+bool kbase_csf_cpu_queue_read_dump_req(struct kbase_context *kctx, -+ struct base_csf_notification *req) -+{ -+ if (atomic_cmpxchg(&kctx->csf.cpu_queue.dump_req_status, -+ BASE_CSF_CPU_QUEUE_DUMP_ISSUED, -+ BASE_CSF_CPU_QUEUE_DUMP_PENDING) != -+ BASE_CSF_CPU_QUEUE_DUMP_ISSUED) { -+ return false; -+ } -+ -+ req->type = BASE_CSF_NOTIFICATION_CPU_QUEUE_DUMP; -+ return true; -+} -+ -+/** -+ * kbasep_csf_cpu_queue_debugfs_show() - Print cpu queue information for per context -+ * -+ * @file: The seq_file for printing to -+ * @data: The debugfs dentry private data, a pointer to kbase_context -+ * -+ * Return: Negative error code or 0 on success. -+ */ -+static int kbasep_csf_cpu_queue_debugfs_show(struct seq_file *file, void *data) -+{ -+ struct kbase_context *kctx = file->private; -+ -+ mutex_lock(&kctx->csf.lock); -+ if (atomic_read(&kctx->csf.cpu_queue.dump_req_status) != -+ BASE_CSF_CPU_QUEUE_DUMP_COMPLETE) { -+ seq_printf(file, "Dump request already started! (try again)\n"); -+ mutex_unlock(&kctx->csf.lock); -+ return -EBUSY; -+ } -+ -+ atomic_set(&kctx->csf.cpu_queue.dump_req_status, BASE_CSF_CPU_QUEUE_DUMP_ISSUED); -+ init_completion(&kctx->csf.cpu_queue.dump_cmp); -+ kbase_event_wakeup(kctx); -+ mutex_unlock(&kctx->csf.lock); -+ -+ seq_printf(file, "CPU Queues table (version:v%u):\n", MALI_CSF_CPU_QUEUE_DEBUGFS_VERSION); -+ -+ wait_for_completion_timeout(&kctx->csf.cpu_queue.dump_cmp, -+ msecs_to_jiffies(3000)); -+ -+ mutex_lock(&kctx->csf.lock); -+ if (kctx->csf.cpu_queue.buffer) { -+ WARN_ON(atomic_read(&kctx->csf.cpu_queue.dump_req_status) != -+ BASE_CSF_CPU_QUEUE_DUMP_PENDING); -+ -+ seq_printf(file, "%s\n", kctx->csf.cpu_queue.buffer); -+ -+ kfree(kctx->csf.cpu_queue.buffer); -+ kctx->csf.cpu_queue.buffer = NULL; -+ kctx->csf.cpu_queue.buffer_size = 0; -+ } -+ else -+ seq_printf(file, "Dump error! (time out)\n"); -+ -+ atomic_set(&kctx->csf.cpu_queue.dump_req_status, -+ BASE_CSF_CPU_QUEUE_DUMP_COMPLETE); -+ -+ mutex_unlock(&kctx->csf.lock); -+ return 0; -+} -+ -+static int kbasep_csf_cpu_queue_debugfs_open(struct inode *in, struct file *file) -+{ -+ return single_open(file, kbasep_csf_cpu_queue_debugfs_show, in->i_private); -+} -+ -+static const struct file_operations kbasep_csf_cpu_queue_debugfs_fops = { -+ .open = kbasep_csf_cpu_queue_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+void kbase_csf_cpu_queue_debugfs_init(struct kbase_context *kctx) -+{ -+ struct dentry *file; -+ -+ if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) -+ return; -+ -+ file = debugfs_create_file("cpu_queue", 0444, kctx->kctx_dentry, -+ kctx, &kbasep_csf_cpu_queue_debugfs_fops); -+ -+ if (IS_ERR_OR_NULL(file)) { -+ dev_warn(kctx->kbdev->dev, -+ "Unable to create cpu queue debugfs entry"); -+ } -+ -+ kctx->csf.cpu_queue.buffer = NULL; -+ kctx->csf.cpu_queue.buffer_size = 0; -+ atomic_set(&kctx->csf.cpu_queue.dump_req_status, -+ BASE_CSF_CPU_QUEUE_DUMP_COMPLETE); -+} -+ -+int kbase_csf_cpu_queue_dump(struct kbase_context *kctx, -+ u64 buffer, size_t buf_size) -+{ -+ int err = 0; -+ -+ size_t alloc_size = buf_size; -+ char *dump_buffer; -+ -+ if (!buffer || !alloc_size) -+ goto done; -+ -+ alloc_size = (alloc_size + PAGE_SIZE) & ~(PAGE_SIZE - 1); -+ dump_buffer = kzalloc(alloc_size, GFP_KERNEL); -+ if (ZERO_OR_NULL_PTR(dump_buffer)) { -+ err = -ENOMEM; -+ goto done; -+ } -+ -+ WARN_ON(kctx->csf.cpu_queue.buffer != NULL); -+ -+ err = copy_from_user(dump_buffer, -+ u64_to_user_ptr(buffer), -+ buf_size); -+ if (err) { -+ kfree(dump_buffer); -+ err = -EFAULT; -+ goto done; -+ } -+ -+ mutex_lock(&kctx->csf.lock); -+ -+ kfree(kctx->csf.cpu_queue.buffer); -+ -+ if (atomic_read(&kctx->csf.cpu_queue.dump_req_status) == -+ BASE_CSF_CPU_QUEUE_DUMP_PENDING) { -+ kctx->csf.cpu_queue.buffer = dump_buffer; -+ kctx->csf.cpu_queue.buffer_size = buf_size; -+ complete_all(&kctx->csf.cpu_queue.dump_cmp); -+ } else { -+ kfree(dump_buffer); -+ } -+ -+ mutex_unlock(&kctx->csf.lock); -+done: -+ return err; -+} -+#else -+/* -+ * Stub functions for when debugfs is disabled -+ */ -+void kbase_csf_cpu_queue_debugfs_init(struct kbase_context *kctx) -+{ -+} -+ -+bool kbase_csf_cpu_queue_read_dump_req(struct kbase_context *kctx, -+ struct base_csf_notification *req) -+{ -+ return false; -+} -+ -+int kbase_csf_cpu_queue_dump(struct kbase_context *kctx, -+ u64 buffer, size_t buf_size) -+{ -+ return 0; -+} -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_cpu_queue_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_cpu_queue_debugfs.h -new file mode 100644 -index 0000000..435a993 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_cpu_queue_debugfs.h -@@ -0,0 +1,90 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_CPU_QUEUE_DEBUGFS_H_ -+#define _KBASE_CSF_CPU_QUEUE_DEBUGFS_H_ -+ -+#include <asm/atomic.h> -+#include <linux/types.h> -+ -+#include "mali_kbase.h" -+ -+/* Forward declaration */ -+struct base_csf_notification; -+ -+#define MALI_CSF_CPU_QUEUE_DEBUGFS_VERSION 0 -+ -+/* CPU queue dump status */ -+/* Dumping is done or no dumping is in progress. */ -+#define BASE_CSF_CPU_QUEUE_DUMP_COMPLETE 0 -+/* Dumping request is pending. */ -+#define BASE_CSF_CPU_QUEUE_DUMP_PENDING 1 -+/* Dumping request is issued to Userspace */ -+#define BASE_CSF_CPU_QUEUE_DUMP_ISSUED 2 -+ -+ -+/** -+ * kbase_csf_cpu_queue_debugfs_init() - Create a debugfs entry for per context cpu queue(s) -+ * -+ * @kctx: The kbase_context for which to create the debugfs entry -+ */ -+void kbase_csf_cpu_queue_debugfs_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_cpu_queue_read_dump_req - Read cpu queue dump request event -+ * -+ * @kctx: The kbase_context which cpu queue dumpped belongs to -+ * @req: Notification with cpu queue dump request. -+ * -+ * Return: true if needs CPU queue dump, or false otherwise. -+ */ -+bool kbase_csf_cpu_queue_read_dump_req(struct kbase_context *kctx, -+ struct base_csf_notification *req); -+ -+/** -+ * kbase_csf_cpu_queue_dump_needed - Check the requirement for cpu queue dump -+ * -+ * @kctx: The kbase_context which cpu queue dumpped belongs to -+ * -+ * Return: true if it needs cpu queue dump, or false otherwise. -+ */ -+static inline bool kbase_csf_cpu_queue_dump_needed(struct kbase_context *kctx) -+{ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ return (atomic_read(&kctx->csf.cpu_queue.dump_req_status) == -+ BASE_CSF_CPU_QUEUE_DUMP_ISSUED); -+#else -+ return false; -+#endif -+} -+ -+/** -+ * kbase_csf_cpu_queue_dump - dump buffer containing cpu queue information to debugfs -+ * -+ * @kctx: The kbase_context which cpu queue dumpped belongs to -+ * @buffer: Buffer containing the cpu queue information. -+ * @buf_size: Buffer size. -+ * -+ * Return: Return 0 for dump successfully, or error code. -+ */ -+int kbase_csf_cpu_queue_dump(struct kbase_context *kctx, -+ u64 buffer, size_t buf_size); -+#endif /* _KBASE_CSF_CPU_QUEUE_DEBUGFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_csg_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_csg_debugfs.c -new file mode 100644 -index 0000000..14deb98 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_csg_debugfs.c -@@ -0,0 +1,591 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_csf_csg_debugfs.h" -+#include <mali_kbase.h> -+#include <linux/seq_file.h> -+#include <linux/delay.h> -+#include <csf/mali_kbase_csf_trace_buffer.h> -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+#include "mali_kbase_csf_tl_reader.h" -+ -+/** -+ * blocked_reason_to_string() - Convert blocking reason id to a string -+ * -+ * @reason_id: blocked_reason -+ * -+ * Return: Suitable string -+ */ -+static const char *blocked_reason_to_string(u32 reason_id) -+{ -+ /* possible blocking reasons of a cs */ -+ static const char *const cs_blocked_reason[] = { -+ [CS_STATUS_BLOCKED_REASON_REASON_UNBLOCKED] = "UNBLOCKED", -+ [CS_STATUS_BLOCKED_REASON_REASON_WAIT] = "WAIT", -+ [CS_STATUS_BLOCKED_REASON_REASON_PROGRESS_WAIT] = -+ "PROGRESS_WAIT", -+ [CS_STATUS_BLOCKED_REASON_REASON_SYNC_WAIT] = "SYNC_WAIT", -+ [CS_STATUS_BLOCKED_REASON_REASON_DEFERRED] = "DEFERRED", -+ [CS_STATUS_BLOCKED_REASON_REASON_RESOURCE] = "RESOURCE", -+ [CS_STATUS_BLOCKED_REASON_REASON_FLUSH] = "FLUSH" -+ }; -+ -+ if (WARN_ON(reason_id >= ARRAY_SIZE(cs_blocked_reason))) -+ return "UNKNOWN_BLOCKED_REASON_ID"; -+ -+ return cs_blocked_reason[reason_id]; -+} -+ -+static void kbasep_csf_scheduler_dump_active_queue_cs_status_wait( -+ struct seq_file *file, u32 wait_status, u32 wait_sync_value, -+ u64 wait_sync_live_value, u64 wait_sync_pointer, u32 sb_status, -+ u32 blocked_reason) -+{ -+#define WAITING "Waiting" -+#define NOT_WAITING "Not waiting" -+ -+ seq_printf(file, "SB_MASK: %d\n", -+ CS_STATUS_WAIT_SB_MASK_GET(wait_status)); -+ seq_printf(file, "PROGRESS_WAIT: %s\n", -+ CS_STATUS_WAIT_PROGRESS_WAIT_GET(wait_status) ? -+ WAITING : NOT_WAITING); -+ seq_printf(file, "PROTM_PEND: %s\n", -+ CS_STATUS_WAIT_PROTM_PEND_GET(wait_status) ? -+ WAITING : NOT_WAITING); -+ seq_printf(file, "SYNC_WAIT: %s\n", -+ CS_STATUS_WAIT_SYNC_WAIT_GET(wait_status) ? -+ WAITING : NOT_WAITING); -+ seq_printf(file, "WAIT_CONDITION: %s\n", -+ CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(wait_status) ? -+ "greater than" : "less or equal"); -+ seq_printf(file, "SYNC_POINTER: 0x%llx\n", wait_sync_pointer); -+ seq_printf(file, "SYNC_VALUE: %d\n", wait_sync_value); -+ seq_printf(file, "SYNC_LIVE_VALUE: 0x%016llx\n", wait_sync_live_value); -+ seq_printf(file, "SB_STATUS: %u\n", -+ CS_STATUS_SCOREBOARDS_NONZERO_GET(sb_status)); -+ seq_printf(file, "BLOCKED_REASON: %s\n", -+ blocked_reason_to_string(CS_STATUS_BLOCKED_REASON_REASON_GET( -+ blocked_reason))); -+} -+ -+static void kbasep_csf_scheduler_dump_active_cs_trace(struct seq_file *file, -+ struct kbase_csf_cmd_stream_info const *const stream) -+{ -+ u32 val = kbase_csf_firmware_cs_input_read(stream, -+ CS_INSTR_BUFFER_BASE_LO); -+ u64 addr = ((u64)kbase_csf_firmware_cs_input_read(stream, -+ CS_INSTR_BUFFER_BASE_HI) << 32) | val; -+ val = kbase_csf_firmware_cs_input_read(stream, -+ CS_INSTR_BUFFER_SIZE); -+ -+ seq_printf(file, "CS_TRACE_BUF_ADDR: 0x%16llx, SIZE: %u\n", addr, val); -+ -+ /* Write offset variable address (pointer) */ -+ val = kbase_csf_firmware_cs_input_read(stream, -+ CS_INSTR_BUFFER_OFFSET_POINTER_LO); -+ addr = ((u64)kbase_csf_firmware_cs_input_read(stream, -+ CS_INSTR_BUFFER_OFFSET_POINTER_HI) << 32) | val; -+ seq_printf(file, "CS_TRACE_BUF_OFFSET_PTR: 0x%16llx\n", addr); -+ -+ /* EVENT_SIZE and EVENT_STATEs */ -+ val = kbase_csf_firmware_cs_input_read(stream, CS_INSTR_CONFIG); -+ seq_printf(file, "TRACE_EVENT_SIZE: 0x%x, TRACE_EVENT_STAES 0x%x\n", -+ CS_INSTR_CONFIG_EVENT_SIZE_GET(val), -+ CS_INSTR_CONFIG_EVENT_STATE_GET(val)); -+} -+ -+/** -+ * kbasep_csf_scheduler_dump_active_queue() - Print GPU command queue -+ * debug information -+ * -+ * @file: seq_file for printing to -+ * @queue: Address of a GPU command queue to examine -+ */ -+static void kbasep_csf_scheduler_dump_active_queue(struct seq_file *file, -+ struct kbase_queue *queue) -+{ -+ u32 *addr; -+ u64 cs_extract; -+ u64 cs_insert; -+ u32 cs_active; -+ u64 wait_sync_pointer; -+ u32 wait_status, wait_sync_value; -+ u32 sb_status; -+ u32 blocked_reason; -+ struct kbase_vmap_struct *mapping; -+ u64 *evt; -+ u64 wait_sync_live_value; -+ -+ if (!queue) -+ return; -+ -+ if (WARN_ON(queue->csi_index == KBASEP_IF_NR_INVALID || -+ !queue->group)) -+ return; -+ -+ /* Ring the doorbell to have firmware update CS_EXTRACT */ -+ kbase_csf_ring_cs_user_doorbell(queue->kctx->kbdev, queue); -+ msleep(100); -+ -+ addr = (u32 *)queue->user_io_addr; -+ cs_insert = addr[CS_INSERT_LO/4] | ((u64)addr[CS_INSERT_HI/4] << 32); -+ -+ addr = (u32 *)(queue->user_io_addr + PAGE_SIZE); -+ cs_extract = addr[CS_EXTRACT_LO/4] | ((u64)addr[CS_EXTRACT_HI/4] << 32); -+ cs_active = addr[CS_ACTIVE/4]; -+ -+#define KBASEP_CSF_DEBUGFS_CS_HEADER_USER_IO \ -+ "Bind Idx, Ringbuf addr, Prio, Insert offset, Extract offset, Active, Doorbell\n" -+ -+ seq_printf(file, KBASEP_CSF_DEBUGFS_CS_HEADER_USER_IO "%8d, %16llx, %4u, %16llx, %16llx, %6u, %8d\n", -+ queue->csi_index, queue->base_addr, queue->priority, -+ cs_insert, cs_extract, cs_active, queue->doorbell_nr); -+ -+ /* Print status information for blocked group waiting for sync object. For on-slot queues, -+ * if cs_trace is enabled, dump the interface's cs_trace configuration. -+ */ -+ if (kbase_csf_scheduler_group_get_slot(queue->group) < 0) { -+ if (CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) { -+ wait_status = queue->status_wait; -+ wait_sync_value = queue->sync_value; -+ wait_sync_pointer = queue->sync_ptr; -+ sb_status = queue->sb_status; -+ blocked_reason = queue->blocked_reason; -+ -+ evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx, wait_sync_pointer, &mapping); -+ if (evt) { -+ wait_sync_live_value = evt[0]; -+ kbase_phy_alloc_mapping_put(queue->kctx, mapping); -+ } else { -+ wait_sync_live_value = U64_MAX; -+ } -+ -+ kbasep_csf_scheduler_dump_active_queue_cs_status_wait( -+ file, wait_status, wait_sync_value, -+ wait_sync_live_value, wait_sync_pointer, -+ sb_status, blocked_reason); -+ } -+ } else { -+ struct kbase_device const *const kbdev = -+ queue->group->kctx->kbdev; -+ struct kbase_csf_cmd_stream_group_info const *const ginfo = -+ &kbdev->csf.global_iface.groups[queue->group->csg_nr]; -+ struct kbase_csf_cmd_stream_info const *const stream = -+ &ginfo->streams[queue->csi_index]; -+ u64 cmd_ptr; -+ u32 req_res; -+ -+ if (WARN_ON(!stream)) -+ return; -+ -+ cmd_ptr = kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_CMD_PTR_LO); -+ cmd_ptr |= (u64)kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_CMD_PTR_HI) << 32; -+ req_res = kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_REQ_RESOURCE); -+ -+ seq_printf(file, "CMD_PTR: 0x%llx\n", cmd_ptr); -+ seq_printf(file, "REQ_RESOURCE [COMPUTE]: %d\n", -+ CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_GET(req_res)); -+ seq_printf(file, "REQ_RESOURCE [FRAGMENT]: %d\n", -+ CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_GET(req_res)); -+ seq_printf(file, "REQ_RESOURCE [TILER]: %d\n", -+ CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_GET(req_res)); -+ seq_printf(file, "REQ_RESOURCE [IDVS]: %d\n", -+ CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_GET(req_res)); -+ -+ wait_status = kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_WAIT); -+ wait_sync_value = kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_WAIT_SYNC_VALUE); -+ wait_sync_pointer = kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_WAIT_SYNC_POINTER_LO); -+ wait_sync_pointer |= (u64)kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_WAIT_SYNC_POINTER_HI) << 32; -+ -+ sb_status = kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_SCOREBOARDS); -+ blocked_reason = kbase_csf_firmware_cs_output( -+ stream, CS_STATUS_BLOCKED_REASON); -+ -+ evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx, wait_sync_pointer, &mapping); -+ if (evt) { -+ wait_sync_live_value = evt[0]; -+ kbase_phy_alloc_mapping_put(queue->kctx, mapping); -+ } else { -+ wait_sync_live_value = U64_MAX; -+ } -+ -+ kbasep_csf_scheduler_dump_active_queue_cs_status_wait( -+ file, wait_status, wait_sync_value, -+ wait_sync_live_value, wait_sync_pointer, sb_status, -+ blocked_reason); -+ /* Dealing with cs_trace */ -+ if (kbase_csf_scheduler_queue_has_trace(queue)) -+ kbasep_csf_scheduler_dump_active_cs_trace(file, stream); -+ else -+ seq_puts(file, "NO CS_TRACE\n"); -+ } -+ -+ seq_puts(file, "\n"); -+} -+ -+/* Waiting timeout for STATUS_UPDATE acknowledgment, in milliseconds */ -+#define CSF_STATUS_UPDATE_TO_MS (100) -+ -+static void kbasep_csf_scheduler_dump_active_group(struct seq_file *file, -+ struct kbase_queue_group *const group) -+{ -+ if (kbase_csf_scheduler_group_get_slot(group) >= 0) { -+ struct kbase_device *const kbdev = group->kctx->kbdev; -+ unsigned long flags; -+ u32 ep_c, ep_r; -+ char exclusive; -+ struct kbase_csf_cmd_stream_group_info const *const ginfo = -+ &kbdev->csf.global_iface.groups[group->csg_nr]; -+ long remaining = -+ kbase_csf_timeout_in_jiffies(CSF_STATUS_UPDATE_TO_MS); -+ u8 slot_priority = -+ kbdev->csf.scheduler.csg_slots[group->csg_nr].priority; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, -+ ~kbase_csf_firmware_csg_output(ginfo, CSG_ACK), -+ CSG_REQ_STATUS_UPDATE_MASK); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ kbase_csf_ring_csg_doorbell(kbdev, group->csg_nr); -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ !((kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ) ^ -+ kbase_csf_firmware_csg_output(ginfo, CSG_ACK)) & -+ CSG_REQ_STATUS_UPDATE_MASK), remaining); -+ -+ ep_c = kbase_csf_firmware_csg_output(ginfo, -+ CSG_STATUS_EP_CURRENT); -+ ep_r = kbase_csf_firmware_csg_output(ginfo, CSG_STATUS_EP_REQ); -+ -+ if (CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_GET(ep_r)) -+ exclusive = 'C'; -+ else if (CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_GET(ep_r)) -+ exclusive = 'F'; -+ else -+ exclusive = '0'; -+ -+ if (!remaining) { -+ dev_err(kbdev->dev, -+ "Timed out for STATUS_UPDATE on group %d on slot %d", -+ group->handle, group->csg_nr); -+ -+ seq_printf(file, "*** Warn: Timed out for STATUS_UPDATE on slot %d\n", -+ group->csg_nr); -+ seq_printf(file, "*** The following group-record is likely stale\n"); -+ } -+ -+ seq_puts(file, "GroupID, CSG NR, CSG Prio, Run State, Priority, C_EP(Alloc/Req), F_EP(Alloc/Req), T_EP(Alloc/Req), Exclusive\n"); -+ seq_printf(file, "%7d, %6d, %8d, %9d, %8d, %11d/%3d, %11d/%3d, %11d/%3d, %9c\n", -+ group->handle, -+ group->csg_nr, -+ slot_priority, -+ group->run_state, -+ group->priority, -+ CSG_STATUS_EP_CURRENT_COMPUTE_EP_GET(ep_c), -+ CSG_STATUS_EP_REQ_COMPUTE_EP_GET(ep_r), -+ CSG_STATUS_EP_CURRENT_FRAGMENT_EP_GET(ep_c), -+ CSG_STATUS_EP_REQ_FRAGMENT_EP_GET(ep_r), -+ CSG_STATUS_EP_CURRENT_TILER_EP_GET(ep_c), -+ CSG_STATUS_EP_REQ_TILER_EP_GET(ep_r), -+ exclusive); -+ } else { -+ seq_puts(file, "GroupID, CSG NR, Run State, Priority\n"); -+ seq_printf(file, "%7d, %6d, %9d, %8d\n", -+ group->handle, -+ group->csg_nr, -+ group->run_state, -+ group->priority); -+ } -+ -+ if (group->run_state != KBASE_CSF_GROUP_TERMINATED) { -+ unsigned int i; -+ -+ seq_puts(file, "Bound queues:\n"); -+ -+ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { -+ kbasep_csf_scheduler_dump_active_queue(file, -+ group->bound_queues[i]); -+ } -+ } -+ -+ seq_puts(file, "\n"); -+} -+ -+/** -+ * kbasep_csf_queue_group_debugfs_show() - Print per-context GPU command queue -+ * group debug information -+ * -+ * @file: The seq_file for printing to -+ * @data: The debugfs dentry private data, a pointer to kbase context -+ * -+ * Return: Negative error code or 0 on success. -+ */ -+static int kbasep_csf_queue_group_debugfs_show(struct seq_file *file, -+ void *data) -+{ -+ u32 gr; -+ struct kbase_context *const kctx = file->private; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ if (WARN_ON(!kctx)) -+ return -EINVAL; -+ -+ seq_printf(file, "MALI_CSF_CSG_DEBUGFS_VERSION: v%u\n", -+ MALI_CSF_CSG_DEBUGFS_VERSION); -+ -+ mutex_lock(&kctx->csf.lock); -+ kbase_csf_scheduler_lock(kbdev); -+ for (gr = 0; gr < MAX_QUEUE_GROUP_NUM; gr++) { -+ struct kbase_queue_group *const group = -+ kctx->csf.queue_groups[gr]; -+ -+ if (group) -+ kbasep_csf_scheduler_dump_active_group(file, group); -+ } -+ kbase_csf_scheduler_unlock(kbdev); -+ mutex_unlock(&kctx->csf.lock); -+ -+ return 0; -+} -+ -+/** -+ * kbasep_csf_scheduler_dump_active_groups() - Print debug info for active -+ * GPU command queue groups -+ * -+ * @file: The seq_file for printing to -+ * @data: The debugfs dentry private data, a pointer to kbase_device -+ * -+ * Return: Negative error code or 0 on success. -+ */ -+static int kbasep_csf_scheduler_dump_active_groups(struct seq_file *file, -+ void *data) -+{ -+ u32 csg_nr; -+ struct kbase_device *kbdev = file->private; -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ -+ seq_printf(file, "MALI_CSF_CSG_DEBUGFS_VERSION: v%u\n", -+ MALI_CSF_CSG_DEBUGFS_VERSION); -+ -+ kbase_csf_scheduler_lock(kbdev); -+ for (csg_nr = 0; csg_nr < num_groups; csg_nr++) { -+ struct kbase_queue_group *const group = -+ kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; -+ -+ if (!group) -+ continue; -+ -+ seq_printf(file, "\nCtx %d_%d\n", group->kctx->tgid, -+ group->kctx->id); -+ -+ kbasep_csf_scheduler_dump_active_group(file, group); -+ } -+ kbase_csf_scheduler_unlock(kbdev); -+ -+ return 0; -+} -+ -+static int kbasep_csf_queue_group_debugfs_open(struct inode *in, -+ struct file *file) -+{ -+ return single_open(file, kbasep_csf_queue_group_debugfs_show, -+ in->i_private); -+} -+ -+static int kbasep_csf_active_queue_groups_debugfs_open(struct inode *in, -+ struct file *file) -+{ -+ return single_open(file, kbasep_csf_scheduler_dump_active_groups, -+ in->i_private); -+} -+ -+static const struct file_operations kbasep_csf_queue_group_debugfs_fops = { -+ .open = kbasep_csf_queue_group_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+void kbase_csf_queue_group_debugfs_init(struct kbase_context *kctx) -+{ -+ struct dentry *file; -+#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) -+ const mode_t mode = 0444; -+#else -+ const mode_t mode = 0400; -+#endif -+ -+ if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) -+ return; -+ -+ file = debugfs_create_file("groups", mode, -+ kctx->kctx_dentry, kctx, &kbasep_csf_queue_group_debugfs_fops); -+ -+ if (IS_ERR_OR_NULL(file)) { -+ dev_warn(kctx->kbdev->dev, -+ "Unable to create per context queue groups debugfs entry"); -+ } -+} -+ -+static const struct file_operations -+ kbasep_csf_active_queue_groups_debugfs_fops = { -+ .open = kbasep_csf_active_queue_groups_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+static int kbasep_csf_debugfs_scheduling_timer_enabled_get( -+ void *data, u64 *val) -+{ -+ struct kbase_device *const kbdev = data; -+ -+ *val = kbase_csf_scheduler_timer_is_enabled(kbdev); -+ -+ return 0; -+} -+ -+static int kbasep_csf_debugfs_scheduling_timer_enabled_set( -+ void *data, u64 val) -+{ -+ struct kbase_device *const kbdev = data; -+ -+ kbase_csf_scheduler_timer_set_enabled(kbdev, val != 0); -+ -+ return 0; -+} -+ -+static int kbasep_csf_debugfs_scheduling_timer_kick_set( -+ void *data, u64 val) -+{ -+ struct kbase_device *const kbdev = data; -+ -+ kbase_csf_scheduler_kick(kbdev); -+ -+ return 0; -+} -+ -+DEFINE_SIMPLE_ATTRIBUTE(kbasep_csf_debugfs_scheduling_timer_enabled_fops, -+ &kbasep_csf_debugfs_scheduling_timer_enabled_get, -+ &kbasep_csf_debugfs_scheduling_timer_enabled_set, -+ "%llu\n"); -+DEFINE_SIMPLE_ATTRIBUTE(kbasep_csf_debugfs_scheduling_timer_kick_fops, -+ NULL, -+ &kbasep_csf_debugfs_scheduling_timer_kick_set, -+ "%llu\n"); -+ -+/** -+ * kbase_csf_debugfs_scheduler_suspend_get() - get if the scheduler is suspended. -+ * -+ * @data: The debugfs dentry private data, a pointer to kbase_device -+ * @val: The debugfs output value, boolean: 1 suspended, 0 otherwise -+ * -+ * Return: 0 -+ */ -+static int kbase_csf_debugfs_scheduler_suspend_get( -+ void *data, u64 *val) -+{ -+ struct kbase_device *kbdev = data; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ kbase_csf_scheduler_lock(kbdev); -+ *val = (scheduler->state == SCHED_SUSPENDED); -+ kbase_csf_scheduler_unlock(kbdev); -+ -+ return 0; -+} -+ -+/** -+ * kbase_csf_debugfs_scheduler_suspend_set() - set the scheduler to suspended. -+ * -+ * @data: The debugfs dentry private data, a pointer to kbase_device -+ * @val: The debugfs input value, boolean: 1 suspend, 0 otherwise -+ * -+ * Return: Negative value if already in requested state, 0 otherwise. -+ */ -+static int kbase_csf_debugfs_scheduler_suspend_set( -+ void *data, u64 val) -+{ -+ struct kbase_device *kbdev = data; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ enum kbase_csf_scheduler_state state; -+ -+ kbase_csf_scheduler_lock(kbdev); -+ state = scheduler->state; -+ kbase_csf_scheduler_unlock(kbdev); -+ -+ if (val && (state != SCHED_SUSPENDED)) -+ kbase_csf_scheduler_pm_suspend(kbdev); -+ else if (!val && (state == SCHED_SUSPENDED)) -+ kbase_csf_scheduler_pm_resume(kbdev); -+ else -+ return -1; -+ -+ return 0; -+} -+ -+DEFINE_SIMPLE_ATTRIBUTE(kbasep_csf_debugfs_scheduler_suspend_fops, -+ &kbase_csf_debugfs_scheduler_suspend_get, -+ &kbase_csf_debugfs_scheduler_suspend_set, -+ "%llu\n"); -+ -+void kbase_csf_debugfs_init(struct kbase_device *kbdev) -+{ -+ debugfs_create_file("active_groups", 0444, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbasep_csf_active_queue_groups_debugfs_fops); -+ -+ debugfs_create_file("scheduling_timer_enabled", 0644, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbasep_csf_debugfs_scheduling_timer_enabled_fops); -+ debugfs_create_file("scheduling_timer_kick", 0200, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbasep_csf_debugfs_scheduling_timer_kick_fops); -+ debugfs_create_file("scheduler_suspend", 0644, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbasep_csf_debugfs_scheduler_suspend_fops); -+ -+ kbase_csf_tl_reader_debugfs_init(kbdev); -+ kbase_csf_firmware_trace_buffer_debugfs_init(kbdev); -+} -+ -+#else -+/* -+ * Stub functions for when debugfs is disabled -+ */ -+void kbase_csf_queue_group_debugfs_init(struct kbase_context *kctx) -+{ -+} -+ -+void kbase_csf_debugfs_init(struct kbase_device *kbdev) -+{ -+} -+ -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_csg_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_csg_debugfs.h -new file mode 100644 -index 0000000..397e657 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_csg_debugfs.h -@@ -0,0 +1,47 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_CSG_DEBUGFS_H_ -+#define _KBASE_CSF_CSG_DEBUGFS_H_ -+ -+/* Forward declarations */ -+struct kbase_device; -+struct kbase_context; -+struct kbase_queue_group; -+ -+#define MALI_CSF_CSG_DEBUGFS_VERSION 0 -+ -+/** -+ * kbase_csf_queue_group_debugfs_init() - Add debugfs entry for queue groups -+ * associated with @kctx. -+ * -+ * @kctx: Pointer to kbase_context -+ */ -+void kbase_csf_queue_group_debugfs_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_debugfs_init() - Add a global debugfs entry for queue groups -+ * -+ * @kbdev: Pointer to the device -+ */ -+void kbase_csf_debugfs_init(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_CSF_CSG_DEBUGFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_defs.h -new file mode 100644 -index 0000000..53526ce ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_defs.h -@@ -0,0 +1,1254 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* Definitions (types, defines, etcs) common to the CSF. -+ * They are placed here to allow the hierarchy of header files to work. -+ */ -+ -+#ifndef _KBASE_CSF_DEFS_H_ -+#define _KBASE_CSF_DEFS_H_ -+ -+#include <linux/types.h> -+#include <linux/wait.h> -+ -+#include "mali_kbase_csf_firmware.h" -+ -+/* Maximum number of KCPU command queues to be created per GPU address space. -+ */ -+#define KBASEP_MAX_KCPU_QUEUES ((size_t)256) -+ -+/* Maximum number of GPU command queue groups to be created per GPU address -+ * space. -+ */ -+#define MAX_QUEUE_GROUP_NUM (256) -+ -+/* Maximum number of GPU tiler heaps to allow to be created per GPU address -+ * space. -+ */ -+#define MAX_TILER_HEAPS (128) -+ -+#define CSF_FIRMWARE_ENTRY_READ (1ul << 0) -+#define CSF_FIRMWARE_ENTRY_WRITE (1ul << 1) -+#define CSF_FIRMWARE_ENTRY_EXECUTE (1ul << 2) -+#define CSF_FIRMWARE_ENTRY_CACHE_MODE (3ul << 3) -+#define CSF_FIRMWARE_ENTRY_PROTECTED (1ul << 5) -+#define CSF_FIRMWARE_ENTRY_SHARED (1ul << 30) -+#define CSF_FIRMWARE_ENTRY_ZERO (1ul << 31) -+ -+/** -+ * enum kbase_csf_bind_state - bind state of the queue -+ * -+ * @KBASE_CSF_QUEUE_UNBOUND: Set when the queue is registered or when the link -+ * between queue and the group to which it was bound or being bound is removed. -+ * @KBASE_CSF_QUEUE_BIND_IN_PROGRESS: Set when the first part of bind operation -+ * has completed i.e. CS_QUEUE_BIND ioctl. -+ * @KBASE_CSF_QUEUE_BOUND: Set when the bind operation has completed i.e. IO -+ * pages have been mapped in the process address space. -+ */ -+enum kbase_csf_queue_bind_state { -+ KBASE_CSF_QUEUE_UNBOUND, -+ KBASE_CSF_QUEUE_BIND_IN_PROGRESS, -+ KBASE_CSF_QUEUE_BOUND, -+}; -+ -+/** -+ * enum kbase_csf_reset_gpu_state - state of the gpu reset -+ * -+ * @KBASE_CSF_RESET_GPU_NOT_PENDING: Set when the GPU reset isn't pending -+ * -+ * @KBASE_CSF_RESET_GPU_PREPARED: Set when kbase_prepare_to_reset_gpu() has -+ * been called. This is just for debugging checks to encourage callers to call -+ * kbase_prepare_to_reset_gpu() before kbase_reset_gpu(). -+ * -+ * @KBASE_CSF_RESET_GPU_COMMITTED: Set when the GPU reset process has been -+ * committed and so will definitely happen, but the procedure to reset the GPU -+ * has not yet begun. Other threads must finish accessing the HW before we -+ * reach %KBASE_CSF_RESET_GPU_HAPPENING. -+ * -+ * @KBASE_CSF_RESET_GPU_HAPPENING: Set when the GPU reset process is occurring -+ * (silent or otherwise), and is actively accessing the HW. Any changes to the -+ * HW in other threads might get lost, overridden, or corrupted. -+ * -+ * @KBASE_CSF_RESET_GPU_COMMITTED_SILENT: Set when the GPU reset process has -+ * been committed but has not started happening. This is used when resetting -+ * the GPU as part of normal behavior (e.g. when exiting protected mode). -+ * Other threads must finish accessing the HW before we reach -+ * %KBASE_CSF_RESET_GPU_HAPPENING. -+ * -+ * @KBASE_CSF_RESET_GPU_FAILED: Set when an error is encountered during the -+ * GPU reset process. No more work could then be executed on GPU, unloading -+ * the Driver module is the only option. -+ */ -+enum kbase_csf_reset_gpu_state { -+ KBASE_CSF_RESET_GPU_NOT_PENDING, -+ KBASE_CSF_RESET_GPU_PREPARED, -+ KBASE_CSF_RESET_GPU_COMMITTED, -+ KBASE_CSF_RESET_GPU_HAPPENING, -+ KBASE_CSF_RESET_GPU_COMMITTED_SILENT, -+ KBASE_CSF_RESET_GPU_FAILED, -+}; -+ -+/** -+ * enum kbase_csf_group_state - state of the GPU command queue group -+ * -+ * @KBASE_CSF_GROUP_INACTIVE: Group is inactive and won't be -+ * considered by scheduler for running on -+ * CSG slot. -+ * @KBASE_CSF_GROUP_RUNNABLE: Group is in the list of runnable groups -+ * and is subjected to time-slice based -+ * scheduling. A start request would be -+ * sent (or already has been sent) if the -+ * group is assigned the CS -+ * group slot for the fist time. -+ * @KBASE_CSF_GROUP_IDLE: Group is currently on a CSG slot -+ * but all the CSs bound to the group have -+ * become either idle or waiting on sync -+ * object. -+ * Group could be evicted from the slot on -+ * the next tick if there are no spare -+ * slots left after scheduling non-idle -+ * queue groups. If the group is kept on -+ * slot then it would be moved to the -+ * RUNNABLE state, also if one of the -+ * queues bound to the group is kicked it -+ * would be moved to the RUNNABLE state. -+ * If the group is evicted from the slot it -+ * would be moved to either -+ * KBASE_CSF_GROUP_SUSPENDED_ON_IDLE or -+ * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC -+ * state. -+ * @KBASE_CSF_GROUP_SUSPENDED: Group was evicted from the CSG slot -+ * and is not running but is still in the -+ * list of runnable groups and subjected -+ * to time-slice based scheduling. A resume -+ * request would be sent when a CSG slot is -+ * re-assigned to the group and once the -+ * resume is complete group would be moved -+ * back to the RUNNABLE state. -+ * @KBASE_CSF_GROUP_SUSPENDED_ON_IDLE: Same as KBASE_CSF_GROUP_SUSPENDED except -+ * that queue group also became idle before -+ * the suspension. This state helps -+ * Scheduler avoid scheduling the idle -+ * groups over the non-idle groups in the -+ * subsequent ticks. If one of the queues -+ * bound to the group is kicked it would be -+ * moved to the SUSPENDED state. -+ * @KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC: Same as GROUP_SUSPENDED_ON_IDLE -+ * except that at least one CS -+ * bound to this group was -+ * waiting for synchronization object -+ * before the suspension. -+ * @KBASE_CSF_GROUP_FAULT_EVICTED: Group is evicted from the scheduler due -+ * to a fault condition, pending to be -+ * terminated. -+ * @KBASE_CSF_GROUP_TERMINATED: Group is no longer schedulable and is -+ * pending to be deleted by Client, all the -+ * queues bound to it have been unbound. -+ */ -+enum kbase_csf_group_state { -+ KBASE_CSF_GROUP_INACTIVE, -+ KBASE_CSF_GROUP_RUNNABLE, -+ KBASE_CSF_GROUP_IDLE, -+ KBASE_CSF_GROUP_SUSPENDED, -+ KBASE_CSF_GROUP_SUSPENDED_ON_IDLE, -+ KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC, -+ KBASE_CSF_GROUP_FAULT_EVICTED, -+ KBASE_CSF_GROUP_TERMINATED, -+}; -+ -+/** -+ * enum kbase_csf_csg_slot_state - state of the command queue group slots under -+ * the scheduler control. -+ * -+ * @CSG_SLOT_READY: The slot is clean and ready to be programmed with a -+ * queue group. -+ * @CSG_SLOT_READY2RUN: The slot has been programmed with a queue group, i.e. a -+ * start or resume request has been sent to the firmware. -+ * @CSG_SLOT_RUNNING: The queue group is running on the slot, acknowledgment -+ * of a start or resume request has been obtained from the -+ * firmware. -+ * @CSG_SLOT_DOWN2STOP: The suspend or terminate request for the queue group on -+ * the slot has been sent to the firmware. -+ * @CSG_SLOT_STOPPED: The queue group is removed from the slot, acknowledgment -+ * of suspend or terminate request has been obtained from -+ * the firmware. -+ * @CSG_SLOT_READY2RUN_TIMEDOUT: The start or resume request sent on the slot -+ * for the queue group timed out. -+ * @CSG_SLOT_DOWN2STOP_TIMEDOUT: The suspend or terminate request for queue -+ * group on the slot timed out. -+ */ -+enum kbase_csf_csg_slot_state { -+ CSG_SLOT_READY, -+ CSG_SLOT_READY2RUN, -+ CSG_SLOT_RUNNING, -+ CSG_SLOT_DOWN2STOP, -+ CSG_SLOT_STOPPED, -+ CSG_SLOT_READY2RUN_TIMEDOUT, -+ CSG_SLOT_DOWN2STOP_TIMEDOUT, -+}; -+ -+/** -+ * enum kbase_csf_scheduler_state - state of the scheduler operational phases. -+ * -+ * @SCHED_BUSY: The scheduler is busy performing on tick schedule -+ * operations, the state of CSG slots -+ * can't be changed. -+ * @SCHED_INACTIVE: The scheduler is inactive, it is allowed to modify the -+ * state of CSG slots by in-cycle -+ * priority scheduling. -+ * @SCHED_SUSPENDED: The scheduler is in low-power mode with scheduling -+ * operations suspended and is not holding the power -+ * management reference. This can happen if the GPU -+ * becomes idle for a duration exceeding a threshold, -+ * or due to a system triggered suspend action. -+ */ -+enum kbase_csf_scheduler_state { -+ SCHED_BUSY, -+ SCHED_INACTIVE, -+ SCHED_SUSPENDED, -+}; -+ -+/** -+ * enum kbase_queue_group_priority - Kbase internal relative priority list. -+ * -+ * @KBASE_QUEUE_GROUP_PRIORITY_REALTIME: The realtime queue group priority. -+ * @KBASE_QUEUE_GROUP_PRIORITY_HIGH: The high queue group priority. -+ * @KBASE_QUEUE_GROUP_PRIORITY_MEDIUM: The medium queue group priority. -+ * @KBASE_QUEUE_GROUP_PRIORITY_LOW: The low queue group priority. -+ * @KBASE_QUEUE_GROUP_PRIORITY_COUNT: The number of priority levels. -+ */ -+enum kbase_queue_group_priority { -+ KBASE_QUEUE_GROUP_PRIORITY_REALTIME = 0, -+ KBASE_QUEUE_GROUP_PRIORITY_HIGH, -+ KBASE_QUEUE_GROUP_PRIORITY_MEDIUM, -+ KBASE_QUEUE_GROUP_PRIORITY_LOW, -+ KBASE_QUEUE_GROUP_PRIORITY_COUNT -+}; -+ -+ -+/** -+ * struct kbase_csf_notification - Event or error generated as part of command -+ * queue execution -+ * -+ * @data: Event or error data returned to userspace -+ * @link: Link to the linked list, &struct_kbase_csf_context.error_list. -+ */ -+struct kbase_csf_notification { -+ struct base_csf_notification data; -+ struct list_head link; -+}; -+ -+/** -+ * struct kbase_queue - Object representing a GPU command queue. -+ * -+ * @kctx: Pointer to the base context with which this GPU command queue -+ * is associated. -+ * @reg: Pointer to the region allocated from the shared -+ * interface segment for mapping the User mode -+ * input/output pages in MCU firmware address space. -+ * @phys: Pointer to the physical pages allocated for the -+ * pair or User mode input/output page -+ * @user_io_addr: Pointer to the permanent kernel mapping of User mode -+ * input/output pages. The pages can be accessed through -+ * the mapping without any cache maintenance. -+ * @handle: Handle returned with bind ioctl for creating a -+ * contiguous User mode mapping of input/output pages & -+ * the hardware doorbell page. -+ * @doorbell_nr: Index of the hardware doorbell page assigned to the -+ * queue. -+ * @db_file_offset: File offset value that is assigned to userspace mapping -+ * created on bind to access the doorbell page. -+ * It is in page units. -+ * @link: Link to the linked list of GPU command queues created per -+ * GPU address space. -+ * @refcount: Reference count, stands for the number of times the queue -+ * has been referenced. The reference is taken when it is -+ * created, when it is bound to the group and also when the -+ * @oom_event_work work item is queued -+ * for it. -+ * @group: Pointer to the group to which this queue is bound. -+ * @queue_reg: Pointer to the VA region allocated for CS buffer. -+ * @oom_event_work: Work item corresponding to the out of memory event for -+ * chunked tiler heap being used for this queue. -+ * @base_addr: Base address of the CS buffer. -+ * @size: Size of the CS buffer. -+ * @priority: Priority of this queue within the group. -+ * @bind_state: Bind state of the queue as enum @kbase_csf_queue_bind_state -+ * @csi_index: The ID of the assigned CS hardware interface. -+ * @enabled: Indicating whether the CS is running, or not. -+ * @status_wait: Value of CS_STATUS_WAIT register of the CS will -+ * be kept when the CS gets blocked by sync wait. -+ * CS_STATUS_WAIT provides information on conditions queue is -+ * blocking on. This is set when the group, to which queue is -+ * bound, is suspended after getting blocked, i.e. in -+ * KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC state. -+ * @sync_ptr: Value of CS_STATUS_WAIT_SYNC_POINTER register of the CS -+ * will be kept when the CS gets blocked by -+ * sync wait. CS_STATUS_WAIT_SYNC_POINTER contains the address -+ * of synchronization object being waited on. -+ * Valid only when @status_wait is set. -+ * @sync_value: Value of CS_STATUS_WAIT_SYNC_VALUE register of the CS -+ * will be kept when the CS gets blocked by -+ * sync wait. CS_STATUS_WAIT_SYNC_VALUE contains the value -+ * tested against the synchronization object. -+ * Valid only when @status_wait is set. -+ * @sb_status: Value indicates which of the scoreboard entries in the queue -+ * are non-zero -+ * @blocked_reason: Value shows if the queue is blocked, and if so, -+ * the reason why it is blocked -+ * @trace_buffer_base: CS trace buffer base address. -+ * @trace_offset_ptr: Pointer to the CS trace buffer offset variable. -+ * @trace_buffer_size: CS trace buffer size for the queue. -+ * @trace_cfg: CS trace configuration parameters. -+ * @error: GPU command queue fatal information to pass to user space. -+ * @fatal_event_work: Work item to handle the CS fatal event reported for this -+ * queue. -+ * @cs_fatal_info: Records additional information about the CS fatal event. -+ * @cs_fatal: Records information about the CS fatal event. -+ */ -+struct kbase_queue { -+ struct kbase_context *kctx; -+ struct kbase_va_region *reg; -+ struct tagged_addr phys[2]; -+ char *user_io_addr; -+ u64 handle; -+ int doorbell_nr; -+ unsigned long db_file_offset; -+ struct list_head link; -+ atomic_t refcount; -+ struct kbase_queue_group *group; -+ struct kbase_va_region *queue_reg; -+ struct work_struct oom_event_work; -+ u64 base_addr; -+ u32 size; -+ u8 priority; -+ s8 csi_index; -+ enum kbase_csf_queue_bind_state bind_state; -+ bool enabled; -+ u32 status_wait; -+ u64 sync_ptr; -+ u32 sync_value; -+ u32 sb_status; -+ u32 blocked_reason; -+ u64 trace_buffer_base; -+ u64 trace_offset_ptr; -+ u32 trace_buffer_size; -+ u32 trace_cfg; -+ struct kbase_csf_notification error; -+ struct work_struct fatal_event_work; -+ u64 cs_fatal_info; -+ u32 cs_fatal; -+}; -+ -+/** -+ * struct kbase_normal_suspend_buffer - Object representing a normal -+ * suspend buffer for queue group. -+ * @reg: Memory region allocated for the normal-mode suspend buffer. -+ * @phy: Array of physical memory pages allocated for the normal- -+ * mode suspend buffer. -+ */ -+struct kbase_normal_suspend_buffer { -+ struct kbase_va_region *reg; -+ struct tagged_addr *phy; -+}; -+ -+/** -+ * struct kbase_protected_suspend_buffer - Object representing a protected -+ * suspend buffer for queue group. -+ * @reg: Memory region allocated for the protected-mode suspend buffer. -+ * @pma: Array of pointer to protected mode allocations containing -+ * information about memory pages allocated for protected mode -+ * suspend buffer. -+ */ -+struct kbase_protected_suspend_buffer { -+ struct kbase_va_region *reg; -+ struct protected_memory_allocation **pma; -+}; -+ -+/** -+ * struct kbase_queue_group - Object representing a GPU command queue group. -+ * -+ * @kctx: Pointer to the kbase context with which this queue group -+ * is associated. -+ * @normal_suspend_buf: Object representing the normal suspend buffer. -+ * Normal-mode suspend buffer that is used for -+ * group context switch. -+ * @protected_suspend_buf: Object representing the protected suspend -+ * buffer. Protected-mode suspend buffer that is -+ * used for group context switch. -+ * @handle: Handle which identifies this queue group. -+ * @csg_nr: Number/index of the CSG to which this queue group is -+ * mapped; KBASEP_CSG_NR_INVALID indicates that the queue -+ * group is not scheduled. -+ * @priority: Priority of the queue group, 0 being the highest, -+ * BASE_QUEUE_GROUP_PRIORITY_COUNT - 1 being the lowest. -+ * @tiler_max: Maximum number of tiler endpoints the group is allowed -+ * to use. -+ * @fragment_max: Maximum number of fragment endpoints the group is -+ * allowed to use. -+ * @compute_max: Maximum number of compute endpoints the group is -+ * allowed to use. -+ * @tiler_mask: Mask of tiler endpoints the group is allowed to use. -+ * @fragment_mask: Mask of fragment endpoints the group is allowed to use. -+ * @compute_mask: Mask of compute endpoints the group is allowed to use. -+ * @group_uid: 32-bit wide unsigned identifier for the group, unique -+ * across all kbase devices and contexts. -+ * @link: Link to this queue group in the 'runnable_groups' list of -+ * the corresponding kctx. -+ * @link_to_schedule: Link to this queue group in the list of prepared groups -+ * to be scheduled, if the group is runnable/suspended. -+ * If the group is idle or waiting for CQS, it would be a -+ * link to the list of idle/blocked groups list. -+ * @run_state: Current state of the queue group. -+ * @prepared_seq_num: Indicates the position of queue group in the list of -+ * prepared groups to be scheduled. -+ * @scan_seq_num: Scan out sequence number before adjusting for dynamic -+ * idle conditions. It is used for setting a group's -+ * onslot priority. It could differ from prepared_seq_number -+ * when there are idle groups. -+ * @faulted: Indicates that a GPU fault occurred for the queue group. -+ * This flag persists until the fault has been queued to be -+ * reported to userspace. -+ * @bound_queues: Array of registered queues bound to this queue group. -+ * @doorbell_nr: Index of the hardware doorbell page assigned to the -+ * group. -+ * @protm_event_work: Work item corresponding to the protected mode entry -+ * event for this queue. -+ * @protm_pending_bitmap: Bit array to keep a track of CSs that -+ * have pending protected mode entry requests. -+ * @error_fatal: An error of type BASE_GPU_QUEUE_GROUP_ERROR_FATAL to be -+ * returned to userspace if such an error has occurred. -+ * @error_timeout: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT -+ * to be returned to userspace if such an error has occurred. -+ * @error_tiler_oom: An error of type BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM -+ * to be returned to userspace if such an error has occurred. -+ * @timer_event_work: Work item to handle the progress timeout fatal event -+ * for the group. -+ */ -+struct kbase_queue_group { -+ struct kbase_context *kctx; -+ struct kbase_normal_suspend_buffer normal_suspend_buf; -+ struct kbase_protected_suspend_buffer protected_suspend_buf; -+ u8 handle; -+ s8 csg_nr; -+ u8 priority; -+ -+ u8 tiler_max; -+ u8 fragment_max; -+ u8 compute_max; -+ -+ u64 tiler_mask; -+ u64 fragment_mask; -+ u64 compute_mask; -+ -+ u32 group_uid; -+ -+ struct list_head link; -+ struct list_head link_to_schedule; -+ enum kbase_csf_group_state run_state; -+ u32 prepared_seq_num; -+ u32 scan_seq_num; -+ bool faulted; -+ -+ struct kbase_queue *bound_queues[MAX_SUPPORTED_STREAMS_PER_GROUP]; -+ -+ int doorbell_nr; -+ struct work_struct protm_event_work; -+ DECLARE_BITMAP(protm_pending_bitmap, MAX_SUPPORTED_STREAMS_PER_GROUP); -+ -+ struct kbase_csf_notification error_fatal; -+ struct kbase_csf_notification error_timeout; -+ struct kbase_csf_notification error_tiler_oom; -+ -+ struct work_struct timer_event_work; -+}; -+ -+/** -+ * struct kbase_csf_kcpu_queue_context - Object representing the kernel CPU -+ * queues for a GPU address space. -+ * -+ * @lock: Lock preventing concurrent access to @array and the @in_use bitmap. -+ * @array: Array of pointers to kernel CPU command queues. -+ * @in_use: Bitmap which indicates which kernel CPU command queues are in use. -+ * @wq: Dedicated workqueue for processing kernel CPU command queues. -+ * @num_cmds: The number of commands that have been enqueued across -+ * all the KCPU command queues. This could be used as a -+ * timestamp to determine the command's enqueueing time. -+ * @jit_cmds_head: A list of the just-in-time memory commands, both -+ * allocate & free, in submission order, protected -+ * by kbase_csf_kcpu_queue_context.lock. -+ * @jit_blocked_queues: A list of KCPU command queues blocked by a pending -+ * just-in-time memory allocation command which will be -+ * reattempted after the impending free of other active -+ * allocations. -+ */ -+struct kbase_csf_kcpu_queue_context { -+ struct mutex lock; -+ struct kbase_kcpu_command_queue *array[KBASEP_MAX_KCPU_QUEUES]; -+ DECLARE_BITMAP(in_use, KBASEP_MAX_KCPU_QUEUES); -+ struct workqueue_struct *wq; -+ u64 num_cmds; -+ -+ struct list_head jit_cmds_head; -+ struct list_head jit_blocked_queues; -+}; -+ -+/** -+ * struct kbase_csf_cpu_queue_context - Object representing the cpu queue -+ * information. -+ * -+ * @buffer: Buffer containing CPU queue information provided by Userspace. -+ * @buffer_size: The size of @buffer. -+ * @dump_req_status: Indicates the current status for CPU queues dump request. -+ * @dump_cmp: Dumping cpu queue completion event. -+ */ -+struct kbase_csf_cpu_queue_context { -+ char *buffer; -+ size_t buffer_size; -+ atomic_t dump_req_status; -+ struct completion dump_cmp; -+}; -+ -+/** -+ * struct kbase_csf_heap_context_allocator - Allocator of heap contexts -+ * -+ * Heap context structures are allocated by the kernel for use by the firmware. -+ * The current implementation subdivides a single GPU memory region for use as -+ * a sparse array. -+ * -+ * @kctx: Pointer to the kbase context with which this allocator is -+ * associated. -+ * @region: Pointer to a GPU memory region from which heap context structures -+ * are allocated. NULL if no heap contexts have been allocated. -+ * @gpu_va: GPU virtual address of the start of the region from which heap -+ * context structures are allocated. 0 if no heap contexts have been -+ * allocated. -+ * @lock: Lock preventing concurrent access to the @in_use bitmap. -+ * @in_use: Bitmap that indicates which heap context structures are currently -+ * allocated (in @region). -+ */ -+struct kbase_csf_heap_context_allocator { -+ struct kbase_context *kctx; -+ struct kbase_va_region *region; -+ u64 gpu_va; -+ struct mutex lock; -+ DECLARE_BITMAP(in_use, MAX_TILER_HEAPS); -+}; -+ -+/** -+ * struct kbase_csf_tiler_heap_context - Object representing the tiler heaps -+ * context for a GPU address space. -+ * -+ * This contains all of the CSF state relating to chunked tiler heaps for one -+ * @kbase_context. It is not the same as a heap context structure allocated by -+ * the kernel for use by the firmware. -+ * -+ * @lock: Lock preventing concurrent access to the tiler heaps. -+ * @list: List of tiler heaps. -+ * @ctx_alloc: Allocator for heap context structures. -+ * @nr_of_heaps: Total number of tiler heaps that were added during the -+ * life time of the context. -+ */ -+struct kbase_csf_tiler_heap_context { -+ struct mutex lock; -+ struct list_head list; -+ struct kbase_csf_heap_context_allocator ctx_alloc; -+ u64 nr_of_heaps; -+}; -+ -+/** -+ * struct kbase_csf_scheduler_context - Object representing the scheduler's -+ * context for a GPU address space. -+ * -+ * @runnable_groups: Lists of runnable GPU command queue groups in the kctx, -+ * one per queue group relative-priority level. -+ * @num_runnable_grps: Total number of runnable groups across all priority -+ * levels in @runnable_groups. -+ * @idle_wait_groups: A list of GPU command queue groups in which all enabled -+ * GPU command queues are idle and at least one of them -+ * is blocked on a sync wait operation. -+ * @num_idle_wait_grps: Length of the @idle_wait_groups list. -+ * @sync_update_wq: Dedicated workqueue to process work items corresponding -+ * to the sync_update events by sync_set/sync_add -+ * instruction execution on CSs bound to groups -+ * of @idle_wait_groups list. -+ * @sync_update_work: work item to process the sync_update events by -+ * sync_set / sync_add instruction execution on command -+ * streams bound to groups of @idle_wait_groups list. -+ * @ngrp_to_schedule: Number of groups added for the context to the -+ * 'groups_to_schedule' list of scheduler instance. -+ */ -+struct kbase_csf_scheduler_context { -+ struct list_head runnable_groups[KBASE_QUEUE_GROUP_PRIORITY_COUNT]; -+ u32 num_runnable_grps; -+ struct list_head idle_wait_groups; -+ u32 num_idle_wait_grps; -+ struct workqueue_struct *sync_update_wq; -+ struct work_struct sync_update_work; -+ u32 ngrp_to_schedule; -+}; -+ -+/** -+ * struct kbase_csf_context - Object representing CSF for a GPU address space. -+ * -+ * @event_pages_head: A list of pages allocated for the event memory used by -+ * the synchronization objects. A separate list would help -+ * in the fast lookup, since the list is expected to be short -+ * as one page would provide the memory for up to 1K -+ * synchronization objects. -+ * KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES is the upper -+ * bound on the size of event memory. -+ * @cookies: Bitmask containing of KBASE_CSF_NUM_USER_IO_PAGES_HANDLE -+ * bits, used for creating the User mode CPU mapping in a -+ * deferred manner of a pair of User mode input/output pages -+ * & a hardware doorbell page. -+ * The pages are allocated when a GPU command queue is -+ * bound to a CSG in kbase_csf_queue_bind. -+ * This helps returning unique handles to Userspace from -+ * kbase_csf_queue_bind and later retrieving the pointer to -+ * queue in the mmap handler. -+ * @user_pages_info: Array containing pointers to queue -+ * structures, used in conjunction with cookies bitmask for -+ * providing a mechansim to create a CPU mapping of -+ * input/output pages & hardware doorbell page. -+ * @lock: Serializes accesses to all members, except for ones that -+ * have their own locks. -+ * @queue_groups: Array of registered GPU command queue groups. -+ * @queue_list: Linked list of GPU command queues not yet deregistered. -+ * Note that queues can persist after deregistration if the -+ * userspace mapping created for them on bind operation -+ * hasn't been removed. -+ * @kcpu_queues: Kernel CPU command queues. -+ * @event_lock: Lock protecting access to @event_callback_list and -+ * @error_list. -+ * @event_callback_list: List of callbacks which are registered to serve CSF -+ * events. -+ * @tiler_heaps: Chunked tiler memory heaps. -+ * @wq: Dedicated workqueue to process work items corresponding -+ * to the OoM events raised for chunked tiler heaps being -+ * used by GPU command queues, and progress timeout events. -+ * @link: Link to this csf context in the 'runnable_kctxs' list of -+ * the scheduler instance -+ * @user_reg_vma: Pointer to the vma corresponding to the virtual mapping -+ * of the USER register page. Currently used only for sanity -+ * checking. -+ * @sched: Object representing the scheduler's context -+ * @error_list: List for CS fatal errors in this context. -+ * Link of fatal error is -+ * &struct_kbase_csf_notification.link. -+ * @event_lock needs to be held to access this list. -+ * @cpu_queue: CPU queue information. Only be available when DEBUG_FS -+ * is enabled. -+ */ -+struct kbase_csf_context { -+ struct list_head event_pages_head; -+ DECLARE_BITMAP(cookies, KBASE_CSF_NUM_USER_IO_PAGES_HANDLE); -+ struct kbase_queue *user_pages_info[ -+ KBASE_CSF_NUM_USER_IO_PAGES_HANDLE]; -+ struct mutex lock; -+ struct kbase_queue_group *queue_groups[MAX_QUEUE_GROUP_NUM]; -+ struct list_head queue_list; -+ struct kbase_csf_kcpu_queue_context kcpu_queues; -+ spinlock_t event_lock; -+ struct list_head event_callback_list; -+ struct kbase_csf_tiler_heap_context tiler_heaps; -+ struct workqueue_struct *wq; -+ struct list_head link; -+ struct vm_area_struct *user_reg_vma; -+ struct kbase_csf_scheduler_context sched; -+ struct list_head error_list; -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ struct kbase_csf_cpu_queue_context cpu_queue; -+#endif -+}; -+ -+/** -+ * struct kbase_csf_reset_gpu - Object containing the members required for -+ * GPU reset handling. -+ * @workq: Workqueue to execute the GPU reset work item @work. -+ * @work: Work item for performing the GPU reset. -+ * @wait: Wait queue used to wait for the GPU reset completion. -+ * @sem: RW Semaphore to ensure no other thread attempts to use the -+ * GPU whilst a reset is in process. Unlike traditional -+ * semaphores and wait queues, this allows Linux's lockdep -+ * mechanism to check for deadlocks involving reset waits. -+ * @state: Tracks if the GPU reset is in progress or not. -+ * The state is represented by enum @kbase_csf_reset_gpu_state. -+ */ -+struct kbase_csf_reset_gpu { -+ struct workqueue_struct *workq; -+ struct work_struct work; -+ wait_queue_head_t wait; -+ struct rw_semaphore sem; -+ atomic_t state; -+}; -+ -+/** -+ * struct kbase_csf_csg_slot - Object containing members for tracking the state -+ * of CSG slots. -+ * @resident_group: pointer to the queue group that is resident on the CSG slot. -+ * @state: state of the slot as per enum @kbase_csf_csg_slot_state. -+ * @trigger_jiffies: value of jiffies when change in slot state is recorded. -+ * @priority: dynamic priority assigned to CSG slot. -+ */ -+struct kbase_csf_csg_slot { -+ struct kbase_queue_group *resident_group; -+ atomic_t state; -+ unsigned long trigger_jiffies; -+ u8 priority; -+}; -+ -+/** -+ * struct kbase_csf_scheduler - Object representing the scheduler used for -+ * CSF for an instance of GPU platform device. -+ * @lock: Lock to serialize the scheduler operations and -+ * access to the data members. -+ * @interrupt_lock: Lock to protect members accessed by interrupt -+ * handler. -+ * @state: The operational phase the scheduler is in. Primarily -+ * used for indicating what in-cycle schedule actions -+ * are allowed. -+ * @doorbell_inuse_bitmap: Bitmap of hardware doorbell pages keeping track of -+ * which pages are currently available for assignment -+ * to clients. -+ * @csg_inuse_bitmap: Bitmap to keep a track of CSG slots -+ * that are currently in use. -+ * @csg_slots: The array for tracking the state of CS -+ * group slots. -+ * @runnable_kctxs: List of Kbase contexts that have runnable command -+ * queue groups. -+ * @groups_to_schedule: List of runnable queue groups prepared on every -+ * scheduler tick. The dynamic priority of the CSG -+ * slot assigned to a group will depend upon the -+ * position of group in the list. -+ * @ngrp_to_schedule: Number of groups in the @groups_to_schedule list, -+ * incremented when a group is added to the list, used -+ * to record the position of group in the list. -+ * @num_active_address_spaces: Number of GPU address space slots that would get -+ * used to program the groups in @groups_to_schedule -+ * list on all the available CSG -+ * slots. -+ * @num_csg_slots_for_tick: Number of CSG slots that can be -+ * active in the given tick/tock. This depends on the -+ * value of @num_active_address_spaces. -+ * @remaining_tick_slots: Tracking the number of remaining available slots -+ * for @num_csg_slots_for_tick during the scheduling -+ * operation in a tick/tock. -+ * @idle_groups_to_schedule: List of runnable queue groups, in which all GPU -+ * command queues became idle or are waiting for -+ * synchronization object, prepared on every -+ * scheduler tick. The groups in this list are -+ * appended to the tail of @groups_to_schedule list -+ * after the scan out so that the idle groups aren't -+ * preferred for scheduling over the non-idle ones. -+ * @csg_scan_count_for_tick: CSG scanout count for assign the scan_seq_num for -+ * each scanned out group during scheduling operation -+ * in a tick/tock. -+ * @total_runnable_grps: Total number of runnable groups across all KCTXs. -+ * @csgs_events_enable_mask: Use for temporary masking off asynchronous events -+ * from firmware (such as OoM events) before a group -+ * is suspended. -+ * @csg_slots_idle_mask: Bit array for storing the mask of CS -+ * group slots for which idle notification was -+ * received. -+ * @csg_slots_prio_update: Bit array for tracking slots that have an on-slot -+ * priority update operation. -+ * @last_schedule: Time in jiffies recorded when the last "tick" or -+ * "tock" schedule operation concluded. Used for -+ * evaluating the exclusion window for in-cycle -+ * schedule operation. -+ * @timer_enabled: Whether the CSF scheduler wakes itself up for -+ * periodic scheduling tasks. If this value is 0 -+ * then it will only perform scheduling under the -+ * influence of external factors e.g., IRQs, IOCTLs. -+ * @wq: Dedicated workqueue to execute the @tick_work. -+ * @tick_timer: High-resolution timer employed to schedule tick -+ * workqueue items (kernel-provided delayed_work -+ * items do not use hrtimer and for some reason do -+ * not provide sufficiently reliable periodicity). -+ * @tick_work: Work item that performs the "schedule on tick" -+ * operation to implement timeslice-based scheduling. -+ * @tock_work: Work item that would perform the schedule on tock -+ * operation to implement the asynchronous scheduling. -+ * @ping_work: Work item that would ping the firmware at regular -+ * intervals, only if there is a single active CSG -+ * slot, to check if firmware is alive and would -+ * initiate a reset if the ping request isn't -+ * acknowledged. -+ * @top_ctx: Pointer to the Kbase context corresponding to the -+ * @top_grp. -+ * @top_grp: Pointer to queue group inside @groups_to_schedule -+ * list that was assigned the highest slot priority. -+ * @tock_pending_request: A "tock" request is pending: a group that is not -+ * currently on the GPU demands to be scheduled. -+ * @active_protm_grp: Indicates if firmware has been permitted to let GPU -+ * enter protected mode with the given group. On exit -+ * from protected mode the pointer is reset to NULL. -+ * @gpu_idle_fw_timer_enabled: Whether the CSF scheduler has activiated the -+ * firmware idle hysteresis timer for preparing a -+ * GPU suspend on idle. -+ * @gpu_idle_work: Work item for facilitating the scheduler to bring -+ * the GPU to a low-power mode on becoming idle. -+ * @non_idle_offslot_grps: Count of off-slot non-idle groups. Reset during -+ * the scheduler active phase in a tick. It then -+ * tracks the count of non-idle groups across all the -+ * other phases. -+ * @non_idle_scanout_grps: Count on the non-idle groups in the scan-out -+ * list at the scheduling prepare stage. -+ * @pm_active_count: Count indicating if the scheduler is owning a power -+ * management reference count. Reference is taken when -+ * the count becomes 1 and is dropped when the count -+ * becomes 0. It is used to enable the power up of MCU -+ * after GPU and L2 cache have been powered up. So when -+ * this count is zero, MCU will not be powered up. -+ * @csg_scheduling_period_ms: Duration of Scheduling tick in milliseconds. -+ * @tick_timer_active: Indicates whether the @tick_timer is effectively -+ * active or not, as the callback function of -+ * @tick_timer will enqueue @tick_work only if this -+ * flag is true. This is mainly useful for the case -+ * when scheduling tick needs to be advanced from -+ * interrupt context, without actually deactivating -+ * the @tick_timer first and then enqueing @tick_work. -+ */ -+struct kbase_csf_scheduler { -+ struct mutex lock; -+ spinlock_t interrupt_lock; -+ enum kbase_csf_scheduler_state state; -+ DECLARE_BITMAP(doorbell_inuse_bitmap, CSF_NUM_DOORBELL); -+ DECLARE_BITMAP(csg_inuse_bitmap, MAX_SUPPORTED_CSGS); -+ struct kbase_csf_csg_slot *csg_slots; -+ struct list_head runnable_kctxs; -+ struct list_head groups_to_schedule; -+ u32 ngrp_to_schedule; -+ u32 num_active_address_spaces; -+ u32 num_csg_slots_for_tick; -+ u32 remaining_tick_slots; -+ struct list_head idle_groups_to_schedule; -+ u32 csg_scan_count_for_tick; -+ u32 total_runnable_grps; -+ DECLARE_BITMAP(csgs_events_enable_mask, MAX_SUPPORTED_CSGS); -+ DECLARE_BITMAP(csg_slots_idle_mask, MAX_SUPPORTED_CSGS); -+ DECLARE_BITMAP(csg_slots_prio_update, MAX_SUPPORTED_CSGS); -+ unsigned long last_schedule; -+ bool timer_enabled; -+ struct workqueue_struct *wq; -+ struct hrtimer tick_timer; -+ struct work_struct tick_work; -+ struct delayed_work tock_work; -+ struct delayed_work ping_work; -+ struct kbase_context *top_ctx; -+ struct kbase_queue_group *top_grp; -+ bool tock_pending_request; -+ struct kbase_queue_group *active_protm_grp; -+ bool gpu_idle_fw_timer_enabled; -+ struct work_struct gpu_idle_work; -+ atomic_t non_idle_offslot_grps; -+ u32 non_idle_scanout_grps; -+ u32 pm_active_count; -+ unsigned int csg_scheduling_period_ms; -+ bool tick_timer_active; -+}; -+ -+/** -+ * Number of GPU cycles per unit of the global progress timeout. -+ */ -+#define GLB_PROGRESS_TIMER_TIMEOUT_SCALE ((u64)1024) -+ -+/** -+ * Maximum value of the global progress timeout. -+ */ -+#define GLB_PROGRESS_TIMER_TIMEOUT_MAX \ -+ ((GLB_PROGRESS_TIMER_TIMEOUT_MASK >> \ -+ GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) * \ -+ GLB_PROGRESS_TIMER_TIMEOUT_SCALE) -+ -+/** -+ * Default GLB_PWROFF_TIMER_TIMEOUT value in unit of micro-seconds. -+ */ -+#define DEFAULT_GLB_PWROFF_TIMEOUT_US (800) -+ -+/** -+ * In typical operations, the management of the shader core power transitions -+ * is delegated to the MCU/firmware. However, if the host driver is configured -+ * to take direct control, one needs to disable the MCU firmware GLB_PWROFF -+ * timer. -+ */ -+#define DISABLE_GLB_PWROFF_TIMER (0) -+ -+/* Index of the GPU_ACTIVE counter within the CSHW counter block */ -+#define GPU_ACTIVE_CNT_IDX (4) -+ -+/** -+ * Maximum number of sessions that can be managed by the IPA Control component. -+ */ -+#if MALI_UNIT_TEST -+#define KBASE_IPA_CONTROL_MAX_SESSIONS ((size_t)8) -+#else -+#define KBASE_IPA_CONTROL_MAX_SESSIONS ((size_t)2) -+#endif -+ -+/** -+ * enum kbase_ipa_core_type - Type of counter block for performance counters -+ * -+ * @KBASE_IPA_CORE_TYPE_CSHW: CS Hardware counters. -+ * @KBASE_IPA_CORE_TYPE_MEMSYS: Memory System counters. -+ * @KBASE_IPA_CORE_TYPE_TILER: Tiler counters. -+ * @KBASE_IPA_CORE_TYPE_SHADER: Shader Core counters. -+ * @KBASE_IPA_CORE_TYPE_NUM: Number of core types. -+ */ -+enum kbase_ipa_core_type { -+ KBASE_IPA_CORE_TYPE_CSHW = 0, -+ KBASE_IPA_CORE_TYPE_MEMSYS, -+ KBASE_IPA_CORE_TYPE_TILER, -+ KBASE_IPA_CORE_TYPE_SHADER, -+ KBASE_IPA_CORE_TYPE_NUM -+}; -+ -+/** -+ * Number of configurable counters per type of block on the IPA Control -+ * interface. -+ */ -+#define KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS ((size_t)8) -+ -+/** -+ * Total number of configurable counters existing on the IPA Control interface. -+ */ -+#define KBASE_IPA_CONTROL_MAX_COUNTERS \ -+ ((size_t)KBASE_IPA_CORE_TYPE_NUM * KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS) -+ -+/** -+ * struct kbase_ipa_control_prfcnt - Session for a single performance counter -+ * -+ * @latest_raw_value: Latest raw value read from the counter. -+ * @scaling_factor: Factor raw value shall be multiplied by. -+ * @accumulated_diff: Partial sum of scaled and normalized values from -+ * previous samples. This represent all the values -+ * that were read before the latest raw value. -+ * @type: Type of counter block for performance counter. -+ * @select_idx: Index of the performance counter as configured on -+ * the IPA Control interface. -+ * @gpu_norm: Indicating whether values shall be normalized by -+ * GPU frequency. If true, returned values represent -+ * an interval of time expressed in seconds (when the -+ * scaling factor is set to 1). -+ */ -+struct kbase_ipa_control_prfcnt { -+ u64 latest_raw_value; -+ u64 scaling_factor; -+ u64 accumulated_diff; -+ enum kbase_ipa_core_type type; -+ u8 select_idx; -+ bool gpu_norm; -+}; -+ -+/** -+ * struct kbase_ipa_control_session - Session for an IPA Control client -+ * -+ * @prfcnts: Sessions for individual performance counters. -+ * @num_prfcnts: Number of performance counters. -+ * @active: Indicates whether this slot is in use or not -+ * @last_query_time: Time of last query, in ns -+ * @protm_time: Amount of time (in ns) that GPU has been in protected -+ */ -+struct kbase_ipa_control_session { -+ struct kbase_ipa_control_prfcnt prfcnts[KBASE_IPA_CONTROL_MAX_COUNTERS]; -+ size_t num_prfcnts; -+ bool active; -+ u64 last_query_time; -+ u64 protm_time; -+}; -+ -+/** -+ * struct kbase_ipa_control_prfcnt_config - Performance counter configuration -+ * -+ * @idx: Index of the performance counter inside the block, as specified -+ * in the GPU architecture. -+ * @refcount: Number of client sessions bound to this counter. -+ * -+ * This structure represents one configurable performance counter of -+ * the IPA Control interface. The entry may be mapped to a specific counter -+ * by one or more client sessions. The counter is considered to be unused -+ * if it isn't part of any client session. -+ */ -+struct kbase_ipa_control_prfcnt_config { -+ u8 idx; -+ u8 refcount; -+}; -+ -+/** -+ * struct kbase_ipa_control_prfcnt_block - Block of performance counters -+ * -+ * @select: Current performance counter configuration. -+ * @num_available_counters: Number of counters that are not already configured. -+ * -+ */ -+struct kbase_ipa_control_prfcnt_block { -+ struct kbase_ipa_control_prfcnt_config -+ select[KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS]; -+ size_t num_available_counters; -+}; -+ -+/** -+ * struct kbase_ipa_control - Manager of the IPA Control interface. -+ * -+ * @blocks: Current configuration of performance counters -+ * for the IPA Control interface. -+ * @sessions: State of client sessions, storing information -+ * like performance counters the client subscribed to -+ * and latest value read from each counter. -+ * @lock: Spinlock to serialize access by concurrent clients. -+ * @rtm_listener_data: Private data for allocating a GPU frequency change -+ * listener. -+ * @num_active_sessions: Number of sessions opened by clients. -+ * @cur_gpu_rate: Current GPU top-level operating frequency, in Hz. -+ * @rtm_listener_data: Private data for allocating a GPU frequency change -+ * listener. -+ * @protm_start: Time (in ns) at which the GPU entered protected mode -+ */ -+struct kbase_ipa_control { -+ struct kbase_ipa_control_prfcnt_block blocks[KBASE_IPA_CORE_TYPE_NUM]; -+ struct kbase_ipa_control_session -+ sessions[KBASE_IPA_CONTROL_MAX_SESSIONS]; -+ spinlock_t lock; -+ void *rtm_listener_data; -+ size_t num_active_sessions; -+ u32 cur_gpu_rate; -+ u64 protm_start; -+}; -+ -+/** -+ * struct kbase_csf_firmware_interface - Interface in the MCU firmware -+ * -+ * @node: Interface objects are on the kbase_device:csf.firmware_interfaces -+ * list using this list_head to link them -+ * @phys: Array of the physical (tagged) addresses making up this interface -+ * @name: NULL-terminated string naming the interface -+ * @num_pages: Number of entries in @phys and @pma (and length of the interface) -+ * @virtual: Starting GPU virtual address this interface is mapped at -+ * @flags: bitmask of CSF_FIRMWARE_ENTRY_* conveying the interface attributes -+ * @data_start: Offset into firmware image at which the interface data starts -+ * @data_end: Offset into firmware image at which the interface data ends -+ * @kernel_map: A kernel mapping of the memory or NULL if not required to be -+ * mapped in the kernel -+ * @pma: Array of pointers to protected memory allocations. -+ */ -+struct kbase_csf_firmware_interface { -+ struct list_head node; -+ struct tagged_addr *phys; -+ char *name; -+ u32 num_pages; -+ u32 virtual; -+ u32 flags; -+ u32 data_start; -+ u32 data_end; -+ void *kernel_map; -+ struct protected_memory_allocation **pma; -+}; -+ -+/* -+ * struct kbase_csf_hwcnt - Object containing members for handling the dump of -+ * HW counters. -+ * -+ * @request_pending: Flag set when HWC requested and used for HWC sample -+ * done interrupt. -+ * @enable_pending: Flag set when HWC enable status change and used for -+ * enable done interrupt. -+ */ -+struct kbase_csf_hwcnt { -+ bool request_pending; -+ bool enable_pending; -+}; -+ -+/** -+ * struct kbase_csf_device - Object representing CSF for an instance of GPU -+ * platform device. -+ * -+ * @mcu_mmu: MMU page tables for the MCU firmware -+ * @firmware_interfaces: List of interfaces defined in the firmware image -+ * @firmware_config: List of configuration options within the firmware -+ * image -+ * @firmware_timeline_metadata: List of timeline meta-data within the firmware -+ * image -+ * @fw_cfg_kobj: Pointer to the kobject corresponding to the sysf -+ * directory that contains a sub-directory for each -+ * of the configuration option present in the -+ * firmware image. -+ * @firmware_trace_buffers: List of trace buffers described in the firmware -+ * image. -+ * @shared_interface: Pointer to the interface object containing info for -+ * the memory area shared between firmware & host. -+ * @shared_reg_rbtree: RB tree of the memory regions allocated from the -+ * shared interface segment in MCU firmware address -+ * space. -+ * @db_filp: Pointer to a dummy file, that alongwith -+ * @db_file_offsets, facilitates the use of unqiue -+ * file offset for the userspace mapping created -+ * for Hw Doorbell pages. The userspace mapping -+ * is made to point to this file inside the mmap -+ * handler. -+ * @db_file_offsets: Counter that is incremented every time a GPU -+ * command queue is bound to provide a unique file -+ * offset range for @db_filp file, so that pte of -+ * Doorbell page can be zapped through the kernel -+ * function unmap_mapping_range(). It is incremented -+ * in page units. -+ * @dummy_db_page: Address of the dummy page that is mapped in place -+ * of the real Hw doorbell page for the active GPU -+ * command queues after they are stopped or after the -+ * GPU is powered down. -+ * @dummy_user_reg_page: Address of the dummy page that is mapped in place -+ * of the real User register page just before the GPU -+ * is powered down. The User register page is mapped -+ * in the address space of every process, that created -+ * a Base context, to enable the access to LATEST_FLUSH -+ * register from userspace. -+ * @mali_file_inode: Pointer to the inode corresponding to mali device -+ * file. This is needed in order to switch to the -+ * @dummy_user_reg_page on GPU power down. -+ * All instances of the mali device file will point to -+ * the same inode. -+ * @reg_lock: Lock to serialize the MCU firmware related actions -+ * that affect all contexts such as allocation of -+ * regions from shared interface area, assignment of -+ * of hardware doorbell pages, assignment of CSGs, -+ * sending global requests. -+ * @event_wait: Wait queue to wait for receiving csf events, i.e. -+ * the interrupt from CSF firmware, or scheduler state -+ * changes. -+ * @interrupt_received: Flag set when the interrupt is received from CSF fw -+ * @global_iface: The result of parsing the global interface -+ * structure set up by the firmware, including the -+ * CSGs, CSs, and their properties -+ * @scheduler: The CS scheduler instance. -+ * @reset: Contain members required for GPU reset handling. -+ * @progress_timeout: Maximum number of GPU clock cycles without forward -+ * progress to allow, for all tasks running on -+ * hardware endpoints (e.g. shader cores), before -+ * terminating a GPU command queue group. -+ * Must not exceed @GLB_PROGRESS_TIMER_TIMEOUT_MAX. -+ * @pma_dev: Pointer to protected memory allocator device. -+ * @firmware_inited: Flag for indicating that the cold-boot stage of -+ * the MCU has completed. -+ * @firmware_reloaded: Flag for indicating a firmware reload operation -+ * in GPU reset has completed. -+ * @firmware_reload_needed: Flag for indicating that the firmware needs to be -+ * reloaded as part of the GPU reset action. -+ * @firmware_hctl_core_pwr: Flag for indicating that the host diver is in -+ * charge of the shader core's power transitions, and -+ * the mcu_core_pwroff timeout feature is disabled -+ * (i.e. configured 0 in the register field). If -+ * false, the control is delegated to the MCU. -+ * @firmware_reload_work: Work item for facilitating the procedural actions -+ * on reloading the firmware. -+ * @glb_init_request_pending: Flag to indicate that Global requests have been -+ * sent to the FW after MCU was re-enabled and their -+ * acknowledgement is pending. -+ * @fw_error_work: Work item for handling the firmware internal error -+ * fatal event. -+ * @ipa_control: IPA Control component manager. -+ * @mcu_core_pwroff_dur_us: Sysfs attribute for the glb_pwroff timeout input -+ * in unit of micro-seconds. The firmware does not use -+ * it directly. -+ * @mcu_core_pwroff_dur_count: The counterpart of the glb_pwroff timeout input -+ * in interface required format, ready to be used -+ * directly in the firmware. -+ * @mcu_core_pwroff_reg_shadow: The actual value that has been programed into -+ * the glb_pwoff register. This is separated from -+ * the @p mcu_core_pwroff_dur_count as an update -+ * to the latter is asynchronous. -+ * @gpu_idle_hysteresis_ms: Sysfs attribute for the idle hysteresis time -+ * window in unit of ms. The firmware does not use it -+ * directly. -+ * @gpu_idle_dur_count: The counterpart of the hysteresis time window in -+ * interface required format, ready to be used -+ * directly in the firmware. -+ * @fw_timeout_ms: Timeout value (in milliseconds) used when waiting -+ * for any request sent to the firmware. -+ * @hwcnt: Contain members required for handling the dump of -+ * HW counters. -+ */ -+struct kbase_csf_device { -+ struct kbase_mmu_table mcu_mmu; -+ struct list_head firmware_interfaces; -+ struct list_head firmware_config; -+ struct list_head firmware_timeline_metadata; -+ struct kobject *fw_cfg_kobj; -+ struct kbase_csf_trace_buffers firmware_trace_buffers; -+ void *shared_interface; -+ struct rb_root shared_reg_rbtree; -+ struct file *db_filp; -+ u32 db_file_offsets; -+ struct tagged_addr dummy_db_page; -+ struct tagged_addr dummy_user_reg_page; -+ struct inode *mali_file_inode; -+ struct mutex reg_lock; -+ wait_queue_head_t event_wait; -+ bool interrupt_received; -+ struct kbase_csf_global_iface global_iface; -+ struct kbase_csf_scheduler scheduler; -+ struct kbase_csf_reset_gpu reset; -+ atomic64_t progress_timeout; -+ struct protected_memory_allocator_device *pma_dev; -+ bool firmware_inited; -+ bool firmware_reloaded; -+ bool firmware_reload_needed; -+ bool firmware_hctl_core_pwr; -+ struct work_struct firmware_reload_work; -+ bool glb_init_request_pending; -+ struct work_struct fw_error_work; -+ struct kbase_ipa_control ipa_control; -+ u32 mcu_core_pwroff_dur_us; -+ u32 mcu_core_pwroff_dur_count; -+ u32 mcu_core_pwroff_reg_shadow; -+ u32 gpu_idle_hysteresis_ms; -+ u32 gpu_idle_dur_count; -+ unsigned int fw_timeout_ms; -+ struct kbase_csf_hwcnt hwcnt; -+}; -+ -+/** -+ * struct kbase_as - Object representing an address space of GPU. -+ * @number: Index at which this address space structure is present -+ * in an array of address space structures embedded inside -+ * the &struct kbase_device. -+ * @pf_wq: Workqueue for processing work items related to -+ * Page fault, Bus fault and GPU fault handling. -+ * @work_pagefault: Work item for the Page fault handling. -+ * @work_busfault: Work item for the Bus fault handling. -+ * @work_gpufault: Work item for the GPU fault handling. -+ * @pf_data: Data relating to Page fault. -+ * @bf_data: Data relating to Bus fault. -+ * @gf_data: Data relating to GPU fault. -+ * @current_setup: Stores the MMU configuration for this address space. -+ */ -+struct kbase_as { -+ int number; -+ struct workqueue_struct *pf_wq; -+ struct work_struct work_pagefault; -+ struct work_struct work_busfault; -+ struct work_struct work_gpufault; -+ struct kbase_fault pf_data; -+ struct kbase_fault bf_data; -+ struct kbase_fault gf_data; -+ struct kbase_mmu_setup current_setup; -+}; -+ -+#endif /* _KBASE_CSF_DEFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware.c -new file mode 100644 -index 0000000..1b31122 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware.c -@@ -0,0 +1,2337 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase.h" -+#include "mali_kbase_csf_firmware_cfg.h" -+#include "mali_kbase_csf_trace_buffer.h" -+#include "mali_kbase_csf_timeout.h" -+#include "mali_kbase_mem.h" -+#include "mali_kbase_reset_gpu.h" -+#include "mali_kbase_ctx_sched.h" -+#include "mali_kbase_csf_scheduler.h" -+#include "device/mali_kbase_device.h" -+#include "backend/gpu/mali_kbase_pm_internal.h" -+#include "tl/mali_kbase_timeline_priv.h" -+#include "mali_kbase_csf_tl_reader.h" -+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" -+#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> -+ -+#include <linux/list.h> -+#include <linux/slab.h> -+#include <linux/firmware.h> -+#include <linux/mman.h> -+#include <linux/string.h> -+#include <linux/mutex.h> -+#if (KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE) -+#include <linux/set_memory.h> -+#endif -+#include <mmu/mali_kbase_mmu.h> -+#include <asm/arch_timer.h> -+ -+#define MALI_MAX_FIRMWARE_NAME_LEN ((size_t)20) -+ -+ -+static char fw_name[MALI_MAX_FIRMWARE_NAME_LEN] = "mali_csffw.bin"; -+module_param_string(fw_name, fw_name, sizeof(fw_name), 0644); -+MODULE_PARM_DESC(fw_name, "firmware image"); -+ -+/* The waiting time for firmware to boot */ -+static unsigned int csf_firmware_boot_timeout_ms = 500; -+module_param(csf_firmware_boot_timeout_ms, uint, 0444); -+MODULE_PARM_DESC(csf_firmware_boot_timeout_ms, -+ "Maximum time to wait for firmware to boot."); -+ -+#ifdef CONFIG_MALI_DEBUG -+/* Makes Driver wait indefinitely for an acknowledgment for the different -+ * requests it sends to firmware. Otherwise the timeouts interfere with the -+ * use of debugger for source-level debugging of firmware as Driver initiates -+ * a GPU reset when a request times out, which always happen when a debugger -+ * is connected. -+ */ -+bool fw_debug; /* Default value of 0/false */ -+module_param(fw_debug, bool, 0444); -+MODULE_PARM_DESC(fw_debug, -+ "Enables effective use of a debugger for debugging firmware code."); -+#endif -+ -+#define FIRMWARE_HEADER_MAGIC (0xC3F13A6Eul) -+#define FIRMWARE_HEADER_VERSION (0ul) -+#define FIRMWARE_HEADER_LENGTH (0x14ul) -+ -+#define CSF_FIRMWARE_ENTRY_SUPPORTED_FLAGS \ -+ (CSF_FIRMWARE_ENTRY_READ | \ -+ CSF_FIRMWARE_ENTRY_WRITE | \ -+ CSF_FIRMWARE_ENTRY_EXECUTE | \ -+ CSF_FIRMWARE_ENTRY_PROTECTED | \ -+ CSF_FIRMWARE_ENTRY_SHARED | \ -+ CSF_FIRMWARE_ENTRY_ZERO | \ -+ CSF_FIRMWARE_ENTRY_CACHE_MODE) -+ -+#define CSF_FIRMWARE_ENTRY_TYPE_INTERFACE (0) -+#define CSF_FIRMWARE_ENTRY_TYPE_CONFIGURATION (1) -+#define CSF_FIRMWARE_ENTRY_TYPE_FUTF_TEST (2) -+#define CSF_FIRMWARE_ENTRY_TYPE_TRACE_BUFFER (3) -+#define CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA (4) -+ -+#define CSF_FIRMWARE_CACHE_MODE_NONE (0ul << 3) -+#define CSF_FIRMWARE_CACHE_MODE_CACHED (1ul << 3) -+#define CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT (2ul << 3) -+#define CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT (3ul << 3) -+ -+#define INTERFACE_ENTRY_NAME_OFFSET (0x14) -+ -+#define TL_METADATA_ENTRY_NAME_OFFSET (0x8) -+ -+#define CSF_MAX_FW_STOP_LOOPS (100000) -+ -+#define CSF_GLB_REQ_CFG_MASK \ -+ (GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK | \ -+ GLB_REQ_CFG_PWROFF_TIMER_MASK) -+ -+static inline u32 input_page_read(const u32 *const input, const u32 offset) -+{ -+ WARN_ON(offset % sizeof(u32)); -+ -+ return input[offset / sizeof(u32)]; -+} -+ -+static inline void input_page_write(u32 *const input, const u32 offset, -+ const u32 value) -+{ -+ WARN_ON(offset % sizeof(u32)); -+ -+ input[offset / sizeof(u32)] = value; -+} -+ -+static inline void input_page_partial_write(u32 *const input, const u32 offset, -+ u32 value, u32 mask) -+{ -+ WARN_ON(offset % sizeof(u32)); -+ -+ input[offset / sizeof(u32)] = -+ (input_page_read(input, offset) & ~mask) | (value & mask); -+} -+ -+static inline u32 output_page_read(const u32 *const output, const u32 offset) -+{ -+ WARN_ON(offset % sizeof(u32)); -+ -+ return output[offset / sizeof(u32)]; -+} -+ -+static unsigned int entry_type(u32 header) -+{ -+ return header & 0xFF; -+} -+static unsigned int entry_size(u32 header) -+{ -+ return (header >> 8) & 0xFF; -+} -+static bool entry_update(u32 header) -+{ -+ return (header >> 30) & 0x1; -+} -+static bool entry_optional(u32 header) -+{ -+ return (header >> 31) & 0x1; -+} -+ -+/** -+ * struct firmware_timeline_metadata - -+ * Timeline metadata item within the MCU firmware -+ * -+ * @node: List head linking all timeline metadata to -+ * kbase_device:csf.firmware_timeline_metadata. -+ * @name: NUL-terminated string naming the metadata. -+ * @data: Metadata content. -+ * @size: Metadata size. -+ */ -+struct firmware_timeline_metadata { -+ struct list_head node; -+ char *name; -+ char *data; -+ size_t size; -+}; -+ -+/* The shared interface area, used for communicating with firmware, is managed -+ * like a virtual memory zone. Reserve the virtual space from that zone -+ * corresponding to shared interface entry parsed from the firmware image. -+ * The shared_reg_rbtree should have been initialized before calling this -+ * function. -+ */ -+static int setup_shared_iface_static_region(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_firmware_interface *interface = -+ kbdev->csf.shared_interface; -+ struct kbase_va_region *reg; -+ int ret = -ENOMEM; -+ -+ if (!interface) -+ return -EINVAL; -+ -+ reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, -+ interface->num_pages, KBASE_REG_ZONE_MCU_SHARED); -+ if (reg) { -+ ret = kbase_add_va_region_rbtree(kbdev, reg, -+ interface->virtual, interface->num_pages, 1); -+ if (ret) -+ kfree(reg); -+ else -+ reg->flags &= ~KBASE_REG_FREE; -+ } -+ -+ return ret; -+} -+ -+static int wait_mcu_status_value(struct kbase_device *kbdev, u32 val) -+{ -+ u32 max_loops = CSF_MAX_FW_STOP_LOOPS; -+ -+ /* wait for the MCU_STATUS register to reach the given status value */ -+ while (--max_loops && -+ (kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS)) != val)) { -+ } -+ -+ return (max_loops == 0) ? -1 : 0; -+} -+ -+void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) -+{ -+ if (wait_mcu_status_value(kbdev, MCU_CNTRL_DISABLE) < 0) -+ dev_err(kbdev->dev, "MCU failed to get disabled"); -+} -+ -+static void wait_for_firmware_stop(struct kbase_device *kbdev) -+{ -+ if (wait_mcu_status_value(kbdev, MCU_CNTRL_DISABLE) < 0) { -+ /* This error shall go away once MIDJM-2371 is closed */ -+ dev_err(kbdev->dev, "Firmware failed to stop"); -+ } -+} -+ -+static void stop_csf_firmware(struct kbase_device *kbdev) -+{ -+ /* Stop the MCU firmware */ -+ kbase_csf_firmware_disable_mcu(kbdev); -+ -+ wait_for_firmware_stop(kbdev); -+} -+ -+static void wait_for_firmware_boot(struct kbase_device *kbdev) -+{ -+ const long wait_timeout = -+ kbase_csf_timeout_in_jiffies(csf_firmware_boot_timeout_ms); -+ long remaining; -+ -+ /* Firmware will generate a global interface interrupt once booting -+ * is complete -+ */ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ kbdev->csf.interrupt_received == true, wait_timeout); -+ -+ if (!remaining) -+ dev_err(kbdev->dev, "Timed out waiting for fw boot completion"); -+ -+ kbdev->csf.interrupt_received = false; -+} -+ -+static void boot_csf_firmware(struct kbase_device *kbdev) -+{ -+ kbase_csf_firmware_enable_mcu(kbdev); -+ -+ wait_for_firmware_boot(kbdev); -+} -+ -+static void wait_ready(struct kbase_device *kbdev) -+{ -+ u32 max_loops = KBASE_AS_INACTIVE_MAX_LOOPS; -+ u32 val; -+ -+ val = kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)); -+ -+ /* Wait for a while for the update command to take effect */ -+ while (--max_loops && (val & AS_STATUS_AS_ACTIVE)) -+ val = kbase_reg_read(kbdev, MMU_AS_REG(MCU_AS_NR, AS_STATUS)); -+ -+ if (max_loops == 0) -+ dev_err(kbdev->dev, "AS_ACTIVE bit stuck, might be caused by slow/unstable GPU clock or possible faulty FPGA connector\n"); -+} -+ -+static void unload_mmu_tables(struct kbase_device *kbdev) -+{ -+ unsigned long irq_flags; -+ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -+ if (kbdev->pm.backend.gpu_powered) -+ kbase_mmu_disable_as(kbdev, MCU_AS_NR); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+} -+ -+static void load_mmu_tables(struct kbase_device *kbdev) -+{ -+ unsigned long irq_flags; -+ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags); -+ kbase_mmu_update(kbdev, &kbdev->csf.mcu_mmu, MCU_AS_NR); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ /* Wait for a while for the update command to take effect */ -+ wait_ready(kbdev); -+} -+ -+/** -+ * convert_mem_flags() - Convert firmware memory flags to GPU region flags -+ * -+ * Return: GPU memory region flags -+ * -+ * @kbdev: Instance of GPU platform device (used to determine system coherency) -+ * @flags: Flags of an "interface memory setup" section in a firmware image -+ * @cm: appropriate cache mode chosen for the "interface memory setup" -+ * section, which could be different from the cache mode requested by -+ * firmware. -+ */ -+static unsigned long convert_mem_flags(const struct kbase_device * const kbdev, -+ const u32 flags, u32 *cm) -+{ -+ unsigned long mem_flags = 0; -+ u32 cache_mode = flags & CSF_FIRMWARE_ENTRY_CACHE_MODE; -+ bool is_shared = (flags & CSF_FIRMWARE_ENTRY_SHARED) ? true : false; -+ -+ /* The memory flags control the access permissions for the MCU, the -+ * shader cores/tiler are not expected to access this memory -+ */ -+ if (flags & CSF_FIRMWARE_ENTRY_READ) -+ mem_flags |= KBASE_REG_GPU_RD; -+ -+ if (flags & CSF_FIRMWARE_ENTRY_WRITE) -+ mem_flags |= KBASE_REG_GPU_WR; -+ -+ if ((flags & CSF_FIRMWARE_ENTRY_EXECUTE) == 0) -+ mem_flags |= KBASE_REG_GPU_NX; -+ -+ if (flags & CSF_FIRMWARE_ENTRY_PROTECTED) -+ mem_flags |= KBASE_REG_PROTECTED; -+ -+ /* Substitute uncached coherent memory for cached coherent memory if -+ * the system does not support ACE coherency. -+ */ -+ if ((cache_mode == CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT) && -+ (kbdev->system_coherency != COHERENCY_ACE)) -+ cache_mode = CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT; -+ -+ /* Substitute uncached incoherent memory for uncached coherent memory -+ * if the system does not support ACE-Lite coherency. -+ */ -+ if ((cache_mode == CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT) && -+ (kbdev->system_coherency == COHERENCY_NONE)) -+ cache_mode = CSF_FIRMWARE_CACHE_MODE_NONE; -+ -+ *cm = cache_mode; -+ -+ switch (cache_mode) { -+ case CSF_FIRMWARE_CACHE_MODE_NONE: -+ mem_flags |= -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); -+ break; -+ case CSF_FIRMWARE_CACHE_MODE_CACHED: -+ mem_flags |= -+ KBASE_REG_MEMATTR_INDEX( -+ AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY); -+ break; -+ case CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT: -+ case CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT: -+ WARN_ON(!is_shared); -+ mem_flags |= KBASE_REG_SHARE_BOTH | -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); -+ break; -+ default: -+ dev_err(kbdev->dev, -+ "Firmware contains interface with unsupported cache mode\n"); -+ break; -+ } -+ return mem_flags; -+} -+ -+static void load_fw_image_section(struct kbase_device *kbdev, const u8 *data, -+ struct tagged_addr *phys, u32 num_pages, u32 flags, -+ u32 data_start, u32 data_end) -+{ -+ u32 data_pos = data_start; -+ u32 data_len = data_end - data_start; -+ u32 page_num; -+ u32 page_limit; -+ -+ if (flags & CSF_FIRMWARE_ENTRY_ZERO) -+ page_limit = num_pages; -+ else -+ page_limit = (data_len + PAGE_SIZE - 1) / PAGE_SIZE; -+ -+ for (page_num = 0; page_num < page_limit; ++page_num) { -+ struct page *const page = as_page(phys[page_num]); -+ char *const p = kmap_atomic(page); -+ u32 const copy_len = min_t(u32, PAGE_SIZE, data_len); -+ -+ if (copy_len > 0) { -+ memcpy(p, data + data_pos, copy_len); -+ data_pos += copy_len; -+ data_len -= copy_len; -+ } -+ -+ if (flags & CSF_FIRMWARE_ENTRY_ZERO) { -+ u32 const zi_len = PAGE_SIZE - copy_len; -+ -+ memset(p + copy_len, 0, zi_len); -+ } -+ -+ kbase_sync_single_for_device(kbdev, kbase_dma_addr(page), -+ PAGE_SIZE, DMA_TO_DEVICE); -+ kunmap_atomic(p); -+ } -+} -+ -+static int reload_fw_data_sections(struct kbase_device *kbdev) -+{ -+ const u32 magic = FIRMWARE_HEADER_MAGIC; -+ struct kbase_csf_firmware_interface *interface; -+ const struct firmware *firmware; -+ int ret = 0; -+ -+ if (request_firmware(&firmware, fw_name, kbdev->dev) != 0) { -+ dev_err(kbdev->dev, -+ "Failed to reload firmware image '%s'\n", -+ fw_name); -+ return -ENOENT; -+ } -+ -+ /* Do couple of basic sanity checks */ -+ if (firmware->size < FIRMWARE_HEADER_LENGTH) { -+ dev_err(kbdev->dev, "Firmware image unexpectedly too small\n"); -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ if (memcmp(firmware->data, &magic, sizeof(magic)) != 0) { -+ dev_err(kbdev->dev, "Incorrect magic value, firmware image could have been corrupted\n"); -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) { -+ /* Skip reload of text & read only data sections */ -+ if ((interface->flags & CSF_FIRMWARE_ENTRY_EXECUTE) || -+ !(interface->flags & CSF_FIRMWARE_ENTRY_WRITE)) -+ continue; -+ -+ load_fw_image_section(kbdev, firmware->data, interface->phys, -+ interface->num_pages, interface->flags, -+ interface->data_start, interface->data_end); -+ } -+ -+ kbase_csf_firmware_reload_trace_buffers_data(kbdev); -+ -+out: -+ release_firmware(firmware); -+ return ret; -+} -+ -+/** -+ * parse_memory_setup_entry() - Process an "interface memory setup" section -+ * -+ * Read an "interface memory setup" section from the firmware image and create -+ * the necessary memory region including the MMU page tables. If successful -+ * the interface will be added to the kbase_device:csf.firmware_interfaces list. -+ * -+ * Return: 0 if successful, negative error code on failure -+ * -+ * @kbdev: Kbase device structure -+ * @fw: The firmware image containing the section -+ * @entry: Pointer to the start of the section -+ * @size: Size (in bytes) of the section -+ */ -+static int parse_memory_setup_entry(struct kbase_device *kbdev, -+ const struct firmware *fw, -+ const u32 *entry, unsigned int size) -+{ -+ int ret = 0; -+ const u32 flags = entry[0]; -+ const u32 virtual_start = entry[1]; -+ const u32 virtual_end = entry[2]; -+ const u32 data_start = entry[3]; -+ const u32 data_end = entry[4]; -+ u32 num_pages; -+ char *name; -+ struct tagged_addr *phys = NULL; -+ struct kbase_csf_firmware_interface *interface = NULL; -+ bool allocated_pages = false, protected_mode = false; -+ unsigned long mem_flags = 0; -+ u32 cache_mode = 0; -+ struct protected_memory_allocation **pma = NULL; -+ -+ if (data_end < data_start) { -+ dev_err(kbdev->dev, "Firmware corrupt, data_end < data_start (0x%x<0x%x)\n", -+ data_end, data_start); -+ return -EINVAL; -+ } -+ if (virtual_end < virtual_start) { -+ dev_err(kbdev->dev, "Firmware corrupt, virtual_end < virtual_start (0x%x<0x%x)\n", -+ virtual_end, virtual_start); -+ return -EINVAL; -+ } -+ if (data_end > fw->size) { -+ dev_err(kbdev->dev, "Firmware corrupt, file truncated? data_end=0x%x > fw->size=0x%zx\n", -+ data_end, fw->size); -+ return -EINVAL; -+ } -+ -+ if ((virtual_start & ~PAGE_MASK) != 0 || -+ (virtual_end & ~PAGE_MASK) != 0) { -+ dev_err(kbdev->dev, "Firmware corrupt: virtual addresses not page aligned: 0x%x-0x%x\n", -+ virtual_start, virtual_end); -+ return -EINVAL; -+ } -+ -+ if ((flags & CSF_FIRMWARE_ENTRY_SUPPORTED_FLAGS) != flags) { -+ dev_err(kbdev->dev, "Firmware contains interface with unsupported flags (0x%x)\n", -+ flags); -+ return -EINVAL; -+ } -+ -+ if (flags & CSF_FIRMWARE_ENTRY_PROTECTED) -+ protected_mode = true; -+ -+ if (protected_mode && kbdev->csf.pma_dev == NULL) { -+ dev_err(kbdev->dev, -+ "Protected memory allocator not found, Firmware protected mode entry will not be supported"); -+ return 0; -+ } -+ -+ num_pages = (virtual_end - virtual_start) -+ >> PAGE_SHIFT; -+ -+ phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); -+ if (!phys) -+ return -ENOMEM; -+ -+ if (protected_mode) { -+ pma = kbase_csf_protected_memory_alloc(kbdev, phys, num_pages); -+ -+ if (pma == NULL) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ } else { -+ ret = kbase_mem_pool_alloc_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ num_pages, phys, false); -+ if (ret < 0) -+ goto out; -+ } -+ -+ allocated_pages = true; -+ load_fw_image_section(kbdev, fw->data, phys, num_pages, flags, -+ data_start, data_end); -+ -+ /* Allocate enough memory for the struct kbase_csf_firmware_interface and -+ * the name of the interface. An extra byte is allocated to place a -+ * NUL-terminator in. This should already be included according to the -+ * specification but here we add it anyway to be robust against a -+ * corrupt firmware image. -+ */ -+ interface = kmalloc(sizeof(*interface) + -+ size - INTERFACE_ENTRY_NAME_OFFSET + 1, GFP_KERNEL); -+ if (!interface) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ name = (void *)(interface + 1); -+ memcpy(name, entry + (INTERFACE_ENTRY_NAME_OFFSET / sizeof(*entry)), -+ size - INTERFACE_ENTRY_NAME_OFFSET); -+ name[size - INTERFACE_ENTRY_NAME_OFFSET] = 0; -+ -+ interface->name = name; -+ interface->phys = phys; -+ interface->num_pages = num_pages; -+ interface->virtual = virtual_start; -+ interface->kernel_map = NULL; -+ interface->flags = flags; -+ interface->data_start = data_start; -+ interface->data_end = data_end; -+ interface->pma = pma; -+ -+ mem_flags = convert_mem_flags(kbdev, flags, &cache_mode); -+ -+ if (flags & CSF_FIRMWARE_ENTRY_SHARED) { -+ struct page **page_list; -+ u32 i; -+ pgprot_t cpu_map_prot; -+ u32 mem_attr_index = KBASE_REG_MEMATTR_VALUE(mem_flags); -+ -+ /* Since SHARED memory type was used for mapping shared memory -+ * on GPU side, it can be mapped as cached on CPU side on both -+ * types of coherent platforms. -+ */ -+ if ((cache_mode == CSF_FIRMWARE_CACHE_MODE_CACHED_COHERENT) || -+ (cache_mode == CSF_FIRMWARE_CACHE_MODE_UNCACHED_COHERENT)) { -+ WARN_ON(mem_attr_index != -+ AS_MEMATTR_INDEX_SHARED); -+ cpu_map_prot = PAGE_KERNEL; -+ } else { -+ WARN_ON(mem_attr_index != -+ AS_MEMATTR_INDEX_NON_CACHEABLE); -+ cpu_map_prot = pgprot_writecombine(PAGE_KERNEL); -+ } -+ -+ page_list = kmalloc_array(num_pages, sizeof(*page_list), -+ GFP_KERNEL); -+ if (!page_list) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ for (i = 0; i < num_pages; i++) -+ page_list[i] = as_page(phys[i]); -+ -+ interface->kernel_map = vmap(page_list, num_pages, VM_MAP, -+ cpu_map_prot); -+ -+ kfree(page_list); -+ -+ if (!interface->kernel_map) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ } -+ -+ /* Start location of the shared interface area is fixed and is -+ * specified in firmware spec, and so there shall only be a -+ * single entry with that start address. -+ */ -+ if (virtual_start == (KBASE_REG_ZONE_MCU_SHARED_BASE << PAGE_SHIFT)) -+ kbdev->csf.shared_interface = interface; -+ -+ list_add(&interface->node, &kbdev->csf.firmware_interfaces); -+ -+ ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, -+ virtual_start >> PAGE_SHIFT, phys, num_pages, mem_flags, -+ KBASE_MEM_GROUP_CSF_FW); -+ -+ if (ret != 0) { -+ dev_err(kbdev->dev, "Failed to insert firmware pages\n"); -+ /* The interface has been added to the list, so cleanup will -+ * be handled by firmware unloading -+ */ -+ } -+ -+ dev_dbg(kbdev->dev, "Processed section '%s'", name); -+ -+ return ret; -+ -+out: -+ if (allocated_pages) { -+ if (protected_mode) { -+ kbase_csf_protected_memory_free(kbdev, pma, num_pages); -+ } else { -+ kbase_mem_pool_free_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ num_pages, phys, false, false); -+ } -+ } -+ -+ kfree(phys); -+ kfree(interface); -+ return ret; -+} -+ -+/** -+ * parse_timeline_metadata_entry() - Process a "timeline metadata" section -+ * -+ * Return: 0 if successful, negative error code on failure -+ * -+ * @kbdev: Kbase device structure -+ * @fw: Firmware image containing the section -+ * @entry: Pointer to the section -+ * @size: Size (in bytes) of the section -+ */ -+static int parse_timeline_metadata_entry(struct kbase_device *kbdev, -+ const struct firmware *fw, const u32 *entry, unsigned int size) -+{ -+ const u32 data_start = entry[0]; -+ const u32 data_size = entry[1]; -+ const u32 data_end = data_start + data_size; -+ const char *name = (char *)&entry[2]; -+ struct firmware_timeline_metadata *metadata; -+ const unsigned int name_len = -+ size - TL_METADATA_ENTRY_NAME_OFFSET; -+ size_t allocation_size = sizeof(*metadata) + name_len + 1 + data_size; -+ -+ if (data_end > fw->size) { -+ dev_err(kbdev->dev, -+ "Firmware corrupt, file truncated? data_end=0x%x > fw->size=0x%zx", -+ data_end, fw->size); -+ return -EINVAL; -+ } -+ -+ /* Allocate enough space for firmware_timeline_metadata, -+ * its name and the content. -+ */ -+ metadata = kmalloc(allocation_size, GFP_KERNEL); -+ if (!metadata) -+ return -ENOMEM; -+ -+ metadata->name = (char *)(metadata + 1); -+ metadata->data = (char *)(metadata + 1) + name_len + 1; -+ metadata->size = data_size; -+ -+ memcpy(metadata->name, name, name_len); -+ metadata->name[name_len] = 0; -+ -+ /* Copy metadata's content. */ -+ memcpy(metadata->data, fw->data + data_start, data_size); -+ -+ list_add(&metadata->node, &kbdev->csf.firmware_timeline_metadata); -+ -+ dev_dbg(kbdev->dev, "Timeline metadata '%s'", metadata->name); -+ -+ return 0; -+} -+ -+/** -+ * load_firmware_entry() - Process an entry from a firmware image -+ * -+ * Read an entry from a firmware image and do any necessary work (e.g. loading -+ * the data into page accessible to the MCU). -+ * -+ * Unknown entries are ignored if the 'optional' flag is set within the entry, -+ * otherwise the function will fail with -EINVAL -+ * -+ * Return: 0 if successful, negative error code on failure -+ * -+ * @kbdev: Kbase device -+ * @fw: Firmware image containing the entry -+ * @offset: Byte offset within the image of the entry to load -+ * @header: Header word of the entry -+ */ -+static int load_firmware_entry(struct kbase_device *kbdev, -+ const struct firmware *fw, -+ u32 offset, u32 header) -+{ -+ const unsigned int type = entry_type(header); -+ unsigned int size = entry_size(header); -+ const bool optional = entry_optional(header); -+ /* Update is used with configuration and tracebuffer entries to -+ * initiate a FIRMWARE_CONFIG_UPDATE, instead of triggering a -+ * silent reset. -+ */ -+ const bool updatable = entry_update(header); -+ const u32 *entry = (void *)(fw->data + offset); -+ -+ if ((offset % sizeof(*entry)) || (size % sizeof(*entry))) { -+ dev_err(kbdev->dev, "Firmware entry isn't 32 bit aligned, offset=0x%x size=0x%x\n", -+ offset, size); -+ return -EINVAL; -+ } -+ -+ if (size < sizeof(*entry)) { -+ dev_err(kbdev->dev, "Size field too small: %u\n", size); -+ return -EINVAL; -+ } -+ -+ /* Remove the header */ -+ entry++; -+ size -= sizeof(*entry); -+ -+ switch (type) { -+ case CSF_FIRMWARE_ENTRY_TYPE_INTERFACE: -+ /* Interface memory setup */ -+ if (size < INTERFACE_ENTRY_NAME_OFFSET + sizeof(*entry)) { -+ dev_err(kbdev->dev, "Interface memory setup entry too short (size=%u)\n", -+ size); -+ return -EINVAL; -+ } -+ return parse_memory_setup_entry(kbdev, fw, entry, size); -+ case CSF_FIRMWARE_ENTRY_TYPE_CONFIGURATION: -+ /* Configuration option */ -+ if (size < CONFIGURATION_ENTRY_NAME_OFFSET + sizeof(*entry)) { -+ dev_err(kbdev->dev, "Configuration option entry too short (size=%u)\n", -+ size); -+ return -EINVAL; -+ } -+ return kbase_csf_firmware_cfg_option_entry_parse( -+ kbdev, fw, entry, size, updatable); -+ case CSF_FIRMWARE_ENTRY_TYPE_FUTF_TEST: -+#ifndef MALI_KBASE_BUILD -+ /* FW UTF option */ -+ if (size < 2*sizeof(*entry)) { -+ dev_err(kbdev->dev, "FW UTF entry too short (size=%u)\n", -+ size); -+ return -EINVAL; -+ } -+ return mali_kutf_process_fw_utf_entry(kbdev, fw->data, -+ fw->size, entry); -+#endif -+ break; -+ case CSF_FIRMWARE_ENTRY_TYPE_TRACE_BUFFER: -+ /* Trace buffer */ -+ if (size < TRACE_BUFFER_ENTRY_NAME_OFFSET + sizeof(*entry)) { -+ dev_err(kbdev->dev, "Trace Buffer entry too short (size=%u)\n", -+ size); -+ return -EINVAL; -+ } -+ return kbase_csf_firmware_parse_trace_buffer_entry( -+ kbdev, entry, size, updatable); -+ case CSF_FIRMWARE_ENTRY_TYPE_TIMELINE_METADATA: -+ /* Meta data section */ -+ if (size < TL_METADATA_ENTRY_NAME_OFFSET + sizeof(*entry)) { -+ dev_err(kbdev->dev, "Timeline metadata entry too short (size=%u)\n", -+ size); -+ return -EINVAL; -+ } -+ return parse_timeline_metadata_entry(kbdev, fw, entry, size); -+ } -+ -+ if (!optional) { -+ dev_err(kbdev->dev, -+ "Unsupported non-optional entry type %u in firmware\n", -+ type); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+ -+static void free_global_iface(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; -+ -+ if (iface->groups) { -+ unsigned int gid; -+ -+ for (gid = 0; gid < iface->group_num; ++gid) -+ kfree(iface->groups[gid].streams); -+ -+ kfree(iface->groups); -+ iface->groups = NULL; -+ } -+} -+ -+/** -+ * iface_gpu_va_to_cpu - Convert a GPU VA address within the shared interface -+ * region to a CPU address, using the existing mapping. -+ * @kbdev: Device pointer -+ * @gpu_va: GPU VA to convert -+ * -+ * Return: A CPU pointer to the location within the shared interface region, or -+ * NULL on failure. -+ */ -+static inline void *iface_gpu_va_to_cpu(struct kbase_device *kbdev, u32 gpu_va) -+{ -+ struct kbase_csf_firmware_interface *interface = -+ kbdev->csf.shared_interface; -+ u8 *kernel_base = interface->kernel_map; -+ -+ if (gpu_va < interface->virtual || -+ gpu_va >= interface->virtual + interface->num_pages * PAGE_SIZE) { -+ dev_err(kbdev->dev, -+ "Interface address 0x%x not within %u-page region at 0x%x", -+ gpu_va, interface->num_pages, -+ interface->virtual); -+ return NULL; -+ } -+ -+ return (void *)(kernel_base + (gpu_va - interface->virtual)); -+} -+ -+static int parse_cmd_stream_info(struct kbase_device *kbdev, -+ struct kbase_csf_cmd_stream_info *sinfo, -+ u32 *stream_base) -+{ -+ sinfo->kbdev = kbdev; -+ sinfo->features = stream_base[STREAM_FEATURES/4]; -+ sinfo->input = iface_gpu_va_to_cpu(kbdev, -+ stream_base[STREAM_INPUT_VA/4]); -+ sinfo->output = iface_gpu_va_to_cpu(kbdev, -+ stream_base[STREAM_OUTPUT_VA/4]); -+ -+ if (sinfo->input == NULL || sinfo->output == NULL) -+ return -EINVAL; -+ -+ return 0; -+} -+ -+static int parse_cmd_stream_group_info(struct kbase_device *kbdev, -+ struct kbase_csf_cmd_stream_group_info *ginfo, -+ u32 *group_base, u32 group_stride) -+{ -+ unsigned int sid; -+ -+ ginfo->kbdev = kbdev; -+ ginfo->features = group_base[GROUP_FEATURES/4]; -+ ginfo->input = iface_gpu_va_to_cpu(kbdev, -+ group_base[GROUP_INPUT_VA/4]); -+ ginfo->output = iface_gpu_va_to_cpu(kbdev, -+ group_base[GROUP_OUTPUT_VA/4]); -+ -+ if (ginfo->input == NULL || ginfo->output == NULL) -+ return -ENOMEM; -+ -+ ginfo->suspend_size = group_base[GROUP_SUSPEND_SIZE/4]; -+ ginfo->protm_suspend_size = group_base[GROUP_PROTM_SUSPEND_SIZE/4]; -+ ginfo->stream_num = group_base[GROUP_STREAM_NUM/4]; -+ -+ if (ginfo->stream_num < MIN_SUPPORTED_STREAMS_PER_GROUP || -+ ginfo->stream_num > MAX_SUPPORTED_STREAMS_PER_GROUP) { -+ dev_err(kbdev->dev, "CSG with %u CSs out of range %u-%u", -+ ginfo->stream_num, -+ MIN_SUPPORTED_STREAMS_PER_GROUP, -+ MAX_SUPPORTED_STREAMS_PER_GROUP); -+ return -EINVAL; -+ } -+ -+ ginfo->stream_stride = group_base[GROUP_STREAM_STRIDE/4]; -+ -+ if (ginfo->stream_num * ginfo->stream_stride > group_stride) { -+ dev_err(kbdev->dev, -+ "group stride of 0x%x exceeded by %u CSs with stride 0x%x", -+ group_stride, ginfo->stream_num, -+ ginfo->stream_stride); -+ return -EINVAL; -+ } -+ -+ ginfo->streams = kmalloc_array(ginfo->stream_num, -+ sizeof(*ginfo->streams), GFP_KERNEL); -+ -+ if (!ginfo->streams) -+ return -ENOMEM; -+ -+ for (sid = 0; sid < ginfo->stream_num; sid++) { -+ int err; -+ u32 *stream_base = group_base + (STREAM_CONTROL_0 + -+ ginfo->stream_stride * sid) / 4; -+ -+ err = parse_cmd_stream_info(kbdev, &ginfo->streams[sid], -+ stream_base); -+ if (err < 0) { -+ /* caller will free the memory for CSs array */ -+ return err; -+ } -+ } -+ -+ return 0; -+} -+ -+static u32 get_firmware_version(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_firmware_interface *interface = -+ kbdev->csf.shared_interface; -+ u32 *shared_info = interface->kernel_map; -+ -+ return shared_info[GLB_VERSION/4]; -+} -+ -+static int parse_capabilities(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_firmware_interface *interface = -+ kbdev->csf.shared_interface; -+ u32 *shared_info = interface->kernel_map; -+ struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; -+ unsigned int gid; -+ -+ /* All offsets are in bytes, so divide by 4 for access via a u32 pointer -+ */ -+ -+ /* The version number of the global interface is expected to be a -+ * non-zero value. If it's not, the firmware may not have booted. -+ */ -+ iface->version = get_firmware_version(kbdev); -+ if (!iface->version) { -+ dev_err(kbdev->dev, "Version check failed. Firmware may have failed to boot."); -+ return -EINVAL; -+ } -+ -+ -+ iface->kbdev = kbdev; -+ iface->features = shared_info[GLB_FEATURES/4]; -+ iface->input = iface_gpu_va_to_cpu(kbdev, shared_info[GLB_INPUT_VA/4]); -+ iface->output = iface_gpu_va_to_cpu(kbdev, -+ shared_info[GLB_OUTPUT_VA/4]); -+ -+ if (iface->input == NULL || iface->output == NULL) -+ return -ENOMEM; -+ -+ iface->group_num = shared_info[GLB_GROUP_NUM/4]; -+ -+ if (iface->group_num < MIN_SUPPORTED_CSGS || -+ iface->group_num > MAX_SUPPORTED_CSGS) { -+ dev_err(kbdev->dev, -+ "Interface containing %u CSGs outside of range %u-%u", -+ iface->group_num, MIN_SUPPORTED_CSGS, -+ MAX_SUPPORTED_CSGS); -+ return -EINVAL; -+ } -+ -+ iface->group_stride = shared_info[GLB_GROUP_STRIDE/4]; -+ iface->prfcnt_size = shared_info[GLB_PRFCNT_SIZE/4]; -+ -+ if (iface->version >= kbase_csf_interface_version(1, 1, 0)) { -+ iface->instr_features = shared_info[GLB_INSTR_FEATURES / 4]; -+ } else { -+ iface->instr_features = 0; -+ } -+ -+ if ((GROUP_CONTROL_0 + -+ (unsigned long)iface->group_num * iface->group_stride) > -+ (interface->num_pages * PAGE_SIZE)) { -+ dev_err(kbdev->dev, -+ "interface size of %u pages exceeded by %u CSGs with stride 0x%x", -+ interface->num_pages, iface->group_num, -+ iface->group_stride); -+ return -EINVAL; -+ } -+ -+ WARN_ON(iface->groups); -+ -+ iface->groups = kcalloc(iface->group_num, sizeof(*iface->groups), -+ GFP_KERNEL); -+ if (!iface->groups) -+ return -ENOMEM; -+ -+ for (gid = 0; gid < iface->group_num; gid++) { -+ int err; -+ u32 *group_base = shared_info + (GROUP_CONTROL_0 + -+ iface->group_stride * gid) / 4; -+ -+ err = parse_cmd_stream_group_info(kbdev, &iface->groups[gid], -+ group_base, iface->group_stride); -+ if (err < 0) { -+ free_global_iface(kbdev); -+ return err; -+ } -+ } -+ -+ return 0; -+} -+ -+static inline void access_firmware_memory(struct kbase_device *kbdev, -+ u32 gpu_addr, u32 *value, const bool read) -+{ -+ struct kbase_csf_firmware_interface *interface; -+ -+ list_for_each_entry(interface, &kbdev->csf.firmware_interfaces, node) { -+ if ((gpu_addr >= interface->virtual) && -+ (gpu_addr < interface->virtual + (interface->num_pages << PAGE_SHIFT))) { -+ u32 offset_bytes = gpu_addr - interface->virtual; -+ u32 page_num = offset_bytes >> PAGE_SHIFT; -+ u32 offset_in_page = offset_bytes & ~PAGE_MASK; -+ struct page *target_page = as_page( -+ interface->phys[page_num]); -+ u32 *cpu_addr = kmap_atomic(target_page); -+ -+ if (read) { -+ kbase_sync_single_for_device(kbdev, -+ kbase_dma_addr(target_page) + offset_in_page, -+ sizeof(u32), DMA_BIDIRECTIONAL); -+ -+ *value = cpu_addr[offset_in_page >> 2]; -+ } else { -+ cpu_addr[offset_in_page >> 2] = *value; -+ -+ kbase_sync_single_for_device(kbdev, -+ kbase_dma_addr(target_page) + offset_in_page, -+ sizeof(u32), DMA_BIDIRECTIONAL); -+ } -+ -+ kunmap_atomic(cpu_addr); -+ return; -+ } -+ } -+ dev_warn(kbdev->dev, "Invalid GPU VA %x passed\n", gpu_addr); -+} -+ -+void kbase_csf_read_firmware_memory(struct kbase_device *kbdev, -+ u32 gpu_addr, u32 *value) -+{ -+ access_firmware_memory(kbdev, gpu_addr, value, true); -+} -+ -+void kbase_csf_update_firmware_memory(struct kbase_device *kbdev, -+ u32 gpu_addr, u32 value) -+{ -+ access_firmware_memory(kbdev, gpu_addr, &value, false); -+} -+ -+void kbase_csf_firmware_cs_input( -+ const struct kbase_csf_cmd_stream_info *const info, const u32 offset, -+ const u32 value) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ -+ dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); -+ input_page_write(info->input, offset, value); -+} -+ -+u32 kbase_csf_firmware_cs_input_read( -+ const struct kbase_csf_cmd_stream_info *const info, -+ const u32 offset) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ u32 const val = input_page_read(info->input, offset); -+ -+ dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+void kbase_csf_firmware_cs_input_mask( -+ const struct kbase_csf_cmd_stream_info *const info, const u32 offset, -+ const u32 value, const u32 mask) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ -+ dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", -+ offset, value, mask); -+ input_page_partial_write(info->input, offset, value, mask); -+} -+ -+u32 kbase_csf_firmware_cs_output( -+ const struct kbase_csf_cmd_stream_info *const info, const u32 offset) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ u32 const val = output_page_read(info->output, offset); -+ -+ dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+void kbase_csf_firmware_csg_input( -+ const struct kbase_csf_cmd_stream_group_info *const info, -+ const u32 offset, const u32 value) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ -+ dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", -+ offset, value); -+ input_page_write(info->input, offset, value); -+} -+ -+u32 kbase_csf_firmware_csg_input_read( -+ const struct kbase_csf_cmd_stream_group_info *const info, -+ const u32 offset) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ u32 const val = input_page_read(info->input, offset); -+ -+ dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+void kbase_csf_firmware_csg_input_mask( -+ const struct kbase_csf_cmd_stream_group_info *const info, -+ const u32 offset, const u32 value, const u32 mask) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ -+ dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", -+ offset, value, mask); -+ input_page_partial_write(info->input, offset, value, mask); -+} -+ -+u32 kbase_csf_firmware_csg_output( -+ const struct kbase_csf_cmd_stream_group_info *const info, -+ const u32 offset) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ u32 const val = output_page_read(info->output, offset); -+ -+ dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+void kbase_csf_firmware_global_input( -+ const struct kbase_csf_global_iface *const iface, const u32 offset, -+ const u32 value) -+{ -+ const struct kbase_device * const kbdev = iface->kbdev; -+ -+ dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); -+ input_page_write(iface->input, offset, value); -+} -+ -+void kbase_csf_firmware_global_input_mask( -+ const struct kbase_csf_global_iface *const iface, const u32 offset, -+ const u32 value, const u32 mask) -+{ -+ const struct kbase_device * const kbdev = iface->kbdev; -+ -+ dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", -+ offset, value, mask); -+ input_page_partial_write(iface->input, offset, value, mask); -+} -+ -+u32 kbase_csf_firmware_global_input_read( -+ const struct kbase_csf_global_iface *const iface, const u32 offset) -+{ -+ const struct kbase_device * const kbdev = iface->kbdev; -+ u32 const val = input_page_read(iface->input, offset); -+ -+ dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+u32 kbase_csf_firmware_global_output( -+ const struct kbase_csf_global_iface *const iface, const u32 offset) -+{ -+ const struct kbase_device * const kbdev = iface->kbdev; -+ u32 const val = output_page_read(iface->output, offset); -+ -+ dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+/** -+ * handle_internal_firmware_fatal - Handler for CS internal firmware fault. -+ * -+ * @kbdev: Pointer to kbase device -+ * -+ * Report group fatal error to user space for all GPU command queue groups -+ * in the device, terminate them and reset GPU. -+ */ -+static void handle_internal_firmware_fatal(struct kbase_device *const kbdev) -+{ -+ int as; -+ -+ for (as = 0; as < kbdev->nr_hw_address_spaces; as++) { -+ unsigned long flags; -+ struct kbase_context *kctx; -+ struct kbase_fault fault; -+ -+ if (as == MCU_AS_NR) -+ continue; -+ -+ /* Only handle the fault for an active address space. Lock is -+ * taken here to atomically get reference to context in an -+ * active address space and retain its refcount. -+ */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as); -+ -+ if (kctx) { -+ kbase_ctx_sched_retain_ctx_refcount(kctx); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } else { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ continue; -+ } -+ -+ fault = (struct kbase_fault) { -+ .status = GPU_EXCEPTION_TYPE_SW_FAULT_1, -+ }; -+ -+ kbase_csf_ctx_handle_fault(kctx, &fault); -+ kbase_ctx_sched_release_ctx_lock(kctx); -+ } -+ -+ if (kbase_prepare_to_reset_gpu(kbdev, -+ RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) -+ kbase_reset_gpu(kbdev); -+} -+ -+/** -+ * firmware_error_worker - Worker function for handling firmware internal error -+ * -+ * @data: Pointer to a work_struct embedded in kbase device. -+ * -+ * Handle the CS internal firmware error -+ */ -+static void firmware_error_worker(struct work_struct *const data) -+{ -+ struct kbase_device *const kbdev = -+ container_of(data, struct kbase_device, csf.fw_error_work); -+ -+ handle_internal_firmware_fatal(kbdev); -+} -+ -+static bool global_request_complete(struct kbase_device *const kbdev, -+ u32 const req_mask) -+{ -+ struct kbase_csf_global_iface *global_iface = -+ &kbdev->csf.global_iface; -+ bool complete = false; -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ -+ if ((kbase_csf_firmware_global_output(global_iface, GLB_ACK) & -+ req_mask) == -+ (kbase_csf_firmware_global_input_read(global_iface, GLB_REQ) & -+ req_mask)) -+ complete = true; -+ -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ return complete; -+} -+ -+static int wait_for_global_request(struct kbase_device *const kbdev, -+ u32 const req_mask) -+{ -+ const long wait_timeout = -+ kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ long remaining; -+ int err = 0; -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ global_request_complete(kbdev, req_mask), -+ wait_timeout); -+ -+ if (!remaining) { -+ dev_warn(kbdev->dev, "Timed out waiting for global request %x to complete", -+ req_mask); -+ err = -ETIMEDOUT; -+ } -+ -+ return err; -+} -+ -+static void set_global_request( -+ const struct kbase_csf_global_iface *const global_iface, -+ u32 const req_mask) -+{ -+ u32 glb_req; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(global_iface->kbdev); -+ -+ glb_req = kbase_csf_firmware_global_output(global_iface, GLB_ACK); -+ glb_req ^= req_mask; -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, glb_req, -+ req_mask); -+} -+ -+static void enable_endpoints_global( -+ const struct kbase_csf_global_iface *const global_iface, -+ u64 const shader_core_mask) -+{ -+ kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_LO, -+ shader_core_mask & U32_MAX); -+ kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_HI, -+ shader_core_mask >> 32); -+ -+ set_global_request(global_iface, GLB_REQ_CFG_ALLOC_EN_MASK); -+} -+ -+static void enable_shader_poweroff_timer(struct kbase_device *const kbdev, -+ const struct kbase_csf_global_iface *const global_iface) -+{ -+ u32 pwroff_reg; -+ -+ if (kbdev->csf.firmware_hctl_core_pwr) -+ pwroff_reg = -+ GLB_PWROFF_TIMER_TIMER_SOURCE_SET(DISABLE_GLB_PWROFF_TIMER, -+ GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); -+ else -+ pwroff_reg = kbdev->csf.mcu_core_pwroff_dur_count; -+ -+ kbase_csf_firmware_global_input(global_iface, GLB_PWROFF_TIMER, -+ pwroff_reg); -+ set_global_request(global_iface, GLB_REQ_CFG_PWROFF_TIMER_MASK); -+ -+ /* Save the programed reg value in its shadow field */ -+ kbdev->csf.mcu_core_pwroff_reg_shadow = pwroff_reg; -+ -+ dev_dbg(kbdev->dev, "GLB_PWROFF_TIMER set to 0x%.8x\n", pwroff_reg); -+} -+ -+static void set_timeout_global( -+ const struct kbase_csf_global_iface *const global_iface, -+ u64 const timeout) -+{ -+ kbase_csf_firmware_global_input(global_iface, GLB_PROGRESS_TIMER, -+ timeout / GLB_PROGRESS_TIMER_TIMEOUT_SCALE); -+ -+ set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK); -+} -+ -+static void global_init(struct kbase_device *const kbdev, u64 core_mask) -+{ -+ u32 const ack_irq_mask = GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK | -+ GLB_ACK_IRQ_MASK_PING_MASK | -+ GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK | -+ GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK | -+ GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK | -+ GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK | -+ GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK | -+ GLB_ACK_IRQ_MASK_IDLE_EVENT_MASK; -+ -+ const struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ -+ /* Set the coherency mode for protected mode execution */ -+ WARN_ON(kbdev->system_coherency == COHERENCY_ACE); -+ kbase_csf_firmware_global_input(global_iface, GLB_PROTM_COHERENCY, -+ kbdev->system_coherency); -+ -+ /* Update shader core allocation enable mask */ -+ enable_endpoints_global(global_iface, core_mask); -+ enable_shader_poweroff_timer(kbdev, global_iface); -+ -+ set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); -+ -+ /* Unmask the interrupts */ -+ kbase_csf_firmware_global_input(global_iface, -+ GLB_ACK_IRQ_MASK, ack_irq_mask); -+ -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+/** -+ * global_init_on_boot - Sends a global request to control various features. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface -+ * -+ * Currently only the request to enable endpoints and timeout for GPU progress -+ * timer is sent. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+static int global_init_on_boot(struct kbase_device *const kbdev) -+{ -+ unsigned long flags; -+ u64 core_mask; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ core_mask = kbase_pm_ca_get_core_mask(kbdev); -+ kbdev->csf.firmware_hctl_core_pwr = -+ kbase_pm_no_mcu_core_pwroff(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ global_init(kbdev, core_mask); -+ -+ return wait_for_global_request(kbdev, CSF_GLB_REQ_CFG_MASK); -+} -+ -+void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, -+ u64 core_mask) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbdev->csf.glb_init_request_pending = true; -+ kbdev->csf.firmware_hctl_core_pwr = -+ kbase_pm_no_mcu_core_pwroff(kbdev); -+ global_init(kbdev, core_mask); -+} -+ -+bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ WARN_ON(!kbdev->csf.glb_init_request_pending); -+ -+ if (global_request_complete(kbdev, CSF_GLB_REQ_CFG_MASK)) -+ kbdev->csf.glb_init_request_pending = false; -+ -+ return !kbdev->csf.glb_init_request_pending; -+} -+ -+void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, -+ bool update_core_pwroff_timer, bool update_core_mask, u64 core_mask) -+{ -+ unsigned long flags; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ if (update_core_mask) -+ enable_endpoints_global(&kbdev->csf.global_iface, core_mask); -+ if (update_core_pwroff_timer) -+ enable_shader_poweroff_timer(kbdev, &kbdev->csf.global_iface); -+ -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+bool kbase_csf_firmware_core_attr_updated(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ return global_request_complete(kbdev, GLB_REQ_CFG_ALLOC_EN_MASK | -+ GLB_REQ_CFG_PWROFF_TIMER_MASK); -+} -+ -+/** -+ * kbase_csf_firmware_reload_worker() - -+ * reload the fw image and re-enable the MCU -+ * @work: CSF Work item for reloading the firmware. -+ * -+ * This helper function will reload the firmware image and re-enable the MCU. -+ * It is supposed to be called after MCU(GPU) has been reset. -+ * Unlike the initial boot the firmware binary image is not parsed completely. -+ * Only the data sections, which were loaded in memory during the initial boot, -+ * are re-initialized either by zeroing them or copying their data from the -+ * firmware binary image. The memory allocation for the firmware pages and -+ * MMU programming is not needed for the reboot, presuming the firmware binary -+ * file on the filesystem would not change. -+ */ -+static void kbase_csf_firmware_reload_worker(struct work_struct *work) -+{ -+ struct kbase_device *kbdev = container_of(work, struct kbase_device, -+ csf.firmware_reload_work); -+ int err; -+ -+ dev_info(kbdev->dev, "reloading firmware"); -+ -+ /* Reload just the data sections from firmware binary image */ -+ err = reload_fw_data_sections(kbdev); -+ if (err) -+ return; -+ -+ kbase_csf_tl_reader_reset(&kbdev->timeline->csf_tl_reader); -+ -+ /* Reboot the firmware */ -+ kbase_csf_firmware_enable_mcu(kbdev); -+} -+ -+void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbdev->csf.firmware_reloaded = false; -+ -+ if (kbdev->csf.firmware_reload_needed) { -+ kbdev->csf.firmware_reload_needed = false; -+ queue_work(system_wq, &kbdev->csf.firmware_reload_work); -+ } else { -+ kbase_csf_firmware_enable_mcu(kbdev); -+ } -+} -+ -+void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) -+{ -+ u32 version; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (unlikely(!kbdev->csf.firmware_inited)) -+ return; -+ -+ /* Check firmware rebooted properly: we do not expect -+ * the version number to change with a running reboot. -+ */ -+ version = get_firmware_version(kbdev); -+ -+ if (version != kbdev->csf.global_iface.version) -+ dev_err(kbdev->dev, "Version check failed in firmware reboot."); -+ -+ KBASE_KTRACE_ADD(kbdev, FIRMWARE_REBOOT, NULL, 0u); -+ -+ /* Tell MCU state machine to transit to next state */ -+ kbdev->csf.firmware_reloaded = true; -+ kbase_pm_update_state(kbdev); -+} -+ -+static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_ms) -+{ -+#define HYSTERESIS_VAL_UNIT_SHIFT (10) -+ /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ -+ u64 freq = arch_timer_get_cntfrq(); -+ u64 dur_val = dur_ms; -+ u32 cnt_val_u32, reg_val_u32; -+ bool src_system_timestamp = freq > 0; -+ -+ if (!src_system_timestamp) { -+ /* Get the cycle_counter source alternative */ -+ spin_lock(&kbdev->pm.clk_rtm.lock); -+ if (kbdev->pm.clk_rtm.clks[0]) -+ freq = kbdev->pm.clk_rtm.clks[0]->clock_val; -+ else -+ dev_warn(kbdev->dev, "No GPU clock, unexpected intregration issue!"); -+ spin_unlock(&kbdev->pm.clk_rtm.lock); -+ -+ dev_info(kbdev->dev, "Can't get the timestamp frequency, " -+ "use cycle counter format with firmware idle hysteresis!"); -+ } -+ -+ /* Formula for dur_val = ((dur_ms/1000) * freq_HZ) >> 10) */ -+ dur_val = (dur_val * freq) >> HYSTERESIS_VAL_UNIT_SHIFT; -+ dur_val = div_u64(dur_val, 1000); -+ -+ /* Interface limits the value field to S32_MAX */ -+ cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; -+ -+ reg_val_u32 = GLB_IDLE_TIMER_TIMEOUT_SET(0, cnt_val_u32); -+ /* add the source flag */ -+ if (src_system_timestamp) -+ reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET(reg_val_u32, -+ GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); -+ else -+ reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET(reg_val_u32, -+ GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER); -+ -+ return reg_val_u32; -+} -+ -+u32 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev) -+{ -+ return kbdev->csf.gpu_idle_hysteresis_ms; -+} -+ -+u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u32 dur) -+{ -+ unsigned long flags; -+ const u32 hysteresis_val = convert_dur_to_idle_count(kbdev, dur); -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ kbdev->csf.gpu_idle_hysteresis_ms = dur; -+ kbdev->csf.gpu_idle_dur_count = hysteresis_val; -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ dev_dbg(kbdev->dev, "CSF set firmware idle hysteresis count-value: 0x%.8x", -+ hysteresis_val); -+ -+ return hysteresis_val; -+} -+ -+static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u32 dur_us) -+{ -+#define PWROFF_VAL_UNIT_SHIFT (10) -+ /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ -+ u64 freq = arch_timer_get_cntfrq(); -+ u64 dur_val = dur_us; -+ u32 cnt_val_u32, reg_val_u32; -+ bool src_system_timestamp = freq > 0; -+ -+ if (!src_system_timestamp) { -+ /* Get the cycle_counter source alternative */ -+ spin_lock(&kbdev->pm.clk_rtm.lock); -+ if (kbdev->pm.clk_rtm.clks[0]) -+ freq = kbdev->pm.clk_rtm.clks[0]->clock_val; -+ else -+ dev_warn(kbdev->dev, "No GPU clock, unexpected integration issue!"); -+ spin_unlock(&kbdev->pm.clk_rtm.lock); -+ -+ dev_info(kbdev->dev, "Can't get the timestamp frequency, " -+ "use cycle counter with MCU Core Poweroff timer!"); -+ } -+ -+ /* Formula for dur_val = ((dur_us/1e6) * freq_HZ) >> 10) */ -+ dur_val = (dur_val * freq) >> HYSTERESIS_VAL_UNIT_SHIFT; -+ dur_val = div_u64(dur_val, 1000000); -+ -+ /* Interface limits the value field to S32_MAX */ -+ cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; -+ -+ reg_val_u32 = GLB_PWROFF_TIMER_TIMEOUT_SET(0, cnt_val_u32); -+ /* add the source flag */ -+ if (src_system_timestamp) -+ reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET(reg_val_u32, -+ GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); -+ else -+ reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET(reg_val_u32, -+ GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER); -+ -+ return reg_val_u32; -+} -+ -+u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev) -+{ -+ return kbdev->csf.mcu_core_pwroff_dur_us; -+} -+ -+u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur) -+{ -+ unsigned long flags; -+ const u32 pwroff = convert_dur_to_core_pwroff_count(kbdev, dur); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->csf.mcu_core_pwroff_dur_us = dur; -+ kbdev->csf.mcu_core_pwroff_dur_count = pwroff; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ dev_dbg(kbdev->dev, "MCU Core Poweroff input update: 0x%.8x", pwroff); -+ -+ return pwroff; -+} -+ -+ -+int kbase_csf_firmware_early_init(struct kbase_device *kbdev) -+{ -+ init_waitqueue_head(&kbdev->csf.event_wait); -+ kbdev->csf.interrupt_received = false; -+ kbdev->csf.fw_timeout_ms = CSF_FIRMWARE_TIMEOUT_MS; -+ -+ INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); -+ INIT_LIST_HEAD(&kbdev->csf.firmware_config); -+ INIT_LIST_HEAD(&kbdev->csf.firmware_timeline_metadata); -+ INIT_LIST_HEAD(&kbdev->csf.firmware_trace_buffers.list); -+ INIT_WORK(&kbdev->csf.firmware_reload_work, -+ kbase_csf_firmware_reload_worker); -+ INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker); -+ -+ mutex_init(&kbdev->csf.reg_lock); -+ -+ return 0; -+} -+ -+int kbase_csf_firmware_init(struct kbase_device *kbdev) -+{ -+ const struct firmware *firmware; -+ const u32 magic = FIRMWARE_HEADER_MAGIC; -+ u8 version_major, version_minor; -+ u32 version_hash; -+ u32 entry_end_offset; -+ u32 entry_offset; -+ int ret; -+ -+ lockdep_assert_held(&kbdev->fw_load_lock); -+ -+ if (WARN_ON((kbdev->as_free & MCU_AS_BITMASK) == 0)) -+ return -EINVAL; -+ kbdev->as_free &= ~MCU_AS_BITMASK; -+ -+ ret = kbase_mmu_init(kbdev, &kbdev->csf.mcu_mmu, NULL, -+ BASE_MEM_GROUP_DEFAULT); -+ -+ if (ret != 0) { -+ /* Release the address space */ -+ kbdev->as_free |= MCU_AS_BITMASK; -+ return ret; -+ } -+ -+ kbdev->csf.gpu_idle_hysteresis_ms = FIRMWARE_IDLE_HYSTERESIS_TIME_MS; -+ kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count( -+ kbdev, FIRMWARE_IDLE_HYSTERESIS_TIME_MS); -+ -+ kbdev->csf.mcu_core_pwroff_dur_us = DEFAULT_GLB_PWROFF_TIMEOUT_US; -+ kbdev->csf.mcu_core_pwroff_dur_count = convert_dur_to_core_pwroff_count( -+ kbdev, DEFAULT_GLB_PWROFF_TIMEOUT_US); -+ -+ ret = kbase_mcu_shared_interface_region_tracker_init(kbdev); -+ if (ret != 0) { -+ dev_err(kbdev->dev, -+ "Failed to setup the rb tree for managing shared interface segment\n"); -+ goto error; -+ } -+ -+ if (request_firmware(&firmware, fw_name, kbdev->dev) != 0) { -+ dev_err(kbdev->dev, -+ "Failed to load firmware image '%s'\n", -+ fw_name); -+ ret = -ENOENT; -+ goto error; -+ } -+ -+ if (firmware->size < FIRMWARE_HEADER_LENGTH) { -+ dev_err(kbdev->dev, "Firmware too small\n"); -+ ret = -EINVAL; -+ goto error; -+ } -+ -+ if (memcmp(firmware->data, &magic, sizeof(magic)) != 0) { -+ dev_err(kbdev->dev, "Incorrect firmware magic\n"); -+ ret = -EINVAL; -+ goto error; -+ } -+ -+ version_minor = firmware->data[4]; -+ version_major = firmware->data[5]; -+ -+ if (version_major != FIRMWARE_HEADER_VERSION) { -+ dev_err(kbdev->dev, -+ "Firmware header version %d.%d not understood\n", -+ version_major, version_minor); -+ ret = -EINVAL; -+ goto error; -+ } -+ -+ memcpy(&version_hash, &firmware->data[8], sizeof(version_hash)); -+ -+ dev_notice(kbdev->dev, "Loading Mali firmware 0x%x", version_hash); -+ -+ memcpy(&entry_end_offset, &firmware->data[0x10], -+ sizeof(entry_end_offset)); -+ -+ if (entry_end_offset > firmware->size) { -+ dev_err(kbdev->dev, "Firmware image is truncated\n"); -+ ret = -EINVAL; -+ goto error; -+ } -+ -+ entry_offset = FIRMWARE_HEADER_LENGTH; -+ while (entry_offset < entry_end_offset) { -+ u32 header; -+ unsigned int size; -+ -+ memcpy(&header, &firmware->data[entry_offset], sizeof(header)); -+ -+ size = entry_size(header); -+ -+ ret = load_firmware_entry(kbdev, firmware, entry_offset, -+ header); -+ if (ret != 0) { -+ dev_err(kbdev->dev, "Failed to load firmware image\n"); -+ goto error; -+ } -+ entry_offset += size; -+ } -+ -+ if (!kbdev->csf.shared_interface) { -+ dev_err(kbdev->dev, "Shared interface region not found\n"); -+ ret = -EINVAL; -+ goto error; -+ } else { -+ ret = setup_shared_iface_static_region(kbdev); -+ if (ret != 0) { -+ dev_err(kbdev->dev, "Failed to insert a region for shared iface entry parsed from fw image\n"); -+ goto error; -+ } -+ } -+ -+ ret = kbase_csf_firmware_trace_buffers_init(kbdev); -+ if (ret != 0) { -+ dev_err(kbdev->dev, "Failed to initialize trace buffers\n"); -+ goto error; -+ } -+ -+ /* Make sure L2 cache is powered up */ -+ kbase_pm_wait_for_l2_powered(kbdev); -+ -+ /* Load the MMU tables into the selected address space */ -+ load_mmu_tables(kbdev); -+ -+ boot_csf_firmware(kbdev); -+ -+ ret = parse_capabilities(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = kbase_csf_doorbell_mapping_init(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = kbase_csf_scheduler_init(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = kbase_csf_setup_dummy_user_reg_page(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = kbase_csf_timeout_init(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = global_init_on_boot(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = kbase_csf_firmware_cfg_init(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ -+ /* Firmware loaded successfully */ -+ release_firmware(firmware); -+ KBASE_KTRACE_ADD(kbdev, FIRMWARE_BOOT, NULL, -+ (((u64)version_hash) << 32) | -+ (((u64)version_major) << 8) | version_minor); -+ return 0; -+ -+error: -+ kbase_csf_firmware_term(kbdev); -+ release_firmware(firmware); -+ return ret; -+} -+ -+void kbase_csf_firmware_term(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ int ret = 0; -+ -+ cancel_work_sync(&kbdev->csf.fw_error_work); -+ -+ ret = kbase_reset_gpu_wait(kbdev); -+ -+ WARN(ret, "failed to wait for GPU reset"); -+ -+ kbase_csf_firmware_cfg_term(kbdev); -+ -+ kbase_csf_timeout_term(kbdev); -+ -+ kbase_csf_free_dummy_user_reg_page(kbdev); -+ -+ kbase_csf_scheduler_term(kbdev); -+ -+ kbase_csf_doorbell_mapping_term(kbdev); -+ -+ /* Explicitly trigger the disabling of MCU through the state machine and -+ * wait for its completion. It may not have been disabled yet due to the -+ * power policy. -+ */ -+ kbdev->pm.backend.mcu_desired = false; -+ kbase_pm_wait_for_desired_state(kbdev); -+ -+ free_global_iface(kbdev); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->csf.firmware_inited = false; -+ if (WARN_ON(kbdev->pm.backend.mcu_state != KBASE_MCU_OFF)) { -+ kbdev->pm.backend.mcu_state = KBASE_MCU_OFF; -+ stop_csf_firmware(kbdev); -+ } -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ unload_mmu_tables(kbdev); -+ -+ kbase_csf_firmware_trace_buffers_term(kbdev); -+ -+ while (!list_empty(&kbdev->csf.firmware_interfaces)) { -+ struct kbase_csf_firmware_interface *interface; -+ -+ interface = -+ list_first_entry(&kbdev->csf.firmware_interfaces, -+ struct kbase_csf_firmware_interface, -+ node); -+ list_del(&interface->node); -+ -+ vunmap(interface->kernel_map); -+ if (interface->flags & CSF_FIRMWARE_ENTRY_PROTECTED) { -+ kbase_csf_protected_memory_free(kbdev, interface->pma, -+ interface->num_pages); -+ } else { -+ kbase_mem_pool_free_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ interface->num_pages, interface->phys, -+ true, false); -+ } -+ -+ kfree(interface->phys); -+ kfree(interface); -+ } -+ -+ while (!list_empty(&kbdev->csf.firmware_timeline_metadata)) { -+ struct firmware_timeline_metadata *metadata; -+ -+ metadata = list_first_entry( -+ &kbdev->csf.firmware_timeline_metadata, -+ struct firmware_timeline_metadata, -+ node); -+ list_del(&metadata->node); -+ -+ kfree(metadata); -+ } -+ -+#ifndef MALI_KBASE_BUILD -+ mali_kutf_fw_utf_entry_cleanup(kbdev); -+#endif -+ -+ /* This will also free up the region allocated for the shared interface -+ * entry parsed from the firmware image. -+ */ -+ kbase_mcu_shared_interface_region_tracker_term(kbdev); -+ -+ mutex_destroy(&kbdev->csf.reg_lock); -+ -+ kbase_mmu_term(kbdev, &kbdev->csf.mcu_mmu); -+ -+ /* Release the address space */ -+ kbdev->as_free |= MCU_AS_BITMASK; -+} -+ -+void kbase_csf_firmware_enable_gpu_idle_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ const u32 glb_req = -+ kbase_csf_firmware_global_input_read(global_iface, GLB_REQ); -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ -+ /* The scheduler is assumed to only call the enable when its internal -+ * state indicates that the idle timer has previously been disabled. So -+ * on entry the expected field values are: -+ * 1. GLOBAL_INPUT_BLOCK.GLB_REQ.IDLE_ENABLE: 0 -+ * 2. GLOBAL_OUTPUT_BLOCK.GLB_ACK.IDLE_ENABLE: 0, or, on 1 -> 0 -+ */ -+ -+ if (glb_req & GLB_REQ_IDLE_ENABLE_MASK) -+ dev_err(kbdev->dev, "Incoherent scheduler state on REQ_IDLE_ENABLE!"); -+ -+ kbase_csf_firmware_global_input(global_iface, GLB_IDLE_TIMER, -+ kbdev->csf.gpu_idle_dur_count); -+ -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, -+ GLB_REQ_REQ_IDLE_ENABLE, GLB_REQ_IDLE_ENABLE_MASK); -+ -+ dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x", -+ kbdev->csf.gpu_idle_dur_count); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+} -+ -+void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, -+ GLB_REQ_REQ_IDLE_DISABLE, -+ GLB_REQ_IDLE_DISABLE_MASK); -+ dev_dbg(kbdev->dev, "Sending request to disable gpu idle timer"); -+ -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+} -+ -+void kbase_csf_firmware_ping(struct kbase_device *const kbdev) -+{ -+ const struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ set_global_request(global_iface, GLB_REQ_PING_MASK); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+int kbase_csf_firmware_ping_wait(struct kbase_device *const kbdev) -+{ -+ kbase_csf_firmware_ping(kbdev); -+ return wait_for_global_request(kbdev, GLB_REQ_PING_MASK); -+} -+ -+int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, -+ u64 const timeout) -+{ -+ const struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ unsigned long flags; -+ int err; -+ -+ /* The 'reg_lock' is also taken and is held till the update is not -+ * complete, to ensure the update of timeout value by multiple Users -+ * gets serialized. -+ */ -+ mutex_lock(&kbdev->csf.reg_lock); -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ set_timeout_global(global_iface, timeout); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ err = wait_for_global_request(kbdev, GLB_REQ_CFG_PROGRESS_TIMER_MASK); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ return err; -+} -+ -+void kbase_csf_enter_protected_mode(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ unsigned long flags; -+ int err; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ set_global_request(global_iface, GLB_REQ_PROTM_ENTER_MASK); -+ dev_dbg(kbdev->dev, "Sending request to enter protected mode"); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ err = wait_for_global_request(kbdev, GLB_REQ_PROTM_ENTER_MASK); -+ -+ if (!err) { -+ unsigned long irq_flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->protected_mode = true; -+ kbase_ipa_protection_mode_switch_event(kbdev); -+ kbase_ipa_control_protm_entered(kbdev); -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &irq_flags); -+ kbase_hwcnt_backend_csf_protm_entered(&kbdev->hwcnt_gpu_iface); -+ kbase_csf_scheduler_spin_unlock(kbdev, irq_flags); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } -+} -+ -+void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ set_global_request(global_iface, GLB_REQ_HALT_MASK); -+ dev_dbg(kbdev->dev, "Sending request to HALT MCU"); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ unsigned long flags; -+ int err = 0; -+ -+ /* Ensure GPU is powered-up until we complete config update.*/ -+ kbase_csf_scheduler_pm_active(kbdev); -+ -+ /* The 'reg_lock' is also taken and is held till the update is -+ * complete, to ensure the config update gets serialized. -+ */ -+ mutex_lock(&kbdev->csf.reg_lock); -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ -+ set_global_request(global_iface, GLB_REQ_FIRMWARE_CONFIG_UPDATE_MASK); -+ dev_dbg(kbdev->dev, "Sending request for FIRMWARE_CONFIG_UPDATE"); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ err = wait_for_global_request(kbdev, -+ GLB_REQ_FIRMWARE_CONFIG_UPDATE_MASK); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ kbase_csf_scheduler_pm_idle(kbdev); -+ return err; -+} -+ -+/** -+ * copy_grp_and_stm - Copy CS and/or group data -+ * -+ * @iface: Global CSF interface provided by the firmware. -+ * @group_data: Pointer where to store all the group data -+ * (sequentially). -+ * @max_group_num: The maximum number of groups to be read. Can be 0, in -+ * which case group_data is unused. -+ * @stream_data: Pointer where to store all the CS data -+ * (sequentially). -+ * @max_total_stream_num: The maximum number of CSs to be read. -+ * Can be 0, in which case stream_data is unused. -+ * -+ * Return: Total number of CSs, summed across all groups. -+ */ -+static u32 copy_grp_and_stm( -+ const struct kbase_csf_global_iface * const iface, -+ struct basep_cs_group_control * const group_data, -+ u32 max_group_num, -+ struct basep_cs_stream_control * const stream_data, -+ u32 max_total_stream_num) -+{ -+ u32 i, total_stream_num = 0; -+ -+ if (WARN_ON((max_group_num > 0) && !group_data)) -+ max_group_num = 0; -+ -+ if (WARN_ON((max_total_stream_num > 0) && !stream_data)) -+ max_total_stream_num = 0; -+ -+ for (i = 0; i < iface->group_num; i++) { -+ u32 j; -+ -+ if (i < max_group_num) { -+ group_data[i].features = iface->groups[i].features; -+ group_data[i].stream_num = iface->groups[i].stream_num; -+ group_data[i].suspend_size = -+ iface->groups[i].suspend_size; -+ } -+ for (j = 0; j < iface->groups[i].stream_num; j++) { -+ if (total_stream_num < max_total_stream_num) -+ stream_data[total_stream_num].features = -+ iface->groups[i].streams[j].features; -+ total_stream_num++; -+ } -+ } -+ -+ return total_stream_num; -+} -+ -+u32 kbase_csf_firmware_get_glb_iface( -+ struct kbase_device *kbdev, -+ struct basep_cs_group_control *const group_data, -+ u32 const max_group_num, -+ struct basep_cs_stream_control *const stream_data, -+ u32 const max_total_stream_num, u32 *const glb_version, -+ u32 *const features, u32 *const group_num, u32 *const prfcnt_size, -+ u32 *instr_features) -+{ -+ const struct kbase_csf_global_iface * const iface = -+ &kbdev->csf.global_iface; -+ -+ if (WARN_ON(!glb_version) || WARN_ON(!features) || -+ WARN_ON(!group_num) || WARN_ON(!prfcnt_size) || -+ WARN_ON(!instr_features)) -+ return 0; -+ -+ *glb_version = iface->version; -+ *features = iface->features; -+ *group_num = iface->group_num; -+ *prfcnt_size = iface->prfcnt_size; -+ *instr_features = iface->instr_features; -+ -+ return copy_grp_and_stm(iface, group_data, max_group_num, -+ stream_data, max_total_stream_num); -+} -+ -+const char *kbase_csf_firmware_get_timeline_metadata( -+ struct kbase_device *kbdev, const char *name, size_t *size) -+{ -+ struct firmware_timeline_metadata *metadata; -+ -+ list_for_each_entry( -+ metadata, &kbdev->csf.firmware_timeline_metadata, node) { -+ if (!strcmp(metadata->name, name)) { -+ *size = metadata->size; -+ return metadata->data; -+ } -+ } -+ -+ *size = 0; -+ return NULL; -+} -+ -+int kbase_csf_firmware_mcu_shared_mapping_init( -+ struct kbase_device *kbdev, -+ unsigned int num_pages, -+ unsigned long cpu_map_properties, -+ unsigned long gpu_map_properties, -+ struct kbase_csf_mapping *csf_mapping) -+{ -+ struct tagged_addr *phys; -+ struct kbase_va_region *va_reg; -+ struct page **page_list; -+ void *cpu_addr; -+ int i, ret = 0; -+ pgprot_t cpu_map_prot = PAGE_KERNEL; -+ unsigned long gpu_map_prot; -+ -+ if (cpu_map_properties & PROT_READ) -+ cpu_map_prot = PAGE_KERNEL_RO; -+ -+ if (kbdev->system_coherency == COHERENCY_ACE) { -+ gpu_map_prot = -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); -+ } else { -+ gpu_map_prot = -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); -+ cpu_map_prot = pgprot_writecombine(cpu_map_prot); -+ }; -+ -+ phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); -+ if (!phys) -+ goto out; -+ -+ page_list = kmalloc_array(num_pages, sizeof(*page_list), GFP_KERNEL); -+ if (!page_list) -+ goto page_list_alloc_error; -+ -+ ret = kbase_mem_pool_alloc_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ num_pages, phys, false); -+ if (ret <= 0) -+ goto phys_mem_pool_alloc_error; -+ -+ for (i = 0; i < num_pages; i++) -+ page_list[i] = as_page(phys[i]); -+ -+ cpu_addr = vmap(page_list, num_pages, VM_MAP, cpu_map_prot); -+ if (!cpu_addr) -+ goto vmap_error; -+ -+ va_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, -+ num_pages, KBASE_REG_ZONE_MCU_SHARED); -+ if (!va_reg) -+ goto va_region_alloc_error; -+ -+ mutex_lock(&kbdev->csf.reg_lock); -+ ret = kbase_add_va_region_rbtree(kbdev, va_reg, 0, num_pages, 1); -+ va_reg->flags &= ~KBASE_REG_FREE; -+ if (ret) -+ goto va_region_add_error; -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ gpu_map_properties &= (KBASE_REG_GPU_RD | KBASE_REG_GPU_WR); -+ gpu_map_properties |= gpu_map_prot; -+ -+ ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, -+ va_reg->start_pfn, &phys[0], num_pages, -+ gpu_map_properties, KBASE_MEM_GROUP_CSF_FW); -+ if (ret) -+ goto mmu_insert_pages_error; -+ -+ kfree(page_list); -+ csf_mapping->phys = phys; -+ csf_mapping->cpu_addr = cpu_addr; -+ csf_mapping->va_reg = va_reg; -+ csf_mapping->num_pages = num_pages; -+ -+ return 0; -+ -+mmu_insert_pages_error: -+ mutex_lock(&kbdev->csf.reg_lock); -+ kbase_remove_va_region(va_reg); -+va_region_add_error: -+ kbase_free_alloced_region(va_reg); -+ mutex_unlock(&kbdev->csf.reg_lock); -+va_region_alloc_error: -+ vunmap(cpu_addr); -+vmap_error: -+ kbase_mem_pool_free_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ num_pages, phys, false, false); -+ -+phys_mem_pool_alloc_error: -+ kfree(page_list); -+page_list_alloc_error: -+ kfree(phys); -+out: -+ /* Zero-initialize the mapping to make sure that the termination -+ * function doesn't try to unmap or free random addresses. -+ */ -+ csf_mapping->phys = NULL; -+ csf_mapping->cpu_addr = NULL; -+ csf_mapping->va_reg = NULL; -+ csf_mapping->num_pages = 0; -+ -+ return -ENOMEM; -+} -+ -+void kbase_csf_firmware_mcu_shared_mapping_term( -+ struct kbase_device *kbdev, struct kbase_csf_mapping *csf_mapping) -+{ -+ if (csf_mapping->va_reg) { -+ mutex_lock(&kbdev->csf.reg_lock); -+ kbase_remove_va_region(csf_mapping->va_reg); -+ kbase_free_alloced_region(csf_mapping->va_reg); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ } -+ -+ if (csf_mapping->phys) { -+ kbase_mem_pool_free_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ csf_mapping->num_pages, csf_mapping->phys, false, -+ false); -+ } -+ -+ vunmap(csf_mapping->cpu_addr); -+ kfree(csf_mapping->phys); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware.h -new file mode 100644 -index 0000000..60d7065 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware.h -@@ -0,0 +1,811 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_FIRMWARE_H_ -+#define _KBASE_CSF_FIRMWARE_H_ -+ -+#include "device/mali_kbase_device.h" -+#include <uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h> -+ -+/* -+ * PAGE_KERNEL_RO was only defined on 32bit ARM in 4.19 in: -+ * Commit a3266bd49c721e2e0a71f352d83713fbd60caadb -+ * Author: Luis R. Rodriguez <mcgrof@kernel.org> -+ * Date: Fri Aug 17 15:46:29 2018 -0700 -+ * -+ * mm: provide a fallback for PAGE_KERNEL_RO for architectures -+ * -+ * Some architectures do not define certain PAGE_KERNEL_* flags, this is -+ * either because: -+ * -+ * a) The way to implement some of these flags is *not yet ported*, or -+ * b) The architecture *has no way* to describe them -+ * -+ * [snip] -+ * -+ * This can be removed once support of 32bit ARM kernels predating 4.19 is no -+ * longer required. -+ */ -+#ifndef PAGE_KERNEL_RO -+#define PAGE_KERNEL_RO PAGE_KERNEL -+#endif -+ -+/* Address space number to claim for the firmware. */ -+#define MCU_AS_NR 0 -+#define MCU_AS_BITMASK (1 << MCU_AS_NR) -+ -+/* Number of available Doorbells */ -+#define CSF_NUM_DOORBELL ((u8)24) -+ -+/* Offset to the first HW doorbell page */ -+#define CSF_HW_DOORBELL_PAGE_OFFSET ((u32)0x80000) -+ -+/* Size of HW Doorbell page, used to calculate the offset to subsequent pages */ -+#define CSF_HW_DOORBELL_PAGE_SIZE ((u32)0x10000) -+ -+/* Doorbell 0 is used by the driver. */ -+#define CSF_KERNEL_DOORBELL_NR ((u32)0) -+ -+/* Offset of name inside a trace buffer entry in the firmware image */ -+#define TRACE_BUFFER_ENTRY_NAME_OFFSET (0x1C) -+ -+/* All implementations of the host interface with major version 0 must comply -+ * with these restrictions: -+ */ -+/* GLB_GROUP_NUM: At least 3 CSGs, but no more than 31 */ -+#define MIN_SUPPORTED_CSGS 3 -+#define MAX_SUPPORTED_CSGS 31 -+/* GROUP_STREAM_NUM: At least 8 CSs per CSG, but no more than 32 */ -+#define MIN_SUPPORTED_STREAMS_PER_GROUP 8 -+/* Maximum CSs per csg. */ -+#define MAX_SUPPORTED_STREAMS_PER_GROUP 32 -+ -+/* Waiting timeout for status change acknowledgment, in milliseconds */ -+#define CSF_FIRMWARE_TIMEOUT_MS (3000) /* Relaxed to 3000ms from 800ms due to Android */ -+ -+struct kbase_device; -+ -+ -+/** -+ * struct kbase_csf_mapping - Memory mapping for CSF memory. -+ * @phys: Physical memory allocation used by the mapping. -+ * @cpu_addr: Starting CPU address for the mapping. -+ * @va_reg: GPU virtual address region for the mapping. -+ * @num_pages: Size of the mapping, in memory pages. -+ */ -+struct kbase_csf_mapping { -+ struct tagged_addr *phys; -+ void *cpu_addr; -+ struct kbase_va_region *va_reg; -+ unsigned int num_pages; -+}; -+ -+/** -+ * struct kbase_csf_trace_buffers - List and state of firmware trace buffers. -+ * @list: List of trace buffers descriptors. -+ * @mcu_rw: Metadata for the MCU shared memory mapping used for -+ * GPU-readable,writable/CPU-writable variables. -+ * @mcu_write: Metadata for the MCU shared memory mapping used for -+ * GPU-writable/CPU-readable variables. -+ */ -+struct kbase_csf_trace_buffers { -+ struct list_head list; -+ struct kbase_csf_mapping mcu_rw; -+ struct kbase_csf_mapping mcu_write; -+}; -+ -+/** -+ * struct kbase_csf_cmd_stream_info - CSI provided by the firmware. -+ * -+ * @kbdev: Address of the instance of a GPU platform device that implements -+ * this interface. -+ * @features: Bit field of CS features (e.g. which types of jobs -+ * are supported). Bits 7:0 specify the number of work registers(-1). -+ * Bits 11:8 specify the number of scoreboard entries(-1). -+ * @input: Address of CSI input page. -+ * @output: Address of CSI output page. -+ */ -+struct kbase_csf_cmd_stream_info { -+ struct kbase_device *kbdev; -+ u32 features; -+ void *input; -+ void *output; -+}; -+ -+/** -+ * kbase_csf_firmware_cs_input() - Set a word in a CS's input page -+ * -+ * @info: CSI provided by the firmware. -+ * @offset: Offset of the word to be written, in bytes. -+ * @value: Value to be written. -+ */ -+void kbase_csf_firmware_cs_input( -+ const struct kbase_csf_cmd_stream_info *info, u32 offset, u32 value); -+ -+/** -+ * kbase_csf_firmware_cs_input_read() - Read a word in a CS's input page -+ * -+ * Return: Value of the word read from the CS's input page. -+ * -+ * @info: CSI provided by the firmware. -+ * @offset: Offset of the word to be read, in bytes. -+ */ -+u32 kbase_csf_firmware_cs_input_read( -+ const struct kbase_csf_cmd_stream_info *const info, const u32 offset); -+ -+/** -+ * kbase_csf_firmware_cs_input_mask() - Set part of a word in a CS's input page -+ * -+ * @info: CSI provided by the firmware. -+ * @offset: Offset of the word to be modified, in bytes. -+ * @value: Value to be written. -+ * @mask: Bitmask with the bits to be modified set. -+ */ -+void kbase_csf_firmware_cs_input_mask( -+ const struct kbase_csf_cmd_stream_info *info, u32 offset, -+ u32 value, u32 mask); -+ -+/** -+ * kbase_csf_firmware_cs_output() - Read a word in a CS's output page -+ * -+ * Return: Value of the word read from the CS's output page. -+ * -+ * @info: CSI provided by the firmware. -+ * @offset: Offset of the word to be read, in bytes. -+ */ -+u32 kbase_csf_firmware_cs_output( -+ const struct kbase_csf_cmd_stream_info *info, u32 offset); -+/** -+ * struct kbase_csf_cmd_stream_group_info - CSG interface provided by the -+ * firmware. -+ * -+ * @kbdev: Address of the instance of a GPU platform device that implements -+ * this interface. -+ * @features: Bit mask of features. Reserved bits should be 0, and should -+ * be ignored. -+ * @input: Address of global interface input page. -+ * @output: Address of global interface output page. -+ * @suspend_size: Size in bytes for normal suspend buffer for the CSG -+ * @protm_suspend_size: Size in bytes for protected mode suspend buffer -+ * for the CSG. -+ * @stream_num: Number of CSs in the CSG. -+ * @stream_stride: Stride in bytes in JASID0 virtual address between -+ * CS capability structures. -+ * @streams: Address of an array of CS capability structures. -+ */ -+struct kbase_csf_cmd_stream_group_info { -+ struct kbase_device *kbdev; -+ u32 features; -+ void *input; -+ void *output; -+ u32 suspend_size; -+ u32 protm_suspend_size; -+ u32 stream_num; -+ u32 stream_stride; -+ struct kbase_csf_cmd_stream_info *streams; -+}; -+ -+/** -+ * kbase_csf_firmware_csg_input() - Set a word in a CSG's input page -+ * -+ * @info: CSG interface provided by the firmware. -+ * @offset: Offset of the word to be written, in bytes. -+ * @value: Value to be written. -+ */ -+void kbase_csf_firmware_csg_input( -+ const struct kbase_csf_cmd_stream_group_info *info, u32 offset, -+ u32 value); -+ -+/** -+ * kbase_csf_firmware_csg_input_read() - Read a word in a CSG's input page -+ * -+ * Return: Value of the word read from the CSG's input page. -+ * -+ * @info: CSG interface provided by the firmware. -+ * @offset: Offset of the word to be read, in bytes. -+ */ -+u32 kbase_csf_firmware_csg_input_read( -+ const struct kbase_csf_cmd_stream_group_info *info, u32 offset); -+ -+/** -+ * kbase_csf_firmware_csg_input_mask() - Set part of a word in a CSG's -+ * input page -+ * -+ * @info: CSG interface provided by the firmware. -+ * @offset: Offset of the word to be modified, in bytes. -+ * @value: Value to be written. -+ * @mask: Bitmask with the bits to be modified set. -+ */ -+void kbase_csf_firmware_csg_input_mask( -+ const struct kbase_csf_cmd_stream_group_info *info, u32 offset, -+ u32 value, u32 mask); -+ -+/** -+ * kbase_csf_firmware_csg_output()- Read a word in a CSG's output page -+ * -+ * Return: Value of the word read from the CSG's output page. -+ * -+ * @info: CSG interface provided by the firmware. -+ * @offset: Offset of the word to be read, in bytes. -+ */ -+u32 kbase_csf_firmware_csg_output( -+ const struct kbase_csf_cmd_stream_group_info *info, u32 offset); -+ -+/** -+ * struct kbase_csf_global_iface - Global CSF interface -+ * provided by the firmware. -+ * -+ * @kbdev: Address of the instance of a GPU platform device that implements -+ * this interface. -+ * @version: Bits 31:16 hold the major version number and 15:0 hold the minor -+ * version number. A higher minor version is backwards-compatible -+ * with a lower minor version for the same major version. -+ * @features: Bit mask of features (e.g. whether certain types of job can -+ * be suspended). Reserved bits should be 0, and should be ignored. -+ * @input: Address of global interface input page. -+ * @output: Address of global interface output page. -+ * @group_num: Number of CSGs supported. -+ * @group_stride: Stride in bytes in JASID0 virtual address between -+ * CSG capability structures. -+ * @prfcnt_size: Performance counters size. -+ * @instr_features: Instrumentation features. (csf >= 1.1.0) -+ * @groups: Address of an array of CSG capability structures. -+ */ -+struct kbase_csf_global_iface { -+ struct kbase_device *kbdev; -+ u32 version; -+ u32 features; -+ void *input; -+ void *output; -+ u32 group_num; -+ u32 group_stride; -+ u32 prfcnt_size; -+ u32 instr_features; -+ struct kbase_csf_cmd_stream_group_info *groups; -+}; -+ -+/** -+ * kbase_csf_firmware_global_input() - Set a word in the global input page -+ * -+ * @iface: CSF interface provided by the firmware. -+ * @offset: Offset of the word to be written, in bytes. -+ * @value: Value to be written. -+ */ -+void kbase_csf_firmware_global_input( -+ const struct kbase_csf_global_iface *iface, u32 offset, u32 value); -+ -+/** -+ * kbase_csf_firmware_global_input_mask() - Set part of a word in the global -+ * input page -+ * -+ * @iface: CSF interface provided by the firmware. -+ * @offset: Offset of the word to be modified, in bytes. -+ * @value: Value to be written. -+ * @mask: Bitmask with the bits to be modified set. -+ */ -+void kbase_csf_firmware_global_input_mask( -+ const struct kbase_csf_global_iface *iface, u32 offset, -+ u32 value, u32 mask); -+ -+/** -+ * kbase_csf_firmware_global_input_read() - Read a word in a global input page -+ * -+ * Return: Value of the word read from the global input page. -+ * -+ * @info: CSG interface provided by the firmware. -+ * @offset: Offset of the word to be read, in bytes. -+ */ -+u32 kbase_csf_firmware_global_input_read( -+ const struct kbase_csf_global_iface *info, u32 offset); -+ -+/** -+ * kbase_csf_firmware_global_output() - Read a word in the global output page -+ * -+ * Return: Value of the word read from the global output page. -+ * -+ * @iface: CSF interface provided by the firmware. -+ * @offset: Offset of the word to be read, in bytes. -+ */ -+u32 kbase_csf_firmware_global_output( -+ const struct kbase_csf_global_iface *iface, u32 offset); -+ -+/* Calculate the offset to the Hw doorbell page corresponding to the -+ * doorbell number. -+ */ -+static u32 csf_doorbell_offset(int doorbell_nr) -+{ -+ WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); -+ -+ return CSF_HW_DOORBELL_PAGE_OFFSET + -+ (doorbell_nr * CSF_HW_DOORBELL_PAGE_SIZE); -+} -+ -+static inline void kbase_csf_ring_doorbell(struct kbase_device *kbdev, -+ int doorbell_nr) -+{ -+ WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); -+ -+ kbase_reg_write(kbdev, csf_doorbell_offset(doorbell_nr), (u32)1); -+} -+ -+/** -+ * kbase_csf_read_firmware_memory - Read a value in a GPU address -+ * -+ * This function read a value in a GPU address that belongs to -+ * a private firmware memory region. The function assumes that the location -+ * is not permanently mapped on the CPU address space, therefore it maps it -+ * and then unmaps it to access it independently. -+ * -+ * @kbdev: Device pointer -+ * @gpu_addr: GPU address to read -+ * @value: output pointer to which the read value will be written. -+ */ -+void kbase_csf_read_firmware_memory(struct kbase_device *kbdev, -+ u32 gpu_addr, u32 *value); -+ -+/** -+ * kbase_csf_update_firmware_memory - Write a value in a GPU address -+ * -+ * This function writes a given value in a GPU address that belongs to -+ * a private firmware memory region. The function assumes that the destination -+ * is not permanently mapped on the CPU address space, therefore it maps it -+ * and then unmaps it to access it independently. -+ * -+ * @kbdev: Device pointer -+ * @gpu_addr: GPU address to write -+ * @value: Value to write -+ */ -+void kbase_csf_update_firmware_memory(struct kbase_device *kbdev, -+ u32 gpu_addr, u32 value); -+ -+/** -+ * kbase_csf_firmware_early_init() - Early initializatin for the firmware. -+ * @kbdev: Kbase device -+ * -+ * Initialize resources related to the firmware. Must be called at kbase probe. -+ * -+ * Return: 0 if successful, negative error code on failure -+ */ -+int kbase_csf_firmware_early_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_init() - Load the firmware for the CSF MCU -+ * @kbdev: Kbase device -+ * -+ * Request the firmware from user space and load it into memory. -+ * -+ * Return: 0 if successful, negative error code on failure -+ */ -+int kbase_csf_firmware_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_term() - Unload the firmware -+ * @kbdev: Kbase device -+ * -+ * Frees the memory allocated by kbase_csf_firmware_init() -+ */ -+void kbase_csf_firmware_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_ping - Send the ping request to firmware. -+ * -+ * The function sends the ping request to firmware. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_firmware_ping(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_ping_wait - Send the ping request to firmware and waits. -+ * -+ * The function sends the ping request to firmware and waits to confirm it is -+ * alive. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_firmware_ping_wait(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_set_timeout - Set a hardware endpoint progress timeout. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @timeout: The maximum number of GPU cycles that is allowed to elapse -+ * without forward progress before the driver terminates a GPU -+ * command queue group. -+ * -+ * Configures the progress timeout value used by the firmware to decide -+ * when to report that a task is not making progress on an endpoint. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_firmware_set_timeout(struct kbase_device *kbdev, u64 timeout); -+ -+/** -+ * kbase_csf_enter_protected_mode - Send the Global request to firmware to -+ * enter protected mode and wait for its -+ * completion. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_enter_protected_mode(struct kbase_device *kbdev); -+ -+static inline bool kbase_csf_firmware_mcu_halted(struct kbase_device *kbdev) -+{ -+ return (kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS)) == -+ MCU_STATUS_HALTED); -+} -+ -+/** -+ * kbase_csf_firmware_trigger_mcu_halt - Send the Global request to firmware to -+ * halt its operation and bring itself -+ * into a known internal state for warm -+ * boot later. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_enable_mcu - Send the command to enable MCU -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+static inline void kbase_csf_firmware_enable_mcu(struct kbase_device *kbdev) -+{ -+ /* Trigger the boot of MCU firmware, Use the AUTO mode as -+ * otherwise on fast reset, to exit protected mode, MCU will -+ * not reboot by itself to enter normal mode. -+ */ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), MCU_CNTRL_AUTO); -+} -+ -+/** -+ * kbase_csf_firmware_disable_mcu - Send the command to disable MCU -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+static inline void kbase_csf_firmware_disable_mcu(struct kbase_device *kbdev) -+{ -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(MCU_CONTROL), MCU_CNTRL_DISABLE); -+} -+ -+/** -+ * kbase_csf_firmware_disable_mcu_wait - Wait for the MCU to reach disabled -+ * status. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev); -+ -+/** -+ * kbase_trigger_firmware_reload - Trigger the reboot of MCU firmware, for the -+ * cold boot case firmware image would be -+ * reloaded from filesystem into memory. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_reload_completed - The reboot of MCU firmware has -+ * completed. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_global_reinit - Send the Global configuration requests -+ * after the reboot of MCU firmware. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @core_mask: Mask of the enabled shader cores. -+ */ -+void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, -+ u64 core_mask); -+ -+/** -+ * kbase_csf_firmware_global_reinit_complete - Check the Global configuration -+ * requests, sent after the reboot of MCU firmware, have -+ * completed or not. -+ * -+ * Return: true if the Global configuration requests completed otherwise false. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_update_core_attr - Send the Global configuration request -+ * to update the requested core attribute -+ * changes. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @update_core_pwroff_timer: If true, signal the firmware needs to update -+ * the MCU power-off timer value. -+ * @update_core_mask: If true, need to do the core_mask update with -+ * the supplied core_mask value. -+ * @core_mask: New core mask value if update_core_mask is true, -+ * otherwise unused. -+ */ -+void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, -+ bool update_core_pwroff_timer, bool update_core_mask, u64 core_mask); -+ -+/** -+ * kbase_csf_firmware_core_attr_updated - Check the Global configuration -+ * request has completed or not, that was sent to update -+ * the core attributes. -+ * -+ * Return: true if the Global configuration request to update the core -+ * attributes has completed, otherwise false. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+bool kbase_csf_firmware_core_attr_updated(struct kbase_device *kbdev); -+ -+/** -+ * Request the global control block of CSF interface capabilities -+ * -+ * Return: Total number of CSs, summed across all groups. -+ * -+ * @kbdev: Kbase device. -+ * @group_data: Pointer where to store all the group data -+ * (sequentially). -+ * @max_group_num: The maximum number of groups to be read. -+ * Can be 0, in which case group_data is unused. -+ * @stream_data: Pointer where to store all the CS data -+ * (sequentially). -+ * @max_total_stream_num: The maximum number of CSs to be read. -+ * Can be 0, in which case stream_data is unused. -+ * @glb_version: Where to store the global interface version. -+ * @features: Where to store a bit mask of features (e.g. -+ * whether certain types of job can be suspended). -+ * @group_num: Where to store the number of CSGs -+ * supported. -+ * @prfcnt_size: Where to store the size of CSF performance counters, -+ * in bytes. Bits 31:16 hold the size of firmware -+ * performance counter data and 15:0 hold the size of -+ * hardware performance counter data. -+ * @instr_features: Instrumentation features. Bits 7:4 hold the max size -+ * of events. Bits 3:0 hold the offset update rate. -+ * (csf >= 1,1,0) -+ */ -+u32 kbase_csf_firmware_get_glb_iface( -+ struct kbase_device *kbdev, struct basep_cs_group_control *group_data, -+ u32 max_group_num, struct basep_cs_stream_control *stream_data, -+ u32 max_total_stream_num, u32 *glb_version, u32 *features, -+ u32 *group_num, u32 *prfcnt_size, u32 *instr_features); -+ -+/** -+ * Get CSF firmware header timeline metadata content -+ * -+ * Return: The firmware timeline metadata content which match @p name. -+ * -+ * @kbdev: Kbase device. -+ * @name: Name of the metadata which metadata content to be returned. -+ * @size: Metadata size if specified metadata found. -+ */ -+const char *kbase_csf_firmware_get_timeline_metadata(struct kbase_device *kbdev, -+ const char *name, size_t *size); -+ -+/** -+ * kbase_csf_firmware_mcu_shared_mapping_init - -+ * Allocate and map MCU shared memory. -+ * -+ * This helper function allocates memory and maps it on both the CPU -+ * and the GPU address spaces. Most of the properties of the mapping -+ * are implicit and will be automatically determined by the function, -+ * e.g. whether memory is cacheable. -+ * -+ * The client is only expected to specify whether the mapping is readable -+ * or writable in the CPU and the GPU address spaces; any other flag -+ * will be ignored by the function. -+ * -+ * Return: 0 if success, or an error code on failure. -+ * -+ * @kbdev: Kbase device the memory mapping shall belong to. -+ * @num_pages: Number of memory pages to map. -+ * @cpu_map_properties: Either PROT_READ or PROT_WRITE. -+ * @gpu_map_properties: Either KBASE_REG_GPU_RD or KBASE_REG_GPU_WR. -+ * @csf_mapping: Object where to write metadata for the memory mapping. -+ */ -+int kbase_csf_firmware_mcu_shared_mapping_init( -+ struct kbase_device *kbdev, -+ unsigned int num_pages, -+ unsigned long cpu_map_properties, -+ unsigned long gpu_map_properties, -+ struct kbase_csf_mapping *csf_mapping); -+ -+/** -+ * kbase_csf_firmware_mcu_shared_mapping_term - Unmap and free MCU shared memory. -+ * -+ * @kbdev: Device pointer. -+ * @csf_mapping: Metadata of the memory mapping to terminate. -+ */ -+void kbase_csf_firmware_mcu_shared_mapping_term( -+ struct kbase_device *kbdev, struct kbase_csf_mapping *csf_mapping); -+ -+#ifndef MALI_KBASE_BUILD -+/** -+ * mali_kutf_process_fw_utf_entry() - Process the "Firmware UTF tests" section -+ * -+ * Read "Firmware UTF tests" section from the firmware image and create -+ * necessary kutf app+suite+tests. -+ * -+ * Return: 0 if successful, negative error code on failure. In both cases -+ * caller will have to invoke mali_kutf_fw_utf_entry_cleanup for the cleanup -+ * -+ * @kbdev: Kbase device structure -+ * @fw_data: Pointer to the start of firmware binary image loaded from disk -+ * @fw_size: Size (in bytes) of the firmware image -+ * @entry: Pointer to the start of the section -+ */ -+int mali_kutf_process_fw_utf_entry(struct kbase_device *kbdev, -+ const void *fw_data, size_t fw_size, const u32 *entry); -+ -+/** -+ * mali_kutf_fw_utf_entry_cleanup() - Remove the Fw UTF tests debugfs entries -+ * -+ * Destroy the kutf apps+suites+tests created on parsing "Firmware UTF tests" -+ * section from the firmware image. -+ * -+ * @kbdev: Kbase device structure -+ */ -+void mali_kutf_fw_utf_entry_cleanup(struct kbase_device *kbdev); -+#endif -+ -+#ifdef CONFIG_MALI_DEBUG -+extern bool fw_debug; -+#endif -+ -+static inline long kbase_csf_timeout_in_jiffies(const unsigned int msecs) -+{ -+#ifdef CONFIG_MALI_DEBUG -+ return (fw_debug ? MAX_SCHEDULE_TIMEOUT : msecs_to_jiffies(msecs)); -+#else -+ return msecs_to_jiffies(msecs); -+#endif -+} -+ -+/** -+ * kbase_csf_firmware_enable_gpu_idle_timer() - Activate the idle hysteresis -+ * monitoring operation -+ * -+ * Program the firmware interface with its configured hysteresis count value -+ * and enable the firmware to act on it. The Caller is -+ * assumed to hold the kbdev->csf.scheduler.interrupt_lock. -+ * -+ * @kbdev: Kbase device structure -+ */ -+void kbase_csf_firmware_enable_gpu_idle_timer(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_disable_gpu_idle_timer() - Disable the idle time -+ * hysteresis monitoring operation -+ * -+ * Program the firmware interface to disable the idle hysteresis timer. The -+ * Caller is assumed to hold the kbdev->csf.scheduler.interrupt_lock. -+ * -+ * @kbdev: Kbase device structure -+ */ -+void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_get_gpu_idle_hysteresis_time - Get the firmware GPU idle -+ * detection hysteresis duration -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Return: the internally recorded hysteresis (nominal) value. -+ */ -+u32 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_set_gpu_idle_hysteresis_time - Set the firmware GPU idle -+ * detection hysteresis duration -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @dur: The duration value (unit: milliseconds) for the configuring -+ * hysteresis field for GPU idle detection -+ * -+ * The supplied value will be recorded internally without any change. But the -+ * actual field value will be subject to hysteresis source frequency scaling -+ * and maximum value limiting. The default source will be SYSTEM_TIMESTAMP -+ * counter. But in case the platform is not able to supply it, the GPU -+ * CYCLE_COUNTER source will be used as an alternative. Bit-31 on the -+ * returned value is the source configuration flag, and it is set to '1' -+ * when CYCLE_COUNTER alternative source is used. -+ * -+ * Return: the actual internally configured hysteresis field value. -+ */ -+u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u32 dur); -+ -+/** -+ * kbase_csf_firmware_get_mcu_core_pwroff_time - Get the MCU core power-off -+ * time value -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Return: the internally recorded MCU core power-off (nominal) value. The unit -+ * of the value is in micro-seconds. -+ */ -+u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_set_mcu_core_pwroff_time - Set the MCU core power-off -+ * time value -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @dur: The duration value (unit: micro-seconds) for configuring MCU -+ * core power-off timer, when the shader cores' power -+ * transitions are delegated to the MCU (normal operational -+ * mode) -+ * -+ * The supplied value will be recorded internally without any change. But the -+ * actual field value will be subject to core power-off timer source frequency -+ * scaling and maximum value limiting. The default source will be -+ * SYSTEM_TIMESTAMP counter. But in case the platform is not able to supply it, -+ * the GPU CYCLE_COUNTER source will be used as an alternative. Bit-31 on the -+ * returned value is the source configuration flag, and it is set to '1' -+ * when CYCLE_COUNTER alternative source is used. -+ * -+ * The configured MCU core power-off timer will only have effect when the host -+ * driver has delegated the shader cores' power management to MCU. -+ * -+ * Return: the actual internal core power-off timer value in register defined -+ * format. -+ */ -+u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur); -+ -+/** -+ * kbase_csf_interface_version - Helper function to build the full firmware -+ * interface version in a format compatible with -+ * with GLB_VERSION register -+ * -+ * @major: major version of csf interface -+ * @minor: minor version of csf interface -+ * @patch: patch version of csf interface -+ * -+ * Return: firmware interface version -+ */ -+static inline u32 kbase_csf_interface_version(u32 major, u32 minor, u32 patch) -+{ -+ return ((major << GLB_VERSION_MAJOR_SHIFT) | -+ (minor << GLB_VERSION_MINOR_SHIFT) | -+ (patch << GLB_VERSION_PATCH_SHIFT)); -+} -+ -+/** -+ * kbase_csf_trigger_firmware_config_update - Send a firmware config update. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Any changes done to firmware configuration entry or tracebuffer entry -+ * requires a GPU silent reset to reflect the configuration changes -+ * requested, but if Firmware.header.entry.bit(30) is set then we can request a -+ * FIRMWARE_CONFIG_UPDATE rather than doing a silent reset. -+ * -+ * Return: 0 if success, or negative error code on failure. -+ */ -+int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev); -+#endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_cfg.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_cfg.c -new file mode 100644 -index 0000000..f00acb1 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_cfg.c -@@ -0,0 +1,327 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include "mali_kbase_csf_firmware_cfg.h" -+#include <mali_kbase_reset_gpu.h> -+ -+#if CONFIG_SYSFS -+#define CSF_FIRMWARE_CFG_SYSFS_DIR_NAME "firmware_config" -+ -+/** -+ * struct firmware_config - Configuration item within the MCU firmware -+ * -+ * The firmware may expose configuration options. Each option has a name, the -+ * address where the option is controlled and the minimum and maximum values -+ * that the option can take. -+ * -+ * @node: List head linking all options to -+ * kbase_device:csf.firmware_config -+ * @kbdev: Pointer to the Kbase device -+ * @kobj: Kobject corresponding to the sysfs sub-directory, -+ * inside CSF_FIRMWARE_CFG_SYSFS_DIR_NAME directory, -+ * representing the configuration option @name. -+ * @kobj_inited: kobject initialization state -+ * @updatable: Indicates whether config items can be updated with -+ * FIRMWARE_CONFIG_UPDATE -+ * @name: NUL-terminated string naming the option -+ * @address: The address in the firmware image of the configuration option -+ * @min: The lowest legal value of the configuration option -+ * @max: The maximum legal value of the configuration option -+ * @cur_val: The current value of the configuration option -+ */ -+struct firmware_config { -+ struct list_head node; -+ struct kbase_device *kbdev; -+ struct kobject kobj; -+ bool kobj_inited; -+ bool updatable; -+ char *name; -+ u32 address; -+ u32 min; -+ u32 max; -+ u32 cur_val; -+}; -+ -+#define FW_CFG_ATTR(_name, _mode) \ -+ struct attribute fw_cfg_attr_##_name = { \ -+ .name = __stringify(_name), \ -+ .mode = VERIFY_OCTAL_PERMISSIONS(_mode), \ -+ } -+ -+static FW_CFG_ATTR(min, S_IRUGO); -+static FW_CFG_ATTR(max, S_IRUGO); -+static FW_CFG_ATTR(cur, S_IRUGO | S_IWUSR); -+ -+static void fw_cfg_kobj_release(struct kobject *kobj) -+{ -+ struct firmware_config *config = -+ container_of(kobj, struct firmware_config, kobj); -+ -+ kfree(config); -+} -+ -+static ssize_t show_fw_cfg(struct kobject *kobj, -+ struct attribute *attr, char *buf) -+{ -+ struct firmware_config *config = -+ container_of(kobj, struct firmware_config, kobj); -+ struct kbase_device *kbdev = config->kbdev; -+ u32 val = 0; -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ if (attr == &fw_cfg_attr_max) -+ val = config->max; -+ else if (attr == &fw_cfg_attr_min) -+ val = config->min; -+ else if (attr == &fw_cfg_attr_cur) { -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ val = config->cur_val; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } else { -+ dev_warn(kbdev->dev, -+ "Unexpected read from entry %s/%s", -+ config->name, attr->name); -+ return -EINVAL; -+ } -+ -+ return snprintf(buf, PAGE_SIZE, "%u\n", val); -+} -+ -+static ssize_t store_fw_cfg(struct kobject *kobj, -+ struct attribute *attr, -+ const char *buf, -+ size_t count) -+{ -+ struct firmware_config *config = -+ container_of(kobj, struct firmware_config, kobj); -+ struct kbase_device *kbdev = config->kbdev; -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ if (attr == &fw_cfg_attr_cur) { -+ unsigned long flags; -+ u32 val; -+ int ret = kstrtouint(buf, 0, &val); -+ -+ if (ret) { -+ dev_err(kbdev->dev, -+ "Couldn't process %s/%s write operation.\n" -+ "Use format <value>\n", -+ config->name, attr->name); -+ return -EINVAL; -+ } -+ -+ if ((val < config->min) || (val > config->max)) -+ return -EINVAL; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ if (config->cur_val == val) { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ return count; -+ } -+ -+ /* If configuration update cannot be performed with -+ * FIRMWARE_CONFIG_UPDATE then we need to do a -+ * silent reset before we update the memory. -+ */ -+ if (!config->updatable) { -+ /* -+ * If there is already a GPU reset pending then inform -+ * the User to retry the write. -+ */ -+ if (kbase_reset_gpu_silent(kbdev)) { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, -+ flags); -+ return -EAGAIN; -+ } -+ } -+ -+ /* -+ * GPU reset request has been placed, now update the -+ * firmware image. GPU reset will take place only after -+ * hwaccess_lock is released. -+ * Update made to firmware image in memory would not -+ * be lost on GPU reset as configuration entries reside -+ * in the RONLY section of firmware image, which is not -+ * reloaded on firmware reboot due to GPU reset. -+ */ -+ kbase_csf_update_firmware_memory( -+ kbdev, config->address, val); -+ -+ config->cur_val = val; -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* If we can update the config without firmware reset then -+ * we need to just trigger FIRMWARE_CONFIG_UPDATE. -+ */ -+ if (config->updatable) { -+ ret = kbase_csf_trigger_firmware_config_update(kbdev); -+ if (ret) -+ return ret; -+ } -+ -+ /* Wait for the config update to take effect */ -+ if (!config->updatable) -+ kbase_reset_gpu_wait(kbdev); -+ } else { -+ dev_warn(kbdev->dev, -+ "Unexpected write to entry %s/%s", -+ config->name, attr->name); -+ return -EINVAL; -+ } -+ -+ return count; -+} -+ -+static const struct sysfs_ops fw_cfg_ops = { -+ .show = &show_fw_cfg, -+ .store = &store_fw_cfg, -+}; -+ -+static struct attribute *fw_cfg_attrs[] = { -+ &fw_cfg_attr_min, -+ &fw_cfg_attr_max, -+ &fw_cfg_attr_cur, -+ NULL, -+}; -+ -+static struct kobj_type fw_cfg_kobj_type = { -+ .release = &fw_cfg_kobj_release, -+ .sysfs_ops = &fw_cfg_ops, -+ .default_attrs = fw_cfg_attrs, -+}; -+ -+int kbase_csf_firmware_cfg_init(struct kbase_device *kbdev) -+{ -+ struct firmware_config *config; -+ -+ kbdev->csf.fw_cfg_kobj = kobject_create_and_add( -+ CSF_FIRMWARE_CFG_SYSFS_DIR_NAME, &kbdev->dev->kobj); -+ if (!kbdev->csf.fw_cfg_kobj) { -+ kobject_put(kbdev->csf.fw_cfg_kobj); -+ dev_err(kbdev->dev, -+ "Creation of %s sysfs sub-directory failed\n", -+ CSF_FIRMWARE_CFG_SYSFS_DIR_NAME); -+ return -ENOMEM; -+ } -+ -+ list_for_each_entry(config, &kbdev->csf.firmware_config, node) { -+ int err; -+ -+ kbase_csf_read_firmware_memory(kbdev, config->address, -+ &config->cur_val); -+ -+ err = kobject_init_and_add(&config->kobj, &fw_cfg_kobj_type, -+ kbdev->csf.fw_cfg_kobj, "%s", config->name); -+ if (err) { -+ kobject_put(&config->kobj); -+ dev_err(kbdev->dev, -+ "Creation of %s sysfs sub-directory failed\n", -+ config->name); -+ return err; -+ } -+ -+ config->kobj_inited = true; -+ } -+ -+ return 0; -+} -+ -+void kbase_csf_firmware_cfg_term(struct kbase_device *kbdev) -+{ -+ while (!list_empty(&kbdev->csf.firmware_config)) { -+ struct firmware_config *config; -+ -+ config = list_first_entry(&kbdev->csf.firmware_config, -+ struct firmware_config, node); -+ list_del(&config->node); -+ -+ if (config->kobj_inited) { -+ kobject_del(&config->kobj); -+ kobject_put(&config->kobj); -+ } else -+ kfree(config); -+ } -+ -+ kobject_del(kbdev->csf.fw_cfg_kobj); -+ kobject_put(kbdev->csf.fw_cfg_kobj); -+} -+ -+int kbase_csf_firmware_cfg_option_entry_parse(struct kbase_device *kbdev, -+ const struct firmware *fw, -+ const u32 *entry, -+ unsigned int size, bool updatable) -+{ -+ const char *name = (char *)&entry[3]; -+ struct firmware_config *config; -+ const unsigned int name_len = size - CONFIGURATION_ENTRY_NAME_OFFSET; -+ -+ /* Allocate enough space for struct firmware_config and the -+ * configuration option name (with NULL termination) -+ */ -+ config = kzalloc(sizeof(*config) + name_len + 1, GFP_KERNEL); -+ -+ if (!config) -+ return -ENOMEM; -+ -+ config->kbdev = kbdev; -+ config->updatable = updatable; -+ config->name = (char *)(config+1); -+ config->address = entry[0]; -+ config->min = entry[1]; -+ config->max = entry[2]; -+ -+ memcpy(config->name, name, name_len); -+ config->name[name_len] = 0; -+ -+ list_add(&config->node, &kbdev->csf.firmware_config); -+ -+ dev_dbg(kbdev->dev, "Configuration option '%s' at 0x%x range %u-%u", -+ config->name, config->address, -+ config->min, config->max); -+ -+ return 0; -+} -+#else -+int kbase_csf_firmware_cfg_init(struct kbase_device *kbdev) -+{ -+ return 0; -+} -+ -+void kbase_csf_firmware_cfg_term(struct kbase_device *kbdev) -+{ -+ /* !CONFIG_SYSFS: Nothing to do here */ -+} -+ -+int kbase_csf_firmware_cfg_option_entry_parse(struct kbase_device *kbdev, -+ const struct firmware *fw, -+ const u32 *entry, unsigned int size) -+{ -+ return 0; -+} -+#endif /* CONFIG_SYSFS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_cfg.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_cfg.h -new file mode 100644 -index 0000000..080c154 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_cfg.h -@@ -0,0 +1,74 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_FIRMWARE_CFG_H_ -+#define _KBASE_CSF_FIRMWARE_CFG_H_ -+ -+#include <mali_kbase.h> -+#include "mali_kbase_csf_firmware.h" -+#include <linux/firmware.h> -+ -+#define CONFIGURATION_ENTRY_NAME_OFFSET (0xC) -+ -+/** -+ * kbase_csf_firmware_cfg_init - Create the sysfs directory for configuration -+ * options present in firmware image. -+ * -+ * This function would create a sysfs directory and populate it with a -+ * sub-directory, that would contain a file per attribute, for every -+ * configuration option parsed from firmware image. -+ * -+ * @kbdev: Pointer to the Kbase device -+ * -+ * Return: The initialization error code. -+ */ -+int kbase_csf_firmware_cfg_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_cfg_term - Delete the sysfs directory that was created -+ * for firmware configuration options. -+ * -+ * @kbdev: Pointer to the Kbase device -+ * -+ */ -+void kbase_csf_firmware_cfg_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_cfg_option_entry_parse() - Process a -+ * "configuration option" section. -+ * -+ * Read a "configuration option" section adding it to the -+ * kbase_device:csf.firmware_config list. -+ * -+ * Return: 0 if successful, negative error code on failure -+ * -+ * @kbdev: Kbase device structure -+ * @fw: Firmware image containing the section -+ * @entry: Pointer to the section -+ * @size: Size (in bytes) of the section -+ * @updatable: Indicates if entry can be updated with FIRMWARE_CONFIG_UPDATE -+ */ -+int kbase_csf_firmware_cfg_option_entry_parse(struct kbase_device *kbdev, -+ const struct firmware *fw, -+ const u32 *entry, -+ unsigned int size, -+ bool updatable); -+#endif /* _KBASE_CSF_FIRMWARE_CFG_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_no_mali.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_no_mali.c -new file mode 100644 -index 0000000..ae2ad33 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_firmware_no_mali.c -@@ -0,0 +1,1389 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase.h" -+#include "mali_kbase_csf_firmware.h" -+#include "mali_kbase_csf_trace_buffer.h" -+#include "mali_kbase_csf_timeout.h" -+#include "mali_kbase_mem.h" -+#include "mali_kbase_reset_gpu.h" -+#include "mali_kbase_ctx_sched.h" -+#include "device/mali_kbase_device.h" -+#include "backend/gpu/mali_kbase_pm_internal.h" -+#include "mali_kbase_csf_scheduler.h" -+#include "mmu/mali_kbase_mmu.h" -+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" -+ -+#include <linux/list.h> -+#include <linux/slab.h> -+#include <linux/firmware.h> -+#include <linux/mman.h> -+#include <linux/string.h> -+#include <linux/mutex.h> -+#if (KERNEL_VERSION(4, 13, 0) <= LINUX_VERSION_CODE) -+#include <linux/set_memory.h> -+#endif -+#include <asm/arch_timer.h> -+ -+#ifdef CONFIG_MALI_DEBUG -+/* Makes Driver wait indefinitely for an acknowledgment for the different -+ * requests it sends to firmware. Otherwise the timeouts interfere with the -+ * use of debugger for source-level debugging of firmware as Driver initiates -+ * a GPU reset when a request times out, which always happen when a debugger -+ * is connected. -+ */ -+bool fw_debug; /* Default value of 0/false */ -+module_param(fw_debug, bool, 0444); -+MODULE_PARM_DESC(fw_debug, -+ "Enables effective use of a debugger for debugging firmware code."); -+#endif -+ -+#define DUMMY_FW_PAGE_SIZE SZ_4K -+ -+/** -+ * struct dummy_firmware_csi - Represents a dummy interface for MCU firmware CSs -+ * -+ * @cs_kernel_input: CS kernel input memory region -+ * @cs_kernel_output: CS kernel output memory region -+ */ -+struct dummy_firmware_csi { -+ u8 cs_kernel_input[DUMMY_FW_PAGE_SIZE]; -+ u8 cs_kernel_output[DUMMY_FW_PAGE_SIZE]; -+}; -+ -+/** -+ * struct dummy_firmware_csg - Represents a dummy interface for MCU firmware CSGs -+ * -+ * @csg_input: CSG kernel input memory region -+ * @csg_output: CSG kernel output memory region -+ * @csi: Dummy firmware CSIs -+ */ -+struct dummy_firmware_csg { -+ u8 csg_input[DUMMY_FW_PAGE_SIZE]; -+ u8 csg_output[DUMMY_FW_PAGE_SIZE]; -+ struct dummy_firmware_csi csi[8]; -+} dummy_firmware_csg; -+ -+/** -+ * struct dummy_firmware_interface - Represents a dummy interface in the MCU firmware -+ * -+ * @global_input: Global input memory region -+ * @global_output: Global output memory region -+ * @csg: Dummy firmware CSGs -+ * @node: Interface objects are on the kbase_device:csf.firmware_interfaces -+ * list using this list_head to link them -+ */ -+struct dummy_firmware_interface { -+ u8 global_input[DUMMY_FW_PAGE_SIZE]; -+ u8 global_output[DUMMY_FW_PAGE_SIZE]; -+ struct dummy_firmware_csg csg[8]; -+ struct list_head node; -+} dummy_firmware_interface; -+ -+#define CSF_GLB_REQ_CFG_MASK \ -+ (GLB_REQ_CFG_ALLOC_EN_MASK | GLB_REQ_CFG_PROGRESS_TIMER_MASK | \ -+ GLB_REQ_CFG_PWROFF_TIMER_MASK) -+ -+static inline u32 input_page_read(const u32 *const input, const u32 offset) -+{ -+ WARN_ON(offset % sizeof(u32)); -+ -+ return input[offset / sizeof(u32)]; -+} -+ -+static inline void input_page_write(u32 *const input, const u32 offset, -+ const u32 value) -+{ -+ WARN_ON(offset % sizeof(u32)); -+ -+ input[offset / sizeof(u32)] = value; -+} -+ -+static inline void input_page_partial_write(u32 *const input, const u32 offset, -+ u32 value, u32 mask) -+{ -+ WARN_ON(offset % sizeof(u32)); -+ -+ input[offset / sizeof(u32)] = -+ (input_page_read(input, offset) & ~mask) | (value & mask); -+} -+ -+static inline u32 output_page_read(const u32 *const output, const u32 offset) -+{ -+ WARN_ON(offset % sizeof(u32)); -+ -+ return output[offset / sizeof(u32)]; -+} -+ -+static inline void output_page_write(u32 *const output, const u32 offset, -+ const u32 value) -+{ -+ WARN_ON(offset % sizeof(u32)); -+ -+ output[offset / sizeof(u32)] = value; -+} -+ -+/** -+ * invent_memory_setup_entry() - Invent an "interface memory setup" section -+ * -+ * Invent an "interface memory setup" section similar to one from a firmware -+ * image. If successful the interface will be added to the -+ * kbase_device:csf.firmware_interfaces list. -+ * -+ * Return: 0 if successful, negative error code on failure -+ * -+ * @kbdev: Kbase device structure -+ */ -+static int invent_memory_setup_entry(struct kbase_device *kbdev) -+{ -+ struct dummy_firmware_interface *interface = NULL; -+ -+ /* Allocate enough memory for the struct dummy_firmware_interface. -+ */ -+ interface = kzalloc(sizeof(*interface), GFP_KERNEL); -+ if (!interface) -+ return -ENOMEM; -+ -+ kbdev->csf.shared_interface = interface; -+ list_add(&interface->node, &kbdev->csf.firmware_interfaces); -+ -+ /* NO_MALI: Don't insert any firmware pages */ -+ return 0; -+} -+ -+static void free_global_iface(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; -+ -+ if (iface->groups) { -+ unsigned int gid; -+ -+ for (gid = 0; gid < iface->group_num; ++gid) -+ kfree(iface->groups[gid].streams); -+ -+ kfree(iface->groups); -+ iface->groups = NULL; -+ } -+} -+ -+static int invent_cmd_stream_group_info(struct kbase_device *kbdev, -+ struct kbase_csf_cmd_stream_group_info *ginfo, -+ struct dummy_firmware_csg *csg) -+{ -+ unsigned int sid; -+ -+ ginfo->input = csg->csg_input; -+ ginfo->output = csg->csg_output; -+ -+ ginfo->kbdev = kbdev; -+ ginfo->features = 0; -+ ginfo->suspend_size = 64; -+ ginfo->protm_suspend_size = 64; -+ ginfo->stream_num = ARRAY_SIZE(csg->csi); -+ ginfo->stream_stride = 0; -+ -+ ginfo->streams = kcalloc(ginfo->stream_num, sizeof(*ginfo->streams), GFP_KERNEL); -+ if (ginfo->streams == NULL) { -+ return -ENOMEM; -+ } -+ -+ for (sid = 0; sid < ginfo->stream_num; ++sid) { -+ struct kbase_csf_cmd_stream_info *stream = &ginfo->streams[sid]; -+ struct dummy_firmware_csi *csi = &csg->csi[sid]; -+ -+ stream->input = csi->cs_kernel_input; -+ stream->output = csi->cs_kernel_output; -+ -+ stream->kbdev = kbdev; -+ stream->features = -+ STREAM_FEATURES_WORK_REGISTERS_SET(0, 80) | -+ STREAM_FEATURES_SCOREBOARDS_SET(0, 8) | -+ STREAM_FEATURES_COMPUTE_SET(0, 1) | -+ STREAM_FEATURES_FRAGMENT_SET(0, 1) | -+ STREAM_FEATURES_TILER_SET(0, 1); -+ } -+ -+ return 0; -+} -+ -+static int invent_capabilities(struct kbase_device *kbdev) -+{ -+ struct dummy_firmware_interface *interface = kbdev->csf.shared_interface; -+ struct kbase_csf_global_iface *iface = &kbdev->csf.global_iface; -+ unsigned int gid; -+ -+ iface->input = interface->global_input; -+ iface->output = interface->global_output; -+ -+ iface->version = 1; -+ iface->kbdev = kbdev; -+ iface->features = 0; -+ iface->prfcnt_size = 64; -+ -+ if (iface->version >= kbase_csf_interface_version(1, 1, 0)) { -+ /* update rate=1, max event size = 1<<8 = 256 */ -+ iface->instr_features = 0x81; -+ } else { -+ iface->instr_features = 0; -+ } -+ -+ iface->group_num = ARRAY_SIZE(interface->csg); -+ iface->group_stride = 0; -+ -+ iface->groups = kcalloc(iface->group_num, sizeof(*iface->groups), GFP_KERNEL); -+ if (iface->groups == NULL) { -+ return -ENOMEM; -+ } -+ -+ for (gid = 0; gid < iface->group_num; ++gid) { -+ int err; -+ -+ err = invent_cmd_stream_group_info(kbdev, &iface->groups[gid], -+ &interface->csg[gid]); -+ if (err < 0) { -+ free_global_iface(kbdev); -+ return err; -+ } -+ } -+ -+ return 0; -+} -+ -+void kbase_csf_read_firmware_memory(struct kbase_device *kbdev, -+ u32 gpu_addr, u32 *value) -+{ -+ /* NO_MALI: Nothing to do here */ -+} -+ -+ -+void kbase_csf_update_firmware_memory(struct kbase_device *kbdev, -+ u32 gpu_addr, u32 value) -+{ -+ /* NO_MALI: Nothing to do here */ -+} -+ -+void kbase_csf_firmware_cs_input( -+ const struct kbase_csf_cmd_stream_info *const info, const u32 offset, -+ const u32 value) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ -+ dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x\n", offset, value); -+ input_page_write(info->input, offset, value); -+ -+ if (offset == CS_REQ) { -+ /* NO_MALI: Immediately acknowledge requests */ -+ output_page_write(info->output, CS_ACK, value); -+ } -+} -+ -+u32 kbase_csf_firmware_cs_input_read( -+ const struct kbase_csf_cmd_stream_info *const info, -+ const u32 offset) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ u32 const val = input_page_read(info->input, offset); -+ -+ dev_dbg(kbdev->dev, "cs input r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+void kbase_csf_firmware_cs_input_mask( -+ const struct kbase_csf_cmd_stream_info *const info, const u32 offset, -+ const u32 value, const u32 mask) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ -+ dev_dbg(kbdev->dev, "cs input w: reg %08x val %08x mask %08x\n", -+ offset, value, mask); -+ -+ /* NO_MALI: Go through kbase_csf_firmware_cs_input to capture writes */ -+ kbase_csf_firmware_cs_input(info, offset, (input_page_read(info->input, offset) & ~mask) | (value & mask)); -+} -+ -+u32 kbase_csf_firmware_cs_output( -+ const struct kbase_csf_cmd_stream_info *const info, const u32 offset) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ u32 const val = output_page_read(info->output, offset); -+ -+ dev_dbg(kbdev->dev, "cs output r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+void kbase_csf_firmware_csg_input( -+ const struct kbase_csf_cmd_stream_group_info *const info, -+ const u32 offset, const u32 value) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ -+ dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x\n", -+ offset, value); -+ input_page_write(info->input, offset, value); -+ -+ if (offset == CSG_REQ) { -+ /* NO_MALI: Immediately acknowledge requests */ -+ output_page_write(info->output, CSG_ACK, value); -+ } -+} -+ -+u32 kbase_csf_firmware_csg_input_read( -+ const struct kbase_csf_cmd_stream_group_info *const info, -+ const u32 offset) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ u32 const val = input_page_read(info->input, offset); -+ -+ dev_dbg(kbdev->dev, "csg input r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+void kbase_csf_firmware_csg_input_mask( -+ const struct kbase_csf_cmd_stream_group_info *const info, -+ const u32 offset, const u32 value, const u32 mask) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ -+ dev_dbg(kbdev->dev, "csg input w: reg %08x val %08x mask %08x\n", -+ offset, value, mask); -+ -+ /* NO_MALI: Go through kbase_csf_firmware_csg_input to capture writes */ -+ kbase_csf_firmware_csg_input(info, offset, (input_page_read(info->input, offset) & ~mask) | (value & mask)); -+} -+ -+u32 kbase_csf_firmware_csg_output( -+ const struct kbase_csf_cmd_stream_group_info *const info, -+ const u32 offset) -+{ -+ const struct kbase_device * const kbdev = info->kbdev; -+ u32 const val = output_page_read(info->output, offset); -+ -+ dev_dbg(kbdev->dev, "csg output r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+static void -+csf_firmware_prfcnt_process(const struct kbase_csf_global_iface *const iface, -+ const u32 glb_req) -+{ -+ struct kbase_device *kbdev = iface->kbdev; -+ u32 glb_ack = output_page_read(iface->output, GLB_ACK); -+ /* If the value of GLB_REQ.PRFCNT_SAMPLE is different from the value of -+ * GLB_ACK.PRFCNT_SAMPLE, the CSF will sample the performance counters. -+ */ -+ if ((glb_req ^ glb_ack) & GLB_REQ_PRFCNT_SAMPLE_MASK) { -+ /* NO_MALI only uses the first buffer in the ring buffer. */ -+ input_page_write(iface->input, GLB_PRFCNT_EXTRACT, 0); -+ output_page_write(iface->output, GLB_PRFCNT_INSERT, 1); -+ kbase_reg_write(kbdev, GPU_COMMAND, GPU_COMMAND_PRFCNT_SAMPLE); -+ } -+ -+ /* Propagate enable masks to model if request to enable. */ -+ if (glb_req & GLB_REQ_PRFCNT_ENABLE_MASK) { -+ u32 tiler_en, l2_en, sc_en; -+ -+ tiler_en = input_page_read(iface->input, GLB_PRFCNT_TILER_EN); -+ l2_en = input_page_read(iface->input, GLB_PRFCNT_MMU_L2_EN); -+ sc_en = input_page_read(iface->input, GLB_PRFCNT_SHADER_EN); -+ -+ /* NO_MALI platform enabled all CSHW counters by default. */ -+ kbase_reg_write(kbdev, PRFCNT_TILER_EN, tiler_en); -+ kbase_reg_write(kbdev, PRFCNT_MMU_L2_EN, l2_en); -+ kbase_reg_write(kbdev, PRFCNT_SHADER_EN, sc_en); -+ } -+} -+ -+void kbase_csf_firmware_global_input( -+ const struct kbase_csf_global_iface *const iface, const u32 offset, -+ const u32 value) -+{ -+ const struct kbase_device * const kbdev = iface->kbdev; -+ -+ dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x\n", offset, value); -+ input_page_write(iface->input, offset, value); -+ -+ if (offset == GLB_REQ) { -+ csf_firmware_prfcnt_process(iface, value); -+ /* NO_MALI: Immediately acknowledge requests */ -+ output_page_write(iface->output, GLB_ACK, value); -+ } -+} -+ -+void kbase_csf_firmware_global_input_mask( -+ const struct kbase_csf_global_iface *const iface, const u32 offset, -+ const u32 value, const u32 mask) -+{ -+ const struct kbase_device * const kbdev = iface->kbdev; -+ -+ dev_dbg(kbdev->dev, "glob input w: reg %08x val %08x mask %08x\n", -+ offset, value, mask); -+ -+ /* NO_MALI: Go through kbase_csf_firmware_global_input to capture writes */ -+ kbase_csf_firmware_global_input(iface, offset, (input_page_read(iface->input, offset) & ~mask) | (value & mask)); -+} -+ -+u32 kbase_csf_firmware_global_input_read( -+ const struct kbase_csf_global_iface *const iface, const u32 offset) -+{ -+ const struct kbase_device * const kbdev = iface->kbdev; -+ u32 const val = input_page_read(iface->input, offset); -+ -+ dev_dbg(kbdev->dev, "glob input r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+u32 kbase_csf_firmware_global_output( -+ const struct kbase_csf_global_iface *const iface, const u32 offset) -+{ -+ const struct kbase_device * const kbdev = iface->kbdev; -+ u32 const val = output_page_read(iface->output, offset); -+ -+ dev_dbg(kbdev->dev, "glob output r: reg %08x val %08x\n", offset, val); -+ return val; -+} -+ -+/** -+ * handle_internal_firmware_fatal - Handler for CS internal firmware fault. -+ * -+ * @kbdev: Pointer to kbase device -+ * -+ * Report group fatal error to user space for all GPU command queue groups -+ * in the device, terminate them and reset GPU. -+ */ -+static void handle_internal_firmware_fatal(struct kbase_device *const kbdev) -+{ -+ int as; -+ -+ for (as = 0; as < kbdev->nr_hw_address_spaces; as++) { -+ unsigned long flags; -+ struct kbase_context *kctx; -+ struct kbase_fault fault; -+ -+ if (as == MCU_AS_NR) -+ continue; -+ -+ /* Only handle the fault for an active address space. Lock is -+ * taken here to atomically get reference to context in an -+ * active address space and retain its refcount. -+ */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as); -+ -+ if (kctx) { -+ kbase_ctx_sched_retain_ctx_refcount(kctx); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } else { -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ continue; -+ } -+ -+ fault = (struct kbase_fault) { -+ .status = GPU_EXCEPTION_TYPE_SW_FAULT_1, -+ }; -+ -+ kbase_csf_ctx_handle_fault(kctx, &fault); -+ kbase_ctx_sched_release_ctx_lock(kctx); -+ } -+ -+ if (kbase_prepare_to_reset_gpu(kbdev, -+ RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) -+ kbase_reset_gpu(kbdev); -+} -+ -+/** -+ * firmware_error_worker - Worker function for handling firmware internal error -+ * -+ * @data: Pointer to a work_struct embedded in kbase device. -+ * -+ * Handle the CS internal firmware error -+ */ -+static void firmware_error_worker(struct work_struct *const data) -+{ -+ struct kbase_device *const kbdev = -+ container_of(data, struct kbase_device, csf.fw_error_work); -+ -+ handle_internal_firmware_fatal(kbdev); -+} -+ -+static bool global_request_complete(struct kbase_device *const kbdev, -+ u32 const req_mask) -+{ -+ struct kbase_csf_global_iface *global_iface = -+ &kbdev->csf.global_iface; -+ bool complete = false; -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ -+ if ((kbase_csf_firmware_global_output(global_iface, GLB_ACK) & -+ req_mask) == -+ (kbase_csf_firmware_global_input_read(global_iface, GLB_REQ) & -+ req_mask)) -+ complete = true; -+ -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ return complete; -+} -+ -+static int wait_for_global_request(struct kbase_device *const kbdev, -+ u32 const req_mask) -+{ -+ const long wait_timeout = -+ kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ long remaining; -+ int err = 0; -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ global_request_complete(kbdev, req_mask), -+ wait_timeout); -+ -+ if (!remaining) { -+ dev_warn(kbdev->dev, "Timed out waiting for global request %x to complete", -+ req_mask); -+ err = -ETIMEDOUT; -+ } -+ -+ return err; -+} -+ -+static void set_global_request( -+ const struct kbase_csf_global_iface *const global_iface, -+ u32 const req_mask) -+{ -+ u32 glb_req; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(global_iface->kbdev); -+ -+ glb_req = kbase_csf_firmware_global_output(global_iface, GLB_ACK); -+ glb_req ^= req_mask; -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, glb_req, -+ req_mask); -+} -+ -+static void enable_endpoints_global( -+ const struct kbase_csf_global_iface *const global_iface, -+ u64 const shader_core_mask) -+{ -+ kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_LO, -+ shader_core_mask & U32_MAX); -+ kbase_csf_firmware_global_input(global_iface, GLB_ALLOC_EN_HI, -+ shader_core_mask >> 32); -+ -+ set_global_request(global_iface, GLB_REQ_CFG_ALLOC_EN_MASK); -+} -+ -+static void enable_shader_poweroff_timer(struct kbase_device *const kbdev, -+ const struct kbase_csf_global_iface *const global_iface) -+{ -+ u32 pwroff_reg; -+ -+ if (kbdev->csf.firmware_hctl_core_pwr) -+ pwroff_reg = -+ GLB_PWROFF_TIMER_TIMER_SOURCE_SET(DISABLE_GLB_PWROFF_TIMER, -+ GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); -+ else -+ pwroff_reg = kbdev->csf.mcu_core_pwroff_dur_count; -+ -+ kbase_csf_firmware_global_input(global_iface, GLB_PWROFF_TIMER, -+ pwroff_reg); -+ set_global_request(global_iface, GLB_REQ_CFG_PWROFF_TIMER_MASK); -+ -+ /* Save the programed reg value in its shadow field */ -+ kbdev->csf.mcu_core_pwroff_reg_shadow = pwroff_reg; -+} -+ -+static void set_timeout_global( -+ const struct kbase_csf_global_iface *const global_iface, -+ u64 const timeout) -+{ -+ kbase_csf_firmware_global_input(global_iface, GLB_PROGRESS_TIMER, -+ timeout / GLB_PROGRESS_TIMER_TIMEOUT_SCALE); -+ -+ set_global_request(global_iface, GLB_REQ_CFG_PROGRESS_TIMER_MASK); -+} -+ -+static void global_init(struct kbase_device *const kbdev, u64 core_mask) -+{ -+ u32 const ack_irq_mask = GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK | -+ GLB_ACK_IRQ_MASK_PING_MASK | -+ GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK | -+ GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK | -+ GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK | -+ GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK | -+ GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK | -+ GLB_ACK_IRQ_MASK_IDLE_EVENT_MASK; -+ -+ const struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ -+ /* Update shader core allocation enable mask */ -+ enable_endpoints_global(global_iface, core_mask); -+ enable_shader_poweroff_timer(kbdev, global_iface); -+ -+ set_timeout_global(global_iface, kbase_csf_timeout_get(kbdev)); -+ -+ /* Unmask the interrupts */ -+ kbase_csf_firmware_global_input(global_iface, -+ GLB_ACK_IRQ_MASK, ack_irq_mask); -+ -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+/** -+ * global_init_on_boot - Sends a global request to control various features. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Currently only the request to enable endpoints and cycle counter is sent. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+static int global_init_on_boot(struct kbase_device *const kbdev) -+{ -+ unsigned long flags; -+ u64 core_mask; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ core_mask = kbase_pm_ca_get_core_mask(kbdev); -+ kbdev->csf.firmware_hctl_core_pwr = -+ kbase_pm_no_mcu_core_pwroff(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ global_init(kbdev, core_mask); -+ -+ return wait_for_global_request(kbdev, CSF_GLB_REQ_CFG_MASK); -+} -+ -+void kbase_csf_firmware_global_reinit(struct kbase_device *kbdev, -+ u64 core_mask) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbdev->csf.glb_init_request_pending = true; -+ kbdev->csf.firmware_hctl_core_pwr = -+ kbase_pm_no_mcu_core_pwroff(kbdev); -+ global_init(kbdev, core_mask); -+} -+ -+bool kbase_csf_firmware_global_reinit_complete(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ WARN_ON(!kbdev->csf.glb_init_request_pending); -+ -+ if (global_request_complete(kbdev, CSF_GLB_REQ_CFG_MASK)) -+ kbdev->csf.glb_init_request_pending = false; -+ -+ return !kbdev->csf.glb_init_request_pending; -+} -+ -+void kbase_csf_firmware_update_core_attr(struct kbase_device *kbdev, -+ bool update_core_pwroff_timer, bool update_core_mask, u64 core_mask) -+{ -+ unsigned long flags; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ if (update_core_mask) -+ enable_endpoints_global(&kbdev->csf.global_iface, core_mask); -+ if (update_core_pwroff_timer) -+ enable_shader_poweroff_timer(kbdev, &kbdev->csf.global_iface); -+ -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+bool kbase_csf_firmware_core_attr_updated(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ return global_request_complete(kbdev, GLB_REQ_CFG_ALLOC_EN_MASK | -+ GLB_REQ_CFG_PWROFF_TIMER_MASK); -+} -+ -+static void kbase_csf_firmware_reload_worker(struct work_struct *work) -+{ -+ struct kbase_device *kbdev = container_of(work, struct kbase_device, -+ csf.firmware_reload_work); -+ unsigned long flags; -+ -+ /* Reboot the firmware */ -+ kbase_csf_firmware_enable_mcu(kbdev); -+ -+ /* Tell MCU state machine to transit to next state */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->csf.firmware_reloaded = true; -+ kbase_pm_update_state(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+void kbase_csf_firmware_trigger_reload(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbdev->csf.firmware_reloaded = false; -+ -+ if (kbdev->csf.firmware_reload_needed) { -+ kbdev->csf.firmware_reload_needed = false; -+ queue_work(system_wq, &kbdev->csf.firmware_reload_work); -+ } else { -+ kbase_csf_firmware_enable_mcu(kbdev); -+ kbdev->csf.firmware_reloaded = true; -+ } -+} -+ -+void kbase_csf_firmware_reload_completed(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (unlikely(!kbdev->csf.firmware_inited)) -+ return; -+ -+ /* Tell MCU state machine to transit to next state */ -+ kbdev->csf.firmware_reloaded = true; -+ kbase_pm_update_state(kbdev); -+} -+ -+static u32 convert_dur_to_idle_count(struct kbase_device *kbdev, const u32 dur_ms) -+{ -+#define HYSTERESIS_VAL_UNIT_SHIFT (10) -+ /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ -+ u64 freq = arch_timer_get_cntfrq(); -+ u64 dur_val = dur_ms; -+ u32 cnt_val_u32, reg_val_u32; -+ bool src_system_timestamp = freq > 0; -+ -+ if (!src_system_timestamp) { -+ /* Get the cycle_counter source alternative */ -+ spin_lock(&kbdev->pm.clk_rtm.lock); -+ if (kbdev->pm.clk_rtm.clks[0]) -+ freq = kbdev->pm.clk_rtm.clks[0]->clock_val; -+ else -+ dev_warn(kbdev->dev, "No GPU clock, unexpected intregration issue!"); -+ spin_unlock(&kbdev->pm.clk_rtm.lock); -+ -+ dev_info(kbdev->dev, "Can't get the timestamp frequency, " -+ "use cycle counter format with firmware idle hysteresis!"); -+ } -+ -+ /* Formula for dur_val = ((dur_ms/1000) * freq_HZ) >> 10) */ -+ dur_val = (dur_val * freq) >> HYSTERESIS_VAL_UNIT_SHIFT; -+ dur_val = div_u64(dur_val, 1000); -+ -+ /* Interface limits the value field to S32_MAX */ -+ cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; -+ -+ reg_val_u32 = GLB_IDLE_TIMER_TIMEOUT_SET(0, cnt_val_u32); -+ /* add the source flag */ -+ if (src_system_timestamp) -+ reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET(reg_val_u32, -+ GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); -+ else -+ reg_val_u32 = GLB_IDLE_TIMER_TIMER_SOURCE_SET(reg_val_u32, -+ GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER); -+ -+ return reg_val_u32; -+} -+ -+u32 kbase_csf_firmware_get_gpu_idle_hysteresis_time(struct kbase_device *kbdev) -+{ -+ return kbdev->csf.gpu_idle_hysteresis_ms; -+} -+ -+u32 kbase_csf_firmware_set_gpu_idle_hysteresis_time(struct kbase_device *kbdev, u32 dur) -+{ -+ unsigned long flags; -+ const u32 hysteresis_val = convert_dur_to_idle_count(kbdev, dur); -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ kbdev->csf.gpu_idle_hysteresis_ms = dur; -+ kbdev->csf.gpu_idle_dur_count = hysteresis_val; -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ dev_dbg(kbdev->dev, "CSF set firmware idle hysteresis count-value: 0x%.8x", -+ hysteresis_val); -+ -+ return hysteresis_val; -+} -+ -+static u32 convert_dur_to_core_pwroff_count(struct kbase_device *kbdev, const u32 dur_us) -+{ -+#define PWROFF_VAL_UNIT_SHIFT (10) -+ /* Get the cntfreq_el0 value, which drives the SYSTEM_TIMESTAMP */ -+ u64 freq = arch_timer_get_cntfrq(); -+ u64 dur_val = dur_us; -+ u32 cnt_val_u32, reg_val_u32; -+ bool src_system_timestamp = freq > 0; -+ -+ if (!src_system_timestamp) { -+ /* Get the cycle_counter source alternative */ -+ spin_lock(&kbdev->pm.clk_rtm.lock); -+ if (kbdev->pm.clk_rtm.clks[0]) -+ freq = kbdev->pm.clk_rtm.clks[0]->clock_val; -+ else -+ dev_warn(kbdev->dev, "No GPU clock, unexpected integration issue!"); -+ spin_unlock(&kbdev->pm.clk_rtm.lock); -+ -+ dev_info(kbdev->dev, "Can't get the timestamp frequency, " -+ "use cycle counter with MCU Core Poweroff timer!"); -+ } -+ -+ /* Formula for dur_val = ((dur_us/1e6) * freq_HZ) >> 10) */ -+ dur_val = (dur_val * freq) >> HYSTERESIS_VAL_UNIT_SHIFT; -+ dur_val = div_u64(dur_val, 1000000); -+ -+ /* Interface limits the value field to S32_MAX */ -+ cnt_val_u32 = (dur_val > S32_MAX) ? S32_MAX : (u32)dur_val; -+ -+ reg_val_u32 = GLB_PWROFF_TIMER_TIMEOUT_SET(0, cnt_val_u32); -+ /* add the source flag */ -+ if (src_system_timestamp) -+ reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET(reg_val_u32, -+ GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP); -+ else -+ reg_val_u32 = GLB_PWROFF_TIMER_TIMER_SOURCE_SET(reg_val_u32, -+ GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER); -+ -+ return reg_val_u32; -+} -+ -+u32 kbase_csf_firmware_get_mcu_core_pwroff_time(struct kbase_device *kbdev) -+{ -+ return kbdev->csf.mcu_core_pwroff_dur_us; -+} -+ -+u32 kbase_csf_firmware_set_mcu_core_pwroff_time(struct kbase_device *kbdev, u32 dur) -+{ -+ unsigned long flags; -+ const u32 pwroff = convert_dur_to_core_pwroff_count(kbdev, dur); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->csf.mcu_core_pwroff_dur_us = dur; -+ kbdev->csf.mcu_core_pwroff_dur_count = pwroff; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ dev_dbg(kbdev->dev, "MCU Core Poweroff input update: 0x%.8x", pwroff); -+ -+ return pwroff; -+} -+ -+int kbase_csf_firmware_early_init(struct kbase_device *kbdev) -+{ -+ init_waitqueue_head(&kbdev->csf.event_wait); -+ kbdev->csf.interrupt_received = false; -+ kbdev->csf.fw_timeout_ms = CSF_FIRMWARE_TIMEOUT_MS; -+ -+ INIT_LIST_HEAD(&kbdev->csf.firmware_interfaces); -+ INIT_LIST_HEAD(&kbdev->csf.firmware_config); -+ INIT_LIST_HEAD(&kbdev->csf.firmware_trace_buffers.list); -+ INIT_WORK(&kbdev->csf.firmware_reload_work, -+ kbase_csf_firmware_reload_worker); -+ INIT_WORK(&kbdev->csf.fw_error_work, firmware_error_worker); -+ -+ mutex_init(&kbdev->csf.reg_lock); -+ -+ return 0; -+} -+ -+int kbase_csf_firmware_init(struct kbase_device *kbdev) -+{ -+ int ret; -+ -+ lockdep_assert_held(&kbdev->fw_load_lock); -+ -+ if (WARN_ON((kbdev->as_free & MCU_AS_BITMASK) == 0)) -+ return -EINVAL; -+ kbdev->as_free &= ~MCU_AS_BITMASK; -+ -+ ret = kbase_mmu_init(kbdev, &kbdev->csf.mcu_mmu, NULL, -+ BASE_MEM_GROUP_DEFAULT); -+ -+ if (ret != 0) { -+ /* Release the address space */ -+ kbdev->as_free |= MCU_AS_BITMASK; -+ return ret; -+ } -+ -+ kbdev->csf.gpu_idle_hysteresis_ms = FIRMWARE_IDLE_HYSTERESIS_TIME_MS; -+ kbdev->csf.gpu_idle_dur_count = convert_dur_to_idle_count( -+ kbdev, FIRMWARE_IDLE_HYSTERESIS_TIME_MS); -+ -+ ret = kbase_mcu_shared_interface_region_tracker_init(kbdev); -+ if (ret != 0) { -+ dev_err(kbdev->dev, -+ "Failed to setup the rb tree for managing shared interface segment\n"); -+ goto error; -+ } -+ -+ ret = invent_memory_setup_entry(kbdev); -+ if (ret != 0) { -+ dev_err(kbdev->dev, "Failed to load firmware entry\n"); -+ goto error; -+ } -+ -+ /* Make sure L2 cache is powered up */ -+ kbase_pm_wait_for_l2_powered(kbdev); -+ -+ /* NO_MALI: Don't init trace buffers */ -+ -+ /* NO_MALI: Don't load the MMU tables or boot CSF firmware */ -+ -+ ret = invent_capabilities(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = kbase_csf_doorbell_mapping_init(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = kbase_csf_setup_dummy_user_reg_page(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = kbase_csf_scheduler_init(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = kbase_csf_timeout_init(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ ret = global_init_on_boot(kbdev); -+ if (ret != 0) -+ goto error; -+ -+ return 0; -+ -+error: -+ kbase_csf_firmware_term(kbdev); -+ return ret; -+} -+ -+void kbase_csf_firmware_term(struct kbase_device *kbdev) -+{ -+ cancel_work_sync(&kbdev->csf.fw_error_work); -+ -+ kbase_csf_timeout_term(kbdev); -+ -+ /* NO_MALI: Don't stop firmware or unload MMU tables */ -+ -+ kbase_mmu_term(kbdev, &kbdev->csf.mcu_mmu); -+ -+ kbase_csf_scheduler_term(kbdev); -+ -+ kbase_csf_free_dummy_user_reg_page(kbdev); -+ -+ kbase_csf_doorbell_mapping_term(kbdev); -+ -+ free_global_iface(kbdev); -+ -+ /* Release the address space */ -+ kbdev->as_free |= MCU_AS_BITMASK; -+ -+ while (!list_empty(&kbdev->csf.firmware_interfaces)) { -+ struct dummy_firmware_interface *interface; -+ -+ interface = list_first_entry(&kbdev->csf.firmware_interfaces, -+ struct dummy_firmware_interface, node); -+ list_del(&interface->node); -+ -+ /* NO_MALI: No cleanup in dummy interface necessary */ -+ -+ kfree(interface); -+ } -+ -+ /* NO_MALI: No trace buffers to terminate */ -+ -+#ifndef MALI_KBASE_BUILD -+ mali_kutf_fw_utf_entry_cleanup(kbdev); -+#endif -+ -+ mutex_destroy(&kbdev->csf.reg_lock); -+ -+ /* This will also free up the region allocated for the shared interface -+ * entry parsed from the firmware image. -+ */ -+ kbase_mcu_shared_interface_region_tracker_term(kbdev); -+} -+ -+void kbase_csf_firmware_enable_gpu_idle_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ u32 glb_req; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ -+ /* The scheduler is assumed to only call the enable when its internal -+ * state indicates that the idle timer has previously been disabled. So -+ * on entry the expected field values are: -+ * 1. GLOBAL_INPUT_BLOCK.GLB_REQ.IDLE_ENABLE: 0 -+ * 2. GLOBAL_OUTPUT_BLOCK.GLB_ACK.IDLE_ENABLE: 0, or, on 1 -> 0 -+ */ -+ -+ glb_req = kbase_csf_firmware_global_input_read(global_iface, GLB_REQ); -+ if (glb_req & GLB_REQ_IDLE_ENABLE_MASK) -+ dev_err(kbdev->dev, "Incoherent scheduler state on REQ_IDLE_ENABLE!"); -+ -+ kbase_csf_firmware_global_input(global_iface, GLB_IDLE_TIMER, -+ kbdev->csf.gpu_idle_dur_count); -+ -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, -+ GLB_REQ_REQ_IDLE_ENABLE, GLB_REQ_IDLE_ENABLE_MASK); -+ -+ dev_dbg(kbdev->dev, "Enabling GPU idle timer with count-value: 0x%.8x", -+ kbdev->csf.gpu_idle_dur_count); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+} -+ -+void kbase_csf_firmware_disable_gpu_idle_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, -+ GLB_REQ_REQ_IDLE_DISABLE, -+ GLB_REQ_IDLE_DISABLE_MASK); -+ -+ dev_dbg(kbdev->dev, "Sending request to disable gpu idle timer"); -+ -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+} -+ -+void kbase_csf_firmware_ping(struct kbase_device *const kbdev) -+{ -+ const struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ set_global_request(global_iface, GLB_REQ_PING_MASK); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+int kbase_csf_firmware_ping_wait(struct kbase_device *const kbdev) -+{ -+ kbase_csf_firmware_ping(kbdev); -+ return wait_for_global_request(kbdev, GLB_REQ_PING_MASK); -+} -+ -+int kbase_csf_firmware_set_timeout(struct kbase_device *const kbdev, -+ u64 const timeout) -+{ -+ const struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ unsigned long flags; -+ int err; -+ -+ /* The 'reg_lock' is also taken and is held till the update is not -+ * complete, to ensure the update of timeout value by multiple Users -+ * gets serialized. -+ */ -+ mutex_lock(&kbdev->csf.reg_lock); -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ set_timeout_global(global_iface, timeout); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ err = wait_for_global_request(kbdev, GLB_REQ_CFG_PROGRESS_TIMER_MASK); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ return err; -+} -+ -+void kbase_csf_enter_protected_mode(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ set_global_request(global_iface, GLB_REQ_PROTM_ENTER_MASK); -+ dev_dbg(kbdev->dev, "Sending request to enter protected mode"); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ wait_for_global_request(kbdev, GLB_REQ_PROTM_ENTER_MASK); -+} -+ -+void kbase_csf_firmware_trigger_mcu_halt(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ unsigned long flags; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ set_global_request(global_iface, GLB_REQ_HALT_MASK); -+ dev_dbg(kbdev->dev, "Sending request to HALT MCU"); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+int kbase_csf_trigger_firmware_config_update(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ unsigned long flags; -+ int err = 0; -+ -+ /* The 'reg_lock' is also taken and is held till the update is -+ * complete, to ensure the config update gets serialized. -+ */ -+ mutex_lock(&kbdev->csf.reg_lock); -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ -+ set_global_request(global_iface, GLB_REQ_FIRMWARE_CONFIG_UPDATE_MASK); -+ dev_dbg(kbdev->dev, "Sending request for FIRMWARE_CONFIG_UPDATE"); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ err = wait_for_global_request(kbdev, -+ GLB_REQ_FIRMWARE_CONFIG_UPDATE_MASK); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ return err; -+} -+ -+/** -+ * copy_grp_and_stm - Copy CS and/or group data -+ * -+ * @iface: Global CSF interface provided by -+ * the firmware. -+ * @group_data: Pointer where to store all the group data -+ * (sequentially). -+ * @max_group_num: The maximum number of groups to be read. Can be 0, in -+ * which case group_data is unused. -+ * @stream_data: Pointer where to store all the stream data -+ * (sequentially). -+ * @max_total_stream_num: The maximum number of streams to be read. -+ * Can be 0, in which case stream_data is unused. -+ * -+ * Return: Total number of CSs, summed across all groups. -+ */ -+static u32 copy_grp_and_stm( -+ const struct kbase_csf_global_iface * const iface, -+ struct basep_cs_group_control * const group_data, -+ u32 max_group_num, -+ struct basep_cs_stream_control * const stream_data, -+ u32 max_total_stream_num) -+{ -+ u32 i, total_stream_num = 0; -+ -+ if (WARN_ON((max_group_num > 0) && !group_data)) -+ max_group_num = 0; -+ -+ if (WARN_ON((max_total_stream_num > 0) && !stream_data)) -+ max_total_stream_num = 0; -+ -+ for (i = 0; i < iface->group_num; i++) { -+ u32 j; -+ -+ if (i < max_group_num) { -+ group_data[i].features = iface->groups[i].features; -+ group_data[i].stream_num = iface->groups[i].stream_num; -+ group_data[i].suspend_size = -+ iface->groups[i].suspend_size; -+ } -+ for (j = 0; j < iface->groups[i].stream_num; j++) { -+ if (total_stream_num < max_total_stream_num) -+ stream_data[total_stream_num].features = -+ iface->groups[i].streams[j].features; -+ total_stream_num++; -+ } -+ } -+ -+ return total_stream_num; -+} -+ -+u32 kbase_csf_firmware_get_glb_iface( -+ struct kbase_device *kbdev, -+ struct basep_cs_group_control *const group_data, -+ u32 const max_group_num, -+ struct basep_cs_stream_control *const stream_data, -+ u32 const max_total_stream_num, u32 *const glb_version, -+ u32 *const features, u32 *const group_num, u32 *const prfcnt_size, -+ u32 *const instr_features) -+{ -+ const struct kbase_csf_global_iface * const iface = -+ &kbdev->csf.global_iface; -+ -+ if (WARN_ON(!glb_version) || WARN_ON(!features) || -+ WARN_ON(!group_num) || WARN_ON(!prfcnt_size) || -+ WARN_ON(!instr_features)) -+ return 0; -+ -+ *glb_version = iface->version; -+ *features = iface->features; -+ *group_num = iface->group_num; -+ *prfcnt_size = iface->prfcnt_size; -+ *instr_features = iface->instr_features; -+ -+ return copy_grp_and_stm(iface, group_data, max_group_num, -+ stream_data, max_total_stream_num); -+} -+ -+const char *kbase_csf_firmware_get_timeline_metadata( -+ struct kbase_device *kbdev, const char *name, size_t *size) -+{ -+ if (WARN_ON(!kbdev) || -+ WARN_ON(!name) || -+ WARN_ON(!size)) { -+ return NULL; -+ } -+ -+ *size = 0; -+ return NULL; -+} -+ -+void kbase_csf_firmware_disable_mcu_wait(struct kbase_device *kbdev) -+{ -+ /* NO_MALI: Nothing to do here */ -+} -+ -+int kbase_csf_firmware_mcu_shared_mapping_init( -+ struct kbase_device *kbdev, -+ unsigned int num_pages, -+ unsigned long cpu_map_properties, -+ unsigned long gpu_map_properties, -+ struct kbase_csf_mapping *csf_mapping) -+{ -+ struct tagged_addr *phys; -+ struct kbase_va_region *va_reg; -+ struct page **page_list; -+ void *cpu_addr; -+ int i, ret = 0; -+ pgprot_t cpu_map_prot = PAGE_KERNEL; -+ unsigned long gpu_map_prot; -+ -+ if (cpu_map_properties & PROT_READ) -+ cpu_map_prot = PAGE_KERNEL_RO; -+ -+ if (kbdev->system_coherency == COHERENCY_ACE) { -+ gpu_map_prot = -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_DEFAULT_ACE); -+ } else { -+ gpu_map_prot = -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); -+ cpu_map_prot = pgprot_writecombine(cpu_map_prot); -+ }; -+ -+ phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); -+ if (!phys) -+ goto out; -+ -+ page_list = kmalloc_array(num_pages, sizeof(*page_list), GFP_KERNEL); -+ if (!page_list) -+ goto page_list_alloc_error; -+ -+ ret = kbase_mem_pool_alloc_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ num_pages, phys, false); -+ if (ret <= 0) -+ goto phys_mem_pool_alloc_error; -+ -+ for (i = 0; i < num_pages; i++) -+ page_list[i] = as_page(phys[i]); -+ -+ cpu_addr = vmap(page_list, num_pages, VM_MAP, cpu_map_prot); -+ if (!cpu_addr) -+ goto vmap_error; -+ -+ va_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, 0, -+ num_pages, KBASE_REG_ZONE_MCU_SHARED); -+ if (!va_reg) -+ goto va_region_alloc_error; -+ -+ mutex_lock(&kbdev->csf.reg_lock); -+ ret = kbase_add_va_region_rbtree(kbdev, va_reg, 0, num_pages, 1); -+ va_reg->flags &= ~KBASE_REG_FREE; -+ if (ret) -+ goto va_region_add_error; -+ mutex_unlock(&kbdev->csf.reg_lock); -+ -+ gpu_map_properties &= (KBASE_REG_GPU_RD | KBASE_REG_GPU_WR); -+ gpu_map_properties |= gpu_map_prot; -+ -+ ret = kbase_mmu_insert_pages_no_flush(kbdev, &kbdev->csf.mcu_mmu, -+ va_reg->start_pfn, &phys[0], num_pages, -+ gpu_map_properties, KBASE_MEM_GROUP_CSF_FW); -+ if (ret) -+ goto mmu_insert_pages_error; -+ -+ kfree(page_list); -+ csf_mapping->phys = phys; -+ csf_mapping->cpu_addr = cpu_addr; -+ csf_mapping->va_reg = va_reg; -+ csf_mapping->num_pages = num_pages; -+ -+ return 0; -+ -+mmu_insert_pages_error: -+ mutex_lock(&kbdev->csf.reg_lock); -+ kbase_remove_va_region(va_reg); -+va_region_add_error: -+ kbase_free_alloced_region(va_reg); -+ mutex_unlock(&kbdev->csf.reg_lock); -+va_region_alloc_error: -+ vunmap(cpu_addr); -+vmap_error: -+ kbase_mem_pool_free_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ num_pages, phys, false, false); -+ -+phys_mem_pool_alloc_error: -+ kfree(page_list); -+page_list_alloc_error: -+ kfree(phys); -+out: -+ /* Zero-initialize the mapping to make sure that the termination -+ * function doesn't try to unmap or free random addresses. -+ */ -+ csf_mapping->phys = NULL; -+ csf_mapping->cpu_addr = NULL; -+ csf_mapping->va_reg = NULL; -+ csf_mapping->num_pages = 0; -+ -+ return -ENOMEM; -+} -+ -+void kbase_csf_firmware_mcu_shared_mapping_term( -+ struct kbase_device *kbdev, struct kbase_csf_mapping *csf_mapping) -+{ -+ if (csf_mapping->va_reg) { -+ mutex_lock(&kbdev->csf.reg_lock); -+ kbase_remove_va_region(csf_mapping->va_reg); -+ kbase_free_alloced_region(csf_mapping->va_reg); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ } -+ -+ if (csf_mapping->phys) { -+ kbase_mem_pool_free_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ csf_mapping->num_pages, csf_mapping->phys, false, -+ false); -+ } -+ -+ vunmap(csf_mapping->cpu_addr); -+ kfree(csf_mapping->phys); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_heap_context_alloc.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_heap_context_alloc.c -new file mode 100644 -index 0000000..96746c6 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_heap_context_alloc.c -@@ -0,0 +1,195 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include "mali_kbase_csf_heap_context_alloc.h" -+ -+/* Size of one heap context structure, in bytes. */ -+#define HEAP_CTX_SIZE ((size_t)32) -+ -+/* Total size of the GPU memory region allocated for heap contexts, in bytes. */ -+#define HEAP_CTX_REGION_SIZE (MAX_TILER_HEAPS * HEAP_CTX_SIZE) -+ -+/** -+ * sub_alloc - Sub-allocate a heap context from a GPU memory region -+ * -+ * @ctx_alloc: Pointer to the heap context allocator. -+ * -+ * Return: GPU virtual address of the allocated heap context or 0 on failure. -+ */ -+static u64 sub_alloc(struct kbase_csf_heap_context_allocator *const ctx_alloc) -+{ -+ struct kbase_context *const kctx = ctx_alloc->kctx; -+ int heap_nr = 0; -+ size_t ctx_offset = 0; -+ u64 heap_gpu_va = 0; -+ struct kbase_vmap_struct mapping; -+ void *ctx_ptr = NULL; -+ -+ lockdep_assert_held(&ctx_alloc->lock); -+ -+ heap_nr = find_first_zero_bit(ctx_alloc->in_use, -+ MAX_TILER_HEAPS); -+ -+ if (unlikely(heap_nr >= MAX_TILER_HEAPS)) { -+ dev_err(kctx->kbdev->dev, -+ "No free tiler heap contexts in the pool\n"); -+ return 0; -+ } -+ -+ ctx_offset = heap_nr * HEAP_CTX_SIZE; -+ heap_gpu_va = ctx_alloc->gpu_va + ctx_offset; -+ ctx_ptr = kbase_vmap_prot(kctx, heap_gpu_va, -+ HEAP_CTX_SIZE, KBASE_REG_CPU_WR, &mapping); -+ -+ if (unlikely(!ctx_ptr)) { -+ dev_err(kctx->kbdev->dev, -+ "Failed to map tiler heap context %d (0x%llX)\n", -+ heap_nr, heap_gpu_va); -+ return 0; -+ } -+ -+ memset(ctx_ptr, 0, HEAP_CTX_SIZE); -+ kbase_vunmap(ctx_ptr, &mapping); -+ -+ bitmap_set(ctx_alloc->in_use, heap_nr, 1); -+ -+ dev_dbg(kctx->kbdev->dev, "Allocated tiler heap context %d (0x%llX)\n", -+ heap_nr, heap_gpu_va); -+ -+ return heap_gpu_va; -+} -+ -+/** -+ * sub_free - Free a heap context sub-allocated from a GPU memory region -+ * -+ * @ctx_alloc: Pointer to the heap context allocator. -+ * @heap_gpu_va: The GPU virtual address of a heap context structure to free. -+ */ -+static void sub_free(struct kbase_csf_heap_context_allocator *const ctx_alloc, -+ u64 const heap_gpu_va) -+{ -+ struct kbase_context *const kctx = ctx_alloc->kctx; -+ u64 ctx_offset = 0; -+ unsigned int heap_nr = 0; -+ -+ lockdep_assert_held(&ctx_alloc->lock); -+ -+ if (WARN_ON(!ctx_alloc->region)) -+ return; -+ -+ if (WARN_ON(heap_gpu_va < ctx_alloc->gpu_va)) -+ return; -+ -+ ctx_offset = heap_gpu_va - ctx_alloc->gpu_va; -+ -+ if (WARN_ON(ctx_offset >= HEAP_CTX_REGION_SIZE) || -+ WARN_ON(ctx_offset % HEAP_CTX_SIZE)) -+ return; -+ -+ heap_nr = ctx_offset / HEAP_CTX_SIZE; -+ dev_dbg(kctx->kbdev->dev, -+ "Freed tiler heap context %d (0x%llX)\n", heap_nr, heap_gpu_va); -+ -+ bitmap_clear(ctx_alloc->in_use, heap_nr, 1); -+} -+ -+int kbase_csf_heap_context_allocator_init( -+ struct kbase_csf_heap_context_allocator *const ctx_alloc, -+ struct kbase_context *const kctx) -+{ -+ /* We cannot pre-allocate GPU memory here because the -+ * custom VA zone may not have been created yet. -+ */ -+ ctx_alloc->kctx = kctx; -+ ctx_alloc->region = NULL; -+ ctx_alloc->gpu_va = 0; -+ -+ mutex_init(&ctx_alloc->lock); -+ bitmap_zero(ctx_alloc->in_use, MAX_TILER_HEAPS); -+ -+ dev_dbg(kctx->kbdev->dev, -+ "Initialized a tiler heap context allocator\n"); -+ -+ return 0; -+} -+ -+void kbase_csf_heap_context_allocator_term( -+ struct kbase_csf_heap_context_allocator *const ctx_alloc) -+{ -+ struct kbase_context *const kctx = ctx_alloc->kctx; -+ -+ dev_dbg(kctx->kbdev->dev, -+ "Terminating tiler heap context allocator\n"); -+ -+ if (ctx_alloc->region) { -+ kbase_gpu_vm_lock(kctx); -+ ctx_alloc->region->flags &= ~KBASE_REG_NO_USER_FREE; -+ kbase_mem_free_region(kctx, ctx_alloc->region); -+ kbase_gpu_vm_unlock(kctx); -+ } -+ -+ mutex_destroy(&ctx_alloc->lock); -+} -+ -+u64 kbase_csf_heap_context_allocator_alloc( -+ struct kbase_csf_heap_context_allocator *const ctx_alloc) -+{ -+ struct kbase_context *const kctx = ctx_alloc->kctx; -+ u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | -+ BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE; -+ u64 nr_pages = PFN_UP(HEAP_CTX_REGION_SIZE); -+ u64 heap_gpu_va = 0; -+ -+#ifdef CONFIG_MALI_VECTOR_DUMP -+ flags |= BASE_MEM_PROT_CPU_RD; -+#endif -+ -+ mutex_lock(&ctx_alloc->lock); -+ -+ /* If the pool of heap contexts wasn't already allocated then -+ * allocate it. -+ */ -+ if (!ctx_alloc->region) { -+ ctx_alloc->region = kbase_mem_alloc(kctx, nr_pages, nr_pages, -+ 0, &flags, &ctx_alloc->gpu_va); -+ } -+ -+ /* If the pool still isn't allocated then an error occurred. */ -+ if (unlikely(!ctx_alloc->region)) { -+ dev_err(kctx->kbdev->dev, "Failed to allocate a pool of tiler heap contexts\n"); -+ } else { -+ heap_gpu_va = sub_alloc(ctx_alloc); -+ } -+ -+ mutex_unlock(&ctx_alloc->lock); -+ -+ return heap_gpu_va; -+} -+ -+void kbase_csf_heap_context_allocator_free( -+ struct kbase_csf_heap_context_allocator *const ctx_alloc, -+ u64 const heap_gpu_va) -+{ -+ mutex_lock(&ctx_alloc->lock); -+ sub_free(ctx_alloc, heap_gpu_va); -+ mutex_unlock(&ctx_alloc->lock); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_heap_context_alloc.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_heap_context_alloc.h -new file mode 100644 -index 0000000..993db63 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_heap_context_alloc.h -@@ -0,0 +1,75 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+ -+#ifndef _KBASE_CSF_HEAP_CONTEXT_ALLOC_H_ -+#define _KBASE_CSF_HEAP_CONTEXT_ALLOC_H_ -+ -+/** -+ * kbase_csf_heap_context_allocator_init - Initialize an allocator for heap -+ * contexts -+ * @ctx_alloc: Pointer to the heap context allocator to initialize. -+ * @kctx: Pointer to the kbase context. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+int kbase_csf_heap_context_allocator_init( -+ struct kbase_csf_heap_context_allocator *const ctx_alloc, -+ struct kbase_context *const kctx); -+ -+/** -+ * kbase_csf_heap_context_allocator_term - Terminate an allocator for heap -+ * contexts -+ * @ctx_alloc: Pointer to the heap context allocator to terminate. -+ */ -+void kbase_csf_heap_context_allocator_term( -+ struct kbase_csf_heap_context_allocator *const ctx_alloc); -+ -+/** -+ * kbase_csf_heap_context_allocator_alloc - Allocate a heap context structure -+ * -+ * If this function is successful then it returns the address of a -+ * zero-initialized heap context structure for use by the firmware. -+ * -+ * @ctx_alloc: Pointer to the heap context allocator. -+ * -+ * Return: GPU virtual address of the allocated heap context or 0 on failure. -+ */ -+u64 kbase_csf_heap_context_allocator_alloc( -+ struct kbase_csf_heap_context_allocator *const ctx_alloc); -+ -+/** -+ * kbase_csf_heap_context_allocator_free - Free a heap context structure -+ * -+ * This function returns a heap context structure to the free pool of unused -+ * contexts for possible reuse by a future call to -+ * @kbase_csf_heap_context_allocator_alloc. -+ * -+ * @ctx_alloc: Pointer to the heap context allocator. -+ * @heap_gpu_va: The GPU virtual address of a heap context structure that -+ * was allocated for the firmware. -+ */ -+void kbase_csf_heap_context_allocator_free( -+ struct kbase_csf_heap_context_allocator *const ctx_alloc, -+ u64 const heap_gpu_va); -+ -+#endif /* _KBASE_CSF_HEAP_CONTEXT_ALLOC_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu.c -new file mode 100644 -index 0000000..4e26a49 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu.c -@@ -0,0 +1,2258 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <tl/mali_kbase_tracepoints.h> -+#include <mali_kbase_ctx_sched.h> -+#include "device/mali_kbase_device.h" -+#include "mali_kbase_csf.h" -+#include <linux/export.h> -+ -+#if IS_ENABLED(CONFIG_SYNC_FILE) -+#include "mali_kbase_fence.h" -+#include "mali_kbase_sync.h" -+ -+static DEFINE_SPINLOCK(kbase_csf_fence_lock); -+#endif -+ -+static void kcpu_queue_process(struct kbase_kcpu_command_queue *kcpu_queue, -+ bool ignore_waits); -+ -+static void kcpu_queue_process_worker(struct work_struct *data); -+ -+static int kbase_kcpu_map_import_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_import_info *import_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ struct kbase_va_region *reg; -+ int ret = 0; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ /* Take the processes mmap lock */ -+ down_read(kbase_mem_get_process_mmap_lock()); -+ kbase_gpu_vm_lock(kctx); -+ -+ reg = kbase_region_tracker_find_region_enclosing_address(kctx, -+ import_info->handle); -+ -+ if (kbase_is_region_invalid_or_free(reg) || -+ !kbase_mem_is_imported(reg->gpu_alloc->type)) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { -+ /* Pin the physical pages backing the user buffer while -+ * we are in the process context and holding the mmap lock. -+ * The dma mapping & GPU mapping of the pages would be done -+ * when the MAP_IMPORT operation is executed. -+ * -+ * Though the pages would be pinned, no reference is taken -+ * on the physical pages tracking object. When the last -+ * reference to the tracking object is dropped the pages -+ * would be unpinned if they weren't unpinned before. -+ */ -+ ret = kbase_jd_user_buf_pin_pages(kctx, reg); -+ if (ret) -+ goto out; -+ } -+ -+ current_command->type = BASE_KCPU_COMMAND_TYPE_MAP_IMPORT; -+ current_command->info.import.gpu_va = import_info->handle; -+ -+out: -+ kbase_gpu_vm_unlock(kctx); -+ /* Release the processes mmap lock */ -+ up_read(kbase_mem_get_process_mmap_lock()); -+ -+ return ret; -+} -+ -+static int kbase_kcpu_unmap_import_prepare_internal( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_import_info *import_info, -+ struct kbase_kcpu_command *current_command, -+ enum base_kcpu_command_type type) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ struct kbase_va_region *reg; -+ int ret = 0; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ kbase_gpu_vm_lock(kctx); -+ -+ reg = kbase_region_tracker_find_region_enclosing_address(kctx, -+ import_info->handle); -+ -+ if (kbase_is_region_invalid_or_free(reg) || -+ !kbase_mem_is_imported(reg->gpu_alloc->type)) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ if (reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_USER_BUF) { -+ /* The pages should have been pinned when MAP_IMPORT -+ * was enqueued previously. -+ */ -+ if (reg->gpu_alloc->nents != -+ reg->gpu_alloc->imported.user_buf.nr_pages) { -+ ret = -EINVAL; -+ goto out; -+ } -+ } -+ -+ current_command->type = type; -+ current_command->info.import.gpu_va = import_info->handle; -+ -+out: -+ kbase_gpu_vm_unlock(kctx); -+ -+ return ret; -+} -+ -+static int kbase_kcpu_unmap_import_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_import_info *import_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ return kbase_kcpu_unmap_import_prepare_internal(kcpu_queue, -+ import_info, current_command, -+ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT); -+} -+ -+static int kbase_kcpu_unmap_import_force_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_import_info *import_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ return kbase_kcpu_unmap_import_prepare_internal(kcpu_queue, -+ import_info, current_command, -+ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE); -+} -+ -+/** -+ * kbase_jit_add_to_pending_alloc_list() - Pend JIT allocation -+ * -+ * @queue: The queue containing this JIT allocation -+ * @cmd: The JIT allocation that is blocking this queue -+ */ -+static void kbase_jit_add_to_pending_alloc_list( -+ struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command *cmd) -+{ -+ struct kbase_context *const kctx = queue->kctx; -+ struct list_head *target_list_head = -+ &kctx->csf.kcpu_queues.jit_blocked_queues; -+ struct kbase_kcpu_command_queue *blocked_queue; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ list_for_each_entry(blocked_queue, -+ &kctx->csf.kcpu_queues.jit_blocked_queues, -+ jit_blocked) { -+ struct kbase_kcpu_command const*const jit_alloc_cmd = -+ &blocked_queue->commands[blocked_queue->start_offset]; -+ -+ WARN_ON(jit_alloc_cmd->type != BASE_KCPU_COMMAND_TYPE_JIT_ALLOC); -+ if (cmd->enqueue_ts < jit_alloc_cmd->enqueue_ts) { -+ target_list_head = &blocked_queue->jit_blocked; -+ break; -+ } -+ } -+ -+ list_add_tail(&queue->jit_blocked, target_list_head); -+} -+ -+/** -+ * kbase_kcpu_jit_allocate_process() - Process JIT allocation -+ * -+ * @queue: The queue containing this JIT allocation -+ * @cmd: The JIT allocation command -+ */ -+static int kbase_kcpu_jit_allocate_process( -+ struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command *cmd) -+{ -+ struct kbase_context *const kctx = queue->kctx; -+ struct kbase_kcpu_command_jit_alloc_info *alloc_info = -+ &cmd->info.jit_alloc; -+ struct base_jit_alloc_info *info = alloc_info->info; -+ struct kbase_vmap_struct mapping; -+ struct kbase_va_region *reg; -+ u32 count = alloc_info->count; -+ u64 *ptr, new_addr; -+ u32 i; -+ int ret; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ if (alloc_info->blocked) { -+ list_del(&queue->jit_blocked); -+ alloc_info->blocked = false; -+ } -+ -+ if (WARN_ON(!info)) -+ return -EINVAL; -+ -+ /* Check if all JIT IDs are not in use */ -+ for (i = 0; i < count; i++, info++) { -+ /* The JIT ID is still in use so fail the allocation */ -+ if (kctx->jit_alloc[info->id]) { -+ dev_warn(kctx->kbdev->dev, "JIT ID still in use\n"); -+ return -EINVAL; -+ } -+ } -+ -+ /* Now start the allocation loop */ -+ for (i = 0, info = alloc_info->info; i < count; i++, info++) { -+ /* Create a JIT allocation */ -+ reg = kbase_jit_allocate(kctx, info, true); -+ if (!reg) { -+ bool can_block = false; -+ struct kbase_kcpu_command const *jit_cmd; -+ -+ list_for_each_entry(jit_cmd, &kctx->csf.kcpu_queues.jit_cmds_head, info.jit_alloc.node) { -+ if (jit_cmd == cmd) -+ break; -+ -+ if (jit_cmd->type == BASE_KCPU_COMMAND_TYPE_JIT_FREE) { -+ u8 const*const free_ids = jit_cmd->info.jit_free.ids; -+ -+ if (free_ids && *free_ids && kctx->jit_alloc[*free_ids]) { -+ /* -+ * A JIT free which is active -+ * and submitted before this -+ * command. -+ */ -+ can_block = true; -+ break; -+ } -+ } -+ } -+ -+ if (!can_block) { -+ /* -+ * No prior JIT_FREE command is active. Roll -+ * back previous allocations and fail. -+ */ -+ dev_warn_ratelimited(kctx->kbdev->dev, "JIT alloc command failed: %pK\n", cmd); -+ ret = -ENOMEM; -+ goto fail; -+ } -+ -+ /* There are pending frees for an active allocation -+ * so we should wait to see whether they free the -+ * memory. Add to the list of atoms for which JIT -+ * allocation is pending. -+ */ -+ kbase_jit_add_to_pending_alloc_list(queue, cmd); -+ alloc_info->blocked = true; -+ -+ /* Rollback, the whole set will be re-attempted */ -+ while (i-- > 0) { -+ info--; -+ kbase_jit_free(kctx, kctx->jit_alloc[info->id]); -+ kctx->jit_alloc[info->id] = NULL; -+ } -+ -+ return -EAGAIN; -+ } -+ -+ /* Bind it to the user provided ID. */ -+ kctx->jit_alloc[info->id] = reg; -+ } -+ -+ for (i = 0, info = alloc_info->info; i < count; i++, info++) { -+ /* -+ * Write the address of the JIT allocation to the user provided -+ * GPU allocation. -+ */ -+ ptr = kbase_vmap(kctx, info->gpu_alloc_addr, sizeof(*ptr), -+ &mapping); -+ if (!ptr) { -+ ret = -ENOMEM; -+ goto fail; -+ } -+ -+ reg = kctx->jit_alloc[info->id]; -+ new_addr = reg->start_pfn << PAGE_SHIFT; -+ *ptr = new_addr; -+ kbase_vunmap(kctx, &mapping); -+ } -+ -+ return 0; -+ -+fail: -+ /* Roll back completely */ -+ for (i = 0, info = alloc_info->info; i < count; i++, info++) { -+ /* Free the allocations that were successful. -+ * Mark all the allocations including the failed one and the -+ * other un-attempted allocations in the set, so we know they -+ * are in use. -+ */ -+ if (kctx->jit_alloc[info->id]) -+ kbase_jit_free(kctx, kctx->jit_alloc[info->id]); -+ -+ kctx->jit_alloc[info->id] = KBASE_RESERVED_REG_JIT_ALLOC; -+ } -+ -+ return ret; -+} -+ -+static int kbase_kcpu_jit_allocate_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_jit_alloc_info *alloc_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ void __user *data = u64_to_user_ptr(alloc_info->info); -+ struct base_jit_alloc_info *info; -+ u32 count = alloc_info->count; -+ int ret = 0; -+ u32 i; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ if (!data || count > kcpu_queue->kctx->jit_max_allocations || -+ count > ARRAY_SIZE(kctx->jit_alloc)) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ info = kmalloc_array(count, sizeof(*info), GFP_KERNEL); -+ if (!info) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ if (copy_from_user(info, data, sizeof(*info) * count) != 0) { -+ ret = -EINVAL; -+ goto out_free; -+ } -+ -+ for (i = 0; i < count; i++) { -+ ret = kbasep_jit_alloc_validate(kctx, &info[i]); -+ if (ret) -+ goto out_free; -+ } -+ -+ /* Search for duplicate JIT ids */ -+ for (i = 0; i < (count - 1); i++) { -+ u32 j; -+ -+ for (j = (i + 1); j < count; j++) { -+ if (info[i].id == info[j].id) { -+ ret = -EINVAL; -+ goto out_free; -+ } -+ } -+ } -+ -+ current_command->type = BASE_KCPU_COMMAND_TYPE_JIT_ALLOC; -+ list_add_tail(¤t_command->info.jit_alloc.node, -+ &kctx->csf.kcpu_queues.jit_cmds_head); -+ current_command->info.jit_alloc.info = info; -+ current_command->info.jit_alloc.count = count; -+ current_command->info.jit_alloc.blocked = false; -+ -+ return 0; -+out_free: -+ kfree(info); -+out: -+ return ret; -+} -+ -+/** -+ * kbase_kcpu_jit_allocate_finish() - Finish handling the JIT_ALLOC command -+ * -+ * @queue: The queue containing this JIT allocation -+ * @cmd: The JIT allocation command -+ */ -+static void kbase_kcpu_jit_allocate_finish( -+ struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command *cmd) -+{ -+ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); -+ -+ /* Remove this command from the jit_cmds_head list */ -+ list_del(&cmd->info.jit_alloc.node); -+ -+ /* -+ * If we get to this point we must have already cleared the blocked -+ * flag, otherwise it'd be a bug. -+ */ -+ if (WARN_ON(cmd->info.jit_alloc.blocked)) { -+ list_del(&queue->jit_blocked); -+ cmd->info.jit_alloc.blocked = false; -+ } -+ -+ kfree(cmd->info.jit_alloc.info); -+} -+ -+/** -+ * kbase_kcpu_jit_retry_pending_allocs() - Retry blocked JIT_ALLOC commands -+ * -+ * @kctx: The context containing the blocked JIT_ALLOC commands -+ */ -+static void kbase_kcpu_jit_retry_pending_allocs(struct kbase_context *kctx) -+{ -+ struct kbase_kcpu_command_queue *blocked_queue; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ /* -+ * Reschedule all queues blocked by JIT_ALLOC commands. -+ * NOTE: This code traverses the list of blocked queues directly. It -+ * only works as long as the queued works are not executed at the same -+ * time. This precondition is true since we're holding the -+ * kbase_csf_kcpu_queue_context.lock . -+ */ -+ list_for_each_entry(blocked_queue, -+ &kctx->csf.kcpu_queues.jit_blocked_queues, jit_blocked) -+ queue_work(kctx->csf.kcpu_queues.wq, &blocked_queue->work); -+} -+ -+static int kbase_kcpu_jit_free_process(struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command *const cmd) -+{ -+ struct kbase_kcpu_command_jit_free_info const *const free_info = -+ &cmd->info.jit_free; -+ u8 const *const ids = free_info->ids; -+ u32 const count = free_info->count; -+ u32 i; -+ int rc = 0; -+ struct kbase_context *kctx = queue->kctx; -+ -+ if (WARN_ON(!ids)) -+ return -EINVAL; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( -+ queue->kctx->kbdev, queue); -+ -+ for (i = 0; i < count; i++) { -+ u64 pages_used = 0; -+ int item_err = 0; -+ -+ if (!kctx->jit_alloc[ids[i]]) { -+ dev_warn(kctx->kbdev->dev, "invalid JIT free ID\n"); -+ rc = -EINVAL; -+ item_err = rc; -+ } else { -+ struct kbase_va_region *const reg = kctx->jit_alloc[ids[i]]; -+ -+ /* -+ * If the ID is valid but the allocation request failed, still -+ * succeed this command but don't try and free the allocation. -+ */ -+ if (reg != KBASE_RESERVED_REG_JIT_ALLOC) { -+ pages_used = reg->gpu_alloc->nents; -+ kbase_jit_free(kctx, reg); -+ } -+ -+ kctx->jit_alloc[ids[i]] = NULL; -+ } -+ -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( -+ queue->kctx->kbdev, queue, item_err, pages_used); -+ } -+ -+ /* Free the list of ids */ -+ kfree(ids); -+ -+ /* -+ * Remove this command from the jit_cmds_head list and retry pending -+ * allocations. -+ */ -+ list_del(&cmd->info.jit_free.node); -+ kbase_kcpu_jit_retry_pending_allocs(kctx); -+ -+ return rc; -+} -+ -+static int kbase_kcpu_jit_free_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_jit_free_info *free_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ void __user *data = u64_to_user_ptr(free_info->ids); -+ u8 *ids; -+ u32 count = free_info->count; -+ int ret; -+ u32 i; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ /* Sanity checks */ -+ if (!count || count > ARRAY_SIZE(kctx->jit_alloc)) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ /* Copy the information for safe access and future storage */ -+ ids = kmalloc_array(count, sizeof(*ids), GFP_KERNEL); -+ if (!ids) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ if (!data) { -+ ret = -EINVAL; -+ goto out_free; -+ } -+ -+ if (copy_from_user(ids, data, sizeof(*ids) * count)) { -+ ret = -EINVAL; -+ goto out_free; -+ } -+ -+ for (i = 0; i < count; i++) { -+ /* Fail the command if ID sent is zero */ -+ if (!ids[i]) { -+ ret = -EINVAL; -+ goto out_free; -+ } -+ } -+ -+ /* Search for duplicate JIT ids */ -+ for (i = 0; i < (count - 1); i++) { -+ u32 j; -+ -+ for (j = (i + 1); j < count; j++) { -+ if (ids[i] == ids[j]) { -+ ret = -EINVAL; -+ goto out_free; -+ } -+ } -+ } -+ -+ current_command->type = BASE_KCPU_COMMAND_TYPE_JIT_FREE; -+ list_add_tail(¤t_command->info.jit_free.node, -+ &kctx->csf.kcpu_queues.jit_cmds_head); -+ current_command->info.jit_free.ids = ids; -+ current_command->info.jit_free.count = count; -+ -+ return 0; -+out_free: -+ kfree(ids); -+out: -+ return ret; -+} -+ -+static int kbase_csf_queue_group_suspend_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_group_suspend_info *suspend_buf, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ struct kbase_suspend_copy_buffer *sus_buf = NULL; -+ u64 addr = suspend_buf->buffer; -+ u64 page_addr = addr & PAGE_MASK; -+ u64 end_addr = addr + suspend_buf->size - 1; -+ u64 last_page_addr = end_addr & PAGE_MASK; -+ int nr_pages = (last_page_addr - page_addr) / PAGE_SIZE + 1; -+ int pinned_pages = 0, ret = 0; -+ struct kbase_va_region *reg; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ if (suspend_buf->size < -+ kctx->kbdev->csf.global_iface.groups[0].suspend_size) -+ return -EINVAL; -+ -+ ret = kbase_csf_queue_group_handle_is_valid(kctx, -+ suspend_buf->group_handle); -+ if (ret) -+ return ret; -+ -+ sus_buf = kzalloc(sizeof(*sus_buf), GFP_KERNEL); -+ if (!sus_buf) -+ return -ENOMEM; -+ -+ sus_buf->size = suspend_buf->size; -+ sus_buf->nr_pages = nr_pages; -+ sus_buf->offset = addr & ~PAGE_MASK; -+ -+ sus_buf->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL); -+ if (!sus_buf->pages) { -+ ret = -ENOMEM; -+ goto out_clean_sus_buf; -+ } -+ -+ /* Check if the page_addr is a valid GPU VA from SAME_VA zone, -+ * otherwise consider it is a CPU VA corresponding to the Host -+ * memory allocated by userspace. -+ */ -+ kbase_gpu_vm_lock(kctx); -+ reg = kbase_region_tracker_find_region_enclosing_address(kctx, -+ page_addr); -+ -+ if (kbase_is_region_invalid_or_free(reg)) { -+ kbase_gpu_vm_unlock(kctx); -+ pinned_pages = get_user_pages_fast(page_addr, nr_pages, 1, -+ sus_buf->pages); -+ kbase_gpu_vm_lock(kctx); -+ -+ if (pinned_pages < 0) { -+ ret = pinned_pages; -+ goto out_clean_pages; -+ } -+ if (pinned_pages != nr_pages) { -+ ret = -EINVAL; -+ goto out_clean_pages; -+ } -+ } else { -+ struct tagged_addr *page_array; -+ u64 start, end, i; -+ -+ if (!(reg->flags & BASE_MEM_SAME_VA) || -+ reg->nr_pages < nr_pages || -+ kbase_reg_current_backed_size(reg) != -+ reg->nr_pages) { -+ ret = -EINVAL; -+ goto out_clean_pages; -+ } -+ -+ start = PFN_DOWN(page_addr) - reg->start_pfn; -+ end = start + nr_pages; -+ -+ if (end > reg->nr_pages) { -+ ret = -EINVAL; -+ goto out_clean_pages; -+ } -+ -+ sus_buf->cpu_alloc = kbase_mem_phy_alloc_get(reg->cpu_alloc); -+ kbase_mem_phy_alloc_kernel_mapped(reg->cpu_alloc); -+ page_array = kbase_get_cpu_phy_pages(reg); -+ page_array += start; -+ -+ for (i = 0; i < nr_pages; i++, page_array++) -+ sus_buf->pages[i] = as_page(*page_array); -+ } -+ -+ kbase_gpu_vm_unlock(kctx); -+ current_command->type = BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND; -+ current_command->info.suspend_buf_copy.sus_buf = sus_buf; -+ current_command->info.suspend_buf_copy.group_handle = -+ suspend_buf->group_handle; -+ return ret; -+ -+out_clean_pages: -+ kbase_gpu_vm_unlock(kctx); -+ kfree(sus_buf->pages); -+out_clean_sus_buf: -+ kfree(sus_buf); -+ -+ return ret; -+} -+ -+static int kbase_csf_queue_group_suspend_process(struct kbase_context *kctx, -+ struct kbase_suspend_copy_buffer *sus_buf, -+ u8 group_handle) -+{ -+ return kbase_csf_queue_group_suspend(kctx, sus_buf, group_handle); -+} -+ -+static enum kbase_csf_event_callback_action event_cqs_callback(void *param) -+{ -+ struct kbase_kcpu_command_queue *kcpu_queue = -+ (struct kbase_kcpu_command_queue *)param; -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ -+ queue_work(kctx->csf.kcpu_queues.wq, &kcpu_queue->work); -+ -+ return KBASE_CSF_EVENT_CALLBACK_KEEP; -+} -+ -+static void cleanup_cqs_wait(struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command_cqs_wait_info *cqs_wait) -+{ -+ WARN_ON(!cqs_wait->nr_objs); -+ WARN_ON(!cqs_wait->objs); -+ WARN_ON(!cqs_wait->signaled); -+ WARN_ON(!queue->cqs_wait_count); -+ -+ if (--queue->cqs_wait_count == 0) { -+ kbase_csf_event_wait_remove(queue->kctx, -+ event_cqs_callback, queue); -+ } -+ -+ kfree(cqs_wait->signaled); -+ kfree(cqs_wait->objs); -+ cqs_wait->signaled = NULL; -+ cqs_wait->objs = NULL; -+} -+ -+static int kbase_kcpu_cqs_wait_process(struct kbase_device *kbdev, -+ struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command_cqs_wait_info *cqs_wait) -+{ -+ u32 i; -+ -+ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); -+ -+ if (WARN_ON(!cqs_wait->objs)) -+ return -EINVAL; -+ -+ /* Skip the CQS waits that have already been signaled when processing */ -+ for (i = find_first_zero_bit(cqs_wait->signaled, cqs_wait->nr_objs); i < cqs_wait->nr_objs; i++) { -+ if (!test_bit(i, cqs_wait->signaled)) { -+ struct kbase_vmap_struct *mapping; -+ bool sig_set; -+ u32 *evt = (u32 *)kbase_phy_alloc_mapping_get(queue->kctx, -+ cqs_wait->objs[i].addr, &mapping); -+ -+ if (!queue->command_started) { -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( -+ kbdev, queue); -+ queue->command_started = true; -+ KBASE_KTRACE_ADD_CSF_KCPU(kbdev, CQS_WAIT_START, -+ queue, cqs_wait->nr_objs, 0); -+ } -+ -+ if (!evt) { -+ dev_warn(kbdev->dev, -+ "Sync memory %llx already freed", cqs_wait->objs[i].addr); -+ queue->has_error = true; -+ return -EINVAL; -+ } -+ -+ sig_set = evt[BASEP_EVENT_VAL_INDEX] > cqs_wait->objs[i].val; -+ if (sig_set) { -+ bool error = false; -+ -+ bitmap_set(cqs_wait->signaled, i, 1); -+ if ((cqs_wait->inherit_err_flags & (1U << i)) && -+ evt[BASEP_EVENT_ERR_INDEX] > 0) { -+ queue->has_error = true; -+ error = true; -+ } -+ -+ KBASE_KTRACE_ADD_CSF_KCPU(kbdev, CQS_WAIT_END, -+ queue, cqs_wait->objs[i].addr, -+ error); -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( -+ kbdev, queue, -+ evt[BASEP_EVENT_ERR_INDEX]); -+ queue->command_started = false; -+ } -+ -+ kbase_phy_alloc_mapping_put(queue->kctx, mapping); -+ -+ if (!sig_set) -+ break; -+ } -+ } -+ -+ /* For the queue to progress further, all cqs objects should get -+ * signaled. -+ */ -+ return bitmap_full(cqs_wait->signaled, cqs_wait->nr_objs); -+} -+ -+static int kbase_kcpu_cqs_wait_prepare(struct kbase_kcpu_command_queue *queue, -+ struct base_kcpu_command_cqs_wait_info *cqs_wait_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct base_cqs_wait_info *objs; -+ unsigned int nr_objs = cqs_wait_info->nr_objs; -+ -+ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); -+ -+ if (nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS) -+ return -EINVAL; -+ -+ if (!nr_objs) -+ return -EINVAL; -+ -+ objs = kcalloc(nr_objs, sizeof(*objs), GFP_KERNEL); -+ if (!objs) -+ return -ENOMEM; -+ -+ if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_info->objs), -+ nr_objs * sizeof(*objs))) { -+ kfree(objs); -+ return -ENOMEM; -+ } -+ -+ if (++queue->cqs_wait_count == 1) { -+ if (kbase_csf_event_wait_add(queue->kctx, -+ event_cqs_callback, queue)) { -+ kfree(objs); -+ queue->cqs_wait_count--; -+ return -ENOMEM; -+ } -+ } -+ -+ current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_WAIT; -+ current_command->info.cqs_wait.nr_objs = nr_objs; -+ current_command->info.cqs_wait.objs = objs; -+ current_command->info.cqs_wait.inherit_err_flags = -+ cqs_wait_info->inherit_err_flags; -+ -+ current_command->info.cqs_wait.signaled = kcalloc(BITS_TO_LONGS(nr_objs), -+ sizeof(*current_command->info.cqs_wait.signaled), GFP_KERNEL); -+ if (!current_command->info.cqs_wait.signaled) { -+ if (--queue->cqs_wait_count == 0) { -+ kbase_csf_event_wait_remove(queue->kctx, -+ event_cqs_callback, queue); -+ } -+ -+ kfree(objs); -+ return -ENOMEM; -+ } -+ -+ return 0; -+} -+ -+static void kbase_kcpu_cqs_set_process(struct kbase_device *kbdev, -+ struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command_cqs_set_info *cqs_set) -+{ -+ unsigned int i; -+ -+ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); -+ -+ if (WARN_ON(!cqs_set->objs)) -+ return; -+ -+ for (i = 0; i < cqs_set->nr_objs; i++) { -+ struct kbase_vmap_struct *mapping; -+ u32 *evt; -+ -+ evt = (u32 *)kbase_phy_alloc_mapping_get( -+ queue->kctx, cqs_set->objs[i].addr, &mapping); -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET(kbdev, queue, -+ evt ? 0 : 1); -+ -+ if (!evt) { -+ dev_warn(kbdev->dev, -+ "Sync memory %llx already freed", cqs_set->objs[i].addr); -+ queue->has_error = true; -+ } else { -+ evt[BASEP_EVENT_ERR_INDEX] = queue->has_error; -+ /* Set to signaled */ -+ evt[BASEP_EVENT_VAL_INDEX]++; -+ kbase_phy_alloc_mapping_put(queue->kctx, mapping); -+ -+ KBASE_KTRACE_ADD_CSF_KCPU(kbdev, CQS_SET, -+ queue, cqs_set->objs[i].addr, -+ evt[BASEP_EVENT_ERR_INDEX]); -+ } -+ } -+ -+ kbase_csf_event_signal_notify_gpu(queue->kctx); -+ -+ kfree(cqs_set->objs); -+ cqs_set->objs = NULL; -+} -+ -+static int kbase_kcpu_cqs_set_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_cqs_set_info *cqs_set_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ struct base_cqs_set *objs; -+ unsigned int nr_objs = cqs_set_info->nr_objs; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ if (nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS) -+ return -EINVAL; -+ -+ if (!nr_objs) -+ return -EINVAL; -+ -+ objs = kcalloc(nr_objs, sizeof(*objs), GFP_KERNEL); -+ if (!objs) -+ return -ENOMEM; -+ -+ if (copy_from_user(objs, u64_to_user_ptr(cqs_set_info->objs), -+ nr_objs * sizeof(*objs))) { -+ kfree(objs); -+ return -ENOMEM; -+ } -+ -+ current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_SET; -+ current_command->info.cqs_set.nr_objs = nr_objs; -+ current_command->info.cqs_set.objs = objs; -+ -+ return 0; -+} -+ -+static void cleanup_cqs_wait_operation(struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command_cqs_wait_operation_info *cqs_wait_operation) -+{ -+ WARN_ON(!cqs_wait_operation->nr_objs); -+ WARN_ON(!cqs_wait_operation->objs); -+ WARN_ON(!cqs_wait_operation->signaled); -+ WARN_ON(!queue->cqs_wait_count); -+ -+ if (--queue->cqs_wait_count == 0) { -+ kbase_csf_event_wait_remove(queue->kctx, -+ event_cqs_callback, queue); -+ } -+ -+ kfree(cqs_wait_operation->signaled); -+ kfree(cqs_wait_operation->objs); -+ cqs_wait_operation->signaled = NULL; -+ cqs_wait_operation->objs = NULL; -+} -+ -+static int kbase_kcpu_cqs_wait_operation_process(struct kbase_device *kbdev, -+ struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command_cqs_wait_operation_info *cqs_wait_operation) -+{ -+ u32 i; -+ -+ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); -+ -+ if (WARN_ON(!cqs_wait_operation->objs)) -+ return -EINVAL; -+ -+ /* Skip the CQS waits that have already been signaled when processing */ -+ for (i = find_first_zero_bit(cqs_wait_operation->signaled, cqs_wait_operation->nr_objs); i < cqs_wait_operation->nr_objs; i++) { -+ if (!test_bit(i, cqs_wait_operation->signaled)) { -+ struct kbase_vmap_struct *mapping; -+ bool sig_set; -+ u64 *evt = (u64 *)kbase_phy_alloc_mapping_get(queue->kctx, -+ cqs_wait_operation->objs[i].addr, &mapping); -+ -+ /* GPUCORE-28172 RDT to review */ -+ if (!queue->command_started) -+ queue->command_started = true; -+ -+ if (!evt) { -+ dev_warn(kbdev->dev, -+ "Sync memory %llx already freed", cqs_wait_operation->objs[i].addr); -+ queue->has_error = true; -+ return -EINVAL; -+ } -+ -+ switch (cqs_wait_operation->objs[i].operation) { -+ case BASEP_CQS_WAIT_OPERATION_LE: -+ sig_set = *evt <= cqs_wait_operation->objs[i].val; -+ break; -+ case BASEP_CQS_WAIT_OPERATION_GT: -+ sig_set = *evt > cqs_wait_operation->objs[i].val; -+ break; -+ default: -+ dev_warn(kbdev->dev, -+ "Unsupported CQS wait operation %d", cqs_wait_operation->objs[i].operation); -+ -+ kbase_phy_alloc_mapping_put(queue->kctx, mapping); -+ queue->has_error = true; -+ -+ return -EINVAL; -+ } -+ -+ /* Increment evt up to the error_state value depending on the CQS data type */ -+ switch (cqs_wait_operation->objs[i].data_type) { -+ default: -+ dev_warn(kbdev->dev, "Unreachable data_type=%d", cqs_wait_operation->objs[i].data_type); -+ /* Fallthrough - hint to compiler that there's really only 2 options at present */ -+ case BASEP_CQS_DATA_TYPE_U32: -+ evt = (u64 *)((u8 *)evt + sizeof(u32)); -+ break; -+ case BASEP_CQS_DATA_TYPE_U64: -+ evt = (u64 *)((u8 *)evt + sizeof(u64)); -+ break; -+ } -+ -+ if (sig_set) { -+ bitmap_set(cqs_wait_operation->signaled, i, 1); -+ if ((cqs_wait_operation->inherit_err_flags & (1U << i)) && -+ *evt > 0) { -+ queue->has_error = true; -+ } -+ -+ /* GPUCORE-28172 RDT to review */ -+ -+ queue->command_started = false; -+ } -+ -+ kbase_phy_alloc_mapping_put(queue->kctx, mapping); -+ -+ if (!sig_set) -+ break; -+ } -+ } -+ -+ /* For the queue to progress further, all cqs objects should get -+ * signaled. -+ */ -+ return bitmap_full(cqs_wait_operation->signaled, cqs_wait_operation->nr_objs); -+} -+ -+static int kbase_kcpu_cqs_wait_operation_prepare(struct kbase_kcpu_command_queue *queue, -+ struct base_kcpu_command_cqs_wait_operation_info *cqs_wait_operation_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct base_cqs_wait_operation_info *objs; -+ unsigned int nr_objs = cqs_wait_operation_info->nr_objs; -+ -+ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); -+ -+ if (nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS) -+ return -EINVAL; -+ -+ if (!nr_objs) -+ return -EINVAL; -+ -+ objs = kcalloc(nr_objs, sizeof(*objs), GFP_KERNEL); -+ if (!objs) -+ return -ENOMEM; -+ -+ if (copy_from_user(objs, u64_to_user_ptr(cqs_wait_operation_info->objs), -+ nr_objs * sizeof(*objs))) { -+ kfree(objs); -+ return -ENOMEM; -+ } -+ -+ if (++queue->cqs_wait_count == 1) { -+ if (kbase_csf_event_wait_add(queue->kctx, -+ event_cqs_callback, queue)) { -+ kfree(objs); -+ queue->cqs_wait_count--; -+ return -ENOMEM; -+ } -+ } -+ -+ current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION; -+ current_command->info.cqs_wait_operation.nr_objs = nr_objs; -+ current_command->info.cqs_wait_operation.objs = objs; -+ current_command->info.cqs_wait_operation.inherit_err_flags = -+ cqs_wait_operation_info->inherit_err_flags; -+ -+ current_command->info.cqs_wait_operation.signaled = kcalloc(BITS_TO_LONGS(nr_objs), -+ sizeof(*current_command->info.cqs_wait_operation.signaled), GFP_KERNEL); -+ if (!current_command->info.cqs_wait_operation.signaled) { -+ if (--queue->cqs_wait_count == 0) { -+ kbase_csf_event_wait_remove(queue->kctx, -+ event_cqs_callback, queue); -+ } -+ -+ kfree(objs); -+ return -ENOMEM; -+ } -+ -+ return 0; -+} -+ -+static void kbase_kcpu_cqs_set_operation_process( -+ struct kbase_device *kbdev, -+ struct kbase_kcpu_command_queue *queue, -+ struct kbase_kcpu_command_cqs_set_operation_info *cqs_set_operation) -+{ -+ unsigned int i; -+ -+ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); -+ -+ if (WARN_ON(!cqs_set_operation->objs)) -+ return; -+ -+ for (i = 0; i < cqs_set_operation->nr_objs; i++) { -+ struct kbase_vmap_struct *mapping; -+ u64 *evt; -+ -+ evt = (u64 *)kbase_phy_alloc_mapping_get( -+ queue->kctx, cqs_set_operation->objs[i].addr, &mapping); -+ -+ /* GPUCORE-28172 RDT to review */ -+ -+ if (!evt) { -+ dev_warn(kbdev->dev, -+ "Sync memory %llx already freed", cqs_set_operation->objs[i].addr); -+ queue->has_error = true; -+ } else { -+ switch (cqs_set_operation->objs[i].operation) { -+ case BASEP_CQS_SET_OPERATION_ADD: -+ *evt += cqs_set_operation->objs[i].val; -+ break; -+ case BASEP_CQS_SET_OPERATION_SET: -+ *evt = cqs_set_operation->objs[i].val; -+ break; -+ default: -+ dev_warn(kbdev->dev, -+ "Unsupported CQS set operation %d", cqs_set_operation->objs[i].operation); -+ queue->has_error = true; -+ break; -+ } -+ -+ /* Increment evt up to the error_state value depending on the CQS data type */ -+ switch (cqs_set_operation->objs[i].data_type) { -+ default: -+ dev_warn(kbdev->dev, "Unreachable data_type=%d", cqs_set_operation->objs[i].data_type); -+ /* Fallthrough - hint to compiler that there's really only 2 options at present */ -+ case BASEP_CQS_DATA_TYPE_U32: -+ evt = (u64 *)((u8 *)evt + sizeof(u32)); -+ break; -+ case BASEP_CQS_DATA_TYPE_U64: -+ evt = (u64 *)((u8 *)evt + sizeof(u64)); -+ break; -+ } -+ -+ /* GPUCORE-28172 RDT to review */ -+ -+ /* Always propagate errors */ -+ *evt = queue->has_error; -+ -+ kbase_phy_alloc_mapping_put(queue->kctx, mapping); -+ } -+ } -+ -+ kbase_csf_event_signal_notify_gpu(queue->kctx); -+ -+ kfree(cqs_set_operation->objs); -+ cqs_set_operation->objs = NULL; -+} -+ -+static int kbase_kcpu_cqs_set_operation_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_cqs_set_operation_info *cqs_set_operation_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ struct base_cqs_set_operation_info *objs; -+ unsigned int nr_objs = cqs_set_operation_info->nr_objs; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ if (nr_objs > BASEP_KCPU_CQS_MAX_NUM_OBJS) -+ return -EINVAL; -+ -+ if (!nr_objs) -+ return -EINVAL; -+ -+ objs = kcalloc(nr_objs, sizeof(*objs), GFP_KERNEL); -+ if (!objs) -+ return -ENOMEM; -+ -+ if (copy_from_user(objs, u64_to_user_ptr(cqs_set_operation_info->objs), -+ nr_objs * sizeof(*objs))) { -+ kfree(objs); -+ return -ENOMEM; -+ } -+ -+ current_command->type = BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION; -+ current_command->info.cqs_set_operation.nr_objs = nr_objs; -+ current_command->info.cqs_set_operation.objs = objs; -+ -+ return 0; -+} -+ -+#if IS_ENABLED(CONFIG_SYNC_FILE) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+static void kbase_csf_fence_wait_callback(struct fence *fence, -+ struct fence_cb *cb) -+#else -+static void kbase_csf_fence_wait_callback(struct dma_fence *fence, -+ struct dma_fence_cb *cb) -+#endif -+{ -+ struct kbase_kcpu_command_fence_info *fence_info = container_of(cb, -+ struct kbase_kcpu_command_fence_info, fence_cb); -+ struct kbase_kcpu_command_queue *kcpu_queue = fence_info->kcpu_queue; -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ -+ KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, FENCE_WAIT_END, kcpu_queue, -+ fence->context, fence->seqno); -+ -+ /* Resume kcpu command queue processing. */ -+ queue_work(kctx->csf.kcpu_queues.wq, &kcpu_queue->work); -+} -+ -+static void kbase_kcpu_fence_wait_cancel( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct kbase_kcpu_command_fence_info *fence_info) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ if (WARN_ON(!fence_info->fence)) -+ return; -+ -+ if (kcpu_queue->fence_wait_processed) { -+ bool removed = dma_fence_remove_callback(fence_info->fence, -+ &fence_info->fence_cb); -+ -+ if (removed) -+ KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, FENCE_WAIT_END, -+ kcpu_queue, fence_info->fence->context, -+ fence_info->fence->seqno); -+ } -+ -+ /* Release the reference which is kept by the kcpu_queue */ -+ kbase_fence_put(fence_info->fence); -+ kcpu_queue->fence_wait_processed = false; -+ -+ fence_info->fence = NULL; -+} -+ -+/** -+ * kbase_kcpu_fence_wait_process() - Process the kcpu fence wait command -+ * -+ * @kcpu_queue: The queue containing the fence wait command -+ * @fence_info: Reference to a fence for which the command is waiting -+ * -+ * Return: 0 if fence wait is blocked, 1 if it is unblocked, negative error if -+ * an error has occurred and fence should no longer be waited on. -+ */ -+static int kbase_kcpu_fence_wait_process( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct kbase_kcpu_command_fence_info *fence_info) -+{ -+ int fence_status = 0; -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+ struct fence *fence; -+#else -+ struct dma_fence *fence; -+#endif -+ -+ lockdep_assert_held(&kcpu_queue->kctx->csf.kcpu_queues.lock); -+ -+ if (WARN_ON(!fence_info->fence)) -+ return -EINVAL; -+ -+ fence = fence_info->fence; -+ -+ if (kcpu_queue->fence_wait_processed) { -+ fence_status = dma_fence_get_status(fence); -+ } else { -+ int cb_err = dma_fence_add_callback(fence, -+ &fence_info->fence_cb, -+ kbase_csf_fence_wait_callback); -+ -+ KBASE_KTRACE_ADD_CSF_KCPU(kcpu_queue->kctx->kbdev, -+ FENCE_WAIT_START, kcpu_queue, -+ fence->context, fence->seqno); -+ fence_status = cb_err; -+ if (cb_err == 0) -+ kcpu_queue->fence_wait_processed = true; -+ else if (cb_err == -ENOENT) -+ fence_status = dma_fence_get_status(fence); -+ } -+ -+ /* -+ * At this point fence status can contain 3 types of values: -+ * - Value 0 to represent that fence in question is not signalled yet -+ * - Value 1 to represent that fence in question is signalled without -+ * errors -+ * - Negative error code to represent that some error has occurred such -+ * that waiting on it is no longer valid. -+ */ -+ -+ if (fence_status) -+ kbase_kcpu_fence_wait_cancel(kcpu_queue, fence_info); -+ -+ return fence_status; -+} -+ -+static int kbase_kcpu_fence_wait_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_fence_info *fence_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+ struct fence *fence_in; -+#else -+ struct dma_fence *fence_in; -+#endif -+ struct base_fence fence; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence), -+ sizeof(fence))) -+ return -ENOMEM; -+ -+ fence_in = sync_file_get_fence(fence.basep.fd); -+ -+ if (!fence_in) -+ return -ENOENT; -+ -+ current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_WAIT; -+ current_command->info.fence.fence = fence_in; -+ current_command->info.fence.kcpu_queue = kcpu_queue; -+ -+ return 0; -+} -+ -+static int kbase_kcpu_fence_signal_process( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct kbase_kcpu_command_fence_info *fence_info) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+ int ret; -+ -+ if (WARN_ON(!fence_info->fence)) -+ return -EINVAL; -+ -+ ret = dma_fence_signal(fence_info->fence); -+ -+ if (unlikely(ret < 0)) { -+ dev_warn(kctx->kbdev->dev, -+ "fence_signal() failed with %d\n", ret); -+ } -+ -+ KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, FENCE_SIGNAL, kcpu_queue, -+ fence_info->fence->context, -+ fence_info->fence->seqno); -+ -+ dma_fence_put(fence_info->fence); -+ fence_info->fence = NULL; -+ -+ return ret; -+} -+ -+static int kbase_kcpu_fence_signal_prepare( -+ struct kbase_kcpu_command_queue *kcpu_queue, -+ struct base_kcpu_command_fence_info *fence_info, -+ struct kbase_kcpu_command *current_command) -+{ -+ struct kbase_context *const kctx = kcpu_queue->kctx; -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+ struct fence *fence_out; -+#else -+ struct dma_fence *fence_out; -+#endif -+ struct base_fence fence; -+ struct sync_file *sync_file; -+ int ret = 0; -+ int fd; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ if (copy_from_user(&fence, u64_to_user_ptr(fence_info->fence), -+ sizeof(fence))) -+ return -EFAULT; -+ -+ fence_out = kzalloc(sizeof(*fence_out), GFP_KERNEL); -+ if (!fence_out) -+ return -ENOMEM; -+ -+ dma_fence_init(fence_out, -+ &kbase_fence_ops, -+ &kbase_csf_fence_lock, -+ kcpu_queue->fence_context, -+ ++kcpu_queue->fence_seqno); -+ -+#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) -+ /* Take an extra reference to the fence on behalf of the sync file. -+ * This is only needded on older kernels where sync_file_create() -+ * does not take its own reference. This was changed in v4.9.68 -+ * where sync_file_create() now takes its own reference. -+ */ -+ dma_fence_get(fence_out); -+#endif -+ -+ /* create a sync_file fd representing the fence */ -+ sync_file = sync_file_create(fence_out); -+ if (!sync_file) { -+#if (KERNEL_VERSION(4, 9, 67) >= LINUX_VERSION_CODE) -+ dma_fence_put(fence_out); -+#endif -+ ret = -ENOMEM; -+ goto file_create_fail; -+ } -+ -+ fd = get_unused_fd_flags(O_CLOEXEC); -+ if (fd < 0) { -+ ret = fd; -+ goto fd_flags_fail; -+ } -+ -+ fd_install(fd, sync_file->file); -+ -+ fence.basep.fd = fd; -+ -+ current_command->type = BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL; -+ current_command->info.fence.fence = fence_out; -+ -+ if (copy_to_user(u64_to_user_ptr(fence_info->fence), &fence, -+ sizeof(fence))) { -+ ret = -EFAULT; -+ goto fd_flags_fail; -+ } -+ -+ return 0; -+ -+fd_flags_fail: -+ fput(sync_file->file); -+file_create_fail: -+ dma_fence_put(fence_out); -+ -+ return ret; -+} -+#endif /* CONFIG_SYNC_FILE */ -+ -+static void kcpu_queue_process_worker(struct work_struct *data) -+{ -+ struct kbase_kcpu_command_queue *queue = container_of(data, -+ struct kbase_kcpu_command_queue, work); -+ -+ mutex_lock(&queue->kctx->csf.kcpu_queues.lock); -+ -+ kcpu_queue_process(queue, false); -+ -+ mutex_unlock(&queue->kctx->csf.kcpu_queues.lock); -+} -+ -+static int delete_queue(struct kbase_context *kctx, u32 id) -+{ -+ int err = 0; -+ -+ mutex_lock(&kctx->csf.kcpu_queues.lock); -+ -+ if ((id < KBASEP_MAX_KCPU_QUEUES) && kctx->csf.kcpu_queues.array[id]) { -+ struct kbase_kcpu_command_queue *queue = -+ kctx->csf.kcpu_queues.array[id]; -+ -+ KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_DESTROY, -+ queue, queue->num_pending_cmds, queue->cqs_wait_count); -+ -+ /* Drain the remaining work for this queue first and go past -+ * all the waits. -+ */ -+ kcpu_queue_process(queue, true); -+ -+ /* All commands should have been processed */ -+ WARN_ON(queue->num_pending_cmds); -+ -+ /* All CQS wait commands should have been cleaned up */ -+ WARN_ON(queue->cqs_wait_count); -+ -+ kctx->csf.kcpu_queues.array[id] = NULL; -+ bitmap_clear(kctx->csf.kcpu_queues.in_use, id, 1); -+ -+ /* Fire the tracepoint with the mutex held to enforce correct -+ * ordering with the summary stream. -+ */ -+ KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE(kctx->kbdev, queue); -+ -+ mutex_unlock(&kctx->csf.kcpu_queues.lock); -+ -+ cancel_work_sync(&queue->work); -+ -+ kfree(queue); -+ } else { -+ dev_warn(kctx->kbdev->dev, -+ "Attempt to delete a non-existent KCPU queue\n"); -+ mutex_unlock(&kctx->csf.kcpu_queues.lock); -+ err = -EINVAL; -+ } -+ return err; -+} -+ -+static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_INFO( -+ struct kbase_device *kbdev, -+ const struct kbase_kcpu_command_queue *queue, -+ const struct kbase_kcpu_command_jit_alloc_info *jit_alloc, -+ int alloc_status) -+{ -+ u8 i; -+ -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( -+ kbdev, queue); -+ for (i = 0; i < jit_alloc->count; i++) { -+ const u8 id = jit_alloc->info[i].id; -+ const struct kbase_va_region *reg = queue->kctx->jit_alloc[id]; -+ u64 gpu_alloc_addr = 0; -+ u64 mmu_flags = 0; -+ -+ if ((alloc_status == 0) && !WARN_ON(!reg) && -+ !WARN_ON(reg == KBASE_RESERVED_REG_JIT_ALLOC)) { -+#ifdef CONFIG_MALI_VECTOR_DUMP -+ struct tagged_addr phy = {0}; -+#endif /* CONFIG_MALI_VECTOR_DUMP */ -+ -+ gpu_alloc_addr = reg->start_pfn << PAGE_SHIFT; -+#ifdef CONFIG_MALI_VECTOR_DUMP -+ mmu_flags = kbase_mmu_create_ate(kbdev, -+ phy, reg->flags, -+ MIDGARD_MMU_BOTTOMLEVEL, -+ queue->kctx->jit_group_id); -+#endif /* CONFIG_MALI_VECTOR_DUMP */ -+ } -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( -+ kbdev, queue, alloc_status, gpu_alloc_addr, mmu_flags); -+ } -+} -+ -+static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( -+ struct kbase_device *kbdev, -+ const struct kbase_kcpu_command_queue *queue) -+{ -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( -+ kbdev, queue); -+} -+ -+static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END( -+ struct kbase_device *kbdev, -+ const struct kbase_kcpu_command_queue *queue) -+{ -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( -+ kbdev, queue); -+} -+ -+static void kcpu_queue_process(struct kbase_kcpu_command_queue *queue, -+ bool ignore_waits) -+{ -+ struct kbase_device *kbdev = queue->kctx->kbdev; -+ bool process_next = true; -+ size_t i; -+ -+ lockdep_assert_held(&queue->kctx->csf.kcpu_queues.lock); -+ -+ for (i = 0; i != queue->num_pending_cmds; ++i) { -+ struct kbase_kcpu_command *cmd = -+ &queue->commands[(u8)(queue->start_offset + i)]; -+ int status; -+ -+ switch (cmd->type) { -+ case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: -+ if (!queue->command_started) { -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( -+ kbdev, queue); -+ queue->command_started = true; -+ } -+ -+ status = 0; -+#if IS_ENABLED(CONFIG_SYNC_FILE) -+ if (ignore_waits) { -+ kbase_kcpu_fence_wait_cancel(queue, -+ &cmd->info.fence); -+ } else { -+ status = kbase_kcpu_fence_wait_process(queue, -+ &cmd->info.fence); -+ -+ if (status == 0) -+ process_next = false; -+ else if (status < 0) -+ queue->has_error = true; -+ } -+#else -+ dev_warn(kbdev->dev, -+ "unexpected fence wait command found\n"); -+ -+ status = -EINVAL; -+ queue->has_error = true; -+#endif -+ -+ if (process_next) { -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( -+ kbdev, queue, status < 0 ? status : 0); -+ queue->command_started = false; -+ } -+ break; -+ case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( -+ kbdev, queue); -+ -+ status = 0; -+ -+#if IS_ENABLED(CONFIG_SYNC_FILE) -+ status = kbase_kcpu_fence_signal_process( -+ queue, &cmd->info.fence); -+ -+ if (status < 0) -+ queue->has_error = true; -+#else -+ dev_warn(kbdev->dev, -+ "unexpected fence signal command found\n"); -+ -+ status = -EINVAL; -+ queue->has_error = true; -+#endif -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( -+ kbdev, queue, status); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: -+ status = kbase_kcpu_cqs_wait_process(kbdev, queue, -+ &cmd->info.cqs_wait); -+ -+ if (!status && !ignore_waits) { -+ process_next = false; -+ } else { -+ /* Either all CQS objects were signaled or -+ * there was an error or the queue itself is -+ * being deleted. -+ * In all cases can move to the next command. -+ * TBD: handle the error -+ */ -+ cleanup_cqs_wait(queue, &cmd->info.cqs_wait); -+ } -+ -+ break; -+ case BASE_KCPU_COMMAND_TYPE_CQS_SET: -+ kbase_kcpu_cqs_set_process(kbdev, queue, -+ &cmd->info.cqs_set); -+ -+ break; -+ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION: -+ status = kbase_kcpu_cqs_wait_operation_process(kbdev, queue, -+ &cmd->info.cqs_wait_operation); -+ -+ if (!status && !ignore_waits) { -+ process_next = false; -+ } else { -+ /* Either all CQS objects were signaled or -+ * there was an error or the queue itself is -+ * being deleted. -+ * In all cases can move to the next command. -+ * TBD: handle the error -+ */ -+ cleanup_cqs_wait_operation(queue, &cmd->info.cqs_wait_operation); -+ } -+ -+ break; -+ case BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION: -+ kbase_kcpu_cqs_set_operation_process(kbdev, queue, -+ &cmd->info.cqs_set_operation); -+ -+ break; -+ case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: -+ /* Clear the queue's error state */ -+ queue->has_error = false; -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER( -+ kbdev, queue); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: { -+ struct kbase_ctx_ext_res_meta *meta = NULL; -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( -+ kbdev, queue); -+ -+ kbase_gpu_vm_lock(queue->kctx); -+ meta = kbase_sticky_resource_acquire( -+ queue->kctx, cmd->info.import.gpu_va); -+ kbase_gpu_vm_unlock(queue->kctx); -+ -+ if (meta == NULL) { -+ queue->has_error = true; -+ dev_warn(kbdev->dev, -+ "failed to map an external resource\n"); -+ } -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( -+ kbdev, queue, meta ? 0 : 1); -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: { -+ bool ret; -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( -+ kbdev, queue); -+ -+ kbase_gpu_vm_lock(queue->kctx); -+ ret = kbase_sticky_resource_release( -+ queue->kctx, NULL, cmd->info.import.gpu_va); -+ kbase_gpu_vm_unlock(queue->kctx); -+ -+ if (!ret) { -+ queue->has_error = true; -+ dev_warn(kbdev->dev, -+ "failed to release the reference. resource not found\n"); -+ } -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( -+ kbdev, queue, ret ? 0 : 1); -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: { -+ bool ret; -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( -+ kbdev, queue); -+ -+ kbase_gpu_vm_lock(queue->kctx); -+ ret = kbase_sticky_resource_release_force( -+ queue->kctx, NULL, cmd->info.import.gpu_va); -+ kbase_gpu_vm_unlock(queue->kctx); -+ -+ if (!ret) { -+ queue->has_error = true; -+ dev_warn(kbdev->dev, -+ "failed to release the reference. resource not found\n"); -+ } -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( -+ kbdev, queue, ret ? 0 : 1); -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: -+ { -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( -+ kbdev, queue); -+ -+ status = kbase_kcpu_jit_allocate_process(queue, cmd); -+ if (status == -EAGAIN) { -+ process_next = false; -+ } else { -+ if (status != 0) -+ queue->has_error = true; -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_INFO( -+ kbdev, queue, &cmd->info.jit_alloc, -+ status); -+ -+ kbase_kcpu_jit_allocate_finish(queue, cmd); -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( -+ kbdev, queue); -+ } -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_JIT_FREE: -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( -+ kbdev, queue); -+ -+ status = kbase_kcpu_jit_free_process(queue, cmd); -+ if (status) -+ queue->has_error = true; -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_END( -+ kbdev, queue); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: { -+ struct kbase_suspend_copy_buffer *sus_buf = -+ cmd->info.suspend_buf_copy.sus_buf; -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START( -+ kbdev, queue); -+ -+ status = kbase_csf_queue_group_suspend_process( -+ queue->kctx, sus_buf, -+ cmd->info.suspend_buf_copy.group_handle); -+ if (status) -+ queue->has_error = true; -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END( -+ kbdev, queue, status); -+ -+ if (!sus_buf->cpu_alloc) { -+ int i; -+ -+ for (i = 0; i < sus_buf->nr_pages; i++) -+ put_page(sus_buf->pages[i]); -+ } else { -+ kbase_mem_phy_alloc_kernel_unmapped( -+ sus_buf->cpu_alloc); -+ kbase_mem_phy_alloc_put(sus_buf->cpu_alloc); -+ } -+ -+ kfree(sus_buf->pages); -+ kfree(sus_buf); -+ break; -+ } -+#if MALI_UNIT_TEST -+ case BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME: { -+ u64 time = ktime_get_raw_ns(); -+ void *target_page = kmap(*cmd->info.sample_time.page); -+ -+ if (target_page) { -+ memcpy(target_page + -+ cmd->info.sample_time.page_offset, -+ &time, sizeof(time)); -+ kunmap(*cmd->info.sample_time.page); -+ } else { -+ dev_warn(kbdev->dev, -+ "Could not kmap target page\n"); -+ queue->has_error = true; -+ } -+ put_page(*cmd->info.sample_time.page); -+ kfree(cmd->info.sample_time.page); -+ break; -+ } -+#endif /* MALI_UNIT_TEST */ -+ default: -+ dev_warn(kbdev->dev, -+ "Unrecognized command type\n"); -+ break; -+ } /* switch */ -+ -+ /*TBD: error handling */ -+ -+ if (!process_next) -+ break; -+ } -+ -+ if (i > 0) { -+ queue->start_offset += i; -+ queue->num_pending_cmds -= i; -+ -+ /* If an attempt to enqueue commands failed then we must raise -+ * an event in case the client wants to retry now that there is -+ * free space in the buffer. -+ */ -+ if (queue->enqueue_failed) { -+ queue->enqueue_failed = false; -+ kbase_csf_event_signal_cpu_only(queue->kctx); -+ } -+ } -+} -+ -+static size_t kcpu_queue_get_space(struct kbase_kcpu_command_queue *queue) -+{ -+ return KBASEP_KCPU_QUEUE_SIZE - queue->num_pending_cmds; -+} -+ -+static void KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_COMMAND( -+ const struct kbase_kcpu_command_queue *queue, -+ const struct kbase_kcpu_command *cmd) -+{ -+ struct kbase_device *kbdev = queue->kctx->kbdev; -+ -+ switch (cmd->type) { -+ case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( -+ kbdev, queue, cmd->info.fence.fence); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( -+ kbdev, queue, cmd->info.fence.fence); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: -+ { -+ const struct base_cqs_wait_info *waits = -+ cmd->info.cqs_wait.objs; -+ u32 inherit_err_flags = cmd->info.cqs_wait.inherit_err_flags; -+ unsigned int i; -+ -+ for (i = 0; i < cmd->info.cqs_wait.nr_objs; i++) { -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( -+ kbdev, queue, waits[i].addr, waits[i].val, -+ (inherit_err_flags & ((u32)1 << i)) ? 1 : 0); -+ } -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_CQS_SET: -+ { -+ const struct base_cqs_set *sets = cmd->info.cqs_set.objs; -+ unsigned int i; -+ -+ for (i = 0; i < cmd->info.cqs_set.nr_objs; i++) { -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( -+ kbdev, queue, sets[i].addr); -+ } -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION: -+ { -+ /* GPUCORE-28172 RDT to review */ -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION: -+ { -+ /* GPUCORE-28172 RDT to review */ -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER(kbdev, -+ queue); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( -+ kbdev, queue, cmd->info.import.gpu_va); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( -+ kbdev, queue, cmd->info.import.gpu_va); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( -+ kbdev, queue, cmd->info.import.gpu_va); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: -+ { -+ u8 i; -+ -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( -+ kbdev, queue); -+ for (i = 0; i < cmd->info.jit_alloc.count; i++) { -+ const struct base_jit_alloc_info *info = -+ &cmd->info.jit_alloc.info[i]; -+ -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( -+ kbdev, queue, info->gpu_alloc_addr, -+ info->va_pages, info->commit_pages, -+ info->extension, info->id, info->bin_id, -+ info->max_allocations, info->flags, -+ info->usage_id); -+ } -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( -+ kbdev, queue); -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_JIT_FREE: -+ { -+ u8 i; -+ -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( -+ kbdev, queue); -+ for (i = 0; i < cmd->info.jit_free.count; i++) { -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( -+ kbdev, queue, cmd->info.jit_free.ids[i]); -+ } -+ KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( -+ kbdev, queue); -+ break; -+ } -+ case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND( -+ kbdev, queue, cmd->info.suspend_buf_copy.sus_buf, -+ cmd->info.suspend_buf_copy.group_handle); -+ break; -+#if MALI_UNIT_TEST -+ case BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME: -+ /* -+ * This is test-only KCPU command, no need to have a timeline -+ * entry -+ */ -+ break; -+#endif /* MALI_UNIT_TEST */ -+ } -+} -+ -+int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx, -+ struct kbase_ioctl_kcpu_queue_enqueue *enq) -+{ -+ struct kbase_kcpu_command_queue *queue = NULL; -+ void __user *user_cmds = u64_to_user_ptr(enq->addr); -+ int ret = 0; -+ u32 i; -+ -+ /* The offset to the first command that is being processed or yet to -+ * be processed is of u8 type, so the number of commands inside the -+ * queue cannot be more than 256. -+ */ -+ BUILD_BUG_ON(KBASEP_KCPU_QUEUE_SIZE > 256); -+ -+ /* Whilst the backend interface allows enqueueing multiple commands in -+ * a single operation, the Base interface does not expose any mechanism -+ * to do so. And also right now the handling is missing for the case -+ * where multiple commands are submitted and the enqueue of one of the -+ * command in the set fails after successfully enqueuing other commands -+ * in the set. -+ */ -+ if (enq->nr_commands != 1) { -+ dev_err(kctx->kbdev->dev, -+ "More than one commands enqueued\n"); -+ return -EINVAL; -+ } -+ -+ mutex_lock(&kctx->csf.kcpu_queues.lock); -+ -+ if (!kctx->csf.kcpu_queues.array[enq->id]) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ queue = kctx->csf.kcpu_queues.array[enq->id]; -+ -+ if (kcpu_queue_get_space(queue) < enq->nr_commands) { -+ ret = -EBUSY; -+ queue->enqueue_failed = true; -+ goto out; -+ } -+ -+ /* Copy all command's info to the command buffer. -+ * Note: it would be more efficient to process all commands in-line -+ * until we encounter an unresolved CQS_ / FENCE_WAIT, however, the -+ * interface allows multiple commands to be enqueued so we must account -+ * for the possibility to roll back. -+ */ -+ -+ for (i = 0; (i != enq->nr_commands) && !ret; ++i, ++kctx->csf.kcpu_queues.num_cmds) { -+ struct kbase_kcpu_command *kcpu_cmd = -+ &queue->commands[(u8)(queue->start_offset + queue->num_pending_cmds + i)]; -+ struct base_kcpu_command command; -+ unsigned int j; -+ -+ if (copy_from_user(&command, user_cmds, sizeof(command))) { -+ ret = -EFAULT; -+ goto out; -+ } -+ -+ user_cmds = (void __user *)((uintptr_t)user_cmds + -+ sizeof(struct base_kcpu_command)); -+ -+ for (j = 0; j < sizeof(command.padding); j++) { -+ if (command.padding[j] != 0) { -+ dev_dbg(kctx->kbdev->dev, -+ "base_kcpu_command padding not 0\n"); -+ ret = -EINVAL; -+ goto out; -+ } -+ } -+ -+ kcpu_cmd->enqueue_ts = kctx->csf.kcpu_queues.num_cmds; -+ switch (command.type) { -+ case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: -+#if IS_ENABLED(CONFIG_SYNC_FILE) -+ ret = kbase_kcpu_fence_wait_prepare(queue, -+ &command.info.fence, kcpu_cmd); -+#else -+ ret = -EINVAL; -+ dev_warn(kctx->kbdev->dev, "fence wait command unsupported\n"); -+#endif -+ break; -+ case BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: -+#if IS_ENABLED(CONFIG_SYNC_FILE) -+ ret = kbase_kcpu_fence_signal_prepare(queue, -+ &command.info.fence, kcpu_cmd); -+#else -+ ret = -EINVAL; -+ dev_warn(kctx->kbdev->dev, "fence signal command unsupported\n"); -+#endif -+ break; -+ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: -+ ret = kbase_kcpu_cqs_wait_prepare(queue, -+ &command.info.cqs_wait, kcpu_cmd); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_CQS_SET: -+ ret = kbase_kcpu_cqs_set_prepare(queue, -+ &command.info.cqs_set, kcpu_cmd); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION: -+ ret = kbase_kcpu_cqs_wait_operation_prepare(queue, -+ &command.info.cqs_wait_operation, kcpu_cmd); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION: -+ ret = kbase_kcpu_cqs_set_operation_prepare(queue, -+ &command.info.cqs_set_operation, kcpu_cmd); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: -+ kcpu_cmd->type = BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER; -+ ret = 0; -+ break; -+ case BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: -+ ret = kbase_kcpu_map_import_prepare(queue, -+ &command.info.import, kcpu_cmd); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: -+ ret = kbase_kcpu_unmap_import_prepare(queue, -+ &command.info.import, kcpu_cmd); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: -+ ret = kbase_kcpu_unmap_import_force_prepare(queue, -+ &command.info.import, kcpu_cmd); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: -+ ret = kbase_kcpu_jit_allocate_prepare(queue, -+ &command.info.jit_alloc, kcpu_cmd); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_JIT_FREE: -+ ret = kbase_kcpu_jit_free_prepare(queue, -+ &command.info.jit_free, kcpu_cmd); -+ break; -+ case BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: -+ ret = kbase_csf_queue_group_suspend_prepare(queue, -+ &command.info.suspend_buf_copy, -+ kcpu_cmd); -+ break; -+#if MALI_UNIT_TEST -+ case BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME: { -+ int const page_cnt = 1; -+ -+ kcpu_cmd->type = BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME; -+ kcpu_cmd->info.sample_time.page_addr = -+ command.info.sample_time.time & PAGE_MASK; -+ kcpu_cmd->info.sample_time.page_offset = -+ command.info.sample_time.time & ~PAGE_MASK; -+ kcpu_cmd->info.sample_time.page = kcalloc( -+ page_cnt, sizeof(struct page *), GFP_KERNEL); -+ if (!kcpu_cmd->info.sample_time.page) { -+ ret = -ENOMEM; -+ } else { -+ int pinned_pages = get_user_pages_fast( -+ kcpu_cmd->info.sample_time.page_addr, -+ page_cnt, 1, -+ kcpu_cmd->info.sample_time.page); -+ -+ if (pinned_pages < 0) { -+ ret = pinned_pages; -+ kfree(kcpu_cmd->info.sample_time.page); -+ } else if (pinned_pages != page_cnt) { -+ ret = -EINVAL; -+ kfree(kcpu_cmd->info.sample_time.page); -+ } -+ } -+ -+ break; -+ } -+#endif /* MALI_UNIT_TEST */ -+ default: -+ dev_warn(queue->kctx->kbdev->dev, -+ "Unknown command type %u\n", command.type); -+ ret = -EINVAL; -+ break; -+ } -+ } -+ -+ if (!ret) { -+ /* We only instrument the enqueues after all commands have been -+ * successfully enqueued, as if we do them during the enqueue -+ * and there is an error, we won't be able to roll them back -+ * like is done for the command enqueues themselves. -+ */ -+ for (i = 0; i != enq->nr_commands; ++i) { -+ u8 cmd_idx = (u8)(queue->start_offset + queue->num_pending_cmds + i); -+ -+ KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_COMMAND( -+ queue, &queue->commands[cmd_idx]); -+ } -+ -+ queue->num_pending_cmds += enq->nr_commands; -+ kcpu_queue_process(queue, false); -+ } else { -+ /* Roll back the number of enqueued commands */ -+ kctx->csf.kcpu_queues.num_cmds -= i; -+ } -+ -+out: -+ mutex_unlock(&kctx->csf.kcpu_queues.lock); -+ -+ return ret; -+} -+ -+int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx) -+{ -+ int idx; -+ -+ bitmap_zero(kctx->csf.kcpu_queues.in_use, KBASEP_MAX_KCPU_QUEUES); -+ -+ for (idx = 0; idx < KBASEP_MAX_KCPU_QUEUES; ++idx) -+ kctx->csf.kcpu_queues.array[idx] = NULL; -+ -+ kctx->csf.kcpu_queues.wq = alloc_workqueue("mali_kbase_csf_kcpu", -+ WQ_UNBOUND | WQ_HIGHPRI, 0); -+ if (!kctx->csf.kcpu_queues.wq) -+ return -ENOMEM; -+ -+ mutex_init(&kctx->csf.kcpu_queues.lock); -+ -+ kctx->csf.kcpu_queues.num_cmds = 0; -+ -+ return 0; -+} -+ -+void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx) -+{ -+ while (!bitmap_empty(kctx->csf.kcpu_queues.in_use, -+ KBASEP_MAX_KCPU_QUEUES)) { -+ int id = find_first_bit(kctx->csf.kcpu_queues.in_use, -+ KBASEP_MAX_KCPU_QUEUES); -+ -+ if (WARN_ON(!kctx->csf.kcpu_queues.array[id])) -+ clear_bit(id, kctx->csf.kcpu_queues.in_use); -+ else -+ (void)delete_queue(kctx, id); -+ } -+ -+ destroy_workqueue(kctx->csf.kcpu_queues.wq); -+ mutex_destroy(&kctx->csf.kcpu_queues.lock); -+} -+ -+int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx, -+ struct kbase_ioctl_kcpu_queue_delete *del) -+{ -+ return delete_queue(kctx, (u32)del->id); -+} -+ -+int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, -+ struct kbase_ioctl_kcpu_queue_new *newq) -+{ -+ struct kbase_kcpu_command_queue *queue; -+ int idx; -+ int ret = 0; -+ -+ /* The queue id is of u8 type and we use the index of the kcpu_queues -+ * array as an id, so the number of elements in the array can't be -+ * more than 256. -+ */ -+ BUILD_BUG_ON(KBASEP_MAX_KCPU_QUEUES > 256); -+ -+ mutex_lock(&kctx->csf.kcpu_queues.lock); -+ -+ idx = find_first_zero_bit(kctx->csf.kcpu_queues.in_use, -+ KBASEP_MAX_KCPU_QUEUES); -+ if (idx >= (int)KBASEP_MAX_KCPU_QUEUES) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ if (WARN_ON(kctx->csf.kcpu_queues.array[idx])) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ queue = kzalloc(sizeof(*queue), GFP_KERNEL); -+ -+ if (!queue) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ bitmap_set(kctx->csf.kcpu_queues.in_use, idx, 1); -+ kctx->csf.kcpu_queues.array[idx] = queue; -+ queue->kctx = kctx; -+ queue->start_offset = 0; -+ queue->num_pending_cmds = 0; -+#if IS_ENABLED(CONFIG_SYNC_FILE) -+ queue->fence_context = dma_fence_context_alloc(1); -+ queue->fence_seqno = 0; -+ queue->fence_wait_processed = false; -+#endif -+ queue->enqueue_failed = false; -+ queue->command_started = false; -+ INIT_LIST_HEAD(&queue->jit_blocked); -+ queue->has_error = false; -+ INIT_WORK(&queue->work, kcpu_queue_process_worker); -+ queue->id = idx; -+ -+ newq->id = idx; -+ -+ /* Fire the tracepoint with the mutex held to enforce correct ordering -+ * with the summary stream. -+ */ -+ KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( -+ kctx->kbdev, queue, kctx->id, queue->num_pending_cmds); -+ -+ KBASE_KTRACE_ADD_CSF_KCPU(kctx->kbdev, KCPU_QUEUE_NEW, queue, -+ queue->fence_context, 0); -+out: -+ mutex_unlock(&kctx->csf.kcpu_queues.lock); -+ -+ return ret; -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu.h -new file mode 100644 -index 0000000..2f6da55 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu.h -@@ -0,0 +1,356 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_KCPU_H_ -+#define _KBASE_CSF_KCPU_H_ -+ -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+#include <linux/fence.h> -+#else -+#include <linux/dma-fence.h> -+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ -+ -+/* The maximum number of KCPU commands in flight, enqueueing more commands -+ * than this value shall block. -+ */ -+#define KBASEP_KCPU_QUEUE_SIZE ((size_t)256) -+ -+/** -+ * struct kbase_kcpu_command_import_info - Structure which holds information -+ * about the buffer to be imported -+ * -+ * @gpu_va: Address of the buffer to be imported. -+ */ -+struct kbase_kcpu_command_import_info { -+ u64 gpu_va; -+}; -+ -+/** -+ * struct kbase_kcpu_command_fence_info - Structure which holds information -+ * about the fence object enqueued in the kcpu command queue -+ * -+ * @fence_cb: Fence callback -+ * @fence: Fence -+ * @kcpu_queue: kcpu command queue -+ */ -+struct kbase_kcpu_command_fence_info { -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+ struct fence_cb fence_cb; -+ struct fence *fence; -+#else -+ struct dma_fence_cb fence_cb; -+ struct dma_fence *fence; -+#endif /* LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) */ -+ struct kbase_kcpu_command_queue *kcpu_queue; -+}; -+ -+/** -+ * struct kbase_kcpu_command_cqs_set_info - Structure which holds information -+ * about CQS objects for the kcpu CQS set command -+ * -+ * @objs: Array of structures which define CQS objects to be used by -+ * the kcpu command. -+ * @nr_objs: Number of CQS objects in the array. -+ */ -+struct kbase_kcpu_command_cqs_set_info { -+ struct base_cqs_set *objs; -+ unsigned int nr_objs; -+}; -+ -+/** -+ * struct kbase_kcpu_command_cqs_wait_info - Structure which holds information -+ * about CQS objects for the kcpu CQS wait command -+ * -+ * @objs: Array of structures which define CQS objects to be used by -+ * the kcpu command. -+ * @signaled: Bit array used to report the status of the CQS wait objects. -+ * 1 is signaled, 0 otherwise. -+ * @nr_objs: Number of CQS objects in the array. -+ * @inherit_err_flags: Bit-pattern for the CQSs in the array who's error field -+ * to be served as the source for importing into the -+ * queue's error-state. -+ */ -+struct kbase_kcpu_command_cqs_wait_info { -+ struct base_cqs_wait_info *objs; -+ unsigned long *signaled; -+ unsigned int nr_objs; -+ u32 inherit_err_flags; -+}; -+ -+/** -+ * struct kbase_kcpu_command_cqs_set_operation_info - Structure which holds information -+ * about CQS objects for the kcpu CQS timeline set command -+ * -+ * @objs: Array of structures which define CQS timeline objects to be used by -+ * the kcpu command. -+ * @nr_objs: Number of CQS objects in the array. -+ */ -+struct kbase_kcpu_command_cqs_set_operation_info { -+ struct base_cqs_set_operation_info *objs; -+ unsigned int nr_objs; -+}; -+ -+/** -+ * struct kbase_kcpu_command_cqs_wait_operation_info - Structure which holds information -+ * about CQS objects for the kcpu CQS timeline wait command -+ * -+ * @objs: Array of structures which define CQS timeline objects to be used by -+ * the kcpu command. -+ * @signaled: Bit array used to report the status of the CQS wait objects. -+ * 1 is signaled, 0 otherwise. -+ * @nr_objs: Number of CQS objects in the array. -+ */ -+struct kbase_kcpu_command_cqs_wait_operation_info { -+ struct base_cqs_wait_operation_info *objs; -+ unsigned long *signaled; -+ unsigned int nr_objs; -+ u32 inherit_err_flags; -+}; -+ -+/** -+ * struct kbase_kcpu_command_jit_alloc_info - Structure which holds information -+ * needed for the kcpu command for jit allocations -+ * -+ * @node: Used to keep track of all JIT free/alloc commands in submission -+ * order. This must be located in the front of this struct to -+ * match that of kbase_kcpu_command_jit_free_info. -+ * @info: Array of objects of the struct base_jit_alloc_info type which -+ * specify jit allocations to be made by the kcpu command. -+ * @count: Number of jit alloc objects in the array. -+ * @blocked: Whether this allocation has been put into the pending list to -+ * be retried later. -+ */ -+struct kbase_kcpu_command_jit_alloc_info { -+ struct list_head node; -+ struct base_jit_alloc_info *info; -+ u8 count; -+ bool blocked; -+}; -+ -+/** -+ * struct kbase_kcpu_command_jit_free_info - Structure which holds information -+ * needed for the kcpu jit free command -+ * -+ * @node: Used to keep track of all JIT free/alloc commands in submission -+ * order. This must be located in the front of this struct to -+ * match that of kbase_kcpu_command_jit_alloc_info. -+ * @ids: Array of identifiers of jit allocations which are to be freed -+ * by the kcpu command. -+ * @count: Number of elements in the array. -+ */ -+struct kbase_kcpu_command_jit_free_info { -+ struct list_head node; -+ u8 *ids; -+ u8 count; -+}; -+ -+/** -+ * struct kbase_suspend_copy_buffer - information about the suspend buffer -+ * to be copied. -+ * -+ * @size: size of the suspend buffer in bytes. -+ * @pages: pointer to an array of pointers to the pages which contain -+ * the user buffer. -+ * @nr_pages: number of pages. -+ * @offset: offset into the pages -+ * @cpu_alloc: Reference to physical pages of suspend buffer allocation. -+ */ -+struct kbase_suspend_copy_buffer { -+ size_t size; -+ struct page **pages; -+ int nr_pages; -+ size_t offset; -+ struct kbase_mem_phy_alloc *cpu_alloc; -+}; -+ -+/** -+ * struct base_kcpu_command_group_suspend - structure which contains -+ * suspend buffer data captured for a suspended queue group. -+ * -+ * @sus_buf: Pointer to the structure which contains details of the -+ * user buffer and its kernel pinned pages. -+ * @group_handle: Handle to the mapping of CSG. -+ */ -+struct kbase_kcpu_command_group_suspend_info { -+ struct kbase_suspend_copy_buffer *sus_buf; -+ u8 group_handle; -+}; -+ -+#if MALI_UNIT_TEST -+struct kbase_kcpu_command_sample_time_info { -+ u64 page_addr; -+ u64 page_offset; -+ struct page **page; -+}; -+#endif /* MALI_UNIT_TEST */ -+ -+/** -+ * struct kbase_cpu_command - Command which is to be part of the kernel -+ * command queue -+ * -+ * @type: Type of the command. -+ * @enqueue_ts: Denotes the relative time of enqueueing, a smaller value -+ * indicates that it has been enqueued earlier. -+ * @info: Structure which holds information about the command -+ * dependent on the command type. -+ * @info.fence: Fence -+ * @info.cqs_wait: CQS wait -+ * @info.cqs_set: CQS set -+ * @info.import: import -+ * @info.jit_alloc: jit allocation -+ * @info.jit_free: jit deallocation -+ * @info.suspend_buf_copy: suspend buffer copy -+ * @info.sample_time: sample time -+ */ -+struct kbase_kcpu_command { -+ enum base_kcpu_command_type type; -+ u64 enqueue_ts; -+ union { -+ struct kbase_kcpu_command_fence_info fence; -+ struct kbase_kcpu_command_cqs_wait_info cqs_wait; -+ struct kbase_kcpu_command_cqs_set_info cqs_set; -+ struct kbase_kcpu_command_cqs_wait_operation_info cqs_wait_operation; -+ struct kbase_kcpu_command_cqs_set_operation_info cqs_set_operation; -+ struct kbase_kcpu_command_import_info import; -+ struct kbase_kcpu_command_jit_alloc_info jit_alloc; -+ struct kbase_kcpu_command_jit_free_info jit_free; -+ struct kbase_kcpu_command_group_suspend_info suspend_buf_copy; -+#if MALI_UNIT_TEST -+ struct kbase_kcpu_command_sample_time_info sample_time; -+#endif /* MALI_UNIT_TEST */ -+ } info; -+}; -+ -+/** -+ * struct kbase_kcpu_command_queue - a command queue executed by the kernel -+ * -+ * @kctx: The context to which this command queue belongs. -+ * @commands: Array of commands which have been successfully -+ * enqueued to this command queue. -+ * @work: struct work_struct which contains a pointer to -+ * the function which handles processing of kcpu -+ * commands enqueued into a kcpu command queue; -+ * part of kernel API for processing workqueues -+ * @start_offset: Index of the command to be executed next -+ * @id: KCPU command queue ID. -+ * @num_pending_cmds: The number of commands enqueued but not yet -+ * executed or pending -+ * @cqs_wait_count: Tracks the number of CQS wait commands enqueued -+ * @fence_context: The dma-buf fence context number for this kcpu -+ * queue. A unique context number is allocated for -+ * each kcpu queue. -+ * @fence_seqno: The dma-buf fence sequence number for the fence -+ * that is returned on the enqueue of fence signal -+ * command. This is increased every time the -+ * fence signal command is queued. -+ * @fence_wait_processed: Used to avoid reprocessing of the fence wait -+ * command which has blocked the processing of -+ * commands that follow it. -+ * @enqueue_failed: Indicates that no space has become available in -+ * the buffer since an enqueue operation failed -+ * because of insufficient free space. -+ * @command_started: Indicates that the command at the front of the -+ * queue has been started in a previous queue -+ * process, but was not completed due to some -+ * unmet dependencies. Ensures that instrumentation -+ * of the execution start of these commands is only -+ * fired exactly once. -+ * @has_error: Indicates that the kcpu queue is in error mode -+ * or without errors since last cleaned. -+ * @jit_blocked: Used to keep track of command queues blocked -+ * by a pending JIT allocation command. -+ */ -+struct kbase_kcpu_command_queue { -+ struct kbase_context *kctx; -+ struct kbase_kcpu_command commands[KBASEP_KCPU_QUEUE_SIZE]; -+ struct work_struct work; -+ u8 start_offset; -+ u8 id; -+ u16 num_pending_cmds; -+ u32 cqs_wait_count; -+ u64 fence_context; -+ unsigned int fence_seqno; -+ bool fence_wait_processed; -+ bool enqueue_failed; -+ bool command_started; -+ struct list_head jit_blocked; -+ bool has_error; -+}; -+ -+/** -+ * kbase_csf_kcpu_queue_new - Create new KCPU command queue. -+ * -+ * @kctx: Pointer to the kbase context within which the KCPU command -+ * queue will be created. -+ * @newq: Pointer to the structure which contains information about -+ * the new KCPU command queue to be created. -+ */ -+int kbase_csf_kcpu_queue_new(struct kbase_context *kctx, -+ struct kbase_ioctl_kcpu_queue_new *newq); -+ -+/** -+ * kbase_csf_kcpu_queue_delete - Delete KCPU command queue. -+ * -+ * Return: 0 if successful, -EINVAL if the queue ID is invalid. -+ * -+ * @kctx: Pointer to the kbase context from which the KCPU command -+ * queue is to be deleted. -+ * @del: Pointer to the structure which specifies the KCPU command -+ * queue to be deleted. -+ */ -+int kbase_csf_kcpu_queue_delete(struct kbase_context *kctx, -+ struct kbase_ioctl_kcpu_queue_delete *del); -+ -+/** -+ * kbase_csf_kcpu_queue_enqueue - Enqueue a KCPU command into a KCPU command -+ * queue. -+ * -+ * @kctx: Pointer to the kbase context within which the KCPU command -+ * is to be enqueued into the KCPU command queue. -+ * @enq: Pointer to the structure which specifies the KCPU command -+ * as well as the KCPU command queue into which the command -+ * is to be enqueued. -+ */ -+int kbase_csf_kcpu_queue_enqueue(struct kbase_context *kctx, -+ struct kbase_ioctl_kcpu_queue_enqueue *enq); -+ -+/** -+ * kbase_csf_kcpu_queue_context_init - Initialize the kernel CPU queues context -+ * for a GPU address space -+ * -+ * @kctx: Pointer to the kbase context being initialized. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+int kbase_csf_kcpu_queue_context_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_kcpu_queue_context_term - Terminate the kernel CPU queues context -+ * for a GPU address space -+ * -+ * This function deletes any kernel CPU queues that weren't deleted before -+ * context termination. -+ * -+ * @kctx: Pointer to the kbase context being terminated. -+ */ -+void kbase_csf_kcpu_queue_context_term(struct kbase_context *kctx); -+ -+#endif /* _KBASE_CSF_KCPU_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu_debugfs.c -new file mode 100644 -index 0000000..0a2cde0 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu_debugfs.c -@@ -0,0 +1,197 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_csf_kcpu_debugfs.h" -+#include <mali_kbase.h> -+#include <linux/seq_file.h> -+ -+#if IS_ENABLED(CONFIG_SYNC_FILE) -+#include "mali_kbase_sync.h" -+#endif -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ -+/** -+ * kbasep_csf_kcpu_debugfs_print_queue() - Print additional info for KCPU -+ * queues blocked on CQS wait commands. -+ * -+ * @file: The seq_file to print to -+ * @kctx: The context of the KCPU queue -+ * @waits: Pointer to the KCPU CQS wait command info -+ */ -+static void kbasep_csf_kcpu_debugfs_print_cqs_waits(struct seq_file *file, -+ struct kbase_context *kctx, -+ struct kbase_kcpu_command_cqs_wait_info *waits) -+{ -+ unsigned int i; -+ -+ for (i = 0; i < waits->nr_objs; i++) { -+ struct kbase_vmap_struct *mapping; -+ u32 val; -+ char const *msg; -+ u32 *const cpu_ptr = (u32 *)kbase_phy_alloc_mapping_get(kctx, -+ waits->objs[i].addr, &mapping); -+ -+ if (!cpu_ptr) -+ return; -+ -+ val = *cpu_ptr; -+ -+ kbase_phy_alloc_mapping_put(kctx, mapping); -+ -+ msg = (waits->inherit_err_flags && (1U << i)) ? "true" : -+ "false"; -+ seq_printf(file, " %llx(%u > %u, inherit_err: %s), ", -+ waits->objs[i].addr, val, waits->objs[i].val, msg); -+ } -+} -+ -+/** -+ * kbasep_csf_kcpu_debugfs_print_queue() - Print debug data for a KCPU queue -+ * -+ * @file: The seq_file to print to -+ * @kctx: The context of the KCPU queue -+ * @queue: Pointer to the KCPU queue -+ */ -+static void kbasep_csf_kcpu_debugfs_print_queue(struct seq_file *file, -+ struct kbase_context *kctx, -+ struct kbase_kcpu_command_queue *queue) -+{ -+ if (WARN_ON(!queue)) -+ return; -+ -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+ -+ seq_printf(file, "%16u, %11u, %7u, %13llu %8u", -+ queue->num_pending_cmds, queue->enqueue_failed, -+ queue->command_started ? 1 : 0, -+ queue->fence_context, queue->fence_seqno); -+ -+ if (queue->command_started) { -+ struct kbase_kcpu_command *cmd = -+ &queue->commands[queue->start_offset]; -+ switch (cmd->type) { -+#if IS_ENABLED(CONFIG_SYNC_FILE) -+ case BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: -+ { -+ struct kbase_sync_fence_info info; -+ -+ kbase_sync_fence_info_get(cmd->info.fence.fence, &info); -+ seq_printf(file, ", Fence %pK %s %s", -+ info.fence, info.name, -+ kbase_sync_status_string(info.status)); -+ break; -+ } -+#endif -+ case BASE_KCPU_COMMAND_TYPE_CQS_WAIT: -+ seq_puts(file, ", CQS "); -+ kbasep_csf_kcpu_debugfs_print_cqs_waits(file, kctx, -+ &cmd->info.cqs_wait); -+ break; -+ default: -+ seq_puts(file, ", U, Unknown blocking command"); -+ break; -+ } -+ } -+ -+ seq_puts(file, "\n"); -+} -+ -+/** -+ * kbasep_csf_kcpu_debugfs_show() - Print the KCPU queues debug information -+ * -+ * @file: The seq_file for printing to -+ * @data: The debugfs dentry private data, a pointer to kbase_context -+ * -+ * Return: Negative error code or 0 on success. -+ */ -+static int kbasep_csf_kcpu_debugfs_show(struct seq_file *file, void *data) -+{ -+ struct kbase_context *kctx = file->private; -+ unsigned long idx; -+ -+ seq_printf(file, "MALI_CSF_KCPU_DEBUGFS_VERSION: v%u\n", MALI_CSF_KCPU_DEBUGFS_VERSION); -+ seq_puts(file, "Queue Idx(err-mode), Pending Commands, Enqueue err, Blocked, Fence context & seqno, (Wait Type, Additional info)\n"); -+ mutex_lock(&kctx->csf.kcpu_queues.lock); -+ -+ idx = find_first_bit(kctx->csf.kcpu_queues.in_use, -+ KBASEP_MAX_KCPU_QUEUES); -+ -+ while (idx < KBASEP_MAX_KCPU_QUEUES) { -+ struct kbase_kcpu_command_queue *queue = -+ kctx->csf.kcpu_queues.array[idx]; -+ -+ seq_printf(file, "%9lu( %s ), ", idx, -+ queue->has_error ? "InErr" : "NoErr"); -+ kbasep_csf_kcpu_debugfs_print_queue(file, kctx, -+ kctx->csf.kcpu_queues.array[idx]); -+ -+ idx = find_next_bit(kctx->csf.kcpu_queues.in_use, -+ KBASEP_MAX_KCPU_QUEUES, idx + 1); -+ } -+ -+ mutex_unlock(&kctx->csf.kcpu_queues.lock); -+ return 0; -+} -+ -+static int kbasep_csf_kcpu_debugfs_open(struct inode *in, struct file *file) -+{ -+ return single_open(file, kbasep_csf_kcpu_debugfs_show, in->i_private); -+} -+ -+static const struct file_operations kbasep_csf_kcpu_debugfs_fops = { -+ .open = kbasep_csf_kcpu_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx) -+{ -+ struct dentry *file; -+#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) -+ const mode_t mode = 0444; -+#else -+ const mode_t mode = 0400; -+#endif -+ -+ if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) -+ return; -+ -+ file = debugfs_create_file("kcpu_queues", mode, kctx->kctx_dentry, -+ kctx, &kbasep_csf_kcpu_debugfs_fops); -+ -+ if (IS_ERR_OR_NULL(file)) { -+ dev_warn(kctx->kbdev->dev, -+ "Unable to create KCPU debugfs entry"); -+ } -+} -+ -+ -+#else -+/* -+ * Stub functions for when debugfs is disabled -+ */ -+void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx) -+{ -+} -+ -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu_debugfs.h -new file mode 100644 -index 0000000..08f2fda ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_kcpu_debugfs.h -@@ -0,0 +1,37 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_KCPU_DEBUGFS_H_ -+#define _KBASE_CSF_KCPU_DEBUGFS_H_ -+ -+/* Forward declaration */ -+struct kbase_context; -+ -+#define MALI_CSF_KCPU_DEBUGFS_VERSION 0 -+ -+/** -+ * kbase_csf_kcpu_debugfs_init() - Create a debugfs entry for KCPU queues -+ * -+ * @kctx: The kbase_context for which to create the debugfs entry -+ */ -+void kbase_csf_kcpu_debugfs_init(struct kbase_context *kctx); -+ -+#endif /* _KBASE_CSF_KCPU_DEBUGFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_protected_memory.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_protected_memory.c -new file mode 100644 -index 0000000..5997483 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_protected_memory.c -@@ -0,0 +1,119 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_csf_protected_memory.h" -+#include <linux/protected_memory_allocator.h> -+ -+#if IS_ENABLED(CONFIG_OF) -+#include <linux/of_platform.h> -+#endif -+ -+int kbase_csf_protected_memory_init(struct kbase_device *const kbdev) -+{ -+ int err = 0; -+ -+#if IS_ENABLED(CONFIG_OF) -+ struct device_node *pma_node = of_parse_phandle(kbdev->dev->of_node, -+ "protected-memory-allocator", 0); -+ if (!pma_node) { -+ dev_info(kbdev->dev, "Protected memory allocator not available\n"); -+ } else { -+ struct platform_device *const pdev = -+ of_find_device_by_node(pma_node); -+ -+ kbdev->csf.pma_dev = NULL; -+ if (!pdev) { -+ dev_err(kbdev->dev, "Platform device for Protected memory allocator not found\n"); -+ } else { -+ kbdev->csf.pma_dev = platform_get_drvdata(pdev); -+ if (!kbdev->csf.pma_dev) { -+ dev_info(kbdev->dev, "Protected memory allocator is not ready\n"); -+ err = -EPROBE_DEFER; -+ } else if (!try_module_get(kbdev->csf.pma_dev->owner)) { -+ dev_err(kbdev->dev, "Failed to get Protected memory allocator module\n"); -+ err = -ENODEV; -+ } else { -+ dev_info(kbdev->dev, "Protected memory allocator successfully loaded\n"); -+ } -+ } -+ of_node_put(pma_node); -+ } -+#endif -+ -+ return err; -+} -+ -+void kbase_csf_protected_memory_term(struct kbase_device *const kbdev) -+{ -+ if (kbdev->csf.pma_dev) -+ module_put(kbdev->csf.pma_dev->owner); -+} -+ -+struct protected_memory_allocation ** -+ kbase_csf_protected_memory_alloc( -+ struct kbase_device *const kbdev, -+ struct tagged_addr *phys, -+ size_t num_pages) -+{ -+ size_t i; -+ struct protected_memory_allocator_device *pma_dev = -+ kbdev->csf.pma_dev; -+ struct protected_memory_allocation **pma = -+ kmalloc_array(num_pages, sizeof(*pma), GFP_KERNEL); -+ -+ if (WARN_ON(!pma_dev) || WARN_ON(!phys) || !pma) -+ return NULL; -+ -+ for (i = 0; i < num_pages; i++) { -+ pma[i] = pma_dev->ops.pma_alloc_page(pma_dev, -+ KBASE_MEM_POOL_4KB_PAGE_TABLE_ORDER); -+ if (!pma[i]) -+ break; -+ -+ phys[i] = as_tagged(pma_dev->ops.pma_get_phys_addr(pma_dev, -+ pma[i])); -+ } -+ -+ if (i != num_pages) { -+ kbase_csf_protected_memory_free(kbdev, pma, i); -+ return NULL; -+ } -+ -+ return pma; -+} -+ -+void kbase_csf_protected_memory_free( -+ struct kbase_device *const kbdev, -+ struct protected_memory_allocation **pma, -+ size_t num_pages) -+{ -+ size_t i; -+ struct protected_memory_allocator_device *pma_dev = -+ kbdev->csf.pma_dev; -+ -+ if (WARN_ON(!pma_dev) || WARN_ON(!pma)) -+ return; -+ -+ for (i = 0; i < num_pages; i++) -+ pma_dev->ops.pma_free_page(pma_dev, pma[i]); -+ -+ kfree(pma); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_protected_memory.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_protected_memory.h -new file mode 100644 -index 0000000..4c0609e ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_protected_memory.h -@@ -0,0 +1,71 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_PROTECTED_MEMORY_H_ -+#define _KBASE_CSF_PROTECTED_MEMORY_H_ -+ -+#include "mali_kbase.h" -+/** -+ * kbase_csf_protected_memory_init - Initilaise protected memory allocator. -+ * -+ * @kbdev: Device pointer. -+ * -+ * Return: 0 if success, or an error code on failure. -+ */ -+int kbase_csf_protected_memory_init(struct kbase_device *const kbdev); -+ -+/** -+ * kbase_csf_protected_memory_term - Terminate prtotected memory allocator. -+ * -+ * @kbdev: Device pointer. -+ */ -+void kbase_csf_protected_memory_term(struct kbase_device *const kbdev); -+ -+/** -+ * kbase_csf_protected_memory_alloc - Allocate protected memory pages. -+ * -+ * @kbdev: Device pointer. -+ * @phys: Array of physical addresses to be filled in by the protected -+ * memory allocator. -+ * @num_pages: Number of pages requested to be allocated. -+ * -+ * Return: Pointer to an array of protected memory allocations on success, -+ * or NULL on failure. -+ */ -+struct protected_memory_allocation ** -+ kbase_csf_protected_memory_alloc( -+ struct kbase_device *const kbdev, -+ struct tagged_addr *phys, -+ size_t num_pages); -+ -+/** -+ * kbase_csf_protected_memory_free - Free the allocated -+ * protected memory pages -+ * -+ * @kbdev: Device pointer. -+ * @pma: Array of pointer to protected memory allocations. -+ * @num_pages: Number of pages to be freed. -+ */ -+void kbase_csf_protected_memory_free( -+ struct kbase_device *const kbdev, -+ struct protected_memory_allocation **pma, -+ size_t num_pages); -+#endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_reset_gpu.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_reset_gpu.c -new file mode 100644 -index 0000000..f6d61d7 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_reset_gpu.c -@@ -0,0 +1,629 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <mali_kbase_ctx_sched.h> -+#include <mali_kbase_hwcnt_context.h> -+#include <device/mali_kbase_device.h> -+#include <backend/gpu/mali_kbase_irq_internal.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> -+#include <mali_kbase_regs_history_debugfs.h> -+#include <csf/mali_kbase_csf_trace_buffer.h> -+#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> -+#include <mali_kbase_reset_gpu.h> -+ -+/* Waiting timeout for GPU reset to complete */ -+#define GPU_RESET_TIMEOUT_MS (5000) /* 5 seconds */ -+#define DUMP_DWORDS_PER_LINE (4) -+/* 16 characters needed for a 8 byte value in hex & 1 character for space */ -+#define DUMP_HEX_CHARS_PER_DWORD ((2 * 8) + 1) -+#define DUMP_HEX_CHARS_PER_LINE \ -+ (DUMP_DWORDS_PER_LINE * DUMP_HEX_CHARS_PER_DWORD) -+ -+static inline bool -+kbase_csf_reset_state_is_silent(enum kbase_csf_reset_gpu_state state) -+{ -+ return (state == KBASE_CSF_RESET_GPU_COMMITTED_SILENT); -+} -+ -+static inline bool -+kbase_csf_reset_state_is_committed(enum kbase_csf_reset_gpu_state state) -+{ -+ return (state == KBASE_CSF_RESET_GPU_COMMITTED || -+ state == KBASE_CSF_RESET_GPU_COMMITTED_SILENT); -+} -+ -+static inline bool -+kbase_csf_reset_state_is_active(enum kbase_csf_reset_gpu_state state) -+{ -+ return (state == KBASE_CSF_RESET_GPU_HAPPENING); -+} -+ -+/** -+ * DOC: Mechanism for coherent access to the HW with respect to GPU reset -+ * -+ * Access to the HW from non-atomic context outside of the reset thread must -+ * use kbase_reset_gpu_prevent_and_wait() / kbase_reset_gpu_try_prevent(). -+ * -+ * This currently works by taking the &kbase_device's csf.reset.sem, for -+ * 'write' access by the GPU reset thread and 'read' access by every other -+ * thread. The use of this rw_semaphore means: -+ * -+ * - there will be mutual exclusion (and thus waiting) between the thread doing -+ * reset ('writer') and threads trying to access the GPU for 'normal' -+ * operations ('readers') -+ * -+ * - multiple threads may prevent reset from happening without serializing each -+ * other prematurely. Note that at present the wait for reset to finish has -+ * to be done higher up in the driver than actual GPU access, at a point -+ * where it won't cause lock ordering issues. At such a point, some paths may -+ * actually lead to no GPU access, but we would prefer to avoid serializing -+ * at that level -+ * -+ * - lockdep (if enabled in the kernel) will check such uses for deadlock -+ * -+ * If instead &kbase_device's csf.reset.wait &wait_queue_head_t were used on -+ * its own, we'd also need to add a &lockdep_map and appropriate lockdep calls -+ * to make use of lockdep checking in all places where the &wait_queue_head_t -+ * is waited upon or signaled. -+ * -+ * Indeed places where we wait on &kbase_device's csf.reset.wait (such as -+ * kbase_reset_gpu_wait()) are the only places where we need extra call(s) to -+ * lockdep, and they are made on the existing rw_semaphore. -+ * -+ * For non-atomic access, the &kbase_device's csf.reset.state member should be -+ * checked instead, such as by using kbase_reset_gpu_is_active(). -+ * -+ * Ideally the &rw_semaphore should be replaced in future with a single mutex -+ * that protects any access to the GPU, via reset or otherwise. -+ */ -+ -+int kbase_reset_gpu_prevent_and_wait(struct kbase_device *kbdev) -+{ -+ down_read(&kbdev->csf.reset.sem); -+ -+ if (atomic_read(&kbdev->csf.reset.state) == -+ KBASE_CSF_RESET_GPU_FAILED) { -+ up_read(&kbdev->csf.reset.sem); -+ return -ENOMEM; -+ } -+ -+ if (WARN_ON(kbase_reset_gpu_is_active(kbdev))) { -+ up_read(&kbdev->csf.reset.sem); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+KBASE_EXPORT_TEST_API(kbase_reset_gpu_prevent_and_wait); -+ -+int kbase_reset_gpu_try_prevent(struct kbase_device *kbdev) -+{ -+ if (!down_read_trylock(&kbdev->csf.reset.sem)) -+ return -EAGAIN; -+ -+ if (atomic_read(&kbdev->csf.reset.state) == -+ KBASE_CSF_RESET_GPU_FAILED) { -+ up_read(&kbdev->csf.reset.sem); -+ return -ENOMEM; -+ } -+ -+ if (WARN_ON(kbase_reset_gpu_is_active(kbdev))) { -+ up_read(&kbdev->csf.reset.sem); -+ return -EFAULT; -+ } -+ -+ return 0; -+} -+ -+void kbase_reset_gpu_allow(struct kbase_device *kbdev) -+{ -+ up_read(&kbdev->csf.reset.sem); -+} -+KBASE_EXPORT_TEST_API(kbase_reset_gpu_allow); -+ -+void kbase_reset_gpu_assert_prevented(struct kbase_device *kbdev) -+{ -+#if KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE -+ lockdep_assert_held_read(&kbdev->csf.reset.sem); -+#else -+ lockdep_assert_held(&kbdev->csf.reset.sem); -+#endif -+ WARN_ON(kbase_reset_gpu_is_active(kbdev)); -+} -+ -+void kbase_reset_gpu_assert_failed_or_prevented(struct kbase_device *kbdev) -+{ -+ if (atomic_read(&kbdev->csf.reset.state) == KBASE_CSF_RESET_GPU_FAILED) -+ return; -+ -+#if KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE -+ lockdep_assert_held_read(&kbdev->csf.reset.sem); -+#else -+ lockdep_assert_held(&kbdev->csf.reset.sem); -+#endif -+ WARN_ON(kbase_reset_gpu_is_active(kbdev)); -+} -+ -+/* Mark the reset as now happening, and synchronize with other threads that -+ * might be trying to access the GPU -+ */ -+static void kbase_csf_reset_begin_hw_access_sync( -+ struct kbase_device *kbdev, -+ enum kbase_csf_reset_gpu_state initial_reset_state) -+{ -+ unsigned long hwaccess_lock_flags; -+ unsigned long scheduler_spin_lock_flags; -+ -+ /* Note this is a WARN/atomic_set because it is a software issue for a -+ * race to be occurring here -+ */ -+ WARN_ON(!kbase_csf_reset_state_is_committed(initial_reset_state)); -+ -+ down_write(&kbdev->csf.reset.sem); -+ -+ /* Threads in atomic context accessing the HW will hold one of these -+ * locks, so synchronize with them too. -+ */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_lock_flags); -+ kbase_csf_scheduler_spin_lock(kbdev, &scheduler_spin_lock_flags); -+ atomic_set(&kbdev->csf.reset.state, KBASE_RESET_GPU_HAPPENING); -+ kbase_csf_scheduler_spin_unlock(kbdev, scheduler_spin_lock_flags); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, hwaccess_lock_flags); -+} -+ -+/* Mark the reset as finished and allow others threads to once more access the -+ * GPU -+ */ -+static void kbase_csf_reset_end_hw_access(struct kbase_device *kbdev, -+ int err_during_reset, -+ bool firmware_inited) -+{ -+ unsigned long hwaccess_lock_flags; -+ unsigned long scheduler_spin_lock_flags; -+ -+ WARN_ON(!kbase_csf_reset_state_is_active( -+ atomic_read(&kbdev->csf.reset.state))); -+ -+ /* Once again, we synchronize with atomic context threads accessing the -+ * HW, as otherwise any actions they defer could get lost -+ */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_lock_flags); -+ kbase_csf_scheduler_spin_lock(kbdev, &scheduler_spin_lock_flags); -+ -+ if (!err_during_reset) { -+ atomic_set(&kbdev->csf.reset.state, -+ KBASE_CSF_RESET_GPU_NOT_PENDING); -+ } else { -+ dev_err(kbdev->dev, "Reset failed to complete"); -+ atomic_set(&kbdev->csf.reset.state, KBASE_CSF_RESET_GPU_FAILED); -+ } -+ -+ kbase_csf_scheduler_spin_unlock(kbdev, scheduler_spin_lock_flags); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, hwaccess_lock_flags); -+ -+ /* Invoke the scheduling tick after formally finishing the reset, -+ * otherwise the tick might start too soon and notice that reset -+ * is still in progress. -+ */ -+ up_write(&kbdev->csf.reset.sem); -+ wake_up(&kbdev->csf.reset.wait); -+ -+ if (!err_during_reset && likely(firmware_inited)) -+ kbase_csf_scheduler_enable_tick_timer(kbdev); -+} -+ -+static void kbase_csf_debug_dump_registers(struct kbase_device *kbdev) -+{ -+ kbase_io_history_dump(kbdev); -+ -+ dev_err(kbdev->dev, "Register state:"); -+ dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x MCU_STATUS=0x%08x", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(MCU_STATUS))); -+ dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x", -+ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT)), -+ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT)), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS))); -+ dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)), -+ kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK)), -+ kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK))); -+ dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0)), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1))); -+ dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x TILER_CONFIG=0x%08x", -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG)), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG)), -+ kbase_reg_read(kbdev, GPU_CONTROL_REG(TILER_CONFIG))); -+} -+ -+static void kbase_csf_dump_firmware_trace_buffer(struct kbase_device *kbdev) -+{ -+ u8 *buf, *line_str; -+ unsigned int read_size; -+ struct firmware_trace_buffer *tb = -+ kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); -+ -+ if (tb == NULL) { -+ dev_dbg(kbdev->dev, "Can't get the trace buffer, firmware trace dump skipped"); -+ return; -+ } -+ -+ buf = kmalloc(PAGE_SIZE + DUMP_HEX_CHARS_PER_LINE + 1, GFP_KERNEL); -+ if (buf == NULL) { -+ dev_err(kbdev->dev, "Short of memory, firmware trace dump skipped"); -+ return; -+ } -+ line_str = &buf[PAGE_SIZE]; -+ -+ dev_err(kbdev->dev, "Firmware trace buffer dump:"); -+ while ((read_size = kbase_csf_firmware_trace_buffer_read_data(tb, buf, -+ PAGE_SIZE))) { -+ u64 *ptr = (u64 *)buf; -+ u32 num_dwords; -+ -+ for (num_dwords = read_size / sizeof(u64); -+ num_dwords >= DUMP_DWORDS_PER_LINE; -+ num_dwords -= DUMP_DWORDS_PER_LINE) { -+ dev_err(kbdev->dev, "%016llx %016llx %016llx %016llx", -+ ptr[0], ptr[1], ptr[2], ptr[3]); -+ ptr += DUMP_DWORDS_PER_LINE; -+ } -+ -+ if (num_dwords) { -+ int pos = 0; -+ -+ while (num_dwords--) { -+ pos += snprintf(line_str + pos, -+ DUMP_HEX_CHARS_PER_DWORD + 1, -+ "%016llx ", ptr[0]); -+ ptr++; -+ } -+ -+ dev_err(kbdev->dev, "%s", line_str); -+ } -+ } -+ -+ kfree(buf); -+} -+ -+/** -+ * kbase_csf_hwcnt_on_reset_error() - Sets HWCNT to appropriate state in the -+ * event of an error during GPU reset. -+ * @kbdev: Pointer to KBase device -+ */ -+static void kbase_csf_hwcnt_on_reset_error(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ /* Treat this as an unrecoverable error for HWCNT */ -+ kbase_hwcnt_backend_csf_on_unrecoverable_error(&kbdev->hwcnt_gpu_iface); -+ -+ /* Re-enable counters to ensure matching enable/disable pair. -+ * This might reduce the hwcnt disable count to 0, and therefore -+ * trigger actual re-enabling of hwcnt. -+ * However, as the backend is now in the unrecoverable error state, -+ * re-enabling will immediately fail and put the context into the error -+ * state, preventing the hardware from being touched (which could have -+ * risked a hang). -+ */ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+static int kbase_csf_reset_gpu_now(struct kbase_device *kbdev, -+ bool firmware_inited, bool silent) -+{ -+ unsigned long flags; -+ int err; -+ -+ WARN_ON(kbdev->irq_reset_flush); -+ /* The reset must now be happening otherwise other threads will not -+ * have been synchronized with to stop their access to the HW -+ */ -+#if KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE -+ lockdep_assert_held_write(&kbdev->csf.reset.sem); -+#elif KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE -+ lockdep_assert_held_exclusive(&kbdev->csf.reset.sem); -+#else -+ lockdep_assert_held(&kbdev->csf.reset.sem); -+#endif -+ WARN_ON(!kbase_reset_gpu_is_active(kbdev)); -+ -+ /* Reset the scheduler state before disabling the interrupts as suspend -+ * of active CSG slots would also be done as a part of reset. -+ */ -+ if (likely(firmware_inited)) -+ kbase_csf_scheduler_reset(kbdev); -+ cancel_work_sync(&kbdev->csf.firmware_reload_work); -+ -+ dev_dbg(kbdev->dev, "Disable GPU hardware counters.\n"); -+ /* This call will block until counters are disabled. -+ */ -+ kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ spin_lock(&kbdev->mmu_mask_change); -+ kbase_pm_reset_start_locked(kbdev); -+ -+ dev_dbg(kbdev->dev, -+ "We're about to flush out the IRQs and their bottom halves\n"); -+ kbdev->irq_reset_flush = true; -+ -+ /* Disable IRQ to avoid IRQ handlers to kick in after releasing the -+ * spinlock; this also clears any outstanding interrupts -+ */ -+ kbase_pm_disable_interrupts_nolock(kbdev); -+ -+ spin_unlock(&kbdev->mmu_mask_change); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ dev_dbg(kbdev->dev, "Ensure that any IRQ handlers have finished\n"); -+ /* Must be done without any locks IRQ handlers will take. -+ */ -+ kbase_synchronize_irqs(kbdev); -+ -+ dev_dbg(kbdev->dev, "Flush out any in-flight work items\n"); -+ kbase_flush_mmu_wqs(kbdev); -+ -+ dev_dbg(kbdev->dev, -+ "The flush has completed so reset the active indicator\n"); -+ kbdev->irq_reset_flush = false; -+ -+ mutex_lock(&kbdev->pm.lock); -+ if (!silent) -+ dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", -+ RESET_TIMEOUT); -+ -+ /* Output the state of some interesting registers to help in the -+ * debugging of GPU resets, and dump the firmware trace buffer -+ */ -+ if (!silent) { -+ kbase_csf_debug_dump_registers(kbdev); -+ if (likely(firmware_inited)) -+ kbase_csf_dump_firmware_trace_buffer(kbdev); -+ } -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_ipa_control_handle_gpu_reset_pre(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* Tell hardware counters a reset is about to occur. -+ * If the backend is in an unrecoverable error state (e.g. due to -+ * firmware being unresponsive) this will transition the backend out of -+ * it, on the assumption a reset will fix whatever problem there was. -+ */ -+ kbase_hwcnt_backend_csf_on_before_reset(&kbdev->hwcnt_gpu_iface); -+ -+ /* Reset the GPU */ -+ err = kbase_pm_init_hw(kbdev, 0); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ -+ if (WARN_ON(err)) { -+ kbase_csf_hwcnt_on_reset_error(kbdev); -+ return err; -+ } -+ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_ctx_sched_restore_all_as(kbdev); -+ kbase_ipa_control_handle_gpu_reset_post(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ kbase_pm_enable_interrupts(kbdev); -+ -+ mutex_lock(&kbdev->pm.lock); -+ kbase_pm_reset_complete(kbdev); -+ /* Synchronously wait for the reload of firmware to complete */ -+ err = kbase_pm_wait_for_desired_state(kbdev); -+ mutex_unlock(&kbdev->pm.lock); -+ -+ if (WARN_ON(err)) { -+ kbase_csf_hwcnt_on_reset_error(kbdev); -+ return err; -+ } -+ -+ /* Re-enable GPU hardware counters */ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ if (!silent) -+ dev_err(kbdev->dev, "Reset complete"); -+ -+ return 0; -+} -+ -+static void kbase_csf_reset_gpu_worker(struct work_struct *data) -+{ -+ struct kbase_device *kbdev = container_of(data, struct kbase_device, -+ csf.reset.work); -+ bool firmware_inited; -+ unsigned long flags; -+ int err = 0; -+ const enum kbase_csf_reset_gpu_state initial_reset_state = -+ atomic_read(&kbdev->csf.reset.state); -+ -+ /* Ensure any threads (e.g. executing the CSF scheduler) have finished -+ * using the HW -+ */ -+ kbase_csf_reset_begin_hw_access_sync(kbdev, initial_reset_state); -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ firmware_inited = kbdev->csf.firmware_inited; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ if (!kbase_pm_context_active_handle_suspend(kbdev, -+ KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { -+ bool silent = -+ kbase_csf_reset_state_is_silent(initial_reset_state); -+ -+ err = kbase_csf_reset_gpu_now(kbdev, firmware_inited, silent); -+ kbase_pm_context_idle(kbdev); -+ } -+ -+ kbase_disjoint_state_down(kbdev); -+ -+ /* Allow other threads to once again use the GPU */ -+ kbase_csf_reset_end_hw_access(kbdev, err, firmware_inited); -+} -+ -+bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev, unsigned int flags) -+{ -+ if (flags & RESET_FLAGS_HWC_UNRECOVERABLE_ERROR) -+ kbase_hwcnt_backend_csf_on_unrecoverable_error( -+ &kbdev->hwcnt_gpu_iface); -+ -+ if (atomic_cmpxchg(&kbdev->csf.reset.state, -+ KBASE_CSF_RESET_GPU_NOT_PENDING, -+ KBASE_CSF_RESET_GPU_PREPARED) != -+ KBASE_CSF_RESET_GPU_NOT_PENDING) -+ /* Some other thread is already resetting the GPU */ -+ return false; -+ -+ return true; -+} -+KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu); -+ -+bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev, -+ unsigned int flags) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ return kbase_prepare_to_reset_gpu(kbdev, flags); -+} -+ -+void kbase_reset_gpu(struct kbase_device *kbdev) -+{ -+ /* Note this is a WARN/atomic_set because it is a software issue for -+ * a race to be occurring here -+ */ -+ if (WARN_ON(atomic_read(&kbdev->csf.reset.state) != -+ KBASE_RESET_GPU_PREPARED)) -+ return; -+ -+ atomic_set(&kbdev->csf.reset.state, KBASE_CSF_RESET_GPU_COMMITTED); -+ dev_err(kbdev->dev, "Preparing to soft-reset GPU\n"); -+ -+ kbase_disjoint_state_up(kbdev); -+ -+ queue_work(kbdev->csf.reset.workq, &kbdev->csf.reset.work); -+} -+KBASE_EXPORT_TEST_API(kbase_reset_gpu); -+ -+void kbase_reset_gpu_locked(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbase_reset_gpu(kbdev); -+} -+ -+int kbase_reset_gpu_silent(struct kbase_device *kbdev) -+{ -+ if (atomic_cmpxchg(&kbdev->csf.reset.state, -+ KBASE_CSF_RESET_GPU_NOT_PENDING, -+ KBASE_CSF_RESET_GPU_COMMITTED_SILENT) != -+ KBASE_CSF_RESET_GPU_NOT_PENDING) { -+ /* Some other thread is already resetting the GPU */ -+ return -EAGAIN; -+ } -+ -+ kbase_disjoint_state_up(kbdev); -+ -+ queue_work(kbdev->csf.reset.workq, &kbdev->csf.reset.work); -+ -+ return 0; -+} -+ -+bool kbase_reset_gpu_is_active(struct kbase_device *kbdev) -+{ -+ enum kbase_csf_reset_gpu_state reset_state = -+ atomic_read(&kbdev->csf.reset.state); -+ -+ /* For CSF, the reset is considered active only when the reset worker -+ * is actually executing and other threads would have to wait for it to -+ * complete -+ */ -+ return kbase_csf_reset_state_is_active(reset_state); -+} -+ -+int kbase_reset_gpu_wait(struct kbase_device *kbdev) -+{ -+ const long wait_timeout = -+ kbase_csf_timeout_in_jiffies(GPU_RESET_TIMEOUT_MS); -+ long remaining; -+ -+ /* Inform lockdep we might be trying to wait on a reset (as -+ * would've been done with down_read() - which has no 'timeout' -+ * variant), then use wait_event_timeout() to implement the timed -+ * wait. -+ * -+ * in CONFIG_PROVE_LOCKING builds, this should catch potential 'time -+ * bound' deadlocks such as: -+ * - incorrect lock order with respect to others locks -+ * - current thread has prevented reset -+ * - current thread is executing the reset worker -+ */ -+ might_lock_read(&kbdev->csf.reset.sem); -+ -+ remaining = wait_event_timeout( -+ kbdev->csf.reset.wait, -+ (atomic_read(&kbdev->csf.reset.state) == -+ KBASE_CSF_RESET_GPU_NOT_PENDING) || -+ (atomic_read(&kbdev->csf.reset.state) == -+ KBASE_CSF_RESET_GPU_FAILED), -+ wait_timeout); -+ -+ if (!remaining) { -+ dev_warn(kbdev->dev, "Timed out waiting for the GPU reset to complete"); -+ return -ETIMEDOUT; -+ } else if (atomic_read(&kbdev->csf.reset.state) == -+ KBASE_CSF_RESET_GPU_FAILED) { -+ return -ENOMEM; -+ } -+ -+ return 0; -+} -+KBASE_EXPORT_TEST_API(kbase_reset_gpu_wait); -+ -+int kbase_reset_gpu_init(struct kbase_device *kbdev) -+{ -+ kbdev->csf.reset.workq = alloc_workqueue("Mali reset workqueue", 0, 1); -+ if (kbdev->csf.reset.workq == NULL) -+ return -ENOMEM; -+ -+ INIT_WORK(&kbdev->csf.reset.work, kbase_csf_reset_gpu_worker); -+ -+ init_waitqueue_head(&kbdev->csf.reset.wait); -+ init_rwsem(&kbdev->csf.reset.sem); -+ -+ return 0; -+} -+ -+void kbase_reset_gpu_term(struct kbase_device *kbdev) -+{ -+ destroy_workqueue(kbdev->csf.reset.workq); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_scheduler.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_scheduler.c -new file mode 100644 -index 0000000..5b795d6 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_scheduler.c -@@ -0,0 +1,5063 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include "mali_kbase_config_defaults.h" -+#include <mali_kbase_ctx_sched.h> -+#include <mali_kbase_reset_gpu.h> -+#include <mali_kbase_as_fault_debugfs.h> -+#include "mali_kbase_csf.h" -+#include <tl/mali_kbase_tracepoints.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> -+#include <linux/export.h> -+#include <uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h> -+#include <uapi/gpu/arm/midgard/mali_base_kernel.h> -+ -+/* Value to indicate that a queue group is not groups_to_schedule list */ -+#define KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID (U32_MAX) -+ -+/* Waiting timeout for scheduler state change for descheduling a CSG */ -+#define CSG_SCHED_STOP_TIMEOUT_MS (50) -+ -+#define CSG_SUSPEND_ON_RESET_WAIT_TIMEOUT_MS DEFAULT_RESET_TIMEOUT_MS -+ -+/* Maximum number of endpoints which may run tiler jobs. */ -+#define CSG_TILER_MAX ((u8)1) -+ -+/* Maximum dynamic CSG slot priority value */ -+#define MAX_CSG_SLOT_PRIORITY ((u8)15) -+ -+/* CSF scheduler time slice value */ -+#define CSF_SCHEDULER_TIME_TICK_MS (100) /* 100 milliseconds */ -+ -+/* -+ * CSF scheduler time threshold for converting "tock" requests into "tick" if -+ * they come too close to the end of a tick interval. This avoids scheduling -+ * twice in a row. -+ */ -+#define CSF_SCHEDULER_TIME_TICK_THRESHOLD_MS \ -+ CSF_SCHEDULER_TIME_TICK_MS -+ -+#define CSF_SCHEDULER_TIME_TICK_THRESHOLD_JIFFIES \ -+ msecs_to_jiffies(CSF_SCHEDULER_TIME_TICK_THRESHOLD_MS) -+ -+/* Nanoseconds per millisecond */ -+#define NS_PER_MS ((u64)1000 * 1000) -+ -+/* -+ * CSF minimum time to reschedule for a new "tock" request. Bursts of "tock" -+ * requests are not serviced immediately, but shall wait for a minimum time in -+ * order to reduce load on the CSF scheduler thread. -+ */ -+#define CSF_SCHEDULER_TIME_TOCK_JIFFIES 1 /* 1 jiffies-time */ -+ -+/* CS suspended and is idle (empty ring buffer) */ -+#define CS_IDLE_FLAG (1 << 0) -+ -+/* CS suspended and is wait for a CQS condition */ -+#define CS_WAIT_SYNC_FLAG (1 << 1) -+ -+/* 2 GPU address space slots are reserved for MCU and privileged context for HW -+ * counter dumping. TODO remove the slot reserved for latter in GPUCORE-26293. -+ */ -+#define NUM_RESERVED_AS_SLOTS (2) -+ -+static int scheduler_group_schedule(struct kbase_queue_group *group); -+static void remove_group_from_idle_wait(struct kbase_queue_group *const group); -+static -+void insert_group_to_runnable(struct kbase_csf_scheduler *const scheduler, -+ struct kbase_queue_group *const group, -+ enum kbase_csf_group_state run_state); -+static struct kbase_queue_group *scheduler_get_protm_enter_async_group( -+ struct kbase_device *const kbdev, -+ struct kbase_queue_group *const group); -+static struct kbase_queue_group *get_tock_top_group( -+ struct kbase_csf_scheduler *const scheduler); -+static void scheduler_enable_tick_timer_nolock(struct kbase_device *kbdev); -+static int suspend_active_queue_groups(struct kbase_device *kbdev, -+ unsigned long *slot_mask); -+static void schedule_in_cycle(struct kbase_queue_group *group, bool force); -+ -+#define kctx_as_enabled(kctx) (!kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) -+ -+/** -+ * tick_timer_callback() - Callback function for the scheduling tick hrtimer -+ * -+ * @timer: Pointer to the device -+ * -+ * This function will enqueue the scheduling tick work item for immediate -+ * execution, if it has not been queued already. -+ * -+ * Return: enum value to indicate that timer should not be restarted. -+ */ -+static enum hrtimer_restart tick_timer_callback(struct hrtimer *timer) -+{ -+ struct kbase_device *kbdev = container_of(timer, struct kbase_device, -+ csf.scheduler.tick_timer); -+ -+ kbase_csf_scheduler_advance_tick(kbdev); -+ return HRTIMER_NORESTART; -+} -+ -+/** -+ * start_tick_timer() - Start the scheduling tick hrtimer. -+ * -+ * @kbdev: Pointer to the device -+ * -+ * This function will start the scheduling tick hrtimer and is supposed to -+ * be called only from the tick work item function. The tick hrtimer should -+ * should not be active already. -+ */ -+static void start_tick_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ WARN_ON(scheduler->tick_timer_active); -+ if (likely(!work_pending(&scheduler->tick_work))) { -+ scheduler->tick_timer_active = true; -+ -+ hrtimer_start(&scheduler->tick_timer, -+ HR_TIMER_DELAY_MSEC(scheduler->csg_scheduling_period_ms), -+ HRTIMER_MODE_REL); -+ } -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+} -+ -+/** -+ * cancel_tick_timer() - Cancel the scheduling tick hrtimer -+ * -+ * @kbdev: Pointer to the device -+ */ -+static void cancel_tick_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ scheduler->tick_timer_active = false; -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ hrtimer_cancel(&scheduler->tick_timer); -+} -+ -+/** -+ * enqueue_tick_work() - Enqueue the scheduling tick work item -+ * -+ * @kbdev: Pointer to the device -+ * -+ * This function will queue the scheduling tick work item for immediate -+ * execution. This shall only be called when both the tick hrtimer and tick -+ * work item are not active/pending. -+ */ -+static void enqueue_tick_work(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ WARN_ON(scheduler->tick_timer_active); -+ queue_work(scheduler->wq, &scheduler->tick_work); -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+} -+ -+static void release_doorbell(struct kbase_device *kbdev, int doorbell_nr) -+{ -+ WARN_ON(doorbell_nr >= CSF_NUM_DOORBELL); -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ clear_bit(doorbell_nr, kbdev->csf.scheduler.doorbell_inuse_bitmap); -+} -+ -+static int acquire_doorbell(struct kbase_device *kbdev) -+{ -+ int doorbell_nr; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ doorbell_nr = find_first_zero_bit( -+ kbdev->csf.scheduler.doorbell_inuse_bitmap, -+ CSF_NUM_DOORBELL); -+ -+ if (doorbell_nr >= CSF_NUM_DOORBELL) -+ return KBASEP_USER_DB_NR_INVALID; -+ -+ set_bit(doorbell_nr, kbdev->csf.scheduler.doorbell_inuse_bitmap); -+ -+ return doorbell_nr; -+} -+ -+static void unassign_user_doorbell_from_group(struct kbase_device *kbdev, -+ struct kbase_queue_group *group) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (group->doorbell_nr != KBASEP_USER_DB_NR_INVALID) { -+ release_doorbell(kbdev, group->doorbell_nr); -+ group->doorbell_nr = KBASEP_USER_DB_NR_INVALID; -+ } -+} -+ -+static void unassign_user_doorbell_from_queue(struct kbase_device *kbdev, -+ struct kbase_queue *queue) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ mutex_lock(&kbdev->csf.reg_lock); -+ -+ if (queue->doorbell_nr != KBASEP_USER_DB_NR_INVALID) { -+ queue->doorbell_nr = KBASEP_USER_DB_NR_INVALID; -+ /* After this the dummy page would be mapped in */ -+ unmap_mapping_range(kbdev->csf.db_filp->f_inode->i_mapping, -+ queue->db_file_offset << PAGE_SHIFT, PAGE_SIZE, 1); -+ } -+ -+ mutex_unlock(&kbdev->csf.reg_lock); -+} -+ -+static void assign_user_doorbell_to_group(struct kbase_device *kbdev, -+ struct kbase_queue_group *group) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (group->doorbell_nr == KBASEP_USER_DB_NR_INVALID) -+ group->doorbell_nr = acquire_doorbell(kbdev); -+} -+ -+static void assign_user_doorbell_to_queue(struct kbase_device *kbdev, -+ struct kbase_queue *const queue) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ mutex_lock(&kbdev->csf.reg_lock); -+ -+ /* If bind operation for the queue hasn't completed yet, then the -+ * the CSI can't be programmed for the queue -+ * (even in stopped state) and so the doorbell also can't be assigned -+ * to it. -+ */ -+ if ((queue->bind_state == KBASE_CSF_QUEUE_BOUND) && -+ (queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID)) { -+ WARN_ON(queue->group->doorbell_nr == KBASEP_USER_DB_NR_INVALID); -+ queue->doorbell_nr = queue->group->doorbell_nr; -+ -+ /* After this the real Hw doorbell page would be mapped in */ -+ unmap_mapping_range( -+ kbdev->csf.db_filp->f_inode->i_mapping, -+ queue->db_file_offset << PAGE_SHIFT, -+ PAGE_SIZE, 1); -+ } -+ -+ mutex_unlock(&kbdev->csf.reg_lock); -+} -+ -+static void scheduler_doorbell_init(struct kbase_device *kbdev) -+{ -+ int doorbell_nr; -+ -+ bitmap_zero(kbdev->csf.scheduler.doorbell_inuse_bitmap, -+ CSF_NUM_DOORBELL); -+ -+ mutex_lock(&kbdev->csf.scheduler.lock); -+ /* Reserve doorbell 0 for use by kernel driver */ -+ doorbell_nr = acquire_doorbell(kbdev); -+ mutex_unlock(&kbdev->csf.scheduler.lock); -+ -+ WARN_ON(doorbell_nr != CSF_KERNEL_DOORBELL_NR); -+} -+ -+static u32 get_nr_active_csgs(struct kbase_device *kbdev) -+{ -+ u32 nr_active_csgs; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ nr_active_csgs = bitmap_weight(kbdev->csf.scheduler.csg_inuse_bitmap, -+ kbdev->csf.global_iface.group_num); -+ -+ return nr_active_csgs; -+} -+ -+/** -+ * csgs_active - returns true if any of CSG slots are in use -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Return: the interface is actively engaged flag. -+ */ -+static bool csgs_active(struct kbase_device *kbdev) -+{ -+ u32 nr_active_csgs; -+ -+ mutex_lock(&kbdev->csf.scheduler.lock); -+ nr_active_csgs = get_nr_active_csgs(kbdev); -+ mutex_unlock(&kbdev->csf.scheduler.lock); -+ -+ /* Right now if any of the CSG interfaces are in use -+ * then we need to assume that there is some work pending. -+ * In future when we have IDLE notifications from firmware implemented -+ * then we would have a better idea of the pending work. -+ */ -+ return (nr_active_csgs != 0); -+} -+ -+/** -+ * csg_slot_in_use - returns true if a queue group has been programmed on a -+ * given CSG slot. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @slot: Index/number of the CSG slot in question. -+ * -+ * Return: the interface is actively engaged flag. -+ * -+ * Note: Caller must hold the scheduler lock. -+ */ -+static inline bool csg_slot_in_use(struct kbase_device *kbdev, int slot) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ return (kbdev->csf.scheduler.csg_slots[slot].resident_group != NULL); -+} -+ -+static bool queue_group_suspended_locked(struct kbase_queue_group *group) -+{ -+ lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.lock); -+ -+ return (group->run_state == KBASE_CSF_GROUP_SUSPENDED || -+ group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE || -+ group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC); -+} -+ -+static bool queue_group_idle_locked(struct kbase_queue_group *group) -+{ -+ lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.lock); -+ -+ return (group->run_state == KBASE_CSF_GROUP_IDLE || -+ group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE); -+} -+ -+static bool queue_group_scheduled(struct kbase_queue_group *group) -+{ -+ return (group->run_state != KBASE_CSF_GROUP_INACTIVE && -+ group->run_state != KBASE_CSF_GROUP_TERMINATED && -+ group->run_state != KBASE_CSF_GROUP_FAULT_EVICTED); -+} -+ -+static bool queue_group_scheduled_locked(struct kbase_queue_group *group) -+{ -+ lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.lock); -+ -+ return queue_group_scheduled(group); -+} -+ -+/** -+ * scheduler_wait_protm_quit() - Wait for GPU to exit protected mode. -+ * -+ * @kbdev: Pointer to the GPU device -+ * -+ * This function waits for the GPU to exit protected mode which is confirmed -+ * when active_protm_grp is set to NULL. -+ */ -+static void scheduler_wait_protm_quit(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ long wt = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ long remaining; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_WAIT_PROTM_QUIT, NULL, -+ jiffies_to_msecs(wt)); -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ !kbase_csf_scheduler_protected_mode_in_use(kbdev), wt); -+ -+ if (!remaining) -+ dev_warn(kbdev->dev, "Timeout, protm_quit wait skipped"); -+ -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_WAIT_PROTM_QUIT_DONE, NULL, -+ jiffies_to_msecs(remaining)); -+} -+ -+/** -+ * scheduler_force_protm_exit() - Force GPU to exit protected mode. -+ * -+ * @kbdev: Pointer to the GPU device -+ * -+ * This function sends a ping request to the firmware and waits for the GPU -+ * to exit protected mode. -+ */ -+static void scheduler_force_protm_exit(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ kbase_csf_firmware_ping(kbdev); -+ scheduler_wait_protm_quit(kbdev); -+} -+ -+/** -+ * scheduler_timer_is_enabled_nolock() - Check if the scheduler wakes up -+ * automatically for periodic tasks. -+ * -+ * @kbdev: Pointer to the device -+ * -+ * This is a variant of kbase_csf_scheduler_timer_is_enabled() that assumes the -+ * CSF scheduler lock to already have been held. -+ * -+ * Return: true if the scheduler is configured to wake up periodically -+ */ -+static bool scheduler_timer_is_enabled_nolock(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ return kbdev->csf.scheduler.timer_enabled; -+} -+ -+static void enable_gpu_idle_fw_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (scheduler->gpu_idle_fw_timer_enabled) -+ return; -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ -+ /* Update the timer_enabled flag requires holding interrupt_lock */ -+ scheduler->gpu_idle_fw_timer_enabled = true; -+ kbase_csf_firmware_enable_gpu_idle_timer(kbdev); -+ -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+} -+ -+static void disable_gpu_idle_fw_timer_locked(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ lockdep_assert_held(&scheduler->interrupt_lock); -+ -+ /* Update of the timer_enabled flag requires holding interrupt_lock */ -+ if (scheduler->gpu_idle_fw_timer_enabled) { -+ scheduler->gpu_idle_fw_timer_enabled = false; -+ kbase_csf_firmware_disable_gpu_idle_timer(kbdev); -+ } -+} -+ -+static void disable_gpu_idle_fw_timer(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (!scheduler->gpu_idle_fw_timer_enabled) -+ return; -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ disable_gpu_idle_fw_timer_locked(kbdev); -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+} -+ -+static void scheduler_wakeup(struct kbase_device *kbdev, bool kick) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (scheduler->state == SCHED_SUSPENDED) { -+ dev_dbg(kbdev->dev, "Re-activating the Scheduler"); -+ kbase_csf_scheduler_pm_active(kbdev); -+ scheduler->state = SCHED_INACTIVE; -+ -+ if (kick) -+ scheduler_enable_tick_timer_nolock(kbdev); -+ } -+} -+ -+static void scheduler_suspend(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) { -+ dev_dbg(kbdev->dev, "Suspending the Scheduler"); -+ kbase_csf_scheduler_pm_idle(kbdev); -+ scheduler->state = SCHED_SUSPENDED; -+ } -+} -+ -+/** -+ * update_idle_suspended_group_state() - Move the queue group to a non-idle -+ * suspended state. -+ * @group: Pointer to the queue group. -+ * -+ * This function is called to change the state of queue group to non-idle -+ * suspended state, if the group was suspended when all the queues bound to it -+ * became empty or when some queues got blocked on a sync wait & others became -+ * empty. The group is also moved to the runnable list from idle wait list in -+ * the latter case. -+ * So the function gets called when a queue is kicked or sync wait condition -+ * gets satisfied. -+ */ -+static void update_idle_suspended_group_state(struct kbase_queue_group *group) -+{ -+ struct kbase_csf_scheduler *scheduler = -+ &group->kctx->kbdev->csf.scheduler; -+ int new_val; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) { -+ remove_group_from_idle_wait(group); -+ insert_group_to_runnable(scheduler, group, -+ KBASE_CSF_GROUP_SUSPENDED); -+ } else if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE) { -+ group->run_state = KBASE_CSF_GROUP_SUSPENDED; -+ -+ /* If scheduler is not suspended and the given group's -+ * static priority (reflected by the scan_seq_num) is inside -+ * the current tick slot-range, schedules an async tock. -+ */ -+ if (scheduler->state != SCHED_SUSPENDED && -+ group->scan_seq_num < scheduler->num_csg_slots_for_tick) -+ schedule_in_cycle(group, true); -+ } else -+ return; -+ -+ new_val = atomic_inc_return(&scheduler->non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(group->kctx->kbdev, SCHEDULER_NONIDLE_OFFSLOT_INC, -+ group, new_val); -+} -+ -+int kbase_csf_scheduler_group_get_slot_locked(struct kbase_queue_group *group) -+{ -+ struct kbase_csf_scheduler *scheduler = -+ &group->kctx->kbdev->csf.scheduler; -+ int slot_num = group->csg_nr; -+ -+ lockdep_assert_held(&scheduler->interrupt_lock); -+ -+ if (slot_num >= 0) { -+ if (WARN_ON(scheduler->csg_slots[slot_num].resident_group != -+ group)) -+ return -1; -+ } -+ -+ return slot_num; -+} -+ -+int kbase_csf_scheduler_group_get_slot(struct kbase_queue_group *group) -+{ -+ struct kbase_csf_scheduler *scheduler = -+ &group->kctx->kbdev->csf.scheduler; -+ unsigned long flags; -+ int slot_num; -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ slot_num = kbase_csf_scheduler_group_get_slot_locked(group); -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ -+ return slot_num; -+} -+ -+static bool kbasep_csf_scheduler_group_is_on_slot_locked( -+ struct kbase_queue_group *group) -+{ -+ struct kbase_csf_scheduler *scheduler = -+ &group->kctx->kbdev->csf.scheduler; -+ int slot_num = group->csg_nr; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (slot_num >= 0) { -+ if (!WARN_ON(scheduler->csg_slots[slot_num].resident_group != -+ group)) -+ return true; -+ } -+ -+ return false; -+} -+ -+bool kbase_csf_scheduler_group_events_enabled(struct kbase_device *kbdev, -+ struct kbase_queue_group *group) -+{ -+ struct kbase_csf_scheduler *scheduler = -+ &group->kctx->kbdev->csf.scheduler; -+ int slot_num = group->csg_nr; -+ -+ lockdep_assert_held(&scheduler->interrupt_lock); -+ -+ if (WARN_ON(slot_num < 0)) -+ return false; -+ -+ return test_bit(slot_num, scheduler->csgs_events_enable_mask); -+} -+ -+struct kbase_queue_group *kbase_csf_scheduler_get_group_on_slot( -+ struct kbase_device *kbdev, int slot) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); -+ -+ return kbdev->csf.scheduler.csg_slots[slot].resident_group; -+} -+ -+static int halt_stream_sync(struct kbase_queue *queue) -+{ -+ struct kbase_queue_group *group = queue->group; -+ struct kbase_device *kbdev = queue->kctx->kbdev; -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ struct kbase_csf_cmd_stream_group_info *ginfo; -+ struct kbase_csf_cmd_stream_info *stream; -+ int csi_index = queue->csi_index; -+ long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ -+ if (WARN_ON(!group) || -+ WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) -+ return -EINVAL; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ ginfo = &global_iface->groups[group->csg_nr]; -+ stream = &ginfo->streams[csi_index]; -+ -+ if (CS_REQ_STATE_GET(kbase_csf_firmware_cs_input_read(stream, CS_REQ)) == -+ CS_REQ_STATE_START) { -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) -+ == CS_ACK_STATE_START), remaining); -+ -+ if (!remaining) { -+ dev_warn(kbdev->dev, "Timed out waiting for queue to start on csi %d bound to group %d on slot %d", -+ csi_index, group->handle, group->csg_nr); -+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) -+ kbase_reset_gpu(kbdev); -+ -+ return -ETIMEDOUT; -+ } -+ -+ remaining = -+ kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ } -+ -+ /* Set state to STOP */ -+ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, CS_REQ_STATE_STOP, -+ CS_REQ_STATE_MASK); -+ -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_STOP_REQUESTED, group, queue, 0u); -+ kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, group->csg_nr, true); -+ -+ /* Timed wait */ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ (CS_ACK_STATE_GET(kbase_csf_firmware_cs_output(stream, CS_ACK)) -+ == CS_ACK_STATE_STOP), remaining); -+ -+ if (!remaining) { -+ dev_warn(kbdev->dev, "Timed out waiting for queue to stop on csi %d bound to group %d on slot %d", -+ queue->csi_index, group->handle, group->csg_nr); -+ -+ /* TODO GPUCORE-25328: The CSG can't be terminated, the GPU -+ * will be reset as a work-around. -+ */ -+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) -+ kbase_reset_gpu(kbdev); -+ } -+ return (remaining) ? 0 : -ETIMEDOUT; -+} -+ -+static bool can_halt_stream(struct kbase_device *kbdev, -+ struct kbase_queue_group *group) -+{ -+ struct kbase_csf_csg_slot *const csg_slot = -+ kbdev->csf.scheduler.csg_slots; -+ unsigned long flags; -+ bool can_halt; -+ int slot; -+ -+ if (!queue_group_scheduled(group)) -+ return true; -+ -+ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); -+ slot = kbase_csf_scheduler_group_get_slot_locked(group); -+ can_halt = (slot >= 0) && -+ (atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING); -+ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, -+ flags); -+ -+ return can_halt; -+} -+ -+/** -+ * sched_halt_stream() - Stop a GPU queue when its queue group is not running -+ * on a CSG slot. -+ * @queue: Pointer to the GPU queue to stop. -+ * -+ * This function handles stopping gpu queues for groups that are either not on -+ * a CSG slot or are on the slot but undergoing transition to -+ * resume or suspend states. -+ * It waits until the queue group is scheduled on a slot and starts running, -+ * which is needed as groups that were suspended may need to resume all queues -+ * that were enabled and running at the time of suspension. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+static int sched_halt_stream(struct kbase_queue *queue) -+{ -+ struct kbase_queue_group *group = queue->group; -+ struct kbase_device *kbdev = queue->kctx->kbdev; -+ struct kbase_csf_scheduler *const scheduler = -+ &kbdev->csf.scheduler; -+ struct kbase_csf_csg_slot *const csg_slot = -+ kbdev->csf.scheduler.csg_slots; -+ bool retry_needed = false; -+ bool retried = false; -+ long remaining; -+ int slot; -+ int err = 0; -+ -+ if (WARN_ON(!group)) -+ return -EINVAL; -+ -+ lockdep_assert_held(&queue->kctx->csf.lock); -+ lockdep_assert_held(&scheduler->lock); -+ -+ slot = kbase_csf_scheduler_group_get_slot(group); -+ -+ if (slot >= 0) { -+ WARN_ON(atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING); -+ -+ if (atomic_read(&csg_slot[slot].state) == CSG_SLOT_READY2RUN) { -+ dev_dbg(kbdev->dev, "Stopping a queue on csi %d when Group-%d is in under transition to running state", -+ queue->csi_index, group->handle); -+ retry_needed = true; -+ } -+ } -+retry: -+ /* Update the group state so that it can get scheduled soon */ -+ update_idle_suspended_group_state(group); -+ -+ mutex_unlock(&scheduler->lock); -+ -+ /* This function is called when the queue group is either not on a CSG -+ * slot or is on the slot but undergoing transition. -+ * -+ * To stop the queue, the function needs to wait either for the queue -+ * group to be assigned a CSG slot (and that slot has to reach the -+ * running state) or for the eviction of the queue group from the -+ * scheduler's list. -+ * -+ * In order to evaluate the latter condition, the function doesn't -+ * really need to lock the scheduler, as any update to the run_state -+ * of the queue group by sched_evict_group() would be visible due -+ * to implicit barriers provided by the kernel waitqueue macros. -+ * -+ * The group pointer cannot disappear meanwhile, as the high level -+ * CSF context is locked. Therefore, the scheduler would be -+ * the only one to update the run_state of the group. -+ */ -+ remaining = wait_event_timeout( -+ kbdev->csf.event_wait, can_halt_stream(kbdev, group), -+ kbase_csf_timeout_in_jiffies( -+ 20 * kbdev->csf.scheduler.csg_scheduling_period_ms)); -+ -+ mutex_lock(&scheduler->lock); -+ -+ if (remaining && queue_group_scheduled_locked(group)) { -+ slot = kbase_csf_scheduler_group_get_slot(group); -+ -+ /* If the group is still on slot and slot is in running state -+ * then explicitly stop the CSI of the -+ * queue. Otherwise there are different cases to consider -+ * -+ * - If the queue group was already undergoing transition to -+ * resume/start state when this function was entered then it -+ * would not have disabled the CSI of the -+ * queue being stopped and the previous wait would have ended -+ * once the slot was in a running state with CS -+ * interface still enabled. -+ * Now the group is going through another transition either -+ * to a suspend state or to a resume state (it could have -+ * been suspended before the scheduler lock was grabbed). -+ * In both scenarios need to wait again for the group to -+ * come on a slot and that slot to reach the running state, -+ * as that would guarantee that firmware will observe the -+ * CSI as disabled. -+ * -+ * - If the queue group was either off the slot or was -+ * undergoing transition to suspend state on entering this -+ * function, then the group would have been resumed with the -+ * queue's CSI in disabled state. -+ * So now if the group is undergoing another transition -+ * (after the resume) then just need to wait for the state -+ * bits in the ACK register of CSI to be -+ * set to STOP value. It is expected that firmware will -+ * process the stop/disable request of the CS -+ * interface after resuming the group before it processes -+ * another state change request of the group. -+ */ -+ if ((slot >= 0) && -+ (atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING)) { -+ err = halt_stream_sync(queue); -+ } else if (retry_needed && !retried) { -+ retried = true; -+ goto retry; -+ } else if (slot >= 0) { -+ struct kbase_csf_global_iface *global_iface = -+ &kbdev->csf.global_iface; -+ struct kbase_csf_cmd_stream_group_info *ginfo = -+ &global_iface->groups[slot]; -+ struct kbase_csf_cmd_stream_info *stream = -+ &ginfo->streams[queue->csi_index]; -+ u32 cs_req = -+ kbase_csf_firmware_cs_input_read(stream, CS_REQ); -+ -+ if (!WARN_ON(CS_REQ_STATE_GET(cs_req) != -+ CS_REQ_STATE_STOP)) { -+ /* Timed wait */ -+ remaining = wait_event_timeout( -+ kbdev->csf.event_wait, -+ (CS_ACK_STATE_GET( -+ kbase_csf_firmware_cs_output( -+ stream, CS_ACK)) == -+ CS_ACK_STATE_STOP), -+ kbdev->csf.fw_timeout_ms); -+ -+ if (!remaining) { -+ dev_warn(kbdev->dev, -+ "Timed out waiting for queue stop ack on csi %d bound to group %d on slot %d", -+ queue->csi_index, -+ group->handle, group->csg_nr); -+ err = -ETIMEDOUT; -+ } -+ } -+ } -+ } else if (!remaining) { -+ dev_warn(kbdev->dev, "Group-%d failed to get a slot for stopping the queue on csi %d", -+ group->handle, queue->csi_index); -+ err = -ETIMEDOUT; -+ } -+ -+ return err; -+} -+ -+int kbase_csf_scheduler_queue_stop(struct kbase_queue *queue) -+{ -+ struct kbase_device *kbdev = queue->kctx->kbdev; -+ struct kbase_queue_group *group = queue->group; -+ bool const cs_enabled = queue->enabled; -+ int err = 0; -+ -+ if (WARN_ON(!group)) -+ return -EINVAL; -+ -+ kbase_reset_gpu_assert_failed_or_prevented(kbdev); -+ lockdep_assert_held(&queue->kctx->csf.lock); -+ mutex_lock(&kbdev->csf.scheduler.lock); -+ -+ queue->enabled = false; -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_STOP, group, queue, cs_enabled); -+ -+ if (cs_enabled && queue_group_scheduled_locked(group)) { -+ struct kbase_csf_csg_slot *const csg_slot = -+ kbdev->csf.scheduler.csg_slots; -+ int slot = kbase_csf_scheduler_group_get_slot(group); -+ -+ /* Since the group needs to be resumed in order to stop the queue, -+ * check if GPU needs to be powered up. -+ */ -+ scheduler_wakeup(kbdev, true); -+ -+ if ((slot >= 0) && -+ (atomic_read(&csg_slot[slot].state) == CSG_SLOT_RUNNING)) -+ err = halt_stream_sync(queue); -+ else -+ err = sched_halt_stream(queue); -+ -+ unassign_user_doorbell_from_queue(kbdev, queue); -+ } -+ -+ mutex_unlock(&kbdev->csf.scheduler.lock); -+ return err; -+} -+ -+static void update_hw_active(struct kbase_queue *queue, bool active) -+{ -+ CSTD_UNUSED(queue); -+ CSTD_UNUSED(active); -+} -+ -+static void program_cs_extract_init(struct kbase_queue *queue) -+{ -+ u64 *input_addr = (u64 *)queue->user_io_addr; -+ u64 *output_addr = (u64 *)(queue->user_io_addr + PAGE_SIZE); -+ -+ input_addr[CS_EXTRACT_INIT_LO / sizeof(u64)] = -+ output_addr[CS_EXTRACT_LO / sizeof(u64)]; -+} -+ -+static void program_cs_trace_cfg(struct kbase_csf_cmd_stream_info *stream, -+ struct kbase_queue *queue) -+{ -+ struct kbase_device *kbdev = queue->kctx->kbdev; -+ u32 const glb_version = kbdev->csf.global_iface.version; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ /* If cs_trace_command not supported, nothing to program */ -+ if (glb_version < kbase_csf_interface_version(1, 1, 0)) -+ return; -+ -+ /* Program for cs_trace if enabled. In the current arrangement, it is -+ * possible for the context to enable the cs_trace after some queues -+ * has been registered in cs_trace in disabled state. This is tracked by -+ * the queue's trace buffer base address, which had been validated at the -+ * queue's register_ex call. -+ */ -+ if (kbase_csf_scheduler_queue_has_trace(queue)) { -+ u32 cs_cfg = CS_INSTR_CONFIG_JASID_SET( -+ queue->trace_cfg, queue->kctx->as_nr); -+ -+ kbase_csf_firmware_cs_input(stream, CS_INSTR_CONFIG, cs_cfg); -+ kbase_csf_firmware_cs_input(stream, CS_INSTR_BUFFER_SIZE, -+ queue->trace_buffer_size); -+ -+ kbase_csf_firmware_cs_input(stream, CS_INSTR_BUFFER_BASE_LO, -+ queue->trace_buffer_base & U32_MAX); -+ kbase_csf_firmware_cs_input(stream, CS_INSTR_BUFFER_BASE_HI, -+ queue->trace_buffer_base >> 32); -+ -+ kbase_csf_firmware_cs_input( -+ stream, CS_INSTR_BUFFER_OFFSET_POINTER_LO, -+ queue->trace_offset_ptr & U32_MAX); -+ kbase_csf_firmware_cs_input( -+ stream, CS_INSTR_BUFFER_OFFSET_POINTER_HI, -+ queue->trace_offset_ptr >> 32); -+ } else { -+ /* Place the configuration to the disabled condition */ -+ kbase_csf_firmware_cs_input(stream, CS_INSTR_CONFIG, 0); -+ kbase_csf_firmware_cs_input(stream, CS_INSTR_BUFFER_SIZE, 0); -+ } -+} -+ -+static void program_cs(struct kbase_device *kbdev, -+ struct kbase_queue *queue, bool ring_csg_doorbell) -+{ -+ struct kbase_queue_group *group = queue->group; -+ struct kbase_csf_cmd_stream_group_info *ginfo; -+ struct kbase_csf_cmd_stream_info *stream; -+ int csi_index = queue->csi_index; -+ u64 user_input; -+ u64 user_output; -+ -+ if (WARN_ON(!group)) -+ return; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) -+ return; -+ -+ ginfo = &kbdev->csf.global_iface.groups[group->csg_nr]; -+ -+ if (WARN_ON(csi_index < 0) || -+ WARN_ON(csi_index >= ginfo->stream_num)) -+ return; -+ -+ assign_user_doorbell_to_queue(kbdev, queue); -+ if (queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID) -+ return; -+ -+ WARN_ON(queue->doorbell_nr != queue->group->doorbell_nr); -+ -+ if (queue->enabled && queue_group_suspended_locked(group)) -+ program_cs_extract_init(queue); -+ -+ stream = &ginfo->streams[csi_index]; -+ -+ kbase_csf_firmware_cs_input(stream, CS_BASE_LO, -+ queue->base_addr & 0xFFFFFFFF); -+ kbase_csf_firmware_cs_input(stream, CS_BASE_HI, -+ queue->base_addr >> 32); -+ kbase_csf_firmware_cs_input(stream, CS_SIZE, -+ queue->size); -+ -+ user_input = (queue->reg->start_pfn << PAGE_SHIFT); -+ kbase_csf_firmware_cs_input(stream, CS_USER_INPUT_LO, -+ user_input & 0xFFFFFFFF); -+ kbase_csf_firmware_cs_input(stream, CS_USER_INPUT_HI, -+ user_input >> 32); -+ -+ user_output = ((queue->reg->start_pfn + 1) << PAGE_SHIFT); -+ kbase_csf_firmware_cs_input(stream, CS_USER_OUTPUT_LO, -+ user_output & 0xFFFFFFFF); -+ kbase_csf_firmware_cs_input(stream, CS_USER_OUTPUT_HI, -+ user_output >> 32); -+ -+ kbase_csf_firmware_cs_input(stream, CS_CONFIG, -+ (queue->doorbell_nr << 8) | (queue->priority & 0xF)); -+ -+ /* Program the queue's cs_trace configuration */ -+ program_cs_trace_cfg(stream, queue); -+ -+ /* Enable all interrupts for now */ -+ kbase_csf_firmware_cs_input(stream, CS_ACK_IRQ_MASK, ~((u32)0)); -+ -+ /* -+ * Enable the CSG idle notification once the CS's ringbuffer -+ * becomes empty or the CS becomes sync_idle, waiting sync update -+ * or protected mode switch. -+ */ -+ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, -+ CS_REQ_IDLE_EMPTY_MASK | CS_REQ_IDLE_SYNC_WAIT_MASK, -+ CS_REQ_IDLE_EMPTY_MASK | CS_REQ_IDLE_SYNC_WAIT_MASK); -+ -+ /* Set state to START/STOP */ -+ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, -+ queue->enabled ? CS_REQ_STATE_START : CS_REQ_STATE_STOP, -+ CS_REQ_STATE_MASK); -+ -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_START, group, queue, queue->enabled); -+ -+ kbase_csf_ring_cs_kernel_doorbell(kbdev, csi_index, group->csg_nr, -+ ring_csg_doorbell); -+ update_hw_active(queue, true); -+} -+ -+int kbase_csf_scheduler_queue_start(struct kbase_queue *queue) -+{ -+ struct kbase_queue_group *group = queue->group; -+ struct kbase_device *kbdev = queue->kctx->kbdev; -+ bool const cs_enabled = queue->enabled; -+ int err = 0; -+ bool evicted = false; -+ -+ kbase_reset_gpu_assert_prevented(kbdev); -+ lockdep_assert_held(&queue->kctx->csf.lock); -+ -+ if (WARN_ON(!group || queue->bind_state != KBASE_CSF_QUEUE_BOUND)) -+ return -EINVAL; -+ -+ mutex_lock(&kbdev->csf.scheduler.lock); -+ -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_START, group, queue, -+ group->run_state); -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_SYNC_STATUS_WAIT, queue->group, -+ queue, queue->status_wait); -+ -+ if (group->run_state == KBASE_CSF_GROUP_FAULT_EVICTED) { -+ err = -EIO; -+ evicted = true; -+ } else if ((group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) -+ && CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) { -+ dev_dbg(kbdev->dev, "blocked queue(csi_index=%d) of group %d was kicked", -+ queue->csi_index, group->handle); -+ } else { -+ err = scheduler_group_schedule(group); -+ -+ if (!err) { -+ queue->enabled = true; -+ if (kbasep_csf_scheduler_group_is_on_slot_locked(group)) { -+ if (cs_enabled) { -+ /* In normal situation, when a queue is -+ * already running, the queue update -+ * would be a doorbell kick on user -+ * side. However, if such a kick is -+ * shortly following a start or resume, -+ * the queue may actually in transition -+ * hence the said kick would enter the -+ * kernel as the hw_active flag is yet -+ * to be set. The sheduler needs to -+ * give a kick to the corresponding -+ * user door-bell on such a case. -+ */ -+ kbase_csf_ring_cs_user_doorbell(kbdev, queue); -+ } else -+ program_cs(kbdev, queue, true); -+ } -+ queue_delayed_work(system_long_wq, -+ &kbdev->csf.scheduler.ping_work, -+ msecs_to_jiffies(FIRMWARE_PING_INTERVAL_MS)); -+ } -+ } -+ -+ mutex_unlock(&kbdev->csf.scheduler.lock); -+ -+ if (evicted) -+ kbase_csf_term_descheduled_queue_group(group); -+ -+ return err; -+} -+ -+static enum kbase_csf_csg_slot_state update_csg_slot_status( -+ struct kbase_device *kbdev, s8 slot) -+{ -+ struct kbase_csf_csg_slot *csg_slot = -+ &kbdev->csf.scheduler.csg_slots[slot]; -+ struct kbase_csf_cmd_stream_group_info *ginfo = -+ &kbdev->csf.global_iface.groups[slot]; -+ u32 state; -+ enum kbase_csf_csg_slot_state slot_state; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ state = CSG_ACK_STATE_GET(kbase_csf_firmware_csg_output(ginfo, -+ CSG_ACK)); -+ slot_state = atomic_read(&csg_slot->state); -+ -+ switch (slot_state) { -+ case CSG_SLOT_READY2RUN: -+ if ((state == CSG_ACK_STATE_START) || -+ (state == CSG_ACK_STATE_RESUME)) { -+ slot_state = CSG_SLOT_RUNNING; -+ atomic_set(&csg_slot->state, slot_state); -+ csg_slot->trigger_jiffies = jiffies; -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STARTED, csg_slot->resident_group, state); -+ dev_dbg(kbdev->dev, "Group %u running on slot %d\n", -+ csg_slot->resident_group->handle, slot); -+ } -+ break; -+ case CSG_SLOT_DOWN2STOP: -+ if ((state == CSG_ACK_STATE_SUSPEND) || -+ (state == CSG_ACK_STATE_TERMINATE)) { -+ slot_state = CSG_SLOT_STOPPED; -+ atomic_set(&csg_slot->state, slot_state); -+ csg_slot->trigger_jiffies = jiffies; -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOPPED, csg_slot->resident_group, state); -+ dev_dbg(kbdev->dev, "Group %u stopped on slot %d\n", -+ csg_slot->resident_group->handle, slot); -+ } -+ break; -+ case CSG_SLOT_DOWN2STOP_TIMEDOUT: -+ case CSG_SLOT_READY2RUN_TIMEDOUT: -+ case CSG_SLOT_READY: -+ case CSG_SLOT_RUNNING: -+ case CSG_SLOT_STOPPED: -+ break; -+ default: -+ dev_warn(kbdev->dev, "Unknown CSG slot state %d", slot_state); -+ break; -+ } -+ -+ return slot_state; -+} -+ -+static bool csg_slot_running(struct kbase_device *kbdev, s8 slot) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ return (update_csg_slot_status(kbdev, slot) == CSG_SLOT_RUNNING); -+} -+ -+static bool csg_slot_stopped_locked(struct kbase_device *kbdev, s8 slot) -+{ -+ enum kbase_csf_csg_slot_state slot_state; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ slot_state = update_csg_slot_status(kbdev, slot); -+ -+ return (slot_state == CSG_SLOT_STOPPED || -+ slot_state == CSG_SLOT_READY); -+} -+ -+static bool csg_slot_stopped_raw(struct kbase_device *kbdev, s8 slot) -+{ -+ struct kbase_csf_cmd_stream_group_info *ginfo = -+ &kbdev->csf.global_iface.groups[slot]; -+ u32 state; -+ -+ state = CSG_ACK_STATE_GET(kbase_csf_firmware_csg_output(ginfo, -+ CSG_ACK)); -+ -+ if (state == CSG_ACK_STATE_SUSPEND || state == CSG_ACK_STATE_TERMINATE) { -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOPPED, kbdev->csf.scheduler.csg_slots[slot].resident_group, state); -+ dev_dbg(kbdev->dev, "(raw status) slot %d stopped\n", slot); -+ return true; -+ } -+ -+ return false; -+} -+ -+static void halt_csg_slot(struct kbase_queue_group *group, bool suspend) -+{ -+ struct kbase_device *kbdev = group->kctx->kbdev; -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ struct kbase_csf_csg_slot *csg_slot = -+ kbdev->csf.scheduler.csg_slots; -+ s8 slot; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) -+ return; -+ -+ slot = group->csg_nr; -+ -+ /* When in transition, wait for it to complete */ -+ if (atomic_read(&csg_slot[slot].state) == CSG_SLOT_READY2RUN) { -+ long remaining = -+ kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ -+ dev_dbg(kbdev->dev, "slot %d wait for up-running\n", slot); -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ csg_slot_running(kbdev, slot), remaining); -+ if (!remaining) -+ dev_warn(kbdev->dev, -+ "slot %d timed out on up-running\n", slot); -+ } -+ -+ if (csg_slot_running(kbdev, slot)) { -+ unsigned long flags; -+ struct kbase_csf_cmd_stream_group_info *ginfo = -+ &global_iface->groups[slot]; -+ u32 halt_cmd = suspend ? CSG_REQ_STATE_SUSPEND : -+ CSG_REQ_STATE_TERMINATE; -+ -+ dev_dbg(kbdev->dev, "Halting(suspend=%d) group %d of context %d_%d on slot %d", -+ suspend, group->handle, group->kctx->tgid, group->kctx->id, slot); -+ -+ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); -+ /* Set state to SUSPEND/TERMINATE */ -+ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, halt_cmd, -+ CSG_REQ_STATE_MASK); -+ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, -+ flags); -+ atomic_set(&csg_slot[slot].state, CSG_SLOT_DOWN2STOP); -+ csg_slot[slot].trigger_jiffies = jiffies; -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STOP, group, halt_cmd); -+ -+ kbase_csf_ring_csg_doorbell(kbdev, slot); -+ } -+} -+ -+static void term_csg_slot(struct kbase_queue_group *group) -+{ -+ halt_csg_slot(group, false); -+} -+ -+static void suspend_csg_slot(struct kbase_queue_group *group) -+{ -+ halt_csg_slot(group, true); -+} -+ -+/** -+ * evaluate_sync_update() - Evaluate the sync wait condition the GPU command -+ * queue has been blocked on. -+ * -+ * @queue: Pointer to the GPU command queue -+ * -+ * Return: true if sync wait condition is satisfied. -+ */ -+static bool evaluate_sync_update(struct kbase_queue *queue) -+{ -+ struct kbase_vmap_struct *mapping; -+ bool updated = false; -+ u32 *sync_ptr; -+ u32 sync_wait_cond; -+ u32 sync_current_val; -+ struct kbase_device *kbdev; -+ -+ if (WARN_ON(!queue)) -+ return false; -+ -+ kbdev = queue->kctx->kbdev; -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ sync_ptr = kbase_phy_alloc_mapping_get(queue->kctx, queue->sync_ptr, -+ &mapping); -+ -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_SYNC_UPDATE, queue->group, -+ queue, queue->sync_ptr); -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_SYNC_BLOCKED_REASON, -+ queue->group, queue, queue->blocked_reason); -+ -+ if (!sync_ptr) { -+ dev_dbg(queue->kctx->kbdev->dev, "sync memory VA 0x%016llX already freed", -+ queue->sync_ptr); -+ goto out; -+ } -+ -+ sync_wait_cond = -+ CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(queue->status_wait); -+ -+ WARN_ON((sync_wait_cond != CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GT) && -+ (sync_wait_cond != CS_STATUS_WAIT_SYNC_WAIT_CONDITION_LE)); -+ -+ sync_current_val = READ_ONCE(*sync_ptr); -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_SYNC_CURRENT_VAL, queue->group, -+ queue, sync_current_val); -+ -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_SYNC_TEST_VAL, queue->group, -+ queue, queue->sync_value); -+ -+ if (((sync_wait_cond == CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GT) && -+ (sync_current_val > queue->sync_value)) || -+ ((sync_wait_cond == CS_STATUS_WAIT_SYNC_WAIT_CONDITION_LE) && -+ (sync_current_val <= queue->sync_value))) { -+ /* The sync wait condition is satisfied so the group to which -+ * queue is bound can be re-scheduled. -+ */ -+ updated = true; -+ } else { -+ dev_dbg(queue->kctx->kbdev->dev, -+ "sync memory not updated yet(%u)", sync_current_val); -+ } -+ -+ kbase_phy_alloc_mapping_put(queue->kctx, mapping); -+out: -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_SYNC_UPDATE_EVALUATED, -+ queue->group, queue, updated); -+ return updated; -+} -+ -+/** -+ * save_slot_cs() - Save the state for blocked GPU command queue. -+ * -+ * @ginfo: Pointer to the CSG interface used by the group -+ * the queue is bound to. -+ * @queue: Pointer to the GPU command queue. -+ * -+ * This function will check if GPU command queue is blocked on a sync wait and -+ * evaluate the wait condition. If the wait condition isn't satisfied it would -+ * save the state needed to reevaluate the condition in future. -+ * The group to which queue is bound shall be in idle state. -+ * -+ * Return: true if the queue is blocked on a sync wait operation. -+ */ -+static -+bool save_slot_cs(struct kbase_csf_cmd_stream_group_info const *const ginfo, -+ struct kbase_queue *queue) -+{ -+ struct kbase_csf_cmd_stream_info *const stream = -+ &ginfo->streams[queue->csi_index]; -+ u32 status = kbase_csf_firmware_cs_output(stream, CS_STATUS_WAIT); -+ bool is_waiting = false; -+ -+ KBASE_KTRACE_ADD_CSF_GRP_Q(stream->kbdev, QUEUE_SYNC_STATUS_WAIT, -+ queue->group, queue, status); -+ -+ if (CS_STATUS_WAIT_SYNC_WAIT_GET(status)) { -+ queue->status_wait = status; -+ queue->sync_ptr = kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_WAIT_SYNC_POINTER_LO); -+ queue->sync_ptr |= (u64)kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_WAIT_SYNC_POINTER_HI) << 32; -+ queue->sync_value = kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_WAIT_SYNC_VALUE); -+ -+ queue->sb_status = CS_STATUS_SCOREBOARDS_NONZERO_GET( -+ kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_SCOREBOARDS)); -+ queue->blocked_reason = CS_STATUS_BLOCKED_REASON_REASON_GET( -+ kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_BLOCKED_REASON)); -+ -+ if (!evaluate_sync_update(queue)) { -+ is_waiting = true; -+ } else { -+ /* Sync object already got updated & met the condition -+ * thus it doesn't need to be reevaluated and so can -+ * clear the 'status_wait' here. -+ */ -+ queue->status_wait = 0; -+ } -+ } else { -+ /* Invalidate wait status info that would have been recorded if -+ * this queue was blocked when the group (in idle state) was -+ * suspended previously. After that the group could have been -+ * unblocked due to the kicking of another queue bound to it & -+ * so the wait status info would have stuck with this queue. -+ */ -+ queue->status_wait = 0; -+ } -+ -+ return is_waiting; -+} -+ -+/** -+ * Calculate how far in the future an event should be scheduled. -+ * -+ * The objective of this function is making sure that a minimum period of -+ * time is guaranteed between handling two consecutive events. -+ * -+ * This function guarantees a minimum period of time between two consecutive -+ * events: given the minimum period and the distance between the current time -+ * and the last event, the function returns the difference between the two. -+ * However, if more time than the minimum period has already elapsed -+ * since the last event, the function will return 0 to schedule work to handle -+ * the event with the lowest latency possible. -+ * -+ * @last_event: Timestamp of the last event, in jiffies. -+ * @time_now: Timestamp of the new event to handle, in jiffies. -+ * Must be successive to last_event. -+ * @period: Minimum period between two events, in jiffies. -+ * -+ * Return: Time to delay work to handle the current event, in jiffies -+ */ -+static unsigned long get_schedule_delay(unsigned long last_event, -+ unsigned long time_now, -+ unsigned long period) -+{ -+ const unsigned long t_distance = time_now - last_event; -+ const unsigned long delay_t = (t_distance < period) ? -+ (period - t_distance) : 0; -+ -+ return delay_t; -+} -+ -+static void schedule_in_cycle(struct kbase_queue_group *group, bool force) -+{ -+ struct kbase_context *kctx = group->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ /* Only try to schedule work for this event if no requests are pending, -+ * otherwise the function will end up canceling previous work requests, -+ * and scheduler is configured to wake up periodically (or the schedule -+ * of work needs to be enforced in situation such as entering into -+ * protected mode). -+ */ -+ if ((likely(scheduler_timer_is_enabled_nolock(kbdev)) || force) && -+ !scheduler->tock_pending_request) { -+ const unsigned long delay = -+ get_schedule_delay(scheduler->last_schedule, jiffies, -+ CSF_SCHEDULER_TIME_TOCK_JIFFIES); -+ scheduler->tock_pending_request = true; -+ dev_dbg(kbdev->dev, "Kicking async for group %d\n", -+ group->handle); -+ mod_delayed_work(scheduler->wq, &scheduler->tock_work, delay); -+ } -+} -+ -+static -+void insert_group_to_runnable(struct kbase_csf_scheduler *const scheduler, -+ struct kbase_queue_group *const group, -+ enum kbase_csf_group_state run_state) -+{ -+ struct kbase_context *const kctx = group->kctx; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE); -+ -+ if (WARN_ON(group->priority >= KBASE_QUEUE_GROUP_PRIORITY_COUNT)) -+ return; -+ -+ group->run_state = run_state; -+ -+ if (run_state == KBASE_CSF_GROUP_RUNNABLE) -+ group->prepared_seq_num = KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID; -+ -+ list_add_tail(&group->link, -+ &kctx->csf.sched.runnable_groups[group->priority]); -+ kctx->csf.sched.num_runnable_grps++; -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_INSERT_RUNNABLE, group, -+ kctx->csf.sched.num_runnable_grps); -+ -+ /* Add the kctx if not yet in runnable kctxs */ -+ if (kctx->csf.sched.num_runnable_grps == 1) { -+ /* First runnable csg, adds to the runnable_kctxs */ -+ INIT_LIST_HEAD(&kctx->csf.link); -+ list_add_tail(&kctx->csf.link, &scheduler->runnable_kctxs); -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_INSERT_RUNNABLE, kctx, 0u); -+ } -+ -+ scheduler->total_runnable_grps++; -+ -+ if (likely(scheduler_timer_is_enabled_nolock(kbdev)) && -+ (scheduler->total_runnable_grps == 1 || -+ scheduler->state == SCHED_SUSPENDED)) { -+ dev_dbg(kbdev->dev, "Kicking scheduler on first runnable group\n"); -+ /* Fire a scheduling to start the time-slice */ -+ enqueue_tick_work(kbdev); -+ } else -+ schedule_in_cycle(group, false); -+ -+ /* Since a new group has become runnable, check if GPU needs to be -+ * powered up. -+ */ -+ scheduler_wakeup(kbdev, false); -+} -+ -+static -+void remove_group_from_runnable(struct kbase_csf_scheduler *const scheduler, -+ struct kbase_queue_group *group, -+ enum kbase_csf_group_state run_state) -+{ -+ struct kbase_context *kctx = group->kctx; -+ struct kbase_queue_group *new_head_grp; -+ struct list_head *list = -+ &kctx->csf.sched.runnable_groups[group->priority]; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ WARN_ON(!queue_group_scheduled_locked(group)); -+ -+ group->run_state = run_state; -+ list_del_init(&group->link); -+ -+ if (scheduler->top_grp == group) { -+ /* -+ * Note: this disables explicit rotation in the next scheduling -+ * cycle. However, removing the top_grp is the same as an -+ * implicit rotation (e.g. if we instead rotated the top_ctx -+ * and then remove top_grp) -+ * -+ * This implicit rotation is assumed by the scheduler rotate -+ * functions. -+ */ -+ scheduler->top_grp = NULL; -+ -+ /* -+ * Trigger a scheduling tock for a CSG containing protected -+ * content in case there has been any in order to minimise -+ * latency. -+ */ -+ group = scheduler_get_protm_enter_async_group(kctx->kbdev, -+ NULL); -+ if (group) -+ schedule_in_cycle(group, true); -+ } -+ -+ kctx->csf.sched.num_runnable_grps--; -+ KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, GROUP_REMOVE_RUNNABLE, group, -+ kctx->csf.sched.num_runnable_grps); -+ new_head_grp = (!list_empty(list)) ? -+ list_first_entry(list, struct kbase_queue_group, link) : -+ NULL; -+ KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, GROUP_HEAD_RUNNABLE, new_head_grp, -+ 0u); -+ -+ if (kctx->csf.sched.num_runnable_grps == 0) { -+ struct kbase_context *new_head_kctx; -+ struct list_head *kctx_list = &scheduler->runnable_kctxs; -+ /* drop the kctx */ -+ list_del_init(&kctx->csf.link); -+ if (scheduler->top_ctx == kctx) -+ scheduler->top_ctx = NULL; -+ KBASE_KTRACE_ADD(kctx->kbdev, SCHEDULER_REMOVE_RUNNABLE, kctx, -+ 0u); -+ new_head_kctx = (!list_empty(kctx_list)) ? -+ list_first_entry(kctx_list, struct kbase_context, csf.link) : -+ NULL; -+ KBASE_KTRACE_ADD(kctx->kbdev, SCHEDULER_HEAD_RUNNABLE, -+ new_head_kctx, 0u); -+ } -+ -+ WARN_ON(scheduler->total_runnable_grps == 0); -+ scheduler->total_runnable_grps--; -+ if (!scheduler->total_runnable_grps) { -+ dev_dbg(kctx->kbdev->dev, "Scheduler idle has no runnable groups"); -+ cancel_tick_timer(kctx->kbdev); -+ WARN_ON(atomic_read(&scheduler->non_idle_offslot_grps)); -+ if (scheduler->state != SCHED_SUSPENDED) -+ queue_work(system_wq, &scheduler->gpu_idle_work); -+ } -+ KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, -+ scheduler->num_active_address_spaces | -+ (((u64)scheduler->total_runnable_grps) << 32)); -+} -+ -+static void insert_group_to_idle_wait(struct kbase_queue_group *const group) -+{ -+ struct kbase_context *kctx = group->kctx; -+ -+ lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock); -+ -+ WARN_ON(group->run_state != KBASE_CSF_GROUP_IDLE); -+ -+ list_add_tail(&group->link, &kctx->csf.sched.idle_wait_groups); -+ kctx->csf.sched.num_idle_wait_grps++; -+ KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, GROUP_INSERT_IDLE_WAIT, group, -+ kctx->csf.sched.num_idle_wait_grps); -+ group->run_state = KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC; -+ dev_dbg(kctx->kbdev->dev, -+ "Group-%d suspended on sync_wait, total wait_groups: %u\n", -+ group->handle, kctx->csf.sched.num_idle_wait_grps); -+} -+ -+static void remove_group_from_idle_wait(struct kbase_queue_group *const group) -+{ -+ struct kbase_context *kctx = group->kctx; -+ struct list_head *list = &kctx->csf.sched.idle_wait_groups; -+ struct kbase_queue_group *new_head_grp; -+ -+ lockdep_assert_held(&kctx->kbdev->csf.scheduler.lock); -+ -+ WARN_ON(group->run_state != KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC); -+ -+ list_del_init(&group->link); -+ WARN_ON(kctx->csf.sched.num_idle_wait_grps == 0); -+ kctx->csf.sched.num_idle_wait_grps--; -+ KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, GROUP_REMOVE_IDLE_WAIT, group, -+ kctx->csf.sched.num_idle_wait_grps); -+ new_head_grp = (!list_empty(list)) ? -+ list_first_entry(list, struct kbase_queue_group, link) : -+ NULL; -+ KBASE_KTRACE_ADD_CSF_GRP(kctx->kbdev, GROUP_HEAD_IDLE_WAIT, -+ new_head_grp, 0u); -+ group->run_state = KBASE_CSF_GROUP_INACTIVE; -+} -+ -+static void deschedule_idle_wait_group(struct kbase_csf_scheduler *scheduler, -+ struct kbase_queue_group *group) -+{ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (WARN_ON(!group)) -+ return; -+ -+ remove_group_from_runnable(scheduler, group, KBASE_CSF_GROUP_IDLE); -+ insert_group_to_idle_wait(group); -+} -+ -+static void update_offslot_non_idle_cnt_for_faulty_grp(struct kbase_queue_group *group) -+{ -+ struct kbase_device *kbdev = group->kctx->kbdev; -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (group->prepared_seq_num < scheduler->non_idle_scanout_grps) { -+ int new_val = -+ atomic_dec_return(&scheduler->non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_DEC, -+ group, new_val); -+ } -+} -+ -+static void update_offslot_non_idle_cnt_for_onslot_grp(struct kbase_queue_group *group) -+{ -+ struct kbase_device *kbdev = group->kctx->kbdev; -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ WARN_ON(group->csg_nr < 0); -+ -+ if (group->prepared_seq_num < scheduler->non_idle_scanout_grps) { -+ int new_val = -+ atomic_dec_return(&scheduler->non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_DEC, -+ group, new_val); -+ } -+} -+ -+static void update_offslot_non_idle_cnt_on_grp_suspend( -+ struct kbase_queue_group *group) -+{ -+ struct kbase_device *kbdev = group->kctx->kbdev; -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (scheduler->state == SCHED_BUSY) { -+ /* active phase or, async entering the protected mode */ -+ if (group->prepared_seq_num >= -+ scheduler->non_idle_scanout_grps) { -+ /* At scanout, it was tagged as on-slot idle */ -+ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED) { -+ int new_val = atomic_inc_return( -+ &scheduler->non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_INC, -+ group, new_val); -+ } -+ } else { -+ if (group->run_state != KBASE_CSF_GROUP_SUSPENDED) { -+ int new_val = atomic_dec_return( -+ &scheduler->non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_DEC, -+ group, new_val); -+ } -+ } -+ } else { -+ /* async phases */ -+ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED) { -+ int new_val = atomic_inc_return( -+ &scheduler->non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_INC, -+ group, new_val); -+ } -+ } -+} -+ -+static bool confirm_cmd_buf_empty(struct kbase_queue *queue) -+{ -+ bool cs_empty; -+ bool cs_idle; -+ u32 sb_status = 0; -+ -+ struct kbase_device const *const kbdev = queue->group->kctx->kbdev; -+ struct kbase_csf_global_iface const *const iface = -+ &kbdev->csf.global_iface; -+ -+ u32 glb_version = iface->version; -+ -+ u64 *input_addr = (u64 *)queue->user_io_addr; -+ u64 *output_addr = (u64 *)(queue->user_io_addr + PAGE_SIZE); -+ -+ if (glb_version >= kbase_csf_interface_version(1, 0, 0)) { -+ /* CS_STATUS_SCOREBOARD supported from CSF 1.0 */ -+ struct kbase_csf_cmd_stream_group_info const *const ginfo = -+ &kbdev->csf.global_iface.groups[queue->group->csg_nr]; -+ struct kbase_csf_cmd_stream_info const *const stream = -+ &ginfo->streams[queue->csi_index]; -+ -+ sb_status = CS_STATUS_SCOREBOARDS_NONZERO_GET( -+ kbase_csf_firmware_cs_output(stream, -+ CS_STATUS_SCOREBOARDS)); -+ } -+ -+ cs_empty = (input_addr[CS_INSERT_LO / sizeof(u64)] == -+ output_addr[CS_EXTRACT_LO / sizeof(u64)]); -+ cs_idle = cs_empty && (!sb_status); -+ -+ return cs_idle; -+} -+ -+static void save_csg_slot(struct kbase_queue_group *group) -+{ -+ struct kbase_device *kbdev = group->kctx->kbdev; -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ struct kbase_csf_cmd_stream_group_info *ginfo; -+ u32 state; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) -+ return; -+ -+ ginfo = &kbdev->csf.global_iface.groups[group->csg_nr]; -+ -+ state = -+ CSG_ACK_STATE_GET(kbase_csf_firmware_csg_output(ginfo, CSG_ACK)); -+ -+ if (!WARN_ON((state != CSG_ACK_STATE_SUSPEND) && -+ (state != CSG_ACK_STATE_TERMINATE))) { -+ u32 max_streams = ginfo->stream_num; -+ u32 i; -+ bool sync_wait = false; -+ bool idle = kbase_csf_firmware_csg_output(ginfo, CSG_STATUS_STATE) & -+ CSG_STATUS_STATE_IDLE_MASK; -+ for (i = 0; idle && i < max_streams; i++) { -+ struct kbase_queue *const queue = -+ group->bound_queues[i]; -+ -+ if (!queue || !queue->enabled) -+ continue; -+ -+ if (save_slot_cs(ginfo, queue)) -+ sync_wait = true; -+ else { -+ /* Need to confirm if ringbuffer of the GPU -+ * queue is empty or not. A race can arise -+ * between the flush of GPU queue and suspend -+ * of CSG. If a queue is flushed after FW has -+ * set the IDLE bit in CSG_STATUS_STATE, then -+ * Scheduler will incorrectly consider CSG -+ * as idle. And there may not be any further -+ * flush call for the GPU queue, which would -+ * have de-idled the CSG. -+ */ -+ idle = confirm_cmd_buf_empty(queue); -+ } -+ } -+ -+ if (idle) { -+ /* Take the suspended group out of the runnable_groups -+ * list of the context and move it to the -+ * idle_wait_groups list. -+ */ -+ if (sync_wait) -+ deschedule_idle_wait_group(scheduler, group); -+ else { -+ group->run_state = -+ KBASE_CSF_GROUP_SUSPENDED_ON_IDLE; -+ dev_dbg(kbdev->dev, "Group-%d suspended: idle", -+ group->handle); -+ } -+ } else { -+ group->run_state = KBASE_CSF_GROUP_SUSPENDED; -+ } -+ -+ update_offslot_non_idle_cnt_on_grp_suspend(group); -+ } -+} -+ -+/* Cleanup_csg_slot after it has been vacated, ready for next csg run. -+ * Return whether there is a kctx address fault associated with the group -+ * for which the clean-up is done. -+ */ -+static bool cleanup_csg_slot(struct kbase_queue_group *group) -+{ -+ struct kbase_context *kctx = group->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ struct kbase_csf_cmd_stream_group_info *ginfo; -+ s8 slot; -+ struct kbase_csf_csg_slot *csg_slot; -+ unsigned long flags; -+ u32 i; -+ bool as_fault = false; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) -+ return as_fault; -+ -+ slot = group->csg_nr; -+ csg_slot = &kbdev->csf.scheduler.csg_slots[slot]; -+ ginfo = &global_iface->groups[slot]; -+ -+ /* Now loop through all the bound CSs, and clean them via a stop */ -+ for (i = 0; i < ginfo->stream_num; i++) { -+ struct kbase_csf_cmd_stream_info *stream = &ginfo->streams[i]; -+ -+ if (group->bound_queues[i]) { -+ if (group->bound_queues[i]->enabled) { -+ kbase_csf_firmware_cs_input_mask(stream, -+ CS_REQ, CS_REQ_STATE_STOP, -+ CS_REQ_STATE_MASK); -+ } -+ -+ unassign_user_doorbell_from_queue(kbdev, -+ group->bound_queues[i]); -+ } -+ } -+ -+ unassign_user_doorbell_from_group(kbdev, group); -+ -+ /* The csg does not need cleanup other than drop its AS */ -+ spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags); -+ as_fault = kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT); -+ kbase_ctx_sched_release_ctx(kctx); -+ if (unlikely(group->faulted)) -+ as_fault = true; -+ spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); -+ -+ /* now marking the slot is vacant */ -+ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); -+ kbdev->csf.scheduler.csg_slots[slot].resident_group = NULL; -+ clear_bit(slot, kbdev->csf.scheduler.csg_slots_idle_mask); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_IDLE_CLEAR, group, -+ kbdev->csf.scheduler.csg_slots_idle_mask[0]); -+ -+ group->csg_nr = KBASEP_CSG_NR_INVALID; -+ set_bit(slot, kbdev->csf.scheduler.csgs_events_enable_mask); -+ clear_bit(slot, kbdev->csf.scheduler.csg_inuse_bitmap); -+ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); -+ -+ csg_slot->trigger_jiffies = jiffies; -+ atomic_set(&csg_slot->state, CSG_SLOT_READY); -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_CLEANED, group, slot); -+ dev_dbg(kbdev->dev, "Cleanup done for group %d on slot %d\n", -+ group->handle, slot); -+ -+ KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG(kbdev, -+ kbdev->gpu_props.props.raw_props.gpu_id, slot); -+ -+ return as_fault; -+} -+ -+static void update_csg_slot_priority(struct kbase_queue_group *group, u8 prio) -+{ -+ struct kbase_device *kbdev = group->kctx->kbdev; -+ struct kbase_csf_csg_slot *csg_slot; -+ struct kbase_csf_cmd_stream_group_info *ginfo; -+ s8 slot; -+ u8 prev_prio; -+ u32 ep_cfg; -+ u32 csg_req; -+ unsigned long flags; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked(group))) -+ return; -+ -+ slot = group->csg_nr; -+ csg_slot = &kbdev->csf.scheduler.csg_slots[slot]; -+ ginfo = &kbdev->csf.global_iface.groups[slot]; -+ -+ /* CSGs remaining on-slot can be either idle or runnable. -+ * This also applies in protected mode. -+ */ -+ WARN_ON(!((group->run_state == KBASE_CSF_GROUP_RUNNABLE) || -+ (group->run_state == KBASE_CSF_GROUP_IDLE))); -+ -+ /* Update consumes a group from scanout */ -+ update_offslot_non_idle_cnt_for_onslot_grp(group); -+ -+ if (csg_slot->priority == prio) -+ return; -+ -+ /* Read the csg_ep_cfg back for updating the priority field */ -+ ep_cfg = kbase_csf_firmware_csg_input_read(ginfo, CSG_EP_REQ); -+ prev_prio = CSG_EP_REQ_PRIORITY_GET(ep_cfg); -+ ep_cfg = CSG_EP_REQ_PRIORITY_SET(ep_cfg, prio); -+ kbase_csf_firmware_csg_input(ginfo, CSG_EP_REQ, ep_cfg); -+ -+ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); -+ csg_req = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); -+ csg_req ^= CSG_REQ_EP_CFG_MASK; -+ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, csg_req, -+ CSG_REQ_EP_CFG_MASK); -+ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); -+ -+ csg_slot->priority = prio; -+ -+ dev_dbg(kbdev->dev, "Priority for group %d of context %d_%d on slot %d to be updated from %u to %u\n", -+ group->handle, group->kctx->tgid, group->kctx->id, slot, -+ prev_prio, prio); -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_PRIO_UPDATE, group, prev_prio); -+ -+ kbase_csf_ring_csg_doorbell(kbdev, slot); -+ set_bit(slot, kbdev->csf.scheduler.csg_slots_prio_update); -+} -+ -+static void program_csg_slot(struct kbase_queue_group *group, s8 slot, -+ u8 prio) -+{ -+ struct kbase_context *kctx = group->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_csf_global_iface *global_iface = &kbdev->csf.global_iface; -+ const u64 shader_core_mask = -+ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_SHADER); -+ const u64 tiler_core_mask = -+ kbase_pm_get_present_cores(kbdev, KBASE_PM_CORE_TILER); -+ const u64 compute_mask = shader_core_mask & group->compute_mask; -+ const u64 fragment_mask = shader_core_mask & group->fragment_mask; -+ const u64 tiler_mask = tiler_core_mask & group->tiler_mask; -+ const u8 num_cores = kbdev->gpu_props.num_cores; -+ const u8 compute_max = min(num_cores, group->compute_max); -+ const u8 fragment_max = min(num_cores, group->fragment_max); -+ const u8 tiler_max = min(CSG_TILER_MAX, group->tiler_max); -+ struct kbase_csf_cmd_stream_group_info *ginfo; -+ u32 ep_cfg = 0; -+ u32 csg_req; -+ u32 state; -+ int i; -+ unsigned long flags; -+ const u64 normal_suspend_buf = -+ group->normal_suspend_buf.reg->start_pfn << PAGE_SHIFT; -+ struct kbase_csf_csg_slot *csg_slot = -+ &kbdev->csf.scheduler.csg_slots[slot]; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (WARN_ON(slot < 0) && -+ WARN_ON(slot >= global_iface->group_num)) -+ return; -+ -+ WARN_ON(atomic_read(&csg_slot->state) != CSG_SLOT_READY); -+ -+ ginfo = &global_iface->groups[slot]; -+ -+ /* Pick an available address space for this context */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_ctx_sched_retain_ctx(kctx); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ if (kctx->as_nr == KBASEP_AS_NR_INVALID) { -+ dev_warn(kbdev->dev, "Could not get a valid AS for group %d of context %d_%d on slot %d\n", -+ group->handle, kctx->tgid, kctx->id, slot); -+ return; -+ } -+ -+ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); -+ set_bit(slot, kbdev->csf.scheduler.csg_inuse_bitmap); -+ kbdev->csf.scheduler.csg_slots[slot].resident_group = group; -+ group->csg_nr = slot; -+ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); -+ -+ assign_user_doorbell_to_group(kbdev, group); -+ -+ /* Now loop through all the bound & kicked CSs, and program them */ -+ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { -+ struct kbase_queue *queue = group->bound_queues[i]; -+ -+ if (queue) -+ program_cs(kbdev, queue, false); -+ } -+ -+ -+ /* Endpoint programming for CSG */ -+ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_COMPUTE_LO, -+ compute_mask & U32_MAX); -+ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_COMPUTE_HI, -+ compute_mask >> 32); -+ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_FRAGMENT_LO, -+ fragment_mask & U32_MAX); -+ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_FRAGMENT_HI, -+ fragment_mask >> 32); -+ kbase_csf_firmware_csg_input(ginfo, CSG_ALLOW_OTHER, -+ tiler_mask & U32_MAX); -+ -+ -+ ep_cfg = CSG_EP_REQ_COMPUTE_EP_SET(ep_cfg, compute_max); -+ ep_cfg = CSG_EP_REQ_FRAGMENT_EP_SET(ep_cfg, fragment_max); -+ ep_cfg = CSG_EP_REQ_TILER_EP_SET(ep_cfg, tiler_max); -+ ep_cfg = CSG_EP_REQ_PRIORITY_SET(ep_cfg, prio); -+ kbase_csf_firmware_csg_input(ginfo, CSG_EP_REQ, ep_cfg); -+ -+ /* Program the address space number assigned to the context */ -+ kbase_csf_firmware_csg_input(ginfo, CSG_CONFIG, kctx->as_nr); -+ -+ kbase_csf_firmware_csg_input(ginfo, CSG_SUSPEND_BUF_LO, -+ normal_suspend_buf & U32_MAX); -+ kbase_csf_firmware_csg_input(ginfo, CSG_SUSPEND_BUF_HI, -+ normal_suspend_buf >> 32); -+ -+ if (group->protected_suspend_buf.reg) { -+ const u64 protm_suspend_buf = -+ group->protected_suspend_buf.reg->start_pfn << -+ PAGE_SHIFT; -+ kbase_csf_firmware_csg_input(ginfo, CSG_PROTM_SUSPEND_BUF_LO, -+ protm_suspend_buf & U32_MAX); -+ kbase_csf_firmware_csg_input(ginfo, CSG_PROTM_SUSPEND_BUF_HI, -+ protm_suspend_buf >> 32); -+ } -+ -+ /* Enable all interrupts for now */ -+ kbase_csf_firmware_csg_input(ginfo, CSG_ACK_IRQ_MASK, ~((u32)0)); -+ -+ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, flags); -+ csg_req = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); -+ csg_req ^= CSG_REQ_EP_CFG_MASK; -+ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, csg_req, -+ CSG_REQ_EP_CFG_MASK); -+ -+ /* Set state to START/RESUME */ -+ if (queue_group_suspended_locked(group)) { -+ state = CSG_REQ_STATE_RESUME; -+ } else { -+ WARN_ON(group->run_state != KBASE_CSF_GROUP_RUNNABLE); -+ state = CSG_REQ_STATE_START; -+ } -+ -+ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, -+ state, CSG_REQ_STATE_MASK); -+ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); -+ -+ /* Update status before rings the door-bell, marking ready => run */ -+ atomic_set(&csg_slot->state, CSG_SLOT_READY2RUN); -+ csg_slot->trigger_jiffies = jiffies; -+ csg_slot->priority = prio; -+ -+ /* Trace the programming of the CSG on the slot */ -+ KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG(kbdev, -+ kbdev->gpu_props.props.raw_props.gpu_id, group->handle, slot); -+ -+ dev_dbg(kbdev->dev, "Starting group %d of context %d_%d on slot %d with priority %u\n", -+ group->handle, kctx->tgid, kctx->id, slot, prio); -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_START, group, -+ (((u64)ep_cfg) << 32) | -+ ((((u32)kctx->as_nr) & 0xF) << 16) | -+ (state & (CSG_REQ_STATE_MASK >> CS_REQ_STATE_SHIFT))); -+ -+ kbase_csf_ring_csg_doorbell(kbdev, slot); -+ -+ /* Programming a slot consumes a group from scanout */ -+ update_offslot_non_idle_cnt_for_onslot_grp(group); -+} -+ -+static void remove_scheduled_group(struct kbase_device *kbdev, -+ struct kbase_queue_group *group) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ WARN_ON(group->prepared_seq_num == -+ KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID); -+ WARN_ON(list_empty(&group->link_to_schedule)); -+ -+ list_del_init(&group->link_to_schedule); -+ scheduler->ngrp_to_schedule--; -+ group->prepared_seq_num = KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID; -+ group->kctx->csf.sched.ngrp_to_schedule--; -+} -+ -+static void sched_evict_group(struct kbase_queue_group *group, bool fault, -+ bool update_non_idle_offslot_grps_cnt) -+{ -+ struct kbase_context *kctx = group->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (queue_group_scheduled_locked(group)) { -+ u32 i; -+ -+ if (update_non_idle_offslot_grps_cnt && -+ (group->run_state == KBASE_CSF_GROUP_SUSPENDED || -+ group->run_state == KBASE_CSF_GROUP_RUNNABLE)) { -+ int new_val = atomic_dec_return( -+ &scheduler->non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_DEC, -+ group, new_val); -+ } -+ -+ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { -+ if (group->bound_queues[i]) -+ group->bound_queues[i]->enabled = false; -+ } -+ -+ if (group->prepared_seq_num != -+ KBASEP_GROUP_PREPARED_SEQ_NUM_INVALID) -+ remove_scheduled_group(kbdev, group); -+ -+ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) -+ remove_group_from_idle_wait(group); -+ else { -+ remove_group_from_runnable(scheduler, group, -+ KBASE_CSF_GROUP_INACTIVE); -+ } -+ -+ WARN_ON(group->run_state != KBASE_CSF_GROUP_INACTIVE); -+ -+ if (fault) -+ group->run_state = KBASE_CSF_GROUP_FAULT_EVICTED; -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_EVICT_SCHED, group, -+ (((u64)scheduler->total_runnable_grps) << 32) | -+ ((u32)group->run_state)); -+ dev_dbg(kbdev->dev, "group %d exited scheduler, num_runnable_grps %d\n", -+ group->handle, scheduler->total_runnable_grps); -+ /* Notify a group has been evicted */ -+ wake_up_all(&kbdev->csf.event_wait); -+ } -+} -+ -+static int term_group_sync(struct kbase_queue_group *group) -+{ -+ struct kbase_device *kbdev = group->kctx->kbdev; -+ long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ int err = 0; -+ -+ term_csg_slot(group); -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ csg_slot_stopped_locked(kbdev, group->csg_nr), remaining); -+ -+ if (!remaining) { -+ dev_warn(kbdev->dev, "term request timed out for group %d of context %d_%d on slot %d", -+ group->handle, group->kctx->tgid, -+ group->kctx->id, group->csg_nr); -+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) -+ kbase_reset_gpu(kbdev); -+ err = -ETIMEDOUT; -+ } -+ -+ return err; -+} -+ -+void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group) -+{ -+ struct kbase_device *kbdev = group->kctx->kbdev; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ long remaining = -+ kbase_csf_timeout_in_jiffies(CSG_SCHED_STOP_TIMEOUT_MS); -+ bool force = false; -+ -+ kbase_reset_gpu_assert_failed_or_prevented(kbdev); -+ lockdep_assert_held(&group->kctx->csf.lock); -+ mutex_lock(&scheduler->lock); -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_DESCHEDULE, group, group->run_state); -+ while (queue_group_scheduled_locked(group)) { -+ u32 saved_state = scheduler->state; -+ -+ if (!kbasep_csf_scheduler_group_is_on_slot_locked(group)) { -+ sched_evict_group(group, false, true); -+ } else if (saved_state == SCHED_INACTIVE || force) { -+ bool as_faulty; -+ -+ term_group_sync(group); -+ /* Treat the csg been terminated */ -+ as_faulty = cleanup_csg_slot(group); -+ /* remove from the scheduler list */ -+ sched_evict_group(group, as_faulty, false); -+ } -+ -+ /* waiting scheduler state to change */ -+ if (queue_group_scheduled_locked(group)) { -+ mutex_unlock(&scheduler->lock); -+ remaining = wait_event_timeout( -+ kbdev->csf.event_wait, -+ saved_state != scheduler->state, -+ remaining); -+ if (!remaining) { -+ dev_warn(kbdev->dev, "Scheduler state change wait timed out for group %d on slot %d", -+ group->handle, group->csg_nr); -+ force = true; -+ } -+ mutex_lock(&scheduler->lock); -+ } -+ } -+ -+ mutex_unlock(&scheduler->lock); -+} -+ -+/** -+ * scheduler_group_schedule() - Schedule a GPU command queue group on firmware -+ * -+ * @group: Pointer to the queue group to be scheduled. -+ * -+ * This function would enable the scheduling of GPU command queue group on -+ * firmware. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+static int scheduler_group_schedule(struct kbase_queue_group *group) -+{ -+ struct kbase_context *kctx = group->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ lockdep_assert_held(&scheduler->lock); -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_SCHEDULE, group, group->run_state); -+ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC) -+ update_idle_suspended_group_state(group); -+ else if (queue_group_idle_locked(group)) { -+ WARN_ON(kctx->csf.sched.num_runnable_grps == 0); -+ WARN_ON(kbdev->csf.scheduler.total_runnable_grps == 0); -+ -+ if (group->run_state == KBASE_CSF_GROUP_SUSPENDED_ON_IDLE) -+ update_idle_suspended_group_state(group); -+ else { -+ struct kbase_queue_group *protm_grp; -+ unsigned long flags; -+ -+ WARN_ON(!kbasep_csf_scheduler_group_is_on_slot_locked( -+ group)); -+ -+ group->run_state = KBASE_CSF_GROUP_RUNNABLE; -+ -+ /* A normal mode CSG could be idle onslot during -+ * protected mode. In this case clear the -+ * appropriate bit in csg_slots_idle_mask. -+ */ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ protm_grp = scheduler->active_protm_grp; -+ if (protm_grp && protm_grp != group) { -+ clear_bit((unsigned int)group->csg_nr, -+ scheduler->csg_slots_idle_mask); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_IDLE_CLEAR, group, -+ scheduler->csg_slots_idle_mask[0]); -+ } -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, -+ flags); -+ -+ /* If GPU is in protected mode then any doorbells rang -+ * would have no effect. Check if GPU is in protected -+ * mode and if this group has higher priority than the -+ * active protected mode group. If so prompt the FW -+ * to exit protected mode. -+ */ -+ if (protm_grp && -+ group->scan_seq_num < protm_grp->scan_seq_num) { -+ /* Prompt the FW to exit protected mode */ -+ scheduler_force_protm_exit(kbdev); -+ } -+ } -+ } else if (!queue_group_scheduled_locked(group)) { -+ int new_val; -+ insert_group_to_runnable(&kbdev->csf.scheduler, group, -+ KBASE_CSF_GROUP_RUNNABLE); -+ /* A new group into the scheduler */ -+ new_val = atomic_inc_return( -+ &kbdev->csf.scheduler.non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_INC, -+ group, new_val); -+ } -+ -+ /* Since a group has become active now, check if GPU needs to be -+ * powered up. Also rekick the Scheduler. -+ */ -+ scheduler_wakeup(kbdev, true); -+ -+ return 0; -+} -+ -+/** -+ * set_max_csg_slots() - Set the number of available CSG slots -+ * -+ * @kbdev: Pointer of the GPU device. -+ * -+ * This function would set/limit the number of CSG slots that -+ * can be used in the given tick/tock. It would be less than the total CSG -+ * slots supported by firmware if the number of GPU address space slots -+ * required to utilize all the CSG slots is more than the available -+ * address space slots. -+ */ -+static inline void set_max_csg_slots(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; -+ unsigned int max_address_space_slots = -+ kbdev->nr_hw_address_spaces - NUM_RESERVED_AS_SLOTS; -+ -+ WARN_ON(scheduler->num_active_address_spaces > total_csg_slots); -+ -+ if (likely(scheduler->num_active_address_spaces <= -+ max_address_space_slots)) -+ scheduler->num_csg_slots_for_tick = total_csg_slots; -+} -+ -+/** -+ * count_active_address_space() - Count the number of GPU address space slots -+ * -+ * @kbdev: Pointer of the GPU device. -+ * @kctx: Pointer of the Kbase context. -+ * -+ * This function would update the counter that is tracking the number of GPU -+ * address space slots that would be required to program the CS -+ * group slots from the groups at the head of groups_to_schedule list. -+ */ -+static inline void count_active_address_space(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ unsigned int total_csg_slots = kbdev->csf.global_iface.group_num; -+ unsigned int max_address_space_slots = -+ kbdev->nr_hw_address_spaces - NUM_RESERVED_AS_SLOTS; -+ -+ if (scheduler->ngrp_to_schedule <= total_csg_slots) { -+ if (kctx->csf.sched.ngrp_to_schedule == 1) -+ scheduler->num_active_address_spaces++; -+ -+ if (scheduler->num_active_address_spaces <= -+ max_address_space_slots) -+ scheduler->num_csg_slots_for_tick++; -+ } -+} -+ -+/* Two schemes are used in assigning the priority to CSG slots for a given -+ * CSG from the 'groups_to_schedule' list. -+ * This is needed as an idle on-slot group is deprioritized by moving it to -+ * the tail of 'groups_to_schedule' list. As a result it can either get -+ * evicted from the CSG slot in current tick/tock dealing, or its position -+ * can be after the lower priority non-idle groups in the 'groups_to_schedule' -+ * list. The latter case can result in the on-slot subset containing both -+ * non-idle and idle CSGs, and is handled through the 2nd scheme described -+ * below. -+ * -+ * First scheme :- If all the slots are going to be occupied by the non-idle or -+ * idle groups, then a simple assignment of the priority is done as per the -+ * position of a group in the 'groups_to_schedule' list. So maximum priority -+ * gets assigned to the slot of a group which is at the head of the list. -+ * Here the 'groups_to_schedule' list would effectively be ordered as per the -+ * static priority of groups. -+ * -+ * Second scheme :- If the slots are going to be occupied by a mix of idle and -+ * non-idle groups then the priority assignment needs to ensure that the -+ * priority of a slot belonging to a higher priority idle group will always be -+ * greater than the priority of a slot belonging to a lower priority non-idle -+ * group, reflecting the original position of a group in the scan order (i.e -+ * static priority) 'scan_seq_num', which is set during the prepare phase of a -+ * tick/tock before the group is moved to 'idle_groups_to_schedule' list if it -+ * is idle. -+ * The priority range [MAX_CSG_SLOT_PRIORITY, 0] is partitioned with the first -+ * 'slots_for_tick' groups in the original scan order are assigned a priority in -+ * the subrange [MAX_CSG_SLOT_PRIORITY, MAX_CSG_SLOT_PRIORITY - slots_for_tick), -+ * whereas rest of the groups are assigned the priority in the subrange -+ * [MAX_CSG_SLOT_PRIORITY - slots_for_tick, 0]. This way even if an idle higher -+ * priority group ends up after the non-idle lower priority groups in the -+ * 'groups_to_schedule' list, it will get a higher slot priority. And this will -+ * enable the FW to quickly start the execution of higher priority group when it -+ * gets de-idled. -+ */ -+static u8 get_slot_priority(struct kbase_queue_group *group) -+{ -+ struct kbase_csf_scheduler *scheduler = -+ &group->kctx->kbdev->csf.scheduler; -+ u8 slot_prio; -+ u32 slots_for_tick = scheduler->num_csg_slots_for_tick; -+ u32 used_slots = slots_for_tick - scheduler->remaining_tick_slots; -+ /* Check if all the slots are going to be occupied by the non-idle or -+ * idle groups. -+ */ -+ if (scheduler->non_idle_scanout_grps >= slots_for_tick || -+ !scheduler->non_idle_scanout_grps) { -+ slot_prio = (u8)(MAX_CSG_SLOT_PRIORITY - used_slots); -+ } else { -+ /* There will be a mix of idle and non-idle groups. */ -+ if (group->scan_seq_num < slots_for_tick) -+ slot_prio = (u8)(MAX_CSG_SLOT_PRIORITY - -+ group->scan_seq_num); -+ else if (MAX_CSG_SLOT_PRIORITY > (slots_for_tick + used_slots)) -+ slot_prio = (u8)(MAX_CSG_SLOT_PRIORITY - (slots_for_tick + used_slots)); -+ else -+ slot_prio = 0; -+ } -+ return slot_prio; -+} -+ -+/** -+ * update_resident_groups_priority() - Update the priority of resident groups -+ * -+ * @kbdev: The GPU device. -+ * -+ * This function will update the priority of all resident queue groups -+ * that are at the head of groups_to_schedule list, preceding the first -+ * non-resident group. -+ * -+ * This function will also adjust kbase_csf_scheduler.remaining_tick_slots on -+ * the priority update. -+ */ -+static void update_resident_groups_priority(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ u32 num_groups = scheduler->num_csg_slots_for_tick; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ while (!list_empty(&scheduler->groups_to_schedule)) { -+ struct kbase_queue_group *group = -+ list_first_entry(&scheduler->groups_to_schedule, -+ struct kbase_queue_group, -+ link_to_schedule); -+ bool resident = -+ kbasep_csf_scheduler_group_is_on_slot_locked(group); -+ -+ if ((group->prepared_seq_num >= num_groups) || !resident) -+ break; -+ -+ update_csg_slot_priority(group, -+ get_slot_priority(group)); -+ -+ /* Drop the head group from the list */ -+ remove_scheduled_group(kbdev, group); -+ scheduler->remaining_tick_slots--; -+ } -+} -+ -+/** -+ * program_group_on_vacant_csg_slot() - Program a non-resident group on the -+ * given vacant CSG slot. -+ * @kbdev: Pointer to the GPU device. -+ * @slot: Vacant CSG slot number. -+ * -+ * This function will program a non-resident group at the head of -+ * kbase_csf_scheduler.groups_to_schedule list on the given vacant -+ * CSG slot, provided the initial position of the non-resident -+ * group in the list is less than the number of CSG slots and there is -+ * an available GPU address space slot. -+ * kbase_csf_scheduler.remaining_tick_slots would also be adjusted after -+ * programming the slot. -+ */ -+static void program_group_on_vacant_csg_slot(struct kbase_device *kbdev, -+ s8 slot) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ struct kbase_queue_group *const group = -+ list_empty(&scheduler->groups_to_schedule) ? NULL : -+ list_first_entry(&scheduler->groups_to_schedule, -+ struct kbase_queue_group, -+ link_to_schedule); -+ u32 num_groups = scheduler->num_csg_slots_for_tick; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ if (group && (group->prepared_seq_num < num_groups)) { -+ bool ret = kbasep_csf_scheduler_group_is_on_slot_locked(group); -+ -+ if (!WARN_ON(ret)) { -+ if (kctx_as_enabled(group->kctx) && !group->faulted) { -+ program_csg_slot(group, slot, -+ get_slot_priority(group)); -+ -+ if (likely(csg_slot_in_use(kbdev, slot))) { -+ /* Drop the head group from the list */ -+ remove_scheduled_group(kbdev, group); -+ scheduler->remaining_tick_slots--; -+ } -+ } else { -+ update_offslot_non_idle_cnt_for_faulty_grp( -+ group); -+ remove_scheduled_group(kbdev, group); -+ } -+ } -+ } -+} -+ -+/** -+ * program_vacant_csg_slot() - Program the vacant CSG slot with a non-resident -+ * group and update the priority of resident groups. -+ * -+ * @kbdev: Pointer to the GPU device. -+ * @slot: Vacant CSG slot number. -+ * -+ * This function will first update the priority of all resident queue groups -+ * that are at the head of groups_to_schedule list, preceding the first -+ * non-resident group, it will then try to program the given CS -+ * group slot with the non-resident group. Finally update the priority of all -+ * resident queue groups following the non-resident group. -+ * -+ * kbase_csf_scheduler.remaining_tick_slots would also be adjusted. -+ */ -+static void program_vacant_csg_slot(struct kbase_device *kbdev, s8 slot) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ struct kbase_csf_csg_slot *const csg_slot = -+ scheduler->csg_slots; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ WARN_ON(atomic_read(&csg_slot[slot].state) != CSG_SLOT_READY); -+ -+ /* First update priority for already resident groups (if any) -+ * before the non-resident group -+ */ -+ update_resident_groups_priority(kbdev); -+ -+ /* Now consume the vacant slot for the non-resident group */ -+ program_group_on_vacant_csg_slot(kbdev, slot); -+ -+ /* Now update priority for already resident groups (if any) -+ * following the non-resident group -+ */ -+ update_resident_groups_priority(kbdev); -+} -+ -+static bool slots_state_changed(struct kbase_device *kbdev, -+ unsigned long *slots_mask, -+ bool (*state_check_func)(struct kbase_device *, s8)) -+{ -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ DECLARE_BITMAP(changed_slots, MAX_SUPPORTED_CSGS) = {0}; -+ bool changed = false; -+ u32 i; -+ -+ for_each_set_bit(i, slots_mask, num_groups) { -+ if (state_check_func(kbdev, (s8)i)) { -+ set_bit(i, changed_slots); -+ changed = true; -+ } -+ } -+ -+ if (changed) -+ bitmap_copy(slots_mask, changed_slots, MAX_SUPPORTED_CSGS); -+ -+ return changed; -+} -+ -+/** -+ * program_suspending_csg_slots() - Program the CSG slots vacated on suspension -+ * of queue groups running on them. -+ * -+ * @kbdev: Pointer to the GPU device. -+ * -+ * This function will first wait for the ongoing suspension to complete on a -+ * CSG slot and will then program the vacant slot with the -+ * non-resident queue group inside the groups_to_schedule list. -+ * The programming of the non-resident queue group on the vacant slot could -+ * fail due to unavailability of free GPU address space slot and so the -+ * programming is re-attempted after the ongoing suspension has completed -+ * for all the CSG slots. -+ * The priority of resident groups before and after the non-resident group -+ * in the groups_to_schedule list would also be updated. -+ * This would be repeated for all the slots undergoing suspension. -+ * GPU reset would be initiated if the wait for suspend times out. -+ */ -+static void program_suspending_csg_slots(struct kbase_device *kbdev) -+{ -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS); -+ DECLARE_BITMAP(evicted_mask, MAX_SUPPORTED_CSGS) = {0}; -+ bool suspend_wait_failed = false; -+ long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ /* In the current implementation, csgs_events_enable_mask would be used -+ * only to indicate suspending CSGs. -+ */ -+ bitmap_complement(slot_mask, scheduler->csgs_events_enable_mask, -+ MAX_SUPPORTED_CSGS); -+ -+ while (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { -+ DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); -+ -+ bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ slots_state_changed(kbdev, changed, -+ csg_slot_stopped_raw), -+ remaining); -+ -+ if (remaining) { -+ u32 i; -+ -+ for_each_set_bit(i, changed, num_groups) { -+ struct kbase_queue_group *group = -+ scheduler->csg_slots[i].resident_group; -+ -+ if (WARN_ON(!csg_slot_stopped_locked(kbdev, (s8)i))) { -+ continue; -+ } -+ /* The on slot csg is now stopped */ -+ clear_bit(i, slot_mask); -+ -+ if (likely(group)) { -+ bool as_fault; -+ /* Only do save/cleanup if the -+ * group is not terminated during -+ * the sleep. -+ */ -+ save_csg_slot(group); -+ as_fault = cleanup_csg_slot(group); -+ /* If AS fault detected, evict it */ -+ if (as_fault) { -+ sched_evict_group(group, true, true); -+ set_bit(i, evicted_mask); -+ } -+ } -+ -+ program_vacant_csg_slot(kbdev, (s8)i); -+ } -+ } else { -+ u32 i; -+ -+ /* Groups that have failed to suspend in time shall -+ * raise a fatal error as they could no longer be -+ * safely resumed. -+ */ -+ for_each_set_bit(i, slot_mask, num_groups) { -+ struct kbase_queue_group *const group = -+ scheduler->csg_slots[i].resident_group; -+ -+ struct base_gpu_queue_group_error const -+ err_payload = { .error_type = -+ BASE_GPU_QUEUE_GROUP_ERROR_FATAL, -+ .payload = { -+ .fatal_group = { -+ .status = -+ GPU_EXCEPTION_TYPE_SW_FAULT_2, -+ } } }; -+ -+ if (unlikely(group == NULL)) -+ continue; -+ -+ kbase_csf_add_group_fatal_error(group, -+ &err_payload); -+ kbase_event_wakeup(group->kctx); -+ -+ /* TODO GPUCORE-25328: The CSG can't be -+ * terminated, the GPU will be reset as a -+ * work-around. -+ */ -+ dev_warn( -+ kbdev->dev, -+ "Group %d of context %d_%d on slot %u failed to suspend", -+ group->handle, group->kctx->tgid, -+ group->kctx->id, i); -+ -+ /* The group has failed suspension, stop -+ * further examination. -+ */ -+ clear_bit(i, slot_mask); -+ set_bit(i, scheduler->csgs_events_enable_mask); -+ update_offslot_non_idle_cnt_for_onslot_grp( -+ group); -+ } -+ -+ suspend_wait_failed = true; -+ } -+ } -+ -+ if (!bitmap_empty(evicted_mask, MAX_SUPPORTED_CSGS)) -+ dev_info(kbdev->dev, "Scheduler evicting slots: 0x%*pb\n", -+ num_groups, evicted_mask); -+ -+ if (likely(!suspend_wait_failed)) { -+ u32 i; -+ -+ while (scheduler->ngrp_to_schedule && -+ scheduler->remaining_tick_slots) { -+ i = find_first_zero_bit(scheduler->csg_inuse_bitmap, -+ num_groups); -+ if (WARN_ON(i == num_groups)) -+ break; -+ program_vacant_csg_slot(kbdev, (s8)i); -+ if (!csg_slot_in_use(kbdev, (int)i)) { -+ dev_warn(kbdev->dev, "Couldn't use CSG slot %d despite being vacant", i); -+ break; -+ } -+ } -+ } else { -+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) -+ kbase_reset_gpu(kbdev); -+ } -+} -+ -+static void suspend_queue_group(struct kbase_queue_group *group) -+{ -+ unsigned long flags; -+ struct kbase_csf_scheduler *const scheduler = -+ &group->kctx->kbdev->csf.scheduler; -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ /* This shall be used in program_suspending_csg_slots() where we -+ * assume that whilst CSGs are being suspended, this bitmask is not -+ * used by anything else i.e., it indicates only the CSGs going -+ * through suspension. -+ */ -+ clear_bit(group->csg_nr, scheduler->csgs_events_enable_mask); -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ -+ /* If AS fault detected, terminate the group */ -+ if (!kctx_as_enabled(group->kctx) || group->faulted) -+ term_csg_slot(group); -+ else -+ suspend_csg_slot(group); -+} -+ -+static void wait_csg_slots_start(struct kbase_device *kbdev) -+{ -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ long remaining = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; -+ u32 i; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ /* extract start slot flags for check */ -+ for (i = 0; i < num_groups; i++) { -+ if (atomic_read(&scheduler->csg_slots[i].state) == -+ CSG_SLOT_READY2RUN) -+ set_bit(i, slot_mask); -+ } -+ -+ while (!bitmap_empty(slot_mask, MAX_SUPPORTED_CSGS)) { -+ DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); -+ -+ bitmap_copy(changed, slot_mask, MAX_SUPPORTED_CSGS); -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ slots_state_changed(kbdev, changed, csg_slot_running), -+ remaining); -+ -+ if (remaining) { -+ for_each_set_bit(i, changed, num_groups) { -+ struct kbase_queue_group *group = -+ scheduler->csg_slots[i].resident_group; -+ -+ /* The on slot csg is now running */ -+ clear_bit(i, slot_mask); -+ group->run_state = KBASE_CSF_GROUP_RUNNABLE; -+ } -+ } else { -+ dev_warn(kbdev->dev, "Timed out waiting for CSG slots to start, slots: 0x%*pb\n", -+ num_groups, slot_mask); -+ -+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) -+ kbase_reset_gpu(kbdev); -+ break; -+ } -+ } -+} -+ -+/** -+ * group_on_slot_is_idle() - Check if the given slot has a CSG-idle state -+ * flagged after the completion of a CSG status -+ * update command -+ * -+ * This function is called at the start of scheduling tick to check the -+ * idle status of a queue group resident on a CSG slot. -+ * The caller must make sure the corresponding status update command has -+ * been called and completed before checking this status. -+ * -+ * @kbdev: Pointer to the GPU device. -+ * @slot: The given slot for checking an occupying resident group's idle -+ * state. -+ * -+ * Return: true if the group resident on slot is idle, otherwise false. -+ */ -+static bool group_on_slot_is_idle(struct kbase_device *kbdev, -+ unsigned long slot) -+{ -+ struct kbase_csf_cmd_stream_group_info *ginfo = -+ &kbdev->csf.global_iface.groups[slot]; -+ bool idle = kbase_csf_firmware_csg_output(ginfo, CSG_STATUS_STATE) & -+ CSG_STATUS_STATE_IDLE_MASK; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ return idle; -+} -+ -+/** -+ * slots_update_state_changed() - Check the handshake state of a subset of -+ * command group slots. -+ * -+ * Checks the state of a subset of slots selected through the slots_mask -+ * bit_map. Records which slots' handshake completed and send it back in the -+ * slots_done bit_map. -+ * -+ * @kbdev: The GPU device. -+ * @field_mask: The field mask for checking the state in the csg_req/ack. -+ * @slots_mask: A bit_map specifying the slots to check. -+ * @slots_done: A cleared bit_map for returning the slots that -+ * have finished update. -+ * -+ * Return: true if the slots_done is set for at least one slot. -+ * Otherwise false. -+ */ -+static -+bool slots_update_state_changed(struct kbase_device *kbdev, u32 field_mask, -+ const unsigned long *slots_mask, unsigned long *slots_done) -+{ -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ bool changed = false; -+ u32 i; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ for_each_set_bit(i, slots_mask, num_groups) { -+ struct kbase_csf_cmd_stream_group_info const *const ginfo = -+ &kbdev->csf.global_iface.groups[i]; -+ u32 state = kbase_csf_firmware_csg_input_read(ginfo, CSG_REQ); -+ -+ state ^= kbase_csf_firmware_csg_output(ginfo, CSG_ACK); -+ -+ if (!(state & field_mask)) { -+ set_bit(i, slots_done); -+ changed = true; -+ } -+ } -+ -+ return changed; -+} -+ -+/** -+ * wait_csg_slots_handshake_ack - Wait the req/ack handshakes to complete on -+ * the specified groups. -+ * -+ * This function waits for the acknowledgement of the request that have -+ * already been placed for the CSG slots by the caller. Currently used for -+ * the CSG priority update and status update requests. -+ * -+ * @kbdev: Pointer to the GPU device. -+ * @field_mask: The field mask for checking the state in the csg_req/ack. -+ * @slot_mask: Bitmap reflecting the slots, the function will modify -+ * the acknowledged slots by clearing their corresponding -+ * bits. -+ * @wait_in_jiffies: Wait duration in jiffies, controlling the time-out. -+ * -+ * Return: 0 on all specified slots acknowledged; otherwise -ETIMEDOUT. For -+ * timed out condition with unacknowledged slots, their bits remain -+ * set in the slot_mask. -+ */ -+static int wait_csg_slots_handshake_ack(struct kbase_device *kbdev, -+ u32 field_mask, unsigned long *slot_mask, long wait_in_jiffies) -+{ -+ const u32 num_groups = kbdev->csf.global_iface.group_num; -+ long remaining = wait_in_jiffies; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ while (!bitmap_empty(slot_mask, num_groups) && -+ !kbase_reset_gpu_is_active(kbdev)) { -+ DECLARE_BITMAP(dones, MAX_SUPPORTED_CSGS) = { 0 }; -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ slots_update_state_changed(kbdev, field_mask, -+ slot_mask, dones), -+ remaining); -+ -+ if (remaining) -+ bitmap_andnot(slot_mask, slot_mask, dones, num_groups); -+ else -+ /* Timed-out on the wait */ -+ return -ETIMEDOUT; -+ } -+ -+ return 0; -+} -+ -+static void wait_csg_slots_finish_prio_update(struct kbase_device *kbdev) -+{ -+ unsigned long *slot_mask = -+ kbdev->csf.scheduler.csg_slots_prio_update; -+ long wait_time = kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ int ret = wait_csg_slots_handshake_ack(kbdev, CSG_REQ_EP_CFG_MASK, -+ slot_mask, wait_time); -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (ret != 0) { -+ /* The update timeout is not regarded as a serious -+ * issue, no major consequences are expected as a -+ * result, so just warn the case. -+ */ -+ dev_warn( -+ kbdev->dev, -+ "Timeout on CSG_REQ:EP_CFG, skipping the update wait: slot mask=0x%lx", -+ slot_mask[0]); -+ } -+} -+ -+void kbase_csf_scheduler_evict_ctx_slots(struct kbase_device *kbdev, -+ struct kbase_context *kctx, struct list_head *evicted_groups) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ struct kbase_queue_group *group; -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ u32 slot; -+ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ mutex_lock(&scheduler->lock); -+ -+ /* This code is only called during reset, so we don't wait for the CSG -+ * slots to be stopped -+ */ -+ WARN_ON(!kbase_reset_gpu_is_active(kbdev)); -+ -+ KBASE_KTRACE_ADD(kbdev, EVICT_CTX_SLOTS, kctx, 0u); -+ for (slot = 0; slot < num_groups; slot++) { -+ group = kbdev->csf.scheduler.csg_slots[slot].resident_group; -+ if (group && group->kctx == kctx) { -+ bool as_fault; -+ -+ term_csg_slot(group); -+ as_fault = cleanup_csg_slot(group); -+ /* remove the group from the scheduler list */ -+ sched_evict_group(group, as_fault, false); -+ /* return the evicted group to the caller */ -+ list_add_tail(&group->link, evicted_groups); -+ set_bit(slot, slot_mask); -+ } -+ } -+ -+ dev_info(kbdev->dev, "Evicting context %d_%d slots: 0x%*pb\n", -+ kctx->tgid, kctx->id, num_groups, slot_mask); -+ -+ mutex_unlock(&scheduler->lock); -+} -+ -+/** -+ * scheduler_slot_protm_ack - Acknowledging the protected region requests -+ * from the resident group on a given slot. -+ * -+ * The function assumes that the given slot is in stable running state and -+ * has already been judged by the caller on that any pending protected region -+ * requests of the resident group should be acknowledged. -+ * -+ * @kbdev: Pointer to the GPU device. -+ * @group: Pointer to the resident group on the given slot. -+ * @slot: The slot that the given group is actively operating on. -+ * -+ * Return: true if the group has pending protm request(s) and is acknowledged. -+ * The caller should arrange to enter the protected mode for servicing -+ * it. Otherwise return false, indicating the group has no pending protm -+ * request. -+ */ -+static bool scheduler_slot_protm_ack(struct kbase_device *const kbdev, -+ struct kbase_queue_group *const group, -+ const int slot) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ bool protm_ack = false; -+ struct kbase_csf_cmd_stream_group_info *ginfo = -+ &kbdev->csf.global_iface.groups[slot]; -+ u32 max_csi; -+ int i; -+ -+ if (WARN_ON(scheduler->csg_slots[slot].resident_group != group)) -+ return protm_ack; -+ -+ lockdep_assert_held(&scheduler->lock); -+ lockdep_assert_held(&group->kctx->kbdev->csf.scheduler.interrupt_lock); -+ -+ max_csi = ginfo->stream_num; -+ for (i = find_first_bit(group->protm_pending_bitmap, max_csi); -+ i < max_csi; -+ i = find_next_bit(group->protm_pending_bitmap, max_csi, i + 1)) { -+ struct kbase_queue *queue = group->bound_queues[i]; -+ -+ clear_bit(i, group->protm_pending_bitmap); -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, PROTM_PENDING_CLEAR, group, -+ queue, group->protm_pending_bitmap[0]); -+ -+ if (!WARN_ON(!queue) && queue->enabled) { -+ struct kbase_csf_cmd_stream_info *stream = -+ &ginfo->streams[i]; -+ u32 cs_protm_ack = kbase_csf_firmware_cs_output( -+ stream, CS_ACK) & -+ CS_ACK_PROTM_PEND_MASK; -+ u32 cs_protm_req = kbase_csf_firmware_cs_input_read( -+ stream, CS_REQ) & -+ CS_REQ_PROTM_PEND_MASK; -+ -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, CSI_PROTM_ACK, group, -+ queue, cs_protm_ack ^ cs_protm_req); -+ -+ if (cs_protm_ack == cs_protm_req) { -+ dev_dbg(kbdev->dev, -+ "PROTM-ack already done for queue-%d group-%d slot-%d", -+ queue->csi_index, group->handle, slot); -+ continue; -+ } -+ -+ kbase_csf_firmware_cs_input_mask(stream, CS_REQ, -+ cs_protm_ack, -+ CS_ACK_PROTM_PEND_MASK); -+ protm_ack = true; -+ dev_dbg(kbdev->dev, -+ "PROTM-ack for queue-%d, group-%d slot-%d", -+ queue->csi_index, group->handle, slot); -+ } -+ } -+ -+ return protm_ack; -+} -+ -+/** -+ * scheduler_group_check_protm_enter - Request the given group to be evaluated -+ * for triggering the protected mode. -+ * -+ * The function assumes the given group is either an active running group or -+ * the scheduler internally maintained field scheduler->top_grp. -+ * -+ * If the GPU is not already running in protected mode and the input group -+ * has protected region requests from its bound queues, the requests are -+ * acknowledged and the GPU is instructed to enter the protected mode. -+ * -+ * @kbdev: Pointer to the GPU device. -+ * @input_grp: Pointer to the GPU queue group. -+ */ -+static void scheduler_group_check_protm_enter(struct kbase_device *const kbdev, -+ struct kbase_queue_group *const input_grp) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ bool protm_in_use; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ -+ protm_in_use = kbase_csf_scheduler_protected_mode_in_use(kbdev); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_CHECK_PROTM_ENTER, input_grp, -+ protm_in_use); -+ -+ /* Firmware samples the PROTM_PEND ACK bit for CSs when -+ * Host sends PROTM_ENTER global request. So if PROTM_PEND ACK bit -+ * is set for a CS after Host has sent the PROTM_ENTER -+ * Global request, then there is no guarantee that firmware will -+ * notice that prior to switching to protected mode. And firmware -+ * may not again raise the PROTM_PEND interrupt for that CS -+ * later on. To avoid that uncertainty PROTM_PEND ACK bit -+ * is not set for a CS if the request to enter protected -+ * mode has already been sent. It will be set later (after the exit -+ * from protected mode has taken place) when the group to which -+ * CS is bound becomes the top group. -+ * -+ * The actual decision of entering protected mode is hinging on the -+ * input group is the top priority group, or, in case the previous -+ * top-group is evicted from the scheduler during the tick, its would -+ * be replacement, and that it is currently in a stable state (i.e. the -+ * slot state is running). -+ */ -+ if (!protm_in_use && !WARN_ON(!input_grp)) { -+ const int slot = -+ kbase_csf_scheduler_group_get_slot_locked(input_grp); -+ -+ /* check the input_grp is running and requesting protected mode -+ */ -+ if (slot >= 0 && -+ atomic_read(&scheduler->csg_slots[slot].state) == -+ CSG_SLOT_RUNNING) { -+ if (kctx_as_enabled(input_grp->kctx) && -+ scheduler_slot_protm_ack(kbdev, input_grp, slot)) { -+ /* Option of acknowledging to multiple -+ * CSGs from the same kctx is dropped, -+ * after consulting with the -+ * architecture team. See the comment in -+ * GPUCORE-21394. -+ */ -+ -+ /* Disable the idle timer */ -+ disable_gpu_idle_fw_timer_locked(kbdev); -+ -+ /* Switch to protected mode */ -+ scheduler->active_protm_grp = input_grp; -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_ENTER_PROTM, -+ input_grp, 0u); -+ -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ kbase_csf_enter_protected_mode(kbdev); -+ return; -+ } -+ } -+ } -+ -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+} -+ -+static void scheduler_apply(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ const u32 total_csg_slots = kbdev->csf.global_iface.group_num; -+ const u32 available_csg_slots = scheduler->num_csg_slots_for_tick; -+ u32 suspend_cnt = 0; -+ u32 remain_cnt = 0; -+ u32 resident_cnt = 0; -+ struct kbase_queue_group *group; -+ u32 i; -+ u32 spare; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ /* Suspend those resident groups not in the run list */ -+ for (i = 0; i < total_csg_slots; i++) { -+ group = scheduler->csg_slots[i].resident_group; -+ if (group) { -+ resident_cnt++; -+ if (group->prepared_seq_num >= available_csg_slots) { -+ suspend_queue_group(group); -+ suspend_cnt++; -+ } else -+ remain_cnt++; -+ } -+ } -+ -+ /* Initialize the remaining avialable csg slots for the tick/tock */ -+ scheduler->remaining_tick_slots = available_csg_slots; -+ -+ /* If there are spare slots, apply heads in the list */ -+ spare = (available_csg_slots > resident_cnt) ? -+ (available_csg_slots - resident_cnt) : 0; -+ while (!list_empty(&scheduler->groups_to_schedule)) { -+ group = list_first_entry(&scheduler->groups_to_schedule, -+ struct kbase_queue_group, -+ link_to_schedule); -+ -+ if (kbasep_csf_scheduler_group_is_on_slot_locked(group) && -+ group->prepared_seq_num < available_csg_slots) { -+ /* One of the resident remainders */ -+ update_csg_slot_priority(group, -+ get_slot_priority(group)); -+ } else if (spare != 0) { -+ s8 slot = (s8)find_first_zero_bit( -+ kbdev->csf.scheduler.csg_inuse_bitmap, -+ total_csg_slots); -+ -+ if (WARN_ON(slot >= (s8)total_csg_slots)) -+ break; -+ -+ if (!kctx_as_enabled(group->kctx) || group->faulted) { -+ /* Drop the head group and continue */ -+ update_offslot_non_idle_cnt_for_faulty_grp( -+ group); -+ remove_scheduled_group(kbdev, group); -+ continue; -+ } -+ program_csg_slot(group, slot, -+ get_slot_priority(group)); -+ if (unlikely(!csg_slot_in_use(kbdev, slot))) -+ break; -+ -+ spare--; -+ } else -+ break; -+ -+ /* Drop the head csg from the list */ -+ remove_scheduled_group(kbdev, group); -+ if (!WARN_ON(!scheduler->remaining_tick_slots)) -+ scheduler->remaining_tick_slots--; -+ } -+ -+ /* Dealing with groups currently going through suspend */ -+ program_suspending_csg_slots(kbdev); -+} -+ -+static void scheduler_ctx_scan_groups(struct kbase_device *kbdev, -+ struct kbase_context *kctx, int priority) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ struct kbase_queue_group *group; -+ -+ lockdep_assert_held(&scheduler->lock); -+ if (WARN_ON(priority < 0) || -+ WARN_ON(priority >= KBASE_QUEUE_GROUP_PRIORITY_COUNT)) -+ return; -+ -+ if (!kctx_as_enabled(kctx)) -+ return; -+ -+ list_for_each_entry(group, &kctx->csf.sched.runnable_groups[priority], -+ link) { -+ if (WARN_ON(!list_empty(&group->link_to_schedule))) -+ /* This would be a bug */ -+ list_del_init(&group->link_to_schedule); -+ -+ if (unlikely(group->faulted)) -+ continue; -+ -+ /* Set the scanout sequence number, starting from 0 */ -+ group->scan_seq_num = scheduler->csg_scan_count_for_tick++; -+ -+ if (queue_group_idle_locked(group)) { -+ list_add_tail(&group->link_to_schedule, -+ &scheduler->idle_groups_to_schedule); -+ continue; -+ } -+ -+ if (!scheduler->ngrp_to_schedule) { -+ /* keep the top csg's origin */ -+ scheduler->top_ctx = kctx; -+ scheduler->top_grp = group; -+ } -+ -+ list_add_tail(&group->link_to_schedule, -+ &scheduler->groups_to_schedule); -+ group->prepared_seq_num = scheduler->ngrp_to_schedule++; -+ -+ kctx->csf.sched.ngrp_to_schedule++; -+ count_active_address_space(kbdev, kctx); -+ } -+} -+ -+/** -+ * scheduler_rotate_groups() - Rotate the runnable queue groups to provide -+ * fairness of scheduling within a single -+ * kbase_context. -+ * -+ * Since only kbase_csf_scheduler's top_grp (i.e. the queue group assigned -+ * the highest slot priority) is guaranteed to get the resources that it -+ * needs we only rotate the kbase_context corresponding to it - -+ * kbase_csf_scheduler's top_ctx. -+ * -+ * The priority level chosen for rotation is the one containing the previous -+ * scheduling cycle's kbase_csf_scheduler's top_grp. -+ * -+ * In a 'fresh-slice-cycle' this always corresponds to the highest group -+ * priority in use by kbase_csf_scheduler's top_ctx. That is, it's the priority -+ * level of the previous scheduling cycle's first runnable kbase_context. -+ * -+ * We choose this priority level because when higher priority work is -+ * scheduled, we should always cause the scheduler to run and do a scan. The -+ * scan always enumerates the highest priority work first (whether that be -+ * based on process priority or group priority), and thus -+ * kbase_csf_scheduler's top_grp will point to the first of those high priority -+ * groups, which necessarily must be the highest priority group in -+ * kbase_csf_scheduler's top_ctx. The fresh-slice-cycle will run later and pick -+ * up that group appropriately. -+ * -+ * If kbase_csf_scheduler's top_grp was instead evicted (and thus is NULL), -+ * then no explicit rotation occurs on the next fresh-slice-cycle schedule, but -+ * will set up kbase_csf_scheduler's top_ctx again for the next scheduling -+ * cycle. Implicitly, a rotation had already occurred by removing -+ * the kbase_csf_scheduler's top_grp -+ * -+ * If kbase_csf_scheduler's top_grp became idle and all other groups belonging -+ * to kbase_csf_scheduler's top_grp's priority level in kbase_csf_scheduler's -+ * top_ctx are also idle, then the effect of this will be to rotate idle -+ * groups, which might not actually become resident in the next -+ * scheduling slice. However this is acceptable since a queue group becoming -+ * idle is implicitly a rotation (as above with evicted queue groups), as it -+ * automatically allows a new queue group to take the maximum slot priority -+ * whilst the idle kbase_csf_scheduler's top_grp ends up near the back of -+ * the kbase_csf_scheduler's groups_to_schedule list. In this example, it will -+ * be for a group in the next lowest priority level or in absence of those the -+ * next kbase_context's queue groups. -+ * -+ * @kbdev: Pointer to the GPU device. -+ */ -+static void scheduler_rotate_groups(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ struct kbase_context *const top_ctx = scheduler->top_ctx; -+ struct kbase_queue_group *const top_grp = scheduler->top_grp; -+ -+ lockdep_assert_held(&scheduler->lock); -+ if (top_ctx && top_grp) { -+ struct list_head *list = -+ &top_ctx->csf.sched.runnable_groups[top_grp->priority]; -+ -+ WARN_ON(top_grp->kctx != top_ctx); -+ if (!WARN_ON(list_empty(list))) { -+ struct kbase_queue_group *new_head_grp; -+ list_move_tail(&top_grp->link, list); -+ new_head_grp = (!list_empty(list)) ? -+ list_first_entry(list, struct kbase_queue_group, link) : -+ NULL; -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_ROTATE_RUNNABLE, -+ top_grp, top_ctx->csf.sched.num_runnable_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_HEAD_RUNNABLE, -+ new_head_grp, 0u); -+ dev_dbg(kbdev->dev, -+ "groups rotated for a context, num_runnable_groups: %u\n", -+ scheduler->top_ctx->csf.sched.num_runnable_grps); -+ } -+ } -+} -+ -+static void scheduler_rotate_ctxs(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ struct list_head *list = &scheduler->runnable_kctxs; -+ -+ lockdep_assert_held(&scheduler->lock); -+ if (scheduler->top_ctx) { -+ if (!WARN_ON(list_empty(list))) { -+ struct kbase_context *pos; -+ bool found = false; -+ -+ /* Locate the ctx on the list */ -+ list_for_each_entry(pos, list, csf.link) { -+ if (scheduler->top_ctx == pos) { -+ found = true; -+ break; -+ } -+ } -+ -+ if (!WARN_ON(!found)) { -+ struct kbase_context *new_head_kctx; -+ list_move_tail(&pos->csf.link, list); -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_ROTATE_RUNNABLE, pos, -+ 0u); -+ new_head_kctx = (!list_empty(list)) ? -+ list_first_entry(list, struct kbase_context, csf.link) : -+ NULL; -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_HEAD_RUNNABLE, -+ new_head_kctx, 0u); -+ dev_dbg(kbdev->dev, "contexts rotated\n"); -+ } -+ } -+ } -+} -+ -+/** -+ * scheduler_update_idle_slots_status() - Get the status update for the CSG -+ * slots for which the IDLE notification was received -+ * previously. -+ * -+ * This function sends a CSG status update request for all the CSG slots -+ * present in the bitmap scheduler->csg_slots_idle_mask and wait for the -+ * request to complete. -+ * The bits set in the scheduler->csg_slots_idle_mask bitmap are cleared by -+ * this function. -+ * -+ * @kbdev: Pointer to the GPU device. -+ * @csg_bitmap: Bitmap of the CSG slots for which -+ * the status update request completed successfully. -+ * @failed_csg_bitmap: Bitmap of the CSG slots for which -+ * the status update request timedout. -+ */ -+static void scheduler_update_idle_slots_status(struct kbase_device *kbdev, -+ unsigned long *csg_bitmap, unsigned long *failed_csg_bitmap) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ const u32 num_groups = kbdev->csf.global_iface.group_num; -+ struct kbase_csf_global_iface *const global_iface = -+ &kbdev->csf.global_iface; -+ unsigned long flags, i; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ for_each_set_bit(i, scheduler->csg_slots_idle_mask, num_groups) { -+ struct kbase_csf_csg_slot *csg_slot = &scheduler->csg_slots[i]; -+ struct kbase_queue_group *group = csg_slot->resident_group; -+ struct kbase_csf_cmd_stream_group_info *const ginfo = -+ &global_iface->groups[i]; -+ u32 csg_req; -+ -+ clear_bit(i, scheduler->csg_slots_idle_mask); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_IDLE_CLEAR, group, -+ scheduler->csg_slots_idle_mask[0]); -+ if (WARN_ON(!group)) -+ continue; -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_STATUS_UPDATE, group, -+ i); -+ -+ csg_req = kbase_csf_firmware_csg_output(ginfo, CSG_ACK); -+ csg_req ^= CSG_REQ_STATUS_UPDATE_MASK; -+ kbase_csf_firmware_csg_input_mask(ginfo, CSG_REQ, csg_req, -+ CSG_REQ_STATUS_UPDATE_MASK); -+ -+ set_bit(i, csg_bitmap); -+ } -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ -+ /* The groups are aggregated into a single kernel doorbell request */ -+ if (!bitmap_empty(csg_bitmap, num_groups)) { -+ long wt = -+ kbase_csf_timeout_in_jiffies(kbdev->csf.fw_timeout_ms); -+ u32 db_slots = (u32)csg_bitmap[0]; -+ -+ kbase_csf_ring_csg_slots_doorbell(kbdev, db_slots); -+ -+ if (wait_csg_slots_handshake_ack(kbdev, -+ CSG_REQ_STATUS_UPDATE_MASK, csg_bitmap, wt)) { -+ dev_warn( -+ kbdev->dev, -+ "Timeout on CSG_REQ:STATUS_UPDATE, treat groups as not idle: slot mask=0x%lx", -+ csg_bitmap[0]); -+ -+ /* Store the bitmap of timed out slots */ -+ bitmap_copy(failed_csg_bitmap, csg_bitmap, num_groups); -+ csg_bitmap[0] = ~csg_bitmap[0] & db_slots; -+ } else { -+ KBASE_KTRACE_ADD(kbdev, SLOTS_STATUS_UPDATE_ACK, NULL, -+ db_slots); -+ csg_bitmap[0] = db_slots; -+ } -+ } -+} -+ -+/** -+ * scheduler_handle_idle_slots() - Update the idle status of queue groups -+ * resident on CSG slots for which the -+ * IDLE notification was received previously. -+ * -+ * This function is called at the start of scheduling tick/tock to reconfirm -+ * the idle status of queue groups resident on CSG slots for -+ * which idle notification was received previously, i.e. all the CSG slots -+ * present in the bitmap scheduler->csg_slots_idle_mask. -+ * The confirmation is done by sending the CSG status update request to the -+ * firmware. On completion, the firmware will mark the idleness at the -+ * slot's interface CSG_STATUS_STATE register accordingly. -+ * -+ * The run state of the groups resident on still idle CSG slots is changed to -+ * KBASE_CSF_GROUP_IDLE and the bitmap scheduler->csg_slots_idle_mask is -+ * updated accordingly. -+ * The bits corresponding to slots for which the status update request timedout -+ * remain set in scheduler->csg_slots_idle_mask. -+ * -+ * @kbdev: Pointer to the GPU device. -+ */ -+static void scheduler_handle_idle_slots(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ unsigned long flags, i; -+ DECLARE_BITMAP(csg_bitmap, MAX_SUPPORTED_CSGS) = { 0 }; -+ DECLARE_BITMAP(failed_csg_bitmap, MAX_SUPPORTED_CSGS) = { 0 }; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ scheduler_update_idle_slots_status(kbdev, csg_bitmap, -+ failed_csg_bitmap); -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ for_each_set_bit(i, csg_bitmap, num_groups) { -+ struct kbase_csf_csg_slot *csg_slot = &scheduler->csg_slots[i]; -+ struct kbase_queue_group *group = csg_slot->resident_group; -+ -+ if (WARN_ON(atomic_read(&csg_slot->state) != CSG_SLOT_RUNNING)) -+ continue; -+ if (WARN_ON(!group)) -+ continue; -+ if (WARN_ON(group->run_state != KBASE_CSF_GROUP_RUNNABLE && -+ group->run_state != KBASE_CSF_GROUP_IDLE)) -+ continue; -+ if (WARN_ON(group->priority >= KBASE_QUEUE_GROUP_PRIORITY_COUNT)) -+ continue; -+ -+ if (group_on_slot_is_idle(kbdev, i)) { -+ group->run_state = KBASE_CSF_GROUP_IDLE; -+ set_bit(i, scheduler->csg_slots_idle_mask); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_IDLE_SET, -+ group, scheduler->csg_slots_idle_mask[0]); -+ } else -+ group->run_state = KBASE_CSF_GROUP_RUNNABLE; -+ } -+ -+ bitmap_or(scheduler->csg_slots_idle_mask, -+ scheduler->csg_slots_idle_mask, -+ failed_csg_bitmap, num_groups); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, CSG_SLOT_IDLE_SET, NULL, -+ scheduler->csg_slots_idle_mask[0]); -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+} -+ -+static void scheduler_scan_idle_groups(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ struct kbase_queue_group *group, *n; -+ -+ list_for_each_entry_safe(group, n, &scheduler->idle_groups_to_schedule, -+ link_to_schedule) { -+ -+ WARN_ON(!queue_group_idle_locked(group)); -+ -+ if (!scheduler->ngrp_to_schedule) { -+ /* keep the top csg's origin */ -+ scheduler->top_ctx = group->kctx; -+ scheduler->top_grp = group; -+ } -+ -+ group->prepared_seq_num = scheduler->ngrp_to_schedule++; -+ list_move_tail(&group->link_to_schedule, -+ &scheduler->groups_to_schedule); -+ -+ group->kctx->csf.sched.ngrp_to_schedule++; -+ count_active_address_space(kbdev, group->kctx); -+ } -+} -+ -+static void scheduler_rotate(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ /* Dealing with rotation */ -+ scheduler_rotate_groups(kbdev); -+ scheduler_rotate_ctxs(kbdev); -+} -+ -+static struct kbase_queue_group *get_tock_top_group( -+ struct kbase_csf_scheduler *const scheduler) -+{ -+ struct kbase_context *kctx; -+ int i; -+ -+ lockdep_assert_held(&scheduler->lock); -+ for (i = 0; i < KBASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { -+ list_for_each_entry(kctx, -+ &scheduler->runnable_kctxs, csf.link) { -+ struct kbase_queue_group *group; -+ -+ list_for_each_entry(group, -+ &kctx->csf.sched.runnable_groups[i], -+ link) { -+ if (queue_group_idle_locked(group)) -+ continue; -+ -+ return group; -+ } -+ } -+ } -+ -+ return NULL; -+} -+ -+static int suspend_active_groups_on_powerdown(struct kbase_device *kbdev, -+ bool is_suspend) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; -+ -+ int ret = suspend_active_queue_groups(kbdev, slot_mask); -+ -+ if (ret) { -+ /* The suspend of CSGs failed, trigger the GPU reset and wait -+ * for it to complete to be in a deterministic state. -+ */ -+ dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend on power down, slot_mask: 0x%*pb\n", -+ kbdev->csf.global_iface.group_num, slot_mask); -+ -+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) -+ kbase_reset_gpu(kbdev); -+ -+ if (is_suspend) { -+ mutex_unlock(&scheduler->lock); -+ kbase_reset_gpu_wait(kbdev); -+ mutex_lock(&scheduler->lock); -+ } -+ return -1; -+ } -+ -+ /* Check if the groups became active whilst the suspend was ongoing, -+ * but only for the case where the system suspend is not in progress -+ */ -+ if (!is_suspend && atomic_read(&scheduler->non_idle_offslot_grps)) -+ return -1; -+ -+ return 0; -+} -+ -+static bool scheduler_idle_suspendable(struct kbase_device *kbdev) -+{ -+ bool suspend; -+ unsigned long flags; -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (scheduler->state == SCHED_SUSPENDED) -+ return false; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ if (scheduler->total_runnable_grps) { -+ spin_lock(&scheduler->interrupt_lock); -+ -+ /* Check both on-slots and off-slots groups idle status */ -+ suspend = kbase_csf_scheduler_all_csgs_idle(kbdev) && -+ !atomic_read(&scheduler->non_idle_offslot_grps) && -+ kbase_pm_idle_groups_sched_suspendable(kbdev); -+ -+ spin_unlock(&scheduler->interrupt_lock); -+ } else -+ suspend = kbase_pm_no_runnables_sched_suspendable(kbdev); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return suspend; -+} -+ -+static void gpu_idle_worker(struct work_struct *work) -+{ -+ struct kbase_device *kbdev = container_of( -+ work, struct kbase_device, csf.scheduler.gpu_idle_work); -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ bool reset_active = false; -+ bool scheduler_is_idle_suspendable = false; -+ bool all_groups_suspended = false; -+ -+ KBASE_KTRACE_ADD(kbdev, IDLE_WORKER_BEGIN, NULL, 0u); -+ -+#define __ENCODE_KTRACE_INFO(reset, idle, all_suspend) \ -+ (((u32)reset) | (((u32)idle) << 4) | (((u32)all_suspend) << 8)) -+ -+ if (kbase_reset_gpu_try_prevent(kbdev)) { -+ dev_warn(kbdev->dev, "Quit idle for failing to prevent gpu reset.\n"); -+ KBASE_KTRACE_ADD(kbdev, IDLE_WORKER_END, NULL, -+ __ENCODE_KTRACE_INFO(true, false, false)); -+ return; -+ } -+ mutex_lock(&scheduler->lock); -+ -+ /* Cycle completed, disable the firmware idle timer */ -+ disable_gpu_idle_fw_timer(kbdev); -+ scheduler_is_idle_suspendable = scheduler_idle_suspendable(kbdev); -+ reset_active = kbase_reset_gpu_is_active(kbdev); -+ if (scheduler_is_idle_suspendable && !reset_active) { -+ all_groups_suspended = -+ !suspend_active_groups_on_powerdown(kbdev, false); -+ -+ if (all_groups_suspended) { -+ dev_dbg(kbdev->dev, "Scheduler becomes idle suspended now"); -+ scheduler_suspend(kbdev); -+ cancel_tick_timer(kbdev); -+ } else { -+ dev_dbg(kbdev->dev, "Aborting suspend scheduler (grps: %d)", -+ atomic_read(&scheduler->non_idle_offslot_grps)); -+ /* Bring forward the next tick */ -+ kbase_csf_scheduler_advance_tick(kbdev); -+ } -+ } -+ -+ mutex_unlock(&scheduler->lock); -+ kbase_reset_gpu_allow(kbdev); -+ KBASE_KTRACE_ADD(kbdev, IDLE_WORKER_END, NULL, -+ __ENCODE_KTRACE_INFO(reset_active, scheduler_is_idle_suspendable, all_groups_suspended)); -+#undef __ENCODE_KTRACE_INFO -+} -+ -+static int scheduler_prepare(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ int i; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ /* Empty the groups_to_schedule */ -+ while (!list_empty(&scheduler->groups_to_schedule)) { -+ struct kbase_queue_group *grp = -+ list_first_entry(&scheduler->groups_to_schedule, -+ struct kbase_queue_group, -+ link_to_schedule); -+ -+ remove_scheduled_group(kbdev, grp); -+ } -+ -+ /* Pre-scan init scheduler fields */ -+ if (WARN_ON(scheduler->ngrp_to_schedule != 0)) -+ scheduler->ngrp_to_schedule = 0; -+ scheduler->top_ctx = NULL; -+ scheduler->top_grp = NULL; -+ scheduler->csg_scan_count_for_tick = 0; -+ WARN_ON(!list_empty(&scheduler->idle_groups_to_schedule)); -+ scheduler->num_active_address_spaces = 0; -+ scheduler->num_csg_slots_for_tick = 0; -+ bitmap_zero(scheduler->csg_slots_prio_update, MAX_SUPPORTED_CSGS); -+ -+ /* Scan out to run groups */ -+ for (i = 0; i < KBASE_QUEUE_GROUP_PRIORITY_COUNT; ++i) { -+ struct kbase_context *kctx; -+ -+ list_for_each_entry(kctx, &scheduler->runnable_kctxs, csf.link) -+ scheduler_ctx_scan_groups(kbdev, kctx, i); -+ } -+ -+ /* Update this tick's non-idle groups */ -+ scheduler->non_idle_scanout_grps = scheduler->ngrp_to_schedule; -+ -+ /* Initial number of non-idle off-slot groups, before the scheduler's -+ * scheduler_apply() operation. This gives a sensible start point view -+ * of the tick. It will be subject to up/downs during the scheduler -+ * active phase. -+ */ -+ atomic_set(&scheduler->non_idle_offslot_grps, -+ scheduler->non_idle_scanout_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_INC, NULL, -+ scheduler->non_idle_scanout_grps); -+ -+ /* Adds those idle but runnable groups to the scanout list */ -+ scheduler_scan_idle_groups(kbdev); -+ -+ /* After adding the idle CSGs, the two counts should be the same */ -+ WARN_ON(scheduler->csg_scan_count_for_tick != scheduler->ngrp_to_schedule); -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, -+ scheduler->num_active_address_spaces | -+ (((u64)scheduler->ngrp_to_schedule) << 32)); -+ set_max_csg_slots(kbdev); -+ dev_dbg(kbdev->dev, "prepared groups length: %u, num_active_address_spaces: %u\n", -+ scheduler->ngrp_to_schedule, scheduler->num_active_address_spaces); -+ return 0; -+} -+ -+static void scheduler_handle_idle_timer_onoff(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ /* After the scheduler apply operation, the internal variable -+ * scheduler->non_idle_offslot_grps reflects the end-point view -+ * of the count at the end of the active phase. -+ * -+ * Any changes that follow (after the scheduler has dropped the -+ * scheduler->lock), reflects async operations to the scheduler, -+ * such as a group gets killed (evicted) or a new group inserted, -+ * cqs wait-sync triggered state transtion etc. -+ * -+ * The condition for enable the idle timer is that there is no -+ * non-idle groups off-slots. If there is non-idle group off-slot, -+ * the timer should be disabled. -+ */ -+ if (atomic_read(&scheduler->non_idle_offslot_grps)) -+ disable_gpu_idle_fw_timer(kbdev); -+ else -+ enable_gpu_idle_fw_timer(kbdev); -+} -+ -+static void schedule_actions(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ struct kbase_queue_group *protm_grp; -+ int ret; -+ bool skip_idle_slots_update; -+ bool new_protm_top_grp = false; -+ -+ kbase_reset_gpu_assert_prevented(kbdev); -+ lockdep_assert_held(&scheduler->lock); -+ -+ ret = kbase_pm_wait_for_desired_state(kbdev); -+ if (ret) { -+ dev_err(kbdev->dev, "Wait for MCU power on failed"); -+ return; -+ } -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ skip_idle_slots_update = kbase_csf_scheduler_protected_mode_in_use(kbdev); -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ -+ /* Skip updating on-slot idle CSGs if GPU is in protected mode. */ -+ if (!skip_idle_slots_update) -+ scheduler_handle_idle_slots(kbdev); -+ -+ scheduler_prepare(kbdev); -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ protm_grp = scheduler->active_protm_grp; -+ -+ /* Avoid update if the top-group remains unchanged and in protected -+ * mode. For the said case, all the slots update is effectively -+ * competing against the active protected mode group (typically the -+ * top-group). If we update other slots, even on leaving the -+ * top-group slot untouched, the firmware would exit the protected mode -+ * for interacting with the host-driver. After it, as the top-group -+ * would again raise the request for entering protected mode, we would -+ * be actively doing the switching over twice without progressing the -+ * queue jobs. -+ */ -+ if (protm_grp && scheduler->top_grp == protm_grp) { -+ int new_val; -+ dev_dbg(kbdev->dev, "Scheduler keep protm exec: group-%d", -+ protm_grp->handle); -+ new_val = atomic_dec_return(&scheduler->non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_DEC, -+ protm_grp, new_val); -+ } else if (scheduler->top_grp) { -+ if (protm_grp) -+ dev_dbg(kbdev->dev, "Scheduler drop protm exec: group-%d", -+ protm_grp->handle); -+ -+ if (!bitmap_empty(scheduler->top_grp->protm_pending_bitmap, -+ kbdev->csf.global_iface.groups[0].stream_num)) { -+ dev_dbg(kbdev->dev, "Scheduler prepare protm exec: group-%d of context %d_%d", -+ scheduler->top_grp->handle, -+ scheduler->top_grp->kctx->tgid, -+ scheduler->top_grp->kctx->id); -+ -+ /* When entering protected mode all CSG slots can be occupied -+ * but only the protected mode CSG will be running. Any event -+ * that would trigger the execution of an on-slot idle CSG will -+ * need to be handled by the host during protected mode. -+ */ -+ new_protm_top_grp = true; -+ } -+ -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ -+ scheduler_apply(kbdev); -+ -+ /* Post-apply, all the committed groups in this tick are on -+ * slots, time to arrange the idle timer on/off decision. -+ */ -+ scheduler_handle_idle_timer_onoff(kbdev); -+ -+ /* Scheduler is dropping the exec of the previous protm_grp, -+ * Until the protm quit completes, the GPU is effectively -+ * locked in the secure mode. -+ */ -+ if (protm_grp) -+ scheduler_force_protm_exit(kbdev); -+ -+ wait_csg_slots_start(kbdev); -+ wait_csg_slots_finish_prio_update(kbdev); -+ -+ if (new_protm_top_grp) { -+ scheduler_group_check_protm_enter(kbdev, -+ scheduler->top_grp); -+ } -+ -+ return; -+ } -+ -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ return; -+} -+ -+static void schedule_on_tock(struct work_struct *work) -+{ -+ struct kbase_device *kbdev = container_of(work, struct kbase_device, -+ csf.scheduler.tock_work.work); -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ int err = kbase_reset_gpu_try_prevent(kbdev); -+ /* Regardless of whether reset failed or is currently happening, exit -+ * early -+ */ -+ if (err) -+ return; -+ -+ mutex_lock(&scheduler->lock); -+ if (scheduler->state == SCHED_SUSPENDED) -+ goto exit_no_schedule_unlock; -+ -+ WARN_ON(!(scheduler->state == SCHED_INACTIVE)); -+ scheduler->state = SCHED_BUSY; -+ -+ /* Undertaking schedule action steps */ -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_TOCK, NULL, 0u); -+ schedule_actions(kbdev); -+ -+ /* Record time information */ -+ scheduler->last_schedule = jiffies; -+ -+ /* Tock is serviced */ -+ scheduler->tock_pending_request = false; -+ -+ scheduler->state = SCHED_INACTIVE; -+ mutex_unlock(&scheduler->lock); -+ kbase_reset_gpu_allow(kbdev); -+ -+ dev_dbg(kbdev->dev, -+ "Waking up for event after schedule-on-tock completes."); -+ wake_up_all(&kbdev->csf.event_wait); -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_TOCK_END, NULL, 0u); -+ return; -+ -+exit_no_schedule_unlock: -+ mutex_unlock(&scheduler->lock); -+ kbase_reset_gpu_allow(kbdev); -+} -+ -+static void schedule_on_tick(struct work_struct *work) -+{ -+ struct kbase_device *kbdev = container_of(work, struct kbase_device, -+ csf.scheduler.tick_work); -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ int err = kbase_reset_gpu_try_prevent(kbdev); -+ /* Regardless of whether reset failed or is currently happening, exit -+ * early -+ */ -+ if (err) -+ return; -+ -+ mutex_lock(&scheduler->lock); -+ -+ WARN_ON(scheduler->tick_timer_active); -+ if (scheduler->state == SCHED_SUSPENDED) -+ goto exit_no_schedule_unlock; -+ -+ scheduler->state = SCHED_BUSY; -+ /* Do scheduling stuff */ -+ scheduler_rotate(kbdev); -+ -+ /* Undertaking schedule action steps */ -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_TICK, NULL, -+ scheduler->total_runnable_grps); -+ schedule_actions(kbdev); -+ -+ /* Record time information */ -+ scheduler->last_schedule = jiffies; -+ -+ /* Kicking next scheduling if needed */ -+ if (likely(scheduler_timer_is_enabled_nolock(kbdev)) && -+ (scheduler->total_runnable_grps > 0)) { -+ start_tick_timer(kbdev); -+ dev_dbg(kbdev->dev, -+ "scheduling for next tick, num_runnable_groups:%u\n", -+ scheduler->total_runnable_grps); -+ } -+ -+ scheduler->state = SCHED_INACTIVE; -+ mutex_unlock(&scheduler->lock); -+ kbase_reset_gpu_allow(kbdev); -+ -+ dev_dbg(kbdev->dev, "Waking up for event after schedule-on-tick completes."); -+ wake_up_all(&kbdev->csf.event_wait); -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_TICK_END, NULL, -+ scheduler->total_runnable_grps); -+ return; -+ -+exit_no_schedule_unlock: -+ mutex_unlock(&scheduler->lock); -+ kbase_reset_gpu_allow(kbdev); -+} -+ -+static int wait_csg_slots_suspend(struct kbase_device *kbdev, -+ const unsigned long *slot_mask, -+ unsigned int timeout_ms) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ long remaining = kbase_csf_timeout_in_jiffies(timeout_ms); -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ int err = 0; -+ DECLARE_BITMAP(slot_mask_local, MAX_SUPPORTED_CSGS); -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ bitmap_copy(slot_mask_local, slot_mask, MAX_SUPPORTED_CSGS); -+ -+ while (!bitmap_empty(slot_mask_local, MAX_SUPPORTED_CSGS) -+ && remaining) { -+ DECLARE_BITMAP(changed, MAX_SUPPORTED_CSGS); -+ -+ bitmap_copy(changed, slot_mask_local, MAX_SUPPORTED_CSGS); -+ -+ remaining = wait_event_timeout(kbdev->csf.event_wait, -+ slots_state_changed(kbdev, changed, -+ csg_slot_stopped_locked), -+ remaining); -+ -+ if (remaining) { -+ u32 i; -+ -+ for_each_set_bit(i, changed, num_groups) { -+ struct kbase_queue_group *group; -+ -+ if (WARN_ON(!csg_slot_stopped_locked(kbdev, (s8)i))) -+ continue; -+ -+ /* The on slot csg is now stopped */ -+ clear_bit(i, slot_mask_local); -+ -+ group = scheduler->csg_slots[i].resident_group; -+ if (likely(group)) { -+ /* Only do save/cleanup if the -+ * group is not terminated during -+ * the sleep. -+ */ -+ save_csg_slot(group); -+ if (cleanup_csg_slot(group)) -+ sched_evict_group(group, true, true); -+ } -+ } -+ } else { -+ dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend, slot_mask: 0x%*pb\n", -+ num_groups, slot_mask_local); -+ err = -ETIMEDOUT; -+ } -+ } -+ -+ return err; -+} -+ -+static int suspend_active_queue_groups(struct kbase_device *kbdev, -+ unsigned long *slot_mask) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ u32 slot_num; -+ int ret; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ for (slot_num = 0; slot_num < num_groups; slot_num++) { -+ struct kbase_queue_group *group = -+ scheduler->csg_slots[slot_num].resident_group; -+ -+ if (group) { -+ suspend_queue_group(group); -+ set_bit(slot_num, slot_mask); -+ } -+ } -+ -+ ret = wait_csg_slots_suspend(kbdev, slot_mask, kbdev->reset_timeout_ms); -+ return ret; -+} -+ -+static int suspend_active_queue_groups_on_reset(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = { 0 }; -+ int ret; -+ int ret2; -+ -+ mutex_lock(&scheduler->lock); -+ -+ ret = suspend_active_queue_groups(kbdev, slot_mask); -+ -+ if (ret) { -+ dev_warn(kbdev->dev, "Timed out waiting for CSG slots to suspend before reset, slot_mask: 0x%*pb\n", -+ kbdev->csf.global_iface.group_num, slot_mask); -+ } -+ -+ /* Need to flush the GPU cache to ensure suspend buffer -+ * contents are not lost on reset of GPU. -+ * Do this even if suspend operation had timed out for some of -+ * the CSG slots. -+ * In case the scheduler already in suspended state, the -+ * cache clean is required as the async reset request from -+ * the debugfs may race against the scheduler suspend operation -+ * due to the extra context ref-count, which prevents the -+ * L2 powering down cache clean operation in the non racing -+ * case. -+ */ -+ kbase_gpu_start_cache_clean(kbdev); -+ ret2 = kbase_gpu_wait_cache_clean_timeout(kbdev, -+ kbdev->reset_timeout_ms); -+ if (ret2) { -+ dev_warn(kbdev->dev, "Timed out waiting for cache clean to complete before reset"); -+ if (!ret) -+ ret = ret2; -+ } -+ -+ mutex_unlock(&scheduler->lock); -+ -+ return ret; -+} -+ -+/** -+ * scheduler_handle_reset_in_protected_mode() - Update the state of normal mode -+ * groups when reset is done during -+ * protected mode execution. -+ * -+ * @group: Pointer to the device. -+ * -+ * This function is called at the time of GPU reset, before the suspension of -+ * queue groups, to handle the case when the reset is getting performed whilst -+ * GPU is in protected mode. -+ * On entry to protected mode all the groups, except the top group that executes -+ * in protected mode, are implicitly suspended by the FW. Thus this function -+ * simply marks the normal mode groups as suspended (and cleans up the -+ * corresponding CSG slots) to prevent their potential forceful eviction from -+ * the Scheduler. So if GPU was in protected mode and there was no fault, then -+ * only the protected mode group would be suspended in the regular way post exit -+ * from this function. And if GPU was in normal mode, then all on-slot groups -+ * will get suspended in the regular way. -+ * -+ * Return: true if the groups remaining on the CSG slots need to be suspended in -+ * the regular way by sending CSG SUSPEND reqs to FW, otherwise false. -+ */ -+static bool scheduler_handle_reset_in_protected_mode(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ u32 const num_groups = kbdev->csf.global_iface.group_num; -+ struct kbase_queue_group *protm_grp; -+ bool suspend_on_slot_groups; -+ unsigned long flags; -+ u32 csg_nr; -+ -+ mutex_lock(&scheduler->lock); -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ protm_grp = scheduler->active_protm_grp; -+ -+ /* If GPU wasn't in protected mode or had exited it before the GPU reset -+ * then all the on-slot groups can be suspended in the regular way by -+ * sending CSG SUSPEND requests to FW. -+ * If there wasn't a fault for protected mode group, then it would -+ * also need to be suspended in the regular way before the reset. -+ */ -+ suspend_on_slot_groups = !(protm_grp && protm_grp->faulted); -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ -+ if (!protm_grp) -+ goto unlock; -+ -+ /* GPU is in protected mode, so all the on-slot groups barring the -+ * the protected mode group can be marked as suspended right away. -+ */ -+ for (csg_nr = 0; csg_nr < num_groups; csg_nr++) { -+ struct kbase_queue_group *const group = -+ kbdev->csf.scheduler.csg_slots[csg_nr].resident_group; -+ int new_val; -+ -+ if (!group || (group == protm_grp)) -+ continue; -+ -+ cleanup_csg_slot(group); -+ group->run_state = KBASE_CSF_GROUP_SUSPENDED; -+ -+ /* Simply treat the normal mode groups as non-idle. The tick -+ * scheduled after the reset will re-initialize the counter -+ * anyways. -+ */ -+ new_val = atomic_inc_return(&scheduler->non_idle_offslot_grps); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_NONIDLE_OFFSLOT_INC, -+ group, new_val); -+ } -+ -+unlock: -+ mutex_unlock(&scheduler->lock); -+ return suspend_on_slot_groups; -+} -+ -+static void scheduler_inner_reset(struct kbase_device *kbdev) -+{ -+ u32 const num_groups = kbdev->csf.global_iface.group_num; -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ -+ WARN_ON(csgs_active(kbdev)); -+ -+ /* Cancel any potential queued delayed work(s) */ -+ cancel_work_sync(&kbdev->csf.scheduler.gpu_idle_work); -+ cancel_tick_timer(kbdev); -+ cancel_work_sync(&scheduler->tick_work); -+ cancel_delayed_work_sync(&scheduler->tock_work); -+ cancel_delayed_work_sync(&scheduler->ping_work); -+ -+ mutex_lock(&scheduler->lock); -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ bitmap_fill(scheduler->csgs_events_enable_mask, MAX_SUPPORTED_CSGS); -+ if (scheduler->active_protm_grp) -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_EXIT_PROTM, -+ scheduler->active_protm_grp, 0u); -+ scheduler->active_protm_grp = NULL; -+ memset(kbdev->csf.scheduler.csg_slots, 0, -+ num_groups * sizeof(struct kbase_csf_csg_slot)); -+ bitmap_zero(kbdev->csf.scheduler.csg_inuse_bitmap, num_groups); -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ -+ scheduler->top_ctx = NULL; -+ scheduler->top_grp = NULL; -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, SCHEDULER_TOP_GRP, scheduler->top_grp, -+ scheduler->num_active_address_spaces | -+ (((u64)scheduler->total_runnable_grps) << 32)); -+ -+ mutex_unlock(&scheduler->lock); -+} -+ -+void kbase_csf_scheduler_reset(struct kbase_device *kbdev) -+{ -+ struct kbase_context *kctx; -+ -+ WARN_ON(!kbase_reset_gpu_is_active(kbdev)); -+ -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_RESET, NULL, 0u); -+ -+ if (scheduler_handle_reset_in_protected_mode(kbdev) && -+ !suspend_active_queue_groups_on_reset(kbdev)) { -+ /* As all groups have been successfully evicted from the CSG -+ * slots, clear out thee scheduler data fields and return -+ */ -+ scheduler_inner_reset(kbdev); -+ return; -+ } -+ -+ mutex_lock(&kbdev->kctx_list_lock); -+ -+ /* The loop to iterate over the kbase contexts is present due to lock -+ * ordering issue between kctx->csf.lock & kbdev->csf.scheduler.lock. -+ * CSF ioctls first take kctx->csf.lock which is context-specific and -+ * then take kbdev->csf.scheduler.lock for global actions like assigning -+ * a CSG slot. -+ * If the lock ordering constraint was not there then could have -+ * directly looped over the active queue groups. -+ */ -+ list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { -+ /* Firmware reload would reinitialize the CSG & CS interface IO -+ * pages, so just need to internally mark the currently active -+ * queue groups as terminated (similar to the unexpected OoM -+ * event case). -+ * No further work can now get executed for the active groups -+ * (new groups would have to be created to execute work) and -+ * in near future Clients would be duly informed of this -+ * reset. The resources (like User IO pages, GPU queue memory) -+ * allocated for the associated queues would be freed when the -+ * Clients do the teardown when they become aware of the reset. -+ */ -+ kbase_csf_active_queue_groups_reset(kbdev, kctx); -+ } -+ -+ mutex_unlock(&kbdev->kctx_list_lock); -+ -+ /* After queue groups reset, the scheduler data fields clear out */ -+ scheduler_inner_reset(kbdev); -+} -+ -+static void firmware_aliveness_monitor(struct work_struct *work) -+{ -+ struct kbase_device *kbdev = container_of(work, struct kbase_device, -+ csf.scheduler.ping_work.work); -+ int err; -+ -+ /* Ensure that reset will not be occurring while this function is being -+ * executed as otherwise calling kbase_reset_gpu when reset is already -+ * occurring is a programming error. -+ * -+ * We must use the 'try' variant as the Reset worker can try to flush -+ * this workqueue, which would otherwise deadlock here if we tried to -+ * wait for the reset (and thus ourselves) to complete. -+ */ -+ err = kbase_reset_gpu_try_prevent(kbdev); -+ if (err) { -+ /* It doesn't matter whether the value was -EAGAIN or a fatal -+ * error, just stop processing. In case of -EAGAIN, the Reset -+ * worker will restart the scheduler later to resume ping -+ */ -+ return; -+ } -+ -+ mutex_lock(&kbdev->csf.scheduler.lock); -+ -+#ifdef CONFIG_MALI_DEBUG -+ if (fw_debug) { -+ /* ping requests cause distraction in firmware debugging */ -+ goto exit; -+ } -+#endif -+ -+ if (kbdev->csf.scheduler.state == SCHED_SUSPENDED) -+ goto exit; -+ -+ if (get_nr_active_csgs(kbdev) != 1) -+ goto exit; -+ -+ if (kbase_csf_scheduler_protected_mode_in_use(kbdev)) -+ goto exit; -+ -+ if (kbase_pm_context_active_handle_suspend(kbdev, -+ KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE)) { -+ /* Suspend pending - no real need to ping */ -+ goto exit; -+ } -+ -+ kbase_pm_wait_for_desired_state(kbdev); -+ -+ err = kbase_csf_firmware_ping_wait(kbdev); -+ -+ if (err) { -+ /* It is acceptable to enqueue a reset whilst we've prevented -+ * them, it will happen after we've allowed them again -+ */ -+ if (kbase_prepare_to_reset_gpu( -+ kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) -+ kbase_reset_gpu(kbdev); -+ } else if (get_nr_active_csgs(kbdev) == 1) { -+ queue_delayed_work(system_long_wq, -+ &kbdev->csf.scheduler.ping_work, -+ msecs_to_jiffies(FIRMWARE_PING_INTERVAL_MS)); -+ } -+ -+ kbase_pm_context_idle(kbdev); -+exit: -+ mutex_unlock(&kbdev->csf.scheduler.lock); -+ kbase_reset_gpu_allow(kbdev); -+ return; -+} -+ -+int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, -+ struct kbase_suspend_copy_buffer *sus_buf) -+{ -+ struct kbase_context *const kctx = group->kctx; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ int err = 0; -+ -+ kbase_reset_gpu_assert_prevented(kbdev); -+ lockdep_assert_held(&kctx->csf.lock); -+ mutex_lock(&scheduler->lock); -+ -+ if (kbasep_csf_scheduler_group_is_on_slot_locked(group)) { -+ DECLARE_BITMAP(slot_mask, MAX_SUPPORTED_CSGS) = {0}; -+ -+ set_bit(kbase_csf_scheduler_group_get_slot(group), slot_mask); -+ -+ if (!WARN_ON(scheduler->state == SCHED_SUSPENDED)) -+ suspend_queue_group(group); -+ err = wait_csg_slots_suspend(kbdev, slot_mask, -+ kbdev->csf.fw_timeout_ms); -+ if (err) { -+ dev_warn(kbdev->dev, "Timed out waiting for the group %d to suspend on slot %d", -+ group->handle, group->csg_nr); -+ goto exit; -+ } -+ } -+ -+ if (queue_group_suspended_locked(group)) { -+ unsigned int target_page_nr = 0, i = 0; -+ u64 offset = sus_buf->offset; -+ size_t to_copy = sus_buf->size; -+ -+ if (scheduler->state != SCHED_SUSPENDED) { -+ /* Similar to the case of HW counters, need to flush -+ * the GPU cache before reading from the suspend buffer -+ * pages as they are mapped and cached on GPU side. -+ */ -+ kbase_gpu_start_cache_clean(kbdev); -+ kbase_gpu_wait_cache_clean(kbdev); -+ } else { -+ /* Make sure power down transitions have completed, -+ * i.e. L2 has been powered off as that would ensure -+ * its contents are flushed to memory. -+ * This is needed as Scheduler doesn't wait for the -+ * power down to finish. -+ */ -+ kbase_pm_wait_for_desired_state(kbdev); -+ } -+ -+ for (i = 0; i < PFN_UP(sus_buf->size) && -+ target_page_nr < sus_buf->nr_pages; i++) { -+ struct page *pg = -+ as_page(group->normal_suspend_buf.phy[i]); -+ void *sus_page = kmap(pg); -+ -+ if (sus_page) { -+ kbase_sync_single_for_cpu(kbdev, -+ kbase_dma_addr(pg), -+ PAGE_SIZE, DMA_BIDIRECTIONAL); -+ -+ err = kbase_mem_copy_to_pinned_user_pages( -+ sus_buf->pages, sus_page, -+ &to_copy, sus_buf->nr_pages, -+ &target_page_nr, offset); -+ kunmap(pg); -+ if (err) -+ break; -+ } else { -+ err = -ENOMEM; -+ break; -+ } -+ } -+ schedule_in_cycle(group, false); -+ } else { -+ /* If addr-space fault, the group may have been evicted */ -+ err = -EIO; -+ } -+ -+exit: -+ mutex_unlock(&scheduler->lock); -+ return err; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_csf_scheduler_group_copy_suspend_buf); -+ -+/** -+ * group_sync_updated() - Evaluate sync wait condition of all blocked command -+ * queues of the group. -+ * -+ * @group: Pointer to the command queue group that has blocked command queue(s) -+ * bound to it. -+ * -+ * Return: true if sync wait condition is satisfied for at least one blocked -+ * queue of the group. -+ */ -+static bool group_sync_updated(struct kbase_queue_group *group) -+{ -+ bool updated = false; -+ int stream; -+ -+ /* Groups can also be blocked on-slot during protected mode. */ -+ WARN_ON(group->run_state != KBASE_CSF_GROUP_SUSPENDED_ON_WAIT_SYNC && -+ group->run_state != KBASE_CSF_GROUP_IDLE); -+ -+ for (stream = 0; stream < MAX_SUPPORTED_STREAMS_PER_GROUP; ++stream) { -+ struct kbase_queue *const queue = group->bound_queues[stream]; -+ -+ /* To check the necessity of sync-wait evaluation, -+ * we rely on the cached 'status_wait' instead of reading it -+ * directly from shared memory as the CSG has been already -+ * evicted from the CSG slot, thus this CSG doesn't have -+ * valid information in the shared memory. -+ */ -+ if (queue && queue->enabled && -+ CS_STATUS_WAIT_SYNC_WAIT_GET(queue->status_wait)) -+ if (evaluate_sync_update(queue)) { -+ updated = true; -+ queue->status_wait = 0; -+ } -+ } -+ -+ return updated; -+} -+ -+/** -+ * scheduler_get_protm_enter_async_group() - Check if the GPU queue group -+ * can be now allowed to execute in protected mode. -+ * -+ * @kbdev: Pointer to the GPU device. -+ * @group: Pointer to the GPU queue group. -+ * -+ * This function is called outside the scheduling tick/tock to determine -+ * if the given GPU queue group can now execute in protected mode or not. -+ * If the group pointer passed is NULL then the evaluation is done for the -+ * highest priority group on the scheduler maintained group lists without -+ * tick associated rotation actions. This is referred as the 'top-group' -+ * in a tock action sense. -+ * -+ * It returns the same group pointer, that was passed as an argument, if that -+ * group matches the highest priority group and has pending protected region -+ * requests otherwise NULL is returned. -+ * -+ * If the group pointer passed is NULL then the internal evaluated highest -+ * priority group is returned if that has pending protected region requests -+ * otherwise NULL is returned. -+ * -+ * The evaluated highest priority group may not necessarily be the same as the -+ * scheduler->top_grp. This can happen if there is dynamic de-idle update -+ * during the tick interval for some on-slots groups that were idle during the -+ * scheduler normal scheduling action, where the scheduler->top_grp was set. -+ * The recorded scheduler->top_grp is untouched by this evualuation, so will not -+ * affect the scheduler context/priority list rotation arrangement. -+ * -+ * Return: the pointer to queue group that can currently execute in protected -+ * mode or NULL. -+ */ -+static struct kbase_queue_group *scheduler_get_protm_enter_async_group( -+ struct kbase_device *const kbdev, -+ struct kbase_queue_group *const group) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ struct kbase_queue_group *match_grp, *input_grp; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ if (scheduler->state != SCHED_INACTIVE) -+ return NULL; -+ -+ match_grp = get_tock_top_group(scheduler); -+ input_grp = group ? group : match_grp; -+ -+ if (input_grp && (input_grp == match_grp)) { -+ struct kbase_csf_cmd_stream_group_info *ginfo = -+ &kbdev->csf.global_iface.groups[0]; -+ unsigned long *pending = -+ input_grp->protm_pending_bitmap; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ -+ if (kbase_csf_scheduler_protected_mode_in_use(kbdev) || -+ bitmap_empty(pending, ginfo->stream_num)) -+ input_grp = NULL; -+ -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ } else { -+ input_grp = NULL; -+ } -+ -+ return input_grp; -+} -+ -+void kbase_csf_scheduler_group_protm_enter(struct kbase_queue_group *group) -+{ -+ struct kbase_device *const kbdev = group->kctx->kbdev; -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ int err = kbase_reset_gpu_try_prevent(kbdev); -+ /* Regardless of whether reset failed or is currently happening, exit -+ * early -+ */ -+ if (err) -+ return; -+ -+ mutex_lock(&scheduler->lock); -+ -+ /* Check if the group is now eligible for execution in protected mode. */ -+ if (scheduler_get_protm_enter_async_group(kbdev, group)) -+ scheduler_group_check_protm_enter(kbdev, group); -+ -+ mutex_unlock(&scheduler->lock); -+ kbase_reset_gpu_allow(kbdev); -+} -+ -+/** -+ * check_sync_update_for_idle_group_protm() - Check the sync wait condition -+ * for all the queues bound to -+ * the given group. -+ * -+ * @group: Pointer to the group that requires evaluation. -+ * -+ * This function is called if the GPU is in protected mode and there are on -+ * slot idle groups with higher priority than the active protected mode group. -+ * This function will evaluate the sync condition, if any, of all the queues -+ * bound to the given group. -+ * -+ * Return true if the sync condition of at least one queue has been satisfied. -+ */ -+static bool check_sync_update_for_idle_group_protm( -+ struct kbase_queue_group *group) -+{ -+ struct kbase_device *const kbdev = group->kctx->kbdev; -+ struct kbase_csf_scheduler *const scheduler = -+ &kbdev->csf.scheduler; -+ bool sync_update_done = false; -+ int i; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ for (i = 0; i < MAX_SUPPORTED_STREAMS_PER_GROUP; i++) { -+ struct kbase_queue *queue = group->bound_queues[i]; -+ -+ if (queue && queue->enabled && !sync_update_done) { -+ struct kbase_csf_cmd_stream_group_info *const ginfo = -+ &kbdev->csf.global_iface.groups[group->csg_nr]; -+ struct kbase_csf_cmd_stream_info *const stream = -+ &ginfo->streams[queue->csi_index]; -+ u32 status = kbase_csf_firmware_cs_output( -+ stream, CS_STATUS_WAIT); -+ unsigned long flags; -+ -+ KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, QUEUE_SYNC_STATUS_WAIT, -+ queue->group, queue, status); -+ -+ if (!CS_STATUS_WAIT_SYNC_WAIT_GET(status)) -+ continue; -+ -+ /* Save the information of sync object of the command -+ * queue so the callback function, 'group_sync_updated' -+ * can evaluate the sync object when it gets updated -+ * later. -+ */ -+ queue->status_wait = status; -+ queue->sync_ptr = kbase_csf_firmware_cs_output( -+ stream, CS_STATUS_WAIT_SYNC_POINTER_LO); -+ queue->sync_ptr |= (u64)kbase_csf_firmware_cs_output( -+ stream, CS_STATUS_WAIT_SYNC_POINTER_HI) << 32; -+ queue->sync_value = kbase_csf_firmware_cs_output( -+ stream, CS_STATUS_WAIT_SYNC_VALUE); -+ queue->blocked_reason = -+ CS_STATUS_BLOCKED_REASON_REASON_GET( -+ kbase_csf_firmware_cs_output( -+ stream, -+ CS_STATUS_BLOCKED_REASON)); -+ -+ if (!evaluate_sync_update(queue)) -+ continue; -+ -+ /* Update csg_slots_idle_mask and group's run_state */ -+ if (group->run_state != KBASE_CSF_GROUP_RUNNABLE) { -+ /* Only clear the group's idle flag if it has been dealt -+ * with by the scheduler's tick/tock action, otherwise -+ * leave it untouched. -+ */ -+ spin_lock_irqsave(&scheduler->interrupt_lock, -+ flags); -+ clear_bit((unsigned int)group->csg_nr, -+ scheduler->csg_slots_idle_mask); -+ KBASE_KTRACE_ADD_CSF_GRP( -+ kbdev, CSG_SLOT_IDLE_CLEAR, group, -+ scheduler->csg_slots_idle_mask[0]); -+ spin_unlock_irqrestore( -+ &scheduler->interrupt_lock, flags); -+ group->run_state = KBASE_CSF_GROUP_RUNNABLE; -+ } -+ -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_SYNC_UPDATE_DONE, group, 0u); -+ sync_update_done = true; -+ } -+ } -+ -+ return sync_update_done; -+} -+ -+/** -+ * check_sync_update_for_idle_groups_protm() - Check the sync wait condition -+ * for the idle groups on slot -+ * during protected mode. -+ * -+ * @kbdev: Pointer to the GPU device -+ * -+ * This function checks the gpu queues of all the idle groups on slot during -+ * protected mode that has a higher priority than the active protected mode -+ * group. -+ * -+ * Return true if the sync condition of at least one queue in a group has been -+ * satisfied. -+ */ -+static bool check_sync_update_for_idle_groups_protm(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ struct kbase_queue_group *protm_grp; -+ bool exit_protm = false; -+ unsigned long flags; -+ u32 num_groups; -+ u32 i; -+ -+ lockdep_assert_held(&scheduler->lock); -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ protm_grp = scheduler->active_protm_grp; -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+ -+ if (!protm_grp) -+ return exit_protm; -+ -+ num_groups = kbdev->csf.global_iface.group_num; -+ -+ for_each_set_bit(i, scheduler->csg_slots_idle_mask, num_groups) { -+ struct kbase_csf_csg_slot *csg_slot = -+ &scheduler->csg_slots[i]; -+ struct kbase_queue_group *group = csg_slot->resident_group; -+ -+ if (group->scan_seq_num < protm_grp->scan_seq_num) { -+ /* If sync update has been performed for the group that -+ * has a higher priority than the protm group, then we -+ * need to exit protected mode. -+ */ -+ if (check_sync_update_for_idle_group_protm(group)) -+ exit_protm = true; -+ } -+ } -+ -+ return exit_protm; -+} -+ -+/** -+ * check_group_sync_update_worker() - Check the sync wait condition for all the -+ * blocked queue groups -+ * -+ * @work: Pointer to the context-specific work item for evaluating the wait -+ * condition for all the queue groups in idle_wait_groups list. -+ * -+ * This function checks the gpu queues of all the groups present in both -+ * idle_wait_groups list of a context and all on slot idle groups (if GPU -+ * is in protected mode). -+ * If the sync wait condition for at least one queue bound to the group has -+ * been satisfied then the group is moved to the per context list of -+ * runnable groups so that Scheduler can consider scheduling the group -+ * in next tick or exit protected mode. -+ */ -+static void check_group_sync_update_worker(struct work_struct *work) -+{ -+ struct kbase_context *const kctx = container_of(work, -+ struct kbase_context, csf.sched.sync_update_work); -+ struct kbase_device *const kbdev = kctx->kbdev; -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ mutex_lock(&scheduler->lock); -+ -+ KBASE_KTRACE_ADD(kbdev, GROUP_SYNC_UPDATE_WORKER_BEGIN, kctx, 0u); -+ if (kctx->csf.sched.num_idle_wait_grps != 0) { -+ struct kbase_queue_group *group, *temp; -+ -+ list_for_each_entry_safe(group, temp, -+ &kctx->csf.sched.idle_wait_groups, link) { -+ if (group_sync_updated(group)) { -+ /* Move this group back in to the runnable -+ * groups list of the context. -+ */ -+ update_idle_suspended_group_state(group); -+ KBASE_KTRACE_ADD_CSF_GRP(kbdev, GROUP_SYNC_UPDATE_DONE, group, 0u); -+ } -+ } -+ } else { -+ WARN_ON(!list_empty(&kctx->csf.sched.idle_wait_groups)); -+ } -+ -+ if (check_sync_update_for_idle_groups_protm(kbdev)) -+ scheduler_force_protm_exit(kbdev); -+ KBASE_KTRACE_ADD(kbdev, GROUP_SYNC_UPDATE_WORKER_END, kctx, 0u); -+ -+ mutex_unlock(&scheduler->lock); -+} -+ -+static -+enum kbase_csf_event_callback_action check_group_sync_update_cb(void *param) -+{ -+ struct kbase_context *const kctx = param; -+ -+ KBASE_KTRACE_ADD(kctx->kbdev, SYNC_UPDATE_EVENT, kctx, 0u); -+ queue_work(kctx->csf.sched.sync_update_wq, -+ &kctx->csf.sched.sync_update_work); -+ -+ return KBASE_CSF_EVENT_CALLBACK_KEEP; -+} -+ -+int kbase_csf_scheduler_context_init(struct kbase_context *kctx) -+{ -+ int priority; -+ int err; -+ -+ for (priority = 0; priority < KBASE_QUEUE_GROUP_PRIORITY_COUNT; -+ ++priority) { -+ INIT_LIST_HEAD(&kctx->csf.sched.runnable_groups[priority]); -+ } -+ -+ kctx->csf.sched.num_runnable_grps = 0; -+ INIT_LIST_HEAD(&kctx->csf.sched.idle_wait_groups); -+ kctx->csf.sched.num_idle_wait_grps = 0; -+ kctx->csf.sched.ngrp_to_schedule = 0; -+ -+ kctx->csf.sched.sync_update_wq = -+ alloc_ordered_workqueue("mali_kbase_csf_sync_update_wq", -+ WQ_HIGHPRI); -+ if (!kctx->csf.sched.sync_update_wq) { -+ dev_err(kctx->kbdev->dev, -+ "Failed to initialize scheduler context workqueue"); -+ return -ENOMEM; -+ } -+ -+ INIT_WORK(&kctx->csf.sched.sync_update_work, -+ check_group_sync_update_worker); -+ -+ err = kbase_csf_event_wait_add(kctx, check_group_sync_update_cb, kctx); -+ -+ if (err) { -+ dev_err(kctx->kbdev->dev, -+ "Failed to register a sync update callback"); -+ destroy_workqueue(kctx->csf.sched.sync_update_wq); -+ } -+ -+ return err; -+} -+ -+void kbase_csf_scheduler_context_term(struct kbase_context *kctx) -+{ -+ kbase_csf_event_wait_remove(kctx, check_group_sync_update_cb, kctx); -+ cancel_work_sync(&kctx->csf.sched.sync_update_work); -+ destroy_workqueue(kctx->csf.sched.sync_update_wq); -+} -+ -+int kbase_csf_scheduler_init(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ u32 num_groups = kbdev->csf.global_iface.group_num; -+ -+ bitmap_zero(scheduler->csg_inuse_bitmap, num_groups); -+ bitmap_zero(scheduler->csg_slots_idle_mask, num_groups); -+ -+ scheduler->csg_slots = kcalloc(num_groups, -+ sizeof(*scheduler->csg_slots), GFP_KERNEL); -+ if (!scheduler->csg_slots) { -+ dev_err(kbdev->dev, -+ "Failed to allocate memory for csg slot status array\n"); -+ return -ENOMEM; -+ } -+ -+ return 0; -+} -+ -+int kbase_csf_scheduler_early_init(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ scheduler->timer_enabled = true; -+ -+ scheduler->wq = alloc_ordered_workqueue("csf_scheduler_wq", WQ_HIGHPRI); -+ if (!scheduler->wq) { -+ dev_err(kbdev->dev, "Failed to allocate scheduler workqueue\n"); -+ return -ENOMEM; -+ } -+ -+ INIT_WORK(&scheduler->tick_work, schedule_on_tick); -+ INIT_DEFERRABLE_WORK(&scheduler->tock_work, schedule_on_tock); -+ -+ INIT_DEFERRABLE_WORK(&scheduler->ping_work, firmware_aliveness_monitor); -+ BUILD_BUG_ON(CSF_FIRMWARE_TIMEOUT_MS >= FIRMWARE_PING_INTERVAL_MS); -+ -+ mutex_init(&scheduler->lock); -+ spin_lock_init(&scheduler->interrupt_lock); -+ -+ /* Internal lists */ -+ INIT_LIST_HEAD(&scheduler->runnable_kctxs); -+ INIT_LIST_HEAD(&scheduler->groups_to_schedule); -+ INIT_LIST_HEAD(&scheduler->idle_groups_to_schedule); -+ -+ BUILD_BUG_ON(MAX_SUPPORTED_CSGS > -+ (sizeof(scheduler->csgs_events_enable_mask) * BITS_PER_BYTE)); -+ bitmap_fill(scheduler->csgs_events_enable_mask, MAX_SUPPORTED_CSGS); -+ scheduler->state = SCHED_SUSPENDED; -+ scheduler->pm_active_count = 0; -+ scheduler->ngrp_to_schedule = 0; -+ scheduler->total_runnable_grps = 0; -+ scheduler->top_ctx = NULL; -+ scheduler->top_grp = NULL; -+ scheduler->last_schedule = 0; -+ scheduler->tock_pending_request = false; -+ scheduler->active_protm_grp = NULL; -+ scheduler->gpu_idle_fw_timer_enabled = false; -+ scheduler->csg_scheduling_period_ms = CSF_SCHEDULER_TIME_TICK_MS; -+ scheduler_doorbell_init(kbdev); -+ -+ INIT_WORK(&scheduler->gpu_idle_work, gpu_idle_worker); -+ atomic_set(&scheduler->non_idle_offslot_grps, 0); -+ -+ hrtimer_init(&scheduler->tick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ scheduler->tick_timer.function = tick_timer_callback; -+ scheduler->tick_timer_active = false; -+ -+ return 0; -+} -+ -+void kbase_csf_scheduler_term(struct kbase_device *kbdev) -+{ -+ if (kbdev->csf.scheduler.csg_slots) { -+ WARN_ON(atomic_read(&kbdev->csf.scheduler.non_idle_offslot_grps)); -+ WARN_ON(csgs_active(kbdev)); -+ flush_work(&kbdev->csf.scheduler.gpu_idle_work); -+ mutex_lock(&kbdev->csf.scheduler.lock); -+ if (WARN_ON(kbdev->csf.scheduler.state != SCHED_SUSPENDED)) -+ scheduler_suspend(kbdev); -+ mutex_unlock(&kbdev->csf.scheduler.lock); -+ cancel_delayed_work_sync(&kbdev->csf.scheduler.ping_work); -+ cancel_tick_timer(kbdev); -+ cancel_work_sync(&kbdev->csf.scheduler.tick_work); -+ cancel_delayed_work_sync(&kbdev->csf.scheduler.tock_work); -+ mutex_destroy(&kbdev->csf.scheduler.lock); -+ kfree(kbdev->csf.scheduler.csg_slots); -+ kbdev->csf.scheduler.csg_slots = NULL; -+ } -+} -+ -+void kbase_csf_scheduler_early_term(struct kbase_device *kbdev) -+{ -+ if (kbdev->csf.scheduler.wq) -+ destroy_workqueue(kbdev->csf.scheduler.wq); -+} -+ -+/** -+ * scheduler_enable_tick_timer_nolock - Enable the scheduler tick timer. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * This function will restart the scheduler tick so that regular scheduling can -+ * be resumed without any explicit trigger (like kicking of GPU queues). This -+ * is a variant of kbase_csf_scheduler_enable_tick_timer() that assumes the -+ * CSF scheduler lock to already have been held. -+ */ -+static void scheduler_enable_tick_timer_nolock(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&kbdev->csf.scheduler.lock); -+ -+ if (unlikely(!scheduler_timer_is_enabled_nolock(kbdev))) -+ return; -+ -+ WARN_ON((scheduler->state != SCHED_INACTIVE) && -+ (scheduler->state != SCHED_SUSPENDED)); -+ -+ if (scheduler->total_runnable_grps > 0) { -+ enqueue_tick_work(kbdev); -+ dev_dbg(kbdev->dev, "Re-enabling the scheduler timer\n"); -+ } else if (scheduler->state != SCHED_SUSPENDED) { -+ queue_work(system_wq, &scheduler->gpu_idle_work); -+ } -+} -+ -+void kbase_csf_scheduler_enable_tick_timer(struct kbase_device *kbdev) -+{ -+ mutex_lock(&kbdev->csf.scheduler.lock); -+ scheduler_enable_tick_timer_nolock(kbdev); -+ mutex_unlock(&kbdev->csf.scheduler.lock); -+} -+ -+bool kbase_csf_scheduler_timer_is_enabled(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ bool enabled; -+ -+ mutex_lock(&scheduler->lock); -+ enabled = scheduler_timer_is_enabled_nolock(kbdev); -+ mutex_unlock(&scheduler->lock); -+ -+ return enabled; -+} -+ -+void kbase_csf_scheduler_timer_set_enabled(struct kbase_device *kbdev, -+ bool enable) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ bool currently_enabled; -+ -+ mutex_lock(&scheduler->lock); -+ -+ currently_enabled = scheduler_timer_is_enabled_nolock(kbdev); -+ if (currently_enabled && !enable) { -+ scheduler->timer_enabled = false; -+ cancel_tick_timer(kbdev); -+ cancel_delayed_work(&scheduler->tock_work); -+ mutex_unlock(&scheduler->lock); -+ /* The non-sync version to cancel the normal work item is not -+ * available, so need to drop the lock before cancellation. -+ */ -+ cancel_work_sync(&scheduler->tick_work); -+ } else if (!currently_enabled && enable) { -+ scheduler->timer_enabled = true; -+ -+ scheduler_enable_tick_timer_nolock(kbdev); -+ mutex_unlock(&scheduler->lock); -+ } -+} -+ -+void kbase_csf_scheduler_kick(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ mutex_lock(&scheduler->lock); -+ -+ if (unlikely(scheduler_timer_is_enabled_nolock(kbdev))) -+ goto out; -+ -+ if (scheduler->total_runnable_grps > 0) { -+ enqueue_tick_work(kbdev); -+ dev_dbg(kbdev->dev, "Kicking the scheduler manually\n"); -+ } -+ -+out: -+ mutex_unlock(&scheduler->lock); -+} -+ -+void kbase_csf_scheduler_pm_suspend(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ /* Cancel any potential queued delayed work(s) */ -+ cancel_work_sync(&scheduler->tick_work); -+ cancel_delayed_work_sync(&scheduler->tock_work); -+ -+ if (kbase_reset_gpu_prevent_and_wait(kbdev)) { -+ dev_warn(kbdev->dev, -+ "Stop PM suspending for failing to prevent gpu reset.\n"); -+ return; -+ } -+ -+ mutex_lock(&scheduler->lock); -+ -+ disable_gpu_idle_fw_timer(kbdev); -+ -+ if (scheduler->state != SCHED_SUSPENDED) { -+ suspend_active_groups_on_powerdown(kbdev, true); -+ dev_info(kbdev->dev, "Scheduler PM suspend"); -+ scheduler_suspend(kbdev); -+ cancel_tick_timer(kbdev); -+ } -+ mutex_unlock(&scheduler->lock); -+ -+ kbase_reset_gpu_allow(kbdev); -+} -+KBASE_EXPORT_TEST_API(kbase_csf_scheduler_pm_suspend); -+ -+void kbase_csf_scheduler_pm_resume(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ -+ mutex_lock(&scheduler->lock); -+ -+ if (scheduler->total_runnable_grps > 0) { -+ WARN_ON(scheduler->state != SCHED_SUSPENDED); -+ dev_info(kbdev->dev, "Scheduler PM resume"); -+ scheduler_wakeup(kbdev, true); -+ } -+ mutex_unlock(&scheduler->lock); -+} -+KBASE_EXPORT_TEST_API(kbase_csf_scheduler_pm_resume); -+ -+void kbase_csf_scheduler_pm_active(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ u32 prev_count; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ prev_count = kbdev->csf.scheduler.pm_active_count++; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* On 0 => 1, make a pm_ctx_active request */ -+ if (!prev_count) -+ kbase_pm_context_active(kbdev); -+ else -+ WARN_ON(prev_count == U32_MAX); -+} -+KBASE_EXPORT_TEST_API(kbase_csf_scheduler_pm_active); -+ -+void kbase_csf_scheduler_pm_idle(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ u32 prev_count; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ prev_count = kbdev->csf.scheduler.pm_active_count--; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ if (prev_count == 1) -+ kbase_pm_context_idle(kbdev); -+ else -+ WARN_ON(prev_count == 0); -+} -+KBASE_EXPORT_TEST_API(kbase_csf_scheduler_pm_idle); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_scheduler.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_scheduler.h -new file mode 100644 -index 0000000..428ecbe ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_scheduler.h -@@ -0,0 +1,494 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_SCHEDULER_H_ -+#define _KBASE_CSF_SCHEDULER_H_ -+ -+#include "mali_kbase_csf.h" -+ -+/** -+ * kbase_csf_scheduler_queue_start() - Enable the running of GPU command queue -+ * on firmware. -+ * -+ * @queue: Pointer to the GPU command queue to be started. -+ * -+ * This function would enable the start of a CSI, within a -+ * CSG, to which the @queue was bound. -+ * If the CSG is already scheduled and resident, the CSI will be started -+ * right away, otherwise once the group is made resident. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_scheduler_queue_start(struct kbase_queue *queue); -+ -+/** -+ * kbase_csf_scheduler_queue_stop() - Disable the running of GPU command queue -+ * on firmware. -+ * -+ * @queue: Pointer to the GPU command queue to be stopped. -+ * -+ * This function would stop the CSI, within a CSG, to which @queue was bound. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_scheduler_queue_stop(struct kbase_queue *queue); -+ -+/** -+ * kbase_csf_scheduler_group_protm_enter - Handle the protm enter event for the -+ * GPU command queue group. -+ * -+ * @group: The command queue group. -+ * -+ * This function could request the firmware to enter the protected mode -+ * and allow the execution of protected region instructions for all the -+ * bound queues of the group that have protm pending bit set in their -+ * respective CS_ACK register. -+ */ -+void kbase_csf_scheduler_group_protm_enter(struct kbase_queue_group *group); -+ -+/** -+ * kbase_csf_scheduler_group_get_slot() - Checks if a queue group is -+ * programmed on a firmware CSG slot -+ * and returns the slot number. -+ * -+ * @group: The command queue group. -+ * -+ * Return: The slot number, if the group is programmed on a slot. -+ * Otherwise returns a negative number. -+ * -+ * Note: This function should not be used if the interrupt_lock is held. Use -+ * kbase_csf_scheduler_group_get_slot_locked() instead. -+ */ -+int kbase_csf_scheduler_group_get_slot(struct kbase_queue_group *group); -+ -+/** -+ * kbase_csf_scheduler_group_get_slot_locked() - Checks if a queue group is -+ * programmed on a firmware CSG slot -+ * and returns the slot number. -+ * -+ * @group: The command queue group. -+ * -+ * Return: The slot number, if the group is programmed on a slot. -+ * Otherwise returns a negative number. -+ * -+ * Note: Caller must hold the interrupt_lock. -+ */ -+int kbase_csf_scheduler_group_get_slot_locked(struct kbase_queue_group *group); -+ -+/** -+ * kbase_csf_scheduler_group_events_enabled() - Checks if interrupt events -+ * should be handled for a queue group. -+ * -+ * @kbdev: The device of the group. -+ * @group: The queue group. -+ * -+ * Return: true if interrupt events should be handled. -+ * -+ * Note: Caller must hold the interrupt_lock. -+ */ -+bool kbase_csf_scheduler_group_events_enabled(struct kbase_device *kbdev, -+ struct kbase_queue_group *group); -+ -+/** -+ * kbase_csf_scheduler_get_group_on_slot()- Gets the queue group that has been -+ * programmed to a firmware CSG slot. -+ * -+ * @kbdev: The GPU device. -+ * @slot: The slot for which to get the queue group. -+ * -+ * Return: Pointer to the programmed queue group. -+ * -+ * Note: Caller must hold the interrupt_lock. -+ */ -+struct kbase_queue_group *kbase_csf_scheduler_get_group_on_slot( -+ struct kbase_device *kbdev, int slot); -+ -+/** -+ * kbase_csf_scheduler_group_deschedule() - Deschedule a GPU command queue -+ * group from the firmware. -+ * -+ * @group: Pointer to the queue group to be descheduled. -+ * -+ * This function would disable the scheduling of GPU command queue group on -+ * firmware. -+ */ -+void kbase_csf_scheduler_group_deschedule(struct kbase_queue_group *group); -+ -+/** -+ * kbase_csf_scheduler_evict_ctx_slots() - Evict all GPU command queue groups -+ * of a given context that are active -+ * running from the firmware. -+ * -+ * @kbdev: The GPU device. -+ * @kctx: Kbase context for the evict operation. -+ * @evicted_groups: List_head for returning evicted active queue groups. -+ * -+ * This function would disable the scheduling of GPU command queue groups active -+ * on firmware slots from the given Kbase context. The affected groups are -+ * added to the supplied list_head argument. -+ */ -+void kbase_csf_scheduler_evict_ctx_slots(struct kbase_device *kbdev, -+ struct kbase_context *kctx, struct list_head *evicted_groups); -+ -+/** -+ * kbase_csf_scheduler_context_init() - Initialize the context-specific part -+ * for CSF scheduler. -+ * -+ * @kctx: Pointer to kbase context that is being created. -+ * -+ * This function must be called during Kbase context creation. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_scheduler_context_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_scheduler_init - Initialize the CSF scheduler -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * The scheduler does the arbitration for the CSG slots -+ * provided by the firmware between the GPU command queue groups created -+ * by the Clients. -+ * This function must be called after loading firmware and parsing its capabilities. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_scheduler_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_early_init - Early initialization for the CSF scheduler -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Initialize necessary resources such as locks, workqueue for CSF scheduler. -+ * This must be called at kbase probe. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_scheduler_early_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_context_term() - Terminate the context-specific part -+ * for CSF scheduler. -+ * -+ * @kctx: Pointer to kbase context that is being terminated. -+ * -+ * This function must be called during Kbase context termination. -+ */ -+void kbase_csf_scheduler_context_term(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_scheduler_term - Terminate the CSF scheduler. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * This should be called when unload of firmware is done on device -+ * termination. -+ */ -+void kbase_csf_scheduler_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_early_term - Early termination of the CSF scheduler. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * This should be called only when kbase probe fails or gets rmmoded. -+ */ -+void kbase_csf_scheduler_early_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_reset - Reset the state of all active GPU command -+ * queue groups. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * This function will first iterate through all the active/scheduled GPU -+ * command queue groups and suspend them (to avoid losing work for groups -+ * that are not stuck). The groups that could not get suspended would be -+ * descheduled and marked as terminated (which will then lead to unbinding -+ * of all the queues bound to them) and also no more work would be allowed -+ * to execute for them. -+ * -+ * This is similar to the action taken in response to an unexpected OoM event. -+ * No explicit re-initialization is done for CSG & CS interface I/O pages; -+ * instead, that happens implicitly on firmware reload. -+ * -+ * Should be called only after initiating the GPU reset. -+ */ -+void kbase_csf_scheduler_reset(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_enable_tick_timer - Enable the scheduler tick timer. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * This function will restart the scheduler tick so that regular scheduling can -+ * be resumed without any explicit trigger (like kicking of GPU queues). -+ */ -+void kbase_csf_scheduler_enable_tick_timer(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_group_copy_suspend_buf - Suspend a queue -+ * group and copy suspend buffer. -+ * -+ * This function is called to suspend a queue group and copy the suspend_buffer -+ * contents to the input buffer provided. -+ * -+ * @group: Pointer to the queue group to be suspended. -+ * @sus_buf: Pointer to the structure which contains details of the -+ * user buffer and its kernel pinned pages to which we need to copy -+ * the group suspend buffer. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_scheduler_group_copy_suspend_buf(struct kbase_queue_group *group, -+ struct kbase_suspend_copy_buffer *sus_buf); -+ -+/** -+ * kbase_csf_scheduler_lock - Acquire the global Scheduler lock. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * This function will take the global scheduler lock, in order to serialize -+ * against the Scheduler actions, for access to CS IO pages. -+ */ -+static inline void kbase_csf_scheduler_lock(struct kbase_device *kbdev) -+{ -+ mutex_lock(&kbdev->csf.scheduler.lock); -+} -+ -+/** -+ * kbase_csf_scheduler_unlock - Release the global Scheduler lock. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+static inline void kbase_csf_scheduler_unlock(struct kbase_device *kbdev) -+{ -+ mutex_unlock(&kbdev->csf.scheduler.lock); -+} -+ -+/** -+ * kbase_csf_scheduler_spin_lock - Acquire Scheduler interrupt spinlock. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @flags: Pointer to the memory location that would store the previous -+ * interrupt state. -+ * -+ * This function will take the global scheduler lock, in order to serialize -+ * against the Scheduler actions, for access to CS IO pages. -+ */ -+static inline void kbase_csf_scheduler_spin_lock(struct kbase_device *kbdev, -+ unsigned long *flags) -+{ -+ spin_lock_irqsave(&kbdev->csf.scheduler.interrupt_lock, *flags); -+} -+ -+/** -+ * kbase_csf_scheduler_spin_unlock - Release Scheduler interrupt spinlock. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @flags: Previously stored interrupt state when Scheduler interrupt -+ * spinlock was acquired. -+ */ -+static inline void kbase_csf_scheduler_spin_unlock(struct kbase_device *kbdev, -+ unsigned long flags) -+{ -+ spin_unlock_irqrestore(&kbdev->csf.scheduler.interrupt_lock, flags); -+} -+ -+/** -+ * kbase_csf_scheduler_spin_lock_assert_held - Assert if the Scheduler -+ * interrupt spinlock is held. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+static inline void -+kbase_csf_scheduler_spin_lock_assert_held(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); -+} -+ -+/** -+ * kbase_csf_scheduler_timer_is_enabled() - Check if the scheduler wakes up -+ * automatically for periodic tasks. -+ * -+ * @kbdev: Pointer to the device -+ * -+ * Return: true if the scheduler is configured to wake up periodically -+ */ -+bool kbase_csf_scheduler_timer_is_enabled(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_timer_set_enabled() - Enable/disable periodic -+ * scheduler tasks. -+ * -+ * @kbdev: Pointer to the device -+ * @enable: Whether to enable periodic scheduler tasks -+ */ -+void kbase_csf_scheduler_timer_set_enabled(struct kbase_device *kbdev, -+ bool enable); -+ -+/** -+ * kbase_csf_scheduler_kick - Perform pending scheduling tasks once. -+ * -+ * Note: This function is only effective if the scheduling timer is disabled. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_scheduler_kick(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_protected_mode_in_use() - Check if the scheduler is -+ * running with protected mode tasks. -+ * -+ * @kbdev: Pointer to the device -+ * -+ * Return: true if the scheduler is running with protected mode tasks -+ */ -+static inline bool kbase_csf_scheduler_protected_mode_in_use( -+ struct kbase_device *kbdev) -+{ -+ return (kbdev->csf.scheduler.active_protm_grp != NULL); -+} -+ -+/** -+ * kbase_csf_scheduler_pm_active - Perform scheduler power active operation -+ * -+ * Note: This function will increase the scheduler's internal pm_active_count -+ * value, ensuring that both GPU and MCU are powered for access. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_scheduler_pm_active(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_pm_idle - Perform the scheduler power idle operation -+ * -+ * Note: This function will decrease the scheduler's internal pm_active_count -+ * value. On reaching 0, the MCU and GPU could be powered off. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ */ -+void kbase_csf_scheduler_pm_idle(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_pm_resume - Reactivate the scheduler on system resume -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * This function will make the scheduler resume the scheduling of queue groups -+ * and take the power managemenet reference, if there are any runnable groups. -+ */ -+void kbase_csf_scheduler_pm_resume(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_pm_suspend - Idle the scheduler on system suspend -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * This function will make the scheduler suspend all the running queue groups -+ * and drop its power managemenet reference. -+ */ -+void kbase_csf_scheduler_pm_suspend(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_scheduler_all_csgs_idle() - Check if the scheduler internal -+ * runtime used slots are all tagged as idle command queue groups. -+ * -+ * @kbdev: Pointer to the device -+ * -+ * Return: true if all the used slots are tagged as idle CSGs. -+ */ -+static inline bool kbase_csf_scheduler_all_csgs_idle(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->csf.scheduler.interrupt_lock); -+ return bitmap_equal(kbdev->csf.scheduler.csg_slots_idle_mask, -+ kbdev->csf.scheduler.csg_inuse_bitmap, -+ kbdev->csf.global_iface.group_num); -+} -+ -+/** -+ * kbase_csf_scheduler_advance_tick_nolock() - Advance the scheduling tick -+ * -+ * @kbdev: Pointer to the device -+ * -+ * This function advances the scheduling tick by enqueing the tick work item for -+ * immediate execution, but only if the tick hrtimer is active. If the timer -+ * is inactive then the tick work item is already in flight. -+ * The caller must hold the interrupt lock. -+ */ -+static inline void -+kbase_csf_scheduler_advance_tick_nolock(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ -+ lockdep_assert_held(&scheduler->interrupt_lock); -+ -+ if (scheduler->tick_timer_active) { -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_ADVANCE_TICK, NULL, 0u); -+ scheduler->tick_timer_active = false; -+ queue_work(scheduler->wq, &scheduler->tick_work); -+ } else { -+ KBASE_KTRACE_ADD(kbdev, SCHEDULER_NOADVANCE_TICK, NULL, 0u); -+ } -+} -+ -+/** -+ * kbase_csf_scheduler_advance_tick() - Advance the scheduling tick -+ * -+ * @kbdev: Pointer to the device -+ * -+ * This function advances the scheduling tick by enqueing the tick work item for -+ * immediate execution, but only if the tick hrtimer is active. If the timer -+ * is inactive then the tick work item is already in flight. -+ */ -+static inline void kbase_csf_scheduler_advance_tick(struct kbase_device *kbdev) -+{ -+ struct kbase_csf_scheduler *const scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&scheduler->interrupt_lock, flags); -+ kbase_csf_scheduler_advance_tick_nolock(kbdev); -+ spin_unlock_irqrestore(&scheduler->interrupt_lock, flags); -+} -+ -+/** -+ * kbase_csf_scheduler_queue_has_trace() - report whether the queue has been -+ * configured to operate with the -+ * cs_trace feature. -+ * -+ * @queue: Pointer to the queue. -+ * -+ * Return: True if the gpu queue is configured to operate with the cs_trace -+ * feature, otherwise false. -+ */ -+static inline bool kbase_csf_scheduler_queue_has_trace(struct kbase_queue *queue) -+{ -+ lockdep_assert_held(&queue->kctx->kbdev->csf.scheduler.lock); -+ /* In the current arrangement, it is possible for the context to enable -+ * the cs_trace after some queues have been registered with cs_trace in -+ * disabled state. So each queue has its own enabled/disabled condition. -+ */ -+ return (queue->trace_buffer_size && queue->trace_buffer_base); -+} -+ -+#endif /* _KBASE_CSF_SCHEDULER_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c -new file mode 100644 -index 0000000..8ecf235 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c -@@ -0,0 +1,611 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <tl/mali_kbase_tracepoints.h> -+ -+#include "mali_kbase_csf_tiler_heap.h" -+#include "mali_kbase_csf_tiler_heap_def.h" -+#include "mali_kbase_csf_heap_context_alloc.h" -+ -+/** -+ * encode_chunk_ptr - Encode the address and size of a chunk as an integer. -+ * -+ * The size and address of the next chunk in a list are packed into a single -+ * 64-bit value for storage in a chunk's header. This function returns that -+ * value. -+ * -+ * @chunk_size: Size of a tiler heap chunk, in bytes. -+ * @chunk_addr: GPU virtual address of the same tiler heap chunk. -+ * -+ * Return: Next chunk pointer suitable for writing into a chunk header. -+ */ -+static u64 encode_chunk_ptr(u32 const chunk_size, u64 const chunk_addr) -+{ -+ u64 encoded_size, encoded_addr; -+ -+ WARN_ON(chunk_size & ~CHUNK_SIZE_MASK); -+ WARN_ON(chunk_addr & ~CHUNK_ADDR_MASK); -+ -+ encoded_size = -+ (u64)(chunk_size >> CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT) << -+ CHUNK_HDR_NEXT_SIZE_POS; -+ -+ encoded_addr = -+ (chunk_addr >> CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT) << -+ CHUNK_HDR_NEXT_ADDR_POS; -+ -+ return (encoded_size & CHUNK_HDR_NEXT_SIZE_MASK) | -+ (encoded_addr & CHUNK_HDR_NEXT_ADDR_MASK); -+} -+ -+/** -+ * get_last_chunk - Get the last chunk of a tiler heap -+ * -+ * @heap: Pointer to the tiler heap. -+ * -+ * Return: The address of the most recently-linked chunk, or NULL if none. -+ */ -+static struct kbase_csf_tiler_heap_chunk *get_last_chunk( -+ struct kbase_csf_tiler_heap *const heap) -+{ -+ lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); -+ -+ if (list_empty(&heap->chunks_list)) -+ return NULL; -+ -+ return list_last_entry(&heap->chunks_list, -+ struct kbase_csf_tiler_heap_chunk, link); -+} -+ -+/** -+ * link_chunk - Link a chunk into a tiler heap -+ * -+ * Unless the @chunk is the first in the kernel's list of chunks belonging to -+ * a given tiler heap, this function stores the size and address of the @chunk -+ * in the header of the preceding chunk. This requires the GPU memory region -+ * containing the header to be be mapped temporarily, which can fail. -+ * -+ * @heap: Pointer to the tiler heap. -+ * @chunk: Pointer to the heap chunk to be linked. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+static int link_chunk(struct kbase_csf_tiler_heap *const heap, -+ struct kbase_csf_tiler_heap_chunk *const chunk) -+{ -+ struct kbase_csf_tiler_heap_chunk *const prev = get_last_chunk(heap); -+ -+ if (prev) { -+ struct kbase_context *const kctx = heap->kctx; -+ struct kbase_vmap_struct map; -+ u64 *const prev_hdr = kbase_vmap_prot(kctx, prev->gpu_va, -+ sizeof(*prev_hdr), KBASE_REG_CPU_WR, &map); -+ -+ if (unlikely(!prev_hdr)) { -+ dev_err(kctx->kbdev->dev, -+ "Failed to map tiler heap chunk 0x%llX\n", -+ prev->gpu_va); -+ return -ENOMEM; -+ } -+ -+ *prev_hdr = encode_chunk_ptr(heap->chunk_size, chunk->gpu_va); -+ kbase_vunmap(kctx, &map); -+ -+ dev_dbg(kctx->kbdev->dev, -+ "Linked tiler heap chunks, 0x%llX -> 0x%llX\n", -+ prev->gpu_va, chunk->gpu_va); -+ } -+ -+ return 0; -+} -+ -+/** -+ * init_chunk - Initialize and link a tiler heap chunk -+ * -+ * Zero-initialize a new chunk's header (including its pointer to the next -+ * chunk, which doesn't exist yet) and then update the previous chunk's -+ * header to link the new chunk into the chunk list. -+ * -+ * @heap: Pointer to the tiler heap. -+ * @chunk: Pointer to the heap chunk to be initialized and linked. -+ * @link_with_prev: Flag to indicate if the new chunk needs to be linked with -+ * the previously allocated chunk. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+static int init_chunk(struct kbase_csf_tiler_heap *const heap, -+ struct kbase_csf_tiler_heap_chunk *const chunk, bool link_with_prev) -+{ -+ struct kbase_vmap_struct map; -+ struct u64 *chunk_hdr = NULL; -+ struct kbase_context *const kctx = heap->kctx; -+ -+ if (unlikely(chunk->gpu_va & ~CHUNK_ADDR_MASK)) { -+ dev_err(kctx->kbdev->dev, -+ "Tiler heap chunk address is unusable\n"); -+ return -EINVAL; -+ } -+ -+ chunk_hdr = kbase_vmap_prot(kctx, -+ chunk->gpu_va, CHUNK_HDR_SIZE, KBASE_REG_CPU_WR, &map); -+ -+ if (unlikely(!chunk_hdr)) { -+ dev_err(kctx->kbdev->dev, -+ "Failed to map a tiler heap chunk header\n"); -+ return -ENOMEM; -+ } -+ -+ memset(chunk_hdr, 0, CHUNK_HDR_SIZE); -+ kbase_vunmap(kctx, &map); -+ -+ if (link_with_prev) -+ return link_chunk(heap, chunk); -+ else -+ return 0; -+} -+ -+/** -+ * create_chunk - Create a tiler heap chunk -+ * -+ * This function allocates a chunk of memory for a tiler heap and adds it to -+ * the end of the list of chunks associated with that heap. The size of the -+ * chunk is not a parameter because it is configured per-heap not per-chunk. -+ * -+ * @heap: Pointer to the tiler heap for which to allocate memory. -+ * @link_with_prev: Flag to indicate if the chunk to be allocated needs to be -+ * linked with the previously allocated chunk. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+static int create_chunk(struct kbase_csf_tiler_heap *const heap, -+ bool link_with_prev) -+{ -+ int err = 0; -+ struct kbase_context *const kctx = heap->kctx; -+ u64 nr_pages = PFN_UP(heap->chunk_size); -+ u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | -+ BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE | -+ BASE_MEM_COHERENT_LOCAL; -+ struct kbase_csf_tiler_heap_chunk *chunk = NULL; -+ -+ flags |= base_mem_group_id_set(kctx->jit_group_id); -+ -+#if defined(CONFIG_MALI_DEBUG) || defined(CONFIG_MALI_VECTOR_DUMP) -+ flags |= BASE_MEM_PROT_CPU_RD; -+#endif -+ -+ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); -+ -+ chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); -+ if (unlikely(!chunk)) { -+ dev_err(kctx->kbdev->dev, -+ "No kernel memory for a new tiler heap chunk\n"); -+ return -ENOMEM; -+ } -+ -+ /* Allocate GPU memory for the new chunk. */ -+ INIT_LIST_HEAD(&chunk->link); -+ chunk->region = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, -+ &flags, &chunk->gpu_va); -+ -+ if (unlikely(!chunk->region)) { -+ dev_err(kctx->kbdev->dev, -+ "Failed to allocate a tiler heap chunk\n"); -+ err = -ENOMEM; -+ } else { -+ err = init_chunk(heap, chunk, link_with_prev); -+ if (unlikely(err)) { -+ kbase_gpu_vm_lock(kctx); -+ chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; -+ kbase_mem_free_region(kctx, chunk->region); -+ kbase_gpu_vm_unlock(kctx); -+ } -+ } -+ -+ if (unlikely(err)) { -+ kfree(chunk); -+ } else { -+ list_add_tail(&chunk->link, &heap->chunks_list); -+ heap->chunk_count++; -+ -+ dev_dbg(kctx->kbdev->dev, "Created tiler heap chunk 0x%llX\n", -+ chunk->gpu_va); -+ } -+ -+ return err; -+} -+ -+/** -+ * delete_chunk - Delete a tiler heap chunk -+ * -+ * This function frees a tiler heap chunk previously allocated by @create_chunk -+ * and removes it from the list of chunks associated with the heap. -+ * -+ * WARNING: The deleted chunk is not unlinked from the list of chunks used by -+ * the GPU, therefore it is only safe to use this function when -+ * deleting a heap. -+ * -+ * @heap: Pointer to the tiler heap for which @chunk was allocated. -+ * @chunk: Pointer to a chunk to be deleted. -+ */ -+static void delete_chunk(struct kbase_csf_tiler_heap *const heap, -+ struct kbase_csf_tiler_heap_chunk *const chunk) -+{ -+ struct kbase_context *const kctx = heap->kctx; -+ -+ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); -+ -+ kbase_gpu_vm_lock(kctx); -+ chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; -+ kbase_mem_free_region(kctx, chunk->region); -+ kbase_gpu_vm_unlock(kctx); -+ list_del(&chunk->link); -+ heap->chunk_count--; -+ kfree(chunk); -+} -+ -+/** -+ * delete_all_chunks - Delete all chunks belonging to a tiler heap -+ * -+ * This function empties the list of chunks associated with a tiler heap by -+ * freeing all chunks previously allocated by @create_chunk. -+ * -+ * @heap: Pointer to a tiler heap. -+ */ -+static void delete_all_chunks(struct kbase_csf_tiler_heap *heap) -+{ -+ struct list_head *entry = NULL, *tmp = NULL; -+ struct kbase_context *const kctx = heap->kctx; -+ -+ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); -+ -+ list_for_each_safe(entry, tmp, &heap->chunks_list) { -+ struct kbase_csf_tiler_heap_chunk *chunk = list_entry( -+ entry, struct kbase_csf_tiler_heap_chunk, link); -+ -+ delete_chunk(heap, chunk); -+ } -+} -+ -+/** -+ * create_initial_chunks - Create the initial list of chunks for a tiler heap -+ * -+ * This function allocates a given number of chunks for a tiler heap and -+ * adds them to the list of chunks associated with that heap. -+ * -+ * @heap: Pointer to the tiler heap for which to allocate memory. -+ * @nchunks: Number of chunks to create. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+static int create_initial_chunks(struct kbase_csf_tiler_heap *const heap, -+ u32 const nchunks) -+{ -+ int err = 0; -+ u32 i; -+ -+ for (i = 0; (i < nchunks) && likely(!err); i++) -+ err = create_chunk(heap, true); -+ -+ if (unlikely(err)) -+ delete_all_chunks(heap); -+ -+ return err; -+} -+ -+/** -+ * delete_heap - Delete a tiler heap -+ * -+ * This function frees any chunks allocated for a tiler heap previously -+ * initialized by @kbase_csf_tiler_heap_init and removes it from the list of -+ * heaps associated with the kbase context. The heap context structure used by -+ * the firmware is also freed. -+ * -+ * @heap: Pointer to a tiler heap to be deleted. -+ */ -+static void delete_heap(struct kbase_csf_tiler_heap *heap) -+{ -+ struct kbase_context *const kctx = heap->kctx; -+ -+ dev_dbg(kctx->kbdev->dev, "Deleting tiler heap 0x%llX\n", heap->gpu_va); -+ -+ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); -+ -+ delete_all_chunks(heap); -+ -+ /* We could optimize context destruction by not freeing leaked heap -+ * contexts but it doesn't seem worth the extra complexity. -+ */ -+ kbase_csf_heap_context_allocator_free(&kctx->csf.tiler_heaps.ctx_alloc, -+ heap->gpu_va); -+ -+ list_del(&heap->link); -+ -+ WARN_ON(heap->chunk_count); -+ KBASE_TLSTREAM_AUX_TILER_HEAP_STATS(kctx->kbdev, kctx->id, -+ heap->heap_id, 0, 0, heap->max_chunks, heap->chunk_size, 0, -+ heap->target_in_flight, 0); -+ -+ kfree(heap); -+} -+ -+/** -+ * find_tiler_heap - Find a tiler heap from the address of its heap context -+ * -+ * Each tiler heap managed by the kernel has an associated heap context -+ * structure used by the firmware. This function finds a tiler heap object from -+ * the GPU virtual address of its associated heap context. The heap context -+ * should have been allocated by @kbase_csf_heap_context_allocator_alloc in the -+ * same @kctx. -+ * -+ * @kctx: Pointer to the kbase context to search for a tiler heap. -+ * @heap_gpu_va: GPU virtual address of a heap context structure. -+ * -+ * Return: pointer to the tiler heap object, or NULL if not found. -+ */ -+static struct kbase_csf_tiler_heap *find_tiler_heap( -+ struct kbase_context *const kctx, u64 const heap_gpu_va) -+{ -+ struct kbase_csf_tiler_heap *heap = NULL; -+ -+ lockdep_assert_held(&kctx->csf.tiler_heaps.lock); -+ -+ list_for_each_entry(heap, &kctx->csf.tiler_heaps.list, link) { -+ if (heap_gpu_va == heap->gpu_va) -+ return heap; -+ } -+ -+ dev_dbg(kctx->kbdev->dev, "Tiler heap 0x%llX was not found\n", -+ heap_gpu_va); -+ -+ return NULL; -+} -+ -+int kbase_csf_tiler_heap_context_init(struct kbase_context *const kctx) -+{ -+ int err = kbase_csf_heap_context_allocator_init( -+ &kctx->csf.tiler_heaps.ctx_alloc, kctx); -+ -+ if (unlikely(err)) -+ return err; -+ -+ INIT_LIST_HEAD(&kctx->csf.tiler_heaps.list); -+ mutex_init(&kctx->csf.tiler_heaps.lock); -+ -+ dev_dbg(kctx->kbdev->dev, "Initialized a context for tiler heaps\n"); -+ -+ return 0; -+} -+ -+void kbase_csf_tiler_heap_context_term(struct kbase_context *const kctx) -+{ -+ struct list_head *entry = NULL, *tmp = NULL; -+ -+ dev_dbg(kctx->kbdev->dev, "Terminating a context for tiler heaps\n"); -+ -+ mutex_lock(&kctx->csf.tiler_heaps.lock); -+ -+ list_for_each_safe(entry, tmp, &kctx->csf.tiler_heaps.list) { -+ struct kbase_csf_tiler_heap *heap = list_entry( -+ entry, struct kbase_csf_tiler_heap, link); -+ delete_heap(heap); -+ } -+ -+ mutex_unlock(&kctx->csf.tiler_heaps.lock); -+ mutex_destroy(&kctx->csf.tiler_heaps.lock); -+ -+ kbase_csf_heap_context_allocator_term(&kctx->csf.tiler_heaps.ctx_alloc); -+} -+ -+int kbase_csf_tiler_heap_init(struct kbase_context *const kctx, -+ u32 const chunk_size, u32 const initial_chunks, u32 const max_chunks, -+ u16 const target_in_flight, u64 *const heap_gpu_va, -+ u64 *const first_chunk_va) -+{ -+ int err = 0; -+ struct kbase_csf_tiler_heap *heap = NULL; -+ struct kbase_csf_heap_context_allocator *const ctx_alloc = -+ &kctx->csf.tiler_heaps.ctx_alloc; -+ -+ dev_dbg(kctx->kbdev->dev, -+ "Creating a tiler heap with %u chunks (limit: %u) of size %u\n", -+ initial_chunks, max_chunks, chunk_size); -+ -+ if (chunk_size == 0) -+ return -EINVAL; -+ -+ if (chunk_size & ~CHUNK_SIZE_MASK) -+ return -EINVAL; -+ -+ if (initial_chunks == 0) -+ return -EINVAL; -+ -+ if (initial_chunks > max_chunks) -+ return -EINVAL; -+ -+ if (target_in_flight == 0) -+ return -EINVAL; -+ -+ heap = kzalloc(sizeof(*heap), GFP_KERNEL); -+ if (unlikely(!heap)) { -+ dev_err(kctx->kbdev->dev, -+ "No kernel memory for a new tiler heap\n"); -+ return -ENOMEM; -+ } -+ -+ heap->kctx = kctx; -+ heap->chunk_size = chunk_size; -+ heap->max_chunks = max_chunks; -+ heap->target_in_flight = target_in_flight; -+ INIT_LIST_HEAD(&heap->chunks_list); -+ -+ heap->gpu_va = kbase_csf_heap_context_allocator_alloc(ctx_alloc); -+ -+ mutex_lock(&kctx->csf.tiler_heaps.lock); -+ -+ if (unlikely(!heap->gpu_va)) { -+ dev_err(kctx->kbdev->dev, -+ "Failed to allocate a tiler heap context\n"); -+ err = -ENOMEM; -+ } else { -+ err = create_initial_chunks(heap, initial_chunks); -+ if (unlikely(err)) { -+ kbase_csf_heap_context_allocator_free(ctx_alloc, -+ heap->gpu_va); -+ } -+ } -+ -+ if (unlikely(err)) { -+ kfree(heap); -+ } else { -+ struct kbase_csf_tiler_heap_chunk const *first_chunk = -+ list_first_entry(&heap->chunks_list, -+ struct kbase_csf_tiler_heap_chunk, link); -+ -+ kctx->csf.tiler_heaps.nr_of_heaps++; -+ heap->heap_id = kctx->csf.tiler_heaps.nr_of_heaps; -+ list_add(&heap->link, &kctx->csf.tiler_heaps.list); -+ -+ *heap_gpu_va = heap->gpu_va; -+ *first_chunk_va = first_chunk->gpu_va; -+ -+ KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( -+ kctx->kbdev, kctx->id, heap->heap_id, -+ PFN_UP(heap->chunk_size * heap->max_chunks), -+ PFN_UP(heap->chunk_size * heap->chunk_count), -+ heap->max_chunks, heap->chunk_size, heap->chunk_count, -+ heap->target_in_flight, 0); -+ -+ dev_dbg(kctx->kbdev->dev, "Created tiler heap 0x%llX\n", -+ heap->gpu_va); -+ } -+ -+ mutex_unlock(&kctx->csf.tiler_heaps.lock); -+ -+ return err; -+} -+ -+int kbase_csf_tiler_heap_term(struct kbase_context *const kctx, -+ u64 const heap_gpu_va) -+{ -+ int err = 0; -+ struct kbase_csf_tiler_heap *heap = NULL; -+ -+ mutex_lock(&kctx->csf.tiler_heaps.lock); -+ -+ heap = find_tiler_heap(kctx, heap_gpu_va); -+ if (likely(heap)) -+ delete_heap(heap); -+ else -+ err = -EINVAL; -+ -+ mutex_unlock(&kctx->csf.tiler_heaps.lock); -+ -+ return err; -+} -+ -+/** -+ * alloc_new_chunk - Allocate a new chunk for the tiler heap. -+ * -+ * This function will allocate a new chunk for the chunked tiler heap depending -+ * on the settings provided by userspace when the heap was created and the -+ * heap's statistics (like number of render passes in-flight). -+ * -+ * @heap: Pointer to the tiler heap. -+ * @nr_in_flight: Number of render passes that are in-flight, must not be zero. -+ * @pending_frag_count: Number of render passes in-flight with completed vertex/tiler stage. -+ * The minimum value is zero but it must be less or equal to -+ * the total number of render passes in flight -+ * @new_chunk_ptr: Where to store the GPU virtual address & size of the new -+ * chunk allocated for the heap. -+ * -+ * Return: 0 if a new chunk was allocated otherwise an appropriate negative -+ * error code. -+ */ -+static int alloc_new_chunk(struct kbase_csf_tiler_heap *heap, -+ u32 nr_in_flight, u32 pending_frag_count, u64 *new_chunk_ptr) -+{ -+ int err = -ENOMEM; -+ -+ lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); -+ -+ if (WARN_ON(!nr_in_flight) || -+ WARN_ON(pending_frag_count > nr_in_flight)) -+ return -EINVAL; -+ -+ if (nr_in_flight <= heap->target_in_flight) { -+ if (heap->chunk_count < heap->max_chunks) { -+ /* Not exceeded the target number of render passes yet so be -+ * generous with memory. -+ */ -+ err = create_chunk(heap, false); -+ -+ if (likely(!err)) { -+ struct kbase_csf_tiler_heap_chunk *new_chunk = -+ get_last_chunk(heap); -+ if (!WARN_ON(!new_chunk)) { -+ *new_chunk_ptr = -+ encode_chunk_ptr(heap->chunk_size, -+ new_chunk->gpu_va); -+ return 0; -+ } -+ } -+ } else if (pending_frag_count > 0) { -+ err = -EBUSY; -+ } else { -+ err = -ENOMEM; -+ } -+ } else { -+ /* Reached target number of render passes in flight. -+ * Wait for some of them to finish -+ */ -+ err = -EBUSY; -+ } -+ -+ return err; -+} -+ -+int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx, -+ u64 gpu_heap_va, u32 nr_in_flight, u32 pending_frag_count, u64 *new_chunk_ptr) -+{ -+ struct kbase_csf_tiler_heap *heap; -+ int err = -EINVAL; -+ -+ mutex_lock(&kctx->csf.tiler_heaps.lock); -+ -+ heap = find_tiler_heap(kctx, gpu_heap_va); -+ -+ if (likely(heap)) { -+ err = alloc_new_chunk(heap, nr_in_flight, pending_frag_count, -+ new_chunk_ptr); -+ -+ KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( -+ kctx->kbdev, kctx->id, heap->heap_id, -+ PFN_UP(heap->chunk_size * heap->max_chunks), -+ PFN_UP(heap->chunk_size * heap->chunk_count), -+ heap->max_chunks, heap->chunk_size, heap->chunk_count, -+ heap->target_in_flight, nr_in_flight); -+ } -+ -+ mutex_unlock(&kctx->csf.tiler_heaps.lock); -+ -+ return err; -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.h -new file mode 100644 -index 0000000..04c27f7 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.h -@@ -0,0 +1,115 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_TILER_HEAP_H_ -+#define _KBASE_CSF_TILER_HEAP_H_ -+ -+#include <mali_kbase.h> -+ -+/** -+ * kbase_csf_tiler_heap_context_init - Initialize the tiler heaps context for a -+ * GPU address space -+ * -+ * @kctx: Pointer to the kbase context being initialized. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+int kbase_csf_tiler_heap_context_init(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_tiler_heap_context_term - Terminate the tiler heaps context for a -+ * GPU address space -+ * -+ * This function deletes any chunked tiler heaps that weren't deleted before -+ * context termination. -+ * -+ * @kctx: Pointer to the kbase context being terminated. -+ */ -+void kbase_csf_tiler_heap_context_term(struct kbase_context *kctx); -+ -+/** -+ * kbase_csf_tiler_heap_init - Initialize a chunked tiler memory heap. -+ * -+ * @kctx: Pointer to the kbase context in which to allocate resources for the -+ * tiler heap. -+ * @chunk_size: Size of each chunk, in bytes. Must be page-aligned. -+ * @initial_chunks: The initial number of chunks to allocate. Must not be -+ * zero or greater than @max_chunks. -+ * @max_chunks: The maximum number of chunks that the heap should be allowed -+ * to use. Must not be less than @initial_chunks. -+ * @target_in_flight: Number of render-passes that the driver should attempt to -+ * keep in flight for which allocation of new chunks is -+ * allowed. Must not be zero. -+ * @gpu_heap_va: Where to store the GPU virtual address of the context that was -+ * set up for the tiler heap. -+ * @first_chunk_va: Where to store the GPU virtual address of the first chunk -+ * allocated for the heap. This points to the header of the -+ * heap chunk and not to the low address of free memory in it. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+int kbase_csf_tiler_heap_init(struct kbase_context *kctx, -+ u32 chunk_size, u32 initial_chunks, u32 max_chunks, -+ u16 target_in_flight, u64 *gpu_heap_va, -+ u64 *first_chunk_va); -+ -+/** -+ * kbasep_cs_tiler_heap_term - Terminate a chunked tiler memory heap. -+ * -+ * This function will terminate a chunked tiler heap and cause all the chunks -+ * (initial and those added during out-of-memory processing) to be freed. -+ * It is the caller's responsibility to ensure no further operations on this -+ * heap will happen before calling this function. -+ * -+ * @kctx: Pointer to the kbase context in which the tiler heap was initialized. -+ * @gpu_heap_va: The GPU virtual address of the context that was set up for the -+ * tiler heap. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+int kbase_csf_tiler_heap_term(struct kbase_context *kctx, u64 gpu_heap_va); -+ -+/** -+ * kbase_csf_tiler_heap_alloc_new_chunk - Allocate a new chunk for tiler heap. -+ * -+ * This function will allocate a new chunk for the chunked tiler heap depending -+ * on the settings provided by userspace when the heap was created and the -+ * heap's statistics (like number of render passes in-flight). -+ * It would return an appropriate error code if a new chunk couldn't be -+ * allocated. -+ * -+ * @kctx: Pointer to the kbase context in which the tiler heap was initialized. -+ * @gpu_heap_va: GPU virtual address of the heap context. -+ * @nr_in_flight: Number of render passes that are in-flight, must not be zero. -+ * @pending_frag_count: Number of render passes in-flight with completed vertex/tiler stage. -+ * The minimum value is zero but it must be less or equal to -+ * the total number of render passes in flight -+ * @new_chunk_ptr: Where to store the GPU virtual address & size of the new -+ * chunk allocated for the heap. -+ * -+ * Return: 0 if a new chunk was allocated otherwise an appropriate negative -+ * error code (like -EBUSY when a free chunk is expected to be -+ * available upon completion of a render pass and -EINVAL when -+ * invalid value was passed for one of the argument). -+ */ -+int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx, -+ u64 gpu_heap_va, u32 nr_in_flight, u32 pending_frag_count, u64 *new_chunk_ptr); -+#endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_debugfs.c -new file mode 100644 -index 0000000..f46beed ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_debugfs.c -@@ -0,0 +1,106 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_csf_tiler_heap_debugfs.h" -+#include "mali_kbase_csf_tiler_heap_def.h" -+#include <mali_kbase.h> -+#include <linux/seq_file.h> -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ -+/** -+ * kbasep_csf_tiler_heap_debugfs_show() - Print tiler heap information for per context -+ * -+ * @file: The seq_file for printing to -+ * @data: The debugfs dentry private data, a pointer to kbase_context -+ * -+ * Return: Negative error code or 0 on success. -+ */ -+static int kbasep_csf_tiler_heap_debugfs_show(struct seq_file *file, void *data) -+{ -+ struct kbase_context *kctx = file->private; -+ struct kbase_csf_tiler_heap_context *tiler_heaps_p = &kctx->csf.tiler_heaps; -+ struct kbase_csf_tiler_heap *heap; -+ struct kbase_csf_tiler_heap_chunk *chunk; -+ -+ seq_printf(file, "MALI_CSF_TILER_HEAP_DEBUGFS_VERSION: v%u\n", MALI_CSF_TILER_HEAP_DEBUGFS_VERSION); -+ -+ mutex_lock(&tiler_heaps_p->lock); -+ -+ list_for_each_entry(heap, &tiler_heaps_p->list, link) { -+ if (heap->kctx != kctx) -+ continue; -+ -+ seq_printf(file, "HEAP(gpu_va = 0x%llx):\n", heap->gpu_va); -+ seq_printf(file, "\tchunk_size = %u\n", heap->chunk_size); -+ seq_printf(file, "\tchunk_count = %u\n", heap->chunk_count); -+ seq_printf(file, "\tmax_chunks = %u\n", heap->max_chunks); -+ seq_printf(file, "\ttarget_in_flight = %u\n", heap->target_in_flight); -+ -+ list_for_each_entry(chunk, &heap->chunks_list, link) -+ seq_printf(file, "\t\tchunk gpu_va = 0x%llx\n", -+ chunk->gpu_va); -+ } -+ -+ mutex_unlock(&tiler_heaps_p->lock); -+ -+ return 0; -+} -+ -+static int kbasep_csf_tiler_heap_debugfs_open(struct inode *in, struct file *file) -+{ -+ return single_open(file, kbasep_csf_tiler_heap_debugfs_show, in->i_private); -+} -+ -+static const struct file_operations kbasep_csf_tiler_heap_debugfs_fops = { -+ .open = kbasep_csf_tiler_heap_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+void kbase_csf_tiler_heap_debugfs_init(struct kbase_context *kctx) -+{ -+ struct dentry *file; -+ -+ if (WARN_ON(!kctx || IS_ERR_OR_NULL(kctx->kctx_dentry))) -+ return; -+ -+ file = debugfs_create_file("tiler_heaps", 0444, kctx->kctx_dentry, -+ kctx, &kbasep_csf_tiler_heap_debugfs_fops); -+ -+ if (IS_ERR_OR_NULL(file)) { -+ dev_warn(kctx->kbdev->dev, -+ "Unable to create tiler heap debugfs entry"); -+ } -+} -+ -+ -+#else -+/* -+ * Stub functions for when debugfs is disabled -+ */ -+void kbase_csf_tiler_heap_debugfs_init(struct kbase_context *kctx) -+{ -+} -+ -+#endif /* CONFIG_DEBUG_FS */ -+ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_debugfs.h -new file mode 100644 -index 0000000..92ae91a ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_debugfs.h -@@ -0,0 +1,37 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ -+#define _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ -+ -+/* Forward declaration */ -+struct kbase_context; -+ -+#define MALI_CSF_TILER_HEAP_DEBUGFS_VERSION 0 -+ -+/** -+ * kbase_csf_tiler_heap_debugfs_init() - Create a debugfs entry for per context tiler heap -+ * -+ * @kctx: The kbase_context for which to create the debugfs entry -+ */ -+void kbase_csf_tiler_heap_debugfs_init(struct kbase_context *kctx); -+ -+#endif /* _KBASE_CSF_TILER_HEAP_DEBUGFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_def.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_def.h -new file mode 100644 -index 0000000..fb439cf ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap_def.h -@@ -0,0 +1,114 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_TILER_HEAP_DEF_H_ -+#define _KBASE_CSF_TILER_HEAP_DEF_H_ -+ -+#include <mali_kbase.h> -+ -+/* Size of a tiler heap chunk header, in bytes. */ -+#define CHUNK_HDR_SIZE ((size_t)64) -+ -+/* Bit-position of the next chunk's size when stored in a chunk header. */ -+#define CHUNK_HDR_NEXT_SIZE_POS (0) -+ -+/* Bit-position of the next chunk's address when stored in a chunk header. */ -+#define CHUNK_HDR_NEXT_ADDR_POS (12) -+ -+/* Bitmask of the next chunk's size when stored in a chunk header. */ -+#define CHUNK_HDR_NEXT_SIZE_MASK (((u64)1 << CHUNK_HDR_NEXT_ADDR_POS) - 1u) -+ -+/* Bitmask of the address of the next chunk when stored in a chunk header. */ -+#define CHUNK_HDR_NEXT_ADDR_MASK (~CHUNK_HDR_NEXT_SIZE_MASK) -+ -+/* Right-shift before storing the next chunk's size in a chunk header. */ -+#define CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT (12) -+ -+/* Right-shift before storing the next chunk's address in a chunk header. */ -+#define CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT (12) -+ -+/* Bitmask of valid chunk sizes. This is also the maximum chunk size, in bytes. -+ */ -+#define CHUNK_SIZE_MASK \ -+ ((CHUNK_HDR_NEXT_SIZE_MASK >> CHUNK_HDR_NEXT_SIZE_POS) << \ -+ CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT) -+ -+/* Bitmask of valid chunk addresses. This is also the highest address. */ -+#define CHUNK_ADDR_MASK \ -+ ((CHUNK_HDR_NEXT_ADDR_MASK >> CHUNK_HDR_NEXT_ADDR_POS) << \ -+ CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT) -+ -+/** -+ * struct kbase_csf_tiler_heap_chunk - A tiler heap chunk managed by the kernel -+ * -+ * Chunks are allocated upon initialization of a tiler heap or in response to -+ * out-of-memory events from the firmware. Chunks are always fully backed by -+ * physical memory to avoid the overhead of processing GPU page faults. The -+ * allocated GPU memory regions are linked together independent of the list of -+ * kernel objects of this type. -+ * -+ * @link: Link to this chunk in a list of chunks belonging to a -+ * @kbase_csf_tiler_heap. -+ * @region: Pointer to the GPU memory region allocated for the chunk. -+ * @gpu_va: GPU virtual address of the start of the memory region. -+ * This points to the header of the chunk and not to the low address -+ * of free memory within it. -+ */ -+struct kbase_csf_tiler_heap_chunk { -+ struct list_head link; -+ struct kbase_va_region *region; -+ u64 gpu_va; -+}; -+ -+/** -+ * struct kbase_csf_tiler_heap - A tiler heap managed by the kernel -+ * -+ * @kctx: Pointer to the kbase context with which this heap is -+ * associated. -+ * @link: Link to this heap in a list of tiler heaps belonging to -+ * the @kbase_csf_tiler_heap_context. -+ * @chunk_size: Size of each chunk, in bytes. Must be page-aligned. -+ * @chunk_count: The number of chunks currently allocated. Must not be -+ * zero or greater than @max_chunks. -+ * @max_chunks: The maximum number of chunks that the heap should be -+ * allowed to use. Must not be less than @chunk_count. -+ * @target_in_flight: Number of render-passes that the driver should attempt -+ * to keep in flight for which allocation of new chunks is -+ * allowed. Must not be zero. -+ * @gpu_va: The GPU virtual address of the heap context structure that -+ * was allocated for the firmware. This is also used to -+ * uniquely identify the heap. -+ * @heap_id: Unique id representing the heap, assigned during heap -+ * initialization. -+ * @chunks_list: Linked list of allocated chunks. -+ */ -+struct kbase_csf_tiler_heap { -+ struct kbase_context *kctx; -+ struct list_head link; -+ u32 chunk_size; -+ u32 chunk_count; -+ u32 max_chunks; -+ u16 target_in_flight; -+ u64 gpu_va; -+ u64 heap_id; -+ struct list_head chunks_list; -+}; -+#endif /* !_KBASE_CSF_TILER_HEAP_DEF_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_timeout.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_timeout.c -new file mode 100644 -index 0000000..4d93fe5 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_timeout.c -@@ -0,0 +1,178 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <linux/kernel.h> -+#include <linux/device.h> -+#include <linux/mutex.h> -+#include <linux/sysfs.h> -+#include <linux/of.h> -+ -+#include "mali_kbase.h" -+#include "mali_kbase_config_defaults.h" -+#include "mali_kbase_csf_firmware.h" -+#include "mali_kbase_csf_timeout.h" -+#include "mali_kbase_reset_gpu.h" -+#include "backend/gpu/mali_kbase_pm_internal.h" -+ -+/** -+ * set_timeout - set a new global progress timeout. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * @timeout: the maximum number of GPU cycles without forward progress to allow -+ * to elapse before terminating a GPU command queue group. -+ * -+ * Return: 0 on success, or negative on failure -+ * (e.g. -ERANGE if the requested timeout is too large). -+ */ -+static int set_timeout(struct kbase_device *const kbdev, u64 const timeout) -+{ -+ if (timeout > GLB_PROGRESS_TIMER_TIMEOUT_MAX) { -+ dev_err(kbdev->dev, "Timeout %llu is too large.\n", timeout); -+ return -ERANGE; -+ } -+ -+ dev_dbg(kbdev->dev, "New progress timeout: %llu cycles\n", timeout); -+ -+ atomic64_set(&kbdev->csf.progress_timeout, timeout); -+ -+ return 0; -+} -+ -+/** -+ * progress_timeout_store - Store the progress_timeout device attribute. -+ * @dev: The device that has the attribute. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The value written to the sysfs file. -+ * @count: The number of bytes written to the sysfs file. -+ * -+ * This function is called when the progress_timeout sysfs file is written to. -+ * It checks the data written, and if valid updates the progress timeout value. -+ * The function also checks gpu reset status, if the gpu is in reset process, -+ * the function will return an error code (-EBUSY), and no change for timeout -+ * value. -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t progress_timeout_store(struct device * const dev, -+ struct device_attribute * const attr, const char * const buf, -+ size_t const count) -+{ -+ struct kbase_device *const kbdev = dev_get_drvdata(dev); -+ int err; -+ u64 timeout; -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ err = kbase_reset_gpu_try_prevent(kbdev); -+ if (err) { -+ dev_warn(kbdev->dev, -+ "Couldn't process progress_timeout write operation for GPU reset.\n"); -+ return -EBUSY; -+ } -+ -+ err = kstrtou64(buf, 0, &timeout); -+ if (err) -+ dev_err(kbdev->dev, -+ "Couldn't process progress_timeout write operation.\n" -+ "Use format <progress_timeout>\n"); -+ else -+ err = set_timeout(kbdev, timeout); -+ -+ if (!err) { -+ kbase_csf_scheduler_pm_active(kbdev); -+ -+ err = kbase_pm_wait_for_desired_state(kbdev); -+ if (!err) -+ err = kbase_csf_firmware_set_timeout(kbdev, timeout); -+ -+ kbase_csf_scheduler_pm_idle(kbdev); -+ } -+ -+ kbase_reset_gpu_allow(kbdev); -+ if (err) -+ return err; -+ -+ return count; -+} -+ -+/** -+ * progress_timeout_show - Show the progress_timeout device attribute. -+ * @dev: The device that has the attribute. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the global timeout. -+ * -+ * This function is called to get the progress timeout value. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t progress_timeout_show(struct device * const dev, -+ struct device_attribute * const attr, char * const buf) -+{ -+ struct kbase_device *const kbdev = dev_get_drvdata(dev); -+ int err; -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ err = scnprintf(buf, PAGE_SIZE, "%llu\n", kbase_csf_timeout_get(kbdev)); -+ -+ return err; -+ -+} -+ -+static DEVICE_ATTR(progress_timeout, 0644, progress_timeout_show, -+ progress_timeout_store); -+ -+int kbase_csf_timeout_init(struct kbase_device *const kbdev) -+{ -+ u64 timeout = DEFAULT_PROGRESS_TIMEOUT; -+ int err; -+ -+#if IS_ENABLED(CONFIG_OF) -+ err = of_property_read_u64(kbdev->dev->of_node, -+ "progress_timeout", &timeout); -+ if (!err) -+ dev_info(kbdev->dev, "Found progress_timeout = %llu in Devicetree\n", -+ timeout); -+#endif -+ -+ err = set_timeout(kbdev, timeout); -+ if (err) -+ return err; -+ -+ err = sysfs_create_file(&kbdev->dev->kobj, -+ &dev_attr_progress_timeout.attr); -+ if (err) -+ dev_err(kbdev->dev, "SysFS file creation failed\n"); -+ -+ return err; -+} -+ -+void kbase_csf_timeout_term(struct kbase_device * const kbdev) -+{ -+ sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_progress_timeout.attr); -+} -+ -+u64 kbase_csf_timeout_get(struct kbase_device *const kbdev) -+{ -+ return atomic64_read(&kbdev->csf.progress_timeout); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_timeout.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_timeout.h -new file mode 100644 -index 0000000..b406eaa ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_timeout.h -@@ -0,0 +1,66 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_TIMEOUT_H_ -+#define _KBASE_CSF_TIMEOUT_H_ -+ -+struct kbase_device; -+ -+/** -+ * kbase_csf_timeout_init - Initialize the progress timeout. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * Must be zero-initialized. -+ * -+ * The progress timeout is the number of GPU clock cycles allowed to elapse -+ * before the driver terminates a GPU command queue group in which a task is -+ * making no forward progress on an endpoint (e.g. a shader core). This function -+ * determines the initial value and also creates a sysfs file to allow the -+ * timeout to be reconfigured later. -+ * -+ * Reconfigures the global firmware interface to enable the current timeout. -+ * -+ * Return: 0 on success, or negative on failure. -+ */ -+int kbase_csf_timeout_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_timeout_term - Terminate the progress timeout. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Removes the sysfs file which allowed the timeout to be reconfigured. -+ * Does nothing if called on a zero-initialized object. -+ */ -+void kbase_csf_timeout_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_timeout_get - get the current global progress timeout. -+ * -+ * @kbdev: Instance of a GPU platform device that implements a CSF interface. -+ * -+ * Return: the maximum number of GPU cycles that is allowed to elapse without -+ * forward progress before the driver terminates a GPU command queue -+ * group. -+ */ -+u64 kbase_csf_timeout_get(struct kbase_device *const kbdev); -+ -+#endif /* _KBASE_CSF_TIMEOUT_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tl_reader.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tl_reader.c -new file mode 100644 -index 0000000..1824c2d ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tl_reader.c -@@ -0,0 +1,534 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_csf_tl_reader.h" -+ -+#include "mali_kbase_csf_trace_buffer.h" -+#include "mali_kbase_reset_gpu.h" -+ -+#include "tl/mali_kbase_tlstream.h" -+#include "tl/mali_kbase_tl_serialize.h" -+#include "tl/mali_kbase_tracepoints.h" -+ -+#include "mali_kbase_pm.h" -+#include "mali_kbase_hwaccess_time.h" -+ -+#include <linux/gcd.h> -+#include <linux/math64.h> -+#include <asm/arch_timer.h> -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+#include "tl/mali_kbase_timeline_priv.h" -+#include <linux/debugfs.h> -+ -+#if (KERNEL_VERSION(4, 7, 0) > LINUX_VERSION_CODE) -+#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE -+#endif -+#endif -+ -+/* Name of the CSFFW timeline tracebuffer. */ -+#define KBASE_CSFFW_TRACEBUFFER_NAME "timeline" -+/* Name of the timeline header metatadata */ -+#define KBASE_CSFFW_TIMELINE_HEADER_NAME "timeline_header" -+ -+/** -+ * struct kbase_csffw_tl_message - CSFFW timeline message. -+ * -+ * @msg_id: Message ID. -+ * @timestamp: Timestamp of the event. -+ * @cycle_counter: Cycle number of the event. -+ * -+ * Contain fields that are common for all CSFFW timeline messages. -+ */ -+struct kbase_csffw_tl_message { -+ u32 msg_id; -+ u64 timestamp; -+ u64 cycle_counter; -+} __packed __aligned(4); -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+static int kbase_csf_tl_debugfs_poll_interval_read(void *data, u64 *val) -+{ -+ struct kbase_device *kbdev = (struct kbase_device *)data; -+ struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader; -+ -+ *val = self->timer_interval; -+ -+ return 0; -+} -+ -+static int kbase_csf_tl_debugfs_poll_interval_write(void *data, u64 val) -+{ -+ struct kbase_device *kbdev = (struct kbase_device *)data; -+ struct kbase_csf_tl_reader *self = &kbdev->timeline->csf_tl_reader; -+ -+ if (val > KBASE_CSF_TL_READ_INTERVAL_MAX || val < KBASE_CSF_TL_READ_INTERVAL_MIN) { -+ return -EINVAL; -+ } -+ -+ self->timer_interval = (u32)val; -+ -+ return 0; -+} -+ -+DEFINE_DEBUGFS_ATTRIBUTE(kbase_csf_tl_poll_interval_fops, -+ kbase_csf_tl_debugfs_poll_interval_read, -+ kbase_csf_tl_debugfs_poll_interval_write, "%llu\n"); -+ -+ -+void kbase_csf_tl_reader_debugfs_init(struct kbase_device *kbdev) -+{ -+ debugfs_create_file("csf_tl_poll_interval_in_ms", S_IRUGO | S_IWUSR, -+ kbdev->debugfs_instr_directory, kbdev, -+ &kbase_csf_tl_poll_interval_fops); -+ -+} -+#endif -+ -+/** -+ * get_cpu_gpu_time() - Get current CPU and GPU timestamps. -+ * -+ * @kbdev: Kbase device. -+ * @cpu_ts: Output CPU timestamp. -+ * @gpu_ts: Output GPU timestamp. -+ * @gpu_cycle: Output GPU cycle counts. -+ */ -+static void get_cpu_gpu_time( -+ struct kbase_device *kbdev, -+ u64 *cpu_ts, -+ u64 *gpu_ts, -+ u64 *gpu_cycle) -+{ -+ struct timespec64 ts; -+ -+ kbase_pm_context_active(kbdev); -+ kbase_backend_get_gpu_time(kbdev, gpu_cycle, gpu_ts, &ts); -+ kbase_pm_context_idle(kbdev); -+ -+ if (cpu_ts) -+ *cpu_ts = ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; -+} -+ -+ -+/** -+ * kbase_ts_converter_init() - Initialize system timestamp converter. -+ * -+ * @self: System Timestamp Converter instance. -+ * @kbdev: Kbase device pointer -+ * -+ * Return: Zero on success, -1 otherwise. -+ */ -+static int kbase_ts_converter_init( -+ struct kbase_ts_converter *self, -+ struct kbase_device *kbdev) -+{ -+ u64 cpu_ts = 0; -+ u64 gpu_ts = 0; -+ u64 freq; -+ u64 common_factor; -+ -+ get_cpu_gpu_time(kbdev, &cpu_ts, &gpu_ts, NULL); -+ freq = arch_timer_get_cntfrq(); -+ -+ if (!freq) { -+ dev_warn(kbdev->dev, "arch_timer_get_rate() is zero!"); -+ return -1; -+ } -+ -+ common_factor = gcd(NSEC_PER_SEC, freq); -+ -+ self->multiplier = div64_u64(NSEC_PER_SEC, common_factor); -+ self->divisor = div64_u64(freq, common_factor); -+ self->offset = -+ cpu_ts - div64_u64(gpu_ts * self->multiplier, self->divisor); -+ -+ return 0; -+} -+ -+/** -+ * kbase_ts_converter_convert() - Convert GPU timestamp to CPU timestamp. -+ * -+ * @self: System Timestamp Converter instance. -+ * @gpu_ts: System timestamp value to converter. -+ * -+ * Return: The CPU timestamp. -+ */ -+static void kbase_ts_converter_convert( -+ const struct kbase_ts_converter *self, -+ u64 *gpu_ts) -+{ -+ u64 old_gpu_ts = *gpu_ts; -+ *gpu_ts = div64_u64(old_gpu_ts * self->multiplier, -+ self->divisor) + self->offset; -+} -+ -+/** -+ * tl_reader_overflow_notify() - Emit stream overflow tracepoint. -+ * -+ * @self: CSFFW TL Reader instance. -+ * @msg_buf_start: Start of the message. -+ * @msg_buf_end: End of the message buffer. -+ */ -+static void tl_reader_overflow_notify( -+ const struct kbase_csf_tl_reader *self, -+ u8 *const msg_buf_start, -+ u8 *const msg_buf_end) -+{ -+ struct kbase_device *kbdev = self->kbdev; -+ struct kbase_csffw_tl_message message = {0}; -+ -+ /* Reuse the timestamp and cycle count from current event if possible */ -+ if (msg_buf_start + sizeof(message) <= msg_buf_end) -+ memcpy(&message, msg_buf_start, sizeof(message)); -+ -+ KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( -+ kbdev, message.timestamp, message.cycle_counter); -+} -+ -+/** -+ * tl_reader_overflow_check() - Check if an overflow has happened -+ * -+ * @self: CSFFW TL Reader instance. -+ * @event_id: Incoming event id. -+ * -+ * Return: True, if an overflow has happened, False otherwise. -+ */ -+static bool tl_reader_overflow_check( -+ struct kbase_csf_tl_reader *self, -+ u16 event_id) -+{ -+ struct kbase_device *kbdev = self->kbdev; -+ bool has_overflow = false; -+ -+ /* 0 is a special event_id and reserved for the very first tracepoint -+ * after reset, we should skip overflow check when reset happened. -+ */ -+ if (event_id != 0) { -+ has_overflow = self->got_first_event -+ && self->expected_event_id != event_id; -+ -+ if (has_overflow) -+ dev_warn(kbdev->dev, -+ "CSFFW overflow, event_id: %u, expected: %u.", -+ event_id, self->expected_event_id); -+ } -+ -+ self->got_first_event = true; -+ self->expected_event_id = event_id + 1; -+ /* When event_id reaches its max value, it skips 0 and wraps to 1. */ -+ if (self->expected_event_id == 0) -+ self->expected_event_id++; -+ -+ return has_overflow; -+} -+ -+/** -+ * tl_reader_reset() - Reset timeline tracebuffer reader state machine. -+ * -+ * @self: CSFFW TL Reader instance. -+ * -+ * Reset the reader to the default state, i.e. set all the -+ * mutable fields to zero. -+ */ -+static void tl_reader_reset(struct kbase_csf_tl_reader *self) -+{ -+ self->got_first_event = false; -+ self->is_active = false; -+ self->expected_event_id = 0; -+ self->tl_header.btc = 0; -+} -+ -+int kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self) -+{ -+ int ret = 0; -+ struct kbase_device *kbdev = self->kbdev; -+ struct kbase_tlstream *stream = self->stream; -+ -+ u8 *read_buffer = self->read_buffer; -+ const size_t read_buffer_size = sizeof(self->read_buffer); -+ -+ u32 bytes_read; -+ u8 *csffw_data_begin; -+ u8 *csffw_data_end; -+ u8 *csffw_data_it; -+ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&self->read_lock, flags); -+ -+ /* If not running, early exit. */ -+ if (!self->is_active) { -+ spin_unlock_irqrestore(&self->read_lock, flags); -+ return -EBUSY; -+ } -+ -+ /* Copying the whole buffer in a single shot. We assume -+ * that the buffer will not contain partially written messages. -+ */ -+ bytes_read = kbase_csf_firmware_trace_buffer_read_data( -+ self->trace_buffer, read_buffer, read_buffer_size); -+ csffw_data_begin = read_buffer; -+ csffw_data_end = read_buffer + bytes_read; -+ -+ for (csffw_data_it = csffw_data_begin; -+ csffw_data_it < csffw_data_end;) { -+ u32 event_header; -+ u16 event_id; -+ u16 event_size; -+ unsigned long acq_flags; -+ char *buffer; -+ -+ /* Can we safely read event_id? */ -+ if (csffw_data_it + sizeof(event_header) > csffw_data_end) { -+ dev_warn( -+ kbdev->dev, -+ "Unable to parse CSFFW tracebuffer event header."); -+ ret = -EBUSY; -+ break; -+ } -+ -+ /* Read and parse the event header. */ -+ memcpy(&event_header, csffw_data_it, sizeof(event_header)); -+ event_id = (event_header >> 0) & 0xFFFF; -+ event_size = (event_header >> 16) & 0xFFFF; -+ csffw_data_it += sizeof(event_header); -+ -+ /* Detect if an overflow has happened. */ -+ if (tl_reader_overflow_check(self, event_id)) -+ tl_reader_overflow_notify(self, -+ csffw_data_it, -+ csffw_data_end); -+ -+ /* Can we safely read the message body? */ -+ if (csffw_data_it + event_size > csffw_data_end) { -+ dev_warn(kbdev->dev, -+ "event_id: %u, can't read with event_size: %u.", -+ event_id, event_size); -+ ret = -EBUSY; -+ break; -+ } -+ -+ /* Convert GPU timestamp to CPU timestamp. */ -+ { -+ struct kbase_csffw_tl_message *msg = -+ (struct kbase_csffw_tl_message *) csffw_data_it; -+ kbase_ts_converter_convert( -+ &self->ts_converter, -+ &msg->timestamp); -+ } -+ -+ /* Copy the message out to the tl_stream. */ -+ buffer = kbase_tlstream_msgbuf_acquire( -+ stream, event_size, &acq_flags); -+ kbasep_serialize_bytes(buffer, 0, csffw_data_it, event_size); -+ kbase_tlstream_msgbuf_release(stream, acq_flags); -+ csffw_data_it += event_size; -+ } -+ -+ spin_unlock_irqrestore(&self->read_lock, flags); -+ return ret; -+} -+ -+static void kbasep_csf_tl_reader_read_callback(struct timer_list *timer) -+{ -+ struct kbase_csf_tl_reader *self = -+ container_of(timer, struct kbase_csf_tl_reader, read_timer); -+ -+ int rcode; -+ -+ kbase_csf_tl_reader_flush_buffer(self); -+ -+ rcode = mod_timer(&self->read_timer, -+ jiffies + msecs_to_jiffies(self->timer_interval)); -+ -+ CSTD_UNUSED(rcode); -+} -+ -+/** -+ * tl_reader_init_late() - Late CSFFW TL Reader initialization. -+ * -+ * @self: CSFFW TL Reader instance. -+ * @kbdev: Kbase device. -+ * -+ * Late initialization is done once at kbase_csf_tl_reader_start() time. -+ * This is because the firmware image is not parsed -+ * by the kbase_csf_tl_reader_init() time. -+ * -+ * Return: Zero on success, -1 otherwise. -+ */ -+static int tl_reader_init_late( -+ struct kbase_csf_tl_reader *self, -+ struct kbase_device *kbdev) -+{ -+ struct firmware_trace_buffer *tb; -+ size_t hdr_size = 0; -+ const char *hdr = NULL; -+ -+ if (self->kbdev) -+ return 0; -+ -+ tb = kbase_csf_firmware_get_trace_buffer( -+ kbdev, KBASE_CSFFW_TRACEBUFFER_NAME); -+ hdr = kbase_csf_firmware_get_timeline_metadata( -+ kbdev, KBASE_CSFFW_TIMELINE_HEADER_NAME, &hdr_size); -+ -+ if (!tb) { -+ dev_warn( -+ kbdev->dev, -+ "'%s' tracebuffer is not present in the firmware image.", -+ KBASE_CSFFW_TRACEBUFFER_NAME); -+ return -1; -+ } -+ -+ if (!hdr) { -+ dev_warn( -+ kbdev->dev, -+ "'%s' timeline metadata is not present in the firmware image.", -+ KBASE_CSFFW_TIMELINE_HEADER_NAME); -+ return -1; -+ } -+ -+ if (kbase_ts_converter_init(&self->ts_converter, kbdev)) { -+ return -1; -+ } -+ -+ self->kbdev = kbdev; -+ self->trace_buffer = tb; -+ self->tl_header.data = hdr; -+ self->tl_header.size = hdr_size; -+ -+ return 0; -+} -+ -+/** -+ * tl_reader_update_enable_bit() - Update the first bit of a CSFFW tracebuffer. -+ * -+ * @self: CSFFW TL Reader instance. -+ * @value: The value to set. -+ * -+ * Update the first bit of a CSFFW tracebufer and then reset the GPU. -+ * This is to make these changes visible to the MCU. -+ * -+ * Return: 0 on success, or negative error code for failure. -+ */ -+static int tl_reader_update_enable_bit( -+ struct kbase_csf_tl_reader *self, -+ bool value) -+{ -+ int err = 0; -+ -+ err = kbase_csf_firmware_trace_buffer_update_trace_enable_bit( -+ self->trace_buffer, 0, value); -+ -+ return err; -+} -+ -+void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self, -+ struct kbase_tlstream *stream) -+{ -+ self->timer_interval = KBASE_CSF_TL_READ_INTERVAL_DEFAULT; -+ -+ kbase_timer_setup(&self->read_timer, -+ kbasep_csf_tl_reader_read_callback); -+ -+ self->stream = stream; -+ -+ /* This will be initialized by tl_reader_init_late() */ -+ self->kbdev = NULL; -+ self->trace_buffer = NULL; -+ self->tl_header.data = NULL; -+ self->tl_header.size = 0; -+ -+ spin_lock_init(&self->read_lock); -+ -+ tl_reader_reset(self); -+} -+ -+void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self) -+{ -+ del_timer_sync(&self->read_timer); -+} -+ -+int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, -+ struct kbase_device *kbdev) -+{ -+ int rcode; -+ -+ /* If already running, early exit. */ -+ if (self->is_active) -+ return 0; -+ -+ if (tl_reader_init_late(self, kbdev)) { -+ return -EINVAL; -+ } -+ -+ tl_reader_reset(self); -+ -+ self->is_active = true; -+ /* Set bytes to copy to the header size. This is to trigger copying -+ * of the header to the user space. -+ */ -+ self->tl_header.btc = self->tl_header.size; -+ -+ /* Enable the tracebuffer on the CSFFW side. */ -+ rcode = tl_reader_update_enable_bit(self, true); -+ if (rcode != 0) -+ return rcode; -+ -+ rcode = mod_timer(&self->read_timer, -+ jiffies + msecs_to_jiffies(self->timer_interval)); -+ -+ return 0; -+} -+ -+void kbase_csf_tl_reader_stop(struct kbase_csf_tl_reader *self) -+{ -+ unsigned long flags; -+ -+ /* If is not running, early exit. */ -+ if (!self->is_active) -+ return; -+ -+ /* Disable the tracebuffer on the CSFFW side. */ -+ tl_reader_update_enable_bit(self, false); -+ -+ del_timer_sync(&self->read_timer); -+ -+ spin_lock_irqsave(&self->read_lock, flags); -+ -+ tl_reader_reset(self); -+ -+ spin_unlock_irqrestore(&self->read_lock, flags); -+} -+ -+void kbase_csf_tl_reader_reset(struct kbase_csf_tl_reader *self) -+{ -+ u64 gpu_cycle = 0; -+ struct kbase_device *kbdev = self->kbdev; -+ -+ if (!kbdev) -+ return; -+ -+ kbase_csf_tl_reader_flush_buffer(self); -+ -+ get_cpu_gpu_time(kbdev, NULL, NULL, &gpu_cycle); -+ KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET(kbdev, gpu_cycle); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tl_reader.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tl_reader.h -new file mode 100644 -index 0000000..1b0fcd7 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tl_reader.h -@@ -0,0 +1,185 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSFFW_TL_READER_H_ -+#define _KBASE_CSFFW_TL_READER_H_ -+ -+#include <linux/spinlock.h> -+#include <linux/timer.h> -+#include <asm/page.h> -+ -+/* The number of pages used for CSFFW trace buffer. Can be tweaked. */ -+#define KBASE_CSF_TL_BUFFER_NR_PAGES 128 -+/* CSFFW Timeline read polling minimum period in milliseconds. */ -+#define KBASE_CSF_TL_READ_INTERVAL_MIN 20 -+/* CSFFW Timeline read polling default period in milliseconds. */ -+#define KBASE_CSF_TL_READ_INTERVAL_DEFAULT 200 -+/* CSFFW Timeline read polling maximum period in milliseconds. */ -+#define KBASE_CSF_TL_READ_INTERVAL_MAX (60*1000) -+ -+struct firmware_trace_buffer; -+struct kbase_tlstream; -+struct kbase_device; -+ -+/** -+ * struct kbase_ts_converter - -+ * System timestamp to CPU timestamp converter state. -+ * -+ * @multiplier: Numerator of the converter's fraction. -+ * @divisor: Denominator of the converter's fraction. -+ * @offset: Converter's offset term. -+ * -+ * According to Generic timer spec, system timer: -+ * - Increments at a fixed frequency -+ * - Starts operating from zero -+ * -+ * Hence CPU time is a linear function of System Time. -+ * -+ * CPU_ts = alpha * SYS_ts + beta -+ * -+ * Where -+ * - alpha = 10^9/SYS_ts_freq -+ * - beta is calculated by two timer samples taken at the same time: -+ * beta = CPU_ts_s - SYS_ts_s * alpha -+ * -+ * Since alpha is a rational number, we minimizing possible -+ * rounding error by simplifying the ratio. Thus alpha is stored -+ * as a simple `multiplier / divisor` ratio. -+ * -+ */ -+struct kbase_ts_converter { -+ u64 multiplier; -+ u64 divisor; -+ s64 offset; -+}; -+ -+/** -+ * struct kbase_csf_tl_reader - CSFFW timeline reader state. -+ * -+ * @read_timer: Timer used for periodical tracebufer reading. -+ * @timer_interval: Timer polling period in milliseconds. -+ * @stream: Timeline stream where to the tracebuffer content -+ * is copied. -+ * @kbdev: KBase device. -+ * @trace_buffer: CSF Firmware timeline tracebuffer. -+ * @tl_header: CSFFW Timeline header -+ * @tl_header.data: CSFFW Timeline header content. -+ * @tl_header.size: CSFFW Timeline header size. -+ * @tl_header.btc: CSFFW Timeline header remaining bytes to copy to -+ * the user space. -+ * @ts_converter: Timestamp converter state. -+ * @got_first_event: True, if a CSFFW timelime session has been enabled -+ * and the first event was received. -+ * @is_active: True, if a CSFFW timelime session has been enabled. -+ * @expected_event_id: The last 16 bit event ID received from CSFFW. It -+ * is only valid when got_first_event is true. -+ * @read_buffer: Temporary buffer used for CSFFW timeline data -+ * reading from the tracebufer. -+ * @read_lock: CSFFW timeline reader lock. -+ */ -+struct kbase_csf_tl_reader { -+ struct timer_list read_timer; -+ u32 timer_interval; -+ struct kbase_tlstream *stream; -+ -+ struct kbase_device *kbdev; -+ struct firmware_trace_buffer *trace_buffer; -+ struct { -+ const char *data; -+ size_t size; -+ size_t btc; -+ } tl_header; -+ struct kbase_ts_converter ts_converter; -+ -+ bool got_first_event; -+ bool is_active; -+ u16 expected_event_id; -+ -+ u8 read_buffer[PAGE_SIZE * KBASE_CSF_TL_BUFFER_NR_PAGES]; -+ spinlock_t read_lock; -+}; -+ -+/** -+ * kbase_csf_tl_reader_init() - Initialize CSFFW Timelime Stream Reader. -+ * -+ * @self: CSFFW TL Reader instance. -+ * @stream: Destination timeline stream. -+ */ -+void kbase_csf_tl_reader_init(struct kbase_csf_tl_reader *self, -+ struct kbase_tlstream *stream); -+ -+/** -+ * kbase_csf_tl_reader_term() - Terminate CSFFW Timelime Stream Reader. -+ * -+ * @self: CSFFW TL Reader instance. -+ */ -+void kbase_csf_tl_reader_term(struct kbase_csf_tl_reader *self); -+ -+/** -+ * kbase_csf_tl_reader_flush_buffer() - -+ * Flush trace from buffer into CSFFW timeline stream. -+ * -+ * @self: CSFFW TL Reader instance. -+ * -+ * Return: Zero on success, negative error code (EBUSY) otherwise -+ */ -+ -+int kbase_csf_tl_reader_flush_buffer(struct kbase_csf_tl_reader *self); -+ -+/** -+ * kbase_csf_tl_reader_start() - -+ * Start asynchronous copying of CSFFW timeline stream. -+ * -+ * @self: CSFFW TL Reader instance. -+ * @kbdev: Kbase device. -+ * -+ * Return: zero on success, a negative error code otherwise. -+ */ -+int kbase_csf_tl_reader_start(struct kbase_csf_tl_reader *self, -+ struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_tl_reader_stop() - -+ * Stop asynchronous copying of CSFFW timeline stream. -+ * -+ * @self: CSFFW TL Reader instance. -+ */ -+void kbase_csf_tl_reader_stop(struct kbase_csf_tl_reader *self); -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+/** -+ * kbase_csf_tl_reader_debugfs_init() - -+ * Initialize debugfs for CSFFW Timelime Stream Reader. -+ * -+ * @kbdev: Kbase device. -+ */ -+void kbase_csf_tl_reader_debugfs_init(struct kbase_device *kbdev); -+#endif -+ -+/** -+ * kbase_csf_tl_reader_reset() - -+ * Reset CSFFW timeline reader, it should be called before reset CSFFW. -+ * -+ * @self: CSFFW TL Reader instance. -+ */ -+void kbase_csf_tl_reader_reset(struct kbase_csf_tl_reader *self); -+ -+#endif /* _KBASE_CSFFW_TL_READER_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_trace_buffer.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_trace_buffer.c -new file mode 100644 -index 0000000..a6343c8 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_trace_buffer.c -@@ -0,0 +1,688 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase.h" -+#include "mali_kbase_defs.h" -+#include "mali_kbase_csf_firmware.h" -+#include "mali_kbase_csf_trace_buffer.h" -+#include "mali_kbase_reset_gpu.h" -+#include "mali_kbase_csf_tl_reader.h" -+ -+#include <linux/list.h> -+#include <linux/mman.h> -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+#if (KERNEL_VERSION(4, 7, 0) > LINUX_VERSION_CODE) -+#define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE -+#endif -+#endif -+ -+/** -+ * struct firmware_trace_buffer - Trace Buffer within the MCU firmware -+ * -+ * The firmware relays information to the host by writing on memory buffers -+ * which are allocated and partially configured by the host. These buffers -+ * are called Trace Buffers: each of them has a specific purpose and is -+ * identified by a name and a set of memory addresses where the host can -+ * set pointers to host-allocated structures. -+ * -+ * @kbdev: Pointer to the Kbase device. -+ * @node: List head linking all trace buffers to -+ * kbase_device:csf.firmware_trace_buffers -+ * @data_mapping: MCU shared memory mapping used for the data buffer. -+ * @updatable: Indicates whether config items can be updated with -+ * FIRMWARE_CONFIG_UPDATE -+ * @type: The type of the trace buffer. -+ * @trace_enable_entry_count: Number of Trace Enable bits. -+ * @gpu_va: Structure containing all the Firmware addresses -+ * that are accessed by the MCU. -+ * @gpu_va.size_address: The address where the MCU shall read the size of -+ * the data buffer. -+ * @gpu_va.insert_address: The address that shall be dereferenced by the MCU -+ * to write the Insert offset. -+ * @gpu_va.extract_address: The address that shall be dereferenced by the MCU -+ * to read the Extract offset. -+ * @gpu_va.data_address: The address that shall be dereferenced by the MCU -+ * to write the Trace Buffer. -+ * @gpu_va.trace_enable: The address where the MCU shall read the array of -+ * Trace Enable bits describing which trace points -+ * and features shall be enabled. -+ * @cpu_va: Structure containing CPU addresses of variables -+ * which are permanently mapped on the CPU address -+ * space. -+ * @cpu_va.insert_cpu_va: CPU virtual address of the Insert variable. -+ * @cpu_va.extract_cpu_va: CPU virtual address of the Extract variable. -+ * @num_pages: Size of the data buffer, in pages. -+ * @trace_enable_init_mask: Initial value for the trace enable bit mask. -+ * @name: NULL terminated string which contains the name of the trace buffer. -+ */ -+struct firmware_trace_buffer { -+ struct kbase_device *kbdev; -+ struct list_head node; -+ struct kbase_csf_mapping data_mapping; -+ bool updatable; -+ u32 type; -+ u32 trace_enable_entry_count; -+ struct gpu_va { -+ u32 size_address; -+ u32 insert_address; -+ u32 extract_address; -+ u32 data_address; -+ u32 trace_enable; -+ } gpu_va; -+ struct cpu_va { -+ u32 *insert_cpu_va; -+ u32 *extract_cpu_va; -+ } cpu_va; -+ u32 num_pages; -+ u32 trace_enable_init_mask[CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX]; -+ char name[1]; /* this field must be last */ -+}; -+ -+/** -+ * struct firmware_trace_buffer_data - Configuration data for trace buffers -+ * -+ * Describe how to set up a trace buffer interface. -+ * Trace buffers are identified by name and they require a data buffer and -+ * an initial mask of values for the trace enable bits. -+ * -+ * @name: Name identifier of the trace buffer -+ * @trace_enable_init_mask: Initial value to assign to the trace enable bits -+ * @size: Size of the data buffer to allocate for the trace buffer, in pages. -+ * The size of a data buffer must always be a power of 2. -+ */ -+struct firmware_trace_buffer_data { -+ char name[64]; -+ u32 trace_enable_init_mask[CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX]; -+ size_t size; -+}; -+ -+/* -+ * Table of configuration data for trace buffers. -+ * -+ * This table contains the configuration data for the trace buffers that are -+ * expected to be parsed from the firmware. -+ */ -+static const struct firmware_trace_buffer_data -+trace_buffer_data[] = { -+#ifndef MALI_KBASE_BUILD -+ { "fwutf", {0}, 1 }, -+#endif -+ { FW_TRACE_BUF_NAME, {0}, 4 }, -+ { "benchmark", {0}, 2 }, -+ { "timeline", {0}, KBASE_CSF_TL_BUFFER_NR_PAGES }, -+}; -+ -+int kbase_csf_firmware_trace_buffers_init(struct kbase_device *kbdev) -+{ -+ struct firmware_trace_buffer *trace_buffer; -+ int ret = 0; -+ u32 mcu_rw_offset = 0, mcu_write_offset = 0; -+ const u32 cache_line_alignment = kbase_get_cache_line_alignment(kbdev); -+ -+ if (list_empty(&kbdev->csf.firmware_trace_buffers.list)) { -+ dev_dbg(kbdev->dev, "No trace buffers to initialise\n"); -+ return 0; -+ } -+ -+ /* GPU-readable,writable memory used for Extract variables */ -+ ret = kbase_csf_firmware_mcu_shared_mapping_init( -+ kbdev, 1, PROT_WRITE, -+ KBASE_REG_GPU_RD | KBASE_REG_GPU_WR, -+ &kbdev->csf.firmware_trace_buffers.mcu_rw); -+ if (ret != 0) { -+ dev_err(kbdev->dev, "Failed to map GPU-rw MCU shared memory\n"); -+ goto out; -+ } -+ -+ /* GPU-writable memory used for Insert variables */ -+ ret = kbase_csf_firmware_mcu_shared_mapping_init( -+ kbdev, 1, PROT_READ, KBASE_REG_GPU_WR, -+ &kbdev->csf.firmware_trace_buffers.mcu_write); -+ if (ret != 0) { -+ dev_err(kbdev->dev, "Failed to map GPU-writable MCU shared memory\n"); -+ goto out; -+ } -+ -+ list_for_each_entry(trace_buffer, &kbdev->csf.firmware_trace_buffers.list, node) { -+ u32 extract_gpu_va, insert_gpu_va, data_buffer_gpu_va, -+ trace_enable_size_dwords; -+ u32 *extract_cpu_va, *insert_cpu_va; -+ unsigned int i; -+ -+ /* GPU-writable data buffer for the individual trace buffer */ -+ ret = kbase_csf_firmware_mcu_shared_mapping_init( -+ kbdev, trace_buffer->num_pages, PROT_READ, KBASE_REG_GPU_WR, -+ &trace_buffer->data_mapping); -+ if (ret) { -+ dev_err(kbdev->dev, "Failed to map GPU-writable MCU shared memory for a trace buffer\n"); -+ goto out; -+ } -+ -+ extract_gpu_va = -+ (kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg->start_pfn << PAGE_SHIFT) + -+ mcu_rw_offset; -+ extract_cpu_va = (u32*)( -+ kbdev->csf.firmware_trace_buffers.mcu_rw.cpu_addr + -+ mcu_rw_offset); -+ insert_gpu_va = -+ (kbdev->csf.firmware_trace_buffers.mcu_write.va_reg->start_pfn << PAGE_SHIFT) + -+ mcu_write_offset; -+ insert_cpu_va = (u32*)( -+ kbdev->csf.firmware_trace_buffers.mcu_write.cpu_addr + -+ mcu_write_offset); -+ data_buffer_gpu_va = -+ (trace_buffer->data_mapping.va_reg->start_pfn << PAGE_SHIFT); -+ -+ /* Initialize the Extract variable */ -+ *extract_cpu_va = 0; -+ -+ /* Each FW address shall be mapped and set individually, as we can't -+ * assume anything about their location in the memory address space. -+ */ -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.data_address, data_buffer_gpu_va); -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.insert_address, insert_gpu_va); -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.extract_address, extract_gpu_va); -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.size_address, -+ trace_buffer->num_pages << PAGE_SHIFT); -+ -+ trace_enable_size_dwords = -+ (trace_buffer->trace_enable_entry_count + 31) >> 5; -+ -+ for (i = 0; i < trace_enable_size_dwords; i++) { -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.trace_enable + i*4, -+ trace_buffer->trace_enable_init_mask[i]); -+ } -+ -+ /* Store CPU virtual addresses for permanently mapped variables */ -+ trace_buffer->cpu_va.insert_cpu_va = insert_cpu_va; -+ trace_buffer->cpu_va.extract_cpu_va = extract_cpu_va; -+ -+ /* Update offsets */ -+ mcu_write_offset += cache_line_alignment; -+ mcu_rw_offset += cache_line_alignment; -+ } -+ -+out: -+ return ret; -+} -+ -+void kbase_csf_firmware_trace_buffers_term(struct kbase_device *kbdev) -+{ -+ if (list_empty(&kbdev->csf.firmware_trace_buffers.list)) -+ return; -+ -+ while (!list_empty(&kbdev->csf.firmware_trace_buffers.list)) { -+ struct firmware_trace_buffer *trace_buffer; -+ -+ trace_buffer = list_first_entry(&kbdev->csf.firmware_trace_buffers.list, -+ struct firmware_trace_buffer, node); -+ kbase_csf_firmware_mcu_shared_mapping_term(kbdev, &trace_buffer->data_mapping); -+ list_del(&trace_buffer->node); -+ -+ kfree(trace_buffer); -+ } -+ -+ kbase_csf_firmware_mcu_shared_mapping_term( -+ kbdev, &kbdev->csf.firmware_trace_buffers.mcu_rw); -+ kbase_csf_firmware_mcu_shared_mapping_term( -+ kbdev, &kbdev->csf.firmware_trace_buffers.mcu_write); -+} -+ -+int kbase_csf_firmware_parse_trace_buffer_entry(struct kbase_device *kbdev, -+ const u32 *entry, -+ unsigned int size, -+ bool updatable) -+{ -+ const char *name = (char *)&entry[7]; -+ const unsigned int name_len = size - TRACE_BUFFER_ENTRY_NAME_OFFSET; -+ struct firmware_trace_buffer *trace_buffer; -+ unsigned int i; -+ -+ /* Allocate enough space for struct firmware_trace_buffer and the -+ * trace buffer name (with NULL termination). -+ */ -+ trace_buffer = -+ kmalloc(sizeof(*trace_buffer) + name_len + 1, GFP_KERNEL); -+ -+ if (!trace_buffer) -+ return -ENOMEM; -+ -+ memcpy(&trace_buffer->name, name, name_len); -+ trace_buffer->name[name_len] = '\0'; -+ -+ for (i = 0; i < ARRAY_SIZE(trace_buffer_data); i++) { -+ if (!strcmp(trace_buffer_data[i].name, trace_buffer->name)) { -+ unsigned int j; -+ -+ trace_buffer->kbdev = kbdev; -+ trace_buffer->updatable = updatable; -+ trace_buffer->type = entry[0]; -+ trace_buffer->gpu_va.size_address = entry[1]; -+ trace_buffer->gpu_va.insert_address = entry[2]; -+ trace_buffer->gpu_va.extract_address = entry[3]; -+ trace_buffer->gpu_va.data_address = entry[4]; -+ trace_buffer->gpu_va.trace_enable = entry[5]; -+ trace_buffer->trace_enable_entry_count = entry[6]; -+ trace_buffer->num_pages = trace_buffer_data[i].size; -+ -+ for (j = 0; j < CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX; j++) { -+ trace_buffer->trace_enable_init_mask[j] = -+ trace_buffer_data[i].trace_enable_init_mask[j]; -+ } -+ break; -+ } -+ } -+ -+ if (i < ARRAY_SIZE(trace_buffer_data)) { -+ list_add(&trace_buffer->node, &kbdev->csf.firmware_trace_buffers.list); -+ dev_dbg(kbdev->dev, "Trace buffer '%s'", trace_buffer->name); -+ } else { -+ dev_dbg(kbdev->dev, "Unknown trace buffer '%s'", trace_buffer->name); -+ kfree(trace_buffer); -+ } -+ -+ return 0; -+} -+ -+void kbase_csf_firmware_reload_trace_buffers_data(struct kbase_device *kbdev) -+{ -+ struct firmware_trace_buffer *trace_buffer; -+ u32 mcu_rw_offset = 0, mcu_write_offset = 0; -+ const u32 cache_line_alignment = kbase_get_cache_line_alignment(kbdev); -+ -+ list_for_each_entry(trace_buffer, &kbdev->csf.firmware_trace_buffers.list, node) { -+ u32 extract_gpu_va, insert_gpu_va, data_buffer_gpu_va, -+ trace_enable_size_dwords; -+ u32 *extract_cpu_va, *insert_cpu_va; -+ unsigned int i; -+ -+ /* Rely on the fact that all required mappings already exist */ -+ extract_gpu_va = -+ (kbdev->csf.firmware_trace_buffers.mcu_rw.va_reg->start_pfn << PAGE_SHIFT) + -+ mcu_rw_offset; -+ extract_cpu_va = (u32*)( -+ kbdev->csf.firmware_trace_buffers.mcu_rw.cpu_addr + -+ mcu_rw_offset); -+ insert_gpu_va = -+ (kbdev->csf.firmware_trace_buffers.mcu_write.va_reg->start_pfn << PAGE_SHIFT) + -+ mcu_write_offset; -+ insert_cpu_va = (u32*)( -+ kbdev->csf.firmware_trace_buffers.mcu_write.cpu_addr + -+ mcu_write_offset); -+ data_buffer_gpu_va = -+ (trace_buffer->data_mapping.va_reg->start_pfn << PAGE_SHIFT); -+ -+ /* Notice that the function only re-updates firmware memory locations -+ * with information that allows access to the trace buffers without -+ * really resetting their state. For instance, the Insert offset will -+ * not change and, as a consequence, the Extract offset is not going -+ * to be reset to keep consistency. -+ */ -+ -+ /* Each FW address shall be mapped and set individually, as we can't -+ * assume anything about their location in the memory address space. -+ */ -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.data_address, data_buffer_gpu_va); -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.insert_address, insert_gpu_va); -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.extract_address, extract_gpu_va); -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.size_address, -+ trace_buffer->num_pages << PAGE_SHIFT); -+ -+ trace_enable_size_dwords = -+ (trace_buffer->trace_enable_entry_count + 31) >> 5; -+ -+ for (i = 0; i < trace_enable_size_dwords; i++) { -+ kbase_csf_update_firmware_memory( -+ kbdev, trace_buffer->gpu_va.trace_enable + i*4, -+ trace_buffer->trace_enable_init_mask[i]); -+ } -+ -+ /* Store CPU virtual addresses for permanently mapped variables, -+ * as they might have slightly changed. -+ */ -+ trace_buffer->cpu_va.insert_cpu_va = insert_cpu_va; -+ trace_buffer->cpu_va.extract_cpu_va = extract_cpu_va; -+ -+ /* Update offsets */ -+ mcu_write_offset += cache_line_alignment; -+ mcu_rw_offset += cache_line_alignment; -+ } -+} -+ -+struct firmware_trace_buffer *kbase_csf_firmware_get_trace_buffer( -+ struct kbase_device *kbdev, const char *name) -+{ -+ struct firmware_trace_buffer *trace_buffer; -+ -+ list_for_each_entry(trace_buffer, &kbdev->csf.firmware_trace_buffers.list, node) { -+ if (!strcmp(trace_buffer->name, name)) -+ return trace_buffer; -+ } -+ -+ return NULL; -+} -+EXPORT_SYMBOL(kbase_csf_firmware_get_trace_buffer); -+ -+unsigned int kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count( -+ const struct firmware_trace_buffer *trace_buffer) -+{ -+ return trace_buffer->trace_enable_entry_count; -+} -+EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count); -+ -+static void kbasep_csf_firmware_trace_buffer_update_trace_enable_bit( -+ struct firmware_trace_buffer *tb, unsigned int bit, bool value) -+{ -+ struct kbase_device *kbdev = tb->kbdev; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (bit < tb->trace_enable_entry_count) { -+ unsigned int trace_enable_reg_offset = bit >> 5; -+ u32 trace_enable_bit_mask = 1u << (bit & 0x1F); -+ -+ if (value) { -+ tb->trace_enable_init_mask[trace_enable_reg_offset] |= -+ trace_enable_bit_mask; -+ } else { -+ tb->trace_enable_init_mask[trace_enable_reg_offset] &= -+ ~trace_enable_bit_mask; -+ } -+ -+ /* This is not strictly needed as the caller is supposed to -+ * reload the firmware image (through GPU reset) after updating -+ * the bitmask. Otherwise there is no guarantee that firmware -+ * will take into account the updated bitmask for all types of -+ * trace buffers, since firmware could continue to use the -+ * value of bitmask it cached after the boot. -+ */ -+ kbase_csf_update_firmware_memory( -+ kbdev, -+ tb->gpu_va.trace_enable + trace_enable_reg_offset * 4, -+ tb->trace_enable_init_mask[trace_enable_reg_offset]); -+ } -+} -+ -+int kbase_csf_firmware_trace_buffer_update_trace_enable_bit( -+ struct firmware_trace_buffer *tb, unsigned int bit, bool value) -+{ -+ struct kbase_device *kbdev = tb->kbdev; -+ int err = 0; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* If trace buffer update cannot be performed with -+ * FIRMWARE_CONFIG_UPDATE then we need to do a -+ * silent reset before we update the memory. -+ */ -+ if (!tb->updatable) { -+ /* If there is already a GPU reset pending then inform -+ * the User to retry the update. -+ */ -+ if (kbase_reset_gpu_silent(kbdev)) { -+ dev_warn( -+ kbdev->dev, -+ "GPU reset already in progress when enabling firmware timeline."); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ return -EAGAIN; -+ } -+ } -+ -+ kbasep_csf_firmware_trace_buffer_update_trace_enable_bit(tb, bit, -+ value); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ if (tb->updatable) -+ err = kbase_csf_trigger_firmware_config_update(kbdev); -+ -+ return err; -+} -+EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_update_trace_enable_bit); -+ -+bool kbase_csf_firmware_trace_buffer_is_empty( -+ const struct firmware_trace_buffer *trace_buffer) -+{ -+ return *(trace_buffer->cpu_va.insert_cpu_va) == -+ *(trace_buffer->cpu_va.extract_cpu_va); -+} -+EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_is_empty); -+ -+unsigned int kbase_csf_firmware_trace_buffer_read_data( -+ struct firmware_trace_buffer *trace_buffer, u8 *data, unsigned int num_bytes) -+{ -+ unsigned int bytes_copied; -+ u8 *data_cpu_va = trace_buffer->data_mapping.cpu_addr; -+ u32 extract_offset = *(trace_buffer->cpu_va.extract_cpu_va); -+ u32 insert_offset = *(trace_buffer->cpu_va.insert_cpu_va); -+ u32 buffer_size = trace_buffer->num_pages << PAGE_SHIFT; -+ -+ if (insert_offset >= extract_offset) { -+ bytes_copied = min_t(unsigned int, num_bytes, -+ (insert_offset - extract_offset)); -+ memcpy(data, &data_cpu_va[extract_offset], bytes_copied); -+ extract_offset += bytes_copied; -+ } else { -+ unsigned int bytes_copied_head, bytes_copied_tail; -+ -+ bytes_copied_tail = min_t(unsigned int, num_bytes, -+ (buffer_size - extract_offset)); -+ memcpy(data, &data_cpu_va[extract_offset], bytes_copied_tail); -+ -+ bytes_copied_head = min_t(unsigned int, -+ (num_bytes - bytes_copied_tail), insert_offset); -+ memcpy(&data[bytes_copied_tail], data_cpu_va, bytes_copied_head); -+ -+ bytes_copied = bytes_copied_head + bytes_copied_tail; -+ extract_offset += bytes_copied; -+ if (extract_offset >= buffer_size) -+ extract_offset = bytes_copied_head; -+ } -+ -+ *(trace_buffer->cpu_va.extract_cpu_va) = extract_offset; -+ -+ return bytes_copied; -+} -+EXPORT_SYMBOL(kbase_csf_firmware_trace_buffer_read_data); -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ -+#define U32_BITS 32 -+static u64 get_trace_buffer_active_mask64(struct firmware_trace_buffer *tb) -+{ -+ u64 active_mask = tb->trace_enable_init_mask[0]; -+ -+ if (tb->trace_enable_entry_count > U32_BITS) -+ active_mask |= (u64)tb->trace_enable_init_mask[1] << U32_BITS; -+ -+ return active_mask; -+} -+ -+static void update_trace_buffer_active_mask64(struct firmware_trace_buffer *tb, -+ u64 mask) -+{ -+ unsigned int i; -+ -+ for (i = 0; i < tb->trace_enable_entry_count; i++) -+ kbasep_csf_firmware_trace_buffer_update_trace_enable_bit( -+ tb, i, (mask >> i) & 1); -+} -+ -+static int set_trace_buffer_active_mask64(struct firmware_trace_buffer *tb, -+ u64 mask) -+{ -+ struct kbase_device *kbdev = tb->kbdev; -+ unsigned long flags; -+ int err = 0; -+ -+ if (!tb->updatable) { -+ /* If there is already a GPU reset pending, need a retry */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ if (kbase_reset_gpu_silent(kbdev)) -+ err = -EAGAIN; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } -+ -+ if (!err) { -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ update_trace_buffer_active_mask64(tb, mask); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* if we can update the config we need to just trigger -+ * FIRMWARE_CONFIG_UPDATE. -+ */ -+ if (tb->updatable) -+ err = kbase_csf_trigger_firmware_config_update(kbdev); -+ } -+ -+ return err; -+} -+ -+static int kbase_csf_firmware_trace_enable_mask_read(void *data, u64 *val) -+{ -+ struct kbase_device *kbdev = (struct kbase_device *)data; -+ struct firmware_trace_buffer *tb = -+ kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); -+ -+ if (tb == NULL) { -+ dev_err(kbdev->dev, "Couldn't get the firmware trace buffer"); -+ return -EIO; -+ } -+ /* The enabled traces limited to u64 here, regarded practical */ -+ *val = get_trace_buffer_active_mask64(tb); -+ return 0; -+} -+ -+static int kbase_csf_firmware_trace_enable_mask_write(void *data, u64 val) -+{ -+ struct kbase_device *kbdev = (struct kbase_device *)data; -+ struct firmware_trace_buffer *tb = -+ kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); -+ u64 new_mask; -+ unsigned int enable_bits_count; -+ -+ if (tb == NULL) { -+ dev_err(kbdev->dev, "Couldn't get the firmware trace buffer"); -+ return -EIO; -+ } -+ -+ /* Ignore unsupported types */ -+ enable_bits_count = -+ kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count(tb); -+ if (enable_bits_count > 64) { -+ dev_dbg(kbdev->dev, "Limit enabled bits count from %u to 64", -+ enable_bits_count); -+ enable_bits_count = 64; -+ } -+ new_mask = val & ((1 << enable_bits_count) - 1); -+ -+ if (new_mask != get_trace_buffer_active_mask64(tb)) -+ return set_trace_buffer_active_mask64(tb, new_mask); -+ else -+ return 0; -+} -+ -+static int kbasep_csf_firmware_trace_debugfs_open(struct inode *in, -+ struct file *file) -+{ -+ struct kbase_device *kbdev = in->i_private; -+ -+ file->private_data = kbdev; -+ dev_dbg(kbdev->dev, "Opened firmware trace buffer dump debugfs file"); -+ -+ return 0; -+} -+ -+static ssize_t kbasep_csf_firmware_trace_debugfs_read(struct file *file, -+ char __user *buf, size_t size, loff_t *ppos) -+{ -+ struct kbase_device *kbdev = file->private_data; -+ u8 *pbyte; -+ unsigned int n_read; -+ unsigned long not_copied; -+ /* Limit the kernel buffer to no more than two pages */ -+ size_t mem = MIN(size, 2 * PAGE_SIZE); -+ unsigned long flags; -+ -+ struct firmware_trace_buffer *tb = -+ kbase_csf_firmware_get_trace_buffer(kbdev, FW_TRACE_BUF_NAME); -+ -+ if (tb == NULL) { -+ dev_err(kbdev->dev, "Couldn't get the firmware trace buffer"); -+ return -EIO; -+ } -+ -+ pbyte = kmalloc(mem, GFP_KERNEL); -+ if (pbyte == NULL) { -+ dev_err(kbdev->dev, "Couldn't allocate memory for trace buffer dump"); -+ return -ENOMEM; -+ } -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ n_read = kbase_csf_firmware_trace_buffer_read_data(tb, pbyte, mem); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* Do the copy, if we have obtained some trace data */ -+ not_copied = (n_read) ? copy_to_user(buf, pbyte, n_read) : 0; -+ kfree(pbyte); -+ -+ if (!not_copied) { -+ *ppos += n_read; -+ return n_read; -+ } -+ -+ dev_err(kbdev->dev, "Couldn't copy trace buffer data to user space buffer"); -+ return -EFAULT; -+} -+ -+ -+DEFINE_SIMPLE_ATTRIBUTE(kbase_csf_firmware_trace_enable_mask_fops, -+ kbase_csf_firmware_trace_enable_mask_read, -+ kbase_csf_firmware_trace_enable_mask_write, "%llx\n"); -+ -+static const struct file_operations kbasep_csf_firmware_trace_debugfs_fops = { -+ .owner = THIS_MODULE, -+ .open = kbasep_csf_firmware_trace_debugfs_open, -+ .read = kbasep_csf_firmware_trace_debugfs_read, -+ .llseek = no_llseek, -+}; -+ -+void kbase_csf_firmware_trace_buffer_debugfs_init(struct kbase_device *kbdev) -+{ -+ debugfs_create_file("fw_trace_enable_mask", 0644, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbase_csf_firmware_trace_enable_mask_fops); -+ -+ debugfs_create_file("fw_traces", 0444, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbasep_csf_firmware_trace_debugfs_fops); -+} -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_trace_buffer.h b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_trace_buffer.h -new file mode 100644 -index 0000000..b9f481d ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_trace_buffer.h -@@ -0,0 +1,182 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CSF_TRACE_BUFFER_H_ -+#define _KBASE_CSF_TRACE_BUFFER_H_ -+ -+#include <linux/types.h> -+ -+#define CSF_FIRMWARE_TRACE_ENABLE_INIT_MASK_MAX (4) -+#define FW_TRACE_BUF_NAME "fwlog" -+ -+/* Forward declarations */ -+struct firmware_trace_buffer; -+struct kbase_device; -+ -+/** -+ * kbase_csf_firmware_trace_buffers_init - Initialize trace buffers -+ * -+ * Allocate resources for trace buffers. In particular: -+ * - One memory page of GPU-readable, CPU-writable memory is used for -+ * the Extract variables of all trace buffers. -+ * - One memory page of GPU-writable, CPU-readable memory is used for -+ * the Insert variables of all trace buffers. -+ * - A data buffer of GPU-writable, CPU-readable memory is allocated -+ * for each trace buffer. -+ * -+ * After that, firmware addresses are written with pointers to the -+ * insert, extract and data buffer variables. The size and the trace -+ * enable bits are not dereferenced by the GPU and shall be written -+ * in the firmware addresses directly. -+ * -+ * This function relies on the assumption that the list of -+ * firmware_trace_buffer elements in the device has already been -+ * populated with data from the firmware image parsing. -+ * -+ * Return: 0 if success, or an error code on failure. -+ * -+ * @kbdev: Device pointer -+ */ -+int kbase_csf_firmware_trace_buffers_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_trace_buffer_term - Terminate trace buffers -+ * -+ * @kbdev: Device pointer -+ */ -+void kbase_csf_firmware_trace_buffers_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_parse_trace_buffer_entry - Process a "trace buffer" section -+ * -+ * Read a "trace buffer" section adding metadata for the related trace buffer -+ * to the kbase_device:csf.firmware_trace_buffers list. -+ * -+ * Unexpected trace buffers will not be parsed and, as a consequence, -+ * will not be initialized. -+ * -+ * Return: 0 if successful, negative error code on failure. -+ * -+ * @kbdev: Kbase device structure -+ * @entry: Pointer to the section -+ * @size: Size (in bytes) of the section -+ * @updatable: Indicates whether config items can be updated with FIRMWARE_CONFIG_UPDATE -+ */ -+int kbase_csf_firmware_parse_trace_buffer_entry(struct kbase_device *kbdev, -+ const u32 *entry, -+ unsigned int size, -+ bool updatable); -+ -+/** -+ * kbase_csf_firmware_reload_trace_buffers_data - -+ * Reload trace buffers data for firmware reboot -+ * -+ * Helper function used when rebooting the firmware to reload the initial setup -+ * for all the trace buffers which have been previously parsed and initialized. -+ * -+ * Almost all of the operations done in the initialization process are -+ * replicated, with the difference that they might be done in a different order -+ * and that the variables of a given trace buffer may be mapped to different -+ * offsets within the same existing mappings. -+ * -+ * In other words, the re-initialization done by this function will be -+ * equivalent but not necessarily identical to the original initialization. -+ * -+ * @kbdev: Device pointer -+ */ -+void kbase_csf_firmware_reload_trace_buffers_data(struct kbase_device *kbdev); -+ -+/** -+ * kbase_csf_firmware_get_trace_buffer - Get a trace buffer -+ * -+ * Return: handle to a trace buffer, given the name, or NULL if a trace buffer -+ * with that name couldn't be found. -+ * -+ * @kbdev: Device pointer -+ * @name: Name of the trace buffer to find -+ */ -+struct firmware_trace_buffer *kbase_csf_firmware_get_trace_buffer( -+ struct kbase_device *kbdev, const char *name); -+ -+/** -+ * kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count - -+ * Get number of trace enable bits for a trace buffer -+ * -+ * Return: Number of trace enable bits in a trace buffer. -+ * -+ * @trace_buffer: Trace buffer handle -+ */ -+unsigned int kbase_csf_firmware_trace_buffer_get_trace_enable_bits_count( -+ const struct firmware_trace_buffer *trace_buffer); -+ -+/** -+ * kbase_csf_firmware_trace_buffer_update_trace_enable_bit - -+ * Update a trace enable bit -+ * -+ * Update the value of a given trace enable bit. -+ * -+ * @trace_buffer: Trace buffer handle -+ * @bit: Bit to update -+ * @value: New value for the given bit -+ * -+ * Return: 0 if successful, negative error code on failure. -+ */ -+int kbase_csf_firmware_trace_buffer_update_trace_enable_bit( -+ struct firmware_trace_buffer *trace_buffer, unsigned int bit, -+ bool value); -+ -+/** -+ * kbase_csf_firmware_trace_buffer_is_empty - Empty trace buffer predicate -+ * -+ * Return: True if the trace buffer is empty, or false otherwise. -+ * -+ * @trace_buffer: Trace buffer handle -+ */ -+bool kbase_csf_firmware_trace_buffer_is_empty( -+ const struct firmware_trace_buffer *trace_buffer); -+ -+/** -+ * kbase_csf_firmware_trace_buffer_read_data - Read data from a trace buffer -+ * -+ * Read available data from a trace buffer. The client provides a data buffer -+ * of a given size and the maximum number of bytes to read. -+ * -+ * Return: Number of bytes read from the trace buffer. -+ * -+ * @trace_buffer: Trace buffer handle -+ * @data: Pointer to a client-allocated where data shall be written. -+ * @num_bytes: Maximum number of bytes to read from the trace buffer. -+ */ -+unsigned int kbase_csf_firmware_trace_buffer_read_data( -+ struct firmware_trace_buffer *trace_buffer, u8 *data, unsigned int num_bytes); -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+/** -+ * kbase_csf_fw_trace_buffer_debugfs_init() - Add debugfs entries for setting -+ * enable mask and dumping the binary -+ * firmware trace buffer -+ * -+ * @kbdev: Pointer to the device -+ */ -+void kbase_csf_firmware_trace_buffer_debugfs_init(struct kbase_device *kbdev); -+#endif /* CONFIG_DEBUG_FS */ -+ -+#endif /* _KBASE_CSF_TRACE_BUFFER_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/debug/Kbuild -new file mode 100644 -index 0000000..1682c0f ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/Kbuild -@@ -0,0 +1,27 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+mali_kbase-y += debug/mali_kbase_debug_ktrace.o -+ -+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) -+ mali_kbase-y += debug/backend/mali_kbase_debug_ktrace_csf.o -+else -+ mali_kbase-y += debug/backend/mali_kbase_debug_ktrace_jm.o -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_codes_csf.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_codes_csf.h -new file mode 100644 -index 0000000..d05f802 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_codes_csf.h -@@ -0,0 +1,278 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE ***** -+ * ***** DO NOT INCLUDE DIRECTLY ***** -+ * ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** -+ */ -+ -+/* -+ * The purpose of this header file is just to contain a list of trace code -+ * identifiers -+ * -+ * When updating this file, also remember to update -+ * mali_kbase_debug_linux_ktrace_csf.h -+ * -+ * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT -+ * DESCRIBED IN mali_kbase_debug_ktrace_codes.h -+ */ -+ -+#if 0 /* Dummy section to avoid breaking formatting */ -+int dummy_array[] = { -+#endif -+ /* -+ * Generic CSF events -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(EVICT_CTX_SLOTS), -+ /* info_val[0:7] == fw version_minor -+ * info_val[15:8] == fw version_major -+ * info_val[63:32] == fw version_hash -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(FIRMWARE_BOOT), -+ KBASE_KTRACE_CODE_MAKE_CODE(FIRMWARE_REBOOT), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TOCK), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TOCK_END), -+ /* info_val == total number of runnable groups across all kctxs */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TICK), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TICK_END), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_RESET), -+ /* info_val = timeout in ms */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_WAIT_PROTM_QUIT), -+ /* info_val = remaining ms timeout, or 0 if timedout */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_WAIT_PROTM_QUIT_DONE), -+ KBASE_KTRACE_CODE_MAKE_CODE(SYNC_UPDATE_EVENT), -+ KBASE_KTRACE_CODE_MAKE_CODE(SYNC_UPDATE_EVENT_NOTIFY_GPU), -+ -+ /* info_val = JOB_IRQ_STATUS */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSF_INTERRUPT), -+ /* info_val = JOB_IRQ_STATUS */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSF_INTERRUPT_END), -+ /* info_val = JOB_IRQ_STATUS */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_INTERRUPT_PROCESS), -+ /* info_val = GLB_REQ ^ GLB_ACQ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(GLB_REQ_ACQ), -+ /* info_val[31:0] = num non idle offslot groups -+ * info_val[32] = scheduler can suspend on idle -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_CAN_IDLE), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_ADVANCE_TICK), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_NOADVANCE_TICK), -+ /* kctx is added to the back of the list */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_INSERT_RUNNABLE), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_REMOVE_RUNNABLE), -+ /* kctx is moved to the back of the list */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_ROTATE_RUNNABLE), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_HEAD_RUNNABLE), -+ -+ KBASE_KTRACE_CODE_MAKE_CODE(IDLE_WORKER_BEGIN), -+ /* 4-bit encoding of boolean values (ease of reading as hex values) -+ * -+ * info_val[3:0] = was reset active/failed to be prevented -+ * info_val[7:4] = whether scheduler was both idle and suspendable -+ * info_val[11:8] = whether all groups were suspended -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(IDLE_WORKER_END), -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_SYNC_UPDATE_WORKER_BEGIN), -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_SYNC_UPDATE_WORKER_END), -+ -+ /* info_val = bitmask of slots that gave an ACK for STATUS_UPDATE */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SLOTS_STATUS_UPDATE_ACK), -+ -+ /* -+ * Group events -+ */ -+ /* info_val[2:0] == CSG_REQ state issued -+ * info_val[19:16] == as_nr -+ * info_val[63:32] == endpoint config (max number of endpoints allowed) -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_START), -+ /* info_val == CSG_REQ state issued */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STOP), -+ /* info_val == CSG_ACK state */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STARTED), -+ /* info_val == CSG_ACK state */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STOPPED), -+ /* info_val == slot cleaned */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_CLEANED), -+ /* info_val = slot requesting STATUS_UPDATE */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_STATUS_UPDATE), -+ /* info_val = scheduler's new csg_slots_idle_mask[0] -+ * group->csg_nr indicates which bit was set -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_IDLE_SET), -+ /* info_val = scheduler's new csg_slots_idle_mask[0] -+ * group->csg_nr indicates which bit was cleared -+ * -+ * in case of no group, multiple bits may have been updated -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SLOT_IDLE_CLEAR), -+ /* info_val == previous priority */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_PRIO_UPDATE), -+ /* info_val == CSG_REQ ^ CSG_ACK */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_SYNC_UPDATE_INTERRUPT), -+ /* info_val == CSG_REQ ^ CSG_ACK */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_IDLE_INTERRUPT), -+ /* info_val == CSG_REQ ^ CSG_ACK */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_PROGRESS_TIMER_INTERRUPT), -+ /* info_val[31:0] == CSG_REQ ^ CSG_ACQ -+ * info_val[63:32] == CSG_IRQ_REQ ^ CSG_IRQ_ACK -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSG_INTERRUPT_PROCESS_END), -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_SYNC_UPDATE_DONE), -+ /* info_val == run state of the group */ -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_DESCHEDULE), -+ /* info_val == run state of the group */ -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_SCHEDULE), -+ /* info_val[31:0] == new run state of the evicted group -+ * info_val[63:32] == number of runnable groups -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_EVICT_SCHED), -+ -+ /* info_val == new num_runnable_grps -+ * group is added to the back of the list for its priority level -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_INSERT_RUNNABLE), -+ /* info_val == new num_runnable_grps -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_REMOVE_RUNNABLE), -+ /* info_val == num_runnable_grps -+ * group is moved to the back of the list for its priority level -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_ROTATE_RUNNABLE), -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_HEAD_RUNNABLE), -+ /* info_val == new num_idle_wait_grps -+ * group is added to the back of the list -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_INSERT_IDLE_WAIT), -+ /* info_val == new num_idle_wait_grps -+ * group is added to the back of the list -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_REMOVE_IDLE_WAIT), -+ KBASE_KTRACE_CODE_MAKE_CODE(GROUP_HEAD_IDLE_WAIT), -+ -+ /* info_val == is scheduler running with protected mode tasks */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_CHECK_PROTM_ENTER), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_ENTER_PROTM), -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_EXIT_PROTM), -+ /* info_val[31:0] == number of GPU address space slots in use -+ * info_val[63:32] == number of runnable groups -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_TOP_GRP), -+ /* info_val == new count of off-slot non-idle groups -+ * no group indicates it was set rather than incremented -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_NONIDLE_OFFSLOT_INC), -+ /* info_val == new count of off-slot non-idle groups */ -+ KBASE_KTRACE_CODE_MAKE_CODE(SCHEDULER_NONIDLE_OFFSLOT_DEC), -+ -+ KBASE_KTRACE_CODE_MAKE_CODE(PROTM_EVENT_WORKER_BEGIN), -+ KBASE_KTRACE_CODE_MAKE_CODE(PROTM_EVENT_WORKER_END), -+ -+ /* -+ * Group + Queue events -+ */ -+ /* info_val == queue->enabled */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSI_START), -+ /* info_val == queue->enabled before stop */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSI_STOP), -+ KBASE_KTRACE_CODE_MAKE_CODE(CSI_STOP_REQUESTED), -+ /* info_val == CS_REQ ^ CS_ACK that were not processed due to the group -+ * being suspended -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSI_IGNORED_INTERRUPTS_GROUP_SUSPEND), -+ /* info_val == CS_REQ ^ CS_ACK */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSI_FAULT_INTERRUPT), -+ /* info_val == CS_REQ ^ CS_ACK */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSI_TILER_OOM_INTERRUPT), -+ /* info_val == CS_REQ ^ CS_ACK */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSI_PROTM_PEND_INTERRUPT), -+ /* info_val == CS_ACK_PROTM_PEND ^ CS_REQ_PROTM_PEND */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CSI_PROTM_ACK), -+ /* info_val == group->run_State (for group the queue is bound to) */ -+ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_START), -+ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_STOP), -+ /* info_val == contents of CS_STATUS_WAIT_SYNC_POINTER */ -+ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_SYNC_UPDATE), -+ /* info_val == bool for result of the evaluation */ -+ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_SYNC_UPDATE_EVALUATED), -+ /* info_val == contents of CS_STATUS_WAIT */ -+ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_SYNC_STATUS_WAIT), -+ /* info_val == current sync value pointed to by queue->sync_ptr */ -+ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_SYNC_CURRENT_VAL), -+ /* info_val == current value of CS_STATUS_WAIT_SYNC_VALUE */ -+ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_SYNC_TEST_VAL), -+ /* info_val == current value of CS_STATUS_BLOCKED_REASON */ -+ KBASE_KTRACE_CODE_MAKE_CODE(QUEUE_SYNC_BLOCKED_REASON), -+ /* info_val = group's new protm_pending_bitmap[0] -+ * queue->csi_index indicates which bit was set -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(PROTM_PENDING_SET), -+ /* info_val = group's new protm_pending_bitmap[0] -+ * queue->csi_index indicates which bit was cleared -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(PROTM_PENDING_CLEAR), -+ -+ /* -+ * KCPU queue events -+ */ -+ /* KTrace info_val == KCPU queue fence context -+ * KCPU extra_info_val == N/A. -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(KCPU_QUEUE_NEW), -+ /* KTrace info_val == Number of pending commands in KCPU queue when -+ * it is destroyed. -+ * KCPU extra_info_val == Number of CQS wait operations present in -+ * the KCPU queue when it is destroyed. -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(KCPU_QUEUE_DESTROY), -+ /* KTrace info_val == CQS event memory address -+ * KCPU extra_info_val == Upper 32 bits of event memory, i.e. contents -+ * of error field. -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CQS_SET), -+ /* KTrace info_val == Number of CQS objects to be waited upon -+ * KCPU extra_info_val == N/A. -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CQS_WAIT_START), -+ /* KTrace info_val == CQS event memory address -+ * KCPU extra_info_val == 1 if CQS was signaled with an error and queue -+ * inherited the error, otherwise 0. -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(CQS_WAIT_END), -+ /* KTrace info_val == Fence context -+ * KCPU extra_info_val == Fence seqno. -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(FENCE_SIGNAL), -+ /* KTrace info_val == Fence context -+ * KCPU extra_info_val == Fence seqno. -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(FENCE_WAIT_START), -+ /* KTrace info_val == Fence context -+ * KCPU extra_info_val == Fence seqno. -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(FENCE_WAIT_END), -+ -+#if 0 /* Dummy section to avoid breaking formatting */ -+}; -+#endif -+ -+/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_codes_jm.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_codes_jm.h -index d534f30..f419f70 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_codes_jm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_codes_jm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2015, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -30,6 +29,9 @@ - * The purpose of this header file is just to contain a list of trace code - * identifiers - * -+ * When updating this file, also remember to update -+ * mali_kbase_debug_linux_ktrace_jm.h -+ * - * IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT - * DESCRIBED IN mali_kbase_debug_ktrace_codes.h - */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_csf.c -new file mode 100644 -index 0000000..824ca4b ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_csf.c -@@ -0,0 +1,193 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include "debug/mali_kbase_debug_ktrace_internal.h" -+#include "debug/backend/mali_kbase_debug_ktrace_csf.h" -+ -+#if KBASE_KTRACE_TARGET_RBUF -+ -+void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written) -+{ -+ *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), -+ "group,slot,prio,csi,kcpu"), 0); -+} -+ -+void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, -+ char *buffer, int sz, s32 *written) -+{ -+ const union kbase_ktrace_backend * const be_msg = &trace_msg->backend; -+ /* At present, no need to check for KBASE_KTRACE_FLAG_BACKEND, as the -+ * other backend-specific flags currently imply this anyway -+ */ -+ -+ /* group parts */ -+ if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_GROUP) { -+ const s8 slot = be_msg->gpu.csg_nr; -+ /* group,slot, */ -+ *written += MAX(snprintf(buffer + *written, -+ MAX(sz - *written, 0), -+ "%u,%d,", be_msg->gpu.group_handle, slot), 0); -+ -+ /* prio */ -+ if (slot >= 0) -+ *written += MAX(snprintf(buffer + *written, -+ MAX(sz - *written, 0), -+ "%u", be_msg->gpu.slot_prio), 0); -+ -+ /* , */ -+ *written += MAX(snprintf(buffer + *written, -+ MAX(sz - *written, 0), -+ ","), 0); -+ } else { -+ /* No group,slot,prio fields, but ensure ending with "," */ -+ *written += MAX(snprintf(buffer + *written, -+ MAX(sz - *written, 0), -+ ",,,"), 0); -+ } -+ -+ /* queue parts: csi */ -+ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_CSF_QUEUE) -+ *written += MAX(snprintf(buffer + *written, -+ MAX(sz - *written, 0), -+ "%d", be_msg->gpu.csi_index), 0); -+ -+ /* , */ -+ *written += MAX(snprintf(buffer + *written, -+ MAX(sz - *written, 0), -+ ","), 0); -+ -+ if (be_msg->gpu.flags & KBASE_KTRACE_FLAG_CSF_KCPU) { -+ /* kcpu data */ -+ *written += MAX(snprintf(buffer + *written, -+ MAX(sz - *written, 0), -+ "kcpu %d (0x%llx)", -+ be_msg->kcpu.id, -+ be_msg->kcpu.extra_info_val), 0); -+ } -+ -+ /* Don't end with a trailing "," - this is a 'standalone' formatted -+ * msg, caller will handle the delimiters -+ */ -+} -+ -+void kbasep_ktrace_add_csf(struct kbase_device *kbdev, -+ enum kbase_ktrace_code code, struct kbase_queue_group *group, -+ struct kbase_queue *queue, kbase_ktrace_flag_t flags, -+ u64 info_val) -+{ -+ unsigned long irqflags; -+ struct kbase_ktrace_msg *trace_msg; -+ struct kbase_context *kctx = NULL; -+ -+ spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); -+ -+ /* Reserve and update indices */ -+ trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); -+ -+ /* Determine the kctx */ -+ if (group) -+ kctx = group->kctx; -+ else if (queue) -+ kctx = queue->kctx; -+ -+ /* Fill the common part of the message (including backend.gpu.flags) */ -+ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, -+ info_val); -+ -+ /* Indicate to the common code that backend-specific parts will be -+ * valid -+ */ -+ trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_BACKEND; -+ -+ /* Fill the CSF-specific parts of the message -+ * -+ * Generally, no need to use default initializers when queue/group not -+ * present - can usually check the flags instead. -+ */ -+ -+ if (queue) { -+ trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_CSF_QUEUE; -+ trace_msg->backend.gpu.csi_index = queue->csi_index; -+ } -+ -+ if (group) { -+ const s8 slot = group->csg_nr; -+ -+ trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_CSF_GROUP; -+ -+ trace_msg->backend.gpu.csg_nr = slot; -+ -+ if (slot >= 0) { -+ struct kbase_csf_csg_slot *csg_slot = -+ &kbdev->csf.scheduler.csg_slots[slot]; -+ -+ trace_msg->backend.gpu.slot_prio = -+ csg_slot->priority; -+ } -+ /* slot >=0 indicates whether slot_prio valid, so no need to -+ * initialize in the case where it's invalid -+ */ -+ -+ trace_msg->backend.gpu.group_handle = group->handle; -+ } -+ -+ WARN_ON((trace_msg->backend.gpu.flags & ~KBASE_KTRACE_FLAG_ALL)); -+ -+ /* Done */ -+ spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); -+} -+ -+void kbasep_ktrace_add_csf_kcpu(struct kbase_device *kbdev, -+ enum kbase_ktrace_code code, -+ struct kbase_kcpu_command_queue *queue, -+ u64 info_val1, u64 info_val2) -+{ -+ unsigned long irqflags; -+ struct kbase_ktrace_msg *trace_msg; -+ struct kbase_context *kctx = queue->kctx; -+ -+ spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); -+ -+ /* Reserve and update indices */ -+ trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); -+ -+ /* Fill the common part of the message */ -+ kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, 0, -+ info_val1); -+ -+ /* Indicate to the common code that backend-specific parts will be -+ * valid -+ */ -+ trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_BACKEND; -+ -+ /* Fill the KCPU-specific parts of the message */ -+ trace_msg->backend.kcpu.id = queue->id; -+ trace_msg->backend.kcpu.extra_info_val = info_val2; -+ trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_CSF_KCPU; -+ -+ WARN_ON((trace_msg->backend.gpu.flags & ~KBASE_KTRACE_FLAG_ALL)); -+ -+ /* Done */ -+ spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); -+} -+ -+#endif /* KBASE_KTRACE_TARGET_RBUF */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_csf.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_csf.h -new file mode 100644 -index 0000000..0593c30 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_csf.h -@@ -0,0 +1,203 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_DEBUG_KTRACE_CSF_H_ -+#define _KBASE_DEBUG_KTRACE_CSF_H_ -+ -+/* -+ * KTrace target for internal ringbuffer -+ */ -+#if KBASE_KTRACE_TARGET_RBUF -+/** -+ * kbasep_ktrace_add_csf - internal function to add trace about CSF -+ * @kbdev: kbase device -+ * @code: trace code -+ * @group: queue group, or NULL if no queue group -+ * @queue: queue, or NULL if no queue -+ * @flags: flags about the message -+ * @info_val: generic information about @code to add to the trace -+ * -+ * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_CSF() instead. -+ */ -+ -+void kbasep_ktrace_add_csf(struct kbase_device *kbdev, -+ enum kbase_ktrace_code code, struct kbase_queue_group *group, -+ struct kbase_queue *queue, kbase_ktrace_flag_t flags, -+ u64 info_val); -+ -+/** -+ * kbasep_ktrace_add_csf_kcpu - internal function to add trace about the CSF -+ * KCPU queues. -+ * @kbdev: kbase device -+ * @code: trace code -+ * @queue: queue, or NULL if no queue -+ * @info_val1: Main infoval variable with information based on the KCPU -+ * ktrace call. Refer to mali_kbase_debug_ktrace_codes_csf.h -+ * for information on the infoval values. -+ * @info_val2: Extra infoval variable with information based on the KCPU -+ * ktrace call. Refer to mali_kbase_debug_ktrace_codes_csf.h -+ * for information on the infoval values. -+ * -+ * PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_CSF_KCPU() instead. -+ */ -+void kbasep_ktrace_add_csf_kcpu(struct kbase_device *kbdev, -+ enum kbase_ktrace_code code, -+ struct kbase_kcpu_command_queue *queue, -+ u64 info_val1, u64 info_val2); -+ -+#define KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, group, queue, flags, info_val) \ -+ kbasep_ktrace_add_csf(kbdev, KBASE_KTRACE_CODE(code), group, queue, \ -+ flags, info_val) -+ -+#define KBASE_KTRACE_RBUF_ADD_CSF_KCPU(kbdev, code, queue, info_val1, \ -+ info_val2) kbasep_ktrace_add_csf_kcpu(kbdev, KBASE_KTRACE_CODE(code), \ -+ queue, info_val1, info_val2) -+ -+#else /* KBASE_KTRACE_TARGET_RBUF */ -+ -+#define KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, group, queue, flags, info_val) \ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(code);\ -+ CSTD_UNUSED(group);\ -+ CSTD_UNUSED(queue);\ -+ CSTD_UNUSED(flags);\ -+ CSTD_UNUSED(info_val);\ -+ CSTD_NOP(0);\ -+ } while (0) -+ -+#define KBASE_KTRACE_RBUF_ADD_CSF_KCPU(kbdev, code, queue, info_val1, info_val2) \ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(code);\ -+ CSTD_UNUSED(queue);\ -+ CSTD_UNUSED(info_val1);\ -+ CSTD_UNUSED(info_val2);\ -+ } while (0) -+ -+#endif /* KBASE_KTRACE_TARGET_RBUF */ -+ -+/* -+ * KTrace target for Linux's ftrace -+ * -+ * Note: the header file(s) that define the trace_mali_<...> tracepoints are -+ * included by the parent header file -+ */ -+#if KBASE_KTRACE_TARGET_FTRACE -+ -+#define KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, group, queue, info_val) \ -+ trace_mali_##code(kbdev, group, queue, info_val) -+ -+#define KBASE_KTRACE_FTRACE_ADD_KCPU(code, queue, info_val1, info_val2) \ -+ trace_mali_##code(queue, info_val1, info_val2) -+ -+#else /* KBASE_KTRACE_TARGET_FTRACE */ -+ -+#define KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, group, queue, info_val) \ -+ do {\ -+ CSTD_UNUSED(kbdev);\ -+ CSTD_NOP(code);\ -+ CSTD_UNUSED(group);\ -+ CSTD_UNUSED(queue);\ -+ CSTD_UNUSED(info_val);\ -+ CSTD_NOP(0);\ -+ } while (0) -+ -+#define KBASE_KTRACE_FTRACE_ADD_KCPU(code, queue, info_val1, info_val2) \ -+ do {\ -+ CSTD_NOP(code);\ -+ CSTD_UNUSED(queue);\ -+ CSTD_UNUSED(info_val1);\ -+ CSTD_UNUSED(info_val2);\ -+ } while (0) -+ -+#endif /* KBASE_KTRACE_TARGET_FTRACE */ -+ -+/* -+ * Master set of macros to route KTrace to any of the targets -+ */ -+ -+/** -+ * KBASE_KTRACE_ADD_CSF_GRP - Add trace values about a group, with info -+ * @kbdev: kbase device -+ * @code: trace code -+ * @group: queue group, or NULL if no queue group -+ * @info_val: generic information about @code to add to the trace -+ * -+ * Note: Any functions called through this macro will still be evaluated in -+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when -+ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied -+ * to this macro must: -+ * a) be static or static inline, and -+ * b) just return 0 and have no other statements present in the body. -+ */ -+#define KBASE_KTRACE_ADD_CSF_GRP(kbdev, code, group, info_val) \ -+ do { \ -+ /* capture values that could come from non-pure fn calls */ \ -+ struct kbase_queue_group *__group = group; \ -+ u64 __info_val = info_val; \ -+ KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, __group, NULL, 0u, \ -+ __info_val); \ -+ KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, __group, NULL, \ -+ __info_val); \ -+ } while (0) -+ -+/** -+ * KBASE_KTRACE_ADD_CSF_GRP_Q - Add trace values about a group, queue, with info -+ * @kbdev: kbase device -+ * @code: trace code -+ * @group: queue group, or NULL if no queue group -+ * @queue: queue, or NULL if no queue -+ * @info_val: generic information about @code to add to the trace -+ * -+ * Note: Any functions called through this macro will still be evaluated in -+ * Release builds (CONFIG_MALI_DEBUG not defined). Therefore, when -+ * KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied -+ * to this macro must: -+ * a) be static or static inline, and -+ * b) just return 0 and have no other statements present in the body. -+ */ -+#define KBASE_KTRACE_ADD_CSF_GRP_Q(kbdev, code, group, queue, info_val) \ -+ do { \ -+ /* capture values that could come from non-pure fn calls */ \ -+ struct kbase_queue_group *__group = group; \ -+ struct kbase_queue *__queue = queue; \ -+ u64 __info_val = info_val; \ -+ KBASE_KTRACE_RBUF_ADD_CSF(kbdev, code, __group, __queue, 0u, \ -+ __info_val); \ -+ KBASE_KTRACE_FTRACE_ADD_CSF(kbdev, code, __group, \ -+ __queue, __info_val); \ -+ } while (0) -+ -+ -+#define KBASE_KTRACE_ADD_CSF_KCPU(kbdev, code, queue, info_val1, info_val2) \ -+ do { \ -+ /* capture values that could come from non-pure fn calls */ \ -+ struct kbase_kcpu_command_queue *__queue = queue; \ -+ u64 __info_val1 = info_val1; \ -+ u64 __info_val2 = info_val2; \ -+ KBASE_KTRACE_RBUF_ADD_CSF_KCPU(kbdev, code, __queue, \ -+ __info_val1, __info_val2); \ -+ KBASE_KTRACE_FTRACE_ADD_KCPU(code, __queue, \ -+ __info_val1, __info_val2); \ -+ } while (0) -+ -+#endif /* _KBASE_DEBUG_KTRACE_CSF_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_defs_csf.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_defs_csf.h -new file mode 100644 -index 0000000..7f32cd2 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_defs_csf.h -@@ -0,0 +1,116 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ -+#define _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ -+ -+#if KBASE_KTRACE_TARGET_RBUF -+/** -+ * DOC: KTrace version history, CSF variant -+ * -+ * 1.0: -+ * First version, with version information in the header. -+ * -+ * 1.1: -+ * kctx field is no longer a pointer, and is now an ID of the format %d_%u as -+ * used by kctx directories in mali debugfs entries: (tgid creating the kctx), -+ * (unique kctx id) -+ * -+ * ftrace backend now outputs kctx field (as %d_%u format). -+ * -+ * Add fields group, slot, prio, csi into backend-specific part. -+ * -+ * 1.2: -+ * There is a new class of KCPU traces; with this, a new KCPU column in the -+ * ringbuffer RBUF (mali_trace) between csi and info_val, which is empty -+ * for non-kcpu related traces, and usually displays the KCPU Queue ID and -+ * an extra information value. ftrace also displays these KCPU traces. -+ * -+ * 1.3: -+ * Add a lot of extra new traces. Tweak some existing scheduler related traces -+ * to contain extra information information/happen at slightly different times. -+ * SCHEDULER_EXIT_PROTM now has group information -+ */ -+#define KBASE_KTRACE_VERSION_MAJOR 1 -+#define KBASE_KTRACE_VERSION_MINOR 3 -+ -+/* indicates if the trace message has valid queue-group related info. */ -+#define KBASE_KTRACE_FLAG_CSF_GROUP (((kbase_ktrace_flag_t)1) << 0) -+ -+/* indicates if the trace message has valid queue related info. */ -+#define KBASE_KTRACE_FLAG_CSF_QUEUE (((kbase_ktrace_flag_t)1) << 1) -+ -+/* indicates if the trace message has valid KCPU-queue related info. */ -+#define KBASE_KTRACE_FLAG_CSF_KCPU (((kbase_ktrace_flag_t)1) << 2) -+ -+/* Collect all the flags together for debug checking */ -+#define KBASE_KTRACE_FLAG_BACKEND_ALL \ -+ (KBASE_KTRACE_FLAG_CSF_GROUP | KBASE_KTRACE_FLAG_CSF_QUEUE | \ -+ KBASE_KTRACE_FLAG_CSF_KCPU) -+ -+/** -+ * union kbase_ktrace_backend - backend specific part of a trace message -+ * @kcpu: kcpu union member -+ * @kcpu.code: Identifies the event, refer to enum kbase_ktrace_code. -+ * @kcpu.flags: indicates information about the trace message itself. Used -+ * during dumping of the message. -+ * @kcpu.id: ID of the KCPU queue. -+ * @kcpu.extra_info_val: value specific to the type of KCPU event being traced. -+ * Refer to the KPU specific code in enum kbase_ktrace_code in -+ * mali_kbase_debug_ktrace_codes_csf.h -+ * @gpu: gpu union member -+ * @gpu.code: Identifies the event, refer to enum kbase_ktrace_code. -+ * @gpu.flags: indicates information about the trace message itself. Used -+ * during dumping of the message. -+ * @gpu.group_handle: Handle identifying the associated queue group. Only valid -+ * when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. -+ * @gpu.csg_nr: Number/index of the associated queue group's CS group to -+ * which it is mapped, or negative if none associated. Only -+ * valid when @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. -+ * @gpu.slot_prio: The priority of the slot for the associated group, if it -+ * was scheduled. Hence, only valid when @csg_nr >=0 and -+ * @flags contains KBASE_KTRACE_FLAG_CSF_GROUP. -+ * @gpu.csi_index: ID of the associated queue's CS HW interface. -+ * Only valid when @flags contains KBASE_KTRACE_FLAG_CSF_QUEUE. -+ */ -+ -+union kbase_ktrace_backend { -+ /* Place 64 and 32-bit members together */ -+ /* Pack smaller members together */ -+ struct { -+ kbase_ktrace_code_t code; -+ kbase_ktrace_flag_t flags; -+ u8 id; -+ u64 extra_info_val; -+ } kcpu; -+ -+ struct { -+ kbase_ktrace_code_t code; -+ kbase_ktrace_flag_t flags; -+ u8 group_handle; -+ s8 csg_nr; -+ u8 slot_prio; -+ s8 csi_index; -+ } gpu; -+}; -+ -+#endif /* KBASE_KTRACE_TARGET_RBUF */ -+#endif /* _KBASE_DEBUG_KTRACE_DEFS_CSF_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_defs_jm.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_defs_jm.h -index 55b66ad..c01f930 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_defs_jm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_defs_jm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,25 +17,44 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_DEBUG_KTRACE_DEFS_JM_H_ - #define _KBASE_DEBUG_KTRACE_DEFS_JM_H_ - -+#if KBASE_KTRACE_TARGET_RBUF - /** - * DOC: KTrace version history, JM variant -+ * - * 1.0: -- * - Original version (implicit, header did not carry version information) -+ * Original version (implicit, header did not carry version information). -+ * - * 2.0: -- * - Introduced version information into the header -- * - some changes of parameter names in header -- * - trace now uses all 64-bits of info_val -- * - Non-JM specific parts moved to using info_val instead of refcount/gpu_addr -+ * Introduced version information into the header. -+ * -+ * Some changes of parameter names in header. -+ * -+ * Trace now uses all 64-bits of info_val. -+ * -+ * Non-JM specific parts moved to using info_val instead of refcount/gpu_addr. -+ * -+ * 2.1: -+ * kctx field is no longer a pointer, and is now an ID of the format %d_%u as -+ * used by kctx directories in mali debugfs entries: (tgid creating the kctx), -+ * (unique kctx id). -+ * -+ * ftrace backend now outputs kctx field (as %d_%u format). -+ * - */ - #define KBASE_KTRACE_VERSION_MAJOR 2 --#define KBASE_KTRACE_VERSION_MINOR 0 -+#define KBASE_KTRACE_VERSION_MINOR 1 -+#endif /* KBASE_KTRACE_TARGET_RBUF */ -+ -+/* -+ * Note: mali_kbase_debug_ktrace_jm.h needs these value even if the RBUF target -+ * is disabled (they get discarded with CSTD_UNUSED(), but they're still -+ * referenced) -+ */ - - /* indicates if the trace message has a valid refcount member */ - #define KBASE_KTRACE_FLAG_JM_REFCOUNT (((kbase_ktrace_flag_t)1) << 0) -@@ -43,33 +63,47 @@ - /* indicates if the trace message has valid atom related info. */ - #define KBASE_KTRACE_FLAG_JM_ATOM (((kbase_ktrace_flag_t)1) << 2) - -+#if KBASE_KTRACE_TARGET_RBUF -+/* Collect all the flags together for debug checking */ -+#define KBASE_KTRACE_FLAG_BACKEND_ALL \ -+ (KBASE_KTRACE_FLAG_JM_REFCOUNT | KBASE_KTRACE_FLAG_JM_JOBSLOT \ -+ | KBASE_KTRACE_FLAG_JM_ATOM) - - /** -- * struct kbase_ktrace_backend - backend specific part of a trace message -+ * union kbase_ktrace_backend - backend specific part of a trace message -+ * Contains only a struct but is a union such that it is compatible with -+ * generic JM and CSF KTrace calls. - * -- * @atom_udata: Copy of the user data sent for the atom in base_jd_submit. -- * Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags -- * @gpu_addr: GPU address, usually of the job-chain represented by an atom. -- * @atom_number: id of the atom for which trace message was added. Only valid -- * if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags -- * @code: Identifies the event, refer to enum kbase_ktrace_code. -- * @flags: indicates information about the trace message itself. Used -- * during dumping of the message. -- * @jobslot: job-slot for which trace message was added, valid only for -- * job-slot management events. -- * @refcount: reference count for the context, valid for certain events -- * related to scheduler core and policy. -+ * @gpu: gpu union member -+ * @gpu.atom_udata: Copy of the user data sent for the atom in base_jd_submit. -+ * Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags -+ * @gpu.gpu_addr: GPU address, usually of the job-chain represented by an -+ * atom. -+ * @gpu.atom_number: id of the atom for which trace message was added. Only -+ * valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags -+ * @gpu.code: Identifies the event, refer to enum kbase_ktrace_code. -+ * @gpu.flags: indicates information about the trace message itself. Used -+ * during dumping of the message. -+ * @gpu.jobslot: job-slot for which trace message was added, valid only for -+ * job-slot management events. -+ * @gpu.refcount: reference count for the context, valid for certain events -+ * related to scheduler core and policy. - */ --struct kbase_ktrace_backend { -- /* Place 64 and 32-bit members together */ -- u64 atom_udata[2]; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ -- u64 gpu_addr; -- int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ -- /* Pack smaller members together */ -- kbase_ktrace_code_t code; -- kbase_ktrace_flag_t flags; -- u8 jobslot; -- u8 refcount; -+union kbase_ktrace_backend { -+ struct { -+ /* Place 64 and 32-bit members together */ -+ u64 atom_udata[2]; /* Only valid for -+ * KBASE_KTRACE_FLAG_JM_ATOM -+ */ -+ u64 gpu_addr; -+ int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */ -+ /* Pack smaller members together */ -+ kbase_ktrace_code_t code; -+ kbase_ktrace_flag_t flags; -+ u8 jobslot; -+ u8 refcount; -+ } gpu; - }; -+#endif /* KBASE_KTRACE_TARGET_RBUF */ - - #endif /* _KBASE_DEBUG_KTRACE_DEFS_JM_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_jm.c -index e651a09..fed9c1f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_jm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_jm.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,9 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ -+ - #include <mali_kbase.h> - #include "debug/mali_kbase_debug_ktrace_internal.h" - #include "debug/backend/mali_kbase_debug_ktrace_jm.h" -@@ -35,38 +35,39 @@ void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg, - char *buffer, int sz, s32 *written) - { - /* katom */ -- if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_ATOM) -+ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_ATOM) - *written += MAX(snprintf(buffer + *written, - MAX(sz - *written, 0), - "atom %d (ud: 0x%llx 0x%llx)", -- trace_msg->backend.atom_number, -- trace_msg->backend.atom_udata[0], -- trace_msg->backend.atom_udata[1]), 0); -+ trace_msg->backend.gpu.atom_number, -+ trace_msg->backend.gpu.atom_udata[0], -+ trace_msg->backend.gpu.atom_udata[1]), 0); - - /* gpu_addr */ -- if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_BACKEND) -+ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_BACKEND) - *written += MAX(snprintf(buffer + *written, - MAX(sz - *written, 0), -- ",%.8llx,", trace_msg->backend.gpu_addr), 0); -+ ",%.8llx,", trace_msg->backend.gpu.gpu_addr), -+ 0); - else - *written += MAX(snprintf(buffer + *written, - MAX(sz - *written, 0), - ",,"), 0); - - /* jobslot */ -- if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) -+ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT) - *written += MAX(snprintf(buffer + *written, - MAX(sz - *written, 0), -- "%d", trace_msg->backend.jobslot), 0); -+ "%d", trace_msg->backend.gpu.jobslot), 0); - - *written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0), - ","), 0); - - /* refcount */ -- if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) -+ if (trace_msg->backend.gpu.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT) - *written += MAX(snprintf(buffer + *written, - MAX(sz - *written, 0), -- "%d", trace_msg->backend.refcount), 0); -+ "%d", trace_msg->backend.gpu.refcount), 0); - } - - void kbasep_ktrace_add_jm(struct kbase_device *kbdev, -@@ -83,28 +84,31 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, - /* Reserve and update indices */ - trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); - -- /* Fill the common part of the message (including backend.flags) */ -+ /* Fill the common part of the message (including backend.gpu.flags) */ - kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, - info_val); - - /* Indicate to the common code that backend-specific parts will be - * valid - */ -- trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND; -+ trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_BACKEND; - - /* Fill the JM-specific parts of the message */ - if (katom) { -- trace_msg->backend.flags |= KBASE_KTRACE_FLAG_JM_ATOM; -+ trace_msg->backend.gpu.flags |= KBASE_KTRACE_FLAG_JM_ATOM; - -- trace_msg->backend.atom_number = kbase_jd_atom_id(katom->kctx, katom); -- trace_msg->backend.atom_udata[0] = katom->udata.blob[0]; -- trace_msg->backend.atom_udata[1] = katom->udata.blob[1]; -+ trace_msg->backend.gpu.atom_number = -+ kbase_jd_atom_id(katom->kctx, katom); -+ trace_msg->backend.gpu.atom_udata[0] = katom->udata.blob[0]; -+ trace_msg->backend.gpu.atom_udata[1] = katom->udata.blob[1]; - } - -- trace_msg->backend.gpu_addr = gpu_addr; -- trace_msg->backend.jobslot = jobslot; -+ trace_msg->backend.gpu.gpu_addr = gpu_addr; -+ trace_msg->backend.gpu.jobslot = jobslot; - /* Clamp refcount */ -- trace_msg->backend.refcount = MIN((unsigned int)refcount, 0xFF); -+ trace_msg->backend.gpu.refcount = MIN((unsigned int)refcount, 0xFF); -+ -+ WARN_ON((trace_msg->backend.gpu.flags & ~KBASE_KTRACE_FLAG_ALL)); - - /* Done */ - spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_jm.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_jm.h -index c1bacf9..8b09d05 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_jm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_ktrace_jm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_DEBUG_KTRACE_JM_H_ -@@ -47,89 +46,24 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, - kbase_ktrace_flag_t flags, int refcount, int jobslot, - u64 info_val); - --#define KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ -- jobslot) \ -- kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ -- gpu_addr, KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, jobslot, 0) -- --#define KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \ -- jobslot, info_val) \ -+#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, \ -+ refcount, jobslot, info_val) \ - kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ -- gpu_addr, KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, jobslot, \ -- info_val) -- --#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \ -- refcount) \ -- kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ -- gpu_addr, KBASE_KTRACE_FLAG_JM_REFCOUNT, refcount, 0, 0) --#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ -- gpu_addr, refcount, info_val) \ -- kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ -- gpu_addr, KBASE_KTRACE_FLAG_JM_REFCOUNT, refcount, 0, \ -- info_val) -- --#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \ -- kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \ -- gpu_addr, 0, 0, 0, info_val) -+ gpu_addr, flags, refcount, jobslot, info_val) - - #else /* KBASE_KTRACE_TARGET_RBUF */ --#define KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ -- jobslot) \ -- do {\ -- CSTD_UNUSED(kbdev);\ -- CSTD_NOP(code);\ -- CSTD_UNUSED(kctx);\ -- CSTD_UNUSED(katom);\ -- CSTD_UNUSED(gpu_addr);\ -- CSTD_UNUSED(jobslot);\ -- CSTD_NOP(0);\ -- } while (0) -- --#define KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \ -- jobslot, info_val) \ -- do {\ -- CSTD_UNUSED(kbdev);\ -- CSTD_NOP(code);\ -- CSTD_UNUSED(kctx);\ -- CSTD_UNUSED(katom);\ -- CSTD_UNUSED(gpu_addr);\ -- CSTD_UNUSED(jobslot);\ -- CSTD_UNUSED(info_val);\ -- CSTD_NOP(0);\ -- } while (0) - --#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \ -- refcount) \ -+#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, flags, \ -+ refcount, jobslot, info_val) \ - do {\ - CSTD_UNUSED(kbdev);\ - CSTD_NOP(code);\ - CSTD_UNUSED(kctx);\ - CSTD_UNUSED(katom);\ - CSTD_UNUSED(gpu_addr);\ -+ CSTD_UNUSED(flags);\ - CSTD_UNUSED(refcount);\ -- CSTD_NOP(0);\ -- } while (0) -- --#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ -- gpu_addr, refcount, info_val) \ -- do {\ -- CSTD_UNUSED(kbdev);\ -- CSTD_NOP(code);\ -- CSTD_UNUSED(kctx);\ -- CSTD_UNUSED(katom);\ -- CSTD_UNUSED(gpu_addr);\ -- CSTD_UNUSED(info_val);\ -- CSTD_NOP(0);\ -- } while (0) -- --#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \ -- info_val)\ -- do {\ -- CSTD_UNUSED(kbdev);\ -- CSTD_NOP(code);\ -- CSTD_UNUSED(kctx);\ -- CSTD_UNUSED(katom);\ -- CSTD_UNUSED(gpu_addr);\ -+ CSTD_UNUSED(jobslot);\ - CSTD_UNUSED(info_val);\ - CSTD_NOP(0);\ - } while (0) -@@ -137,27 +71,30 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, - - /* - * KTrace target for Linux's ftrace -+ * -+ * Note: the header file(s) that define the trace_mali_<...> tracepoints are -+ * included by the parent header file - */ - #if KBASE_KTRACE_TARGET_FTRACE - #define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ - jobslot) \ -- trace_mali_##code(jobslot, 0) -+ trace_mali_##code(kctx, jobslot, 0) - - #define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \ - gpu_addr, jobslot, info_val) \ -- trace_mali_##code(jobslot, info_val) -+ trace_mali_##code(kctx, jobslot, info_val) - - #define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \ - gpu_addr, refcount) \ -- trace_mali_##code(refcount, 0) -+ trace_mali_##code(kctx, refcount, 0) - - #define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \ - gpu_addr, refcount, info_val) \ -- trace_mali_##code(refcount, info_val) -+ trace_mali_##code(kctx, refcount, info_val) - - #define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \ - info_val) \ -- trace_mali_##code(gpu_addr, info_val) -+ trace_mali_##code(kctx, gpu_addr, info_val) - #else /* KBASE_KTRACE_TARGET_FTRACE */ - #define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \ - jobslot) \ -@@ -247,7 +184,9 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - int __jobslot = jobslot; \ -- KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \ -+ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ -+ KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, \ -+ 0); \ - KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \ - } while (0) - -@@ -275,7 +214,9 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, - u64 __gpu_addr = gpu_addr; \ - int __jobslot = jobslot; \ - u64 __info_val = info_val; \ -- KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \ -+ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ -+ KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, __jobslot, \ -+ __info_val); \ - KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \ - } while (0) - -@@ -301,7 +242,9 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - int __refcount = refcount; \ -- KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \ -+ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ -+ KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, \ -+ 0u); \ - KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \ - } while (0) - -@@ -330,7 +273,9 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, - u64 __gpu_addr = gpu_addr; \ - int __refcount = refcount; \ - u64 __info_val = info_val; \ -- KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \ -+ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ -+ KBASE_KTRACE_FLAG_JM_REFCOUNT, __refcount, 0, \ -+ __info_val); \ - KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \ - } while (0) - -@@ -355,7 +300,8 @@ void kbasep_ktrace_add_jm(struct kbase_device *kbdev, - /* capture values that could come from non-pure function calls */ \ - u64 __gpu_addr = gpu_addr; \ - u64 __info_val = info_val; \ -- KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \ -+ KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, \ -+ 0u, 0, 0, __info_val); \ - KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \ - } while (0) - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_linux_ktrace_csf.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_linux_ktrace_csf.h -new file mode 100644 -index 0000000..4b23fc9 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_linux_ktrace_csf.h -@@ -0,0 +1,241 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * NOTE: This must **only** be included through mali_linux_trace.h, -+ * otherwise it will fail to setup tracepoints correctly -+ */ -+ -+#if !defined(_KBASE_DEBUG_LINUX_KTRACE_CSF_H_) || defined(TRACE_HEADER_MULTI_READ) -+#define _KBASE_DEBUG_LINUX_KTRACE_CSF_H_ -+ -+/* -+ * Generic CSF events - using the common DEFINE_MALI_ADD_EVENT -+ */ -+DEFINE_MALI_ADD_EVENT(EVICT_CTX_SLOTS); -+DEFINE_MALI_ADD_EVENT(FIRMWARE_BOOT); -+DEFINE_MALI_ADD_EVENT(FIRMWARE_REBOOT); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_TOCK); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_TOCK_END); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_TICK); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_TICK_END); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_RESET); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_WAIT_PROTM_QUIT); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_WAIT_PROTM_QUIT_DONE); -+DEFINE_MALI_ADD_EVENT(SYNC_UPDATE_EVENT); -+DEFINE_MALI_ADD_EVENT(SYNC_UPDATE_EVENT_NOTIFY_GPU); -+DEFINE_MALI_ADD_EVENT(CSF_INTERRUPT); -+DEFINE_MALI_ADD_EVENT(CSF_INTERRUPT_END); -+DEFINE_MALI_ADD_EVENT(CSG_INTERRUPT_PROCESS); -+DEFINE_MALI_ADD_EVENT(GLB_REQ_ACQ); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_CAN_IDLE); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_ADVANCE_TICK); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_NOADVANCE_TICK); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_INSERT_RUNNABLE); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_REMOVE_RUNNABLE); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_ROTATE_RUNNABLE); -+DEFINE_MALI_ADD_EVENT(SCHEDULER_HEAD_RUNNABLE); -+DEFINE_MALI_ADD_EVENT(IDLE_WORKER_BEGIN); -+DEFINE_MALI_ADD_EVENT(IDLE_WORKER_END); -+DEFINE_MALI_ADD_EVENT(GROUP_SYNC_UPDATE_WORKER_BEGIN); -+DEFINE_MALI_ADD_EVENT(GROUP_SYNC_UPDATE_WORKER_END); -+DEFINE_MALI_ADD_EVENT(SLOTS_STATUS_UPDATE_ACK); -+ -+DECLARE_EVENT_CLASS(mali_csf_grp_q_template, -+ TP_PROTO(struct kbase_device *kbdev, struct kbase_queue_group *group, -+ struct kbase_queue *queue, u64 info_val), -+ TP_ARGS(kbdev, group, queue, info_val), -+ TP_STRUCT__entry( -+ __field(u64, info_val) -+ __field(pid_t, kctx_tgid) -+ __field(u32, kctx_id) -+ __field(u8, group_handle) -+ __field(s8, csg_nr) -+ __field(u8, slot_prio) -+ __field(s8, csi_index) -+ ), -+ TP_fast_assign( -+ { -+ struct kbase_context *kctx = NULL; -+ -+ __entry->info_val = info_val; -+ /* Note: if required in future, we could record some -+ * flags in __entry about whether the group/queue parts -+ * are valid, and add that to the trace message e.g. -+ * by using __print_flags()/__print_symbolic() -+ */ -+ if (queue) { -+ /* Note: kctx overridden by group->kctx later if group is valid */ -+ kctx = queue->kctx; -+ __entry->csi_index = queue->csi_index; -+ } else { -+ __entry->csi_index = -1; -+ } -+ -+ if (group) { -+ kctx = group->kctx; -+ __entry->group_handle = group->handle; -+ __entry->csg_nr = group->csg_nr; -+ if (group->csg_nr >= 0) -+ __entry->slot_prio = kbdev->csf.scheduler.csg_slots[group->csg_nr].priority; -+ else -+ __entry->slot_prio = 0u; -+ } else { -+ __entry->group_handle = 0u; -+ __entry->csg_nr = -1; -+ __entry->slot_prio = 0u; -+ } -+ __entry->kctx_id = (kctx) ? kctx->id : 0u; -+ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; -+ } -+ -+ ), -+ TP_printk("kctx=%d_%u group=%u slot=%d prio=%u csi=%d info=0x%llx", -+ __entry->kctx_tgid, __entry->kctx_id, -+ __entry->group_handle, __entry->csg_nr, -+ __entry->slot_prio, __entry->csi_index, -+ __entry->info_val) -+); -+ -+/* -+ * Group events -+ */ -+#define DEFINE_MALI_CSF_GRP_EVENT(name) \ -+ DEFINE_EVENT_PRINT(mali_csf_grp_q_template, mali_##name, \ -+ TP_PROTO(struct kbase_device *kbdev, struct kbase_queue_group *group, \ -+ struct kbase_queue *queue, u64 info_val), \ -+ TP_ARGS(kbdev, group, queue, info_val), \ -+ TP_printk("kctx=%d_%u group=%u slot=%d prio=%u info=0x%llx", \ -+ __entry->kctx_tgid, __entry->kctx_id, __entry->group_handle, \ -+ __entry->csg_nr, __entry->slot_prio, __entry->info_val)) -+ -+DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_START); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STOP); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STARTED); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STOPPED); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_CLEANED); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_STATUS_UPDATE); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_IDLE_SET); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_SLOT_IDLE_CLEAR); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_PRIO_UPDATE); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_SYNC_UPDATE_INTERRUPT); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_IDLE_INTERRUPT); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_PROGRESS_TIMER_INTERRUPT); -+DEFINE_MALI_CSF_GRP_EVENT(CSG_INTERRUPT_PROCESS_END); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_SYNC_UPDATE_DONE); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_DESCHEDULE); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_SCHEDULE); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_EVICT_SCHED); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_INSERT_RUNNABLE); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_REMOVE_RUNNABLE); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_ROTATE_RUNNABLE); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_HEAD_RUNNABLE); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_INSERT_IDLE_WAIT); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_REMOVE_IDLE_WAIT); -+DEFINE_MALI_CSF_GRP_EVENT(GROUP_HEAD_IDLE_WAIT); -+DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_CHECK_PROTM_ENTER); -+DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_ENTER_PROTM); -+DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_EXIT_PROTM); -+DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_TOP_GRP); -+DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_NONIDLE_OFFSLOT_INC); -+DEFINE_MALI_CSF_GRP_EVENT(SCHEDULER_NONIDLE_OFFSLOT_DEC); -+DEFINE_MALI_CSF_GRP_EVENT(PROTM_EVENT_WORKER_BEGIN); -+DEFINE_MALI_CSF_GRP_EVENT(PROTM_EVENT_WORKER_END); -+ -+#undef DEFINE_MALI_CSF_GRP_EVENT -+ -+/* -+ * Group + Queue events -+ */ -+#define DEFINE_MALI_CSF_GRP_Q_EVENT(name) \ -+ DEFINE_EVENT(mali_csf_grp_q_template, mali_##name, \ -+ TP_PROTO(struct kbase_device *kbdev, struct kbase_queue_group *group, \ -+ struct kbase_queue *queue, u64 info_val), \ -+ TP_ARGS(kbdev, group, queue, info_val)) -+ -+DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_START); -+DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_STOP); -+DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_STOP_REQUESTED); -+DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_IGNORED_INTERRUPTS_GROUP_SUSPEND); -+DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_FAULT_INTERRUPT); -+DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_TILER_OOM_INTERRUPT); -+DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_PROTM_PEND_INTERRUPT); -+DEFINE_MALI_CSF_GRP_Q_EVENT(CSI_PROTM_ACK); -+DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_START); -+DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_STOP); -+DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_SYNC_UPDATE); -+DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_SYNC_UPDATE_EVALUATED); -+DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_SYNC_STATUS_WAIT); -+DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_SYNC_CURRENT_VAL); -+DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_SYNC_TEST_VAL); -+DEFINE_MALI_CSF_GRP_Q_EVENT(QUEUE_SYNC_BLOCKED_REASON); -+DEFINE_MALI_CSF_GRP_Q_EVENT(PROTM_PENDING_SET); -+DEFINE_MALI_CSF_GRP_Q_EVENT(PROTM_PENDING_CLEAR); -+ -+#undef DEFINE_MALI_CSF_GRP_Q_EVENT -+ -+/* -+ * KCPU queue events -+ */ -+DECLARE_EVENT_CLASS(mali_csf_kcpu_queue_template, -+ TP_PROTO(struct kbase_kcpu_command_queue *queue, -+ u64 info_val1, u64 info_val2), -+ TP_ARGS(queue, info_val1, info_val2), -+ TP_STRUCT__entry( -+ __field(u64, info_val1) -+ __field(u64, info_val2) -+ __field(pid_t, kctx_tgid) -+ __field(u32, kctx_id) -+ __field(u8, id) -+ ), -+ TP_fast_assign( -+ { -+ __entry->info_val1 = info_val1; -+ __entry->info_val2 = info_val2; -+ __entry->kctx_id = queue->kctx->id; -+ __entry->kctx_tgid = queue->kctx->tgid; -+ __entry->id = queue->id; -+ } -+ -+ ), -+ TP_printk("kctx=%d_%u id=%u info_val1=0x%llx info_val2=0x%llx", -+ __entry->kctx_tgid, __entry->kctx_id, __entry->id, -+ __entry->info_val1, __entry->info_val2) -+); -+ -+#define DEFINE_MALI_CSF_KCPU_EVENT(name) \ -+ DEFINE_EVENT(mali_csf_kcpu_queue_template, mali_##name, \ -+ TP_PROTO(struct kbase_kcpu_command_queue *queue, \ -+ u64 info_val1, u64 info_val2), \ -+ TP_ARGS(queue, info_val1, info_val2)) -+ -+DEFINE_MALI_CSF_KCPU_EVENT(KCPU_QUEUE_NEW); -+DEFINE_MALI_CSF_KCPU_EVENT(KCPU_QUEUE_DESTROY); -+DEFINE_MALI_CSF_KCPU_EVENT(CQS_SET); -+DEFINE_MALI_CSF_KCPU_EVENT(CQS_WAIT_START); -+DEFINE_MALI_CSF_KCPU_EVENT(CQS_WAIT_END); -+DEFINE_MALI_CSF_KCPU_EVENT(FENCE_SIGNAL); -+DEFINE_MALI_CSF_KCPU_EVENT(FENCE_WAIT_START); -+DEFINE_MALI_CSF_KCPU_EVENT(FENCE_WAIT_END); -+ -+#undef DEFINE_MALI_CSF_KCPU_EVENT -+ -+#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_CSF_H_) || defined(TRACE_HEADER_MULTI_READ) */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_linux_ktrace_jm.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_linux_ktrace_jm.h -index d964e5a..2e88e69 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_linux_ktrace_jm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/backend/mali_kbase_debug_linux_ktrace_jm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -29,23 +28,28 @@ - #define _KBASE_DEBUG_LINUX_KTRACE_JM_H_ - - DECLARE_EVENT_CLASS(mali_jm_slot_template, -- TP_PROTO(int jobslot, u64 info_val), -- TP_ARGS(jobslot, info_val), -+ TP_PROTO(struct kbase_context *kctx, int jobslot, u64 info_val), -+ TP_ARGS(kctx, jobslot, info_val), - TP_STRUCT__entry( -+ __field(pid_t, kctx_tgid) -+ __field(u32, kctx_id) - __field(unsigned int, jobslot) - __field(u64, info_val) - ), - TP_fast_assign( -+ __entry->kctx_id = (kctx) ? kctx->id : 0u; -+ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->jobslot = jobslot; - __entry->info_val = info_val; - ), -- TP_printk("jobslot=%u info=0x%llx", __entry->jobslot, __entry->info_val) -+ TP_printk("kctx=%d_%u jobslot=%u info=0x%llx", __entry->kctx_tgid, -+ __entry->kctx_id, __entry->jobslot, __entry->info_val) - ); - - #define DEFINE_MALI_JM_SLOT_EVENT(name) \ - DEFINE_EVENT(mali_jm_slot_template, mali_##name, \ -- TP_PROTO(int jobslot, u64 info_val), \ -- TP_ARGS(jobslot, info_val)) -+ TP_PROTO(struct kbase_context *kctx, int jobslot, u64 info_val), \ -+ TP_ARGS(kctx, jobslot, info_val)) - DEFINE_MALI_JM_SLOT_EVENT(JM_SUBMIT); - DEFINE_MALI_JM_SLOT_EVENT(JM_JOB_DONE); - DEFINE_MALI_JM_SLOT_EVENT(JM_UPDATE_HEAD); -@@ -75,23 +79,28 @@ DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ); - #undef DEFINE_MALI_JM_SLOT_EVENT - - DECLARE_EVENT_CLASS(mali_jm_refcount_template, -- TP_PROTO(int refcount, u64 info_val), -- TP_ARGS(refcount, info_val), -+ TP_PROTO(struct kbase_context *kctx, int refcount, u64 info_val), -+ TP_ARGS(kctx, refcount, info_val), - TP_STRUCT__entry( -+ __field(pid_t, kctx_tgid) -+ __field(u32, kctx_id) - __field(unsigned int, refcount) - __field(u64, info_val) - ), - TP_fast_assign( -+ __entry->kctx_id = (kctx) ? kctx->id : 0u; -+ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->refcount = refcount; - __entry->info_val = info_val; - ), -- TP_printk("refcount=%u info=0x%llx", __entry->refcount, __entry->info_val) -+ TP_printk("kctx=%d_%u refcount=%u info=0x%llx", __entry->kctx_tgid, -+ __entry->kctx_id, __entry->refcount, __entry->info_val) - ); - - #define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \ - DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \ -- TP_PROTO(int refcount, u64 info_val), \ -- TP_ARGS(refcount, info_val)) -+ TP_PROTO(struct kbase_context *kctx, int refcount, u64 info_val), \ -+ TP_ARGS(kctx, refcount, info_val)) - DEFINE_MALI_JM_REFCOUNT_EVENT(JS_ADD_JOB); - DEFINE_MALI_JM_REFCOUNT_EVENT(JS_REMOVE_JOB); - DEFINE_MALI_JM_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX); -@@ -106,23 +115,28 @@ DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS); - #undef DEFINE_MALI_JM_REFCOUNT_EVENT - - DECLARE_EVENT_CLASS(mali_jm_add_template, -- TP_PROTO(u64 gpu_addr, u64 info_val), -- TP_ARGS(gpu_addr, info_val), -+ TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), -+ TP_ARGS(kctx, gpu_addr, info_val), - TP_STRUCT__entry( -+ __field(pid_t, kctx_tgid) -+ __field(u32, kctx_id) - __field(u64, gpu_addr) - __field(u64, info_val) - ), - TP_fast_assign( -+ __entry->kctx_id = (kctx) ? kctx->id : 0u; -+ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->gpu_addr = gpu_addr; - __entry->info_val = info_val; - ), -- TP_printk("gpu_addr=0x%llx info=0x%llx", __entry->gpu_addr, __entry->info_val) -+ TP_printk("kctx=%d_%u gpu_addr=0x%llx info=0x%llx", __entry->kctx_tgid, -+ __entry->kctx_id, __entry->gpu_addr, __entry->info_val) - ); - - #define DEFINE_MALI_JM_ADD_EVENT(name) \ - DEFINE_EVENT(mali_jm_add_template, mali_##name, \ -- TP_PROTO(u64 gpu_addr, u64 info_val), \ -- TP_ARGS(gpu_addr, info_val)) -+ TP_PROTO(struct kbase_context *kctx, u64 gpu_addr, u64 info_val), \ -+ TP_ARGS(kctx, gpu_addr, info_val)) - DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER); - DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER_END); - DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL_WORKER); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace.c b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace.c -index 6322abb..9bf8610 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,9 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ -+ - #include <mali_kbase.h> - #include "debug/mali_kbase_debug_ktrace_internal.h" - -@@ -27,11 +27,6 @@ int kbase_ktrace_init(struct kbase_device *kbdev) - #if KBASE_KTRACE_TARGET_RBUF - struct kbase_ktrace_msg *rbuf; - -- /* See also documentation of enum kbase_ktrace_code */ -- compiletime_assert(sizeof(kbase_ktrace_code_t) == sizeof(unsigned long long) || -- KBASE_KTRACE_CODE_COUNT <= (1ull << (sizeof(kbase_ktrace_code_t) * BITS_PER_BYTE)), -- "kbase_ktrace_code_t not wide enough for KBASE_KTRACE_CODE_COUNT"); -- - rbuf = kmalloc_array(KBASE_KTRACE_SIZE, sizeof(*rbuf), GFP_KERNEL); - - if (!rbuf) -@@ -91,15 +86,25 @@ static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg, - - /* Initial part of message: - * -- * secs,thread_id,cpu,code,kctx, -+ * secs,thread_id,cpu,code, - */ - written += MAX(snprintf(buffer + written, MAX(sz - written, 0), -- "%d.%.6d,%d,%d,%s,%p,", -+ "%d.%.6d,%d,%d,%s,", - (int)trace_msg->timestamp.tv_sec, - (int)(trace_msg->timestamp.tv_nsec / 1000), - trace_msg->thread_id, trace_msg->cpu, -- kbasep_ktrace_code_string[trace_msg->backend.code], -- trace_msg->kctx), 0); -+ kbasep_ktrace_code_string[trace_msg->backend.gpu.code]), -+ 0); -+ -+ /* kctx part: */ -+ if (trace_msg->kctx_tgid) { -+ written += MAX(snprintf(buffer + written, MAX(sz - written, 0), -+ "%d_%u", -+ trace_msg->kctx_tgid, trace_msg->kctx_id), 0); -+ } -+ /* Trailing comma */ -+ written += MAX(snprintf(buffer + written, MAX(sz - written, 0), -+ ","), 0); - - /* Backend parts */ - kbasep_ktrace_backend_format_msg(trace_msg, buffer, sz, -@@ -156,11 +161,19 @@ void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace, - - ktime_get_real_ts64(&trace_msg->timestamp); - -- trace_msg->kctx = kctx; -- -+ /* No need to store a flag about whether there was a kctx, tgid==0 is -+ * sufficient -+ */ -+ if (kctx) { -+ trace_msg->kctx_tgid = kctx->tgid; -+ trace_msg->kctx_id = kctx->id; -+ } else { -+ trace_msg->kctx_tgid = 0; -+ trace_msg->kctx_id = 0; -+ } - trace_msg->info_val = info_val; -- trace_msg->backend.code = code; -- trace_msg->backend.flags = flags; -+ trace_msg->backend.gpu.code = code; -+ trace_msg->backend.gpu.flags = flags; - } - - void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, -@@ -170,12 +183,14 @@ void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code, - unsigned long irqflags; - struct kbase_ktrace_msg *trace_msg; - -+ WARN_ON((flags & ~KBASE_KTRACE_FLAG_COMMON_ALL)); -+ - spin_lock_irqsave(&kbdev->ktrace.lock, irqflags); - - /* Reserve and update indices */ - trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace); - -- /* Fill the common part of the message (including backend.flags) */ -+ /* Fill the common part of the message (including backend.gpu.flags) */ - kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags, - info_val); - -@@ -225,7 +240,7 @@ void kbasep_ktrace_dump(struct kbase_device *kbdev) - spin_unlock_irqrestore(&kbdev->ktrace.lock, flags); - } - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - struct trace_seq_state { - struct kbase_ktrace_msg trace_buf[KBASE_KTRACE_SIZE]; - u32 start; -@@ -333,7 +348,7 @@ void kbase_ktrace_debugfs_init(struct kbase_device *kbdev) - - #else /* KBASE_KTRACE_TARGET_RBUF */ - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - void kbase_ktrace_debugfs_init(struct kbase_device *kbdev) - { - CSTD_UNUSED(kbdev); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace.h -index 0dd8b7a..f943696 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -37,7 +36,15 @@ - #ifndef _KBASE_DEBUG_KTRACE_H_ - #define _KBASE_DEBUG_KTRACE_H_ - -+#if KBASE_KTRACE_TARGET_FTRACE -+#include "mali_linux_trace.h" -+#endif -+ -+#if MALI_USE_CSF -+#include "debug/backend/mali_kbase_debug_ktrace_csf.h" -+#else - #include "debug/backend/mali_kbase_debug_ktrace_jm.h" -+#endif - - /** - * kbase_ktrace_init - initialize kbase ktrace. -@@ -58,7 +65,7 @@ void kbase_ktrace_term(struct kbase_device *kbdev); - */ - void kbase_ktrace_hook_wrapper(void *param); - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - /** - * kbase_ktrace_debugfs_init - initialize kbase ktrace for debugfs usage, if - * the selected targets support it. -@@ -140,10 +147,9 @@ void kbasep_ktrace_dump(struct kbase_device *kbdev); - * KTrace target for Linux's ftrace - */ - #if KBASE_KTRACE_TARGET_FTRACE --#include "mali_linux_trace.h" - - #define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \ -- trace_mali_##code(info_val) -+ trace_mali_##code(kctx, info_val) - - #else /* KBASE_KTRACE_TARGET_FTRACE */ - #define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_codes.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_codes.h -index 364ed60..3309834 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_codes.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_codes.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2015, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -30,6 +29,9 @@ - * The purpose of this header file is just to contain a list of trace code - * identifiers - * -+ * When updating this file, also remember to update -+ * mali_kbase_debug_linux_ktrace.h -+ * - * Each identifier is wrapped in a macro, so that its string form and enum form - * can be created - * -@@ -112,6 +114,7 @@ int dummy_array[] = { - KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER), - KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE), - KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER), -+ KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_L2), - KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE), - KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER), - KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED), -@@ -142,9 +145,20 @@ int dummy_array[] = { - KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK), - /* info_val == kctx->refcount */ - KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX), -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ /* -+ * Arbitration events -+ */ -+ KBASE_KTRACE_CODE_MAKE_CODE(ARB_GPU_LOST), -+ KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_STATE), -+ KBASE_KTRACE_CODE_MAKE_CODE(ARB_VM_EVT), -+#endif - -- -+#if MALI_USE_CSF -+#include "debug/backend/mali_kbase_debug_ktrace_codes_csf.h" -+#else - #include "debug/backend/mali_kbase_debug_ktrace_codes_jm.h" -+#endif - /* - * Unused code just to make it easier to not have a comma at the end. - * All other codes MUST come before this -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_defs.h -index d6baaf1..4694b78 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_defs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_DEBUG_KTRACE_DEFS_H_ -@@ -58,6 +57,16 @@ - #define KBASE_KTRACE_TARGET_RBUF 0 - #endif /* KBASE_KTRACE_ENABLE */ - -+/* -+ * Note: Some backends define flags in this type even if the RBUF target is -+ * disabled (they get discarded with CSTD_UNUSED(), but they're still -+ * referenced) -+ */ -+typedef u8 kbase_ktrace_flag_t; -+ -+#if KBASE_KTRACE_TARGET_RBUF -+typedef u8 kbase_ktrace_code_t; -+ - /* - * NOTE: KBASE_KTRACE_VERSION_MAJOR, KBASE_KTRACE_VERSION_MINOR are kept in - * the backend, since updates can be made to one backend in a way that doesn't -@@ -67,20 +76,28 @@ - * updated. - */ - --#if KBASE_KTRACE_TARGET_RBUF --typedef u8 kbase_ktrace_flag_t; --typedef u8 kbase_ktrace_code_t; -- - /* -- * struct kbase_ktrace_backend - backend specific part of a trace message -- * -- * At the very least, this must contain a kbase_ktrace_code_t 'code' member and -- * a kbase_ktrace_flag_t 'flags' member -+ * union kbase_ktrace_backend - backend specific part of a trace message. -+ * At the very least, this must contain a kbase_ktrace_code_t 'code' member -+ * and a kbase_ktrace_flag_t 'flags' inside a "gpu" sub-struct. Should a -+ * backend need several sub structs in its union to optimize the data storage -+ * for different message types, then it can use a "common initial sequence" to -+ * allow 'flags' and 'code' to pack optimally without corrupting them. -+ * Different backends need not share common initial sequences between them, they -+ * only need to ensure they have gpu.flags and gpu.code members, it -+ * is up to the backend then how to order these. - */ --struct kbase_ktrace_backend; -+union kbase_ktrace_backend; -+ -+#endif /* KBASE_KTRACE_TARGET_RBUF */ - -+#if MALI_USE_CSF -+#include "debug/backend/mali_kbase_debug_ktrace_defs_csf.h" -+#else - #include "debug/backend/mali_kbase_debug_ktrace_defs_jm.h" -+#endif - -+#if KBASE_KTRACE_TARGET_RBUF - /* Indicates if the trace message has backend related info. - * - * If not set, consider the &kbase_ktrace_backend part of a &kbase_ktrace_msg -@@ -90,7 +107,14 @@ struct kbase_ktrace_backend; - */ - #define KBASE_KTRACE_FLAG_BACKEND (((kbase_ktrace_flag_t)1) << 7) - --#define KBASE_KTRACE_SHIFT 8 /* 256 entries */ -+/* Collect all the common flags together for debug checking */ -+#define KBASE_KTRACE_FLAG_COMMON_ALL \ -+ (KBASE_KTRACE_FLAG_BACKEND) -+ -+#define KBASE_KTRACE_FLAG_ALL \ -+ (KBASE_KTRACE_FLAG_COMMON_ALL | KBASE_KTRACE_FLAG_BACKEND_ALL) -+ -+#define KBASE_KTRACE_SHIFT (9) /* 512 entries */ - #define KBASE_KTRACE_SIZE (1 << KBASE_KTRACE_SHIFT) - #define KBASE_KTRACE_MASK ((1 << KBASE_KTRACE_SHIFT)-1) - -@@ -121,24 +145,23 @@ enum kbase_ktrace_code { - * added. - * @cpu: indicates which CPU the @thread_id was scheduled on when the - * trace message was added. -- * @kctx: Pointer to the kbase context for which the trace message was -- * added. Will be NULL for certain trace messages associated with -- * the &kbase_device itself, such as power management events. -- * Will point to the appropriate context corresponding to -- * backend-specific events. -+ * @kctx_tgid: Thread group ID of the &kbase_context associated with the -+ * message, or 0 if none associated. -+ * @kctx_id: Unique identifier of the &kbase_context associated with the -+ * message. Only valid if @kctx_tgid != 0. - * @info_val: value specific to the type of event being traced. Refer to the -- * specific code in enum kbase_ktrace_code -+ * specific code in enum kbase_ktrace_code. - * @backend: backend-specific trace information. All backends must implement -- * a minimum common set of members -+ * a minimum common set of members. - */ - struct kbase_ktrace_msg { - struct timespec64 timestamp; - u32 thread_id; - u32 cpu; -- void *kctx; -+ pid_t kctx_tgid; -+ u32 kctx_id; - u64 info_val; -- -- struct kbase_ktrace_backend backend; -+ union kbase_ktrace_backend backend; - }; - - struct kbase_ktrace { -@@ -148,5 +171,17 @@ struct kbase_ktrace { - struct kbase_ktrace_msg *rbuf; - }; - -+ -+static inline void kbase_ktrace_compiletime_asserts(void) -+{ -+ /* See also documentation of enum kbase_ktrace_code */ -+ compiletime_assert(sizeof(kbase_ktrace_code_t) == sizeof(unsigned long long) || -+ KBASE_KTRACE_CODE_COUNT <= (1ull << (sizeof(kbase_ktrace_code_t) * BITS_PER_BYTE)), -+ "kbase_ktrace_code_t not wide enough for KBASE_KTRACE_CODE_COUNT"); -+ compiletime_assert((KBASE_KTRACE_FLAG_BACKEND_ALL & KBASE_KTRACE_FLAG_COMMON_ALL) == 0, -+ "KTrace backend flags intersect with KTrace common flags"); -+ -+} -+ - #endif /* KBASE_KTRACE_TARGET_RBUF */ - #endif /* _KBASE_DEBUG_KTRACE_DEFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_internal.h -index e450760..d9bd351 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_internal.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_ktrace_internal.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_DEBUG_KTRACE_INTERNAL_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_linux_ktrace.h b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_linux_ktrace.h -index 18e4f7c..b56dec4 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_linux_ktrace.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/debug/mali_kbase_debug_linux_ktrace.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -31,21 +30,29 @@ - #if KBASE_KTRACE_TARGET_FTRACE - - DECLARE_EVENT_CLASS(mali_add_template, -- TP_PROTO(u64 info_val), -- TP_ARGS(info_val), -+ TP_PROTO(struct kbase_context *kctx, u64 info_val), -+ TP_ARGS(kctx, info_val), - TP_STRUCT__entry( -+ __field(pid_t, kctx_tgid) -+ __field(u32, kctx_id) - __field(u64, info_val) - ), - TP_fast_assign( -+ __entry->kctx_id = (kctx) ? kctx->id : 0u; -+ __entry->kctx_tgid = (kctx) ? kctx->tgid : 0; - __entry->info_val = info_val; - ), -- TP_printk("info=0x%llx", __entry->info_val) -+ TP_printk("kctx=%d_%u info=0x%llx", __entry->kctx_tgid, -+ __entry->kctx_id, __entry->info_val) - ); - -+/* DEFINE_MALI_ADD_EVENT is available also to backends for backend-specific -+ * simple trace codes -+ */ - #define DEFINE_MALI_ADD_EVENT(name) \ - DEFINE_EVENT(mali_add_template, mali_##name, \ -- TP_PROTO(u64 info_val), \ -- TP_ARGS(info_val)) -+ TP_PROTO(struct kbase_context *kctx, u64 info_val), \ -+ TP_ARGS(kctx, info_val)) - DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY); - DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM); - DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ); -@@ -78,6 +85,7 @@ DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE); - DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER); - DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE); - DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER); -+DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_L2); - DEFINE_MALI_ADD_EVENT(PM_GPU_ON); - DEFINE_MALI_ADD_EVENT(PM_GPU_OFF); - DEFINE_MALI_ADD_EVENT(PM_SET_POLICY); -@@ -89,10 +97,20 @@ DEFINE_MALI_ADD_EVENT(PM_CONTEXT_IDLE); - DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS); - DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK); - DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX); -+#ifdef CONFIG_MALI_ARBITER_SUPPORT - --#undef DEFINE_MALI_ADD_EVENT -+DEFINE_MALI_ADD_EVENT(ARB_GPU_LOST); -+DEFINE_MALI_ADD_EVENT(ARB_VM_STATE); -+DEFINE_MALI_ADD_EVENT(ARB_VM_EVT); - --#include "mali_kbase_debug_linux_ktrace_jm.h" -+#endif -+#if MALI_USE_CSF -+#include "backend/mali_kbase_debug_linux_ktrace_csf.h" -+#else -+#include "backend/mali_kbase_debug_linux_ktrace_jm.h" -+#endif -+ -+#undef DEFINE_MALI_ADD_EVENT - - #endif /* KBASE_KTRACE_TARGET_FTRACE */ - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Makefile b/dvalin/kernel/drivers/gpu/arm/midgard/device/Kbuild -similarity index 56% -rename from dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Makefile -rename to dvalin/kernel/drivers/gpu/arm/midgard/device/Kbuild -index d848e87..90e7024 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Makefile -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/device/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,21 +16,18 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # --# -- --# linux build system bootstrap for out-of-tree module - --# default to building for the host --ARCH ?= $(shell uname -m) -+mali_kbase-y += \ -+ device/mali_kbase_device.o \ -+ device/mali_kbase_device_hw.o - --ifeq ($(KDIR),) --$(error Must specify KDIR to point to the kernel to target)) -+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) -+ mali_kbase-y += \ -+ device/backend/mali_kbase_device_csf.o \ -+ device/backend/mali_kbase_device_hw_csf.o -+else -+ mali_kbase-y += \ -+ device/backend/mali_kbase_device_jm.o \ -+ device/backend/mali_kbase_device_hw_jm.o - endif -- --all: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS=-I$(CURDIR)/../include modules -- --clean: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_csf.c -new file mode 100644 -index 0000000..0c5052b ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_csf.c -@@ -0,0 +1,464 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <device/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> -+ -+#include <mali_kbase_hwaccess_backend.h> -+#include <mali_kbase_hwcnt_backend_csf_if_fw.h> -+#include <mali_kbase_ctx_sched.h> -+#include <mali_kbase_reset_gpu.h> -+#include <csf/mali_kbase_csf.h> -+#include <csf/ipa_control/mali_kbase_csf_ipa_control.h> -+ -+ -+#include <mali_kbase.h> -+#include <backend/gpu/mali_kbase_irq_internal.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> -+#include <backend/gpu/mali_kbase_js_internal.h> -+#include <backend/gpu/mali_kbase_clk_rate_trace_mgr.h> -+#include <csf/mali_kbase_csf_csg_debugfs.h> -+#include <mali_kbase_hwcnt_virtualizer.h> -+#include <mali_kbase_vinstr.h> -+ -+/** -+ * kbase_device_firmware_hwcnt_term - Terminate CSF firmware and HWC -+ * -+ * @kbdev: An instance of the GPU platform device, allocated from the probe -+ * method of the driver. -+ * -+ * When a kbase driver is removed, terminate CSF firmware and hardware counter -+ * components. -+ */ -+static void kbase_device_firmware_hwcnt_term(struct kbase_device *kbdev) -+{ -+ if (kbdev->csf.firmware_inited) { -+ kbase_vinstr_term(kbdev->vinstr_ctx); -+ kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt); -+ kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface); -+ kbase_csf_firmware_term(kbdev); -+ } -+} -+ -+/** -+ * kbase_backend_late_init - Perform any backend-specific initialization. -+ * @kbdev: Device pointer -+ * -+ * Return: 0 on success, or an error code on failure. -+ */ -+static int kbase_backend_late_init(struct kbase_device *kbdev) -+{ -+ int err; -+ -+ err = kbase_hwaccess_pm_init(kbdev); -+ if (err) -+ return err; -+ -+ err = kbase_reset_gpu_init(kbdev); -+ if (err) -+ goto fail_reset_gpu_init; -+ -+ err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT); -+ if (err) -+ goto fail_pm_powerup; -+ -+ err = kbase_backend_timer_init(kbdev); -+ if (err) -+ goto fail_timer; -+ -+#ifdef CONFIG_MALI_DEBUG -+#ifndef CONFIG_MALI_NO_MALI -+ if (kbasep_common_test_interrupt_handlers(kbdev) != 0) { -+ dev_err(kbdev->dev, "Interrupt assignment check failed.\n"); -+ err = -EINVAL; -+ goto fail_interrupt_test; -+ } -+#endif /* !CONFIG_MALI_NO_MALI */ -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ kbase_ipa_control_init(kbdev); -+ -+ /* Initialise the metrics subsystem, it couldn't be initialized earlier -+ * due to dependency on kbase_ipa_control. -+ */ -+ err = kbasep_pm_metrics_init(kbdev); -+ if (err) -+ goto fail_pm_metrics_init; -+ -+ /* Do the initialisation of devfreq. -+ * Devfreq needs backend_timer_init() for completion of its -+ * initialisation and it also needs to catch the first callback -+ * occurrence of the runtime_suspend event for maintaining state -+ * coherence with the backend power management, hence needs to be -+ * placed before the kbase_pm_context_idle(). -+ */ -+ err = kbase_backend_devfreq_init(kbdev); -+ if (err) -+ goto fail_devfreq_init; -+ -+ /* Update gpuprops with L2_FEATURES if applicable */ -+ err = kbase_gpuprops_update_l2_features(kbdev); -+ if (err) -+ goto fail_update_l2_features; -+ -+ init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); -+ -+ kbase_pm_context_idle(kbdev); -+ -+ mutex_init(&kbdev->fw_load_lock); -+ -+ return 0; -+ -+fail_update_l2_features: -+ kbase_backend_devfreq_term(kbdev); -+fail_devfreq_init: -+ kbasep_pm_metrics_term(kbdev); -+fail_pm_metrics_init: -+ kbase_ipa_control_term(kbdev); -+ -+#ifdef CONFIG_MALI_DEBUG -+#ifndef CONFIG_MALI_NO_MALI -+fail_interrupt_test: -+#endif /* !CONFIG_MALI_NO_MALI */ -+#endif /* CONFIG_MALI_DEBUG */ -+ -+ kbase_backend_timer_term(kbdev); -+fail_timer: -+ kbase_pm_context_idle(kbdev); -+ kbase_hwaccess_pm_halt(kbdev); -+fail_pm_powerup: -+ kbase_reset_gpu_term(kbdev); -+fail_reset_gpu_init: -+ kbase_hwaccess_pm_term(kbdev); -+ -+ return err; -+} -+ -+/** -+ * kbase_backend_late_term - Perform any backend-specific termination. -+ * @kbdev: Device pointer -+ */ -+static void kbase_backend_late_term(struct kbase_device *kbdev) -+{ -+ kbase_backend_devfreq_term(kbdev); -+ kbasep_pm_metrics_term(kbdev); -+ kbase_ipa_control_term(kbdev); -+ kbase_hwaccess_pm_halt(kbdev); -+ kbase_reset_gpu_term(kbdev); -+ kbase_hwaccess_pm_term(kbdev); -+} -+ -+/** -+ * kbase_csf_early_init - Early initialization for firmware & scheduler. -+ * @kbdev: Device pointer -+ * -+ * Return: 0 on success, error code otherwise. -+ */ -+static int kbase_csf_early_init(struct kbase_device *kbdev) -+{ -+ int err = kbase_csf_firmware_early_init(kbdev); -+ -+ if (err) -+ return err; -+ -+ err = kbase_csf_scheduler_early_init(kbdev); -+ -+ return err; -+} -+ -+/** -+ * kbase_csf_early_init - Early termination for firmware & scheduler. -+ * @kbdev: Device pointer -+ */ -+static void kbase_csf_early_term(struct kbase_device *kbdev) -+{ -+ kbase_csf_scheduler_early_term(kbdev); -+} -+ -+/** -+ * kbase_device_hwcnt_backend_csf_if_init - Create hardware counter backend -+ * firmware interface. -+ * @kbdev: Device pointer -+ */ -+static int kbase_device_hwcnt_backend_csf_if_init(struct kbase_device *kbdev) -+{ -+ return kbase_hwcnt_backend_csf_if_fw_create( -+ kbdev, &kbdev->hwcnt_backend_csf_if_fw); -+} -+ -+/** -+ * kbase_device_hwcnt_backend_csf_if_term - Terminate hardware counter backend -+ * firmware interface. -+ * @kbdev: Device pointer -+ */ -+static void kbase_device_hwcnt_backend_csf_if_term(struct kbase_device *kbdev) -+{ -+ kbase_hwcnt_backend_csf_if_fw_destroy(&kbdev->hwcnt_backend_csf_if_fw); -+} -+ -+/** -+ * kbase_device_hwcnt_backend_csf_init - Create hardware counter backend. -+ * @kbdev: Device pointer -+ */ -+ -+static int kbase_device_hwcnt_backend_csf_init(struct kbase_device *kbdev) -+{ -+ return kbase_hwcnt_backend_csf_create( -+ &kbdev->hwcnt_backend_csf_if_fw, -+ KBASE_HWCNT_BACKEND_CSF_RING_BUFFER_COUNT, -+ &kbdev->hwcnt_gpu_iface); -+} -+ -+/** -+ * kbase_device_hwcnt_backend_csf_term - Terminate hardware counter backend. -+ * @kbdev: Device pointer -+ */ -+static void kbase_device_hwcnt_backend_csf_term(struct kbase_device *kbdev) -+{ -+ kbase_hwcnt_backend_csf_destroy(&kbdev->hwcnt_gpu_iface); -+} -+ -+static const struct kbase_device_init dev_init[] = { -+ { assign_irqs, NULL, "IRQ search failed" }, -+ { registers_map, registers_unmap, "Register map failed" }, -+ { power_control_init, power_control_term, -+ "Power control initialization failed" }, -+ { kbase_device_io_history_init, kbase_device_io_history_term, -+ "Register access history initialization failed" }, -+ { kbase_device_early_init, kbase_device_early_term, -+ "Early device initialization failed" }, -+ { kbase_device_populate_max_freq, NULL, -+ "Populating max frequency failed" }, -+ { kbase_device_misc_init, kbase_device_misc_term, -+ "Miscellaneous device initialization failed" }, -+ { kbase_device_pcm_dev_init, kbase_device_pcm_dev_term, -+ "Priority control manager initialization failed" }, -+ { kbase_ctx_sched_init, kbase_ctx_sched_term, -+ "Context scheduler initialization failed" }, -+ { kbase_mem_init, kbase_mem_term, -+ "Memory subsystem initialization failed" }, -+ { kbase_csf_protected_memory_init, kbase_csf_protected_memory_term, -+ "Protected memory allocator initialization failed" }, -+ { kbase_device_coherency_init, NULL, "Device coherency init failed" }, -+ { kbase_protected_mode_init, kbase_protected_mode_term, -+ "Protected mode subsystem initialization failed" }, -+ { kbase_device_list_init, kbase_device_list_term, -+ "Device list setup failed" }, -+ { kbase_device_timeline_init, kbase_device_timeline_term, -+ "Timeline stream initialization failed" }, -+ { kbase_clk_rate_trace_manager_init, kbase_clk_rate_trace_manager_term, -+ "Clock rate trace manager initialization failed" }, -+ { kbase_device_hwcnt_backend_csf_if_init, -+ kbase_device_hwcnt_backend_csf_if_term, -+ "GPU hwcnt backend CSF interface creation failed" }, -+ { kbase_device_hwcnt_backend_csf_init, -+ kbase_device_hwcnt_backend_csf_term, -+ "GPU hwcnt backend creation failed" }, -+ { kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, -+ "GPU hwcnt context initialization failed" }, -+ { kbase_backend_late_init, kbase_backend_late_term, -+ "Late backend initialization failed" }, -+ { kbase_csf_early_init, kbase_csf_early_term, -+ "Early CSF initialization failed" }, -+ { NULL, kbase_device_firmware_hwcnt_term, NULL }, -+#ifdef MALI_KBASE_BUILD -+ { kbase_device_debugfs_init, kbase_device_debugfs_term, -+ "DebugFS initialization failed" }, -+ /* Sysfs init needs to happen before registering the device with -+ * misc_register(), otherwise it causes a race condition between -+ * registering the device and a uevent event being generated for -+ * userspace, causing udev rules to run which might expect certain -+ * sysfs attributes present. As a result of the race condition -+ * we avoid, some Mali sysfs entries may have appeared to udev -+ * to not exist. -+ * For more information, see -+ * https://www.kernel.org/doc/Documentation/driver-model/device.txt, the -+ * paragraph that starts with "Word of warning", currently the -+ * second-last paragraph. -+ */ -+ { kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed" }, -+ { kbase_device_misc_register, kbase_device_misc_deregister, -+ "Misc device registration failed" }, -+ { kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer, -+ "GPU property population failed" }, -+ { kbase_device_late_init, kbase_device_late_term, -+ "Late device initialization failed" }, -+#endif -+}; -+ -+static void kbase_device_term_partial(struct kbase_device *kbdev, -+ unsigned int i) -+{ -+ while (i-- > 0) { -+ if (dev_init[i].term) -+ dev_init[i].term(kbdev); -+ } -+} -+ -+void kbase_device_term(struct kbase_device *kbdev) -+{ -+ kbdev->csf.mali_file_inode = NULL; -+ kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init)); -+ kbase_mem_halt(kbdev); -+} -+ -+int kbase_device_init(struct kbase_device *kbdev) -+{ -+ int err = 0; -+ unsigned int i = 0; -+ -+ dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME); -+ -+ kbase_device_id_init(kbdev); -+ kbase_disjoint_init(kbdev); -+ -+ for (i = 0; i < ARRAY_SIZE(dev_init); i++) { -+ if (dev_init[i].init) { -+ err = dev_init[i].init(kbdev); -+ if (err) { -+ dev_err(kbdev->dev, "%s error = %d\n", -+ dev_init[i].err_mes, err); -+ kbase_device_term_partial(kbdev, i); -+ break; -+ } -+ } -+ } -+ -+ return err; -+} -+ -+/** -+ * kbase_device_hwcnt_csf_deferred_init - Initialize CSF deferred HWC components -+ * -+ * @kbdev: An instance of the GPU platform device, allocated from the probe -+ * method of the driver. -+ * -+ * Hardware counter components depending on firmware are initialized after CSF -+ * firmware is loaded. -+ * -+ * @return 0 on success. An error code on failure. -+ */ -+static int kbase_device_hwcnt_csf_deferred_init(struct kbase_device *kbdev) -+{ -+ int ret = 0; -+ -+ /* For CSF GPUs, HWC metadata needs to query information from CSF -+ * firmware, so the initialization of HWC metadata only can be called -+ * after firmware initialized, but firmware initialization depends on -+ * HWC backend initialization, so we need to separate HWC backend -+ * metadata initialization from HWC backend initialization. -+ */ -+ ret = kbase_hwcnt_backend_csf_metadata_init(&kbdev->hwcnt_gpu_iface); -+ if (ret) { -+ dev_err(kbdev->dev, -+ "GPU hwcnt backend metadata creation failed"); -+ return ret; -+ } -+ -+ ret = kbase_hwcnt_virtualizer_init( -+ kbdev->hwcnt_gpu_ctx, -+ KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS, -+ &kbdev->hwcnt_gpu_virt); -+ if (ret) { -+ dev_err(kbdev->dev, -+ "GPU hwcnt virtualizer initialization failed"); -+ goto virt_fail; -+ } -+ -+ ret = kbase_vinstr_init(kbdev->hwcnt_gpu_virt, &kbdev->vinstr_ctx); -+ if (ret) { -+ dev_err(kbdev->dev, -+ "Virtual instrumentation initialization failed"); -+ goto vinstr_fail; -+ } -+ -+ return ret; -+ -+vinstr_fail: -+ kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt); -+ -+virt_fail: -+ kbase_hwcnt_backend_csf_metadata_term(&kbdev->hwcnt_gpu_iface); -+ return ret; -+} -+ -+/** -+ * kbase_csf_firmware_deferred_init - Load and initialize CSF firmware -+ * -+ * @kbdev: An instance of the GPU platform device, allocated from the probe -+ * method of the driver. -+ * -+ * Called when a device file is opened for the first time. -+ * To meet Android GKI vendor guideline, firmware load is deferred at -+ * the time when @ref kbase_open is called for the first time. -+ * -+ * @return 0 on success. An error code on failure. -+ */ -+static int kbase_csf_firmware_deferred_init(struct kbase_device *kbdev) -+{ -+ int err = 0; -+ -+ lockdep_assert_held(&kbdev->fw_load_lock); -+ -+ kbase_pm_context_active(kbdev); -+ -+ err = kbase_csf_firmware_init(kbdev); -+ if (!err) { -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->pm.backend.mcu_state = KBASE_MCU_ON; -+ kbdev->csf.firmware_inited = true; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } else { -+ dev_err(kbdev->dev, "Firmware initialization failed"); -+ } -+ -+ kbase_pm_context_idle(kbdev); -+ -+ return err; -+} -+ -+int kbase_device_firmware_init_once(struct kbase_device *kbdev) -+{ -+ int ret = 0; -+ -+ mutex_lock(&kbdev->fw_load_lock); -+ -+ if (!kbdev->csf.firmware_inited) { -+ ret = kbase_csf_firmware_deferred_init(kbdev); -+ if (ret) -+ goto out; -+ -+ ret = kbase_device_hwcnt_csf_deferred_init(kbdev); -+ if (ret) { -+ kbase_csf_firmware_term(kbdev); -+ goto out; -+ } -+ -+ kbase_csf_debugfs_init(kbdev); -+ } -+ -+out: -+ mutex_unlock(&kbdev->fw_load_lock); -+ -+ return ret; -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_hw_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_hw_csf.c -new file mode 100644 -index 0000000..3fce637 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_hw_csf.c -@@ -0,0 +1,163 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <gpu/mali_kbase_gpu_fault.h> -+#include <backend/gpu/mali_kbase_instr_internal.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> -+#include <device/mali_kbase_device.h> -+#include <mali_kbase_reset_gpu.h> -+#include <mmu/mali_kbase_mmu.h> -+#include <mali_kbase_ctx_sched.h> -+ -+/** -+ * kbase_report_gpu_fault - Report a GPU fault of the device. -+ * -+ * @kbdev: Kbase device pointer -+ * @status: Fault status -+ * @as_nr: Faulty address space -+ * @as_valid: true if address space is valid -+ * -+ * This function is called from the interrupt handler when a GPU fault occurs. -+ */ -+static void kbase_report_gpu_fault(struct kbase_device *kbdev, u32 status, -+ u32 as_nr, bool as_valid) -+{ -+ u64 address = (u64) kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; -+ -+ address |= kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); -+ -+ /* Report GPU fault for all contexts in case either -+ * the address space is invalid or it's MCU address space. -+ */ -+ meson_gpu_fault ++; -+ kbase_mmu_gpu_fault_interrupt(kbdev, status, as_nr, address, as_valid); -+} -+ -+static void kbase_gpu_fault_interrupt(struct kbase_device *kbdev) -+{ -+ const u32 status = kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FAULTSTATUS)); -+ const bool as_valid = status & GPU_FAULTSTATUS_JASID_VALID_FLAG; -+ const u32 as_nr = (status & GPU_FAULTSTATUS_JASID_MASK) >> -+ GPU_FAULTSTATUS_JASID_SHIFT; -+ bool bus_fault = (status & GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) == -+ GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT; -+ -+ if (bus_fault) { -+ /* If as_valid, reset gpu when ASID is for MCU. */ -+ if (!as_valid || (as_nr == MCU_AS_NR)) { -+ kbase_report_gpu_fault(kbdev, status, as_nr, as_valid); -+ -+ dev_err(kbdev->dev, "GPU bus fault triggering gpu-reset ...\n"); -+ if (kbase_prepare_to_reset_gpu( -+ kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) -+ kbase_reset_gpu(kbdev); -+ } else { -+ /* Handle Bus fault */ -+ if (kbase_mmu_bus_fault_interrupt(kbdev, status, as_nr)) -+ dev_warn(kbdev->dev, -+ "fail to handle GPU bus fault ...\n"); -+ } -+ } else -+ kbase_report_gpu_fault(kbdev, status, as_nr, as_valid); -+} -+ -+void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) -+{ -+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); -+ if (val & GPU_FAULT) -+ kbase_gpu_fault_interrupt(kbdev); -+ -+ if (val & GPU_PROTECTED_FAULT) { -+ struct kbase_csf_scheduler *scheduler = &kbdev->csf.scheduler; -+ unsigned long flags; -+ -+ dev_err_ratelimited(kbdev->dev, "GPU fault in protected mode"); -+ -+ /* Mask the protected fault interrupt to avoid the potential -+ * deluge of such interrupts. It will be unmasked on GPU reset. -+ */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), -+ GPU_IRQ_REG_ALL & ~GPU_PROTECTED_FAULT); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ if (!WARN_ON(!kbase_csf_scheduler_protected_mode_in_use( -+ kbdev))) { -+ struct base_gpu_queue_group_error const -+ err_payload = { .error_type = -+ BASE_GPU_QUEUE_GROUP_ERROR_FATAL, -+ .payload = { -+ .fatal_group = { -+ .status = -+ GPU_EXCEPTION_TYPE_SW_FAULT_0, -+ } } }; -+ -+ scheduler->active_protm_grp->faulted = true; -+ kbase_csf_add_group_fatal_error( -+ scheduler->active_protm_grp, &err_payload); -+ kbase_event_wakeup(scheduler->active_protm_grp->kctx); -+ } -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+ -+ if (kbase_prepare_to_reset_gpu( -+ kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) -+ kbase_reset_gpu(kbdev); -+ } -+ -+ if (val & RESET_COMPLETED) -+ kbase_pm_reset_done(kbdev); -+ -+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val); -+ -+ /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must -+ * be called after the IRQ has been cleared. This is because it might -+ * trigger further power transitions and we don't want to miss the -+ * interrupt raised to notify us that these further transitions have -+ * finished. The same applies to kbase_clean_caches_done() - if another -+ * clean was queued, it might trigger another clean, which might -+ * generate another interrupt which shouldn't be missed. -+ */ -+ -+ if (val & CLEAN_CACHES_COMPLETED) -+ kbase_clean_caches_done(kbdev); -+ -+ if (val & (POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ)) { -+ kbase_pm_power_changed(kbdev); -+ } else if (val & CLEAN_CACHES_COMPLETED) { -+ /* If cache line evict messages can be lost when shader cores -+ * power down then we need to flush the L2 cache before powering -+ * down cores. When the flush completes, the shaders' state -+ * machine needs to be re-invoked to proceed with powering down -+ * cores. -+ */ -+ if (kbdev->pm.backend.l2_always_on || -+ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) -+ kbase_pm_power_changed(kbdev); -+ } -+ -+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_hw_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_hw_jm.c -new file mode 100644 -index 0000000..384e385 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_hw_jm.c -@@ -0,0 +1,98 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <gpu/mali_kbase_gpu_fault.h> -+#include <backend/gpu/mali_kbase_instr_internal.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> -+#include <device/mali_kbase_device.h> -+#include <mali_kbase_reset_gpu.h> -+#include <mmu/mali_kbase_mmu.h> -+ -+/** -+ * kbase_report_gpu_fault - Report a GPU fault. -+ * @kbdev: Kbase device pointer -+ * @multiple: Zero if only GPU_FAULT was raised, non-zero if MULTIPLE_GPU_FAULTS -+ * was also set -+ * -+ * This function is called from the interrupt handler when a GPU fault occurs. -+ * It reports the details of the fault using dev_warn(). -+ */ -+static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple) -+{ -+ u32 status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS)); -+ u64 address = (u64) kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; -+ -+ address |= kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); -+ meson_gpu_fault ++; -+ dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx", -+ status, -+ kbase_gpu_exception_name(status & 0xFF), -+ address); -+ if (multiple) -+ dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n"); -+} -+ -+void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val) -+{ -+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val); -+ if (val & GPU_FAULT) -+ kbase_report_gpu_fault(kbdev, val & MULTIPLE_GPU_FAULTS); -+ -+ if (val & RESET_COMPLETED) -+ kbase_pm_reset_done(kbdev); -+ -+ if (val & PRFCNT_SAMPLE_COMPLETED) -+ kbase_instr_hwcnt_sample_done(kbdev); -+ -+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val); -+ -+ /* kbase_pm_check_transitions (called by kbase_pm_power_changed) must -+ * be called after the IRQ has been cleared. This is because it might -+ * trigger further power transitions and we don't want to miss the -+ * interrupt raised to notify us that these further transitions have -+ * finished. The same applies to kbase_clean_caches_done() - if another -+ * clean was queued, it might trigger another clean, which might -+ * generate another interrupt which shouldn't be missed. -+ */ -+ -+ if (val & CLEAN_CACHES_COMPLETED) -+ kbase_clean_caches_done(kbdev); -+ -+ if (val & POWER_CHANGED_ALL) { -+ kbase_pm_power_changed(kbdev); -+ } else if (val & CLEAN_CACHES_COMPLETED) { -+ /* If cache line evict messages can be lost when shader cores -+ * power down then we need to flush the L2 cache before powering -+ * down cores. When the flush completes, the shaders' state -+ * machine needs to be re-invoked to proceed with powering down -+ * cores. -+ */ -+ if (kbdev->pm.backend.l2_always_on || -+ kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_921)) -+ kbase_pm_power_changed(kbdev); -+ } -+ -+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_jm.c -index fbba2e7..6a6ab60 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_jm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/device/backend/mali_kbase_device_jm.c -@@ -1,12 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -17,21 +17,17 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#include "../mali_kbase_device_internal.h" --#include "../mali_kbase_device.h" -+#include <device/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> -+#include <mali_kbase_hwaccess_instr.h> - - #include <mali_kbase_config_defaults.h> - #include <mali_kbase_hwaccess_backend.h> - #include <mali_kbase_ctx_sched.h> - #include <mali_kbase_reset_gpu.h> - --#ifdef CONFIG_MALI_NO_MALI --#include <mali_kbase_model_linux.h> --#endif - - #ifdef CONFIG_MALI_ARBITER_SUPPORT - #include <arbiter/mali_kbase_arbiter_pm.h> -@@ -43,6 +39,7 @@ - #include <backend/gpu/mali_kbase_js_internal.h> - #include <backend/gpu/mali_kbase_pm_internal.h> - #include <mali_kbase_dummy_job_wa.h> -+#include <backend/gpu/mali_kbase_clk_rate_trace_mgr.h> - - /** - * kbase_backend_late_init - Perform any backend-specific initialization. -@@ -95,9 +92,6 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) - if (err) - goto fail_devfreq_init; - -- /* Idle the GPU and/or cores, if the policy wants it to */ -- kbase_pm_context_idle(kbdev); -- - /* Update gpuprops with L2_FEATURES if applicable */ - err = kbase_gpuprops_update_l2_features(kbdev); - if (err) -@@ -105,9 +99,15 @@ static int kbase_backend_late_init(struct kbase_device *kbdev) - - init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait); - -+ /* Idle the GPU and/or cores, if the policy wants it to */ -+ kbase_pm_context_idle(kbdev); -+ -+ mutex_init(&kbdev->fw_load_lock); -+ - return 0; - - fail_update_l2_features: -+ kbase_backend_devfreq_term(kbdev); - fail_devfreq_init: - kbase_job_slot_term(kbdev); - fail_job_slot: -@@ -120,6 +120,7 @@ fail_interrupt_test: - - kbase_backend_timer_term(kbdev); - fail_timer: -+ kbase_pm_context_idle(kbdev); - kbase_hwaccess_pm_halt(kbdev); - fail_pm_powerup: - kbase_reset_gpu_term(kbdev); -@@ -144,57 +145,65 @@ static void kbase_backend_late_term(struct kbase_device *kbdev) - kbase_hwaccess_pm_term(kbdev); - } - -+static int kbase_device_hwcnt_backend_jm_init(struct kbase_device *kbdev) -+{ -+ return kbase_hwcnt_backend_jm_create(kbdev, &kbdev->hwcnt_gpu_iface); -+} -+ -+static void kbase_device_hwcnt_backend_jm_term(struct kbase_device *kbdev) -+{ -+ kbase_hwcnt_backend_jm_destroy(&kbdev->hwcnt_gpu_iface); -+} -+ - static const struct kbase_device_init dev_init[] = { --#ifdef CONFIG_MALI_NO_MALI -- {kbase_gpu_device_create, kbase_gpu_device_destroy, -- "Dummy model initialization failed"}, --#else -- {assign_irqs, NULL, -- "IRQ search failed"}, -- {registers_map, registers_unmap, -- "Register map failed"}, --#endif -- {kbase_device_io_history_init, kbase_device_io_history_term, -- "Register access history initialization failed"}, -- {kbase_device_pm_init, kbase_device_pm_term, -- "Power management initialization failed"}, -- {kbase_device_early_init, kbase_device_early_term, -- "Early device initialization failed"}, -- {kbase_device_populate_max_freq, NULL, -- "Populating max frequency failed"}, -- {kbase_device_misc_init, kbase_device_misc_term, -- "Miscellaneous device initialization failed"}, -- {kbase_ctx_sched_init, kbase_ctx_sched_term, -- "Context scheduler initialization failed"}, -- {kbase_mem_init, kbase_mem_term, -- "Memory subsystem initialization failed"}, -- {kbase_device_coherency_init, NULL, -- "Device coherency init failed"}, -- {kbase_protected_mode_init, kbase_protected_mode_term, -- "Protected mode subsystem initialization failed"}, -- {kbase_device_list_init, kbase_device_list_term, -- "Device list setup failed"}, -- {kbasep_js_devdata_init, kbasep_js_devdata_term, -- "Job JS devdata initialization failed"}, -- {kbase_device_timeline_init, kbase_device_timeline_term, -- "Timeline stream initialization failed"}, -- {kbase_device_hwcnt_backend_gpu_init, -- kbase_device_hwcnt_backend_gpu_term, -- "GPU hwcnt backend creation failed"}, -- {kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, -- "GPU hwcnt context initialization failed"}, -- {kbase_device_hwcnt_virtualizer_init, -- kbase_device_hwcnt_virtualizer_term, -- "GPU hwcnt virtualizer initialization failed"}, -- {kbase_device_vinstr_init, kbase_device_vinstr_term, -- "Virtual instrumentation initialization failed"}, -- {kbase_backend_late_init, kbase_backend_late_term, -- "Late backend initialization failed"}, -+ { assign_irqs, NULL, "IRQ search failed" }, -+ { registers_map, registers_unmap, "Register map failed" }, -+ { kbase_device_io_history_init, kbase_device_io_history_term, -+ "Register access history initialization failed" }, -+ { kbase_device_pm_init, kbase_device_pm_term, -+ "Power management initialization failed" }, -+ { kbase_device_early_init, kbase_device_early_term, -+ "Early device initialization failed" }, -+ { kbase_device_populate_max_freq, NULL, -+ "Populating max frequency failed" }, -+ { kbase_device_misc_init, kbase_device_misc_term, -+ "Miscellaneous device initialization failed" }, -+ { kbase_device_pcm_dev_init, kbase_device_pcm_dev_term, -+ "Priority control manager initialization failed" }, -+ { kbase_ctx_sched_init, kbase_ctx_sched_term, -+ "Context scheduler initialization failed" }, -+ { kbase_mem_init, kbase_mem_term, -+ "Memory subsystem initialization failed" }, -+ { kbase_device_coherency_init, NULL, "Device coherency init failed" }, -+ { kbase_protected_mode_init, kbase_protected_mode_term, -+ "Protected mode subsystem initialization failed" }, -+ { kbase_device_list_init, kbase_device_list_term, -+ "Device list setup failed" }, -+ { kbasep_js_devdata_init, kbasep_js_devdata_term, -+ "Job JS devdata initialization failed" }, -+ { kbase_device_timeline_init, kbase_device_timeline_term, -+ "Timeline stream initialization failed" }, -+ { kbase_clk_rate_trace_manager_init, kbase_clk_rate_trace_manager_term, -+ "Clock rate trace manager initialization failed" }, -+ { kbase_instr_backend_init, kbase_instr_backend_term, -+ "Instrumentation backend initialization failed" }, -+ { kbase_device_hwcnt_backend_jm_init, -+ kbase_device_hwcnt_backend_jm_term, -+ "GPU hwcnt backend creation failed" }, -+ { kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term, -+ "GPU hwcnt context initialization failed" }, -+ { kbase_device_hwcnt_virtualizer_init, -+ kbase_device_hwcnt_virtualizer_term, -+ "GPU hwcnt virtualizer initialization failed" }, -+ { kbase_device_vinstr_init, kbase_device_vinstr_term, -+ "Virtual instrumentation initialization failed" }, -+ { kbase_backend_late_init, kbase_backend_late_term, -+ "Late backend initialization failed" }, - #ifdef MALI_KBASE_BUILD -- {kbase_debug_job_fault_dev_init, kbase_debug_job_fault_dev_term, -- "Job fault debug initialization failed"}, -- {kbase_device_debugfs_init, kbase_device_debugfs_term, -- "DebugFS initialization failed"}, -+ { kbase_debug_job_fault_dev_init, kbase_debug_job_fault_dev_term, -+ "Job fault debug initialization failed" }, -+ { kbase_device_debugfs_init, kbase_device_debugfs_term, -+ "DebugFS initialization failed" }, - /* Sysfs init needs to happen before registering the device with - * misc_register(), otherwise it causes a race condition between - * registering the device and a uevent event being generated for -@@ -207,17 +216,15 @@ static const struct kbase_device_init dev_init[] = { - * paragraph that starts with "Word of warning", currently the - * second-last paragraph. - */ -- {kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed"}, -- {kbase_device_misc_register, kbase_device_misc_deregister, -- "Misc device registration failed"}, --#ifdef CONFIG_MALI_BUSLOG -- {buslog_init, buslog_term, "Bus log client registration failed"}, -+ { kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed" }, -+ { kbase_device_misc_register, kbase_device_misc_deregister, -+ "Misc device registration failed" }, -+ { kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer, -+ "GPU property population failed" }, - #endif -- {kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer, -- "GPU property population failed"}, --#endif -- {kbase_dummy_job_wa_load, kbase_dummy_job_wa_cleanup, -- "Dummy job workaround load failed"}, -+ { NULL, kbase_dummy_job_wa_cleanup, NULL }, -+ { kbase_device_late_init, kbase_device_late_term, -+ "Late device initialization failed" }, - }; - - static void kbase_device_term_partial(struct kbase_device *kbdev, -@@ -247,14 +254,34 @@ int kbase_device_init(struct kbase_device *kbdev) - kbase_disjoint_init(kbdev); - - for (i = 0; i < ARRAY_SIZE(dev_init); i++) { -- err = dev_init[i].init(kbdev); -- if (err) { -- dev_err(kbdev->dev, "%s error = %d\n", -+ if (dev_init[i].init) { -+ err = dev_init[i].init(kbdev); -+ if (err) { -+ if (err != -EPROBE_DEFER) -+ dev_err(kbdev->dev, "%s error = %d\n", - dev_init[i].err_mes, err); -- kbase_device_term_partial(kbdev, i); -- break; -+ kbase_device_term_partial(kbdev, i); -+ break; -+ } - } - } - - return err; - } -+ -+int kbase_device_firmware_init_once(struct kbase_device *kbdev) -+{ -+ int ret = 0; -+ -+ mutex_lock(&kbdev->fw_load_lock); -+ -+ if (!kbdev->dummy_job_wa_loaded) { -+ ret = kbase_dummy_job_wa_load(kbdev); -+ if (!ret) -+ kbdev->dummy_job_wa_loaded = true; -+ } -+ -+ mutex_unlock(&kbdev->fw_load_lock); -+ -+ return ret; -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device.c b/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device.c -index 76f14e5..1ebd8aa 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device.c -@@ -1,12 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -17,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Base kernel device APIs - */ -@@ -34,12 +30,14 @@ - #include <linux/module.h> - #include <linux/of_platform.h> - #include <linux/types.h> -+#include <linux/oom.h> - - #include <mali_kbase.h> - #include <mali_kbase_defs.h> - #include <mali_kbase_hwaccess_instr.h> - #include <mali_kbase_hw.h> - #include <mali_kbase_config_defaults.h> -+#include <linux/priority_control_manager.h> - - #include <tl/mali_kbase_timeline.h> - #include "mali_kbase_vinstr.h" -@@ -50,6 +48,7 @@ - #include "mali_kbase_device_internal.h" - #include "backend/gpu/mali_kbase_pm_internal.h" - #include "backend/gpu/mali_kbase_irq_internal.h" -+#include "mali_kbase_regs_history_debugfs.h" - - #ifdef CONFIG_MALI_ARBITER_SUPPORT - #include "arbiter/mali_kbase_arbiter_pm.h" -@@ -75,64 +74,152 @@ struct kbase_device *kbase_device_alloc(void) - return kzalloc(sizeof(struct kbase_device), GFP_KERNEL); - } - --static int kbase_device_as_init(struct kbase_device *kbdev, int i) -+/** -+ * kbase_device_all_as_init() - Initialise address space objects of the device. -+ * -+ * @kbdev: Pointer to kbase device. -+ * -+ * Return: 0 on success otherwise non-zero. -+ */ -+static int kbase_device_all_as_init(struct kbase_device *kbdev) - { -- kbdev->as[i].number = i; -- kbdev->as[i].bf_data.addr = 0ULL; -- kbdev->as[i].pf_data.addr = 0ULL; -+ int i, err = 0; - -- kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%d", 0, 1, i); -- if (!kbdev->as[i].pf_wq) -- return -EINVAL; -+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { -+ err = kbase_mmu_as_init(kbdev, i); -+ if (err) -+ break; -+ } - -- INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker); -- INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker); -+ if (err) { -+ while (i-- > 0) -+ kbase_mmu_as_term(kbdev, i); -+ } - -- return 0; -+ return err; - } - --static void kbase_device_as_term(struct kbase_device *kbdev, int i) -+static void kbase_device_all_as_term(struct kbase_device *kbdev) - { -- destroy_workqueue(kbdev->as[i].pf_wq); -+ int i; -+ -+ for (i = 0; i < kbdev->nr_hw_address_spaces; i++) -+ kbase_mmu_as_term(kbdev, i); - } - --static int kbase_device_all_as_init(struct kbase_device *kbdev) -+int kbase_device_pcm_dev_init(struct kbase_device *const kbdev) - { -- int i, err; -+ int err = 0; - -- for (i = 0; i < kbdev->nr_hw_address_spaces; i++) { -- err = kbase_device_as_init(kbdev, i); -- if (err) -- goto free_workqs; -- } -+#if IS_ENABLED(CONFIG_OF) -+ struct device_node *prio_ctrl_node; - -- return 0; -- --free_workqs: -- for (; i > 0; i--) -- kbase_device_as_term(kbdev, i); -+ /* Check to see whether or not a platform specific priority control manager -+ * is available. -+ */ -+ prio_ctrl_node = of_parse_phandle(kbdev->dev->of_node, -+ "priority-control-manager", 0); -+ if (!prio_ctrl_node) { -+ dev_info(kbdev->dev, -+ "No priority control manager is configured"); -+ } else { -+ struct platform_device *const pdev = -+ of_find_device_by_node(prio_ctrl_node); -+ -+ if (!pdev) { -+ dev_err(kbdev->dev, -+ "The configured priority control manager was not found"); -+ } else { -+ struct priority_control_manager_device *pcm_dev = -+ platform_get_drvdata(pdev); -+ if (!pcm_dev) { -+ dev_info(kbdev->dev, "Priority control manager is not ready"); -+ err = -EPROBE_DEFER; -+ } else if (!try_module_get(pcm_dev->owner)) { -+ dev_err(kbdev->dev, "Failed to get priority control manager module"); -+ err = -ENODEV; -+ } else { -+ dev_info(kbdev->dev, "Priority control manager successfully loaded"); -+ kbdev->pcm_dev = pcm_dev; -+ } -+ } -+ of_node_put(prio_ctrl_node); -+ } -+#endif /* CONFIG_OF */ - - return err; - } - --static void kbase_device_all_as_term(struct kbase_device *kbdev) -+void kbase_device_pcm_dev_term(struct kbase_device *const kbdev) - { -- int i; -+ if (kbdev->pcm_dev) -+ module_put(kbdev->pcm_dev->owner); -+} - -- for (i = 0; i < kbdev->nr_hw_address_spaces; i++) -- kbase_device_as_term(kbdev, i); -+#define KBASE_PAGES_TO_KIB(pages) (((unsigned int)pages) << (PAGE_SHIFT - 10)) -+ -+/** -+ * mali_oom_notifier_handler - Mali driver out-of-memory handler -+ * -+ * @nb - notifier block - used to retrieve kbdev pointer -+ * @action - action (unused) -+ * @data - data pointer (unused) -+ * This function simply lists memory usage by the Mali driver, per GPU device, -+ * for diagnostic purposes. -+ */ -+static int mali_oom_notifier_handler(struct notifier_block *nb, -+ unsigned long action, void *data) -+{ -+ struct kbase_device *kbdev; -+ struct kbase_context *kctx = NULL; -+ unsigned long kbdev_alloc_total; -+ -+ if (WARN_ON(nb == NULL)) -+ return NOTIFY_BAD; -+ -+ kbdev = container_of(nb, struct kbase_device, oom_notifier_block); -+ -+ kbdev_alloc_total = -+ KBASE_PAGES_TO_KIB(atomic_read(&(kbdev->memdev.used_pages))); -+ -+ dev_err(kbdev->dev, "OOM notifier: dev %s %lu kB\n", kbdev->devname, -+ kbdev_alloc_total); -+ -+ mutex_lock(&kbdev->kctx_list_lock); -+ -+ list_for_each_entry (kctx, &kbdev->kctx_list, kctx_list_link) { -+ struct pid *pid_struct; -+ struct task_struct *task; -+ unsigned long task_alloc_total = -+ KBASE_PAGES_TO_KIB(atomic_read(&(kctx->used_pages))); -+ -+ rcu_read_lock(); -+ pid_struct = find_get_pid(kctx->pid); -+ task = pid_task(pid_struct, PIDTYPE_PID); -+ -+ dev_err(kbdev->dev, -+ "OOM notifier: tsk %s tgid (%u) pid (%u) %lu kB\n", -+ task ? task->comm : "[null task]", kctx->tgid, -+ kctx->pid, task_alloc_total); -+ -+ put_pid(pid_struct); -+ rcu_read_unlock(); -+ } -+ -+ mutex_unlock(&kbdev->kctx_list_lock); -+ return NOTIFY_OK; - } - - int kbase_device_misc_init(struct kbase_device * const kbdev) - { - int err; --#ifdef CONFIG_ARM64 -+#if IS_ENABLED(CONFIG_ARM64) - struct device_node *np = NULL; - #endif /* CONFIG_ARM64 */ - - spin_lock_init(&kbdev->mmu_mask_change); - mutex_init(&kbdev->mmu_hw_mutex); --#ifdef CONFIG_ARM64 -+#if IS_ENABLED(CONFIG_ARM64) - kbdev->cci_snoop_enabled = false; - np = kbdev->dev->of_node; - if (np != NULL) { -@@ -153,6 +240,7 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) - } - } - #endif /* CONFIG_ARM64 */ -+ - /* Get the list of workarounds for issues on the current HW - * (identified by the GPU_ID register) - */ -@@ -169,11 +257,6 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) - if (err) - goto fail; - -- /* On Linux 4.0+, dma coherency is determined from device tree */ --#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0) -- set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops); --#endif -- - /* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our - * device structure was created by device-tree - */ -@@ -194,9 +277,7 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) - - err = kbase_device_all_as_init(kbdev); - if (err) -- goto as_init_failed; -- -- spin_lock_init(&kbdev->hwcnt.lock); -+ goto dma_set_mask_failed; - - err = kbase_ktrace_init(kbdev); - if (err) -@@ -208,30 +289,28 @@ int kbase_device_misc_init(struct kbase_device * const kbdev) - - atomic_set(&kbdev->ctx_num, 0); - -- err = kbase_instr_backend_init(kbdev); -- if (err) -- goto term_trace; -- - kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD; - - kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS; - -- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -- kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); -- else -- kbdev->mmu_mode = kbase_mmu_mode_get_lpae(); -+ kbdev->mmu_mode = kbase_mmu_mode_get_aarch64(); - - mutex_init(&kbdev->kctx_list_lock); - INIT_LIST_HEAD(&kbdev->kctx_list); - -- spin_lock_init(&kbdev->hwaccess_lock); -+ dev_dbg(kbdev->dev, "Registering mali_oom_notifier_handlern"); -+ kbdev->oom_notifier_block.notifier_call = mali_oom_notifier_handler; -+ err = register_oom_notifier(&kbdev->oom_notifier_block); - -+ if (err) { -+ dev_err(kbdev->dev, -+ "Unable to register OOM notifier for Mali - but will continue\n"); -+ kbdev->oom_notifier_block.notifier_call = NULL; -+ } - return 0; --term_trace: -- kbase_ktrace_term(kbdev); -+ - term_as: - kbase_device_all_as_term(kbdev); --as_init_failed: - dma_set_mask_failed: - fail: - return err; -@@ -247,11 +326,12 @@ void kbase_device_misc_term(struct kbase_device *kbdev) - kbase_debug_assert_register_hook(NULL, NULL); - #endif - -- kbase_instr_backend_term(kbdev); -- - kbase_ktrace_term(kbdev); - - kbase_device_all_as_term(kbdev); -+ -+ if (kbdev->oom_notifier_block.notifier_call) -+ unregister_oom_notifier(&kbdev->oom_notifier_block); - } - - void kbase_device_free(struct kbase_device *kbdev) -@@ -271,16 +351,6 @@ void kbase_increment_device_id(void) - kbase_dev_nr++; - } - --int kbase_device_hwcnt_backend_gpu_init(struct kbase_device *kbdev) --{ -- return kbase_hwcnt_backend_gpu_create(kbdev, &kbdev->hwcnt_gpu_iface); --} -- --void kbase_device_hwcnt_backend_gpu_term(struct kbase_device *kbdev) --{ -- kbase_hwcnt_backend_gpu_destroy(&kbdev->hwcnt_gpu_iface); --} -- - int kbase_device_hwcnt_context_init(struct kbase_device *kbdev) - { - return kbase_hwcnt_context_init(&kbdev->hwcnt_gpu_iface, -@@ -400,7 +470,18 @@ int kbase_device_early_init(struct kbase_device *kbdev) - /* We're done accessing the GPU registers for now. */ - kbase_pm_register_access_disable(kbdev); - -+ /* This spinlock has to be initialized before installing interrupt -+ * handlers that require to hold it to process interrupts. -+ */ -+ spin_lock_init(&kbdev->hwaccess_lock); -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ if (kbdev->arb.arb_if) -+ err = kbase_arbiter_pm_install_interrupts(kbdev); -+ else -+ err = kbase_install_interrupts(kbdev); -+#else - err = kbase_install_interrupts(kbdev); -+#endif - if (err) - goto fail_interrupts; - -@@ -427,3 +508,17 @@ void kbase_device_early_term(struct kbase_device *kbdev) - kbase_pm_runtime_term(kbdev); - kbasep_platform_device_term(kbdev); - } -+ -+int kbase_device_late_init(struct kbase_device *kbdev) -+{ -+ int err; -+ -+ err = kbasep_platform_device_late_init(kbdev); -+ -+ return err; -+} -+ -+void kbase_device_late_term(struct kbase_device *kbdev) -+{ -+ kbasep_platform_device_late_term(kbdev); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device.h b/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device.h -index 16f1d70..517c16b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> -@@ -47,6 +46,19 @@ void kbase_device_put_list(const struct list_head *dev_list); - */ - void kbase_increment_device_id(void); - -+/** -+ * kbase_device_firmware_init_once - Initialize firmware and HWC -+ * -+ * @kbdev: An instance of the GPU platform device, allocated from the probe -+ * method of the driver. -+ * -+ * When a device file is opened for the first time, -+ * load firmware and initialize hardware counter components. -+ * -+ * @return 0 on success. An error code on failure. -+ */ -+int kbase_device_firmware_init_once(struct kbase_device *kbdev); -+ - /** - * kbase_device_init - Device initialisation. - * -@@ -69,3 +81,109 @@ int kbase_device_init(struct kbase_device *kbdev); - * - */ - void kbase_device_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_reg_write - write to GPU register -+ * @kbdev: Kbase device pointer -+ * @offset: Offset of register -+ * @value: Value to write -+ * -+ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). -+ */ -+void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value); -+ -+/** -+ * kbase_reg_read - read from GPU register -+ * @kbdev: Kbase device pointer -+ * @offset: Offset of register -+ * -+ * Caller must ensure the GPU is powered (@kbdev->pm.gpu_powered != false). -+ * -+ * Return: Value in desired register -+ */ -+u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset); -+ -+/** -+ * kbase_is_gpu_removed() - Has the GPU been removed. -+ * @kbdev: Kbase device pointer -+ * -+ * When Kbase takes too long to give up the GPU, the Arbiter -+ * can remove it. This will then be followed by a GPU lost event. -+ * This function will return true if the GPU has been removed. -+ * When this happens register reads will be zero. A zero GPU_ID is -+ * invalid so this is used to detect when GPU is removed. -+ * -+ * Return: True if GPU removed -+ */ -+bool kbase_is_gpu_removed(struct kbase_device *kbdev); -+ -+/** -+ * kbase_gpu_start_cache_clean - Start a cache clean -+ * @kbdev: Kbase device -+ * -+ * Issue a cache clean and invalidate command to hardware. This function will -+ * take hwaccess_lock. -+ */ -+void kbase_gpu_start_cache_clean(struct kbase_device *kbdev); -+ -+/** -+ * kbase_gpu_start_cache_clean_nolock - Start a cache clean -+ * @kbdev: Kbase device -+ * -+ * Issue a cache clean and invalidate command to hardware. hwaccess_lock -+ * must be held by the caller. -+ */ -+void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev); -+ -+/** -+ * kbase_gpu_wait_cache_clean - Wait for cache cleaning to finish -+ * @kbdev: Kbase device -+ * -+ * This function will take hwaccess_lock, and may sleep. -+ */ -+void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev); -+ -+/** -+ * kbase_gpu_wait_cache_clean_timeout - Wait for certain time for cache -+ * cleaning to finish -+ * @kbdev: Kbase device -+ * @wait_timeout_ms: Time in milliseconds, to wait for cache clean to complete. -+ * -+ * This function will take hwaccess_lock, and may sleep. This is supposed to be -+ * called from paths (like GPU reset) where an indefinite wait for the -+ * completion of cache clean operation can cause deadlock, as the operation may -+ * never complete. -+ * -+ * Return: 0 if successful or a negative error code on failure. -+ */ -+int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev, -+ unsigned int wait_timeout_ms); -+ -+/** -+ * kbase_gpu_cache_clean_wait_complete - Called after the cache cleaning is -+ * finished. Would also be called after -+ * the GPU reset. -+ * @kbdev: Kbase device -+ * -+ * Caller must hold the hwaccess_lock. -+ */ -+void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev); -+ -+/** -+ * kbase_clean_caches_done - Issue preiously queued cache clean request or -+ * wake up the requester that issued cache clean. -+ * @kbdev: Kbase device -+ * -+ * Caller must hold the hwaccess_lock. -+ */ -+void kbase_clean_caches_done(struct kbase_device *kbdev); -+ -+/** -+ * kbase_gpu_interrupt - GPU interrupt handler -+ * @kbdev: Kbase device pointer -+ * @val: The value of the GPU IRQ status register which triggered the call -+ * -+ * This function is called from the interrupt handler when a GPU irq is to be -+ * handled. -+ */ -+void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device_hw.c b/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device_hw.c -new file mode 100644 -index 0000000..e80559a ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device_hw.c -@@ -0,0 +1,182 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2014-2016, 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <gpu/mali_kbase_gpu_fault.h> -+#include <backend/gpu/mali_kbase_instr_internal.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> -+#include <device/mali_kbase_device.h> -+#include <mali_kbase_reset_gpu.h> -+#include <mmu/mali_kbase_mmu.h> -+ -+#if !IS_ENABLED(CONFIG_MALI_NO_MALI) -+void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value) -+{ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); -+ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); -+ -+ writel(value, kbdev->reg + offset); -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ if (unlikely(kbdev->io_history.enabled)) -+ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, -+ value, 1); -+#endif /* CONFIG_DEBUG_FS */ -+ dev_dbg(kbdev->dev, "w: reg %08x val %08x", offset, value); -+} -+ -+KBASE_EXPORT_TEST_API(kbase_reg_write); -+ -+u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset) -+{ -+ u32 val; -+ -+ KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered); -+ KBASE_DEBUG_ASSERT(kbdev->dev != NULL); -+ -+ val = readl(kbdev->reg + offset); -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ if (unlikely(kbdev->io_history.enabled)) -+ kbase_io_history_add(&kbdev->io_history, kbdev->reg + offset, -+ val, 0); -+#endif /* CONFIG_DEBUG_FS */ -+ dev_dbg(kbdev->dev, "r: reg %08x val %08x", offset, val); -+ -+ return val; -+} -+ -+KBASE_EXPORT_TEST_API(kbase_reg_read); -+ -+bool kbase_is_gpu_removed(struct kbase_device *kbdev) -+{ -+ u32 val; -+ -+ val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID)); -+ -+ return val == 0; -+} -+#endif /* !IS_ENABLED(CONFIG_MALI_NO_MALI) */ -+ -+void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev) -+{ -+ u32 irq_mask; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (kbdev->cache_clean_in_progress) { -+ /* If this is called while another clean is in progress, we -+ * can't rely on the current one to flush any new changes in -+ * the cache. Instead, trigger another cache clean immediately -+ * after this one finishes. -+ */ -+ kbdev->cache_clean_queued = true; -+ return; -+ } -+ -+ /* Enable interrupt */ -+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), -+ irq_mask | CLEAN_CACHES_COMPLETED); -+ -+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CLEAN_INV_CACHES); -+ -+ kbdev->cache_clean_in_progress = true; -+} -+ -+void kbase_gpu_start_cache_clean(struct kbase_device *kbdev) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_gpu_start_cache_clean_nolock(kbdev); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+void kbase_gpu_cache_clean_wait_complete(struct kbase_device *kbdev) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbdev->cache_clean_queued = false; -+ kbdev->cache_clean_in_progress = false; -+ wake_up(&kbdev->cache_clean_wait); -+} -+ -+void kbase_clean_caches_done(struct kbase_device *kbdev) -+{ -+ u32 irq_mask; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ if (kbdev->cache_clean_queued) { -+ kbdev->cache_clean_queued = false; -+ -+ KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CLEAN_INV_CACHES); -+ } else { -+ /* Disable interrupt */ -+ irq_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK)); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), -+ irq_mask & ~CLEAN_CACHES_COMPLETED); -+ -+ kbase_gpu_cache_clean_wait_complete(kbdev); -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+static inline bool get_cache_clean_flag(struct kbase_device *kbdev) -+{ -+ bool cache_clean_in_progress; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ cache_clean_in_progress = kbdev->cache_clean_in_progress; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return cache_clean_in_progress; -+} -+ -+void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev) -+{ -+ while (get_cache_clean_flag(kbdev)) { -+ wait_event_interruptible(kbdev->cache_clean_wait, -+ !kbdev->cache_clean_in_progress); -+ } -+} -+ -+int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev, -+ unsigned int wait_timeout_ms) -+{ -+ long remaining = msecs_to_jiffies(wait_timeout_ms); -+ -+ while (remaining && get_cache_clean_flag(kbdev)) { -+ remaining = wait_event_timeout(kbdev->cache_clean_wait, -+ !kbdev->cache_clean_in_progress, -+ remaining); -+ } -+ -+ return (remaining ? 0 : -ETIMEDOUT); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device_internal.h -index 9f96db0..d422407 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device_internal.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/device/mali_kbase_device_internal.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> -@@ -43,9 +42,6 @@ void kbase_device_vinstr_term(struct kbase_device *kbdev); - int kbase_device_timeline_init(struct kbase_device *kbdev); - void kbase_device_timeline_term(struct kbase_device *kbdev); - --int kbase_device_hwcnt_backend_gpu_init(struct kbase_device *kbdev); --void kbase_device_hwcnt_backend_gpu_term(struct kbase_device *kbdev); -- - int kbase_device_hwcnt_context_init(struct kbase_device *kbdev); - void kbase_device_hwcnt_context_term(struct kbase_device *kbdev); - -@@ -76,3 +72,17 @@ int kbase_device_early_init(struct kbase_device *kbdev); - * @kbdev: Device pointer - */ - void kbase_device_early_term(struct kbase_device *kbdev); -+ -+/** -+ * kbase_device_late_init - Complete any device-specific initialization. -+ * @kbdev: Device pointer -+ * -+ * Return: 0 on success, or an error code on failure. -+ */ -+int kbase_device_late_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_device_late_term - Complete any device-specific termination. -+ * @kbdev: Device pointer -+ */ -+void kbase_device_late_term(struct kbase_device *kbdev); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Kconfig b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/Kbuild -similarity index 65% -rename from dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Kconfig -rename to dvalin/kernel/drivers/gpu/arm/midgard/gpu/Kbuild -index 0cdb474..c3ab811 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Kconfig -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,14 +16,12 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - -+mali_kbase-y += gpu/mali_kbase_gpu.o - --config MALI_KUTF -- tristate "Mali Kernel Unit Test Framework" -- default m -- help -- Enables MALI testing framework. To compile it as a module, -- choose M here - this will generate a single module called kutf. -+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) -+ mali_kbase-y += gpu/backend/mali_kbase_gpu_fault_csf.o -+else -+ mali_kbase-y += gpu/backend/mali_kbase_gpu_fault_jm.o -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_fault_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_fault_csf.c -new file mode 100644 -index 0000000..f9d4c14 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_fault_csf.c -@@ -0,0 +1,104 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h> -+#include <gpu/mali_kbase_gpu_fault.h> -+ -+const char *kbase_gpu_exception_name(u32 const exception_code) -+{ -+ const char *e; -+ -+ switch (exception_code) { -+ /* CS exceptions */ -+ case CS_FAULT_EXCEPTION_TYPE_CS_RESOURCE_TERMINATED: -+ e = "CS_RESOURCE_TERMINATED"; -+ break; -+ case CS_FAULT_EXCEPTION_TYPE_CS_INHERIT_FAULT: -+ e = "CS_INHERIT_FAULT"; -+ break; -+ /* CS fatal exceptions */ -+ case CS_FATAL_EXCEPTION_TYPE_CS_CONFIG_FAULT: -+ e = "CS_CONFIG_FAULT"; -+ break; -+ case CS_FATAL_EXCEPTION_TYPE_CS_ENDPOINT_FAULT: -+ e = "FATAL_CS_ENDPOINT_FAULT"; -+ break; -+ case CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT: -+ e = "FATAL_CS_BUS_FAULT"; -+ break; -+ case CS_FATAL_EXCEPTION_TYPE_CS_INVALID_INSTRUCTION: -+ e = "FATAL_CS_INVALID_INSTRUCTION"; -+ break; -+ case CS_FATAL_EXCEPTION_TYPE_CS_CALL_STACK_OVERFLOW: -+ e = "FATAL_CS_CALL_STACK_OVERFLOW"; -+ break; -+ /* Shader exceptions */ -+ case CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_PC: -+ e = "INSTR_INVALID_PC"; -+ break; -+ case CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_ENC: -+ e = "INSTR_INVALID_ENC"; -+ break; -+ case CS_FAULT_EXCEPTION_TYPE_INSTR_BARRIER_FAULT: -+ e = "INSTR_BARRIER_FAULT"; -+ break; -+ /* Misc exceptions */ -+ case CS_FAULT_EXCEPTION_TYPE_DATA_INVALID_FAULT: -+ e = "DATA_INVALID_FAULT"; -+ break; -+ case CS_FAULT_EXCEPTION_TYPE_TILE_RANGE_FAULT: -+ e = "TILE_RANGE_FAULT"; -+ break; -+ case CS_FAULT_EXCEPTION_TYPE_ADDR_RANGE_FAULT: -+ e = "ADDR_RANGE_FAULT"; -+ break; -+ case CS_FAULT_EXCEPTION_TYPE_IMPRECISE_FAULT: -+ e = "IMPRECISE_FAULT"; -+ break; -+ /* FW exceptions */ -+ case CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR: -+ e = "FIRMWARE_INTERNAL_ERROR"; -+ break; -+ case CS_FAULT_EXCEPTION_TYPE_RESOURCE_EVICTION_TIMEOUT: -+ e = "RESOURCE_EVICTION_TIMEOUT"; -+ break; -+ /* GPU Fault */ -+ case GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT: -+ e = "GPU_BUS_FAULT"; -+ break; -+ case GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_SHAREABILITY_FAULT: -+ e = "GPU_SHAREABILITY_FAULT"; -+ break; -+ case GPU_FAULTSTATUS_EXCEPTION_TYPE_SYSTEM_SHAREABILITY_FAULT: -+ e = "SYSTEM_SHAREABILITY_FAULT"; -+ break; -+ case GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_CACHEABILITY_FAULT: -+ e = "GPU_CACHEABILITY_FAULT"; -+ break; -+ /* Any other exception code is unknown */ -+ default: -+ e = "UNKNOWN"; -+ break; -+ } -+ -+ return e; -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_fault_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_fault_jm.c -index 63132dc..37015cc 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_fault_jm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_fault_jm.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,13 +17,11 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> - --#include "../mali_kbase_gpu_fault.h" -+#include <gpu/mali_kbase_gpu_fault.h> - - const char *kbase_gpu_exception_name(u32 const exception_code) - { -@@ -119,8 +118,6 @@ const char *kbase_gpu_exception_name(u32 const exception_code) - e = "TRANSLATION_FAULT"; - break; - case 0xC8: -- e = "PERMISSION_FAULT"; -- break; - case 0xC9: - case 0xCA: - case 0xCB: -@@ -141,8 +138,6 @@ const char *kbase_gpu_exception_name(u32 const exception_code) - e = "TRANSTAB_BUS_FAULT"; - break; - case 0xD8: -- e = "ACCESS_FLAG"; -- break; - case 0xD9: - case 0xDA: - case 0xDB: -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu.c b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu.c -index 3128db4..8a84ef5 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,11 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> - #include <mali_kbase_defs.h> -+#include <gpu/mali_kbase_gpu_fault.h> - - const char *kbase_gpu_access_type_name(u32 fault_status) - { -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_fault.h b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_fault.h -index b59b9d1..d1e9f77 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_fault.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_fault.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,14 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_GPU_FAULT_H_ - #define _KBASE_GPU_FAULT_H_ - --/** Returns the name associated with a Mali exception code -- * -+/** -+ * kbase_gpu_exception_name() - -+ * Returns the name associated with a Mali exception code - * @exception_code: exception code - * - * This function is called from the interrupt handler when a GPU fault occurs. -@@ -33,17 +33,6 @@ - */ - const char *kbase_gpu_exception_name(u32 exception_code); - --/** Returns the name associated with a Mali fatal exception code -- * -- * @fatal_exception_code: fatal exception code -- * -- * This function is called from the interrupt handler when a GPU fatal -- * exception occurs. -- * -- * Return: name associated with the fatal exception code -- */ --const char *kbase_gpu_fatal_exception_name(u32 const fatal_exception_code); -- - /** - * kbase_gpu_access_type_name - Convert MMU_AS_CONTROL.FAULTSTATUS.ACCESS_TYPE - * into string. -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_regmap.h b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_regmap.h -index 759f30d..47e7781 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_regmap.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_regmap.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,422 +17,17 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_GPU_REGMAP_H_ - #define _KBASE_GPU_REGMAP_H_ - --#include "mali_kbase_gpu_coherency.h" --#include "mali_kbase_gpu_id.h" --#include "backend/mali_kbase_gpu_regmap_jm.h" -- --/* Begin Register Offsets */ --/* GPU control registers */ -- --#define GPU_CONTROL_BASE 0x0000 --#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r)) --#define GPU_ID 0x000 /* (RO) GPU and revision identifier */ --#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */ --#define TILER_FEATURES 0x00C /* (RO) Tiler Features */ --#define MEM_FEATURES 0x010 /* (RO) Memory system features */ --#define MMU_FEATURES 0x014 /* (RO) MMU features */ --#define AS_PRESENT 0x018 /* (RO) Address space slots present */ --#define GPU_IRQ_RAWSTAT 0x020 /* (RW) */ --#define GPU_IRQ_CLEAR 0x024 /* (WO) */ --#define GPU_IRQ_MASK 0x028 /* (RW) */ --#define GPU_IRQ_STATUS 0x02C /* (RO) */ -- --#define GPU_COMMAND 0x030 /* (WO) */ --#define GPU_STATUS 0x034 /* (RO) */ -- --#define GPU_DBGEN (1 << 8) /* DBGEN wire status */ -- --#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */ --#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */ --#define GPU_FAULTADDRESS_HI 0x044 /* (RO) GPU exception fault address, high word */ -- --#define L2_CONFIG 0x048 /* (RW) Level 2 cache configuration */ -- --#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */ --#define SUPER_L2_COHERENT (1 << 1) /* Shader cores within a core -- * supergroup are l2 coherent -- */ -- --#define PWR_KEY 0x050 /* (WO) Power manager key register */ --#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ --#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ -- --#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ --#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ --#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ --#define TIMESTAMP_HI 0x09C /* (RO) Global time stamp counter, high word */ -- --#define THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ --#define THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ --#define THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ --#define THREAD_FEATURES 0x0AC /* (RO) Thread features */ --#define THREAD_TLS_ALLOC 0x310 /* (RO) Number of threads per core that TLS must be allocated for */ -- --#define TEXTURE_FEATURES_0 0x0B0 /* (RO) Support flags for indexed texture formats 0..31 */ --#define TEXTURE_FEATURES_1 0x0B4 /* (RO) Support flags for indexed texture formats 32..63 */ --#define TEXTURE_FEATURES_2 0x0B8 /* (RO) Support flags for indexed texture formats 64..95 */ --#define TEXTURE_FEATURES_3 0x0BC /* (RO) Support flags for texture order */ -- --#define TEXTURE_FEATURES_REG(n) GPU_CONTROL_REG(TEXTURE_FEATURES_0 + ((n) << 2)) -- --#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ --#define SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */ -- --#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ --#define TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */ -- --#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ --#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */ -- --#define STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */ --#define STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */ -- --#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */ --#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */ -- --#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */ --#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */ -- --#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */ --#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */ -- --#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */ --#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */ -- --#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */ --#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */ -- --#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */ --#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */ -- --#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */ --#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */ -- --#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */ --#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */ -- --#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */ --#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */ -- --#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */ --#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */ -- --#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */ --#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */ -- --#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */ --#define STACK_PWROFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */ -- --#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */ --#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */ -- --#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */ --#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */ -- --#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ --#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ -- --#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ --#define STACK_PWRTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ -- --#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */ --#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */ -- --#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */ --#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */ -- --#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */ --#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */ -- --#define COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */ --#define COHERENCY_ENABLE 0x304 /* (RW) Coherency enable */ -- --#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration (implementation-specific) */ --#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration (implementation-specific) */ --#define L2_MMU_CONFIG 0xF0C /* (RW) L2 cache and MMU configuration (implementation-specific) */ -- --/* Job control registers */ -- --#define JOB_CONTROL_BASE 0x1000 -- --#define JOB_CONTROL_REG(r) (JOB_CONTROL_BASE + (r)) -- --#define JOB_IRQ_RAWSTAT 0x000 /* Raw interrupt status register */ --#define JOB_IRQ_CLEAR 0x004 /* Interrupt clear register */ --#define JOB_IRQ_MASK 0x008 /* Interrupt mask register */ --#define JOB_IRQ_STATUS 0x00C /* Interrupt status register */ -- --/* MMU control registers */ -- --#define MEMORY_MANAGEMENT_BASE 0x2000 --#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r)) -- --#define MMU_IRQ_RAWSTAT 0x000 /* (RW) Raw interrupt status register */ --#define MMU_IRQ_CLEAR 0x004 /* (WO) Interrupt clear register */ --#define MMU_IRQ_MASK 0x008 /* (RW) Interrupt mask register */ --#define MMU_IRQ_STATUS 0x00C /* (RO) Interrupt status register */ -- --#define MMU_AS0 0x400 /* Configuration registers for address space 0 */ --#define MMU_AS1 0x440 /* Configuration registers for address space 1 */ --#define MMU_AS2 0x480 /* Configuration registers for address space 2 */ --#define MMU_AS3 0x4C0 /* Configuration registers for address space 3 */ --#define MMU_AS4 0x500 /* Configuration registers for address space 4 */ --#define MMU_AS5 0x540 /* Configuration registers for address space 5 */ --#define MMU_AS6 0x580 /* Configuration registers for address space 6 */ --#define MMU_AS7 0x5C0 /* Configuration registers for address space 7 */ --#define MMU_AS8 0x600 /* Configuration registers for address space 8 */ --#define MMU_AS9 0x640 /* Configuration registers for address space 9 */ --#define MMU_AS10 0x680 /* Configuration registers for address space 10 */ --#define MMU_AS11 0x6C0 /* Configuration registers for address space 11 */ --#define MMU_AS12 0x700 /* Configuration registers for address space 12 */ --#define MMU_AS13 0x740 /* Configuration registers for address space 13 */ --#define MMU_AS14 0x780 /* Configuration registers for address space 14 */ --#define MMU_AS15 0x7C0 /* Configuration registers for address space 15 */ -- --/* MMU address space control registers */ -- --#define MMU_AS_REG(n, r) (MMU_REG(MMU_AS0 + ((n) << 6)) + (r)) -- --#define AS_TRANSTAB_LO 0x00 /* (RW) Translation Table Base Address for address space n, low word */ --#define AS_TRANSTAB_HI 0x04 /* (RW) Translation Table Base Address for address space n, high word */ --#define AS_MEMATTR_LO 0x08 /* (RW) Memory attributes for address space n, low word. */ --#define AS_MEMATTR_HI 0x0C /* (RW) Memory attributes for address space n, high word. */ --#define AS_LOCKADDR_LO 0x10 /* (RW) Lock region address for address space n, low word */ --#define AS_LOCKADDR_HI 0x14 /* (RW) Lock region address for address space n, high word */ --#define AS_COMMAND 0x18 /* (WO) MMU command register for address space n */ --#define AS_FAULTSTATUS 0x1C /* (RO) MMU fault status register for address space n */ --#define AS_FAULTADDRESS_LO 0x20 /* (RO) Fault Address for address space n, low word */ --#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */ --#define AS_STATUS 0x28 /* (RO) Status flags for address space n */ -- --/* (RW) Translation table configuration for address space n, low word */ --#define AS_TRANSCFG_LO 0x30 --/* (RW) Translation table configuration for address space n, high word */ --#define AS_TRANSCFG_HI 0x34 --/* (RO) Secondary fault address for address space n, low word */ --#define AS_FAULTEXTRA_LO 0x38 --/* (RO) Secondary fault address for address space n, high word */ --#define AS_FAULTEXTRA_HI 0x3C -- --/* End Register Offsets */ -- --/* IRQ flags */ --#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ --#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ --#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. */ --#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ --#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down. */ -- --#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ --#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ -- --/* Include POWER_CHANGED_SINGLE in debug builds for use in irq latency test. -- */ --#define GPU_IRQ_REG_COMMON (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ -- | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) -+#include <uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_regmap.h> - -+/* Include POWER_CHANGED_SINGLE in debug builds for use in irq latency test. */ - #ifdef CONFIG_MALI_DEBUG -+#undef GPU_IRQ_REG_ALL - #define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE) --#else /* CONFIG_MALI_DEBUG */ --#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON) - #endif /* CONFIG_MALI_DEBUG */ - --/* -- * MMU_IRQ_RAWSTAT register values. Values are valid also for -- * MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. -- */ -- --#define MMU_PAGE_FAULT_FLAGS 16 -- --/* Macros returning a bitmask to retrieve page fault or bus error flags from -- * MMU registers */ --#define MMU_PAGE_FAULT(n) (1UL << (n)) --#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) -- --/* -- * Begin LPAE MMU TRANSTAB register values -- */ --#define AS_TRANSTAB_LPAE_ADDR_SPACE_MASK 0xfffff000 --#define AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED (0u << 0) --#define AS_TRANSTAB_LPAE_ADRMODE_IDENTITY (1u << 1) --#define AS_TRANSTAB_LPAE_ADRMODE_TABLE (3u << 0) --#define AS_TRANSTAB_LPAE_READ_INNER (1u << 2) --#define AS_TRANSTAB_LPAE_SHARE_OUTER (1u << 4) -- --#define AS_TRANSTAB_LPAE_ADRMODE_MASK 0x00000003 -- --/* -- * Begin AARCH64 MMU TRANSTAB register values -- */ --#define MMU_HW_OUTA_BITS 40 --#define AS_TRANSTAB_BASE_MASK ((1ULL << MMU_HW_OUTA_BITS) - (1ULL << 4)) -- --/* -- * Begin MMU STATUS register values -- */ --#define AS_STATUS_AS_ACTIVE 0x01 -- --#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3) --#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3) --#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3) --#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3) --#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3) --#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3) --#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3) -- --#define AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT 0 --#define AS_FAULTSTATUS_EXCEPTION_TYPE_MASK (0xFF << AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) --#define AS_FAULTSTATUS_EXCEPTION_TYPE_GET(reg_val) \ -- (((reg_val)&AS_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) -- --#define AS_FAULTSTATUS_ACCESS_TYPE_SHIFT 8 --#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3 << AS_FAULTSTATUS_ACCESS_TYPE_SHIFT) --#define AS_FAULTSTATUS_ACCESS_TYPE_GET(reg_val) \ -- (((reg_val)&AS_FAULTSTATUS_ACCESS_TYPE_MASK) >> AS_FAULTSTATUS_ACCESS_TYPE_SHIFT) -- --#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0) --#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1) --#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2) --#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3) -- --#define AS_FAULTSTATUS_SOURCE_ID_SHIFT 16 --#define AS_FAULTSTATUS_SOURCE_ID_MASK (0xFFFF << AS_FAULTSTATUS_SOURCE_ID_SHIFT) --#define AS_FAULTSTATUS_SOURCE_ID_GET(reg_val) \ -- (((reg_val)&AS_FAULTSTATUS_SOURCE_ID_MASK) >> AS_FAULTSTATUS_SOURCE_ID_SHIFT) -- --/* -- * Begin MMU TRANSCFG register values -- */ --#define AS_TRANSCFG_ADRMODE_LEGACY 0 --#define AS_TRANSCFG_ADRMODE_UNMAPPED 1 --#define AS_TRANSCFG_ADRMODE_IDENTITY 2 --#define AS_TRANSCFG_ADRMODE_AARCH64_4K 6 --#define AS_TRANSCFG_ADRMODE_AARCH64_64K 8 -- --#define AS_TRANSCFG_ADRMODE_MASK 0xF -- --/* -- * Begin TRANSCFG register values -- */ --#define AS_TRANSCFG_PTW_MEMATTR_MASK (3ull << 24) --#define AS_TRANSCFG_PTW_MEMATTR_NON_CACHEABLE (1ull << 24) --#define AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK (2ull << 24) -- --#define AS_TRANSCFG_PTW_SH_MASK ((3ull << 28)) --#define AS_TRANSCFG_PTW_SH_OS (2ull << 28) --#define AS_TRANSCFG_PTW_SH_IS (3ull << 28) --#define AS_TRANSCFG_R_ALLOCATE (1ull << 30) -- --/* -- * Begin Command Values -- */ -- --/* AS_COMMAND register commands */ --#define AS_COMMAND_NOP 0x00 /* NOP Operation */ --#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ --#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ --#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ --#define AS_COMMAND_FLUSH 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs -- (deprecated - only for use with T60x) */ --#define AS_COMMAND_FLUSH_PT 0x04 /* Flush all L2 caches then issue a flush region command to all MMUs */ --#define AS_COMMAND_FLUSH_MEM 0x05 /* Wait for memory accesses to complete, flush all the L1s cache then -- flush all L2 caches then issue a flush region command to all MMUs */ -- --/* GPU_STATUS values */ --#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ --#define GPU_STATUS_PROTECTED_MODE_ACTIVE (1 << 7) /* Set if protected mode is active */ -- --/* PRFCNT_CONFIG register values */ --#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ --#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ --#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ -- --/* The performance counters are disabled. */ --#define PRFCNT_CONFIG_MODE_OFF 0 --/* The performance counters are enabled, but are only written out when a -- * PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. -- */ --#define PRFCNT_CONFIG_MODE_MANUAL 1 --/* The performance counters are enabled, and are written out each time a tile -- * finishes rendering. -- */ --#define PRFCNT_CONFIG_MODE_TILE 2 -- --/* AS<n>_MEMATTR values from MMU_MEMATTR_STAGE1: */ --/* Use GPU implementation-defined caching policy. */ --#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x88ull --/* The attribute set to force all resources to be cached. */ --#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x8Full --/* Inner write-alloc cache setup, no outer caching */ --#define AS_MEMATTR_WRITE_ALLOC 0x8Dull -- --/* Use GPU implementation-defined caching policy. */ --#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull --/* The attribute set to force all resources to be cached. */ --#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full --/* Inner write-alloc cache setup, no outer caching */ --#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull --/* Set to implementation defined, outer caching */ --#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull --/* Set to write back memory, outer caching */ --#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull --/* There is no LPAE support for non-cacheable, since the memory type is always -- * write-back. -- * Marking this setting as reserved for LPAE -- */ --#define AS_MEMATTR_LPAE_NON_CACHEABLE_RESERVED -- --/* L2_MMU_CONFIG register */ --#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23) --#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT) -- --/* End L2_MMU_CONFIG register */ -- --/* THREAD_* registers */ -- --/* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ --#define IMPLEMENTATION_UNSPECIFIED 0 --#define IMPLEMENTATION_SILICON 1 --#define IMPLEMENTATION_FPGA 2 --#define IMPLEMENTATION_MODEL 3 -- --/* Default values when registers are not supported by the implemented hardware */ --#define THREAD_MT_DEFAULT 256 --#define THREAD_MWS_DEFAULT 256 --#define THREAD_MBS_DEFAULT 256 --#define THREAD_MR_DEFAULT 1024 --#define THREAD_MTQ_DEFAULT 4 --#define THREAD_MTGS_DEFAULT 10 -- --/* End THREAD_* registers */ -- --/* SHADER_CONFIG register */ --#define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) --#define SC_TLS_HASH_ENABLE (1ul << 17) --#define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) --#define SC_VAR_ALGORITHM (1ul << 29) --/* End SHADER_CONFIG register */ -- --/* TILER_CONFIG register */ --#define TC_CLOCK_GATE_OVERRIDE (1ul << 0) --/* End TILER_CONFIG register */ -- --/* L2_CONFIG register */ --#define L2_CONFIG_SIZE_SHIFT 16 --#define L2_CONFIG_SIZE_MASK (0xFFul << L2_CONFIG_SIZE_SHIFT) --#define L2_CONFIG_HASH_SHIFT 24 --#define L2_CONFIG_HASH_MASK (0xFFul << L2_CONFIG_HASH_SHIFT) --/* End L2_CONFIG register */ -- --/* IDVS_GROUP register */ --#define IDVS_GROUP_SIZE_SHIFT (16) --#define IDVS_GROUP_MAX_SIZE (0x3F) -- - #endif /* _KBASE_GPU_REGMAP_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/Kbuild -index 3d9cf80..96977e9 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2016-2018 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2016-2018, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,14 +16,20 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - mali_kbase-y += \ -- ipa/mali_kbase_ipa_simple.o \ -- ipa/mali_kbase_ipa.o \ -- ipa/mali_kbase_ipa_vinstr_g7x.o \ -- ipa/mali_kbase_ipa_vinstr_common.o -+ ipa/mali_kbase_ipa_simple.o \ -+ ipa/mali_kbase_ipa.o -+ -+mali_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o - --mali_kbase-$(CONFIG_DEBUG_FS) += ipa/mali_kbase_ipa_debugfs.o -\ No newline at end of file -+ifeq ($(MALI_USE_CSF),1) -+ mali_kbase-y += \ -+ ipa/backend/mali_kbase_ipa_counter_csf.o \ -+ ipa/backend/mali_kbase_ipa_counter_common_csf.o -+else -+ mali_kbase-y += \ -+ ipa/backend/mali_kbase_ipa_counter_jm.o \ -+ ipa/backend/mali_kbase_ipa_counter_common_jm.o -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_csf.c -new file mode 100644 -index 0000000..81dc56b ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_csf.c -@@ -0,0 +1,457 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_ipa_counter_common_csf.h" -+#include "ipa/mali_kbase_ipa_debugfs.h" -+ -+#define DEFAULT_SCALING_FACTOR 5 -+ -+/* If the value of GPU_ACTIVE is below this, use the simple model -+ * instead, to avoid extrapolating small amounts of counter data across -+ * large sample periods. -+ */ -+#define DEFAULT_MIN_SAMPLE_CYCLES 10000 -+ -+/* Typical value for the sampling interval is expected to be less than 100ms, -+ * So 5 seconds is a reasonable upper limit for the time gap between the -+ * 2 samples. -+ */ -+#define MAX_SAMPLE_INTERVAL_MS ((s64)5000) -+ -+/* Maximum increment that is expected for a counter value during a sampling -+ * interval is derived assuming -+ * - max sampling interval of 1 second. -+ * - max GPU frequency of 2 GHz. -+ * - max number of cores as 32. -+ * - max increment of 4 in per core counter value at every clock cycle. -+ * -+ * So max increment = 2 * 10^9 * 32 * 4 = ~2^38. -+ * If a counter increases by an amount greater than this value, then an error -+ * will be returned and the simple power model will be used. -+ */ -+#define MAX_COUNTER_INCREMENT (((u64)1 << 38) - 1) -+ -+static inline s64 kbase_ipa_add_saturate(s64 a, s64 b) -+{ -+ s64 rtn; -+ -+ if (a > 0 && (S64_MAX - a) < b) -+ rtn = S64_MAX; -+ else if (a < 0 && (S64_MIN - a) > b) -+ rtn = S64_MIN; -+ else -+ rtn = a + b; -+ -+ return rtn; -+} -+ -+static s64 kbase_ipa_group_energy(s32 coeff, u64 counter_value) -+{ -+ /* Range: 0 < counter_value < 2^38 */ -+ -+ /* Range: -2^59 < ret < 2^59 (as -2^21 < coeff < 2^21) */ -+ return counter_value * (s64)coeff; -+} -+ -+/** -+ * kbase_ipa_attach_ipa_control() - register with kbase_ipa_control -+ * @model_data: Pointer to counter model data -+ * -+ * Register IPA counter model as a client of kbase_ipa_control, which -+ * provides an interface to retreive the accumulated value of hardware -+ * counters to calculate energy consumption. -+ * -+ * Return: 0 on success, or an error code. -+ */ -+static int -+kbase_ipa_attach_ipa_control(struct kbase_ipa_counter_model_data *model_data) -+{ -+ struct kbase_device *kbdev = model_data->kbdev; -+ struct kbase_ipa_control_perf_counter *perf_counters; -+ u32 cnt_idx = 0; -+ int err; -+ size_t i; -+ -+ /* Value for GPU_ACTIVE counter also needs to be queried. It is required -+ * for the normalization of top-level and shader core counters. -+ */ -+ model_data->num_counters = 1 + model_data->num_top_level_cntrs + -+ model_data->num_shader_cores_cntrs; -+ -+ perf_counters = kcalloc(model_data->num_counters, -+ sizeof(*perf_counters), GFP_KERNEL); -+ -+ if (!perf_counters) { -+ dev_err(kbdev->dev, -+ "Failed to allocate memory for perf_counters array"); -+ return -ENOMEM; -+ } -+ -+ /* Fill in the description for GPU_ACTIVE counter which is always -+ * needed, as mentioned above, regardless of the energy model used -+ * by the CSF GPUs. -+ */ -+ perf_counters[cnt_idx].type = KBASE_IPA_CORE_TYPE_CSHW; -+ perf_counters[cnt_idx].idx = GPU_ACTIVE_CNT_IDX; -+ perf_counters[cnt_idx].gpu_norm = false; -+ perf_counters[cnt_idx].scaling_factor = 1; -+ cnt_idx++; -+ -+ for (i = 0; i < model_data->num_top_level_cntrs; ++i) { -+ const struct kbase_ipa_counter *counter = -+ &model_data->top_level_cntrs_def[i]; -+ -+ perf_counters[cnt_idx].type = counter->counter_block_type; -+ perf_counters[cnt_idx].idx = counter->counter_block_offset; -+ perf_counters[cnt_idx].gpu_norm = false; -+ perf_counters[cnt_idx].scaling_factor = 1; -+ cnt_idx++; -+ } -+ -+ for (i = 0; i < model_data->num_shader_cores_cntrs; ++i) { -+ const struct kbase_ipa_counter *counter = -+ &model_data->shader_cores_cntrs_def[i]; -+ -+ perf_counters[cnt_idx].type = counter->counter_block_type; -+ perf_counters[cnt_idx].idx = counter->counter_block_offset; -+ perf_counters[cnt_idx].gpu_norm = false; -+ perf_counters[cnt_idx].scaling_factor = 1; -+ cnt_idx++; -+ } -+ -+ err = kbase_ipa_control_register(kbdev, perf_counters, -+ model_data->num_counters, -+ &model_data->ipa_control_client); -+ if (err) -+ dev_err(kbdev->dev, -+ "Failed to register IPA with kbase_ipa_control"); -+ -+ kfree(perf_counters); -+ return err; -+} -+ -+/** -+ * kbase_ipa_detach_ipa_control() - De-register from kbase_ipa_control. -+ * @model_data: Pointer to counter model data -+ */ -+static void -+kbase_ipa_detach_ipa_control(struct kbase_ipa_counter_model_data *model_data) -+{ -+ if (model_data->ipa_control_client) { -+ kbase_ipa_control_unregister(model_data->kbdev, -+ model_data->ipa_control_client); -+ model_data->ipa_control_client = NULL; -+ } -+} -+ -+static int calculate_coeff(struct kbase_ipa_counter_model_data *model_data, -+ const struct kbase_ipa_counter *const cnt_defs, -+ size_t num_counters, s32 *counter_coeffs, -+ u64 *counter_values, u32 active_cycles, u32 *coeffp) -+{ -+ u64 coeff = 0, coeff_mul = 0; -+ s64 total_energy = 0; -+ size_t i; -+ -+ /* Range for the 'counter_value' is [0, 2^38) -+ * Range for the 'coeff' is [-2^21, 2^21] -+ * So range for the 'group_energy' is [-2^59, 2^59) and range for the -+ * 'total_energy' is +/- 2^59 * number of IPA groups (~16), i.e. -+ * [-2^63, 2^63). -+ */ -+ for (i = 0; i < num_counters; i++) { -+ s32 coeff = counter_coeffs[i]; -+ u64 counter_value = counter_values[i]; -+ s64 group_energy = kbase_ipa_group_energy(coeff, counter_value); -+ -+ if (counter_value > MAX_COUNTER_INCREMENT) { -+ dev_warn(model_data->kbdev->dev, -+ "Increment in counter %s more than expected", -+ cnt_defs[i].name); -+ return -ERANGE; -+ } -+ -+ total_energy = -+ kbase_ipa_add_saturate(total_energy, group_energy); -+ } -+ -+ /* Range: 0 <= coeff < 2^63 */ -+ if (total_energy >= 0) -+ coeff = total_energy; -+ else -+ dev_dbg(model_data->kbdev->dev, -+ "Energy value came negative as %lld", total_energy); -+ -+ /* Range: 0 <= coeff < 2^63 (because active_cycles >= 1). However, this -+ * can be constrained further: the value of counters that are being -+ * used for dynamic power estimation can only increment by about 128 -+ * maximum per clock cycle. This is because max number of shader -+ * cores is expected to be 32 (max number of L2 slices is expected to -+ * be 8) and some counters (per shader core) like SC_BEATS_RD_TEX_EXT & -+ * SC_EXEC_STARVE_ARITH can increment by 4 every clock cycle. -+ * Each "beat" is defined as 128 bits and each shader core can -+ * (currently) do 512 bits read and 512 bits write to/from the L2 -+ * cache per cycle, so the SC_BEATS_RD_TEX_EXT counter can increment -+ * [0, 4] per shader core per cycle. -+ * We can thus write the range of 'coeff' in terms of active_cycles: -+ * -+ * coeff = SUM(coeffN * counterN * num_cores_for_counterN) -+ * coeff <= SUM(coeffN * counterN) * max_cores -+ * coeff <= num_IPA_groups * max_coeff * max_counter * max_cores -+ * (substitute max_counter = 2^2 * active_cycles) -+ * coeff <= num_IPA_groups * max_coeff * 2^2 * active_cycles * max_cores -+ * coeff <= 2^4 * 2^21 * 2^2 * active_cycles * 2^5 -+ * coeff <= 2^32 * active_cycles -+ * -+ * So after the division: 0 <= coeff <= 2^32 -+ */ -+ coeff = div_u64(coeff, active_cycles); -+ -+ /* Not all models were derived at the same reference voltage. Voltage -+ * scaling is done by multiplying by V^2, so we need to *divide* by -+ * Vref^2 here. -+ * Range: 0 <= coeff <= 2^35 -+ */ -+ coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); -+ /* Range: 0 <= coeff <= 2^38 */ -+ coeff = div_u64(coeff * 1000, max(model_data->reference_voltage, 1)); -+ -+ /* Scale by user-specified integer factor. -+ * Range: 0 <= coeff_mul < 2^43 -+ */ -+ coeff_mul = coeff * model_data->scaling_factor; -+ -+ /* The power models have results with units -+ * mW/(MHz V^2), i.e. nW/(Hz V^2). With precision of 1/1000000, this -+ * becomes fW/(Hz V^2), which are the units of coeff_mul. However, -+ * kbase_scale_dynamic_power() expects units of pW/(Hz V^2), so divide -+ * by 1000. -+ * Range: 0 <= coeff_mul < 2^33 -+ */ -+ coeff_mul = div_u64(coeff_mul, 1000u); -+ -+ /* Clamp to a sensible range - 2^16 gives about 14W at 400MHz/750mV */ -+ *coeffp = clamp(coeff_mul, (u64)0, (u64)1 << 16); -+ -+ return 0; -+} -+ -+int kbase_ipa_counter_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) -+{ -+ struct kbase_ipa_counter_model_data *model_data = -+ (struct kbase_ipa_counter_model_data *)model->model_data; -+ struct kbase_device *kbdev = model->kbdev; -+ s32 *counter_coeffs_p = model_data->counter_coeffs; -+ u64 *cnt_values_p = model_data->counter_values; -+ const u64 num_counters = model_data->num_counters; -+ u32 active_cycles; -+ ktime_t now, diff; -+ s64 diff_ms; -+ int ret; -+ -+ lockdep_assert_held(&kbdev->ipa.lock); -+ -+ /* The last argument is supposed to be a pointer to the location that -+ * will store the time for which GPU has been in protected mode since -+ * last query. This can be passed as NULL as counter model itself will -+ * not be used when GPU enters protected mode, as IPA is supposed to -+ * switch to the simple power model. -+ */ -+ ret = kbase_ipa_control_query(kbdev, -+ model_data->ipa_control_client, -+ cnt_values_p, num_counters, NULL); -+ if (WARN_ON(ret)) -+ return ret; -+ -+ now = ktime_get(); -+ diff = ktime_sub(now, kbdev->ipa.last_sample_time); -+ diff_ms = ktime_to_ms(diff); -+ -+ kbdev->ipa.last_sample_time = now; -+ -+ /* The counter values cannot be relied upon if the sampling interval was -+ * too long. Typically this will happen when the polling is started -+ * after the temperature has risen above a certain trip point. After -+ * that regular calls every 25-100 ms interval are expected. -+ */ -+ if (diff_ms > MAX_SAMPLE_INTERVAL_MS) { -+ dev_dbg(kbdev->dev, -+ "Last sample was taken %lld milli seconds ago", -+ diff_ms); -+ return -EOVERFLOW; -+ } -+ -+ /* Range: 0 (GPU not used at all), to the max sampling interval, say -+ * 1 seconds, * max GPU frequency (GPU 100% utilized). -+ * 0 <= active_cycles <= 1 * ~2GHz -+ * 0 <= active_cycles < 2^31 -+ */ -+ if (*cnt_values_p > U32_MAX) { -+ dev_warn(kbdev->dev, -+ "Increment in GPU_ACTIVE counter more than expected"); -+ return -ERANGE; -+ } -+ -+ active_cycles = (u32)*cnt_values_p; -+ -+ /* If the value of the active_cycles is less than the threshold, then -+ * return an error so that IPA framework can approximate using the -+ * cached simple model results instead. This may be more accurate -+ * than extrapolating using a very small counter dump. -+ */ -+ if (active_cycles < (u32)max(model_data->min_sample_cycles, 0)) -+ return -ENODATA; -+ -+ /* Range: 1 <= active_cycles < 2^31 */ -+ active_cycles = max(1u, active_cycles); -+ -+ cnt_values_p++; -+ ret = calculate_coeff(model_data, model_data->top_level_cntrs_def, -+ model_data->num_top_level_cntrs, -+ counter_coeffs_p, cnt_values_p, active_cycles, -+ &coeffp[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]); -+ if (ret) -+ return ret; -+ -+ cnt_values_p += model_data->num_top_level_cntrs; -+ counter_coeffs_p += model_data->num_top_level_cntrs; -+ ret = calculate_coeff(model_data, model_data->shader_cores_cntrs_def, -+ model_data->num_shader_cores_cntrs, -+ counter_coeffs_p, cnt_values_p, active_cycles, -+ &coeffp[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]); -+ -+ return ret; -+} -+ -+void kbase_ipa_counter_reset_data(struct kbase_ipa_model *model) -+{ -+ struct kbase_ipa_counter_model_data *model_data = -+ (struct kbase_ipa_counter_model_data *)model->model_data; -+ u64 *cnt_values_p = model_data->counter_values; -+ const u64 num_counters = model_data->num_counters; -+ int ret; -+ -+ lockdep_assert_held(&model->kbdev->ipa.lock); -+ -+ ret = kbase_ipa_control_query(model->kbdev, -+ model_data->ipa_control_client, -+ cnt_values_p, num_counters, NULL); -+ WARN_ON(ret); -+} -+ -+int kbase_ipa_counter_common_model_init(struct kbase_ipa_model *model, -+ const struct kbase_ipa_counter *top_level_cntrs_def, -+ size_t num_top_level_cntrs, -+ const struct kbase_ipa_counter *shader_cores_cntrs_def, -+ size_t num_shader_cores_cntrs, -+ s32 reference_voltage) -+{ -+ struct kbase_ipa_counter_model_data *model_data; -+ s32 *counter_coeffs_p; -+ int err = 0; -+ size_t i; -+ -+ if (!model || !top_level_cntrs_def || !shader_cores_cntrs_def || -+ !num_top_level_cntrs || !num_shader_cores_cntrs) -+ return -EINVAL; -+ -+ model_data = kzalloc(sizeof(*model_data), GFP_KERNEL); -+ if (!model_data) -+ return -ENOMEM; -+ -+ model_data->kbdev = model->kbdev; -+ -+ model_data->top_level_cntrs_def = top_level_cntrs_def; -+ model_data->num_top_level_cntrs = num_top_level_cntrs; -+ -+ model_data->shader_cores_cntrs_def = shader_cores_cntrs_def; -+ model_data->num_shader_cores_cntrs = num_shader_cores_cntrs; -+ -+ model->model_data = (void *)model_data; -+ -+ counter_coeffs_p = model_data->counter_coeffs; -+ -+ for (i = 0; i < model_data->num_top_level_cntrs; ++i) { -+ const struct kbase_ipa_counter *counter = -+ &model_data->top_level_cntrs_def[i]; -+ -+ *counter_coeffs_p = counter->coeff_default_value; -+ -+ err = kbase_ipa_model_add_param_s32( -+ model, counter->name, counter_coeffs_p, 1, false); -+ if (err) -+ goto exit; -+ -+ counter_coeffs_p++; -+ } -+ -+ for (i = 0; i < model_data->num_shader_cores_cntrs; ++i) { -+ const struct kbase_ipa_counter *counter = -+ &model_data->shader_cores_cntrs_def[i]; -+ -+ *counter_coeffs_p = counter->coeff_default_value; -+ -+ err = kbase_ipa_model_add_param_s32( -+ model, counter->name, counter_coeffs_p, 1, false); -+ if (err) -+ goto exit; -+ -+ counter_coeffs_p++; -+ } -+ -+ model_data->scaling_factor = DEFAULT_SCALING_FACTOR; -+ err = kbase_ipa_model_add_param_s32( -+ model, "scale", &model_data->scaling_factor, 1, false); -+ if (err) -+ goto exit; -+ -+ model_data->min_sample_cycles = DEFAULT_MIN_SAMPLE_CYCLES; -+ err = kbase_ipa_model_add_param_s32(model, "min_sample_cycles", -+ &model_data->min_sample_cycles, 1, -+ false); -+ if (err) -+ goto exit; -+ -+ model_data->reference_voltage = reference_voltage; -+ err = kbase_ipa_model_add_param_s32(model, "reference_voltage", -+ &model_data->reference_voltage, 1, -+ false); -+ if (err) -+ goto exit; -+ -+ err = kbase_ipa_attach_ipa_control(model_data); -+ -+exit: -+ if (err) { -+ kbase_ipa_model_param_free_all(model); -+ kfree(model_data); -+ } -+ return err; -+} -+ -+void kbase_ipa_counter_common_model_term(struct kbase_ipa_model *model) -+{ -+ struct kbase_ipa_counter_model_data *model_data = -+ (struct kbase_ipa_counter_model_data *)model->model_data; -+ -+ kbase_ipa_detach_ipa_control(model_data); -+ kfree(model_data); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_csf.h b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_csf.h -new file mode 100644 -index 0000000..37d2efc ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_csf.h -@@ -0,0 +1,159 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_IPA_COUNTER_COMMON_CSF_H_ -+#define _KBASE_IPA_COUNTER_COMMON_CSF_H_ -+ -+#include "mali_kbase.h" -+#include "csf/ipa_control/mali_kbase_csf_ipa_control.h" -+ -+/* Maximum number of HW counters used by the IPA counter model. */ -+#define KBASE_IPA_MAX_COUNTER_DEF_NUM 24 -+ -+struct kbase_ipa_counter_model_data; -+ -+/** -+ * struct kbase_ipa_counter_model_data - IPA counter model context per device -+ * @kbdev: Pointer to kbase device -+ * @ipa_control_client: Handle returned on registering IPA counter model as a -+ * client of kbase_ipa_control. -+ * @top_level_cntrs_def: Array of description of HW counters used by the IPA -+ * counter model for top-level. -+ * @num_top_level_cntrs: Number of elements in @top_level_cntrs_def array. -+ * @shader_cores_cntrs_def: Array of description of HW counters used by the IPA -+ * counter model for shader cores. -+ * @num_shader_cores_cntrs: Number of elements in @shader_cores_cntrs_def array. -+ * @counter_coeffs: Buffer to store coefficient value used for HW counters -+ * @counter_values: Buffer to store the accumulated value of HW counters -+ * retreived from kbase_ipa_control. -+ * @num_counters: Number of counters queried from kbase_ipa_control. -+ * @reference_voltage: voltage, in mV, of the operating point used when -+ * deriving the power model coefficients. Range approx -+ * 0.1V - 5V (~= 8V): 2^7 <= reference_voltage <= 2^13 -+ * @scaling_factor: User-specified power scaling factor. This is an -+ * integer, which is multiplied by the power coefficient -+ * just before OPP scaling. -+ * Range approx 0-32: 0 < scaling_factor < 2^5 -+ * @min_sample_cycles: If the value of the GPU_ACTIVE counter (the number of -+ * cycles the GPU was working) is less than -+ * min_sample_cycles, the counter model will return an -+ * error, causing the IPA framework to approximate using -+ * the cached simple model results instead. This may be -+ * more accurate than extrapolating using a very small -+ * counter dump. -+ */ -+struct kbase_ipa_counter_model_data { -+ struct kbase_device *kbdev; -+ void *ipa_control_client; -+ const struct kbase_ipa_counter *top_level_cntrs_def; -+ size_t num_top_level_cntrs; -+ const struct kbase_ipa_counter *shader_cores_cntrs_def; -+ size_t num_shader_cores_cntrs; -+ s32 counter_coeffs[KBASE_IPA_MAX_COUNTER_DEF_NUM]; -+ u64 counter_values[KBASE_IPA_MAX_COUNTER_DEF_NUM]; -+ u64 num_counters; -+ s32 reference_voltage; -+ s32 scaling_factor; -+ s32 min_sample_cycles; -+}; -+ -+/** -+ * struct kbase_ipa_counter - represents a single HW counter used by IPA model -+ * @name: Name of the HW counter used by IPA counter model -+ * for energy estimation. -+ * @coeff_default_value: Default value of coefficient for the counter. -+ * Coefficients are interpreted as fractions where the -+ * denominator is 1000000. -+ * @counter_block_offset: Index to the counter within the counter block of -+ * type @counter_block_type. -+ * @counter_block_type: Type of the counter block. -+ */ -+struct kbase_ipa_counter { -+ const char *name; -+ s32 coeff_default_value; -+ u32 counter_block_offset; -+ enum kbase_ipa_core_type counter_block_type; -+}; -+ -+/** -+ * kbase_ipa_counter_dynamic_coeff() - calculate dynamic power based on HW counters -+ * @model: pointer to instantiated model -+ * @coeffp: pointer to location where calculated power, in -+ * pW/(Hz V^2), is stored for top level and shader cores. -+ * -+ * This is a GPU-agnostic implementation of the get_dynamic_coeff() -+ * function of an IPA model. It relies on the model being populated -+ * with GPU-specific attributes at initialization time. -+ * -+ * Return: 0 on success, or an error code. -+ */ -+int kbase_ipa_counter_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); -+ -+/** -+ * kbase_ipa_counter_reset_data() - Reset the counters data used for dynamic -+ * power estimation -+ * @model: pointer to instantiated model -+ * -+ * Retrieve the accumulated value of HW counters from the kbase_ipa_control -+ * component, without doing any processing, which is effectively a reset as the -+ * next call to kbase_ipa_counter_dynamic_coeff() will see the increment in -+ * counter values from this point onwards. -+ */ -+void kbase_ipa_counter_reset_data(struct kbase_ipa_model *model); -+ -+/** -+ * kbase_ipa_counter_common_model_init() - initialize ipa power model -+ * @model: Pointer to the ipa power model to initialize -+ * @top_level_cntrs_def: Array corresponding to the HW counters used in the -+ * top level counter model, contains the counter index, -+ * default value of the coefficient. -+ * @num_top_level_cntrs: Number of elements in the array @top_level_cntrs_def -+ * @shader_cores_cntrs_def: Array corresponding to the HW counters used in the -+ * shader cores counter model, contains the counter index, -+ * default value of the coefficient. -+ * @num_shader_cores_cntrs: Number of elements in the array -+ * @shader_cores_cntrs_def. -+ * @reference_voltage: voltage, in mV, of the operating point used when -+ * deriving the power model coefficients. -+ * -+ * This function performs initialization steps common for ipa counter based -+ * model of all CSF GPUs. The set of counters and their respective weights -+ * could be different for each GPU. The tuple of counter index and weight -+ * is passed via @top_level_cntrs_def and @shader_cores_cntrs_def array. -+ * -+ * Return: 0 on success, error code otherwise -+ */ -+int kbase_ipa_counter_common_model_init(struct kbase_ipa_model *model, -+ const struct kbase_ipa_counter *top_level_cntrs_def, -+ size_t num_top_level_cntrs, -+ const struct kbase_ipa_counter *shader_cores_cntrs_def, -+ size_t num_shader_cores_cntrs, -+ s32 reference_voltage); -+/** -+ * kbase_ipa_counter_common_model_term() - terminate ipa power model -+ * @model: ipa power model to terminate -+ * -+ * This function performs all necessary steps to terminate ipa power model -+ * including clean up of resources allocated to hold model data. -+ */ -+void kbase_ipa_counter_common_model_term(struct kbase_ipa_model *model); -+ -+#endif /* _KBASE_IPA_COUNTER_COMMON_CSF_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_vinstr_common.c b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_jm.c -similarity index 95% -rename from dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_vinstr_common.c -rename to dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_jm.c -index 9fae8f1..4737b0e 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_vinstr_common.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_jm.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2017-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#include "mali_kbase_ipa_vinstr_common.h" --#include "mali_kbase_ipa_debugfs.h" -+#include "mali_kbase_ipa_counter_common_jm.h" -+#include "ipa/mali_kbase_ipa_debugfs.h" - - #define DEFAULT_SCALING_FACTOR 5 - -@@ -145,6 +144,9 @@ int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data) - - kbase_hwcnt_enable_map_enable_all(&enable_map); - -+ /* Disable cycle counter only. */ -+ enable_map.clk_enable_map = 0; -+ - errcode = kbase_hwcnt_virtualizer_client_create( - hvirt, &enable_map, &model_data->hvirt_cli); - kbase_hwcnt_enable_map_free(&enable_map); -@@ -270,6 +272,12 @@ err0: - return err; - } - -+void kbase_ipa_vinstr_reset_data(struct kbase_ipa_model *model) -+{ -+ /* Currently not implemented */ -+ WARN_ON_ONCE(1); -+} -+ - int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, - const struct kbase_ipa_group *ipa_groups_def, - size_t ipa_group_size, -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_vinstr_common.h b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_jm.h -similarity index 85% -rename from dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_vinstr_common.h -rename to dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_jm.h -index 46e3cd4..3486a9b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_vinstr_common.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_common_jm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifndef _KBASE_IPA_VINSTR_COMMON_H_ --#define _KBASE_IPA_VINSTR_COMMON_H_ -+#ifndef _KBASE_IPA_COUNTER_COMMON_JM_H_ -+#define _KBASE_IPA_COUNTER_COMMON_JM_H_ - - #include "mali_kbase.h" - #include "mali_kbase_hwcnt_virtualizer.h" -@@ -42,11 +41,13 @@ - - struct kbase_ipa_model_vinstr_data; - --typedef u32 (*kbase_ipa_get_active_cycles_callback)(struct kbase_ipa_model_vinstr_data *); -+typedef u32 -+kbase_ipa_get_active_cycles_callback(struct kbase_ipa_model_vinstr_data *); - - /** - * struct kbase_ipa_model_vinstr_data - IPA context per device - * @kbdev: pointer to kbase device -+ * @group_values: values of coefficients for IPA groups - * @groups_def: Array of IPA groups. - * @groups_def_num: Number of elements in the array of IPA groups. - * @get_active_cycles: Callback to return number of active cycles during -@@ -73,7 +74,7 @@ struct kbase_ipa_model_vinstr_data { - s32 group_values[KBASE_IPA_MAX_GROUP_DEF_NUM]; - const struct kbase_ipa_group *groups_def; - size_t groups_def_num; -- kbase_ipa_get_active_cycles_callback get_active_cycles; -+ kbase_ipa_get_active_cycles_callback *get_active_cycles; - struct kbase_hwcnt_virtualizer_client *hvirt_cli; - struct kbase_hwcnt_dump_buffer dump_buf; - s32 reference_voltage; -@@ -102,7 +103,7 @@ struct kbase_ipa_group { - * @model_data: pointer to model data - * @coeff: model coefficient. Unity is ~2^20, so range approx - * +/- 4.0: -2^22 < coeff < 2^22 -- * @counter offset in bytes of the counter used to calculate energy -+ * @counter: offset in bytes of the counter used to calculate energy - * for IPA group - * - * Calculate energy estimation based on hardware counter `counter' -@@ -149,7 +150,7 @@ s64 kbase_ipa_single_counter( - - /** - * attach_vinstr() - attach a vinstr_buffer to an IPA model. -- * @model_data pointer to model data -+ * @model_data: pointer to model data - * - * Attach a vinstr_buffer to an IPA model. The vinstr_buffer - * allows access to the hardware counters used to calculate -@@ -161,7 +162,7 @@ int kbase_ipa_attach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); - - /** - * detach_vinstr() - detach a vinstr_buffer from an IPA model. -- * @model_data pointer to model data -+ * @model_data: pointer to model data - * - * Detach a vinstr_buffer from an IPA model. - */ -@@ -181,6 +182,19 @@ void kbase_ipa_detach_vinstr(struct kbase_ipa_model_vinstr_data *model_data); - */ - int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); - -+/** -+ * kbase_ipa_vinstr_reset_data() - Reset the counters data used for dynamic -+ * power estimation -+ * @model: pointer to instantiated model -+ * -+ * Currently it is not implemented for JM GPUs. -+ * When implemented it is expected to retrieve the accumulated value of HW -+ * counters from the Vinstr component, without doing any processing, which is -+ * effectively a reset as the next call to kbase_ipa_counter_dynamic_coeff() -+ * will see the increment in counter values from this point onwards. -+ */ -+void kbase_ipa_vinstr_reset_data(struct kbase_ipa_model *model); -+ - /** - * kbase_ipa_vinstr_common_model_init() - initialize ipa power model - * @model: ipa power model to initialize -@@ -202,7 +216,7 @@ int kbase_ipa_vinstr_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp); - int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, - const struct kbase_ipa_group *ipa_groups_def, - size_t ipa_group_size, -- kbase_ipa_get_active_cycles_callback get_active_cycles, -+ kbase_ipa_get_active_cycles_callback *get_active_cycles, - s32 reference_voltage); - - /** -@@ -214,4 +228,4 @@ int kbase_ipa_vinstr_common_model_init(struct kbase_ipa_model *model, - */ - void kbase_ipa_vinstr_common_model_term(struct kbase_ipa_model *model); - --#endif /* _KBASE_IPA_VINSTR_COMMON_H_ */ -+#endif /* _KBASE_IPA_COUNTER_COMMON_JM_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_csf.c -new file mode 100644 -index 0000000..1852c3c ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_csf.c -@@ -0,0 +1,171 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_ipa_counter_common_csf.h" -+#include "mali_kbase.h" -+ -+/* MEMSYS counter block offsets */ -+#define L2_RD_MSG_IN (16) -+#define L2_WR_MSG_IN (18) -+#define L2_READ_LOOKUP (26) -+#define L2_EXT_WRITE_NOSNP_FULL (43) -+ -+/* SC counter block offsets */ -+#define FRAG_QUADS_EZS_UPDATE (13) -+#define FULL_QUAD_WARPS (21) -+#define EXEC_INSTR_FMA (27) -+#define EXEC_INSTR_CVT (28) -+#define TEX_FILT_NUM_OPS (39) -+#define LS_MEM_READ_SHORT (45) -+#define LS_MEM_WRITE_SHORT (47) -+#define VARY_SLOT_16 (51) -+ -+/* Tiler counter block offsets */ -+#define IDVS_POS_SHAD_STALL (23) -+#define PREFETCH_STALL (25) -+#define VFETCH_POS_READ_WAIT (29) -+#define VFETCH_VERTEX_WAIT (30) -+#define IDVS_VAR_SHAD_STALL (38) -+ -+#define COUNTER_DEF(cnt_name, coeff, cnt_idx, block_type) \ -+ { \ -+ .name = cnt_name, \ -+ .coeff_default_value = coeff, \ -+ .counter_block_offset = cnt_idx, \ -+ .counter_block_type = block_type, \ -+ } -+ -+#define CSHW_COUNTER_DEF(cnt_name, coeff, cnt_idx) \ -+ COUNTER_DEF(cnt_name, coeff, cnt_idx, KBASE_IPA_CORE_TYPE_CSHW) -+ -+#define MEMSYS_COUNTER_DEF(cnt_name, coeff, cnt_idx) \ -+ COUNTER_DEF(cnt_name, coeff, cnt_idx, KBASE_IPA_CORE_TYPE_MEMSYS) -+ -+#define SC_COUNTER_DEF(cnt_name, coeff, cnt_idx) \ -+ COUNTER_DEF(cnt_name, coeff, cnt_idx, KBASE_IPA_CORE_TYPE_SHADER) -+ -+#define TILER_COUNTER_DEF(cnt_name, coeff, cnt_idx) \ -+ COUNTER_DEF(cnt_name, coeff, cnt_idx, KBASE_IPA_CORE_TYPE_TILER) -+ -+/* Tables of description of HW counters used by IPA counter model. -+ * -+ * These tables provide a description of each performance counter -+ * used by the top level counter model for energy estimation. -+ */ -+static const struct kbase_ipa_counter ipa_top_level_cntrs_def_todx[] = { -+ MEMSYS_COUNTER_DEF("l2_rd_msg_in", 295631, L2_RD_MSG_IN), -+ MEMSYS_COUNTER_DEF("l2_ext_write_nosnp_ull", 325168, L2_EXT_WRITE_NOSNP_FULL), -+ -+ TILER_COUNTER_DEF("prefetch_stall", 145435, PREFETCH_STALL), -+ TILER_COUNTER_DEF("idvs_var_shad_stall", -171917, IDVS_VAR_SHAD_STALL), -+ TILER_COUNTER_DEF("idvs_pos_shad_stall", 109980, IDVS_POS_SHAD_STALL), -+ TILER_COUNTER_DEF("vfetch_pos_read_wait", -119118, VFETCH_POS_READ_WAIT), -+}; -+ -+ -+/* These tables provide a description of each performance counter -+ * used by the shader cores counter model for energy estimation. -+ */ -+static const struct kbase_ipa_counter ipa_shader_core_cntrs_def_todx[] = { -+ SC_COUNTER_DEF("exec_instr_fma", 505449, EXEC_INSTR_FMA), -+ SC_COUNTER_DEF("tex_filt_num_operations", 574869, TEX_FILT_NUM_OPS), -+ SC_COUNTER_DEF("ls_mem_read_short", 60917, LS_MEM_READ_SHORT), -+ SC_COUNTER_DEF("frag_quads_ezs_update", 694555, FRAG_QUADS_EZS_UPDATE), -+ SC_COUNTER_DEF("ls_mem_write_short", 698290, LS_MEM_WRITE_SHORT), -+ SC_COUNTER_DEF("vary_slot_16", 181069, VARY_SLOT_16), -+}; -+ -+ -+#define IPA_POWER_MODEL_OPS(gpu, init_token) \ -+ const struct kbase_ipa_model_ops kbase_ ## gpu ## _ipa_model_ops = { \ -+ .name = "mali-" #gpu "-power-model", \ -+ .init = kbase_ ## init_token ## _power_model_init, \ -+ .term = kbase_ipa_counter_common_model_term, \ -+ .get_dynamic_coeff = kbase_ipa_counter_dynamic_coeff, \ -+ .reset_counter_data = kbase_ipa_counter_reset_data, \ -+ }; \ -+ KBASE_EXPORT_TEST_API(kbase_ ## gpu ## _ipa_model_ops) -+ -+#define STANDARD_POWER_MODEL(gpu, reference_voltage) \ -+ static int kbase_ ## gpu ## _power_model_init(\ -+ struct kbase_ipa_model *model) \ -+ { \ -+ BUILD_BUG_ON((1 + \ -+ ARRAY_SIZE(ipa_top_level_cntrs_def_ ## gpu) +\ -+ ARRAY_SIZE(ipa_shader_core_cntrs_def_ ## gpu)) > \ -+ KBASE_IPA_MAX_COUNTER_DEF_NUM); \ -+ return kbase_ipa_counter_common_model_init(model, \ -+ ipa_top_level_cntrs_def_ ## gpu, \ -+ ARRAY_SIZE(ipa_top_level_cntrs_def_ ## gpu), \ -+ ipa_shader_core_cntrs_def_ ## gpu, \ -+ ARRAY_SIZE(ipa_shader_core_cntrs_def_ ## gpu), \ -+ (reference_voltage)); \ -+ } \ -+ IPA_POWER_MODEL_OPS(gpu, gpu) -+ -+ -+#define ALIAS_POWER_MODEL(gpu, as_gpu) \ -+ IPA_POWER_MODEL_OPS(gpu, as_gpu) -+ -+/* Reference voltage value is 750 mV. -+ */ -+STANDARD_POWER_MODEL(todx, 750); -+ -+ -+/* Assuming LODX is an alias of TODX for IPA */ -+ALIAS_POWER_MODEL(lodx, todx); -+ -+static const struct kbase_ipa_model_ops *ipa_counter_model_ops[] = { -+ &kbase_todx_ipa_model_ops, &kbase_lodx_ipa_model_ops, -+}; -+ -+const struct kbase_ipa_model_ops *kbase_ipa_counter_model_ops_find( -+ struct kbase_device *kbdev, const char *name) -+{ -+ int i; -+ -+ for (i = 0; i < ARRAY_SIZE(ipa_counter_model_ops); ++i) { -+ const struct kbase_ipa_model_ops *ops = -+ ipa_counter_model_ops[i]; -+ -+ if (!strcmp(ops->name, name)) -+ return ops; -+ } -+ -+ dev_err(kbdev->dev, "power model \'%s\' not found\n", name); -+ -+ return NULL; -+} -+ -+const char *kbase_ipa_counter_model_name_from_id(u32 gpu_id) -+{ -+ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> -+ GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ -+ switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { -+ case GPU_ID2_PRODUCT_TODX: -+ return "mali-todx-power-model"; -+ case GPU_ID2_PRODUCT_LODX: -+ return "mali-lodx-power-model"; -+ default: -+ return NULL; -+ } -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_vinstr_g7x.c b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_jm.c -similarity index 83% -rename from dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_vinstr_g7x.c -rename to dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_jm.c -index 270b75e..2f4c9d9 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_vinstr_g7x.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/backend/mali_kbase_ipa_counter_jm.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2016-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2016-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,11 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ -+ - #include <linux/thermal.h> - --#include "mali_kbase_ipa_vinstr_common.h" -+#include "mali_kbase_ipa_counter_common_jm.h" - #include "mali_kbase.h" - - -@@ -97,7 +97,6 @@ static u32 kbase_g7x_power_model_get_sc_counter(struct kbase_ipa_model_vinstr_da - const u32 sc_base = MEMSYS_BASE + - (model_data->kbdev->gpu_props.props.l2_props.num_l2_slices * - KBASE_IPA_NR_BYTES_PER_BLOCK); -- - return sc_base + counter_block_offset; - } - -@@ -178,7 +177,7 @@ static u32 kbase_g7x_get_active_cycles( - return kbase_ipa_single_counter(model_data, 1, counter); - } - --/** Table of IPA group definitions. -+/* Table of IPA group definitions. - * - * For each IPA group, this table defines a function to access the given performance block counter (or counters, - * if the operation needs to be iterated on multiple blocks) and calculate energy estimation. -@@ -415,6 +414,39 @@ static const struct kbase_ipa_group ipa_groups_def_tbex[] = { - }, - }; - -+static const struct kbase_ipa_group ipa_groups_def_tbax[] = { -+ { -+ .name = "l2_access", -+ .default_value = 599800, -+ .op = kbase_g7x_sum_all_memsys_blocks, -+ .counter_block_offset = MEMSYS_L2_ANY_LOOKUP, -+ }, -+ { -+ .name = "exec_instr_msg", -+ .default_value = 1830200, -+ .op = kbase_g7x_sum_all_shader_cores, -+ .counter_block_offset = SC_EXEC_INSTR_MSG, -+ }, -+ { -+ .name = "exec_instr_fma", -+ .default_value = 407300, -+ .op = kbase_g7x_sum_all_shader_cores, -+ .counter_block_offset = SC_EXEC_INSTR_FMA, -+ }, -+ { -+ .name = "tex_filt_num_operations", -+ .default_value = 224500, -+ .op = kbase_g7x_sum_all_shader_cores, -+ .counter_block_offset = SC_TEX_FILT_NUM_OPERATIONS, -+ }, -+ { -+ .name = "gpu_active", -+ .default_value = 153800, -+ .op = kbase_g7x_jm_single_counter, -+ .counter_block_offset = JM_GPU_ACTIVE, -+ }, -+}; -+ - - #define IPA_POWER_MODEL_OPS(gpu, init_token) \ - const struct kbase_ipa_model_ops kbase_ ## gpu ## _ipa_model_ops = { \ -@@ -422,6 +454,7 @@ static const struct kbase_ipa_group ipa_groups_def_tbex[] = { - .init = kbase_ ## init_token ## _power_model_init, \ - .term = kbase_ipa_vinstr_common_model_term, \ - .get_dynamic_coeff = kbase_ipa_vinstr_dynamic_coeff, \ -+ .reset_counter_data = kbase_ipa_vinstr_reset_data, \ - }; \ - KBASE_EXPORT_TEST_API(kbase_ ## gpu ## _ipa_model_ops) - -@@ -449,8 +482,74 @@ STANDARD_POWER_MODEL(g52_r1, 1000); - STANDARD_POWER_MODEL(g51, 1000); - STANDARD_POWER_MODEL(g77, 1000); - STANDARD_POWER_MODEL(tbex, 1000); -+STANDARD_POWER_MODEL(tbax, 1000); - - /* g52 is an alias of g76 (TNOX) for IPA */ - ALIAS_POWER_MODEL(g52, g76); - /* tnax is an alias of g77 (TTRX) for IPA */ - ALIAS_POWER_MODEL(tnax, g77); -+ -+static const struct kbase_ipa_model_ops *ipa_counter_model_ops[] = { -+ &kbase_g71_ipa_model_ops, -+ &kbase_g72_ipa_model_ops, -+ &kbase_g76_ipa_model_ops, -+ &kbase_g52_ipa_model_ops, -+ &kbase_g52_r1_ipa_model_ops, -+ &kbase_g51_ipa_model_ops, -+ &kbase_g77_ipa_model_ops, -+ &kbase_tnax_ipa_model_ops, -+ &kbase_tbex_ipa_model_ops, -+ &kbase_tbax_ipa_model_ops -+}; -+ -+const struct kbase_ipa_model_ops *kbase_ipa_counter_model_ops_find( -+ struct kbase_device *kbdev, const char *name) -+{ -+ int i; -+ -+ for (i = 0; i < ARRAY_SIZE(ipa_counter_model_ops); ++i) { -+ const struct kbase_ipa_model_ops *ops = -+ ipa_counter_model_ops[i]; -+ -+ if (!strcmp(ops->name, name)) -+ return ops; -+ } -+ -+ dev_err(kbdev->dev, "power model \'%s\' not found\n", name); -+ -+ return NULL; -+} -+ -+const char *kbase_ipa_counter_model_name_from_id(u32 gpu_id) -+{ -+ const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> -+ GPU_ID_VERSION_PRODUCT_ID_SHIFT; -+ -+ switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { -+ case GPU_ID2_PRODUCT_TMIX: -+ return "mali-g71-power-model"; -+ case GPU_ID2_PRODUCT_THEX: -+ return "mali-g72-power-model"; -+ case GPU_ID2_PRODUCT_TNOX: -+ return "mali-g76-power-model"; -+ case GPU_ID2_PRODUCT_TSIX: -+ return "mali-g51-power-model"; -+ case GPU_ID2_PRODUCT_TGOX: -+ if ((gpu_id & GPU_ID2_VERSION_MAJOR) == -+ (0 << GPU_ID2_VERSION_MAJOR_SHIFT)) -+ /* g52 aliased to g76 power-model's ops */ -+ return "mali-g52-power-model"; -+ else -+ return "mali-g52_r1-power-model"; -+ case GPU_ID2_PRODUCT_TNAX: -+ return "mali-tnax-power-model"; -+ case GPU_ID2_PRODUCT_TTRX: -+ return "mali-g77-power-model"; -+ case GPU_ID2_PRODUCT_TBEX: -+ return "mali-tbex-power-model"; -+ case GPU_ID2_PRODUCT_TBAX: -+ return "mali-tbax-power-model"; -+ default: -+ return NULL; -+ } -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c -old mode 100644 -new mode 100755 -index d663ccb..24d7b06 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2016-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2016-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,9 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ -+ - #include <linux/thermal.h> - #include <linux/devfreq_cooling.h> - #include <linux/of.h> -@@ -27,30 +27,19 @@ - #include "mali_kbase_ipa_debugfs.h" - #include "mali_kbase_ipa_simple.h" - #include "backend/gpu/mali_kbase_pm_internal.h" -- --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)) -+#include "backend/gpu/mali_kbase_devfreq.h" - #include <linux/pm_opp.h> --#else --#include <linux/opp.h> --#define dev_pm_opp_find_freq_exact opp_find_freq_exact --#define dev_pm_opp_get_voltage opp_get_voltage --#define dev_pm_opp opp --#endif - - #define KBASE_IPA_FALLBACK_MODEL_NAME "mali-simple-power-model" - --static const struct kbase_ipa_model_ops *kbase_ipa_all_model_ops[] = { -- &kbase_simple_ipa_model_ops, -- &kbase_g71_ipa_model_ops, -- &kbase_g72_ipa_model_ops, -- &kbase_g76_ipa_model_ops, -- &kbase_g52_ipa_model_ops, -- &kbase_g52_r1_ipa_model_ops, -- &kbase_g51_ipa_model_ops, -- &kbase_g77_ipa_model_ops, -- &kbase_tnax_ipa_model_ops, -- &kbase_tbex_ipa_model_ops --}; -+/* Polling by thermal governor starts when the temperature exceeds the certain -+ * trip point. In order to have meaningful value for the counters, when the -+ * polling starts and first call to kbase_get_real_power() is made, it is -+ * required to reset the counter values every now and then. -+ * It is reasonable to do the reset every second if no polling is being done, -+ * the counter model implementation also assumes max sampling interval of 1 sec. -+ */ -+#define RESET_INTERVAL_MS ((s64)1000) - - int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) - { -@@ -71,53 +60,24 @@ int kbase_ipa_model_recalculate(struct kbase_ipa_model *model) - } - - const struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, -- const char *name) -+ const char *name) - { -- int i; -+ if (!strcmp(name, kbase_simple_ipa_model_ops.name)) -+ return &kbase_simple_ipa_model_ops; - -- for (i = 0; i < ARRAY_SIZE(kbase_ipa_all_model_ops); ++i) { -- const struct kbase_ipa_model_ops *ops = kbase_ipa_all_model_ops[i]; -- -- if (!strcmp(ops->name, name)) -- return ops; -- } -- -- dev_err(kbdev->dev, "power model \'%s\' not found\n", name); -- -- return NULL; -+ return kbase_ipa_counter_model_ops_find(kbdev, name); - } - KBASE_EXPORT_TEST_API(kbase_ipa_model_ops_find); - - const char *kbase_ipa_model_name_from_id(u32 gpu_id) - { -- const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >> -- GPU_ID_VERSION_PRODUCT_ID_SHIFT; -- -- switch (GPU_ID2_MODEL_MATCH_VALUE(prod_id)) { -- case GPU_ID2_PRODUCT_TMIX: -- return "mali-g71-power-model"; -- case GPU_ID2_PRODUCT_THEX: -- return "mali-g72-power-model"; -- case GPU_ID2_PRODUCT_TNOX: -- return "mali-g76-power-model"; -- case GPU_ID2_PRODUCT_TSIX: -- return "mali-g51-power-model"; -- case GPU_ID2_PRODUCT_TGOX: -- if ((gpu_id & GPU_ID2_VERSION_MAJOR) == -- (0 << GPU_ID2_VERSION_MAJOR_SHIFT)) -- /* g52 aliased to g76 power-model's ops */ -- return "mali-g52-power-model"; -- else -- return "mali-g52_r1-power-model"; -- case GPU_ID2_PRODUCT_TNAX: -- return "mali-tnax-power-model"; -- case GPU_ID2_PRODUCT_TTRX: -- return "mali-g77-power-model"; -- case GPU_ID2_PRODUCT_TBEX: -- return "mali-tbex-power-model"; -- default: -+ const char* model_name = -+ kbase_ipa_counter_model_name_from_id(gpu_id); -+ -+ if (!model_name) - return KBASE_IPA_FALLBACK_MODEL_NAME; -- } -+ else -+ return model_name; - } - KBASE_EXPORT_TEST_API(kbase_ipa_model_name_from_id); - -@@ -364,6 +324,8 @@ int kbase_ipa_init(struct kbase_device *kbdev) - kbdev->ipa.configured_model = default_model; - } - -+ kbdev->ipa.last_sample_time = ktime_get(); -+ - end: - if (err) - kbase_ipa_term_locked(kbdev); -@@ -418,7 +380,8 @@ static u32 kbase_scale_dynamic_power(const u32 c, const u32 freq, - const u32 v2f = v2f_big / 1000; - - /* Range (working backwards from next line): 0 < v2fc < 2^23 uW. -- * Must be < 2^42 to avoid overflowing the return value. */ -+ * Must be < 2^42 to avoid overflowing the return value. -+ */ - const u64 v2fc = (u64) c * (u64) v2f; - - /* Range: 0 < v2fc / 1000 < 2^13 mW */ -@@ -514,8 +477,9 @@ static u32 get_static_power_locked(struct kbase_device *kbdev, - return power; - } - --#if defined(CONFIG_MALI_PWRSOFT_765) || \ -- LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+#if KERNEL_VERSION(5, 10, 0) > LINUX_VERSION_CODE -+#if defined(CONFIG_MALI_PWRSOFT_765) || \ -+ KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE - static unsigned long kbase_get_static_power(struct devfreq *df, - unsigned long voltage) - #else -@@ -524,8 +488,8 @@ static unsigned long kbase_get_static_power(unsigned long voltage) - { - struct kbase_ipa_model *model; - u32 power = 0; --#if defined(CONFIG_MALI_PWRSOFT_765) || \ -- LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+#if defined(CONFIG_MALI_PWRSOFT_765) || \ -+ KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE - struct kbase_device *kbdev = dev_get_drvdata(&df->dev); - #else - struct kbase_device *kbdev = kbase_find_device(-1); -@@ -541,16 +505,55 @@ static unsigned long kbase_get_static_power(unsigned long voltage) - - mutex_unlock(&kbdev->ipa.lock); - --#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ -- LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) -+#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ -+ KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) - kbase_release_device(kbdev); - #endif - - return power; - } -+#endif /* KERNEL_VERSION(5, 10, 0) > LINUX_VERSION_CODE */ -+ -+/** -+ * opp_translate_freq_voltage() - Translate nominal OPP frequency from -+ * devicetree into the real frequency for -+ * top-level and shader cores. -+ * @kbdev: Device pointer -+ * @nominal_freq: Nominal frequency in Hz. -+ * @nominal_voltage: Nominal voltage, in mV. -+ * @freqs: Pointer to array of real frequency values. -+ * @volts: Pointer to array of voltages. -+ * -+ * If there are 2 clock domains, then top-level and shader cores can operate -+ * at different frequency and voltage level. The nominal frequency ("opp-hz") -+ * used by devfreq from the devicetree may not be same as the real frequency -+ * at which top-level and shader cores are operating, so a translation is -+ * needed. -+ * Nominal voltage shall always be same as the real voltage for top-level. -+ */ -+static void opp_translate_freq_voltage(struct kbase_device *kbdev, -+ unsigned long nominal_freq, -+ unsigned long nominal_voltage, -+ unsigned long *freqs, -+ unsigned long *volts) -+{ -+ u64 core_mask; -+ -+ kbase_devfreq_opp_translate(kbdev, nominal_freq, &core_mask, -+ freqs, volts); -+ CSTD_UNUSED(core_mask); -+ -+ if (kbdev->nr_clocks == 1) { -+ freqs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = -+ freqs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]; -+ volts[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = -+ volts[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]; -+ } -+} - --#if defined(CONFIG_MALI_PWRSOFT_765) || \ -- LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+#if KERNEL_VERSION(5, 10, 0) > LINUX_VERSION_CODE -+#if defined(CONFIG_MALI_PWRSOFT_765) || \ -+ KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE - static unsigned long kbase_get_dynamic_power(struct devfreq *df, - unsigned long freq, - unsigned long voltage) -@@ -560,10 +563,13 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq, - #endif - { - struct kbase_ipa_model *model; -- u32 power_coeff = 0, power = 0; -+ unsigned long freqs[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; -+ unsigned long volts[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; -+ u32 power_coeffs[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; -+ u32 power = 0; - int err = 0; --#if defined(CONFIG_MALI_PWRSOFT_765) || \ -- LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+#if defined(CONFIG_MALI_PWRSOFT_765) || \ -+ KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE - struct kbase_device *kbdev = dev_get_drvdata(&df->dev); - #else - struct kbase_device *kbdev = kbase_find_device(-1); -@@ -576,34 +582,53 @@ static unsigned long kbase_get_dynamic_power(unsigned long freq, - - model = kbdev->ipa.fallback_model; - -- err = model->ops->get_dynamic_coeff(model, &power_coeff); -- -- if (!err) -- power = kbase_scale_dynamic_power(power_coeff, freq, voltage); -- else -+ err = model->ops->get_dynamic_coeff(model, power_coeffs); -+ -+ if (!err) { -+ opp_translate_freq_voltage(kbdev, freq, voltage, freqs, volts); -+ -+ power = kbase_scale_dynamic_power( -+ power_coeffs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL], -+ freqs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL], -+ volts[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]); -+ -+ /* Here unlike kbase_get_real_power(), shader core frequency is -+ * used for the scaling as simple power model is used to obtain -+ * the value of dynamic coefficient (which is is a fixed value -+ * retrieved from the device tree). -+ */ -+ power += kbase_scale_dynamic_power( -+ power_coeffs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES], -+ freqs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES], -+ volts[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]); -+ } else - dev_err_ratelimited(kbdev->dev, - "Model %s returned error code %d\n", - model->ops->name, err); - - mutex_unlock(&kbdev->ipa.lock); - --#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ -- LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)) -+#if !(defined(CONFIG_MALI_PWRSOFT_765) || \ -+ KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) - kbase_release_device(kbdev); - #endif - - return power; - } -+#endif /* KERNEL_VERSION(5, 10, 0) > LINUX_VERSION_CODE */ - - int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, - unsigned long freq, - unsigned long voltage) - { - struct kbase_ipa_model *model; -- u32 power_coeff = 0; -- int err = 0; -+ unsigned long freqs[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; -+ unsigned long volts[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; -+ u32 power_coeffs[KBASE_IPA_BLOCK_TYPE_NUM] = {0}; - struct kbasep_pm_metrics diff; - u64 total_time; -+ bool skip_utilization_scaling = false; -+ int err = 0; - - lockdep_assert_held(&kbdev->ipa.lock); - -@@ -611,30 +636,62 @@ int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, - - model = get_current_model(kbdev); - -- err = model->ops->get_dynamic_coeff(model, &power_coeff); -+ err = model->ops->get_dynamic_coeff(model, power_coeffs); - - /* If the counter model returns an error (e.g. switching back to - * protected mode and failing to read counters, or a counter sample - * with too few cycles), revert to the fallback model. - */ - if (err && model != kbdev->ipa.fallback_model) { -+ /* No meaningful scaling for GPU utilization can be done if -+ * the sampling interval was too long. This is equivalent to -+ * assuming GPU was busy throughout (similar to what is done -+ * during protected mode). -+ */ -+ if (err == -EOVERFLOW) -+ skip_utilization_scaling = true; -+ - model = kbdev->ipa.fallback_model; -- err = model->ops->get_dynamic_coeff(model, &power_coeff); -+ err = model->ops->get_dynamic_coeff(model, power_coeffs); - } - -- if (err) -+ if (WARN_ON(err)) - return err; - -- *power = kbase_scale_dynamic_power(power_coeff, freq, voltage); -+ opp_translate_freq_voltage(kbdev, freq, voltage, freqs, volts); - -- /* time_busy / total_time cannot be >1, so assigning the 64-bit -- * result of div_u64 to *power cannot overflow. -- */ -- total_time = diff.time_busy + (u64) diff.time_idle; -- *power = div_u64(*power * (u64) diff.time_busy, -- max(total_time, 1ull)); -+ *power = kbase_scale_dynamic_power( -+ power_coeffs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL], -+ freqs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL], -+ volts[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]); -+ -+ if (power_coeffs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]) { -+ unsigned long freq = freqs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]; - -- *power += get_static_power_locked(kbdev, model, voltage); -+ /* As per the HW team, the top-level frequency needs to be used -+ * for the scaling if the counter based model was used as -+ * counter values are normalized with the GPU_ACTIVE counter -+ * value, which increments at the rate of top-level frequency. -+ */ -+ if (model != kbdev->ipa.fallback_model) -+ freq = freqs[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]; -+ -+ *power += kbase_scale_dynamic_power( -+ power_coeffs[KBASE_IPA_BLOCK_TYPE_SHADER_CORES], -+ freq, volts[KBASE_IPA_BLOCK_TYPE_SHADER_CORES]); -+ } -+ -+ if (!skip_utilization_scaling) { -+ /* time_busy / total_time cannot be >1, so assigning the 64-bit -+ * result of div_u64 to *power cannot overflow. -+ */ -+ total_time = diff.time_busy + (u64) diff.time_idle; -+ *power = div_u64(*power * (u64) diff.time_busy, -+ max(total_time, 1ull)); -+ } -+ -+ *power += get_static_power_locked(kbdev, model, -+ volts[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL]); - - return err; - } -@@ -658,18 +715,42 @@ int kbase_get_real_power(struct devfreq *df, u32 *power, - } - KBASE_EXPORT_TEST_API(kbase_get_real_power); - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) --struct devfreq_cooling_ops kbase_ipa_power_model_ops = { --#else - struct devfreq_cooling_power kbase_ipa_power_model_ops = { --#endif --#ifdef CONFIG_MALI_DEVFREQ -+#if KERNEL_VERSION(5, 10, 0) > LINUX_VERSION_CODE - .get_static_power = &kbase_get_static_power, - .get_dynamic_power = &kbase_get_dynamic_power, --#endif --#if defined(CONFIG_MALI_PWRSOFT_765) || \ -- LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0) -+#endif /* KERNEL_VERSION(5, 10, 0) > LINUX_VERSION_CODE */ -+#if defined(CONFIG_MALI_PWRSOFT_765) || \ -+ KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE - .get_real_power = &kbase_get_real_power, - #endif - }; - KBASE_EXPORT_TEST_API(kbase_ipa_power_model_ops); -+ -+void kbase_ipa_reset_data(struct kbase_device *kbdev) -+{ -+ ktime_t now, diff; -+ s64 elapsed_time; -+ -+ mutex_lock(&kbdev->ipa.lock); -+ -+ now = ktime_get(); -+ diff = ktime_sub(now, kbdev->ipa.last_sample_time); -+ elapsed_time = ktime_to_ms(diff); -+ -+ if (elapsed_time > RESET_INTERVAL_MS) { -+ struct kbasep_pm_metrics diff; -+ struct kbase_ipa_model *model; -+ -+ kbase_pm_get_dvfs_metrics( -+ kbdev, &kbdev->ipa.last_metrics, &diff); -+ -+ model = get_current_model(kbdev); -+ if (model != kbdev->ipa.fallback_model) -+ model->ops->reset_counter_data(model); -+ -+ kbdev->ipa.last_sample_time = ktime_get(); -+ } -+ -+ mutex_unlock(&kbdev->ipa.lock); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h -index 92aace9..c668af9 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2016-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2016-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_IPA_H_ -@@ -27,6 +26,20 @@ - - struct devfreq; - -+/** -+ * enum kbase_ipa_block_type - Type of block for which power estimation is done. -+ * -+ * @KBASE_IPA_BLOCK_TYPE_TOP_LEVEL: Top-level block, that covers CSHW, -+ * MEMSYS, Tiler. -+ * @KBASE_IPA_BLOCK_TYPE_SHADER_CORES: All Shader cores. -+ * @KBASE_IPA_BLOCK_TYPE_NUM: Number of blocks. -+ */ -+enum kbase_ipa_block_type { -+ KBASE_IPA_BLOCK_TYPE_TOP_LEVEL, -+ KBASE_IPA_BLOCK_TYPE_SHADER_CORES, -+ KBASE_IPA_BLOCK_TYPE_NUM -+}; -+ - /** - * struct kbase_ipa_model - Object describing a particular IPA model. - * @kbdev: pointer to kbase device -@@ -89,7 +102,8 @@ struct kbase_ipa_model_ops { - int (*init)(struct kbase_ipa_model *model); - /* Called immediately after init(), or when a parameter is changed, so - * that any coefficients derived from model parameters can be -- * recalculated. */ -+ * recalculated -+ */ - int (*recalculate)(struct kbase_ipa_model *model); - void (*term)(struct kbase_ipa_model *model); - /* -@@ -101,7 +115,9 @@ struct kbase_ipa_model_ops { - * is then scaled by the IPA framework according to the current OPP's - * frequency and voltage. - * -- * Return: 0 on success, or an error code. -+ * Return: 0 on success, or an error code. -EOVERFLOW error code will -+ * indicate that sampling interval was too large and no meaningful -+ * scaling for GPU utiliation can be done. - */ - int (*get_dynamic_coeff)(struct kbase_ipa_model *model, u32 *coeffp); - /* -@@ -115,6 +131,18 @@ struct kbase_ipa_model_ops { - * Return: 0 on success, or an error code. - */ - int (*get_static_coeff)(struct kbase_ipa_model *model, u32 *coeffp); -+ -+ /* -+ * reset_counter_data() - Reset the HW counter data used for calculating -+ * dynamic power coefficient -+ * @model: pointer to model -+ * -+ * This method is currently applicable only to the counter based model. -+ * The next call to get_dynamic_coeff() will have to calculate the -+ * dynamic power coefficient based on the HW counter data generated -+ * from this point onwards. -+ */ -+ void (*reset_counter_data)(struct kbase_ipa_model *model); - }; - - /** -@@ -163,6 +191,17 @@ int kbase_ipa_model_recalculate(struct kbase_ipa_model *model); - const struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device *kbdev, - const char *name); - -+/** -+ * kbase_ipa_counter_model_ops_find - Lookup an IPA counter model using its name -+ * @kbdev: pointer to kbase device -+ * @name: name of counter model to lookup -+ * -+ * Return: Pointer to counter model's 'ops' structure, or NULL if the lookup -+ * failed. -+ */ -+const struct kbase_ipa_model_ops *kbase_ipa_counter_model_ops_find( -+ struct kbase_device *kbdev, const char *name); -+ - /** - * kbase_ipa_model_name_from_id - Find the best model for a given GPU ID - * @gpu_id: GPU ID of GPU the model will be used for -@@ -172,6 +211,16 @@ const struct kbase_ipa_model_ops *kbase_ipa_model_ops_find(struct kbase_device * - */ - const char *kbase_ipa_model_name_from_id(u32 gpu_id); - -+/** -+ * kbase_ipa_counter_model_name_from_id - Find the best counter model for a -+ * given GPU ID -+ * @gpu_id: GPU ID of GPU the counter model will be used for -+ * -+ * Return: The name of the appropriate counter-based model, or NULL if the -+ * no counter model exists. -+ */ -+const char *kbase_ipa_counter_model_name_from_id(u32 gpu_id); -+ - /** - * kbase_ipa_init_model - Initilaize the particular IPA model - * @kbdev: pointer to kbase device -@@ -183,7 +232,7 @@ const char *kbase_ipa_model_name_from_id(u32 gpu_id); - * Return: pointer to kbase_ipa_model on success, NULL on error - */ - struct kbase_ipa_model *kbase_ipa_init_model(struct kbase_device *kbdev, -- const struct kbase_ipa_model_ops *ops); -+ const struct kbase_ipa_model_ops *ops); - /** - * kbase_ipa_term_model - Terminate the particular IPA model - * @model: pointer to the IPA model object, already initialized -@@ -202,16 +251,6 @@ void kbase_ipa_term_model(struct kbase_ipa_model *model); - */ - void kbase_ipa_protection_mode_switch_event(struct kbase_device *kbdev); - --extern const struct kbase_ipa_model_ops kbase_g71_ipa_model_ops; --extern const struct kbase_ipa_model_ops kbase_g72_ipa_model_ops; --extern const struct kbase_ipa_model_ops kbase_g76_ipa_model_ops; --extern const struct kbase_ipa_model_ops kbase_g52_ipa_model_ops; --extern const struct kbase_ipa_model_ops kbase_g52_r1_ipa_model_ops; --extern const struct kbase_ipa_model_ops kbase_g51_ipa_model_ops; --extern const struct kbase_ipa_model_ops kbase_g77_ipa_model_ops; --extern const struct kbase_ipa_model_ops kbase_tnax_ipa_model_ops; --extern const struct kbase_ipa_model_ops kbase_tbex_ipa_model_ops; -- - /** - * kbase_get_real_power() - get the real power consumption of the GPU - * @df: dynamic voltage and frequency scaling information for the GPU. -@@ -237,11 +276,20 @@ int kbase_get_real_power_locked(struct kbase_device *kbdev, u32 *power, - unsigned long voltage); - #endif /* MALI_UNIT_TEST */ - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) --extern struct devfreq_cooling_ops kbase_ipa_power_model_ops; --#else - extern struct devfreq_cooling_power kbase_ipa_power_model_ops; --#endif -+ -+/** -+ * kbase_ipa_reset_data() - Reset the data required for power estimation. -+ * @kbdev: Pointer to kbase device. -+ * -+ * This function is called to ensure a meaningful baseline for -+ * kbase_get_real_power(), when thermal governor starts the polling, and -+ * that is achieved by updating the GPU utilization metrics and retrieving -+ * the accumulated value of HW counters. -+ * Basically this function collects all the data required for power estimation -+ * but does not process it. -+ */ -+void kbase_ipa_reset_data(struct kbase_device *kbdev); - - #else /* !(defined(CONFIG_MALI_DEVFREQ) && defined(CONFIG_DEVFREQ_THERMAL)) */ - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c -index 30a3b7d..5976389 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2017-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/debugfs.h> -@@ -28,7 +27,7 @@ - #include "mali_kbase_ipa.h" - #include "mali_kbase_ipa_debugfs.h" - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0)) -+#if (KERNEL_VERSION(4, 7, 0) > LINUX_VERSION_CODE) - #define DEFINE_DEBUGFS_ATTRIBUTE DEFINE_SIMPLE_ATTRIBUTE - #endif - -@@ -160,7 +159,8 @@ int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, - return -ENOMEM; - - /* 'name' is stack-allocated for array elements, so copy it into -- * heap-allocated storage */ -+ * heap-allocated storage -+ */ - param->name = kstrdup(name, GFP_KERNEL); - - if (!param->name) { -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h -index a983d9c..f690367 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_debugfs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_IPA_DEBUGFS_H_ -@@ -28,7 +27,7 @@ enum kbase_ipa_model_param_type { - PARAM_TYPE_STRING, - }; - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - - void kbase_ipa_debugfs_init(struct kbase_device *kbdev); - int kbase_ipa_model_param_add(struct kbase_ipa_model *model, const char *name, -@@ -63,6 +62,9 @@ static inline int kbase_ipa_model_param_add(struct kbase_ipa_model *model, - static inline void kbase_ipa_model_param_free_all(struct kbase_ipa_model *model) - { } - -+static inline void kbase_ipa_model_param_set_s32(struct kbase_ipa_model *model, -+ const char *name, s32 val) -+{ } - #endif /* CONFIG_DEBUG_FS */ - - #endif /* _KBASE_IPA_DEBUGFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c -index 852559e..55f1d1c 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2016-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2016-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,13 +17,11 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <uapi/linux/thermal.h> - #include <linux/thermal.h> --#ifdef CONFIG_DEVFREQ_THERMAL -+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL) - #include <linux/devfreq_cooling.h> - #endif - #include <linux/of.h> -@@ -34,20 +33,18 @@ - #include "mali_kbase_ipa_simple.h" - #include "mali_kbase_ipa_debugfs.h" - --#if MALI_UNIT_TEST -- --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) --static unsigned long dummy_temp; -+/* This is used if the dynamic power for top-level is estimated separately -+ * through the counter model. To roughly match the contribution of top-level -+ * power in the total dynamic power, when calculated through counter model, -+ * this scalar is used for the dynamic coefficient specified in the device tree -+ * for simple power model. This value was provided by the HW team after -+ * taking all the power data collected and dividing top level power by shader -+ * core power and then averaging it across all samples. -+ */ -+#define TOP_LEVEL_DYN_COEFF_SCALER (3) - --static int kbase_simple_power_model_get_dummy_temp( -- struct thermal_zone_device *tz, -- unsigned long *temp) --{ -- *temp = READ_ONCE(dummy_temp); -- return 0; --} -+#if MALI_UNIT_TEST - --#else - static int dummy_temp; - - static int kbase_simple_power_model_get_dummy_temp( -@@ -57,7 +54,6 @@ static int kbase_simple_power_model_get_dummy_temp( - *temp = READ_ONCE(dummy_temp); - return 0; - } --#endif - - /* Intercept calls to the kernel function using a macro */ - #ifdef thermal_zone_get_temp -@@ -143,16 +139,13 @@ static u32 calculate_temp_scaling_factor(s32 ts[4], s64 t) - - /* We can't call thermal_zone_get_temp() directly in model_static_coeff(), - * because we don't know if tz->lock is held in the same thread. So poll it in -- * a separate thread to get around this. */ -+ * a separate thread to get around this. -+ */ - static int poll_temperature(void *data) - { - struct kbase_ipa_model_simple_data *model_data = - (struct kbase_ipa_model_simple_data *) data; --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0) -- unsigned long temp; --#else - int temp; --#endif - - while (!kthread_should_stop()) { - struct thermal_zone_device *tz = READ_ONCE(model_data->gpu_tz); -@@ -208,7 +201,21 @@ static int model_dynamic_coeff(struct kbase_ipa_model *model, u32 *coeffp) - struct kbase_ipa_model_simple_data *model_data = - (struct kbase_ipa_model_simple_data *) model->model_data; - -+#if MALI_USE_CSF -+ /* On CSF GPUs, the dynamic power for top-level and shader cores is -+ * estimated separately. Currently there is a single dynamic -+ * coefficient value provided in the device tree for simple model. -+ * As per the discussion with HW team the coefficient value needs to -+ * be scaled down for top-level to limit its contribution in the -+ * total dyanmic power. -+ */ -+ coeffp[KBASE_IPA_BLOCK_TYPE_TOP_LEVEL] = -+ model_data->dynamic_coefficient / TOP_LEVEL_DYN_COEFF_SCALER; -+ coeffp[KBASE_IPA_BLOCK_TYPE_SHADER_CORES] = -+ model_data->dynamic_coefficient; -+#else - *coeffp = model_data->dynamic_coefficient; -+#endif - - return 0; - } -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.h b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.h -index fed67d5..fb174e2 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/ipa/mali_kbase_ipa_simple.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_IPA_SIMPLE_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_defs.h -index aac561b..c490f1c 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_defs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Definitions (types, defines, etcs) specific to Job Manager Kbase. - * They are placed here to allow the hierarchy of header files to work. -@@ -129,7 +126,7 @@ - /* Reset the GPU after each atom completion */ - #define KBASE_SERIALIZE_RESET (1 << 2) - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - /** - * struct base_job_fault_event - keeps track of the atom which faulted or which - * completed after the faulty atom but before the -@@ -409,6 +406,16 @@ struct kbase_ext_res { - * sync through soft jobs and for the implicit - * synchronization required on access to external - * resources. -+ * @dma_fence.fence_in: Input fence -+ * @dma_fence.fence: Points to the dma-buf output fence for this atom. -+ * @dma_fence.context: The dma-buf fence context number for this atom. A -+ * unique context number is allocated to each katom in -+ * the context on context creation. -+ * @dma_fence.seqno: The dma-buf fence sequence number for this atom. This -+ * is increased every time this katom uses dma-buf fence -+ * @dma_fence.callbacks: List of all callbacks set up to wait on other fences -+ * @dma_fence.dep_count: Atomic counter of number of outstandind dma-buf fence -+ * dependencies for this atom. - * @event_code: Event code for the job chain represented by the atom, - * both HW and low-level SW events are represented by - * event codes. -@@ -443,6 +450,8 @@ struct kbase_ext_res { - * @blocked: flag indicating that atom's resubmission to GPU is - * blocked till the work item is scheduled to return the - * atom to JS. -+ * @seq_nr: user-space sequence number, to order atoms in some -+ * temporal order - * @pre_dep: Pointer to atom that this atom has same-slot - * dependency on - * @post_dep: Pointer to atom that has same-slot dependency on -@@ -477,11 +486,19 @@ struct kbase_ext_res { - * when transitioning into or out of protected mode. - * Atom will be either entering or exiting the - * protected mode. -+ * @protected_state.enter: entering the protected mode. -+ * @protected_state.exit: exiting the protected mode. - * @runnable_tree_node: The node added to context's job slot specific rb tree - * when the atom becomes runnable. - * @age: Age of atom relative to other atoms in the context, - * is snapshot of the age_count counter in kbase - * context. -+ * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. -+ * @renderpass_id:Renderpass identifier used to associate an atom that has -+ * BASE_JD_REQ_START_RENDERPASS set in its core requirements -+ * with an atom that has BASE_JD_REQ_END_RENDERPASS set. -+ * @jc_fragment: Set of GPU fragment job chains -+ * @retry_count: TODO: Not used,to be removed - */ - struct kbase_jd_atom { - struct work_struct work; -@@ -496,9 +513,9 @@ struct kbase_jd_atom { - struct list_head jd_item; - bool in_jd_list; - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - u8 jit_ids[2]; --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - u16 nr_extres; - struct kbase_ext_res *extres; -@@ -516,7 +533,6 @@ struct kbase_jd_atom { - * when working with this sub struct - */ - #if defined(CONFIG_SYNC_FILE) -- /* Input fence */ - #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence_in; - #else -@@ -539,14 +555,7 @@ struct kbase_jd_atom { - #else - struct dma_fence *fence; - #endif -- /* The dma-buf fence context number for this atom. A unique -- * context number is allocated to each katom in the context on -- * context creation. -- */ - unsigned int context; -- /* The dma-buf fence sequence number for this atom. This is -- * increased every time this katom uses dma-buf fence. -- */ - atomic_t seqno; - /* This contains a list of all callbacks set up to wait on - * other fences. This atom must be held back from JS until all -@@ -593,7 +602,7 @@ struct kbase_jd_atom { - - wait_queue_head_t completed; - enum kbase_jd_atom_state status; --#ifdef CONFIG_GPU_TRACEPOINTS -+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) - int work_id; - #endif - int slot_nr; -@@ -608,6 +617,8 @@ struct kbase_jd_atom { - - atomic_t blocked; - -+ u64 seq_nr; -+ - struct kbase_jd_atom *pre_dep; - struct kbase_jd_atom *post_dep; - -@@ -616,7 +627,7 @@ struct kbase_jd_atom { - - u32 flush_id; - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - struct base_job_fault_event fault_event; - #endif - struct list_head queue; -@@ -778,6 +789,7 @@ struct kbase_jd_renderpass { - * @jit_pending_alloc: A list of just-in-time memory allocation - * soft-jobs which will be reattempted after the - * impending free of other active allocations. -+ * @max_priority: Max priority level allowed for this context. - */ - struct kbase_jd_context { - struct mutex lock; -@@ -792,12 +804,13 @@ struct kbase_jd_context { - u32 job_nr; - size_t tb_wrap_offset; - --#ifdef CONFIG_GPU_TRACEPOINTS -+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) - atomic_t work_id; - #endif - - struct list_head jit_atoms_head; - struct list_head jit_pending_alloc; -+ int max_priority; - }; - - /** -@@ -815,4 +828,27 @@ struct jsctx_queue { - struct list_head x_dep_head; - }; - -+/** -+ * struct kbase_as - Object representing an address space of GPU. -+ * @number: Index at which this address space structure is present -+ * in an array of address space structures embedded inside -+ * the &struct kbase_device. -+ * @pf_wq: Workqueue for processing work items related to -+ * Page fault and Bus fault handling. -+ * @work_pagefault: Work item for the Page fault handling. -+ * @work_busfault: Work item for the Bus fault handling. -+ * @pf_data: Data relating to Page fault. -+ * @bf_data: Data relating to Bus fault. -+ * @current_setup: Stores the MMU configuration for this address space. -+ */ -+struct kbase_as { -+ int number; -+ struct workqueue_struct *pf_wq; -+ struct work_struct work_pagefault; -+ struct work_struct work_busfault; -+ struct kbase_fault pf_data; -+ struct kbase_fault bf_data; -+ struct kbase_mmu_setup current_setup; -+}; -+ - #endif /* _KBASE_JM_DEFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_js.h b/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_js.h -index 6c222ce..5e0c4bc 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_js.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_js.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -32,6 +31,7 @@ - - /** - * kbasep_js_devdata_init - Initialize the Job Scheduler -+ * @kbdev: The kbase_device to operate on - * - * The struct kbasep_js_device_data sub-structure of kbdev must be zero - * initialized before passing to the kbasep_js_devdata_init() function. This is -@@ -41,6 +41,7 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev); - - /** - * kbasep_js_devdata_halt - Halt the Job Scheduler. -+ * @kbdev: The kbase_device to operate on - * - * It is safe to call this on kbdev even if it the kbasep_js_device_data - * sub-structure was never initialized/failed initialization, to give efficient -@@ -58,6 +59,7 @@ void kbasep_js_devdata_halt(struct kbase_device *kbdev); - - /** - * kbasep_js_devdata_term - Terminate the Job Scheduler -+ * @kbdev: The kbase_device to operate on - * - * It is safe to call this on kbdev even if it the kbasep_js_device_data - * sub-structure was never initialized/failed initialization, to give efficient -@@ -75,6 +77,7 @@ void kbasep_js_devdata_term(struct kbase_device *kbdev); - /** - * kbasep_js_kctx_init - Initialize the Scheduling Component of a - * struct kbase_context on the Job Scheduler. -+ * @kctx: The kbase_context to operate on - * - * This effectively registers a struct kbase_context with a Job Scheduler. - * -@@ -89,6 +92,7 @@ int kbasep_js_kctx_init(struct kbase_context *const kctx); - /** - * kbasep_js_kctx_term - Terminate the Scheduling Component of a - * struct kbase_context on the Job Scheduler -+ * @kctx: The kbase_context to operate on - * - * This effectively de-registers a struct kbase_context from its Job Scheduler - * -@@ -108,6 +112,8 @@ void kbasep_js_kctx_term(struct kbase_context *kctx); - * kbasep_js_add_job - Add a job chain to the Job Scheduler, - * and take necessary actions to - * schedule the context/run the job. -+ * @kctx: The kbase_context to operate on -+ * @atom: Atom to add - * - * This atomically does the following: - * * Update the numbers of jobs information -@@ -151,7 +157,10 @@ bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom); - /** - * kbasep_js_remove_job - Remove a job chain from the Job Scheduler, - * except for its 'retained state'. -- * -+ * @kbdev: The kbase_device to operate on -+ * @kctx: The kbase_context to operate on -+ * @atom: Atom to remove -+* - * Completely removing a job requires several calls: - * * kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of - * the atom -@@ -185,6 +194,9 @@ void kbasep_js_remove_job(struct kbase_device *kbdev, - * kbasep_js_remove_cancelled_job - Completely remove a job chain from the - * Job Scheduler, in the case - * where the job chain was cancelled. -+ * @kbdev: The kbase_device to operate on -+ * @kctx: The kbase_context to operate on -+ * @katom: Atom to remove - * - * This is a variant of kbasep_js_remove_job() that takes care of removing all - * of the retained state too. This is generally useful for cancelled atoms, -@@ -215,6 +227,9 @@ bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, - * kbasep_js_runpool_requeue_or_kill_ctx - Handling the requeuing/killing of a - * context that was evicted from the - * policy queue or runpool. -+ * @kbdev: The kbase_device to operate on -+ * @kctx: The kbase_context to operate on -+ * @has_pm_ref: tells whether to release Power Manager active reference - * - * This should be used whenever handing off a context that has been evicted - * from the policy queue or the runpool: -@@ -242,6 +257,8 @@ void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, - /** - * kbasep_js_runpool_release_ctx - Release a refcount of a context being busy, - * allowing it to be scheduled out. -+ * @kbdev: The kbase_device to operate on -+ * @kctx: The kbase_context to operate on - * - * When the refcount reaches zero and the context might be scheduled out - * (depending on whether the Scheduling Policy has deemed it so, or if it has -@@ -296,6 +313,9 @@ void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, - * kbasep_js_runpool_release_ctx_and_katom_retained_state - Variant of - * kbasep_js_runpool_release_ctx() that handles additional - * actions from completing an atom. -+ * @kbdev: KBase device -+ * @kctx: KBase context -+ * @katom_retained_state: Retained state from the atom - * - * This is usually called as part of completing an atom and releasing the - * refcount on the context held by the atom. -@@ -315,8 +335,12 @@ void kbasep_js_runpool_release_ctx_and_katom_retained_state( - struct kbasep_js_atom_retained_state *katom_retained_state); - - /** -- * kbasep_js_runpool_release_ctx_nolock - Variant of -- * kbase_js_runpool_release_ctx() that assumes that -+ * kbasep_js_runpool_release_ctx_nolock - -+ * Variant of kbase_js_runpool_release_ctx() w/out locks -+ * @kbdev: KBase device -+ * @kctx: KBase context -+ * -+ * Variant of kbase_js_runpool_release_ctx() that assumes that - * kbasep_js_device_data::runpool_mutex and - * kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not - * attempt to schedule new contexts. -@@ -326,6 +350,8 @@ void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev, - - /** - * kbasep_js_schedule_privileged_ctx - Schedule in a privileged context -+ * @kbdev: KBase device -+ * @kctx: KBase context - * - * This schedules a context in regardless of the context priority. - * If the runpool is full, a context will be forced out of the runpool and the -@@ -351,6 +377,8 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, - /** - * kbasep_js_release_privileged_ctx - Release a privileged context, - * allowing it to be scheduled out. -+ * @kbdev: KBase device -+ * @kctx: KBase context - * - * See kbasep_js_runpool_release_ctx for potential side effects. - * -@@ -368,6 +396,7 @@ void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev, - - /** - * kbase_js_try_run_jobs - Try to submit the next job on each slot -+ * @kbdev: KBase device - * - * The following locks may be used: - * * kbasep_js_device_data::runpool_mutex -@@ -378,6 +407,7 @@ void kbase_js_try_run_jobs(struct kbase_device *kbdev); - /** - * kbasep_js_suspend - Suspend the job scheduler during a Power Management - * Suspend event. -+ * @kbdev: KBase device - * - * Causes all contexts to be removed from the runpool, and prevents any - * contexts from (re)entering the runpool. -@@ -401,6 +431,7 @@ void kbasep_js_suspend(struct kbase_device *kbdev); - /** - * kbasep_js_resume - Resume the Job Scheduler after a Power Management - * Resume event. -+ * @kbdev: KBase device - * - * This restores the actions from kbasep_js_suspend(): - * * Schedules contexts back into the runpool -@@ -412,7 +443,7 @@ void kbasep_js_resume(struct kbase_device *kbdev); - * kbase_js_dep_resolved_submit - Submit an atom to the job scheduler. - * - * @kctx: Context pointer -- * @atom: Pointer to the atom to submit -+ * @katom: Pointer to the atom to submit - * - * The atom is enqueued on the context's ringbuffer. The caller must have - * ensured that all dependencies can be represented in the ringbuffer. -@@ -457,7 +488,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js); - * kbase_js_unpull - Return an atom to the job scheduler ringbuffer. - * - * @kctx: Context pointer -- * @atom: Pointer to the atom to unpull -+ * @katom: Pointer to the atom to unpull - * - * An atom is 'unpulled' if execution is stopped but intended to be returned to - * later. The most common reason for this is that the atom has been -@@ -584,7 +615,6 @@ void kbase_js_set_timeouts(struct kbase_device *kbdev); - */ - void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority); - -- - /** - * kbase_js_update_ctx_priority - update the context priority - * -@@ -603,6 +633,8 @@ void kbase_js_update_ctx_priority(struct kbase_context *kctx); - /** - * kbasep_js_is_submit_allowed - Check that a context is allowed to submit - * jobs on this policy -+ * @js_devdata: KBase Job Scheduler Device Data -+ * @kctx: KBase context - * - * The purpose of this abstraction is to hide the underlying data size, - * and wrap up the long repeated line of code. -@@ -625,13 +657,15 @@ static inline bool kbasep_js_is_submit_allowed( - test_bit = (u16) (1u << kctx->as_nr); - - is_allowed = (bool) (js_devdata->runpool_irq.submit_allowed & test_bit); -- dev_dbg(kctx->kbdev->dev, "JS: submit %s allowed on %p (as=%d)", -+ dev_dbg(kctx->kbdev->dev, "JS: submit %s allowed on %pK (as=%d)", - is_allowed ? "is" : "isn't", (void *)kctx, kctx->as_nr); - return is_allowed; - } - - /** - * kbasep_js_set_submit_allowed - Allow a context to submit jobs on this policy -+ * @js_devdata: KBase Job Scheduler Device Data -+ * @kctx: KBase context - * - * The purpose of this abstraction is to hide the underlying data size, - * and wrap up the long repeated line of code. -@@ -650,7 +684,7 @@ static inline void kbasep_js_set_submit_allowed( - - set_bit = (u16) (1u << kctx->as_nr); - -- dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", -+ dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %pK (as=%d)", - kctx, kctx->as_nr); - - js_devdata->runpool_irq.submit_allowed |= set_bit; -@@ -659,6 +693,8 @@ static inline void kbasep_js_set_submit_allowed( - /** - * kbasep_js_clear_submit_allowed - Prevent a context from submitting more - * jobs on this policy -+ * @js_devdata: KBase Job Scheduler Device Data -+ * @kctx: KBase context - * - * The purpose of this abstraction is to hide the underlying data size, - * and wrap up the long repeated line of code. -@@ -679,13 +715,17 @@ static inline void kbasep_js_clear_submit_allowed( - clear_bit = (u16) (1u << kctx->as_nr); - clear_mask = ~clear_bit; - -- dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", -+ dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %pK (as=%d)", - kctx, kctx->as_nr); - - js_devdata->runpool_irq.submit_allowed &= clear_mask; - } - - /** -+ * kbasep_js_atom_retained_state_init_invalid - -+ * Create an initial 'invalid' atom retained state -+ * @retained_state: pointer where to create and initialize the state -+ * - * Create an initial 'invalid' atom retained state, that requires no - * atom-related work to be done on releasing with - * kbasep_js_runpool_release_ctx_and_katom_retained_state() -@@ -699,6 +739,10 @@ static inline void kbasep_js_atom_retained_state_init_invalid( - } - - /** -+ * kbasep_js_atom_retained_state_copy() - Copy atom state -+ * @retained_state: where to copy -+ * @katom: where to copy from -+ * - * Copy atom state that can be made available after jd_done_nolock() is called - * on that atom. - */ -@@ -743,7 +787,7 @@ static inline bool kbasep_js_has_atom_finished( - * kbasep_js_atom_retained_state_is_valid - Determine whether a struct - * kbasep_js_atom_retained_state - * is valid -- * @katom_retained_state the atom's retained state to check -+ * @katom_retained_state: the atom's retained state to check - * - * An invalid struct kbasep_js_atom_retained_state is allowed, and indicates - * that the code should just ignore it. -@@ -759,6 +803,8 @@ static inline bool kbasep_js_atom_retained_state_is_valid( - - /** - * kbase_js_runpool_inc_context_count - Increment number of running contexts. -+ * @kbdev: KBase device -+ * @kctx: KBase context - * - * The following locking conditions are made on the caller: - * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. -@@ -795,6 +841,8 @@ static inline void kbase_js_runpool_inc_context_count( - /** - * kbase_js_runpool_dec_context_count - decrement number of running contexts. - * -+ * @kbdev: KBase device -+ * @kctx: KBase context - * The following locking conditions are made on the caller: - * * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex. - * * The caller must hold the kbasep_js_device_data::runpool_mutex -@@ -889,4 +937,17 @@ static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio) - return kbasep_js_relative_priority_to_atom[prio_idx]; - } - -+/** -+ * kbase_js_priority_check - Check the priority requested -+ * -+ * @kbdev: Device pointer -+ * @priority: Requested priority -+ * -+ * This will determine whether the requested priority can be satisfied. -+ * -+ * Return: The same or lower priority than requested. -+ */ -+ -+base_jd_prio kbase_js_priority_check(struct kbase_device *kbdev, base_jd_prio priority); -+ - #endif /* _KBASE_JM_JS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_js_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_js_defs.h -index 0b48615..75152fb 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_js_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_js_defs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,33 +17,15 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /** -- * @file mali_kbase_js.h -- * Job Scheduler Type Definitions -+ * DOC: Job Scheduler Type Definitions - */ - - #ifndef _KBASE_JS_DEFS_H_ - #define _KBASE_JS_DEFS_H_ - --/** -- * @addtogroup base_api -- * @{ -- */ -- --/** -- * @addtogroup base_kbase_api -- * @{ -- */ -- --/** -- * @addtogroup kbase_js -- * @{ -- */ - /* Forward decls */ - struct kbase_device; - struct kbase_jd_atom; -@@ -50,11 +33,14 @@ struct kbase_jd_atom; - - typedef u32 kbase_context_flags; - --/** Callback function run on all of a context's jobs registered with the Job -- * Scheduler */ --typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom); -+/* -+ * typedef kbasep_js_ctx_job_cb - Callback function run on all of a context's -+ * jobs registered with the Job Scheduler -+ */ -+typedef void kbasep_js_ctx_job_cb(struct kbase_device *kbdev, -+ struct kbase_jd_atom *katom); - --/** -+/* - * @brief Maximum number of jobs that can be submitted to a job slot whilst - * inside the IRQ handler. - * -@@ -65,7 +51,15 @@ typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd - #define KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ 2 - - /** -- * @brief Context attributes -+ * enum kbasep_js_ctx_attr - Context attributes -+ * @KBASEP_JS_CTX_ATTR_COMPUTE: Attribute indicating a context that contains -+ * Compute jobs. -+ * @KBASEP_JS_CTX_ATTR_NON_COMPUTE: Attribute indicating a context that contains -+ * Non-Compute jobs. -+ * @KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES: Attribute indicating that a context -+ * contains compute-job atoms that aren't restricted to a coherent group, -+ * and can run on all cores. -+ * @KBASEP_JS_CTX_ATTR_COUNT: Must be the last in the enum - * - * Each context attribute can be thought of as a boolean value that caches some - * state information about either the runpool, or the context: -@@ -82,61 +76,70 @@ typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd - * - The runpool holds a refcount of how many contexts in the runpool have this - * attribute. - * - The context holds a refcount of how many atoms have this attribute. -+ * -+ * KBASEP_JS_CTX_ATTR_COMPUTE: -+ * Attribute indicating a context that contains Compute jobs. That is, -+ * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE -+ * -+ * @note A context can be both 'Compute' and 'Non Compute' if it contains -+ * both types of jobs. -+ * -+ * KBASEP_JS_CTX_ATTR_NON_COMPUTE: -+ * Attribute indicating a context that contains Non-Compute jobs. That is, -+ * the context has some jobs that are \b not of type @ref -+ * BASE_JD_REQ_ONLY_COMPUTE. -+ * -+ * @note A context can be both 'Compute' and 'Non Compute' if it contains -+ * both types of jobs. -+ * -+ * KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES: -+ * Attribute indicating that a context contains compute-job atoms that -+ * aren't restricted to a coherent group, and can run on all cores. -+ * -+ * Specifically, this is when the atom's \a core_req satisfy: -+ * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 -+ * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups -+ * -+ * Such atoms could be blocked from running if one of the coherent groups -+ * is being used by another job slot, so tracking this context attribute -+ * allows us to prevent such situations. -+ * -+ * @note This doesn't take into account the 1-coregroup case, where all -+ * compute atoms would effectively be able to run on 'all cores', but -+ * contexts will still not always get marked with this attribute. Instead, -+ * it is the caller's responsibility to take into account the number of -+ * coregroups when interpreting this attribute. -+ * -+ * @note Whilst Tiler atoms are normally combined with -+ * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without -+ * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy -+ * enough to handle anyway. -+ * -+ * - */ - enum kbasep_js_ctx_attr { -- /** Attribute indicating a context that contains Compute jobs. That is, -- * the context has jobs of type @ref BASE_JD_REQ_ONLY_COMPUTE -- * -- * @note A context can be both 'Compute' and 'Non Compute' if it contains -- * both types of jobs. -- */ - KBASEP_JS_CTX_ATTR_COMPUTE, -- -- /** Attribute indicating a context that contains Non-Compute jobs. That is, -- * the context has some jobs that are \b not of type @ref -- * BASE_JD_REQ_ONLY_COMPUTE. -- * -- * @note A context can be both 'Compute' and 'Non Compute' if it contains -- * both types of jobs. -- */ - KBASEP_JS_CTX_ATTR_NON_COMPUTE, -- -- /** Attribute indicating that a context contains compute-job atoms that -- * aren't restricted to a coherent group, and can run on all cores. -- * -- * Specifically, this is when the atom's \a core_req satisfy: -- * - (\a core_req & (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE | BASE_JD_REQ_T) // uses slot 1 or slot 2 -- * - && !(\a core_req & BASE_JD_REQ_COHERENT_GROUP) // not restricted to coherent groups -- * -- * Such atoms could be blocked from running if one of the coherent groups -- * is being used by another job slot, so tracking this context attribute -- * allows us to prevent such situations. -- * -- * @note This doesn't take into account the 1-coregroup case, where all -- * compute atoms would effectively be able to run on 'all cores', but -- * contexts will still not always get marked with this attribute. Instead, -- * it is the caller's responsibility to take into account the number of -- * coregroups when interpreting this attribute. -- * -- * @note Whilst Tiler atoms are normally combined with -- * BASE_JD_REQ_COHERENT_GROUP, it is possible to send such atoms without -- * BASE_JD_REQ_COHERENT_GROUP set. This is an unlikely case, but it's easy -- * enough to handle anyway. -- */ - KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES, -- -- /** Must be the last in the enum */ - KBASEP_JS_CTX_ATTR_COUNT - }; - - enum { -- /** Bit indicating that new atom should be started because this atom completed */ -+ /* -+ * Bit indicating that new atom should be started because this atom -+ * completed -+ */ - KBASE_JS_ATOM_DONE_START_NEW_ATOMS = (1u << 0), -- /** Bit indicating that the atom was evicted from the JS_NEXT registers */ -+ /* -+ * Bit indicating that the atom was evicted from the JS_NEXT registers -+ */ - KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT = (1u << 1) - }; - --/** Combination of KBASE_JS_ATOM_DONE_<...> bits */ -+/** -+ * typedef kbasep_js_atom_done_code - Combination of KBASE_JS_ATOM_DONE_<...> -+ * bits -+ */ - typedef u32 kbasep_js_atom_done_code; - - /* -@@ -168,7 +171,9 @@ enum { - * Internal atom priority defines for kbase_jd_atom::sched_prio - */ - enum { -- KBASE_JS_ATOM_SCHED_PRIO_HIGH = 0, -+ KBASE_JS_ATOM_SCHED_PRIO_FIRST = 0, -+ KBASE_JS_ATOM_SCHED_PRIO_REALTIME = KBASE_JS_ATOM_SCHED_PRIO_FIRST, -+ KBASE_JS_ATOM_SCHED_PRIO_HIGH, - KBASE_JS_ATOM_SCHED_PRIO_MED, - KBASE_JS_ATOM_SCHED_PRIO_LOW, - KBASE_JS_ATOM_SCHED_PRIO_COUNT, -@@ -183,7 +188,70 @@ enum { - #define KBASE_JS_ATOM_SCHED_PRIO_DEFAULT KBASE_JS_ATOM_SCHED_PRIO_MED - - /** -- * @brief KBase Device Data Job Scheduler sub-structure -+ * struct kbasep_js_device_data - KBase Device Data Job Scheduler sub-structure -+ * @runpool_irq: Sub-structure to collect together Job Scheduling data used in -+ * IRQ context. The hwaccess_lock must be held when accessing. -+ * @runpool_irq.submit_allowed: Bitvector indicating whether a currently -+ * scheduled context is allowed to submit jobs. When bit 'N' is set in -+ * this, it indicates whether the context bound to address space 'N' is -+ * allowed to submit jobs. -+ * @runpool_irq.ctx_attr_ref_count: Array of Context Attributes Ref_counters: -+ * Each is large enough to hold a refcount of the number of contexts -+ * that can fit into the runpool. This is currently BASE_MAX_NR_AS. -+ * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store -+ * the refcount. Hence, it's not worthwhile reducing this to -+ * bit-manipulation on u32s to save space (where in contrast, 4 bit -+ * sub-fields would be easy to do and would save space). -+ * Whilst this must not become negative, the sign bit is used for: -+ * - error detection in debug builds -+ * - Optimization: it is undefined for a signed int to overflow, and so -+ * the compiler can optimize for that never happening (thus, no masking -+ * is required on updating the variable) -+ * @runpool_irq.slot_affinities: Affinity management and tracking. Bitvector -+ * to aid affinity checking. Element 'n' bit 'i' indicates that slot 'n' -+ * is using core i (i.e. slot_affinity_refcount[n][i] > 0) -+ * @runpool_irq.slot_affinity_refcount: Array of fefcount for each core owned -+ * by each slot. Used to generate the slot_affinities array of bitvectors. -+ * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, -+ * because it is refcounted only when a job is definitely about to be -+ * submitted to a slot, and is de-refcounted immediately after a job -+ * finishes -+ * @schedule_sem: Scheduling semaphore. This must be held when calling -+ * kbase_jm_kick() -+ * @ctx_list_pullable: List of contexts that can currently be pulled from -+ * @ctx_list_unpullable: List of contexts that can not currently be pulled -+ * from, but have jobs currently running. -+ * @nr_user_contexts_running: Number of currently scheduled user contexts -+ * (excluding ones that are not submitting jobs) -+ * @nr_all_contexts_running: Number of currently scheduled contexts (including -+ * ones that are not submitting jobs) -+ * @js_reqs: Core Requirements to match up with base_js_atom's core_req memeber -+ * @note This is a write-once member, and so no locking is required to -+ * read -+ * @scheduling_period_ns: Value for JS_SCHEDULING_PERIOD_NS -+ * @soft_stop_ticks: Value for JS_SOFT_STOP_TICKS -+ * @soft_stop_ticks_cl: Value for JS_SOFT_STOP_TICKS_CL -+ * @hard_stop_ticks_ss: Value for JS_HARD_STOP_TICKS_SS -+ * @hard_stop_ticks_cl: Value for JS_HARD_STOP_TICKS_CL -+ * @hard_stop_ticks_dumping: Value for JS_HARD_STOP_TICKS_DUMPING -+ * @gpu_reset_ticks_ss: Value for JS_RESET_TICKS_SS -+ * @gpu_reset_ticks_cl: Value for JS_RESET_TICKS_CL -+ * @gpu_reset_ticks_dumping: Value for JS_RESET_TICKS_DUMPING -+ * @ctx_timeslice_ns: Value for JS_CTX_TIMESLICE_NS -+ * @suspended_soft_jobs_list: List of suspended soft jobs -+ * @softstop_always: Support soft-stop on a single context -+ * @init_status:The initialized-flag is placed at the end, to avoid -+ * cache-pollution (we should only be using this during init/term paths). -+ * @note This is a write-once member, and so no locking is required to -+ * read -+ * @nr_contexts_pullable:Number of contexts that can currently be pulled from -+ * @nr_contexts_runnable:Number of contexts that can either be pulled from or -+ * arecurrently running -+ * @soft_job_timeout_ms:Value for JS_SOFT_JOB_TIMEOUT -+ * @queue_mutex: Queue Lock, used to access the Policy's queue of contexts -+ * independently of the Run Pool. -+ * Of course, you don't need the Run Pool lock to access this. -+ * @runpool_mutex: Run Pool mutex, for managing contexts within the runpool. - * - * This encapsulates the current context of the Job Scheduler on a particular - * device. This context is global to the device, and is not tied to any -@@ -191,121 +259,49 @@ enum { - * - * nr_contexts_running and as_free are optimized for packing together (by making - * them smaller types than u32). The operations on them should rarely involve -- * masking. The use of signed types for arithmetic indicates to the compiler that -- * the value will not rollover (which would be undefined behavior), and so under -- * the Total License model, it is free to make optimizations based on that (i.e. -- * to remove masking). -+ * masking. The use of signed types for arithmetic indicates to the compiler -+ * that the value will not rollover (which would be undefined behavior), and so -+ * under the Total License model, it is free to make optimizations based on -+ * that (i.e. to remove masking). - */ - struct kbasep_js_device_data { -- /* Sub-structure to collect together Job Scheduling data used in IRQ -- * context. The hwaccess_lock must be held when accessing. */ - struct runpool_irq { -- /** Bitvector indicating whether a currently scheduled context is allowed to submit jobs. -- * When bit 'N' is set in this, it indicates whether the context bound to address space -- * 'N' is allowed to submit jobs. -- */ - u16 submit_allowed; -- -- /** Context Attributes: -- * Each is large enough to hold a refcount of the number of contexts -- * that can fit into the runpool. This is currently BASE_MAX_NR_AS -- * -- * Note that when BASE_MAX_NR_AS==16 we need 5 bits (not 4) to store -- * the refcount. Hence, it's not worthwhile reducing this to -- * bit-manipulation on u32s to save space (where in contrast, 4 bit -- * sub-fields would be easy to do and would save space). -- * -- * Whilst this must not become negative, the sign bit is used for: -- * - error detection in debug builds -- * - Optimization: it is undefined for a signed int to overflow, and so -- * the compiler can optimize for that never happening (thus, no masking -- * is required on updating the variable) */ - s8 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; -- -- /* -- * Affinity management and tracking -- */ -- /** Bitvector to aid affinity checking. Element 'n' bit 'i' indicates -- * that slot 'n' is using core i (i.e. slot_affinity_refcount[n][i] > 0) */ - u64 slot_affinities[BASE_JM_MAX_NR_SLOTS]; -- /** Refcount for each core owned by each slot. Used to generate the -- * slot_affinities array of bitvectors -- * -- * The value of the refcount will not exceed BASE_JM_SUBMIT_SLOTS, -- * because it is refcounted only when a job is definitely about to be -- * submitted to a slot, and is de-refcounted immediately after a job -- * finishes */ - s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64]; - } runpool_irq; -- -- /** -- * Scheduling semaphore. This must be held when calling -- * kbase_jm_kick() -- */ - struct semaphore schedule_sem; -- -- /** -- * List of contexts that can currently be pulled from -- */ -- struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; -- /** -- * List of contexts that can not currently be pulled from, but have -- * jobs currently running. -- */ -- struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS][KBASE_JS_ATOM_SCHED_PRIO_COUNT]; -- -- /** Number of currently scheduled user contexts (excluding ones that are not submitting jobs) */ -+ struct list_head ctx_list_pullable[BASE_JM_MAX_NR_SLOTS] -+ [KBASE_JS_ATOM_SCHED_PRIO_COUNT]; -+ struct list_head ctx_list_unpullable[BASE_JM_MAX_NR_SLOTS] -+ [KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - s8 nr_user_contexts_running; -- /** Number of currently scheduled contexts (including ones that are not submitting jobs) */ - s8 nr_all_contexts_running; -- -- /** Core Requirements to match up with base_js_atom's core_req memeber -- * @note This is a write-once member, and so no locking is required to read */ - base_jd_core_req js_reqs[BASE_JM_MAX_NR_SLOTS]; - -- u32 scheduling_period_ns; /*< Value for JS_SCHEDULING_PERIOD_NS */ -- u32 soft_stop_ticks; /*< Value for JS_SOFT_STOP_TICKS */ -- u32 soft_stop_ticks_cl; /*< Value for JS_SOFT_STOP_TICKS_CL */ -- u32 hard_stop_ticks_ss; /*< Value for JS_HARD_STOP_TICKS_SS */ -- u32 hard_stop_ticks_cl; /*< Value for JS_HARD_STOP_TICKS_CL */ -- u32 hard_stop_ticks_dumping; /*< Value for JS_HARD_STOP_TICKS_DUMPING */ -- u32 gpu_reset_ticks_ss; /*< Value for JS_RESET_TICKS_SS */ -- u32 gpu_reset_ticks_cl; /*< Value for JS_RESET_TICKS_CL */ -- u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */ -- u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */ -+ u32 scheduling_period_ns; -+ u32 soft_stop_ticks; -+ u32 soft_stop_ticks_cl; -+ u32 hard_stop_ticks_ss; -+ u32 hard_stop_ticks_cl; -+ u32 hard_stop_ticks_dumping; -+ u32 gpu_reset_ticks_ss; -+ u32 gpu_reset_ticks_cl; -+ u32 gpu_reset_ticks_dumping; -+ u32 ctx_timeslice_ns; - -- /** List of suspended soft jobs */ - struct list_head suspended_soft_jobs_list; - - #ifdef CONFIG_MALI_DEBUG -- /* Support soft-stop on a single context */ - bool softstop_always; - #endif /* CONFIG_MALI_DEBUG */ -- -- /** The initalized-flag is placed at the end, to avoid cache-pollution (we should -- * only be using this during init/term paths). -- * @note This is a write-once member, and so no locking is required to read */ - int init_status; -- -- /* Number of contexts that can currently be pulled from */ - u32 nr_contexts_pullable; -- -- /* Number of contexts that can either be pulled from or are currently -- * running */ - atomic_t nr_contexts_runnable; -- -- /** Value for JS_SOFT_JOB_TIMEOUT */ - atomic_t soft_job_timeout_ms; -- -- /** -- * Queue Lock, used to access the Policy's queue of contexts -- * independently of the Run Pool. -- * -- * Of course, you don't need the Run Pool lock to access this. -- */ - struct mutex queue_mutex; -- -- /** -+ /* - * Run Pool mutex, for managing contexts within the runpool. - * Unless otherwise specified, you must hold this lock whilst accessing - * any members that follow -@@ -317,61 +313,59 @@ struct kbasep_js_device_data { - }; - - /** -- * @brief KBase Context Job Scheduling information structure -+ * struct kbasep_js_kctx_info - KBase Context Job Scheduling information -+ * structure -+ * @ctx: Job Scheduler Context information sub-structure.Its members are -+ * accessed regardless of whether the context is: -+ * - In the Policy's Run Pool -+ * - In the Policy's Queue -+ * - Not queued nor in the Run Pool. -+ * You must obtain the @ctx.jsctx_mutex before accessing any other members -+ * of this substructure. -+ * You may not access any of its members from IRQ context. -+ * @ctx.jsctx_mutex: Job Scheduler Context lock -+ * @ctx.nr_jobs: Number of jobs <b>ready to run</b> - does \em not include -+ * the jobs waiting in the dispatcher, and dependency-only -+ * jobs. See kbase_jd_context::job_nr for such jobs -+ * @ctx.ctx_attr_ref_count: Context Attributes ref count. Each is large enough -+ * to hold a refcount of the number of atoms on the context. -+ * @ctx.is_scheduled_wait: Wait queue to wait for KCTX_SHEDULED flag state -+ * changes. -+ * @ctx.ctx_list_entry: Link implementing JS queues. Context can be present on -+ * one list per job slot. -+ * @init_status: The initalized-flag is placed at the end, to avoid -+ * cache-pollution (we should only be using this during init/term paths) - * - * This is a substructure in the struct kbase_context that encapsulates all the - * scheduling information. - */ - struct kbasep_js_kctx_info { -- -- /** -- * Job Scheduler Context information sub-structure. These members are -- * accessed regardless of whether the context is: -- * - In the Policy's Run Pool -- * - In the Policy's Queue -- * - Not queued nor in the Run Pool. -- * -- * You must obtain the jsctx_mutex before accessing any other members of -- * this substructure. -- * -- * You may not access any of these members from IRQ context. -- */ - struct kbase_jsctx { -- struct mutex jsctx_mutex; /**< Job Scheduler Context lock */ -+ struct mutex jsctx_mutex; - -- /** Number of jobs <b>ready to run</b> - does \em not include the jobs waiting in -- * the dispatcher, and dependency-only jobs. See kbase_jd_context::job_nr -- * for such jobs*/ - u32 nr_jobs; -- -- /** Context Attributes: -- * Each is large enough to hold a refcount of the number of atoms on -- * the context. **/ - u32 ctx_attr_ref_count[KBASEP_JS_CTX_ATTR_COUNT]; -- -- /** -- * Wait queue to wait for KCTX_SHEDULED flag state changes. -- * */ - wait_queue_head_t is_scheduled_wait; -- -- /** Link implementing JS queues. Context can be present on one -- * list per job slot -- */ - struct list_head ctx_list_entry[BASE_JM_MAX_NR_SLOTS]; - } ctx; -- -- /* The initalized-flag is placed at the end, to avoid cache-pollution (we should -- * only be using this during init/term paths) */ - int init_status; - }; - --/** Subset of atom state that can be available after jd_done_nolock() is called -+/** -+ * struct kbasep_js_atom_retained_state - Subset of atom state. -+ * @event_code: to determine whether the atom has finished -+ * @core_req: core requirements -+ * @sched_priority: priority -+ * @device_nr: Core group atom was executed on -+ * -+ * Subset of atom state that can be available after jd_done_nolock() is called - * on that atom. A copy must be taken via kbasep_js_atom_retained_state_copy(), -- * because the original atom could disappear. */ -+ * because the original atom could disappear. -+ */ - struct kbasep_js_atom_retained_state { -- /** Event code - to determine whether the atom has finished */ -+ /* Event code - to determine whether the atom has finished */ - enum base_jd_event_code event_code; -- /** core requirements */ -+ /* core requirements */ - base_jd_core_req core_req; - /* priority */ - int sched_priority; -@@ -380,30 +374,23 @@ struct kbasep_js_atom_retained_state { - - }; - --/** -+/* - * Value signifying 'no retry on a slot required' for: - * - kbase_js_atom_retained_state::retry_submit_on_slot - * - kbase_jd_atom::retry_submit_on_slot - */ - #define KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID (-1) - --/** -- * base_jd_core_req value signifying 'invalid' for a kbase_jd_atom_retained_state. -- * -- * @see kbase_atom_retained_state_is_valid() -+/* -+ * base_jd_core_req value signifying 'invalid' for a -+ * kbase_jd_atom_retained_state. See kbase_atom_retained_state_is_valid() - */ - #define KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID BASE_JD_REQ_DEP - --/** -- * @brief The JS timer resolution, in microseconds -- * -+/* -+ * The JS timer resolution, in microseconds - * Any non-zero difference in time will be at least this size. - */ - #define KBASEP_JS_TICK_RESOLUTION_US 1 - -- -- /** @} *//* end group kbase_js */ -- /** @} *//* end group base_kbase_api */ -- /** @} *//* end group base_api */ -- --#endif /* _KBASE_JS_DEFS_H_ */ -+#endif /* _KBASE_JS_DEFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h -index 6885f8d..93cd05f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_hwconfig_features.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, -@@ -50,12 +49,12 @@ enum base_hw_feature { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_COHERENCY_REG, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_TLS_HASHING, - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, -+ BASE_HW_FEATURE_ASN_HASH, - BASE_HW_FEATURE_END - }; - -@@ -85,7 +84,6 @@ static const enum base_hw_feature base_hw_features_tMIx[] = { - BASE_HW_FEATURE_THREAD_GROUP_SPLIT, - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_END - }; - -@@ -112,7 +110,6 @@ static const enum base_hw_feature base_hw_features_tHEx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_END - }; - -@@ -139,7 +136,6 @@ static const enum base_hw_feature base_hw_features_tSIx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_END - }; - -@@ -166,7 +162,6 @@ static const enum base_hw_feature base_hw_features_tDVx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_END - }; - -@@ -193,7 +188,6 @@ static const enum base_hw_feature base_hw_features_tNOx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_TLS_HASHING, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_END -@@ -222,7 +216,6 @@ static const enum base_hw_feature base_hw_features_tGOx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_TLS_HASHING, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_END -@@ -250,7 +243,6 @@ static const enum base_hw_feature base_hw_features_tTRx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_END -@@ -278,7 +270,6 @@ static const enum base_hw_feature base_hw_features_tNAx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_END -@@ -306,14 +297,13 @@ static const enum base_hw_feature base_hw_features_tBEx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_END - }; - --static const enum base_hw_feature base_hw_features_tDUx[] = { -+static const enum base_hw_feature base_hw_features_tBAx[] = { - BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, - BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, - BASE_HW_FEATURE_XAFFINITY, -@@ -335,14 +325,13 @@ static const enum base_hw_feature base_hw_features_tDUx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_END - }; - --static const enum base_hw_feature base_hw_features_tODx[] = { -+static const enum base_hw_feature base_hw_features_tDUx[] = { - BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, - BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, - BASE_HW_FEATURE_XAFFINITY, -@@ -364,13 +353,13 @@ static const enum base_hw_feature base_hw_features_tODx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, -+ BASE_HW_FEATURE_IDVS_GROUP_SIZE, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_END - }; - --static const enum base_hw_feature base_hw_features_tGRx[] = { -+static const enum base_hw_feature base_hw_features_tODx[] = { - BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, - BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, - BASE_HW_FEATURE_XAFFINITY, -@@ -392,13 +381,12 @@ static const enum base_hw_feature base_hw_features_tGRx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_END - }; - --static const enum base_hw_feature base_hw_features_tVAx[] = { -+static const enum base_hw_feature base_hw_features_tGRx[] = { - BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, - BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, - BASE_HW_FEATURE_XAFFINITY, -@@ -420,13 +408,12 @@ static const enum base_hw_feature base_hw_features_tVAx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_END - }; - --static const enum base_hw_feature base_hw_features_tTUx[] = { -+static const enum base_hw_feature base_hw_features_tVAx[] = { - BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, - BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, - BASE_HW_FEATURE_XAFFINITY, -@@ -448,39 +435,10 @@ static const enum base_hw_feature base_hw_features_tTUx[] = { - BASE_HW_FEATURE_FLUSH_REDUCTION, - BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, - BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, - BASE_HW_FEATURE_L2_CONFIG, - BASE_HW_FEATURE_CLEAN_ONLY_SAFE, - BASE_HW_FEATURE_END - }; - --static const enum base_hw_feature base_hw_features_tE2x[] = { -- BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION, -- BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS, -- BASE_HW_FEATURE_XAFFINITY, -- BASE_HW_FEATURE_WARPING, -- BASE_HW_FEATURE_INTERPIPE_REG_ALIASING, -- BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS, -- BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL, -- BASE_HW_FEATURE_BRNDOUT_CC, -- BASE_HW_FEATURE_BRNDOUT_KILL, -- BASE_HW_FEATURE_LD_ST_LEA_TEX, -- BASE_HW_FEATURE_LD_ST_TILEBUFFER, -- BASE_HW_FEATURE_LINEAR_FILTER_FLOAT, -- BASE_HW_FEATURE_MRT, -- BASE_HW_FEATURE_MSAA_16X, -- BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE, -- BASE_HW_FEATURE_OUT_OF_ORDER_EXEC, -- BASE_HW_FEATURE_T7XX_PAIRING_RULES, -- BASE_HW_FEATURE_TEST4_DATUM_MODE, -- BASE_HW_FEATURE_FLUSH_REDUCTION, -- BASE_HW_FEATURE_PROTECTED_DEBUG_MODE, -- BASE_HW_FEATURE_COHERENCY_REG, -- BASE_HW_FEATURE_AARCH64_MMU, -- BASE_HW_FEATURE_IDVS_GROUP_SIZE, -- BASE_HW_FEATURE_L2_CONFIG, -- BASE_HW_FEATURE_CLEAN_ONLY_SAFE, -- BASE_HW_FEATURE_END --}; - - #endif /* _BASE_HWCONFIG_FEATURES_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h -index 3966069..beda1e4 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_hwconfig_issues.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* AUTOMATICALLY GENERATED FILE. If you want to amend the issues/features, -@@ -59,6 +58,7 @@ enum base_hw_issue { - BASE_HW_ISSUE_TTRX_3470, - BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_TTRX_3485, -+ BASE_HW_ISSUE_GPU2019_3212, - BASE_HW_ISSUE_END - }; - -@@ -532,79 +532,89 @@ static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = { - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_tDUx_r0p0[] = { -+static const enum base_hw_issue base_hw_issues_tBAx_r0p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, -+ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, - BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, -+ BASE_HW_ISSUE_TTRX_3470, -+ BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_model_tDUx[] = { -- BASE_HW_ISSUE_5736, -+static const enum base_hw_issue base_hw_issues_tBAx_r1p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, -+ BASE_HW_ISSUE_TTRX_2968_TTRX_3162, -+ BASE_HW_ISSUE_TTRX_921, - BASE_HW_ISSUE_TTRX_3414, - BASE_HW_ISSUE_TTRX_3083, -+ BASE_HW_ISSUE_TTRX_3470, -+ BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { -- BASE_HW_ISSUE_9435, -- BASE_HW_ISSUE_TSIX_2033, -- BASE_HW_ISSUE_TTRX_1337, -- BASE_HW_ISSUE_END --}; -- --static const enum base_hw_issue base_hw_issues_model_tODx[] = { -+static const enum base_hw_issue base_hw_issues_model_tBAx[] = { - BASE_HW_ISSUE_5736, - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, -+ BASE_HW_ISSUE_TTRX_3414, -+ BASE_HW_ISSUE_TTRX_3083, -+ BASE_HW_ISSUE_TTRX_3470, -+ BASE_HW_ISSUE_TTRX_3464, - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { -+static const enum base_hw_issue base_hw_issues_tDUx_r0p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, -+ BASE_HW_ISSUE_TTRX_921, -+ BASE_HW_ISSUE_TTRX_3414, -+ BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_model_tGRx[] = { -+static const enum base_hw_issue base_hw_issues_model_tDUx[] = { - BASE_HW_ISSUE_5736, - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, -+ BASE_HW_ISSUE_TTRX_3414, -+ BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { -+static const enum base_hw_issue base_hw_issues_tODx_r0p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, -+ BASE_HW_ISSUE_GPU2019_3212, - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_model_tVAx[] = { -+static const enum base_hw_issue base_hw_issues_model_tODx[] = { - BASE_HW_ISSUE_5736, - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, -+ BASE_HW_ISSUE_GPU2019_3212, - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = { -+static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_model_tTUx[] = { -+static const enum base_hw_issue base_hw_issues_model_tGRx[] = { - BASE_HW_ISSUE_5736, - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, -@@ -612,24 +622,20 @@ static const enum base_hw_issue base_hw_issues_model_tTUx[] = { - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_tE2x_r0p0[] = { -+static const enum base_hw_issue base_hw_issues_tVAx_r0p0[] = { - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, -- BASE_HW_ISSUE_TTRX_921, -- BASE_HW_ISSUE_TTRX_3414, -- BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_END - }; - --static const enum base_hw_issue base_hw_issues_model_tE2x[] = { -+static const enum base_hw_issue base_hw_issues_model_tVAx[] = { - BASE_HW_ISSUE_5736, - BASE_HW_ISSUE_9435, - BASE_HW_ISSUE_TSIX_2033, - BASE_HW_ISSUE_TTRX_1337, -- BASE_HW_ISSUE_TTRX_3414, -- BASE_HW_ISSUE_TTRX_3083, - BASE_HW_ISSUE_END - }; - -+ - #endif /* _BASE_HWCONFIG_ISSUES_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase.h -index 907142d..56db8ca 100755 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #ifndef _KBASE_H_ - #define _KBASE_H_ - -@@ -38,7 +35,7 @@ - #include <linux/mutex.h> - #include <linux/rwsem.h> - #include <linux/sched.h> --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)) -+#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) - #include <linux/sched/mm.h> - #endif - #include <linux/slab.h> -@@ -46,8 +43,9 @@ - #include <linux/vmalloc.h> - #include <linux/wait.h> - #include <linux/workqueue.h> -+#include <linux/interrupt.h> - --#include "mali_base_kernel.h" -+#include <uapi/gpu/arm/midgard/mali_base_kernel.h> - #include <mali_kbase_linux.h> - - /* -@@ -66,26 +64,40 @@ - #include "mali_kbase_gpu_memory_debugfs.h" - #include "mali_kbase_mem_profile_debugfs.h" - #include "mali_kbase_gpuprops.h" --#include "mali_kbase_ioctl.h" -+#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h> -+#if !MALI_USE_CSF - #include "mali_kbase_debug_job_fault.h" - #include "mali_kbase_jd_debugfs.h" - #include "mali_kbase_jm.h" - #include "mali_kbase_js.h" -+#endif /* !MALI_USE_CSF */ - - #include "ipa/mali_kbase_ipa.h" - --#ifdef CONFIG_GPU_TRACEPOINTS -+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) - #include <trace/events/gpu.h> - #endif - - #include "mali_linux_trace.h" - -+#if MALI_USE_CSF -+#include "csf/mali_kbase_csf.h" -+#endif - - #ifndef u64_to_user_ptr - /* Introduced in Linux v4.6 */ - #define u64_to_user_ptr(x) ((void __user *)(uintptr_t)x) - #endif - -+#if MALI_USE_CSF -+/* Physical memory group ID for CSF user I/O. -+ */ -+#define KBASE_MEM_GROUP_CSF_IO BASE_MEM_GROUP_DEFAULT -+ -+/* Physical memory group ID for CSF firmware. -+ */ -+#define KBASE_MEM_GROUP_CSF_FW BASE_MEM_GROUP_DEFAULT -+#endif - - /* Physical memory group ID for a special page which can alias several regions. - */ -@@ -140,9 +152,9 @@ void kbase_release_device(struct kbase_device *kbdev); - * the flag @ref KBASE_REG_TILER_ALIGN_TOP (check the flags of the kbase - * region): - * - alignment offset is set to the difference between the kbase region -- * extent (converted from the original value in pages to bytes) and the kbase -+ * extension (converted from the original value in pages to bytes) and the kbase - * region initial_commit (also converted from the original value in pages to -- * bytes); alignment mask is set to the kbase region extent in bytes and -+ * bytes); alignment mask is set to the kbase region extension in bytes and - * decremented by 1. - * - * Return: if successful, address of the unmapped area aligned as required; -@@ -184,7 +196,7 @@ void kbase_device_pm_term(struct kbase_device *kbdev); - int power_control_init(struct kbase_device *kbdev); - void power_control_term(struct kbase_device *kbdev); - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - void kbase_device_debugfs_term(struct kbase_device *kbdev); - int kbase_device_debugfs_init(struct kbase_device *kbdev); - #else /* CONFIG_DEBUG_FS */ -@@ -201,11 +213,8 @@ void registers_unmap(struct kbase_device *kbdev); - - int kbase_device_coherency_init(struct kbase_device *kbdev); - --#ifdef CONFIG_MALI_BUSLOG --int buslog_init(struct kbase_device *kbdev); --void buslog_term(struct kbase_device *kbdev); --#endif - -+#if !MALI_USE_CSF - int kbase_jd_init(struct kbase_context *kctx); - void kbase_jd_exit(struct kbase_context *kctx); - -@@ -213,9 +222,9 @@ void kbase_jd_exit(struct kbase_context *kctx); - * kbase_jd_submit - Submit atoms to the job dispatcher - * - * @kctx: The kbase context to submit to -- * @user_addr: The address in user space of the struct base_jd_atom_v2 array -+ * @user_addr: The address in user space of the struct base_jd_atom array - * @nr_atoms: The number of atoms in the array -- * @stride: sizeof(struct base_jd_atom_v2) -+ * @stride: sizeof(struct base_jd_atom) - * @uk6_atom: true if the atoms are legacy atoms (struct base_jd_atom_v2_uk6) - * - * Return: 0 on success or error code -@@ -306,9 +315,12 @@ void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action, - void kbase_job_check_leave_disjoint(struct kbase_device *kbdev, - struct kbase_jd_atom *target_katom); - -+#endif /* !MALI_USE_CSF */ - - void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event); -+#if !MALI_USE_CSF - int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent); -+#endif /* !MALI_USE_CSF */ - int kbase_event_pending(struct kbase_context *ctx); - int kbase_event_init(struct kbase_context *kctx); - void kbase_event_close(struct kbase_context *kctx); -@@ -372,6 +384,7 @@ static inline void kbase_free_user_buffer( - */ - int kbase_mem_copy_from_extres(struct kbase_context *kctx, - struct kbase_debug_copy_buffer *buf_data); -+#if !MALI_USE_CSF - int kbase_process_soft_job(struct kbase_jd_atom *katom); - int kbase_prepare_soft_job(struct kbase_jd_atom *katom); - void kbase_finish_soft_job(struct kbase_jd_atom *katom); -@@ -387,18 +400,21 @@ int kbase_soft_event_update(struct kbase_context *kctx, - - void kbasep_soft_job_timeout_worker(struct timer_list *timer); - void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt); -+#endif /* !MALI_USE_CSF */ - - void kbasep_as_do_poke(struct work_struct *work); - - /** - * Check whether a system suspend is in progress, or has already been suspended -+ * @kbdev: The kbase device structure for the device - * - * The caller should ensure that either kbdev->pm.active_count_lock is held, or - * a dmb was executed recently (to ensure the value is most - * up-to-date). However, without a lock the value could change afterwards. - * -- * @return false if a suspend is not in progress -- * @return !=false otherwise -+ * Return: -+ * * false if a suspend is not in progress -+ * * !=false otherwise - */ - static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) - { -@@ -419,7 +435,27 @@ static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev) - */ - static inline bool kbase_pm_is_gpu_lost(struct kbase_device *kbdev) - { -- return kbdev->pm.gpu_lost; -+ return (atomic_read(&kbdev->pm.gpu_lost) == 0 ? false : true); -+} -+ -+/* -+ * Set or clear gpu lost state -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @gpu_lost: true to activate GPU lost state, FALSE is deactive it -+ * -+ * Puts power management code into gpu lost state or takes it out of the -+ * state. Once in gpu lost state new GPU jobs will no longer be -+ * scheduled. -+ */ -+static inline void kbase_pm_set_gpu_lost(struct kbase_device *kbdev, -+ bool gpu_lost) -+{ -+ const int new_val = (gpu_lost ? 1 : 0); -+ const int cur_val = atomic_xchg(&kbdev->pm.gpu_lost, new_val); -+ -+ if (new_val != cur_val) -+ KBASE_KTRACE_ADD(kbdev, ARB_GPU_LOST, NULL, new_val); - } - #endif - -@@ -455,9 +491,12 @@ void kbase_pm_metrics_start(struct kbase_device *kbdev); - */ - void kbase_pm_metrics_stop(struct kbase_device *kbdev); - -+#if !MALI_USE_CSF - /** - * Return the atom's ID, as was originally supplied by userspace in -- * base_jd_atom_v2::atom_number -+ * base_jd_atom::atom_number -+ * @kctx: KBase context pointer -+ * @katom: Atome for which to return ID - */ - static inline int kbase_jd_atom_id(struct kbase_context *kctx, struct kbase_jd_atom *katom) - { -@@ -484,6 +523,7 @@ static inline struct kbase_jd_atom *kbase_jd_atom_from_id( - { - return &kctx->jctx.atoms[id]; - } -+#endif /* !MALI_USE_CSF */ - - /** - * Initialize the disjoint state -@@ -508,7 +548,7 @@ static inline struct kbase_jd_atom *kbase_jd_atom_from_id( - * The disjoint event counter is also incremented immediately whenever a job is soft stopped - * and during context creation. - * -- * @param kbdev The kbase device -+ * @kbdev: The kbase device - * - * Return: 0 on success and non-zero value on failure. - */ -@@ -518,7 +558,7 @@ void kbase_disjoint_init(struct kbase_device *kbdev); - * Increase the count of disjoint events - * called when a disjoint event has happened - * -- * @param kbdev The kbase device -+ * @kbdev: The kbase device - */ - void kbase_disjoint_event(struct kbase_device *kbdev); - -@@ -528,14 +568,14 @@ void kbase_disjoint_event(struct kbase_device *kbdev); - * This should be called when something happens which could be disjoint if the GPU - * is in a disjoint state. The state refcount keeps track of this. - * -- * @param kbdev The kbase device -+ * @kbdev: The kbase device - */ - void kbase_disjoint_event_potential(struct kbase_device *kbdev); - - /** - * Returns the count of disjoint events - * -- * @param kbdev The kbase device -+ * @kbdev: The kbase device - * @return the count of disjoint events - */ - u32 kbase_disjoint_event_get(struct kbase_device *kbdev); -@@ -547,7 +587,7 @@ u32 kbase_disjoint_event_get(struct kbase_device *kbdev); - * eventually after the disjoint state has completed @ref kbase_disjoint_state_down - * should be called - * -- * @param kbdev The kbase device -+ * @kbdev: The kbase device - */ - void kbase_disjoint_state_up(struct kbase_device *kbdev); - -@@ -558,68 +598,43 @@ void kbase_disjoint_state_up(struct kbase_device *kbdev); - * - * Called after @ref kbase_disjoint_state_up once the disjoint state is over - * -- * @param kbdev The kbase device -+ * @kbdev: The kbase device - */ - void kbase_disjoint_state_down(struct kbase_device *kbdev); - - /** -- * If a job is soft stopped and the number of contexts is >= this value -- * it is reported as a disjoint event -- */ --#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2 -- --#if !defined(UINT64_MAX) -- #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) --#endif -- --#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) -- --/* kbase_io_history_init - initialize data struct for register access history -- * -- * @kbdev The register history to initialize -- * @n The number of register accesses that the buffer could hold -+ * kbase_device_pcm_dev_init() - Initialize the priority control manager device - * -- * @return 0 if successfully initialized, failure otherwise -- */ --int kbase_io_history_init(struct kbase_io_history *h, u16 n); -- --/* kbase_io_history_term - uninit all resources for the register access history -+ * @kbdev: Pointer to the structure for the kbase device - * -- * @h The register history to terminate -- */ --void kbase_io_history_term(struct kbase_io_history *h); -- --/* kbase_io_history_dump - print the register history to the kernel ring buffer -+ * Pointer to the priority control manager device is retrieved from the device -+ * tree and a reference is taken on the module implementing the callbacks for -+ * priority control manager operations. - * -- * @kbdev Pointer to kbase_device containing the register history to dump -+ * Return: 0 if successful, or an error code on failure - */ --void kbase_io_history_dump(struct kbase_device *kbdev); -+int kbase_device_pcm_dev_init(struct kbase_device *const kbdev); - - /** -- * kbase_io_history_resize - resize the register access history buffer. -+ * kbase_device_pcm_dev_term() - Performs priority control manager device -+ * deinitialization. - * -- * @h: Pointer to a valid register history to resize -- * @new_size: Number of accesses the buffer could hold -+ * @kbdev: Pointer to the structure for the kbase device - * -- * A successful resize will clear all recent register accesses. -- * If resizing fails for any reason (e.g., could not allocate memory, invalid -- * buffer size) then the original buffer will be kept intact. -- * -- * @return 0 if the buffer was resized, failure otherwise -+ * Reference is released on the module implementing the callbacks for priority -+ * control manager operations. - */ --int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size); -- --#else /* CONFIG_DEBUG_FS */ -+void kbase_device_pcm_dev_term(struct kbase_device *const kbdev); - --#define kbase_io_history_init(...) ((int)0) -- --#define kbase_io_history_term CSTD_NOP -- --#define kbase_io_history_dump CSTD_NOP -- --#define kbase_io_history_resize CSTD_NOP -+/** -+ * If a job is soft stopped and the number of contexts is >= this value -+ * it is reported as a disjoint event -+ */ -+#define KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD 2 - --#endif /* CONFIG_DEBUG_FS */ -+#if !defined(UINT64_MAX) -+ #define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL) -+#endif - - /*meson graphics start */ - extern int meson_gpu_data_invalid_count; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c -index 2e2e394..027eb8c 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2016-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2016-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/debugfs.h> -@@ -26,7 +25,7 @@ - #include <mali_kbase_as_fault_debugfs.h> - #include <device/mali_kbase_device.h> - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - #ifdef CONFIG_MALI_DEBUG - - static int kbase_as_fault_read(struct seq_file *sfile, void *data) -@@ -80,7 +79,7 @@ static const struct file_operations as_fault_fops = { - */ - void kbase_as_fault_debugfs_init(struct kbase_device *kbdev) - { --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - #ifdef CONFIG_MALI_DEBUG - uint i; - char as_name[64]; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h -index 496d8b1..919fbc1 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_as_fault_debugfs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2016, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_AS_FAULT_DEBUG_FS_H -@@ -39,7 +38,7 @@ void kbase_as_fault_debugfs_init(struct kbase_device *kbdev); - static inline void - kbase_as_fault_debugfs_new(struct kbase_device *kbdev, int as_no) - { --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - #ifdef CONFIG_MALI_DEBUG - kbdev->debugfs_as_read_bitmap |= (1ULL << as_no); - #endif /* CONFIG_DEBUG_FS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_bits.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_bits.h -index 2c11093..a085fd8 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_bits.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_bits.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,17 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ --/* -- * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. - */ - - #ifndef _KBASE_BITS_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c -index 27a03cf..af51ed8 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cache_policy.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Cache Policy API. - */ -@@ -58,10 +55,11 @@ void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, - { - dma_sync_single_for_device(kbdev->dev, handle, size, dir); - } -- -+KBASE_EXPORT_TEST_API(kbase_sync_single_for_device); - - void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, - size_t size, enum dma_data_direction dir) - { - dma_sync_single_for_cpu(kbdev->dev, handle, size, dir); - } -+KBASE_EXPORT_TEST_API(kbase_sync_single_for_cpu); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h -index 8a1e529..7da33a6 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cache_policy.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2013, 2015 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2013, 2015, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Cache Policy API. - */ -@@ -30,7 +27,7 @@ - #define _KBASE_CACHE_POLICY_H_ - - #include "mali_kbase.h" --#include "mali_base_kernel.h" -+#include <uapi/gpu/arm/midgard/mali_base_kernel.h> - - /** - * kbase_cache_enabled - Choose the cache policy for a specific region -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_caps.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_caps.h -new file mode 100644 -index 0000000..c232e21 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_caps.h -@@ -0,0 +1,61 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/** -+ * DOC: Driver Capability Queries. -+ */ -+ -+#ifndef _KBASE_CAPS_H_ -+#define _KBASE_CAPS_H_ -+ -+#include <linux/types.h> -+ -+typedef enum mali_kbase_cap { -+ MALI_KBASE_CAP_SYSTEM_MONITOR = 0, -+ MALI_KBASE_CAP_JIT_PRESSURE_LIMIT, -+ MALI_KBASE_CAP_MEM_GROW_ON_GPF, -+ MALI_KBASE_CAP_MEM_PROTECTED, -+ MALI_KBASE_NUM_CAPS -+} mali_kbase_cap; -+ -+extern bool mali_kbase_supports_cap(unsigned long api_version, mali_kbase_cap cap); -+ -+static inline bool mali_kbase_supports_system_monitor(unsigned long api_version) -+{ -+ return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_SYSTEM_MONITOR); -+} -+ -+static inline bool mali_kbase_supports_jit_pressure_limit(unsigned long api_version) -+{ -+ return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_JIT_PRESSURE_LIMIT); -+} -+ -+static inline bool mali_kbase_supports_mem_grow_on_gpf(unsigned long api_version) -+{ -+ return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_GROW_ON_GPF); -+} -+ -+static inline bool mali_kbase_supports_mem_protected(unsigned long api_version) -+{ -+ return mali_kbase_supports_cap(api_version, MALI_KBASE_CAP_MEM_PROTECTED); -+} -+ -+#endif /* __KBASE_CAPS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ccswe.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ccswe.c -new file mode 100644 -index 0000000..6a1e7e4 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ccswe.c -@@ -0,0 +1,100 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_ccswe.h" -+#include "mali_kbase_linux.h" -+ -+#include <linux/math64.h> -+#include <linux/time.h> -+ -+static u64 kbasep_ccswe_cycle_at_no_lock( -+ struct kbase_ccswe *self, u64 timestamp_ns) -+{ -+ s64 diff_s, diff_ns; -+ u32 gpu_freq; -+ -+ lockdep_assert_held(&self->access); -+ -+ diff_ns = timestamp_ns - self->timestamp_ns; -+ gpu_freq = diff_ns > 0 ? self->gpu_freq : self->prev_gpu_freq; -+ -+ diff_s = div_s64(diff_ns, NSEC_PER_SEC); -+ diff_ns -= diff_s * NSEC_PER_SEC; -+ -+ return self->cycles_elapsed + diff_s * gpu_freq -+ + div_s64(diff_ns * gpu_freq, NSEC_PER_SEC); -+} -+ -+void kbase_ccswe_init(struct kbase_ccswe *self) -+{ -+ memset(self, 0, sizeof(*self)); -+ -+ spin_lock_init(&self->access); -+} -+ -+u64 kbase_ccswe_cycle_at(struct kbase_ccswe *self, u64 timestamp_ns) -+{ -+ unsigned long flags; -+ u64 result; -+ -+ spin_lock_irqsave(&self->access, flags); -+ result = kbasep_ccswe_cycle_at_no_lock(self, timestamp_ns); -+ spin_unlock_irqrestore(&self->access, flags); -+ -+ return result; -+} -+ -+void kbase_ccswe_freq_change( -+ struct kbase_ccswe *self, u64 timestamp_ns, u32 gpu_freq) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&self->access, flags); -+ -+ /* The time must go only forward. */ -+ if (WARN_ON(timestamp_ns < self->timestamp_ns)) -+ goto exit; -+ -+ /* If this is the first frequency change, cycles_elapsed is zero. */ -+ if (self->timestamp_ns) -+ self->cycles_elapsed = kbasep_ccswe_cycle_at_no_lock( -+ self, timestamp_ns); -+ -+ self->timestamp_ns = timestamp_ns; -+ self->prev_gpu_freq = self->gpu_freq; -+ self->gpu_freq = gpu_freq; -+exit: -+ spin_unlock_irqrestore(&self->access, flags); -+} -+ -+void kbase_ccswe_reset(struct kbase_ccswe *self) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&self->access, flags); -+ -+ self->timestamp_ns = 0; -+ self->cycles_elapsed = 0; -+ self->gpu_freq = 0; -+ self->prev_gpu_freq = 0; -+ -+ spin_unlock_irqrestore(&self->access, flags); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ccswe.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ccswe.h -new file mode 100644 -index 0000000..8e55ffc ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ccswe.h -@@ -0,0 +1,96 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_CCSWE_H_ -+#define _KBASE_CCSWE_H_ -+ -+#include <linux/spinlock.h> -+ -+/** -+ * struct kbase_ccswe - Cycle count software estimator. -+ * -+ * @access: Spinlock protecting this structure access. -+ * @timestamp_ns: Timestamp(ns) when the last frequency change -+ * occurred. -+ * @cycles_elapsed: Number of cycles elapsed before the last frequency -+ * change -+ * @gpu_freq: Current GPU frequency(Hz) value. -+ * @prev_gpu_freq: Previous GPU frequency(Hz) before the last frequency -+ * change. -+ */ -+struct kbase_ccswe { -+ spinlock_t access; -+ u64 timestamp_ns; -+ u64 cycles_elapsed; -+ u32 gpu_freq; -+ u32 prev_gpu_freq; -+}; -+ -+/** -+ * kbase_ccswe_init() - initialize the cycle count estimator. -+ * -+ * @self: Cycles count software estimator instance. -+ */ -+void kbase_ccswe_init(struct kbase_ccswe *self); -+ -+ -+/** -+ * kbase_ccswe_cycle_at() - Estimate cycle count at given timestamp. -+ * -+ * @self: Cycles count software estimator instance. -+ * @timestamp_ns: The timestamp(ns) for cycle count estimation. -+ * -+ * The timestamp must be bigger than the timestamp of the penultimate -+ * frequency change. If only one frequency change occurred, the -+ * timestamp must be bigger than the timestamp of the frequency change. -+ * This is to allow the following code to be executed w/o synchronization. -+ * If lines below executed atomically, it is safe to assume that only -+ * one frequency change may happen in between. -+ * -+ * u64 ts = ktime_get_raw_ns(); -+ * u64 cycle = kbase_ccswe_cycle_at(&ccswe, ts) -+ * -+ * Returns: estimated value of cycle count at a given time. -+ */ -+u64 kbase_ccswe_cycle_at(struct kbase_ccswe *self, u64 timestamp_ns); -+ -+/** -+ * kbase_ccswe_freq_change() - update GPU frequency. -+ * -+ * @self: Cycles count software estimator instance. -+ * @timestamp_ns: Timestamp(ns) when frequency change occurred. -+ * @gpu_freq: New GPU frequency value. -+ * -+ * The timestamp must be bigger than the timestamp of the previous -+ * frequency change. The function is to be called at the frequency -+ * change moment (not later). -+ */ -+void kbase_ccswe_freq_change( -+ struct kbase_ccswe *self, u64 timestamp_ns, u32 gpu_freq); -+ -+/** -+ * kbase_ccswe_reset() - reset estimator state -+ * -+ * @self: Cycles count software estimator instance. -+ */ -+void kbase_ccswe_reset(struct kbase_ccswe *self); -+ -+#endif /* _KBASE_CCSWE_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config.c -index ce7070d..37dbca1 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2015,2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2015, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #include <mali_kbase.h> - #include <mali_kbase_defs.h> - #include <mali_kbase_config_defaults.h> -@@ -46,3 +43,62 @@ void kbasep_platform_device_term(struct kbase_device *kbdev) - platform_funcs_p->platform_term_func(kbdev); - } - -+int kbasep_platform_device_late_init(struct kbase_device *kbdev) -+{ -+ struct kbase_platform_funcs_conf *platform_funcs_p; -+ -+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; -+ if (platform_funcs_p && platform_funcs_p->platform_late_init_func) -+ platform_funcs_p->platform_late_init_func(kbdev); -+ -+ return 0; -+} -+ -+void kbasep_platform_device_late_term(struct kbase_device *kbdev) -+{ -+ struct kbase_platform_funcs_conf *platform_funcs_p; -+ -+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; -+ if (platform_funcs_p && platform_funcs_p->platform_late_term_func) -+ platform_funcs_p->platform_late_term_func(kbdev); -+} -+ -+#if !MALI_USE_CSF -+int kbasep_platform_context_init(struct kbase_context *kctx) -+{ -+ struct kbase_platform_funcs_conf *platform_funcs_p; -+ -+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; -+ if (platform_funcs_p && platform_funcs_p->platform_handler_context_init_func) -+ return platform_funcs_p->platform_handler_context_init_func(kctx); -+ -+ return 0; -+} -+ -+void kbasep_platform_context_term(struct kbase_context *kctx) -+{ -+ struct kbase_platform_funcs_conf *platform_funcs_p; -+ -+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; -+ if (platform_funcs_p && platform_funcs_p->platform_handler_context_term_func) -+ platform_funcs_p->platform_handler_context_term_func(kctx); -+} -+ -+void kbasep_platform_event_atom_submit(struct kbase_jd_atom *katom) -+{ -+ struct kbase_platform_funcs_conf *platform_funcs_p; -+ -+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; -+ if (platform_funcs_p && platform_funcs_p->platform_handler_atom_submit_func) -+ platform_funcs_p->platform_handler_atom_submit_func(katom); -+} -+ -+void kbasep_platform_event_atom_complete(struct kbase_jd_atom *katom) -+{ -+ struct kbase_platform_funcs_conf *platform_funcs_p; -+ -+ platform_funcs_p = (struct kbase_platform_funcs_conf *)PLATFORM_FUNCS; -+ if (platform_funcs_p && platform_funcs_p->platform_handler_atom_complete_func) -+ platform_funcs_p->platform_handler_atom_complete_func(katom); -+} -+#endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config.h -index 69723ea..e7eb334 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2017, 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2017, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_config.h -- * Configuration API and Attributes for KBase -+ * DOC: Configuration API and Attributes for KBase - */ - - #ifndef _KBASE_CONFIG_H_ -@@ -32,37 +28,31 @@ - - #include <linux/mm.h> - #include <mali_malisw.h> --#include <mali_kbase_backend_config.h> -+#include <backend/gpu/mali_kbase_backend_config.h> - #include <linux/rbtree.h> - --/** -- * @addtogroup base_api -- * @{ -- */ -- --/** -- * @addtogroup base_kbase_api -- * @{ -- */ -- --/** -- * @addtogroup kbase_config Configuration API and Attributes -- * @{ -- */ -- - /* Forward declaration of struct kbase_device */ - struct kbase_device; - -+#if !MALI_USE_CSF -+/* Forward declaration of struct kbase_context */ -+struct kbase_context; -+ -+/* Forward declaration of struct kbase_atom */ -+struct kbase_jd_atom; -+#endif -+ - /** -- * kbase_platform_funcs_conf - Specifies platform init/term function pointers -+ * struct kbase_platform_funcs_conf - Specifies platform integration function -+ * pointers for DDK events such as device init and term. - * - * Specifies the functions pointers for platform specific initialization and -- * termination. By default no functions are required. No additional platform -- * specific control is necessary. -+ * termination as well as other events. By default no functions are required. -+ * No additional platform specific control is necessary. - */ - struct kbase_platform_funcs_conf { - /** -- * platform_init_func - platform specific init function pointer -+ * @platform_init_func: platform specific init function pointer - * @kbdev - kbase_device pointer - * - * Returns 0 on success, negative error code otherwise. -@@ -77,7 +67,7 @@ struct kbase_platform_funcs_conf { - */ - int (*platform_init_func)(struct kbase_device *kbdev); - /** -- * platform_term_func - platform specific termination function pointer -+ * @platform_term_func: platform specific termination function pointer - * @kbdev - kbase_device pointer - * - * Function pointer for platform specific termination or NULL if no -@@ -88,6 +78,84 @@ struct kbase_platform_funcs_conf { - * can be accessed (and possibly terminated) in here. - */ - void (*platform_term_func)(struct kbase_device *kbdev); -+ -+ /** -+ * @platform_late_init_func: platform specific late init function pointer -+ * @kbdev - kbase_device pointer -+ * -+ * Function pointer to inform that the kbase driver initialization completed -+ * or NULL if no such function is required. At this point the GPU driver will be -+ * fully initialized. -+ * -+ * The platform specific private pointer kbase_device::platform_context -+ * can be accessed (and possibly terminated) in here. -+ */ -+ int (*platform_late_init_func)(struct kbase_device *kbdev); -+ -+ /** -+ * @platform_late_term_func: platform specific late termination function pointer -+ * @kbdev - kbase_device pointer -+ * -+ * Function pointer for platform specific termination or NULL if no -+ * termination function is required. At this point the GPU driver will complete -+ * termination process -+ * -+ * The platform specific private pointer kbase_device::platform_context -+ * can be accessed (and possibly terminated) in here. -+ */ -+ void (*platform_late_term_func)(struct kbase_device *kbdev); -+ -+#if !MALI_USE_CSF -+ /** -+ * @platform_handler_context_init_func: platform specific handler for -+ * when a new kbase_context is created. -+ * @kctx - kbase_context pointer -+ * -+ * Returns 0 on success, negative error code otherwise. -+ * -+ * Function pointer for platform specific initialization of a kernel -+ * context or NULL if not required. Called at the last stage of kernel -+ * context initialization. -+ */ -+ int (*platform_handler_context_init_func)(struct kbase_context *kctx); -+ /** -+ * @platform_handler_context_term_func: platform specific handler for -+ * when a kbase_context is terminated. -+ * @kctx - kbase_context pointer -+ * -+ * Function pointer for platform specific termination of a kernel -+ * context or NULL if not required. Called at the first stage of kernel -+ * context termination. -+ */ -+ void (*platform_handler_context_term_func)(struct kbase_context *kctx); -+ /** -+ * @platform_handler_atom_submit_func: platform specific handler for -+ * when a kbase_jd_atom is submitted. -+ * @katom - kbase_jd_atom pointer -+ * -+ * Function pointer for platform specific handling at the point when an -+ * atom is submitted to the GPU or set to NULL if not required. The -+ * function cannot assume that it is running in a process context. -+ * -+ * Context: The caller must hold the hwaccess_lock. Function must be -+ * runnable in an interrupt context. -+ */ -+ void (*platform_handler_atom_submit_func)(struct kbase_jd_atom *katom); -+ /** -+ * @platform_handler_atom_complete_func: platform specific handler for -+ * when a kbase_jd_atom completes. -+ * @katom - kbase_jd_atom pointer -+ * -+ * Function pointer for platform specific handling at the point when an -+ * atom stops running on the GPU or set to NULL if not required. The -+ * function cannot assume that it is running in a process context. -+ * -+ * Context: The caller must hold the hwaccess_lock. Function must be -+ * runnable in an interrupt context. -+ */ -+ void (*platform_handler_atom_complete_func)( -+ struct kbase_jd_atom *katom); -+#endif - }; - - /* -@@ -223,7 +291,90 @@ struct kbase_pm_callback_conf { - int (*soft_reset_callback)(struct kbase_device *kbdev); - }; - --#ifdef CONFIG_OF -+/* struct kbase_gpu_clk_notifier_data - Data for clock rate change notifier. -+ * -+ * Pointer to this structure is supposed to be passed to the gpu clock rate -+ * change notifier function. This structure is deliberately aligned with the -+ * common clock framework notification structure 'struct clk_notifier_data' -+ * and such alignment should be maintained. -+ * -+ * @gpu_clk_handle: Handle of the GPU clock for which notifier was registered. -+ * @old_rate: Previous rate of this GPU clock in Hz. -+ * @new_rate: New rate of this GPU clock in Hz. -+ */ -+struct kbase_gpu_clk_notifier_data { -+ void *gpu_clk_handle; -+ unsigned long old_rate; -+ unsigned long new_rate; -+}; -+ -+/** -+ * struct kbase_clk_rate_trace_op_conf - Specifies GPU clock rate trace -+ * operations. -+ * -+ * Specifies the functions pointers for platform specific GPU clock rate trace -+ * operations. By default no functions are required. -+ */ -+struct kbase_clk_rate_trace_op_conf { -+ /** -+ * @enumerate_gpu_clk: Enumerate a GPU clock on the given index -+ * @kbdev - kbase_device pointer -+ * @index - GPU clock index -+ * -+ * Returns a handle unique to the given GPU clock, or NULL if the clock -+ * array has been exhausted at the given index value. -+ * -+ * Kbase will use this function pointer to enumerate the existence of a -+ * GPU clock on the given index. -+ */ -+ void *(*enumerate_gpu_clk)(struct kbase_device *kbdev, -+ unsigned int index); -+ -+ /** -+ * @get_gpu_clk_rate: Get the current rate for an enumerated clock. -+ * @kbdev - kbase_device pointer -+ * @gpu_clk_handle - Handle unique to the enumerated GPU clock -+ * -+ * Returns current rate of the GPU clock in unit of Hz. -+ */ -+ unsigned long (*get_gpu_clk_rate)(struct kbase_device *kbdev, -+ void *gpu_clk_handle); -+ -+ /** -+ * @gpu_clk_notifier_register: Register a clock rate change notifier. -+ * @kbdev - kbase_device pointer -+ * @gpu_clk_handle - Handle unique to the enumerated GPU clock -+ * @nb - notifier block containing the callback function -+ * pointer -+ * -+ * Returns 0 on success, negative error code otherwise. -+ * -+ * This function pointer is used to register a callback function that -+ * is supposed to be invoked whenever the rate of clock corresponding -+ * to @gpu_clk_handle changes. -+ * @nb contains the pointer to callback function. -+ * The callback function expects the pointer of type -+ * 'struct kbase_gpu_clk_notifier_data' as the third argument. -+ */ -+ int (*gpu_clk_notifier_register)(struct kbase_device *kbdev, -+ void *gpu_clk_handle, struct notifier_block *nb); -+ -+ /** -+ * @gpu_clk_notifier_unregister: Unregister clock rate change notifier -+ * @kbdev - kbase_device pointer -+ * @gpu_clk_handle - Handle unique to the enumerated GPU clock -+ * @nb - notifier block containing the callback function -+ * pointer -+ * -+ * This function pointer is used to unregister a callback function that -+ * was previously registered to get notified of the change in rate -+ * of clock corresponding to @gpu_clk_handle. -+ */ -+ void (*gpu_clk_notifier_unregister)(struct kbase_device *kbdev, -+ void *gpu_clk_handle, struct notifier_block *nb); -+}; -+ -+#if IS_ENABLED(CONFIG_OF) - struct kbase_platform_config { - }; - #else -@@ -253,7 +404,7 @@ struct kbase_platform_config { - #endif /* CONFIG_OF */ - - /** -- * @brief Gets the pointer to platform config. -+ * kbase_get_platform_config - Gets the pointer to platform config. - * - * @return Pointer to the platform config - */ -@@ -284,6 +435,83 @@ int kbasep_platform_device_init(struct kbase_device *kbdev); - */ - void kbasep_platform_device_term(struct kbase_device *kbdev); - -+/** -+ * kbasep_platform_device_late_init: - Platform specific call to finish hardware -+ * initialization -+ * @kbdev: kbase device pointer -+ * -+ * Function calls a platform defined routine if specified in the configuration -+ * attributes. The routine can initialize any hardware and context state that -+ * is required for the GPU block to function. -+ * -+ * Return: 0 if no errors have been found in the config. -+ * Negative error code otherwise. -+ */ -+int kbasep_platform_device_late_init(struct kbase_device *kbdev); -+ -+/** -+ * kbasep_platform_device_late_term - Platform specific call to finish hardware -+ * termination -+ * @kbdev: Kbase device pointer -+ * -+ * Function calls a platform defined routine if specified in the configuration -+ * attributes. The routine can destroy any platform specific context state and -+ * shut down any hardware functionality that are outside of the Power Management -+ * callbacks. -+ * -+ */ -+void kbasep_platform_device_late_term(struct kbase_device *kbdev); -+ -+#if !MALI_USE_CSF -+/** -+ * kbasep_platform_context_init - Platform specific callback when a kernel -+ * context is created -+ * @kctx: kbase_context pointer -+ * -+ * Function calls a platform defined routine if specified in the configuration -+ * attributes. The routine can initialize any per kernel context structures -+ * that are required for the GPU block to function. -+ * -+ * Return: 0 if no errors were encountered. Negative error code otherwise. -+ */ -+int kbasep_platform_context_init(struct kbase_context *kctx); -+ -+/** -+ * kbasep_platform_context_term - Platform specific callback when a kernel -+ * context is terminated -+ * @kctx: kbase_context pointer -+ * -+ * Function calls a platform defined routine if specified in the configuration -+ * attributes. The routine should terminate any per kernel context structures -+ * created as part of &kbasep_platform_context_init. -+ * -+ */ -+void kbasep_platform_context_term(struct kbase_context *kctx); -+ -+/** -+ * kbasep_platform_event_atom_submit - Platform specific callback when an atom -+ * is submitted to the GPU -+ * @katom: kbase_jd_atom pointer -+ * -+ * Function calls a platform defined routine if specified in the configuration -+ * attributes. The routine should not assume that it is in a process context. -+ * -+ * Return: 0 if no errors were encountered. Negative error code otherwise. -+ */ -+void kbasep_platform_event_atom_submit(struct kbase_jd_atom *katom); -+ -+/** -+ * kbasep_platform_event_atom_complete - Platform specific callback when an atom -+ * has stopped running on the GPU -+ * @katom: kbase_jd_atom pointer -+ * -+ * Function calls a platform defined routine if specified in the configuration -+ * attributes. The routine should not assume that it is in a process context. -+ * -+ */ -+void kbasep_platform_event_atom_complete(struct kbase_jd_atom *katom); -+#endif -+ - #ifndef CONFIG_OF - /** - * kbase_platform_register - Register a platform device for the GPU -@@ -304,8 +532,4 @@ int kbase_platform_register(void); - void kbase_platform_unregister(void); - #endif - -- /** @} *//* end group kbase_config */ -- /** @} *//* end group base_kbase_api */ -- /** @} *//* end group base_api */ -- - #endif /* _KBASE_CONFIG_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h -index e079281..63c36e2 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_config_defaults.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2013-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2013-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,14 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -- * @file mali_kbase_config_defaults.h -- * -- * Default values for configuration settings -+ * DOC: Default values for configuration settings - * - */ - -@@ -88,29 +85,38 @@ enum { - }; - - /** -- * Default period for DVFS sampling -+ * Default period for DVFS sampling (can be overridden by platform header) - */ -+#ifndef DEFAULT_PM_DVFS_PERIOD - #define DEFAULT_PM_DVFS_PERIOD 100 /* 100ms */ -+#endif - - /** - * Power Management poweroff tick granuality. This is in nanoseconds to -- * allow HR timer support. -+ * allow HR timer support (can be overridden by platform header). - * - * On each scheduling tick, the power manager core may decide to: - * -# Power off one or more shader cores - * -# Power off the entire GPU - */ -+#ifndef DEFAULT_PM_GPU_POWEROFF_TICK_NS - #define DEFAULT_PM_GPU_POWEROFF_TICK_NS (400000) /* 400us */ -+#endif - - /** - * Power Manager number of ticks before shader cores are powered off -+ * (can be overridden by platform header). - */ -+#ifndef DEFAULT_PM_POWEROFF_TICK_SHADER - #define DEFAULT_PM_POWEROFF_TICK_SHADER (2) /* 400-800us */ -+#endif - - /** -- * Default scheduling tick granuality -+ * Default scheduling tick granuality (can be overridden by platform header) - */ -+#ifndef DEFAULT_JS_SCHEDULING_PERIOD_NS - #define DEFAULT_JS_SCHEDULING_PERIOD_NS (100000000u) /* 100ms */ -+#endif - - /** - * Default minimum number of scheduling ticks before jobs are soft-stopped. -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_core_linux.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_core_linux.c -index fb2353e..ce84219 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_core_linux.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_core_linux.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> -@@ -28,37 +27,39 @@ - #ifdef CONFIG_MALI_DEVFREQ - #include <linux/devfreq.h> - #include <backend/gpu/mali_kbase_devfreq.h> --#ifdef CONFIG_DEVFREQ_THERMAL -+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL) - #include <ipa/mali_kbase_ipa_debugfs.h> - #endif /* CONFIG_DEVFREQ_THERMAL */ - #endif /* CONFIG_MALI_DEVFREQ */ --#ifdef CONFIG_MALI_NO_MALI --#include "mali_kbase_model_linux.h" --#include <backend/gpu/mali_kbase_model_dummy.h> --#endif /* CONFIG_MALI_NO_MALI */ - #include "mali_kbase_mem_profile_debugfs_buf_size.h" --#include "mali_kbase_debug_mem_view.h" - #include "mali_kbase_mem.h" - #include "mali_kbase_mem_pool_debugfs.h" -+#include "mali_kbase_mem_pool_group.h" - #include "mali_kbase_debugfs_helper.h" --#if !MALI_CUSTOMER_RELEASE --#include "mali_kbase_regs_dump_debugfs.h" --#endif /* !MALI_CUSTOMER_RELEASE */ - #include "mali_kbase_regs_history_debugfs.h" - #include <mali_kbase_hwaccess_backend.h> - #include <mali_kbase_hwaccess_time.h> -+#if !MALI_USE_CSF - #include <mali_kbase_hwaccess_jm.h> --#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS -+#endif /* !MALI_USE_CSF */ -+#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS - #include <mali_kbase_hwaccess_instr.h> - #endif --#include <mali_kbase_ctx_sched.h> - #include <mali_kbase_reset_gpu.h> --#include <backend/gpu/mali_kbase_device_internal.h> --#include "mali_kbase_ioctl.h" -+#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h> -+#if !MALI_USE_CSF -+#include "mali_kbase_kinstr_jm.h" -+#endif - #include "mali_kbase_hwcnt_context.h" - #include "mali_kbase_hwcnt_virtualizer.h" - #include "mali_kbase_hwcnt_legacy.h" - #include "mali_kbase_vinstr.h" -+#if MALI_USE_CSF -+#include "csf/mali_kbase_csf_firmware.h" -+#include "csf/mali_kbase_csf_tiler_heap.h" -+#include "csf/mali_kbase_csf_csg_debugfs.h" -+#include "csf/mali_kbase_csf_cpu_queue_debugfs.h" -+#endif - #ifdef CONFIG_MALI_ARBITER_SUPPORT - #include "arbiter/mali_kbase_arbiter_pm.h" - #endif -@@ -68,7 +69,8 @@ - #ifdef CONFIG_MALI_CINSTR_GWT - #include "mali_kbase_gwt.h" - #endif --#include "mali_kbase_pm_internal.h" -+#include "backend/gpu/mali_kbase_pm_internal.h" -+#include "mali_kbase_dvfs_debugfs.h" - - #include <linux/module.h> - #include <linux/init.h> -@@ -99,13 +101,7 @@ - - #include <mali_kbase_config.h> - -- --#if (KERNEL_VERSION(3, 13, 0) <= LINUX_VERSION_CODE) - #include <linux/pm_opp.h> --#else --#include <linux/opp.h> --#endif -- - #include <linux/pm_runtime.h> - - #include <tl/mali_kbase_timeline.h> -@@ -114,6 +110,8 @@ - #include <device/mali_kbase_device.h> - #include <context/mali_kbase_context.h> - -+#include <mali_kbase_caps.h> -+ - /* GPU IRQ Tags */ - #define JOB_IRQ_TAG 0 - #define MMU_IRQ_TAG 1 -@@ -121,6 +119,72 @@ - - #define KERNEL_SIDE_DDK_VERSION_STRING "K:" MALI_RELEASE_NAME "(GPL)" - -+/** -+ * KBASE_API_VERSION - KBase API Version -+ * @major: Kernel major version -+ * @minor: Kernel minor version -+ */ -+#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ -+ (((minor) & 0xFFF) << 8) | \ -+ ((0 & 0xFF) << 0)) -+ -+#define KBASE_API_MIN(api_version) ((api_version >> 8) & 0xFFF) -+#define KBASE_API_MAJ(api_version) ((api_version >> 20) & 0xFFF) -+ -+/** -+ * typedef mali_kbase_capability_def - kbase capabilities table -+ */ -+typedef struct mali_kbase_capability_def { -+ u16 required_major; -+ u16 required_minor; -+} mali_kbase_capability_def; -+ -+/* -+ * This must be kept in-sync with mali_kbase_cap -+ * -+ * TODO: The alternative approach would be to embed the cap enum values -+ * in the table. Less efficient but potentially safer. -+ */ -+static mali_kbase_capability_def kbase_caps_table[MALI_KBASE_NUM_CAPS] = { -+#if MALI_USE_CSF -+ { 1, 0 }, /* SYSTEM_MONITOR */ -+ { 1, 0 }, /* JIT_PRESSURE_LIMIT */ -+ { 1, 0 }, /* MEM_GROW_ON_GPF */ -+ { 1, 0 } /* MEM_PROTECTED */ -+#else -+ { 11, 15 }, /* SYSTEM_MONITOR */ -+ { 11, 25 }, /* JIT_PRESSURE_LIMIT */ -+ { 11, 2 }, /* MEM_GROW_ON_GPF */ -+ { 11, 2 } /* MEM_PROTECTED */ -+#endif -+}; -+ -+/** -+ * mali_kbase_supports_cap - Query whether a kbase capability is supported -+ * -+ * @api_version: API version to convert -+ * @cap: Capability to query for - see mali_kbase_caps.h -+ */ -+bool mali_kbase_supports_cap(unsigned long api_version, mali_kbase_cap cap) -+{ -+ bool supported = false; -+ unsigned long required_ver; -+ -+ mali_kbase_capability_def const *cap_def; -+ -+ if (WARN_ON(cap < 0)) -+ return false; -+ -+ if (WARN_ON(cap >= MALI_KBASE_NUM_CAPS)) -+ return false; -+ -+ cap_def = &kbase_caps_table[(int)cap]; -+ required_ver = KBASE_API_VERSION(cap_def->required_major, cap_def->required_minor); -+ supported = (api_version >= required_ver); -+ -+ return supported; -+} -+ - /** - * kbase_file_new - Create an object representing a device file - * -@@ -152,7 +216,7 @@ static struct kbase_file *kbase_file_new(struct kbase_device *const kbdev, - } - - /** -- * kbase_file_get_api_version - Set the application programmer interface version -+ * kbase_file_set_api_version - Set the application programmer interface version - * - * @kfile: A device file created by kbase_file_new() - * @major: Major version number (must not exceed 12 bits) -@@ -271,7 +335,7 @@ static void kbase_file_delete(struct kbase_file *const kfile) - if (atomic_read(&kfile->setup_state) == KBASE_FILE_COMPLETE) { - struct kbase_context *kctx = kfile->kctx; - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - kbasep_mem_profile_debugfs_remove(kctx); - #endif - -@@ -326,31 +390,18 @@ static int kbase_api_handshake(struct kbase_file *kfile, - * the flags have been set. Originally it was created on file open - * (with job submission disabled) but we don't support that usage. - */ -- if (kbase_file_get_api_version(kfile) < KBASE_API_VERSION(11, 15)) -+ if (!mali_kbase_supports_system_monitor(kbase_file_get_api_version(kfile))) - err = kbase_file_create_kctx(kfile, - BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED); - - return err; - } - --/** -- * enum mali_error - Mali error codes shared with userspace -- * -- * This is subset of those common Mali errors that can be returned to userspace. -- * Values of matching user and kernel space enumerators MUST be the same. -- * MALI_ERROR_NONE is guaranteed to be 0. -- * -- * @MALI_ERROR_NONE: Success -- * @MALI_ERROR_OUT_OF_GPU_MEMORY: Not used in the kernel driver -- * @MALI_ERROR_OUT_OF_MEMORY: Memory allocation failure -- * @MALI_ERROR_FUNCTION_FAILED: Generic error code -- */ --enum mali_error { -- MALI_ERROR_NONE = 0, -- MALI_ERROR_OUT_OF_GPU_MEMORY, -- MALI_ERROR_OUT_OF_MEMORY, -- MALI_ERROR_FUNCTION_FAILED, --}; -+static int kbase_api_handshake_dummy(struct kbase_file *kfile, -+ struct kbase_ioctl_version_check *version) -+{ -+ return -EPERM; -+} - - static struct kbase_device *to_kbase_device(struct device *dev) - { -@@ -377,7 +428,7 @@ int assign_irqs(struct kbase_device *kbdev) - return -ENOENT; - } - --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - if (!strncasecmp(irq_res->name, "JOB", 4)) { - irqtag = JOB_IRQ_TAG; - } else if (!strncasecmp(irq_res->name, "MMU", 4)) { -@@ -428,10 +479,10 @@ void kbase_release_device(struct kbase_device *kbdev) - } - EXPORT_SYMBOL(kbase_release_device); - --#ifdef CONFIG_DEBUG_FS --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && \ -- !(LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 28) && \ -- LINUX_VERSION_CODE < KERNEL_VERSION(4, 5, 0)) -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && \ -+ !(KERNEL_VERSION(4, 4, 28) <= LINUX_VERSION_CODE && \ -+ KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE) - /* - * Older versions, before v4.6, of the kernel doesn't have - * kstrtobool_from_user(), except longterm 4.4.y which had it added in 4.4.28 -@@ -545,7 +596,7 @@ static int kbase_file_create_kctx(struct kbase_file *const kfile, - { - struct kbase_device *kbdev = NULL; - struct kbase_context *kctx = NULL; --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - char kctx_name[64]; - #endif - -@@ -576,9 +627,11 @@ static int kbase_file_create_kctx(struct kbase_file *const kfile, - if (kbdev->infinite_cache_active_default) - kbase_ctx_flag_set(kctx, KCTX_INFINITE_CACHE); - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - snprintf(kctx_name, 64, "%d_%d", kctx->tgid, kctx->id); - -+ mutex_init(&kctx->mem_profile_lock); -+ - kctx->kctx_dentry = debugfs_create_dir(kctx_name, - kbdev->debugfs_ctx_directory); - -@@ -599,8 +652,6 @@ static int kbase_file_create_kctx(struct kbase_file *const kfile, - debugfs_create_file("force_same_va", 0600, kctx->kctx_dentry, - kctx, &kbase_force_same_va_fops); - -- mutex_init(&kctx->mem_profile_lock); -- - kbase_context_debugfs_init(kctx); - } - #endif /* CONFIG_DEBUG_FS */ -@@ -612,7 +663,6 @@ static int kbase_file_create_kctx(struct kbase_file *const kfile, - - return 0; - } -- - static int kbase_open(struct inode *inode, struct file *filp) - { - struct kbase_device *kbdev = NULL; -@@ -624,6 +674,13 @@ static int kbase_open(struct inode *inode, struct file *filp) - if (!kbdev) - return -ENODEV; - -+ /* Device-wide firmware load is moved here from probing to comply with -+ * Android GKI vendor guideline. -+ */ -+ ret = kbase_device_firmware_init_once(kbdev); -+ if (ret) -+ goto out; -+ - kfile = kbase_file_new(kbdev, filp); - if (!kfile) { - ret = -ENOMEM; -@@ -635,7 +692,7 @@ static int kbase_open(struct inode *inode, struct file *filp) - - return 0; - -- out: -+out: - kbase_release_device(kbdev); - return ret; - } -@@ -663,11 +720,13 @@ static int kbase_api_set_flags(struct kbase_file *kfile, - /* For backward compatibility, the context may have been created before - * the flags were set. - */ -- if (api_version >= KBASE_API_VERSION(11, 15)) { -+ if (mali_kbase_supports_system_monitor(api_version)) { - err = kbase_file_create_kctx(kfile, flags->create_flags); - } else { -+#if !MALI_USE_CSF - struct kbasep_js_kctx_info *js_kctx_info = NULL; - unsigned long irq_flags = 0; -+#endif - - /* If setup is incomplete (e.g. because the API version - * wasn't set) then we have to give up. -@@ -676,6 +735,12 @@ static int kbase_api_set_flags(struct kbase_file *kfile, - if (unlikely(!kctx)) - return -EPERM; - -+#if MALI_USE_CSF -+ /* On CSF GPUs Job Manager interface isn't used to submit jobs -+ * (there are no job slots). So the legacy job manager path to -+ * submit jobs needs to remain disabled for CSF GPUs. -+ */ -+#else - js_kctx_info = &kctx->jctx.sched_info; - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags); -@@ -687,11 +752,13 @@ static int kbase_api_set_flags(struct kbase_file *kfile, - - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -+#endif - } - - return err; - } - -+#if !MALI_USE_CSF - static int kbase_api_job_submit(struct kbase_context *kctx, - struct kbase_ioctl_job_submit *submit) - { -@@ -699,6 +766,7 @@ static int kbase_api_job_submit(struct kbase_context *kctx, - submit->nr_atoms, - submit->stride, false); - } -+#endif /* !MALI_USE_CSF */ - - static int kbase_api_get_gpuprops(struct kbase_context *kctx, - struct kbase_ioctl_get_gpuprops *get_props) -@@ -724,11 +792,13 @@ static int kbase_api_get_gpuprops(struct kbase_context *kctx, - return kprops->prop_buffer_size; - } - -+#if !MALI_USE_CSF - static int kbase_api_post_term(struct kbase_context *kctx) - { - kbase_event_close(kctx); - return 0; - } -+#endif /* !MALI_USE_CSF */ - - static int kbase_api_mem_alloc(struct kbase_context *kctx, - union kbase_ioctl_mem_alloc *alloc) -@@ -762,11 +832,23 @@ static int kbase_api_mem_alloc(struct kbase_context *kctx, - flags |= BASE_MEM_SAME_VA; - } - -+#if MALI_USE_CSF -+ /* If CSF event memory allocation, need to force certain flags. -+ * SAME_VA - GPU address needs to be used as a CPU address, explicit -+ * mmap has to be avoided. -+ * CACHED_CPU - Frequent access to the event memory by CPU. -+ * COHERENT_SYSTEM - No explicit cache maintenance around the access -+ * to event memory so need to leverage the coherency support. -+ */ -+ if (flags & BASE_MEM_CSF_EVENT) { -+ flags |= (BASE_MEM_SAME_VA | -+ BASE_MEM_CACHED_CPU | -+ BASE_MEM_COHERENT_SYSTEM); -+ } -+#endif - -- reg = kbase_mem_alloc(kctx, alloc->in.va_pages, -- alloc->in.commit_pages, -- alloc->in.extent, -- &flags, &gpu_va); -+ reg = kbase_mem_alloc(kctx, alloc->in.va_pages, alloc->in.commit_pages, -+ alloc->in.extension, &flags, &gpu_va); - - if (!reg) - return -ENOMEM; -@@ -790,6 +872,14 @@ static int kbase_api_mem_free(struct kbase_context *kctx, - return kbase_mem_free(kctx, free->gpu_addr); - } - -+#if !MALI_USE_CSF -+static int kbase_api_kinstr_jm_fd(struct kbase_context *kctx, -+ union kbase_kinstr_jm_fd *arg) -+{ -+ return kbase_kinstr_jm_get_fd(kctx->kinstr_jm, arg); -+} -+#endif -+ - static int kbase_api_hwcnt_reader_setup(struct kbase_context *kctx, - struct kbase_ioctl_hwcnt_reader_setup *setup) - { -@@ -883,17 +973,6 @@ static int kbase_api_get_cpu_gpu_timeinfo(struct kbase_context *kctx, - return 0; - } - --#ifdef CONFIG_MALI_NO_MALI --static int kbase_api_hwcnt_set(struct kbase_context *kctx, -- struct kbase_ioctl_hwcnt_values *values) --{ -- gpu_model_set_dummy_prfcnt_sample( -- (u32 __user *)(uintptr_t)values->data, -- values->size); -- -- return 0; --} --#endif - - static int kbase_api_disjoint_query(struct kbase_context *kctx, - struct kbase_ioctl_disjoint_query *query) -@@ -1058,10 +1137,7 @@ static int kbase_api_mem_alias(struct kbase_context *kctx, - u64 flags; - int err; - -- if (alias->in.nents == 0 || alias->in.nents > 2048) -- return -EINVAL; -- -- if (alias->in.stride > (U64_MAX / 2048)) -+ if (alias->in.nents == 0 || alias->in.nents > BASE_MEM_ALIAS_MAX_ENTS) - return -EINVAL; - - ai = vmalloc(sizeof(*ai) * alias->in.nents); -@@ -1185,6 +1261,7 @@ static int kbase_api_mem_profile_add(struct kbase_context *kctx, - return kbasep_mem_profile_debugfs_insert(kctx, buf, data->len); - } - -+#if !MALI_USE_CSF - static int kbase_api_soft_event_update(struct kbase_context *kctx, - struct kbase_ioctl_soft_event_update *update) - { -@@ -1193,6 +1270,7 @@ static int kbase_api_soft_event_update(struct kbase_context *kctx, - - return kbase_soft_event_update(kctx, update->event, update->new_status); - } -+#endif /* !MALI_USE_CSF */ - - static int kbase_api_sticky_resource_map(struct kbase_context *kctx, - struct kbase_ioctl_sticky_resource_map *map) -@@ -1263,18 +1341,6 @@ static int kbase_api_sticky_resource_unmap(struct kbase_context *kctx, - } - - #if MALI_UNIT_TEST --static int kbase_api_tlstream_test(struct kbase_context *kctx, -- struct kbase_ioctl_tlstream_test *test) --{ -- kbase_timeline_test( -- kctx->kbdev, -- test->tpw_count, -- test->msg_delay, -- test->msg_count, -- test->aux_msg); -- -- return 0; --} - - static int kbase_api_tlstream_stats(struct kbase_context *kctx, - struct kbase_ioctl_tlstream_stats *stats) -@@ -1287,55 +1353,266 @@ static int kbase_api_tlstream_stats(struct kbase_context *kctx, - } - #endif /* MALI_UNIT_TEST */ - -+#if MALI_USE_CSF -+static int kbasep_cs_event_signal(struct kbase_context *kctx) -+{ -+ kbase_csf_event_signal_notify_gpu(kctx); -+ return 0; -+} -+ -+static int kbasep_cs_queue_register(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_register *reg) -+{ -+ kctx->jit_group_id = BASE_MEM_GROUP_DEFAULT; -+ -+ return kbase_csf_queue_register(kctx, reg); -+} -+ -+static int kbasep_cs_queue_register_ex(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_register_ex *reg) -+{ -+ kctx->jit_group_id = BASE_MEM_GROUP_DEFAULT; -+ -+ return kbase_csf_queue_register_ex(kctx, reg); -+} -+ -+static int kbasep_cs_queue_terminate(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_terminate *term) -+{ -+ kbase_csf_queue_terminate(kctx, term); -+ -+ return 0; -+} -+ -+static int kbasep_cs_queue_bind(struct kbase_context *kctx, -+ union kbase_ioctl_cs_queue_bind *bind) -+{ -+ return kbase_csf_queue_bind(kctx, bind); -+} -+ -+static int kbasep_cs_queue_kick(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_kick *kick) -+{ -+ return kbase_csf_queue_kick(kctx, kick); -+} -+ -+static int kbasep_cs_queue_group_create(struct kbase_context *kctx, -+ union kbase_ioctl_cs_queue_group_create *create) -+{ -+ return kbase_csf_queue_group_create(kctx, create); -+} -+ -+static int kbasep_cs_queue_group_terminate(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_queue_group_term *term) -+{ -+ kbase_csf_queue_group_terminate(kctx, term->group_handle); -+ -+ return 0; -+} -+ -+static int kbasep_kcpu_queue_new(struct kbase_context *kctx, -+ struct kbase_ioctl_kcpu_queue_new *new) -+{ -+ return kbase_csf_kcpu_queue_new(kctx, new); -+} -+ -+static int kbasep_kcpu_queue_delete(struct kbase_context *kctx, -+ struct kbase_ioctl_kcpu_queue_delete *delete) -+{ -+ return kbase_csf_kcpu_queue_delete(kctx, delete); -+} -+ -+static int kbasep_kcpu_queue_enqueue(struct kbase_context *kctx, -+ struct kbase_ioctl_kcpu_queue_enqueue *enqueue) -+{ -+ return kbase_csf_kcpu_queue_enqueue(kctx, enqueue); -+} -+ -+static int kbasep_cs_tiler_heap_init(struct kbase_context *kctx, -+ union kbase_ioctl_cs_tiler_heap_init *heap_init) -+{ -+ kctx->jit_group_id = heap_init->in.group_id; -+ -+ return kbase_csf_tiler_heap_init(kctx, heap_init->in.chunk_size, -+ heap_init->in.initial_chunks, heap_init->in.max_chunks, -+ heap_init->in.target_in_flight, -+ &heap_init->out.gpu_heap_va, &heap_init->out.first_chunk_va); -+} -+ -+static int kbasep_cs_tiler_heap_term(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_tiler_heap_term *heap_term) -+{ -+ return kbase_csf_tiler_heap_term(kctx, heap_term->gpu_heap_va); -+} -+ -+static int kbase_ioctl_cs_get_glb_iface(struct kbase_context *kctx, -+ union kbase_ioctl_cs_get_glb_iface *param) -+{ -+ struct basep_cs_stream_control *stream_data = NULL; -+ struct basep_cs_group_control *group_data = NULL; -+ void __user *user_groups, *user_streams; -+ int err = 0; -+ u32 const max_group_num = param->in.max_group_num; -+ u32 const max_total_stream_num = param->in.max_total_stream_num; -+ -+ if (max_group_num > MAX_SUPPORTED_CSGS) -+ return -EINVAL; -+ -+ if (max_total_stream_num > -+ MAX_SUPPORTED_CSGS * MAX_SUPPORTED_STREAMS_PER_GROUP) -+ return -EINVAL; -+ -+ user_groups = u64_to_user_ptr(param->in.groups_ptr); -+ user_streams = u64_to_user_ptr(param->in.streams_ptr); -+ -+ if (max_group_num > 0) { -+ if (!user_groups) -+ err = -EINVAL; -+ else { -+ group_data = kcalloc(max_group_num, -+ sizeof(*group_data), GFP_KERNEL); -+ if (!group_data) -+ err = -ENOMEM; -+ } -+ } -+ -+ if (max_total_stream_num > 0) { -+ if (!user_streams) -+ err = -EINVAL; -+ else { -+ stream_data = kcalloc(max_total_stream_num, -+ sizeof(*stream_data), GFP_KERNEL); -+ if (!stream_data) -+ err = -ENOMEM; -+ } -+ } -+ -+ if (!err) { -+ param->out.total_stream_num = kbase_csf_firmware_get_glb_iface( -+ kctx->kbdev, group_data, max_group_num, stream_data, -+ max_total_stream_num, ¶m->out.glb_version, -+ ¶m->out.features, ¶m->out.group_num, -+ ¶m->out.prfcnt_size, ¶m->out.instr_features); -+ -+ if (copy_to_user(user_groups, group_data, -+ MIN(max_group_num, param->out.group_num) * -+ sizeof(*group_data))) -+ err = -EFAULT; -+ } -+ -+ if (!err) -+ if (copy_to_user(user_streams, stream_data, -+ MIN(max_total_stream_num, param->out.total_stream_num) * -+ sizeof(*stream_data))) -+ err = -EFAULT; -+ -+ kfree(group_data); -+ kfree(stream_data); -+ return err; -+} -+ -+static int kbasep_ioctl_cs_cpu_queue_dump(struct kbase_context *kctx, -+ struct kbase_ioctl_cs_cpu_queue_info *cpu_queue_info) -+{ -+ return kbase_csf_cpu_queue_dump(kctx, cpu_queue_info->buffer, -+ cpu_queue_info->size); -+} -+ -+#endif /* MALI_USE_CSF */ -+ -+static int kbasep_ioctl_context_priority_check(struct kbase_context *kctx, -+ struct kbase_ioctl_context_priority_check *priority_check) -+{ -+#if MALI_USE_CSF -+ priority_check->priority = kbase_csf_priority_check(kctx->kbdev, priority_check->priority); -+#else -+ base_jd_prio req_priority = (base_jd_prio)priority_check->priority; -+ -+ priority_check->priority = (u8)kbase_js_priority_check(kctx->kbdev, req_priority); -+#endif -+ return 0; -+} - --#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ -- do { \ -- BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ -- return function(arg); \ -+#define KBASE_HANDLE_IOCTL(cmd, function, arg) \ -+ do { \ -+ int ret; \ -+ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_NONE); \ -+ dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ -+ ret = function(arg); \ -+ dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, \ -+ #function); \ -+ return ret; \ - } while (0) - --#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ -- do { \ -- type param; \ -- int err; \ -- BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ -- BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ -- err = copy_from_user(¶m, uarg, sizeof(param)); \ -- if (err) \ -- return -EFAULT; \ -- return function(arg, ¶m); \ -+#define KBASE_HANDLE_IOCTL_IN(cmd, function, type, arg) \ -+ do { \ -+ type param; \ -+ int ret, err; \ -+ dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ -+ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_WRITE); \ -+ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ -+ err = copy_from_user(¶m, uarg, sizeof(param)); \ -+ if (err) \ -+ return -EFAULT; \ -+ ret = function(arg, ¶m); \ -+ dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, \ -+ #function); \ -+ return ret; \ - } while (0) - --#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ -- do { \ -- type param; \ -- int ret, err; \ -- BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ -- BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ -- memset(¶m, 0, sizeof(param)); \ -- ret = function(arg, ¶m); \ -- err = copy_to_user(uarg, ¶m, sizeof(param)); \ -- if (err) \ -- return -EFAULT; \ -- return ret; \ -+#define KBASE_HANDLE_IOCTL_OUT(cmd, function, type, arg) \ -+ do { \ -+ type param; \ -+ int ret, err; \ -+ dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ -+ BUILD_BUG_ON(_IOC_DIR(cmd) != _IOC_READ); \ -+ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ -+ memset(¶m, 0, sizeof(param)); \ -+ ret = function(arg, ¶m); \ -+ err = copy_to_user(uarg, ¶m, sizeof(param)); \ -+ if (err) \ -+ return -EFAULT; \ -+ dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, \ -+ #function); \ -+ return ret; \ - } while (0) - --#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ -- do { \ -- type param; \ -- int ret, err; \ -- BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE|_IOC_READ)); \ -- BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ -- err = copy_from_user(¶m, uarg, sizeof(param)); \ -- if (err) \ -- return -EFAULT; \ -- ret = function(arg, ¶m); \ -- err = copy_to_user(uarg, ¶m, sizeof(param)); \ -- if (err) \ -- return -EFAULT; \ -- return ret; \ -+#define KBASE_HANDLE_IOCTL_INOUT(cmd, function, type, arg) \ -+ do { \ -+ type param; \ -+ int ret, err; \ -+ dev_dbg(arg->kbdev->dev, "Enter ioctl %s\n", #function); \ -+ BUILD_BUG_ON(_IOC_DIR(cmd) != (_IOC_WRITE | _IOC_READ)); \ -+ BUILD_BUG_ON(sizeof(param) != _IOC_SIZE(cmd)); \ -+ err = copy_from_user(¶m, uarg, sizeof(param)); \ -+ if (err) \ -+ return -EFAULT; \ -+ ret = function(arg, ¶m); \ -+ err = copy_to_user(uarg, ¶m, sizeof(param)); \ -+ if (err) \ -+ return -EFAULT; \ -+ dev_dbg(arg->kbdev->dev, "Return %d from ioctl %s\n", ret, \ -+ #function); \ -+ return ret; \ - } while (0) - -+static int kbasep_ioctl_set_limited_core_count(struct kbase_context *kctx, -+ struct kbase_ioctl_set_limited_core_count *set_limited_core_count) -+{ -+ const u64 shader_core_mask = -+ kbase_pm_get_present_cores(kctx->kbdev, KBASE_PM_CORE_SHADER); -+ const u64 limited_core_mask = -+ ((u64)1 << (set_limited_core_count->max_core_count)) - 1; -+ -+ if ((shader_core_mask & limited_core_mask) == 0) { -+ /* At least one shader core must be available after applying the mask */ -+ return -EINVAL; -+ } -+ -+ kctx->limited_core_mask = limited_core_mask; -+ return 0; -+} -+ - static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) - { - struct kbase_file *const kfile = filp->private_data; -@@ -1352,6 +1629,13 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) - kfile); - break; - -+ case KBASE_IOCTL_VERSION_CHECK_RESERVED: -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_VERSION_CHECK_RESERVED, -+ kbase_api_handshake_dummy, -+ struct kbase_ioctl_version_check, -+ kfile); -+ break; -+ - case KBASE_IOCTL_SET_FLAGS: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_FLAGS, - kbase_api_set_flags, -@@ -1366,23 +1650,27 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) - - /* Normal ioctls */ - switch (cmd) { -+#if !MALI_USE_CSF - case KBASE_IOCTL_JOB_SUBMIT: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_JOB_SUBMIT, - kbase_api_job_submit, - struct kbase_ioctl_job_submit, - kctx); - break; -+#endif /* !MALI_USE_CSF */ - case KBASE_IOCTL_GET_GPUPROPS: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_GET_GPUPROPS, - kbase_api_get_gpuprops, - struct kbase_ioctl_get_gpuprops, - kctx); - break; -+#if !MALI_USE_CSF - case KBASE_IOCTL_POST_TERM: - KBASE_HANDLE_IOCTL(KBASE_IOCTL_POST_TERM, - kbase_api_post_term, - kctx); - break; -+#endif /* !MALI_USE_CSF */ - case KBASE_IOCTL_MEM_ALLOC: - KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_MEM_ALLOC, - kbase_api_mem_alloc, -@@ -1515,12 +1803,14 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) - kctx); - break; - -+#if !MALI_USE_CSF - case KBASE_IOCTL_SOFT_EVENT_UPDATE: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SOFT_EVENT_UPDATE, - kbase_api_soft_event_update, - struct kbase_ioctl_soft_event_update, - kctx); - break; -+#endif /* !MALI_USE_CSF */ - - case KBASE_IOCTL_STICKY_RESOURCE_MAP: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_STICKY_RESOURCE_MAP, -@@ -1536,6 +1826,14 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) - break; - - /* Instrumentation. */ -+#if !MALI_USE_CSF -+ case KBASE_IOCTL_KINSTR_JM_FD: -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_KINSTR_JM_FD, -+ kbase_api_kinstr_jm_fd, -+ union kbase_kinstr_jm_fd, -+ kctx); -+ break; -+#endif - case KBASE_IOCTL_HWCNT_READER_SETUP: - KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_READER_SETUP, - kbase_api_hwcnt_reader_setup, -@@ -1564,14 +1862,6 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) - union kbase_ioctl_get_cpu_gpu_timeinfo, - kctx); - break; --#ifdef CONFIG_MALI_NO_MALI -- case KBASE_IOCTL_HWCNT_SET: -- KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_HWCNT_SET, -- kbase_api_hwcnt_set, -- struct kbase_ioctl_hwcnt_values, -- kctx); -- break; --#endif - #ifdef CONFIG_MALI_CINSTR_GWT - case KBASE_IOCTL_CINSTR_GWT_START: - KBASE_HANDLE_IOCTL(KBASE_IOCTL_CINSTR_GWT_START, -@@ -1590,13 +1880,98 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) - kctx); - break; - #endif --#if MALI_UNIT_TEST -- case KBASE_IOCTL_TLSTREAM_TEST: -- KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_TLSTREAM_TEST, -- kbase_api_tlstream_test, -- struct kbase_ioctl_tlstream_test, -+#if MALI_USE_CSF -+ case KBASE_IOCTL_CS_EVENT_SIGNAL: -+ KBASE_HANDLE_IOCTL(KBASE_IOCTL_CS_EVENT_SIGNAL, -+ kbasep_cs_event_signal, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_QUEUE_REGISTER: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_REGISTER, -+ kbasep_cs_queue_register, -+ struct kbase_ioctl_cs_queue_register, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_QUEUE_REGISTER_EX: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_REGISTER_EX, -+ kbasep_cs_queue_register_ex, -+ struct kbase_ioctl_cs_queue_register_ex, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_QUEUE_TERMINATE: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_TERMINATE, -+ kbasep_cs_queue_terminate, -+ struct kbase_ioctl_cs_queue_terminate, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_QUEUE_BIND: -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_QUEUE_BIND, -+ kbasep_cs_queue_bind, -+ union kbase_ioctl_cs_queue_bind, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_QUEUE_KICK: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_KICK, -+ kbasep_cs_queue_kick, -+ struct kbase_ioctl_cs_queue_kick, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_QUEUE_GROUP_CREATE: -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_QUEUE_GROUP_CREATE, -+ kbasep_cs_queue_group_create, -+ union kbase_ioctl_cs_queue_group_create, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE, -+ kbasep_cs_queue_group_terminate, -+ struct kbase_ioctl_cs_queue_group_term, -+ kctx); -+ break; -+ case KBASE_IOCTL_KCPU_QUEUE_CREATE: -+ KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_KCPU_QUEUE_CREATE, -+ kbasep_kcpu_queue_new, -+ struct kbase_ioctl_kcpu_queue_new, -+ kctx); -+ break; -+ case KBASE_IOCTL_KCPU_QUEUE_DELETE: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_KCPU_QUEUE_DELETE, -+ kbasep_kcpu_queue_delete, -+ struct kbase_ioctl_kcpu_queue_delete, -+ kctx); -+ break; -+ case KBASE_IOCTL_KCPU_QUEUE_ENQUEUE: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_KCPU_QUEUE_ENQUEUE, -+ kbasep_kcpu_queue_enqueue, -+ struct kbase_ioctl_kcpu_queue_enqueue, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_TILER_HEAP_INIT: -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_TILER_HEAP_INIT, -+ kbasep_cs_tiler_heap_init, -+ union kbase_ioctl_cs_tiler_heap_init, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_TILER_HEAP_TERM: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_TILER_HEAP_TERM, -+ kbasep_cs_tiler_heap_term, -+ struct kbase_ioctl_cs_tiler_heap_term, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_GET_GLB_IFACE: -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CS_GET_GLB_IFACE, -+ kbase_ioctl_cs_get_glb_iface, -+ union kbase_ioctl_cs_get_glb_iface, -+ kctx); -+ break; -+ case KBASE_IOCTL_CS_CPU_QUEUE_DUMP: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_CS_CPU_QUEUE_DUMP, -+ kbasep_ioctl_cs_cpu_queue_dump, -+ struct kbase_ioctl_cs_cpu_queue_info, - kctx); - break; -+#endif /* MALI_USE_CSF */ -+#if MALI_UNIT_TEST - case KBASE_IOCTL_TLSTREAM_STATS: - KBASE_HANDLE_IOCTL_OUT(KBASE_IOCTL_TLSTREAM_STATS, - kbase_api_tlstream_stats, -@@ -1604,6 +1979,18 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) - kctx); - break; - #endif /* MALI_UNIT_TEST */ -+ case KBASE_IOCTL_CONTEXT_PRIORITY_CHECK: -+ KBASE_HANDLE_IOCTL_INOUT(KBASE_IOCTL_CONTEXT_PRIORITY_CHECK, -+ kbasep_ioctl_context_priority_check, -+ struct kbase_ioctl_context_priority_check, -+ kctx); -+ break; -+ case KBASE_IOCTL_SET_LIMITED_CORE_COUNT: -+ KBASE_HANDLE_IOCTL_IN(KBASE_IOCTL_SET_LIMITED_CORE_COUNT, -+ kbasep_ioctl_set_limited_core_count, -+ struct kbase_ioctl_set_limited_core_count, -+ kctx); -+ break; - } - - dev_warn(kbdev->dev, "Unknown ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); -@@ -1611,6 +1998,51 @@ static long kbase_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) - return -ENOIOCTLCMD; - } - -+#if MALI_USE_CSF -+static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) -+{ -+ struct kbase_file *const kfile = filp->private_data; -+ struct kbase_context *const kctx = -+ kbase_file_get_kctx_if_setup_complete(kfile); -+ struct base_csf_notification event_data = { -+ .type = BASE_CSF_NOTIFICATION_EVENT }; -+ const size_t data_size = sizeof(event_data); -+ bool read_event = false, read_error = false; -+ -+ if (unlikely(!kctx)) -+ return -EPERM; -+ -+ if (atomic_read(&kctx->event_count)) -+ read_event = true; -+ else -+ read_error = kbase_csf_read_error(kctx, &event_data); -+ -+ if (!read_event && !read_error) { -+ bool dump = kbase_csf_cpu_queue_read_dump_req(kctx, -+ &event_data); -+ /* This condition is not treated as an error. -+ * It is possible that event handling thread was woken up due -+ * to a fault/error that occurred for a queue group, but before -+ * the corresponding fault data was read by the thread the -+ * queue group was already terminated by the userspace. -+ */ -+ if (!dump) -+ dev_dbg(kctx->kbdev->dev, -+ "Neither event nor error signaled"); -+ } -+ -+ if (copy_to_user(buf, &event_data, data_size) != 0) { -+ dev_warn(kctx->kbdev->dev, -+ "Failed to copy data\n"); -+ return -EFAULT; -+ } -+ -+ if (read_event) -+ atomic_set(&kctx->event_count, 0); -+ -+ return data_size; -+} -+#else /* MALI_USE_CSF */ - static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos) - { - struct kbase_file *const kfile = filp->private_data; -@@ -1654,6 +2086,7 @@ static ssize_t kbase_read(struct file *filp, char __user *buf, size_t count, lof - out: - return out_count * sizeof(uevent); - } -+#endif /* MALI_USE_CSF */ - - static unsigned int kbase_poll(struct file *filp, poll_table *wait) - { -@@ -1674,19 +2107,31 @@ static unsigned int kbase_poll(struct file *filp, poll_table *wait) - void kbase_event_wakeup(struct kbase_context *kctx) - { - KBASE_DEBUG_ASSERT(kctx); -- -+ dev_dbg(kctx->kbdev->dev, "Waking event queue for context %pK\n", -+ (void *)kctx); - wake_up_interruptible(&kctx->event_queue); - } - - KBASE_EXPORT_TEST_API(kbase_event_wakeup); - -+#if MALI_USE_CSF -+int kbase_event_pending(struct kbase_context *ctx) -+{ -+ WARN_ON_ONCE(!ctx); -+ -+ return (atomic_read(&ctx->event_count) != 0) || -+ kbase_csf_error_pending(ctx) || -+ kbase_csf_cpu_queue_dump_needed(ctx); -+} -+#else - int kbase_event_pending(struct kbase_context *ctx) - { - KBASE_DEBUG_ASSERT(ctx); - - return (atomic_read(&ctx->event_count) != 0) || -- (atomic_read(&ctx->event_closed) != 0); -+ (atomic_read(&ctx->event_closed) != 0); - } -+#endif - - KBASE_EXPORT_TEST_API(kbase_event_pending); - -@@ -1740,42 +2185,166 @@ static const struct file_operations kbase_fops = { - .get_unmapped_area = kbase_get_unmapped_area, - }; - --/** -- * show_policy - Show callback for the power_policy sysfs file. -- * -- * This function is called to get the contents of the power_policy sysfs -- * file. This is a list of the available policies with the currently active one -- * surrounded by square brackets. -- * -- * @dev: The device this sysfs file is for -- * @attr: The attributes of the sysfs file -- * @buf: The output buffer for the sysfs file contents -- * -- * Return: The number of bytes output to @buf. -- */ --static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf) -+static ssize_t show_gpu_memory(struct device *dev, struct device_attribute *attr, char * const buf) - { - struct kbase_device *kbdev; -- const struct kbase_pm_policy *current_policy; -- const struct kbase_pm_policy *const *policy_list; -- int policy_count; -- int i; - ssize_t ret = 0; -+ struct list_head *entry; -+ const struct list_head *kbdev_list; - - kbdev = to_kbase_device(dev); - - if (!kbdev) - return -ENODEV; - -- current_policy = kbase_pm_get_policy(kbdev); -+ kbdev_list = kbase_device_get_list(); -+ list_for_each(entry, kbdev_list) { -+ struct kbase_device *kbdev = NULL; -+ struct kbase_context *kctx; -+ -+ kbdev = list_entry(entry, struct kbase_device, entry); -+ /* output the total memory usage and cap for this device */ -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "%-16s %-16s %10u\n", -+ kbdev->devname, -+ "total used_pages", -+ atomic_read(&(kbdev->memdev.used_pages))); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "----------------------------------------------------\n"); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "%-16s %-16s %-16s\n", -+ "kctx", "pid", "used_pages"); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "----------------------------------------------------\n"); -+ mutex_lock(&kbdev->kctx_list_lock); -+ list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { -+ /* output the memory usage and cap for each kctx -+ * opened on this device */ -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "%p %10u %10u\n", -+ kctx, -+ kctx->tgid, -+ atomic_read(&(kctx->used_pages))); -+ } -+ mutex_unlock(&kbdev->kctx_list_lock); -+ } - -- policy_count = kbase_pm_list_policies(kbdev, &policy_list); -+ kbase_device_put_list(kbdev_list); - -- for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { -- if (policy_list[i] == current_policy) -- ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); -- else -- ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); -+ -+ return ret; -+} -+ -+static ssize_t set_gpu_memory(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ ssize_t err = count; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ return err; -+} -+ -+static DEVICE_ATTR(gpu_memory, S_IRUGO | S_IWUSR, show_gpu_memory, set_gpu_memory); -+ -+static ssize_t show_ctx_mem_pool_size(struct device *dev, struct device_attribute *attr, char * const buf) -+{ -+ struct list_head *entry; -+ const struct list_head *kbdev_list; -+ ssize_t ret = 0; -+ int i = 0; -+ struct kbase_device *const kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ kbdev_list = kbase_device_get_list(); -+ list_for_each(entry, kbdev_list) { -+ struct kbase_device *kbdev = NULL; -+ struct kbase_context *kctx; -+ -+ kbdev = list_entry(entry, struct kbase_device, entry); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "%-16s %-16s %-16s\n", -+ "kctx", "pid", "cached_pages"); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "----------------------------------------------------\n"); -+ mutex_lock(&kbdev->kctx_list_lock); -+ list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { -+ /* output the memory cached and cap for each kctx -+ * opened on this device */ -+ unsigned long cached_mem = 0; -+ for (i = 0; i < MEMORY_GROUP_MANAGER_NR_GROUPS; i++) -+ //pr_info("[%d]:kctx->mem_pools.small[%d] = %d", kctx->tgid, i, kctx->mem_pools.small[i].cur_size); -+ cached_mem += kctx->mem_pools.small[i].cur_size; -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "%p %10u %10lu\n", -+ kctx, -+ kctx->tgid, -+ cached_mem); -+ } -+ mutex_unlock(&kbdev->kctx_list_lock); -+ } -+ -+ kbase_device_put_list(kbdev_list); -+ -+ return ret; -+} -+ -+static ssize_t set_ctx_mem_pool_size(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ ssize_t err = count; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ return err; -+} -+ -+static DEVICE_ATTR(ctx_mem_pool_size, S_IRUGO | S_IWUSR, show_ctx_mem_pool_size, set_ctx_mem_pool_size); -+ -+/** -+ * show_policy - Show callback for the power_policy sysfs file. -+ * -+ * This function is called to get the contents of the power_policy sysfs -+ * file. This is a list of the available policies with the currently active one -+ * surrounded by square brackets. -+ * -+ * @dev: The device this sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The output buffer for the sysfs file contents -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_policy(struct device *dev, struct device_attribute *attr, char *const buf) -+{ -+ struct kbase_device *kbdev; -+ const struct kbase_pm_policy *current_policy; -+ const struct kbase_pm_policy *const *policy_list; -+ int policy_count; -+ int i; -+ ssize_t ret = 0; -+ -+ kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ current_policy = kbase_pm_get_policy(kbdev); -+ -+ policy_count = kbase_pm_list_policies(kbdev, &policy_list); -+ -+ for (i = 0; i < policy_count && ret < PAGE_SIZE; i++) { -+ if (policy_list[i] == current_policy) -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s] ", policy_list[i]->name); -+ else -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", policy_list[i]->name); - } - - if (ret < PAGE_SIZE - 1) { -@@ -1800,7 +2369,7 @@ static ssize_t show_policy(struct device *dev, struct device_attribute *attr, ch - * @dev: The device with sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file -- * @count: The number of bytes written to the sysfs file -+ * @count: The number of bytes to write to the sysfs file - * - * Return: @count if the function succeeded. An error code on failure. - */ -@@ -1859,6 +2428,7 @@ static DEVICE_ATTR(power_policy, S_IRUGO | S_IWUSR, show_policy, set_policy); - static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, char * const buf) - { - struct kbase_device *kbdev; -+ unsigned long flags; - ssize_t ret = 0; - - kbdev = to_kbase_device(dev); -@@ -1866,6 +2436,19 @@ static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, - if (!kbdev) - return -ENODEV; - -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+#if MALI_USE_CSF -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "Current debug core mask : 0x%llX\n", -+ kbdev->pm.debug_core_mask); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "Current desired core mask : 0x%llX\n", -+ kbase_pm_ca_get_core_mask(kbdev)); -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, -+ "Current in use core mask : 0x%llX\n", -+ kbdev->pm.backend.shaders_avail); -+#else - ret += scnprintf(buf + ret, PAGE_SIZE - ret, - "Current core mask (JS0) : 0x%llX\n", - kbdev->pm.debug_core_mask[0]); -@@ -1875,10 +2458,14 @@ static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, - ret += scnprintf(buf + ret, PAGE_SIZE - ret, - "Current core mask (JS2) : 0x%llX\n", - kbdev->pm.debug_core_mask[2]); -+#endif /* MALI_USE_CSF */ -+ - ret += scnprintf(buf + ret, PAGE_SIZE - ret, - "Available core mask : 0x%llX\n", - kbdev->gpu_props.props.raw_props.shader_present); - -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ - return ret; - } - -@@ -1890,24 +2477,42 @@ static ssize_t show_core_mask(struct device *dev, struct device_attribute *attr, - * @dev: The device with sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file -- * @count: The number of bytes written to the sysfs file -+ * @count: The number of bytes to write to the sysfs file - * - * Return: @count if the function succeeded. An error code on failure. - */ - static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) - { - struct kbase_device *kbdev; -+#if MALI_USE_CSF -+ u64 new_core_mask; -+#else - u64 new_core_mask[3]; -- int items, i; -+ u64 group0_core_mask; -+ int i; -+#endif /* MALI_USE_CSF */ -+ -+ int items; - ssize_t err = count; - unsigned long flags; -- u64 shader_present, group0_core_mask; -+ u64 shader_present; - - kbdev = to_kbase_device(dev); - - if (!kbdev) - return -ENODEV; - -+#if MALI_USE_CSF -+ items = sscanf(buf, "%llx", &new_core_mask); -+ -+ if (items != 1) { -+ dev_err(kbdev->dev, -+ "Couldn't process core mask write operation.\n" -+ "Use format <core_mask>\n"); -+ err = -EINVAL; -+ goto end; -+ } -+#else - items = sscanf(buf, "%llx %llx %llx", - &new_core_mask[0], &new_core_mask[1], - &new_core_mask[2]); -@@ -1922,11 +2527,35 @@ static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, - - if (items == 1) - new_core_mask[1] = new_core_mask[2] = new_core_mask[0]; -+#endif - - mutex_lock(&kbdev->pm.lock); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - shader_present = kbdev->gpu_props.props.raw_props.shader_present; -+ -+#if MALI_USE_CSF -+ if ((new_core_mask & shader_present) != new_core_mask) { -+ dev_err(dev, -+ "Invalid core mask 0x%llX: Includes non-existent cores (present = 0x%llX)", -+ new_core_mask, shader_present); -+ err = -EINVAL; -+ goto unlock; -+ -+ } else if (!(new_core_mask & shader_present & -+ kbdev->pm.backend.ca_cores_enabled)) { -+ dev_err(dev, -+ "Invalid core mask 0x%llX: No intersection with currently available cores (present = 0x%llX, CA enabled = 0x%llX\n", -+ new_core_mask, -+ kbdev->gpu_props.props.raw_props.shader_present, -+ kbdev->pm.backend.ca_cores_enabled); -+ err = -EINVAL; -+ goto unlock; -+ } -+ -+ if (kbdev->pm.debug_core_mask != new_core_mask) -+ kbase_pm_set_debug_core_mask(kbdev, new_core_mask); -+#else - group0_core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask; - - for (i = 0; i < 3; ++i) { -@@ -1949,6 +2578,11 @@ static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, - new_core_mask[i], i, group0_core_mask); - err = -EINVAL; - goto unlock; -+ } else if (!(new_core_mask[i] & kbdev->gpu_props.curr_config.shader_present)) { -+ dev_err(dev, "Invalid core mask 0x%llX for JS %d: No intersection with current core mask 0x%llX\n", -+ new_core_mask[i], i, kbdev->gpu_props.curr_config.shader_present); -+ err = -EINVAL; -+ goto unlock; - } - } - -@@ -1961,6 +2595,7 @@ static ssize_t set_core_mask(struct device *dev, struct device_attribute *attr, - kbase_pm_set_debug_core_mask(kbdev, new_core_mask[0], - new_core_mask[1], new_core_mask[2]); - } -+#endif /* MALI_USE_CSF */ - - unlock: - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -@@ -1978,6 +2613,7 @@ end: - */ - static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask); - -+#if !MALI_USE_CSF - /** - * set_soft_job_timeout - Store callback for the soft_job_timeout sysfs - * file. -@@ -1985,7 +2621,7 @@ static DEVICE_ATTR(core_mask, S_IRUGO | S_IWUSR, show_core_mask, set_core_mask); - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. - * @buf: The value written to the sysfs file. -- * @count: The number of bytes written to the sysfs file. -+ * @count: The number of bytes to write to the sysfs file. - * - * This allows setting the timeout for software jobs. Waiting soft event wait - * jobs will be cancelled after this period expires, while soft fence wait jobs -@@ -2078,7 +2714,7 @@ static u32 timeout_ms_to_ticks(struct kbase_device *kbdev, long timeout_ms, - * @dev: The device with sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file -- * @count: The number of bytes written to the sysfs file -+ * @count: The number of bytes to write to the sysfs file - * - * Return: @count if the function succeeded. An error code on failure. - */ -@@ -2255,7 +2891,7 @@ static u32 get_new_js_timeout( - * @dev: The device the sysfs file is for - * @attr: The attributes of the sysfs file - * @buf: The value written to the sysfs file -- * @count: The number of bytes written to the sysfs file -+ * @count: The number of bytes to write to the sysfs file - * - * This function is called when the js_scheduling_period sysfs file is written - * to. It checks the data written, and if valid updates the js_scheduling_period -@@ -2295,7 +2931,8 @@ static ssize_t set_js_scheduling_period(struct device *dev, - - /* If no contexts have been scheduled since js_timeouts was last written - * to, the new timeouts might not have been latched yet. So check if an -- * update is pending and use the new values if necessary. */ -+ * update is pending and use the new values if necessary. -+ */ - - /* Use previous 'new' scheduling period as a base if present. */ - old_period = js_data->scheduling_period_ns; -@@ -2420,9 +3057,10 @@ static ssize_t show_js_softstop_always(struct device *dev, - */ - static DEVICE_ATTR(js_softstop_always, S_IRUGO | S_IWUSR, show_js_softstop_always, set_js_softstop_always); - #endif /* CONFIG_MALI_DEBUG */ -+#endif /* !MALI_USE_CSF */ - - #ifdef CONFIG_MALI_DEBUG --typedef void (kbasep_debug_command_func) (struct kbase_device *); -+typedef void kbasep_debug_command_func(struct kbase_device *); - - enum kbasep_debug_command_code { - KBASEP_DEBUG_COMMAND_DUMPTRACE, -@@ -2568,24 +3206,20 @@ static ssize_t kbase_show_gpuinfo(struct device *dev, - .name = "Mali-G77" }, - { .id = GPU_ID2_PRODUCT_TBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, - .name = "Mali-G78" }, -+ { .id = GPU_ID2_PRODUCT_TBAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -+ .name = "Mali-G78AE" }, - { .id = GPU_ID2_PRODUCT_LBEX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, - .name = "Mali-G68" }, - { .id = GPU_ID2_PRODUCT_TNAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, - .name = "Mali-G57" }, - { .id = GPU_ID2_PRODUCT_TODX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -- .name = "Mali-TODX" }, -+ .name = "Mali-G710" }, -+ { .id = GPU_ID2_PRODUCT_LODX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -+ .name = "Mali-G610" }, - { .id = GPU_ID2_PRODUCT_TGRX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -- .name = "Mali-TGRX" }, -+ .name = "Mali-G510" }, - { .id = GPU_ID2_PRODUCT_TVAX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -- .name = "Mali-TVAX" }, -- { .id = GPU_ID2_PRODUCT_LODX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -- .name = "Mali-LODX" }, -- { .id = GPU_ID2_PRODUCT_TTUX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -- .name = "Mali-TTUX" }, -- { .id = GPU_ID2_PRODUCT_LTUX >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -- .name = "Mali-LTUX" }, -- { .id = GPU_ID2_PRODUCT_TE2X >> GPU_ID_VERSION_PRODUCT_ID_SHIFT, -- .name = "Mali-TE2X" }, -+ .name = "Mali-G310" }, - }; - const char *product_name = "(Unknown Mali GPU)"; - struct kbase_device *kbdev; -@@ -2728,7 +3362,8 @@ static ssize_t set_pm_poweroff(struct device *dev, - - stt = &kbdev->pm.backend.shader_tick_timer; - stt->configured_interval = HR_TIMER_DELAY_NSEC(gpu_poweroff_time); -- stt->configured_ticks = poweroff_shader_ticks; -+ stt->default_ticks = poweroff_shader_ticks; -+ stt->configured_ticks = stt->default_ticks; - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -@@ -2766,7 +3401,7 @@ static ssize_t show_pm_poweroff(struct device *dev, - stt = &kbdev->pm.backend.shader_tick_timer; - ret = scnprintf(buf, PAGE_SIZE, "%llu %u 0\n", - ktime_to_ns(stt->configured_interval), -- stt->configured_ticks); -+ stt->default_ticks); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - -@@ -2776,6 +3411,75 @@ static ssize_t show_pm_poweroff(struct device *dev, - static DEVICE_ATTR(pm_poweroff, S_IRUGO | S_IWUSR, show_pm_poweroff, - set_pm_poweroff); - -+#if MALI_USE_CSF -+/** -+ * set_idle_hysteresis_time - Store callback for CSF idle_hysteresis_time -+ * sysfs file. -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * This function is called when the idle_hysteresis_time sysfs file is -+ * written to. -+ * -+ * This file contains values of the idle idle hysteresis duration. -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_idle_hysteresis_time(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ u32 dur; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ if (kstrtou32(buf, 0, &dur)) { -+ dev_err(kbdev->dev, "Couldn't process idle_hysteresis_time write operation.\n" -+ "Use format <idle_hysteresis_time>\n"); -+ return -EINVAL; -+ } -+ -+ kbase_csf_firmware_set_gpu_idle_hysteresis_time(kbdev, dur); -+ -+ return count; -+} -+ -+/** -+ * show_idle_hysteresis_time - Show callback for CSF idle_hysteresis_time -+ * sysfs entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the GPU information. -+ * -+ * This function is called to get the current idle hysteresis duration in ms. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_idle_hysteresis_time(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ u32 dur; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ dur = kbase_csf_firmware_get_gpu_idle_hysteresis_time(kbdev); -+ ret = scnprintf(buf, PAGE_SIZE, "%u\n", dur); -+ -+ return ret; -+} -+ -+static DEVICE_ATTR(idle_hysteresis_time, S_IRUGO | S_IWUSR, -+ show_idle_hysteresis_time, set_idle_hysteresis_time); -+#endif -+ - /** - * set_reset_timeout - Store callback for the reset_timeout sysfs file. - * @dev: The device with sysfs file is for -@@ -3013,122 +3717,359 @@ static DEVICE_ATTR(lp_mem_pool_max_size, S_IRUGO | S_IWUSR, show_lp_mem_pool_max - set_lp_mem_pool_max_size); - - /** -- * show_js_ctx_scheduling_mode - Show callback for js_ctx_scheduling_mode sysfs -- * entry. -+ * show_simplified_mem_pool_max_size - Show the maximum size for the memory -+ * pool 0 of small (4KiB) pages. - * @dev: The device this sysfs file is for. - * @attr: The attributes of the sysfs file. -- * @buf: The output buffer to receive the context scheduling mode information. -+ * @buf: The output buffer to receive the max size. - * -- * This function is called to get the context scheduling mode being used by JS. -+ * This function is called to get the maximum size for the memory pool 0 of -+ * small (4KiB) pages. It is assumed that the maximum size value is same for -+ * all the pools. - * - * Return: The number of bytes output to @buf. - */ --static ssize_t show_js_ctx_scheduling_mode(struct device *dev, -+static ssize_t show_simplified_mem_pool_max_size(struct device *dev, - struct device_attribute *attr, char * const buf) - { -- struct kbase_device *kbdev; -+ struct kbase_device *const kbdev = to_kbase_device(dev); - -- kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - -- return scnprintf(buf, PAGE_SIZE, "%u\n", kbdev->js_ctx_scheduling_mode); -+ return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, -+ kbdev->mem_pools.small, 1, kbase_mem_pool_debugfs_max_size); - } - - /** -- * set_js_ctx_scheduling_mode - Set callback for js_ctx_scheduling_mode sysfs -- * entry. -- * @dev: The device this sysfs file is for. -- * @attr: The attributes of the sysfs file. -- * @buf: The value written to the sysfs file. -- * @count: The number of bytes written to the sysfs file. -+ * set_simplified_mem_pool_max_size - Set the same maximum size for all the -+ * memory pools of small (4KiB) pages. -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file - * -- * This function is called when the js_ctx_scheduling_mode sysfs file is written -- * to. It checks the data written, and if valid updates the ctx scheduling mode -- * being by JS. -+ * This function is called to set the same maximum size for all the memory -+ * pools of small (4KiB) pages. - * -- * Return: @count if the function succeeded. An error code on failure. -+ * Return: The number of bytes output to @buf. - */ --static ssize_t set_js_ctx_scheduling_mode(struct device *dev, -+static ssize_t set_simplified_mem_pool_max_size(struct device *dev, - struct device_attribute *attr, const char *buf, size_t count) - { -- struct kbase_context *kctx; -- u32 new_js_ctx_scheduling_mode; -- struct kbase_device *kbdev; -- unsigned long flags; -- int ret; -+ struct kbase_device *const kbdev = to_kbase_device(dev); -+ unsigned long new_size; -+ int gid; -+ int err; - -- kbdev = to_kbase_device(dev); - if (!kbdev) - return -ENODEV; - -- ret = kstrtouint(buf, 0, &new_js_ctx_scheduling_mode); -- if (ret || new_js_ctx_scheduling_mode >= KBASE_JS_PRIORITY_MODE_COUNT) { -- dev_err(kbdev->dev, "Couldn't process js_ctx_scheduling_mode" -- " write operation.\n" -- "Use format <js_ctx_scheduling_mode>\n"); -+ err = kstrtoul(buf, 0, &new_size); -+ if (err) - return -EINVAL; -- } -- -- if (new_js_ctx_scheduling_mode == kbdev->js_ctx_scheduling_mode) -- return count; -- -- mutex_lock(&kbdev->kctx_list_lock); -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- -- /* Update the context priority mode */ -- kbdev->js_ctx_scheduling_mode = new_js_ctx_scheduling_mode; -- -- /* Adjust priority of all the contexts as per the new mode */ -- list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) -- kbase_js_update_ctx_priority(kctx); -- -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -- mutex_unlock(&kbdev->kctx_list_lock); - -- dev_dbg(kbdev->dev, "JS ctx scheduling mode: %u\n", new_js_ctx_scheduling_mode); -+ for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) -+ kbase_mem_pool_debugfs_set_max_size( -+ kbdev->mem_pools.small, gid, (size_t)new_size); - - return count; - } - --static DEVICE_ATTR(js_ctx_scheduling_mode, S_IRUGO | S_IWUSR, -- show_js_ctx_scheduling_mode, -- set_js_ctx_scheduling_mode); -- --#ifdef MALI_KBASE_BUILD --#ifdef CONFIG_DEBUG_FS -- --/* Number of entries in serialize_jobs_settings[] */ --#define NR_SERIALIZE_JOBS_SETTINGS 5 --/* Maximum string length in serialize_jobs_settings[].name */ --#define MAX_SERIALIZE_JOBS_NAME_LEN 16 -- --static struct --{ -- char *name; -- u8 setting; --} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { -- {"none", 0}, -- {"intra-slot", KBASE_SERIALIZE_INTRA_SLOT}, -- {"inter-slot", KBASE_SERIALIZE_INTER_SLOT}, -- {"full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT}, -- {"full-reset", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | -- KBASE_SERIALIZE_RESET} --}; -+static DEVICE_ATTR(max_size, 0600, show_simplified_mem_pool_max_size, -+ set_simplified_mem_pool_max_size); - - /** -- * kbasep_serialize_jobs_seq_show - Show callback for the serialize_jobs debugfs -- * file -- * @sfile: seq_file pointer -- * @data: Private callback data -+ * show_simplified_lp_mem_pool_max_size - Show the maximum size for the memory -+ * pool 0 of large (2MiB) pages. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the total current pool size. - * -- * This function is called to get the contents of the serialize_jobs debugfs -- * file. This is a list of the available settings with the currently active one -- * surrounded by square brackets. -+ * This function is called to get the maximum size for the memory pool 0 of -+ * large (2MiB) pages. It is assumed that the maximum size value is same for -+ * all the pools. - * -- * Return: 0 on success, or an error code on error -+ * Return: The number of bytes output to @buf. - */ --static int kbasep_serialize_jobs_seq_show(struct seq_file *sfile, void *data) -+static ssize_t show_simplified_lp_mem_pool_max_size(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *const kbdev = to_kbase_device(dev); -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ return kbase_debugfs_helper_get_attr_to_string(buf, PAGE_SIZE, -+ kbdev->mem_pools.large, 1, kbase_mem_pool_debugfs_max_size); -+} -+ -+/** -+ * set_simplified_lp_mem_pool_max_size - Set the same maximum size for all the -+ * memory pools of large (2MiB) pages. -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * This function is called to set the same maximum size for all the memory -+ * pools of large (2MiB) pages. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t set_simplified_lp_mem_pool_max_size(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *const kbdev = to_kbase_device(dev); -+ unsigned long new_size; -+ int gid; -+ int err; -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ err = kstrtoul(buf, 0, &new_size); -+ if (err) -+ return -EINVAL; -+ -+ for (gid = 0; gid < MEMORY_GROUP_MANAGER_NR_GROUPS; ++gid) -+ kbase_mem_pool_debugfs_set_max_size( -+ kbdev->mem_pools.large, gid, (size_t)new_size); -+ -+ return count; -+} -+ -+static DEVICE_ATTR(lp_max_size, 0600, show_simplified_lp_mem_pool_max_size, -+ set_simplified_lp_mem_pool_max_size); -+ -+/** -+ * show_simplified_ctx_default_max_size - Show the default maximum size for the -+ * memory pool 0 of small (4KiB) pages. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the pool size. -+ * -+ * This function is called to get the default ctx maximum size for the memory -+ * pool 0 of small (4KiB) pages. It is assumed that maximum size value is same -+ * for all the pools. The maximum size for the pool of large (2MiB) pages will -+ * be same as max size of the pool of small (4KiB) pages in terms of bytes. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_simplified_ctx_default_max_size(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev = to_kbase_device(dev); -+ size_t max_size; -+ -+ if (!kbdev) -+ return -ENODEV; -+ -+ max_size = kbase_mem_pool_config_debugfs_max_size( -+ kbdev->mem_pool_defaults.small, 0); -+ -+ return scnprintf(buf, PAGE_SIZE, "%zu\n", max_size); -+} -+ -+/** -+ * set_simplified_ctx_default_max_size - Set the same default maximum size for -+ * all the pools created for new -+ * contexts. This covers the pool of -+ * large pages as well and its max size -+ * will be same as max size of the pool -+ * of small pages in terms of bytes. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The value written to the sysfs file. -+ * @count: The number of bytes written to the sysfs file. -+ * -+ * This function is called to set the same maximum size for all pools created -+ * for new contexts. -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_simplified_ctx_default_max_size(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_device *kbdev; -+ unsigned long new_size; -+ int err; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ err = kstrtoul(buf, 0, &new_size); -+ if (err) -+ return -EINVAL; -+ -+ kbase_mem_pool_group_config_set_max_size( -+ &kbdev->mem_pool_defaults, (size_t)new_size); -+ -+ return count; -+} -+ -+static DEVICE_ATTR(ctx_default_max_size, 0600, -+ show_simplified_ctx_default_max_size, -+ set_simplified_ctx_default_max_size); -+ -+#if !MALI_USE_CSF -+/** -+ * show_js_ctx_scheduling_mode - Show callback for js_ctx_scheduling_mode sysfs -+ * entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the context scheduling mode information. -+ * -+ * This function is called to get the context scheduling mode being used by JS. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_js_ctx_scheduling_mode(struct device *dev, -+ struct device_attribute *attr, char * const buf) -+{ -+ struct kbase_device *kbdev; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ return scnprintf(buf, PAGE_SIZE, "%u\n", kbdev->js_ctx_scheduling_mode); -+} -+ -+/** -+ * set_js_ctx_scheduling_mode - Set callback for js_ctx_scheduling_mode sysfs -+ * entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The value written to the sysfs file. -+ * @count: The number of bytes written to the sysfs file. -+ * -+ * This function is called when the js_ctx_scheduling_mode sysfs file is written -+ * to. It checks the data written, and if valid updates the ctx scheduling mode -+ * being by JS. -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t set_js_ctx_scheduling_mode(struct device *dev, -+ struct device_attribute *attr, const char *buf, size_t count) -+{ -+ struct kbase_context *kctx; -+ u32 new_js_ctx_scheduling_mode; -+ struct kbase_device *kbdev; -+ unsigned long flags; -+ int ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = kstrtouint(buf, 0, &new_js_ctx_scheduling_mode); -+ if (ret || new_js_ctx_scheduling_mode >= KBASE_JS_PRIORITY_MODE_COUNT) { -+ dev_err(kbdev->dev, "Couldn't process js_ctx_scheduling_mode" -+ " write operation.\n" -+ "Use format <js_ctx_scheduling_mode>\n"); -+ return -EINVAL; -+ } -+ -+ if (new_js_ctx_scheduling_mode == kbdev->js_ctx_scheduling_mode) -+ return count; -+ -+ mutex_lock(&kbdev->kctx_list_lock); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ /* Update the context priority mode */ -+ kbdev->js_ctx_scheduling_mode = new_js_ctx_scheduling_mode; -+ -+ /* Adjust priority of all the contexts as per the new mode */ -+ list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) -+ kbase_js_update_ctx_priority(kctx); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->kctx_list_lock); -+ -+ dev_dbg(kbdev->dev, "JS ctx scheduling mode: %u\n", new_js_ctx_scheduling_mode); -+ -+ return count; -+} -+ -+static DEVICE_ATTR(js_ctx_scheduling_mode, S_IRUGO | S_IWUSR, -+ show_js_ctx_scheduling_mode, -+ set_js_ctx_scheduling_mode); -+ -+#ifdef MALI_KBASE_BUILD -+ -+/* Number of entries in serialize_jobs_settings[] */ -+#define NR_SERIALIZE_JOBS_SETTINGS 5 -+/* Maximum string length in serialize_jobs_settings[].name */ -+#define MAX_SERIALIZE_JOBS_NAME_LEN 16 -+ -+static struct -+{ -+ char *name; -+ u8 setting; -+} serialize_jobs_settings[NR_SERIALIZE_JOBS_SETTINGS] = { -+ {"none", 0}, -+ {"intra-slot", KBASE_SERIALIZE_INTRA_SLOT}, -+ {"inter-slot", KBASE_SERIALIZE_INTER_SLOT}, -+ {"full", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT}, -+ {"full-reset", KBASE_SERIALIZE_INTRA_SLOT | KBASE_SERIALIZE_INTER_SLOT | -+ KBASE_SERIALIZE_RESET} -+}; -+ -+/** -+ * update_serialize_jobs_setting - Update the serialization setting for the -+ * submission of GPU jobs. -+ * -+ * This function is called when the serialize_jobs sysfs/debugfs file is -+ * written to. It matches the requested setting against the available settings -+ * and if a matching setting is found updates kbdev->serialize_jobs. -+ * -+ * @kbdev: An instance of the GPU platform device, allocated from the probe -+ * method of the driver. -+ * @buf: Buffer containing the value written to the sysfs/debugfs file. -+ * @count: The number of bytes to write to the sysfs/debugfs file. -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t update_serialize_jobs_setting(struct kbase_device *kbdev, -+ const char *buf, size_t count) -+{ -+ int i; -+ bool valid = false; -+ -+ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { -+ if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { -+ kbdev->serialize_jobs = -+ serialize_jobs_settings[i].setting; -+ valid = true; -+ break; -+ } -+ } -+ -+ if (!valid) { -+ dev_err(kbdev->dev, "serialize_jobs: invalid setting"); -+ return -EINVAL; -+ } -+ -+ return count; -+} -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+/** -+ * kbasep_serialize_jobs_seq_debugfs_show - Show callback for the serialize_jobs -+ * debugfs file -+ * @sfile: seq_file pointer -+ * @data: Private callback data -+ * -+ * This function is called to get the contents of the serialize_jobs debugfs -+ * file. This is a list of the available settings with the currently active one -+ * surrounded by square brackets. -+ * -+ * Return: 0 on success, or an error code on error -+ */ -+static int kbasep_serialize_jobs_seq_debugfs_show(struct seq_file *sfile, -+ void *data) - { - struct kbase_device *kbdev = sfile->private; - int i; -@@ -3169,8 +4110,6 @@ static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, - struct seq_file *s = file->private_data; - struct kbase_device *kbdev = s->private; - char buf[MAX_SERIALIZE_JOBS_NAME_LEN]; -- int i; -- bool valid = false; - - CSTD_UNUSED(ppos); - -@@ -3180,21 +4119,7 @@ static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, - - buf[count] = 0; - -- for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { -- if (sysfs_streq(serialize_jobs_settings[i].name, buf)) { -- kbdev->serialize_jobs = -- serialize_jobs_settings[i].setting; -- valid = true; -- break; -- } -- } -- -- if (!valid) { -- dev_err(kbdev->dev, "serialize_jobs: invalid setting\n"); -- return -EINVAL; -- } -- -- return count; -+ return update_serialize_jobs_setting(kbdev, buf, count); - } - - /** -@@ -3208,7 +4133,8 @@ static ssize_t kbasep_serialize_jobs_debugfs_write(struct file *file, - static int kbasep_serialize_jobs_debugfs_open(struct inode *in, - struct file *file) - { -- return single_open(file, kbasep_serialize_jobs_seq_show, in->i_private); -+ return single_open(file, kbasep_serialize_jobs_seq_debugfs_show, -+ in->i_private); - } - - static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { -@@ -3221,27 +4147,101 @@ static const struct file_operations kbasep_serialize_jobs_debugfs_fops = { - }; - - #endif /* CONFIG_DEBUG_FS */ -+ -+/** -+ * show_serialize_jobs_sysfs - Show callback for serialize_jobs sysfs file. -+ * -+ * This function is called to get the contents of the serialize_jobs sysfs -+ * file. This is a list of the available settings with the currently active -+ * one surrounded by square brackets. -+ * -+ * @dev: The device this sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The output buffer for the sysfs file contents -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t show_serialize_jobs_sysfs(struct device *dev, -+ struct device_attribute *attr, -+ char *buf) -+{ -+ struct kbase_device *kbdev = to_kbase_device(dev); -+ ssize_t ret = 0; -+ int i; -+ -+ for (i = 0; i < NR_SERIALIZE_JOBS_SETTINGS; i++) { -+ if (kbdev->serialize_jobs == -+ serialize_jobs_settings[i].setting) -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "[%s]", -+ serialize_jobs_settings[i].name); -+ else -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s ", -+ serialize_jobs_settings[i].name); -+ } -+ -+ if (ret < PAGE_SIZE - 1) { -+ ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); -+ } else { -+ buf[PAGE_SIZE - 2] = '\n'; -+ buf[PAGE_SIZE - 1] = '\0'; -+ ret = PAGE_SIZE - 1; -+ } -+ -+ return ret; -+} -+ -+/** -+ * store_serialize_jobs_sysfs - Store callback for serialize_jobs sysfs file. -+ * -+ * This function is called when the serialize_jobs sysfs file is written to. -+ * It matches the requested setting against the available settings and if a -+ * matching setting is found updates kbdev->serialize_jobs. -+ * -+ * @dev: The device this sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes to write to the sysfs file -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t store_serialize_jobs_sysfs(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) -+{ -+ return update_serialize_jobs_setting(to_kbase_device(dev), buf, count); -+} -+ -+static DEVICE_ATTR(serialize_jobs, 0600, show_serialize_jobs_sysfs, -+ store_serialize_jobs_sysfs); - #endif /* MALI_KBASE_BUILD */ -+#endif /* !MALI_USE_CSF */ - - static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) - { - struct kbase_device *kbdev = container_of(data, struct kbase_device, - protected_mode_hwcnt_disable_work); -+ spinlock_t *backend_lock; - unsigned long flags; - - bool do_disable; - -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+#if MALI_USE_CSF -+ backend_lock = &kbdev->csf.scheduler.interrupt_lock; -+#else -+ backend_lock = &kbdev->hwaccess_lock; -+#endif -+ -+ spin_lock_irqsave(backend_lock, flags); - do_disable = !kbdev->protected_mode_hwcnt_desired && - !kbdev->protected_mode_hwcnt_disabled; -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ spin_unlock_irqrestore(backend_lock, flags); - - if (!do_disable) - return; - - kbase_hwcnt_context_disable(kbdev->hwcnt_gpu_ctx); - -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ spin_lock_irqsave(backend_lock, flags); - do_disable = !kbdev->protected_mode_hwcnt_desired && - !kbdev->protected_mode_hwcnt_disabled; - -@@ -3251,7 +4251,9 @@ static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) - * the state machine. - */ - kbdev->protected_mode_hwcnt_disabled = true; -+#if !MALI_USE_CSF - kbase_backend_slot_update(kbdev); -+#endif /* !MALI_USE_CSF */ - } else { - /* Protected mode state was updated while we were doing the - * disable, so we need to undo the disable we just performed. -@@ -3259,9 +4261,10 @@ static void kbasep_protected_mode_hwcnt_disable_worker(struct work_struct *data) - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - } - -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ spin_unlock_irqrestore(backend_lock, flags); - } - -+#ifndef PLATFORM_PROTECTED_CALLBACKS - static int kbasep_protected_mode_enable(struct protected_mode_device *pdev) - { - struct kbase_device *kbdev = pdev->data; -@@ -3281,6 +4284,9 @@ static const struct protected_mode_ops kbasep_native_protected_ops = { - .protected_mode_disable = kbasep_protected_mode_disable - }; - -+#define PLATFORM_PROTECTED_CALLBACKS (&kbasep_native_protected_ops) -+#endif /* PLATFORM_PROTECTED_CALLBACKS */ -+ - int kbase_protected_mode_init(struct kbase_device *kbdev) - { - /* Use native protected ops */ -@@ -3289,7 +4295,7 @@ int kbase_protected_mode_init(struct kbase_device *kbdev) - if (!kbdev->protected_dev) - return -ENOMEM; - kbdev->protected_dev->data = kbdev; -- kbdev->protected_ops = &kbasep_native_protected_ops; -+ kbdev->protected_ops = PLATFORM_PROTECTED_CALLBACKS; - INIT_WORK(&kbdev->protected_mode_hwcnt_disable_work, - kbasep_protected_mode_hwcnt_disable_worker); - kbdev->protected_mode_hwcnt_desired = true; -@@ -3303,15 +4309,6 @@ void kbase_protected_mode_term(struct kbase_device *kbdev) - kfree(kbdev->protected_dev); - } - --#ifdef CONFIG_MALI_NO_MALI --static int kbase_common_reg_map(struct kbase_device *kbdev) --{ -- return 0; --} --static void kbase_common_reg_unmap(struct kbase_device * const kbdev) --{ --} --#else /* CONFIG_MALI_NO_MALI */ - static int kbase_common_reg_map(struct kbase_device *kbdev) - { - int err = 0; -@@ -3347,7 +4344,6 @@ static void kbase_common_reg_unmap(struct kbase_device * const kbdev) - kbdev->reg_size = 0; - } - } --#endif /* CONFIG_MALI_NO_MALI */ - - int registers_map(struct kbase_device * const kbdev) - { -@@ -3367,6 +4363,15 @@ int registers_map(struct kbase_device * const kbdev) - kbdev->reg_start = reg_res->start; - kbdev->reg_size = resource_size(reg_res); - -+#if MALI_USE_CSF -+ if (kbdev->reg_size < -+ (CSF_HW_DOORBELL_PAGE_OFFSET + -+ CSF_NUM_DOORBELL * CSF_HW_DOORBELL_PAGE_SIZE)) { -+ dev_err(kbdev->dev, "Insufficient register space, will override to the required size\n"); -+ kbdev->reg_size = CSF_HW_DOORBELL_PAGE_OFFSET + -+ CSF_NUM_DOORBELL * CSF_HW_DOORBELL_PAGE_SIZE; -+ } -+#endif - - err = kbase_common_reg_map(kbdev); - if (err) { -@@ -3448,6 +4453,7 @@ int kbase_device_pm_init(struct kbase_device *kbdev) - u32 gpu_model_id; - - if (kbase_is_pv_enabled(kbdev->dev->of_node)) { -+ dev_info(kbdev->dev, "Arbitration interface enabled\n"); - if (kbase_is_pm_enabled(kbdev->dev->of_node)) { - /* Arbitration AND power management invalid */ - dev_err(kbdev->dev, "Invalid combination of arbitration AND power management\n"); -@@ -3471,13 +4477,16 @@ int kbase_device_pm_init(struct kbase_device *kbdev) - gpu_model_id = GPU_ID2_MODEL_MATCH_VALUE(product_id); - - if (gpu_model_id != GPU_ID2_PRODUCT_TGOX -- && gpu_model_id != GPU_ID2_PRODUCT_TNOX) { -+ && gpu_model_id != GPU_ID2_PRODUCT_TNOX -+ && gpu_model_id != GPU_ID2_PRODUCT_TBAX) { - kbase_arbiter_pm_early_term(kbdev); - dev_err(kbdev->dev, "GPU platform not suitable for arbitration\n"); - return -EPERM; - } - } - } else { -+ kbdev->arb.arb_if = NULL; -+ kbdev->arb.arb_dev = NULL; - err = power_control_init(kbdev); - } - #else -@@ -3489,7 +4498,7 @@ int kbase_device_pm_init(struct kbase_device *kbdev) - void kbase_device_pm_term(struct kbase_device *kbdev) - { - #ifdef CONFIG_MALI_ARBITER_SUPPORT --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - if (kbase_is_pv_enabled(kbdev->dev->of_node)) - kbase_arbiter_pm_early_term(kbdev); - else -@@ -3502,7 +4511,7 @@ void kbase_device_pm_term(struct kbase_device *kbdev) - - int power_control_init(struct kbase_device *kbdev) - { --#if KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE || !defined(CONFIG_OF) -+#ifndef CONFIG_OF - /* Power control initialization requires at least the capability to get - * regulators and clocks from the device tree, as well as parsing - * arrays of unsigned integer values. -@@ -3597,12 +4606,6 @@ int power_control_init(struct kbase_device *kbdev) - * on the device tree of the platform shouldn't prevent the driver - * from completing its initialization. - */ --#if (KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE && \ -- !defined(LSK_OPPV2_BACKPORT)) -- err = of_init_opp_table(kbdev->dev); -- CSTD_UNUSED(err); --#else -- - #if defined(CONFIG_PM_OPP) - #if ((KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) && \ - defined(CONFIG_REGULATOR)) -@@ -3614,8 +4617,6 @@ int power_control_init(struct kbase_device *kbdev) - err = dev_pm_opp_of_add_table(kbdev->dev); - CSTD_UNUSED(err); - #endif /* CONFIG_PM_OPP */ -- --#endif /* KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE */ - return 0; - - clocks_probe_defer: -@@ -3624,20 +4625,13 @@ clocks_probe_defer: - regulator_put(kbdev->regulators[i]); - #endif - return err; --#endif /* KERNEL_VERSION(3, 18, 0) > LINUX_VERSION_CODE */ -+#endif /* CONFIG_OF */ - } - - void power_control_term(struct kbase_device *kbdev) - { - unsigned int i; - --#if (KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE && \ -- !defined(LSK_OPPV2_BACKPORT)) --#if KERNEL_VERSION(3, 19, 0) <= LINUX_VERSION_CODE -- of_free_opp_table(kbdev->dev); --#endif --#else -- - #if defined(CONFIG_PM_OPP) - dev_pm_opp_of_remove_table(kbdev->dev); - #if ((KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) && \ -@@ -3647,8 +4641,6 @@ void power_control_term(struct kbase_device *kbdev) - #endif /* (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE */ - #endif /* CONFIG_PM_OPP */ - --#endif /* KERNEL_VERSION(4, 4, 0) > LINUX_VERSION_CODE */ -- - for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { - if (kbdev->clocks[i]) { - if (__clk_is_enabled(kbdev->clocks[i])) -@@ -3659,24 +4651,23 @@ void power_control_term(struct kbase_device *kbdev) - break; - } - --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 12, 0)) && defined(CONFIG_OF) \ -- && defined(CONFIG_REGULATOR) -+#if defined(CONFIG_OF) && defined(CONFIG_REGULATOR) - for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { - if (kbdev->regulators[i]) { - regulator_put(kbdev->regulators[i]); - kbdev->regulators[i] = NULL; - } - } --#endif /* LINUX_VERSION_CODE >= 3, 12, 0 */ -+#endif - } - - #ifdef MALI_KBASE_BUILD --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - - static void trigger_reset(struct kbase_device *kbdev) - { - kbase_pm_context_active(kbdev); -- if (kbase_prepare_to_reset_gpu(kbdev)) -+ if (kbase_prepare_to_reset_gpu(kbdev, RESET_FLAGS_NONE)) - kbase_reset_gpu(kbdev); - kbase_pm_context_idle(kbdev); - } -@@ -3704,7 +4695,7 @@ DEFINE_SIMPLE_ATTRIBUTE(fops_##type##_quirks, type##_quirks_get,\ - MAKE_QUIRK_ACCESSORS(sc); - MAKE_QUIRK_ACCESSORS(tiler); - MAKE_QUIRK_ACCESSORS(mmu); --MAKE_QUIRK_ACCESSORS(jm); -+MAKE_QUIRK_ACCESSORS(gpu); - - static ssize_t kbase_device_debugfs_reset_write(struct file *file, - const char __user *ubuf, size_t count, loff_t *ppos) -@@ -3825,7 +4816,9 @@ int kbase_device_debugfs_init(struct kbase_device *kbdev) - kbdev->mali_debugfs_directory = debugfs_create_dir(kbdev->devname, - NULL); - if (!kbdev->mali_debugfs_directory) { -- dev_err(kbdev->dev, "Couldn't create mali debugfs directory\n"); -+ dev_err(kbdev->dev, -+ "Couldn't create mali debugfs directory: %s\n", -+ kbdev->devname); - err = -ENOMEM; - goto out; - } -@@ -3838,6 +4831,14 @@ int kbase_device_debugfs_init(struct kbase_device *kbdev) - goto out; - } - -+ kbdev->debugfs_instr_directory = debugfs_create_dir("instrumentation", -+ kbdev->mali_debugfs_directory); -+ if (!kbdev->debugfs_instr_directory) { -+ dev_err(kbdev->dev, "Couldn't create mali debugfs instrumentation directory\n"); -+ err = -ENOMEM; -+ goto out; -+ } -+ - debugfs_ctx_defaults_directory = debugfs_create_dir("defaults", - kbdev->debugfs_ctx_directory); - if (!debugfs_ctx_defaults_directory) { -@@ -3846,20 +4847,20 @@ int kbase_device_debugfs_init(struct kbase_device *kbdev) - goto out; - } - --#if !MALI_CUSTOMER_RELEASE -- kbasep_regs_dump_debugfs_init(kbdev); --#endif /* !MALI_CUSTOMER_RELEASE */ - kbasep_regs_history_debugfs_init(kbdev); - -+#if !MALI_USE_CSF - kbase_debug_job_fault_debugfs_init(kbdev); -+#endif /* !MALI_USE_CSF */ - - kbasep_gpu_memory_debugfs_init(kbdev); - kbase_as_fault_debugfs_init(kbdev); --#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS -+#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS - kbase_instr_backend_debugfs_init(kbdev); - #endif - /* fops_* variables created by invocations of macro -- * MAKE_QUIRK_ACCESSORS() above. */ -+ * MAKE_QUIRK_ACCESSORS() above. -+ */ - debugfs_create_file("quirks_sc", 0644, - kbdev->mali_debugfs_directory, kbdev, - &fops_sc_quirks); -@@ -3869,9 +4870,8 @@ int kbase_device_debugfs_init(struct kbase_device *kbdev) - debugfs_create_file("quirks_mmu", 0644, - kbdev->mali_debugfs_directory, kbdev, - &fops_mmu_quirks); -- debugfs_create_file("quirks_jm", 0644, -- kbdev->mali_debugfs_directory, kbdev, -- &fops_jm_quirks); -+ debugfs_create_file("quirks_gpu", 0644, kbdev->mali_debugfs_directory, -+ kbdev, &fops_gpu_quirks); - - debugfs_create_bool("infinite_cache", mode, - debugfs_ctx_defaults_directory, -@@ -3900,16 +4900,20 @@ int kbase_device_debugfs_init(struct kbase_device *kbdev) - kbase_ktrace_debugfs_init(kbdev); - - #ifdef CONFIG_MALI_DEVFREQ --#ifdef CONFIG_DEVFREQ_THERMAL -+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL) - if (kbdev->devfreq) - kbase_ipa_debugfs_init(kbdev); - #endif /* CONFIG_DEVFREQ_THERMAL */ - #endif /* CONFIG_MALI_DEVFREQ */ - -+#if !MALI_USE_CSF - debugfs_create_file("serialize_jobs", S_IRUGO | S_IWUSR, - kbdev->mali_debugfs_directory, kbdev, - &kbasep_serialize_jobs_debugfs_fops); - -+#endif -+ kbase_dvfs_status_debugfs_init(kbdev); -+ - return 0; - - out: -@@ -3926,7 +4930,7 @@ void kbase_device_debugfs_term(struct kbase_device *kbdev) - - int kbase_device_coherency_init(struct kbase_device *kbdev) - { --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - u32 supported_coherency_bitmap = - kbdev->gpu_props.props.raw_props.coherency_mode; - const void *coherency_override_dts; -@@ -3953,7 +4957,7 @@ int kbase_device_coherency_init(struct kbase_device *kbdev) - kbdev->system_coherency = COHERENCY_NONE; - - /* device tree may override the coherency */ --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - coherency_override_dts = of_get_property(kbdev->dev->of_node, - "system-coherency", - NULL); -@@ -3961,6 +4965,17 @@ int kbase_device_coherency_init(struct kbase_device *kbdev) - - override_coherency = be32_to_cpup(coherency_override_dts); - -+#if MALI_USE_CSF && !IS_ENABLED(CONFIG_MALI_NO_MALI) -+ /* ACE coherency mode is not supported by Driver on CSF GPUs. -+ * Return an error to signal the invalid device tree configuration. -+ */ -+ if (override_coherency == COHERENCY_ACE) { -+ dev_err(kbdev->dev, -+ "ACE coherency not supported, wrong DT configuration"); -+ return -EINVAL; -+ } -+#endif -+ - if ((override_coherency <= COHERENCY_NONE) && - (supported_coherency_bitmap & - COHERENCY_FEATURE_BIT(override_coherency))) { -@@ -3984,67 +4999,226 @@ int kbase_device_coherency_init(struct kbase_device *kbdev) - return 0; - } - --#ifdef CONFIG_MALI_BUSLOG - --/* Callback used by the kbase bus logger client, to initiate a GPU reset -- * when the bus log is restarted. GPU reset is used as reference point -- * in HW bus log analyses. -+#if MALI_USE_CSF -+/** -+ * csg_scheduling_period_store - Store callback for the csg_scheduling_period -+ * sysfs file. -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * This function is called when the csg_scheduling_period sysfs file is written -+ * to. It checks the data written, and if valid updates the reset timeout. -+ * -+ * Return: @count if the function succeeded. An error code on failure. - */ --static void kbase_logging_started_cb(void *data) -+static ssize_t csg_scheduling_period_store(struct device *dev, -+ struct device_attribute *attr, -+ const char *buf, size_t count) - { -- struct kbase_device *kbdev = (struct kbase_device *)data; -+ struct kbase_device *kbdev; -+ int ret; -+ unsigned int csg_scheduling_period; - -- if (kbase_prepare_to_reset_gpu(kbdev)) -- kbase_reset_gpu(kbdev); -- dev_info(kbdev->dev, "KBASE - Bus logger restarted\n"); -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = kstrtouint(buf, 0, &csg_scheduling_period); -+ if (ret || csg_scheduling_period == 0) { -+ dev_err(kbdev->dev, -+ "Couldn't process csg_scheduling_period write operation.\n" -+ "Use format 'csg_scheduling_period_ms', and csg_scheduling_period_ms > 0\n"); -+ return -EINVAL; -+ } -+ -+ kbase_csf_scheduler_lock(kbdev); -+ kbdev->csf.scheduler.csg_scheduling_period_ms = csg_scheduling_period; -+ dev_dbg(kbdev->dev, "CSG scheduling period: %ums\n", -+ csg_scheduling_period); -+ kbase_csf_scheduler_unlock(kbdev); -+ -+ return count; - } - --int buslog_init(struct kbase_device *kbdev) -+/** -+ * csg_scheduling_period_show - Show callback for the csg_scheduling_period -+ * sysfs entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the GPU information. -+ * -+ * This function is called to get the current reset timeout. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t csg_scheduling_period_show(struct device *dev, -+ struct device_attribute *attr, -+ char *const buf) - { -- int err = 0; -+ struct kbase_device *kbdev; -+ ssize_t ret; - -- err = bl_core_client_register(kbdev->devname, -- kbase_logging_started_cb, -- kbdev, &kbdev->buslogger, -- THIS_MODULE, NULL); -- if (err == 0) -- bl_core_set_threshold(kbdev->buslogger, 1024*1024*1024); -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; - -- return err; -+ ret = scnprintf(buf, PAGE_SIZE, "%u\n", -+ kbdev->csf.scheduler.csg_scheduling_period_ms); -+ -+ return ret; - } - --void buslog_term(struct kbase_device *kbdev) -+static DEVICE_ATTR(csg_scheduling_period, 0644, csg_scheduling_period_show, -+ csg_scheduling_period_store); -+ -+/** -+ * fw_timeout_store - Store callback for the fw_timeout sysfs file. -+ * @dev: The device with sysfs file is for -+ * @attr: The attributes of the sysfs file -+ * @buf: The value written to the sysfs file -+ * @count: The number of bytes written to the sysfs file -+ * -+ * This function is called when the fw_timeout sysfs file is written to. It -+ * checks the data written, and if valid updates the reset timeout. -+ * -+ * Return: @count if the function succeeded. An error code on failure. -+ */ -+static ssize_t fw_timeout_store(struct device *dev, -+ struct device_attribute *attr, const char *buf, -+ size_t count) - { -- bl_core_client_unregister(kbdev->buslogger); -+ struct kbase_device *kbdev; -+ int ret; -+ unsigned int fw_timeout; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = kstrtouint(buf, 0, &fw_timeout); -+ if (ret || fw_timeout == 0) { -+ dev_err(kbdev->dev, "%s\n%s\n%u", -+ "Couldn't process fw_timeout write operation.", -+ "Use format 'fw_timeout_ms', and fw_timeout_ms > 0", -+ FIRMWARE_PING_INTERVAL_MS); -+ return -EINVAL; -+ } -+ -+ kbase_csf_scheduler_lock(kbdev); -+ kbdev->csf.fw_timeout_ms = fw_timeout; -+ kbase_csf_scheduler_unlock(kbdev); -+ dev_dbg(kbdev->dev, "Firmware timeout: %ums\n", fw_timeout); -+ -+ return count; - } --#endif -+ -+/** -+ * fw_timeout_show - Show callback for the firmware timeout sysfs entry. -+ * @dev: The device this sysfs file is for. -+ * @attr: The attributes of the sysfs file. -+ * @buf: The output buffer to receive the GPU information. -+ * -+ * This function is called to get the current reset timeout. -+ * -+ * Return: The number of bytes output to @buf. -+ */ -+static ssize_t fw_timeout_show(struct device *dev, -+ struct device_attribute *attr, char *const buf) -+{ -+ struct kbase_device *kbdev; -+ ssize_t ret; -+ -+ kbdev = to_kbase_device(dev); -+ if (!kbdev) -+ return -ENODEV; -+ -+ ret = scnprintf(buf, PAGE_SIZE, "%u\n", kbdev->csf.fw_timeout_ms); -+ -+ return ret; -+} -+ -+static DEVICE_ATTR(fw_timeout, 0644, fw_timeout_show, fw_timeout_store); -+#endif /* MALI_USE_CSF */ -+ -+static struct attribute *kbase_scheduling_attrs[] = { -+#if !MALI_USE_CSF -+ &dev_attr_serialize_jobs.attr, -+#endif /* !MALI_USE_CSF */ -+ NULL -+}; - - static struct attribute *kbase_attrs[] = { - #ifdef CONFIG_MALI_DEBUG - &dev_attr_debug_command.attr, -+#if !MALI_USE_CSF - &dev_attr_js_softstop_always.attr, -+#endif /* !MALI_USE_CSF */ - #endif -+#if !MALI_USE_CSF - &dev_attr_js_timeouts.attr, - &dev_attr_soft_job_timeout.attr, -+#endif /* !MALI_USE_CSF */ - &dev_attr_gpuinfo.attr, - &dev_attr_dvfs_period.attr, - &dev_attr_pm_poweroff.attr, -+#if MALI_USE_CSF -+ &dev_attr_idle_hysteresis_time.attr, -+#endif - &dev_attr_reset_timeout.attr, -+#if !MALI_USE_CSF - &dev_attr_js_scheduling_period.attr, -+#else -+ &dev_attr_csg_scheduling_period.attr, -+ &dev_attr_fw_timeout.attr, -+#endif /* !MALI_USE_CSF */ - &dev_attr_power_policy.attr, - &dev_attr_core_mask.attr, -+ &dev_attr_gpu_memory.attr, - &dev_attr_mem_pool_size.attr, - &dev_attr_mem_pool_max_size.attr, - &dev_attr_lp_mem_pool_size.attr, - &dev_attr_lp_mem_pool_max_size.attr, -+#if !MALI_USE_CSF - &dev_attr_js_ctx_scheduling_mode.attr, -+#endif /* !MALI_USE_CSF */ -+ NULL -+}; -+ -+static struct attribute *kbase_mempool_attrs[] = { -+ &dev_attr_max_size.attr, -+ &dev_attr_lp_max_size.attr, -+ &dev_attr_ctx_default_max_size.attr, - NULL - }; - -+#define SYSFS_SCHEDULING_GROUP "scheduling" -+static const struct attribute_group kbase_scheduling_attr_group = { -+ .name = SYSFS_SCHEDULING_GROUP, -+ .attrs = kbase_scheduling_attrs, -+}; -+ -+#define SYSFS_MEMPOOL_GROUP "mempool" -+static const struct attribute_group kbase_mempool_attr_group = { -+ .name = SYSFS_MEMPOOL_GROUP, -+ .attrs = kbase_mempool_attrs, -+}; -+ - static const struct attribute_group kbase_attr_group = { - .attrs = kbase_attrs, - }; - -+static struct attribute *ctx_attrs[] = { -+ &dev_attr_ctx_mem_pool_size.attr, -+ NULL -+}; -+ -+static const struct attribute_group kbase_ctx_attr_group = { -+ .attrs = ctx_attrs, -+}; -+ - int kbase_sysfs_init(struct kbase_device *kbdev) - { - int err = 0; -@@ -4056,11 +5230,39 @@ int kbase_sysfs_init(struct kbase_device *kbdev) - kbdev->mdev.mode = 0666; - - err = sysfs_create_group(&kbdev->dev->kobj, &kbase_attr_group); -+ err += sysfs_create_group(&kbdev->dev->kobj, &kbase_ctx_attr_group); -+ if (err) -+ return err; -+ -+ err = sysfs_create_group(&kbdev->dev->kobj, -+ &kbase_scheduling_attr_group); -+ if (err) { -+ dev_err(kbdev->dev, "Creation of %s sysfs group failed", -+ SYSFS_SCHEDULING_GROUP); -+ sysfs_remove_group(&kbdev->dev->kobj, -+ &kbase_attr_group); -+ return err; -+ } -+ -+ err = sysfs_create_group(&kbdev->dev->kobj, -+ &kbase_mempool_attr_group); -+ if (err) { -+ dev_err(kbdev->dev, "Creation of %s sysfs group failed", -+ SYSFS_MEMPOOL_GROUP); -+ sysfs_remove_group(&kbdev->dev->kobj, -+ &kbase_scheduling_attr_group); -+ sysfs_remove_group(&kbdev->dev->kobj, -+ &kbase_attr_group); -+ } -+ - return err; - } - - void kbase_sysfs_term(struct kbase_device *kbdev) - { -+ sysfs_remove_group(&kbdev->dev->kobj, &kbase_ctx_attr_group); -+ sysfs_remove_group(&kbdev->dev->kobj, &kbase_mempool_attr_group); -+ sysfs_remove_group(&kbdev->dev->kobj, &kbase_scheduling_attr_group); - sysfs_remove_group(&kbdev->dev->kobj, &kbase_attr_group); - put_device(kbdev->dev); - } -@@ -4119,7 +5321,8 @@ static int kbase_platform_device_probe(struct platform_device *pdev) - - if (err) { - if (err == -EPROBE_DEFER) -- dev_err(kbdev->dev, "Device initialization Deferred\n"); -+ dev_info(kbdev->dev, -+ "Device initialization Deferred\n"); - else - dev_err(kbdev->dev, "Device initialization failed\n"); - -@@ -4161,8 +5364,11 @@ static int kbase_device_suspend(struct device *dev) - - kbase_pm_suspend(kbdev); - --#if defined(CONFIG_MALI_DEVFREQ) && \ -- (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ kbase_pm_metrics_stop(kbdev); -+#endif -+ -+#ifdef CONFIG_MALI_DEVFREQ - dev_dbg(dev, "Callback %s\n", __func__); - if (kbdev->devfreq) { - kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_SUSPEND); -@@ -4190,8 +5396,11 @@ static int kbase_device_resume(struct device *dev) - - kbase_pm_resume(kbdev); - --#if defined(CONFIG_MALI_DEVFREQ) && \ -- (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ kbase_pm_metrics_start(kbdev); -+#endif -+ -+#ifdef CONFIG_MALI_DEVFREQ - dev_dbg(dev, "Callback %s\n", __func__); - if (kbdev->devfreq) { - mutex_lock(&kbdev->pm.lock); -@@ -4224,8 +5433,12 @@ static int kbase_device_runtime_suspend(struct device *dev) - return -ENODEV; - - dev_dbg(dev, "Callback %s\n", __func__); --#if defined(CONFIG_MALI_DEVFREQ) && \ -- (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ kbase_pm_metrics_stop(kbdev); -+#endif -+ -+#ifdef CONFIG_MALI_DEVFREQ - if (kbdev->devfreq) - kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_SUSPEND); - #endif -@@ -4263,8 +5476,11 @@ static int kbase_device_runtime_resume(struct device *dev) - dev_dbg(dev, "runtime resume\n"); - } - --#if defined(CONFIG_MALI_DEVFREQ) && \ -- (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0)) -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+ kbase_pm_metrics_start(kbdev); -+#endif -+ -+#ifdef CONFIG_MALI_DEVFREQ - if (kbdev->devfreq) - kbase_devfreq_enqueue_work(kbdev, DEVFREQ_WORK_RESUME); - #endif -@@ -4317,7 +5533,7 @@ static const struct dev_pm_ops kbase_pm_ops = { - #endif /* KBASE_PM_RUNTIME */ - }; - --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - static const struct of_device_id kbase_dt_ids[] = { - { .compatible = "arm,malit6xx" }, - { .compatible = "arm,mali-midgard" }, -@@ -4332,9 +5548,9 @@ static struct platform_driver kbase_platform_driver = { - .remove = kbase_platform_device_remove, - .driver = { - .name = kbase_drv_name, -- .owner = THIS_MODULE, - .pm = &kbase_pm_ops, - .of_match_table = of_match_ptr(kbase_dt_ids), -+ .probe_type = PROBE_PREFER_ASYNCHRONOUS, - }, - }; - -@@ -4342,7 +5558,7 @@ static struct platform_driver kbase_platform_driver = { - * The driver will not provide a shortcut to create the Mali platform device - * anymore when using Device Tree. - */ --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - module_platform_driver(kbase_platform_driver); - #else - -@@ -4377,6 +5593,7 @@ MODULE_LICENSE("GPL"); - MODULE_VERSION(MALI_RELEASE_NAME " (UK version " \ - __stringify(BASE_UK_VERSION_MAJOR) "." \ - __stringify(BASE_UK_VERSION_MINOR) ")"); -+MODULE_SOFTDEP("pre: memory_group_manager"); - - #define CREATE_TRACE_POINTS - /* Create the trace points (otherwise we just get code to call a tracepoint) */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cs_experimental.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cs_experimental.h -index e1fffc3..4dc09e4 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cs_experimental.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_cs_experimental.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,18 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ -- --/* -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * - */ - - #ifndef _KBASE_CS_EXPERIMENTAL_H_ -@@ -41,9 +30,6 @@ - */ - static inline void mali_kbase_print_cs_experimental(void) - { --#if MALI_JIT_PRESSURE_LIMIT -- pr_info("mali_kbase: JIT_PRESSURE_LIMIT (experimental) enabled"); --#endif /* MALI_JIT_PRESSURE_LIMIT */ - #if MALI_INCREMENTAL_RENDERING - pr_info("mali_kbase: INCREMENTAL_RENDERING (experimental) enabled"); - #endif /* MALI_INCREMENTAL_RENDERING */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c -index cea91bc..d06380d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,13 +17,9 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> --#include <mali_kbase_config_defaults.h> -- - #include <mali_kbase_defs.h> - #include "mali_kbase_ctx_sched.h" - #include "tl/mali_kbase_tracepoints.h" -@@ -46,7 +43,8 @@ int kbase_ctx_sched_init(struct kbase_device *kbdev) - int as_present = (1U << kbdev->nr_hw_address_spaces) - 1; - - /* These two must be recalculated if nr_hw_address_spaces changes -- * (e.g. for HW workarounds) */ -+ * (e.g. for HW workarounds) -+ */ - kbdev->nr_user_address_spaces = kbdev->nr_hw_address_spaces; - kbdev->as_free = as_present; /* All ASs initially free */ - -@@ -212,6 +210,13 @@ void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev) - for (i = 0; i != kbdev->nr_hw_address_spaces; ++i) { - struct kbase_context *kctx; - -+#if MALI_USE_CSF -+ if ((i == MCU_AS_NR) && kbdev->csf.firmware_inited) { -+ kbase_mmu_update(kbdev, &kbdev->csf.mcu_mmu, -+ MCU_AS_NR); -+ continue; -+ } -+#endif - kctx = kbdev->as_to_kctx[i]; - if (kctx) { - if (atomic_read(&kctx->refcount)) { -@@ -254,7 +259,7 @@ struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( - - found_kctx = kbdev->as_to_kctx[as_nr]; - -- if (found_kctx != NULL) -+ if (!WARN_ON(found_kctx == NULL)) - kbase_ctx_sched_retain_ctx_refcount(found_kctx); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -@@ -264,6 +269,21 @@ struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( - - struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, - size_t as_nr) -+{ -+ unsigned long flags; -+ struct kbase_context *found_kctx; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ found_kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as_nr); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return found_kctx; -+} -+ -+struct kbase_context *kbase_ctx_sched_as_to_ctx_nolock( -+ struct kbase_device *kbdev, size_t as_nr) - { - struct kbase_context *found_kctx; - -@@ -273,13 +293,14 @@ struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, - if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) - return NULL; - -- found_kctx = kbdev->as_to_kctx[as_nr]; -+ lockdep_assert_held(&kbdev->hwaccess_lock); - -- if (WARN_ON(!found_kctx)) -- return NULL; -+ found_kctx = kbdev->as_to_kctx[as_nr]; - -- if (WARN_ON(atomic_read(&found_kctx->refcount) <= 0)) -- return NULL; -+ if (found_kctx) { -+ if (atomic_read(&found_kctx->refcount) <= 0) -+ found_kctx = NULL; -+ } - - return found_kctx; - } -@@ -342,3 +363,40 @@ void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx) - - spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags); - } -+ -+#if MALI_USE_CSF -+bool kbase_ctx_sched_inc_refcount_if_as_valid(struct kbase_context *kctx) -+{ -+ struct kbase_device *kbdev; -+ bool added_ref = false; -+ unsigned long flags; -+ -+ if (WARN_ON(kctx == NULL)) -+ return added_ref; -+ -+ kbdev = kctx->kbdev; -+ -+ if (WARN_ON(kbdev == NULL)) -+ return added_ref; -+ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ if ((kctx->as_nr != KBASEP_AS_NR_INVALID) && -+ (kctx == kbdev->as_to_kctx[kctx->as_nr])) { -+ atomic_inc(&kctx->refcount); -+ -+ if (kbdev->as_free & (1u << kctx->as_nr)) -+ kbdev->as_free &= ~(1u << kctx->as_nr); -+ -+ KBASE_KTRACE_ADD(kbdev, SCHED_RETAIN_CTX_NOLOCK, kctx, -+ kbase_ktrace_get_ctx_refcnt(kctx)); -+ added_ref = true; -+ } -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ return added_ref; -+} -+#endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h -index 1affa71..334724f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ctx_sched.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_CTX_SCHED_H_ -@@ -26,7 +25,7 @@ - #include <mali_kbase.h> - - /** -- * The Context Scheduler manages address space assignment and reference -+ * DOC: The Context Scheduler manages address space assignment and reference - * counting to kbase_context. The interface has been designed to minimise - * interactions between the Job Scheduler and Power Management/MMU to support - * the existing Job Scheduler interface. -@@ -41,7 +40,7 @@ - */ - - /** -- * kbase_ctx_sched_init - Initialise the context scheduler -+ * kbase_ctx_sched_init() - Initialise the context scheduler - * @kbdev: The device for which the context scheduler needs to be initialised - * - * This must be called during device initialisation. The number of hardware -@@ -167,6 +166,21 @@ struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount( - struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev, - size_t as_nr); - -+/** -+ * kbase_ctx_sched_as_to_ctx_nolock - Lookup a context based on its current -+ * address space. -+ * @kbdev: The device for which the returned context must belong -+ * @as_nr: address space assigned to the context of interest -+ * -+ * The following lock must be held by the caller: -+ * * kbase_device::hwaccess_lock -+ * -+ * Return: a valid struct kbase_context on success or NULL on failure, -+ * indicating that no context was found in as_nr. -+ */ -+struct kbase_context *kbase_ctx_sched_as_to_ctx_nolock( -+ struct kbase_device *kbdev, size_t as_nr); -+ - /** - * kbase_ctx_sched_inc_refcount_nolock - Refcount a context as being busy, - * preventing it from being scheduled out. -@@ -206,4 +220,22 @@ bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx); - */ - void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx); - -+#if MALI_USE_CSF -+/** -+ * kbase_ctx_sched_inc_refcount_if_as_valid - Refcount the context if it has GPU -+ * address space slot assigned to it. -+ * -+ * @kctx: Context to be refcounted -+ * -+ * This function takes a reference on the context if it has a GPU address space -+ * slot assigned to it. The address space slot will not be available for -+ * re-assignment until the reference is released. -+ * -+ * Return: true if refcount succeeded and the address space slot will not be -+ * reassigned, false if the refcount failed (because the address space slot -+ * was not assigned). -+ */ -+bool kbase_ctx_sched_inc_refcount_if_as_valid(struct kbase_context *kctx); -+#endif -+ - #endif /* _KBASE_CTX_SCHED_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug.c -index 118f787..6d3b109 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2014 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2014, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #include <mali_kbase.h> - - static struct kbasep_debug_assert_cb kbasep_debug_assert_registered_cb = { -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug.h -index 2fdb72d..10a3c85 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2015, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2015, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #ifndef _KBASE_DEBUG_H - #define _KBASE_DEBUG_H - -@@ -43,7 +40,7 @@ - #endif /* KBASE_DEBUG_DISABLE_ASSERTS */ - - /** Function type that is called on an KBASE_DEBUG_ASSERT() or KBASE_DEBUG_ASSERT_MSG() */ --typedef void (kbase_debug_assert_hook) (void *); -+typedef void kbase_debug_assert_hook(void *); - - struct kbasep_debug_assert_cb { - kbase_debug_assert_hook *func; -@@ -51,9 +48,9 @@ struct kbasep_debug_assert_cb { - }; - - /** -- * @def KBASEP_DEBUG_PRINT_TRACE -- * @brief Private macro containing the format of the trace to display before every message -- * @sa KBASE_DEBUG_SKIP_TRACE, KBASE_DEBUG_SKIP_FUNCTION_NAME -+ * KBASEP_DEBUG_PRINT_TRACE - Private macro containing the format of the trace -+ * to display before every message @sa KBASE_DEBUG_SKIP_TRACE, -+ * KBASE_DEBUG_SKIP_FUNCTION_NAME - */ - #if !KBASE_DEBUG_SKIP_TRACE - #define KBASEP_DEBUG_PRINT_TRACE \ -@@ -68,21 +65,22 @@ struct kbasep_debug_assert_cb { - #endif - - /** -- * @def KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) -- * @brief (Private) system printing function associated to the @ref KBASE_DEBUG_ASSERT_MSG event. -- * @param trace location in the code from where the message is printed -- * @param function function from where the message is printed -- * @param ... Format string followed by format arguments. -+ * KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) - (Private) system printing -+ * function associated to the @ref KBASE_DEBUG_ASSERT_MSG event. -+ * @trace: location in the code from where the message is printed -+ * @function: function from where the message is printed -+ * @...: Format string followed by format arguments. -+ * - * @note function parameter cannot be concatenated with other strings - */ - /* Select the correct system output function*/ - #ifdef CONFIG_MALI_DEBUG --#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...)\ -- do { \ -- pr_err("Mali<ASSERT>: %s function:%s ", trace, function);\ -- pr_err(__VA_ARGS__);\ -- pr_err("\n");\ -- } while (false) -+#define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) \ -+ do { \ -+ pr_err("Mali<ASSERT>: %s function:%s ", trace, function); \ -+ pr_err(__VA_ARGS__); \ -+ pr_err("\n"); \ -+ } while (false) - #else - #define KBASEP_DEBUG_ASSERT_OUT(trace, function, ...) CSTD_NOP() - #endif -@@ -94,12 +92,12 @@ struct kbasep_debug_assert_cb { - #endif - - /** -- * @def KBASE_DEBUG_ASSERT(expr) -- * @brief Calls @ref KBASE_PRINT_ASSERT and prints the expression @a expr if @a expr is false -+ * KBASE_DEBUG_ASSERT(expr) - Calls @ref KBASE_PRINT_ASSERT and prints the -+ * expression @a expr if @a expr is false -+ * @expr: Boolean expression - * - * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 - * -- * @param expr Boolean expression - */ - #define KBASE_DEBUG_ASSERT(expr) \ - KBASE_DEBUG_ASSERT_MSG(expr, #expr) -@@ -107,15 +105,15 @@ struct kbasep_debug_assert_cb { - #if KBASE_DEBUG_DISABLE_ASSERTS - #define KBASE_DEBUG_ASSERT_MSG(expr, ...) CSTD_NOP() - #else -- /** -- * @def KBASE_DEBUG_ASSERT_MSG(expr, ...) -- * @brief Calls @ref KBASEP_DEBUG_ASSERT_OUT and prints the given message if @a expr is false -- * -- * @note This macro does nothing if the flag @ref KBASE_DEBUG_DISABLE_ASSERTS is set to 1 -- * -- * @param expr Boolean expression -- * @param ... Message to display when @a expr is false, as a format string followed by format arguments. -- */ -+/** -+ * KBASE_DEBUG_ASSERT_MSG() - Calls @ref KBASEP_DEBUG_ASSERT_OUT and prints the -+ * given message if @a expr is false -+ * @expr: Boolean expression -+ * @...: Message to display when @a expr is false, as a format string followed -+ * by format arguments. -+ * -+ * This macro does nothing if the flag KBASE_DEBUG_DISABLE_ASSERTS is set to 1 -+ */ - #define KBASE_DEBUG_ASSERT_MSG(expr, ...) \ - do { \ - if (!(expr)) { \ -@@ -127,10 +125,8 @@ struct kbasep_debug_assert_cb { - #endif /* KBASE_DEBUG_DISABLE_ASSERTS */ - - /** -- * @def KBASE_DEBUG_CODE( X ) -- * @brief Executes the code inside the macro only in debug mode -- * -- * @param X Code to compile only in debug mode. -+ * KBASE_DEBUG_CODE( X ) - Executes the code inside the macro only in debug mode -+ * @X: Code to compile only in debug mode. - */ - #ifdef CONFIG_MALI_DEBUG - #define KBASE_DEBUG_CODE(X) X -@@ -141,7 +137,9 @@ struct kbasep_debug_assert_cb { - /** @} */ - - /** -- * @brief Register a function to call on ASSERT -+ * kbase_debug_assert_register_hook - Register a function to call on ASSERT -+ * @func: the function to call when an assert is triggered. -+ * @param: the parameter to pass to \a func when calling it - * - * Such functions will \b only be called during Debug mode, and for debugging - * features \b only. Do not rely on them to be called in general use. -@@ -151,13 +149,12 @@ struct kbasep_debug_assert_cb { - * @note This function is not thread-safe, and should only be used to - * register/deregister once in the module's lifetime. - * -- * @param[in] func the function to call when an assert is triggered. -- * @param[in] param the parameter to pass to \a func when calling it - */ - void kbase_debug_assert_register_hook(kbase_debug_assert_hook *func, void *param); - - /** -- * @brief Call a debug assert hook previously registered with kbase_debug_assert_register_hook() -+ * kbasep_debug_assert_call_hook - Call a debug assert hook previously -+ * registered with kbase_debug_assert_register_hook() - * - * @note This function is not thread-safe with respect to multiple threads - * registering functions and parameters with -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c -index dbc774d..4f021b3 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,13 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> - #include <linux/spinlock.h> - #include <mali_kbase_hwaccess_jm.h> - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - - static bool kbase_is_job_fault_event_pending(struct kbase_device *kbdev) - { -@@ -518,23 +517,24 @@ void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev) - /* - * Initialize the relevant data structure per context - */ --void kbase_debug_job_fault_context_init(struct kbase_context *kctx) -+int kbase_debug_job_fault_context_init(struct kbase_context *kctx) - { - - /* We need allocate double size register range - * Because this memory will keep the register address and value - */ - kctx->reg_dump = vmalloc(0x4000 * 2); -- if (kctx->reg_dump == NULL) -- return; -- -- if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == false) { -- vfree(kctx->reg_dump); -- kctx->reg_dump = NULL; -+ if (kctx->reg_dump != NULL) { -+ if (kbase_debug_job_fault_reg_snapshot_init(kctx, 0x4000) == -+ false) { -+ vfree(kctx->reg_dump); -+ kctx->reg_dump = NULL; -+ } -+ INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); -+ atomic_set(&kctx->job_fault_count, 0); - } -- INIT_LIST_HEAD(&kctx->job_fault_resume_event_list); -- atomic_set(&kctx->job_fault_count, 0); - -+ return 0; - } - - /* -@@ -549,6 +549,14 @@ void kbase_debug_job_fault_kctx_unblock(struct kbase_context *kctx) - { - WARN_ON(!kbase_ctx_flag(kctx, KCTX_DYING)); - -+ /* Return early if the job fault part of the kbase_device is not -+ * initialized yet. An error can happen during the device probe after -+ * the privileged Kbase context was created for the HW counter dumping -+ * but before the job fault part is initialized. -+ */ -+ if (!kctx->kbdev->job_fault_resume_workq) -+ return; -+ - kbase_ctx_remove_pending_event(kctx); - } - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h -index ef69627..39aeed0 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_job_fault.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2016, 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2016, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_DEBUG_JOB_FAULT_H -@@ -54,8 +53,9 @@ void kbase_debug_job_fault_dev_term(struct kbase_device *kbdev); - * kbase_debug_job_fault_context_init - Initialize the relevant - * data structure per context - * @kctx: KBase context pointer -+ * @return 0 on success - */ --void kbase_debug_job_fault_context_init(struct kbase_context *kctx); -+int kbase_debug_job_fault_context_init(struct kbase_context *kctx); - - /** - * kbase_debug_job_fault_context_term - Release the relevant -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c -index 4788137..5a99b5e 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2013-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2013-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -30,11 +29,7 @@ - #include <linux/list.h> - #include <linux/file.h> - --#ifdef CONFIG_DEBUG_FS -- --#if (KERNEL_VERSION(4, 1, 0) > LINUX_VERSION_CODE) --#define get_file_rcu(x) atomic_long_inc_not_zero(&(x)->f_count) --#endif -+#if IS_ENABLED(CONFIG_DEBUG_FS) - - struct debug_mem_mapping { - struct list_head node; -@@ -179,6 +174,13 @@ static int debug_mem_zone_open(struct rb_root *rbtree, - /* Empty region - ignore */ - continue; - -+ if (reg->flags & KBASE_REG_PROTECTED) { -+ /* CPU access to protected memory is forbidden - so -+ * skip this GPU virtual region. -+ */ -+ continue; -+ } -+ - mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); - if (!mapping) { - ret = -ENOMEM; -@@ -222,19 +224,19 @@ static int debug_mem_open(struct inode *i, struct file *file) - kbase_gpu_vm_lock(kctx); - - ret = debug_mem_zone_open(&kctx->reg_rbtree_same, mem_data); -- if (0 != ret) { -+ if (ret != 0) { - kbase_gpu_vm_unlock(kctx); - goto out; - } - - ret = debug_mem_zone_open(&kctx->reg_rbtree_custom, mem_data); -- if (0 != ret) { -+ if (ret != 0) { - kbase_gpu_vm_unlock(kctx); - goto out; - } - - ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data); -- if (0 != ret) { -+ if (ret != 0) { - kbase_gpu_vm_unlock(kctx); - goto out; - } -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h -index b948b7c..d034832 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debug_mem_view.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2013-2015, 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2013-2015, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_DEBUG_MEM_VIEW_H -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debugfs_helper.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debugfs_helper.c -index 37e507b..973739f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debugfs_helper.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debugfs_helper.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/debugfs.h> -@@ -46,10 +45,9 @@ - * - * Return: 0 if success, negative error code otherwise. - */ --static int set_attr_from_string( -- char *const buf, -- void *const array, size_t const nelems, -- kbase_debugfs_helper_set_attr_fn const set_attr_fn) -+static int -+set_attr_from_string(char *const buf, void *const array, size_t const nelems, -+ kbase_debugfs_helper_set_attr_fn *const set_attr_fn) - { - size_t index, err = 0; - char *ptr = buf; -@@ -90,9 +88,62 @@ static int set_attr_from_string( - return err; - } - -+int kbase_debugfs_string_validator(char *const buf) -+{ -+ size_t index; -+ int err = 0; -+ char *ptr = buf; -+ -+ for (index = 0; *ptr; ++index) { -+ unsigned long test_number; -+ size_t len; -+ -+ /* Drop leading spaces */ -+ while (*ptr == ' ') -+ ptr++; -+ -+ /* Strings passed into the validator will be NULL terminated -+ * by nature, so here strcspn only needs to delimit by -+ * newlines, spaces and NULL terminator (delimited natively). -+ */ -+ len = strcspn(ptr, "\n "); -+ if (len == 0) { -+ /* No more values (allow this) */ -+ break; -+ } -+ -+ /* Substitute a nul terminator for a space character to make -+ * the substring valid for kstrtoul, and then replace it back. -+ */ -+ if (ptr[len] == ' ') { -+ ptr[len] = '\0'; -+ err = kstrtoul(ptr, 0, &test_number); -+ ptr[len] = ' '; -+ -+ /* len should only be incremented if there is a valid -+ * number to follow - otherwise this will skip over -+ * the NULL terminator in cases with no ending newline -+ */ -+ len++; -+ } else { -+ /* This would occur at the last element before a space -+ * or a NULL terminator. -+ */ -+ err = kstrtoul(ptr, 0, &test_number); -+ } -+ -+ if (err) -+ break; -+ /* Skip the substring (including any premature nul terminator) -+ */ -+ ptr += len; -+ } -+ return err; -+} -+ - int kbase_debugfs_helper_set_attr_from_string( - const char *const buf, void *const array, size_t const nelems, -- kbase_debugfs_helper_set_attr_fn const set_attr_fn) -+ kbase_debugfs_helper_set_attr_fn *const set_attr_fn) - { - char *const wbuf = kstrdup(buf, GFP_KERNEL); - int err = 0; -@@ -100,6 +151,13 @@ int kbase_debugfs_helper_set_attr_from_string( - if (!wbuf) - return -ENOMEM; - -+ /* validate string before actually writing values */ -+ err = kbase_debugfs_string_validator(wbuf); -+ if (err) { -+ kfree(wbuf); -+ return err; -+ } -+ - err = set_attr_from_string(wbuf, array, nelems, - set_attr_fn); - -@@ -108,9 +166,9 @@ int kbase_debugfs_helper_set_attr_from_string( - } - - ssize_t kbase_debugfs_helper_get_attr_to_string( -- char *const buf, size_t const size, -- void *const array, size_t const nelems, -- kbase_debugfs_helper_get_attr_fn const get_attr_fn) -+ char *const buf, size_t const size, void *const array, -+ size_t const nelems, -+ kbase_debugfs_helper_get_attr_fn *const get_attr_fn) - { - ssize_t total = 0; - size_t index; -@@ -128,10 +186,10 @@ ssize_t kbase_debugfs_helper_get_attr_to_string( - return total; - } - --int kbase_debugfs_helper_seq_write(struct file *const file, -- const char __user *const ubuf, size_t const count, -- size_t const nelems, -- kbase_debugfs_helper_set_attr_fn const set_attr_fn) -+int kbase_debugfs_helper_seq_write( -+ struct file *const file, const char __user *const ubuf, -+ size_t const count, size_t const nelems, -+ kbase_debugfs_helper_set_attr_fn *const set_attr_fn) - { - const struct seq_file *const sfile = file->private_data; - void *const array = sfile->private; -@@ -154,6 +212,14 @@ int kbase_debugfs_helper_seq_write(struct file *const file, - } - - buf[count] = '\0'; -+ -+ /* validate string before actually writing values */ -+ err = kbase_debugfs_string_validator(buf); -+ if (err) { -+ kfree(buf); -+ return err; -+ } -+ - err = set_attr_from_string(buf, - array, nelems, set_attr_fn); - kfree(buf); -@@ -161,9 +227,9 @@ int kbase_debugfs_helper_seq_write(struct file *const file, - return err; - } - --int kbase_debugfs_helper_seq_read(struct seq_file *const sfile, -- size_t const nelems, -- kbase_debugfs_helper_get_attr_fn const get_attr_fn) -+int kbase_debugfs_helper_seq_read( -+ struct seq_file *const sfile, size_t const nelems, -+ kbase_debugfs_helper_get_attr_fn *const get_attr_fn) - { - void *const array = sfile->private; - size_t index; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debugfs_helper.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debugfs_helper.h -index c3c9efa..4c69d8b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debugfs_helper.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_debugfs_helper.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_DEBUGFS_HELPER_H_ -@@ -31,8 +30,8 @@ - * @index: An element index. The valid range depends on the use-case. - * @value: Attribute value to be set. - */ --typedef void (*kbase_debugfs_helper_set_attr_fn)( -- void *array, size_t index, size_t value); -+typedef void kbase_debugfs_helper_set_attr_fn(void *array, size_t index, -+ size_t value); - - /** - * kbase_debugfs_helper_set_attr_from_string - Parse a string to reconfigure an -@@ -56,7 +55,30 @@ typedef void (*kbase_debugfs_helper_set_attr_fn)( - */ - int kbase_debugfs_helper_set_attr_from_string( - const char *buf, void *array, size_t nelems, -- kbase_debugfs_helper_set_attr_fn set_attr_fn); -+ kbase_debugfs_helper_set_attr_fn *set_attr_fn); -+ -+/** -+ * kbase_debugfs_string_validator - Validate a string to be written to a -+ * debugfs file for any incorrect formats -+ * or wrong values. -+ * -+ * This function is to be used before any writes to debugfs values are done -+ * such that any strings with erroneous values (such as octal 09 or -+ * hexadecimal 0xGH are fully ignored) - without this validation, any correct -+ * values before the first incorrect one will still be entered into the -+ * debugfs file. This essentially iterates the values through kstrtoul to see -+ * if it is valid. -+ * -+ * It is largely similar to set_attr_from_string to iterate through the values -+ * of the input string. This function also requires the input string to be -+ * writable. -+ * -+ * @buf: Null-terminated string to validate. -+ * -+ * Return: 0 with no error, else -22 (the invalid return value of kstrtoul) if -+ * any value in the string was wrong or with an incorrect format. -+ */ -+int kbase_debugfs_string_validator(char *const buf); - - /** - * typedef kbase_debugfs_helper_get_attr_fn - Type of function to get an -@@ -67,8 +89,7 @@ int kbase_debugfs_helper_set_attr_from_string( - * - * Return: Value of attribute. - */ --typedef size_t (*kbase_debugfs_helper_get_attr_fn)( -- void *array, size_t index); -+typedef size_t kbase_debugfs_helper_get_attr_fn(void *array, size_t index); - - /** - * kbase_debugfs_helper_get_attr_to_string - Construct a formatted string -@@ -89,7 +110,7 @@ typedef size_t (*kbase_debugfs_helper_get_attr_fn)( - */ - ssize_t kbase_debugfs_helper_get_attr_to_string( - char *buf, size_t size, void *array, size_t nelems, -- kbase_debugfs_helper_get_attr_fn get_attr_fn); -+ kbase_debugfs_helper_get_attr_fn *get_attr_fn); - - /** - * kbase_debugfs_helper_seq_read - Implements reads from a virtual file for an -@@ -110,8 +131,8 @@ ssize_t kbase_debugfs_helper_get_attr_to_string( - * Return: 0 if success, negative error code otherwise. - */ - int kbase_debugfs_helper_seq_read( -- struct seq_file *const sfile, size_t const nelems, -- kbase_debugfs_helper_get_attr_fn const get_attr_fn); -+ struct seq_file *sfile, size_t nelems, -+ kbase_debugfs_helper_get_attr_fn *get_attr_fn); - - /** - * kbase_debugfs_helper_seq_write - Implements writes to a virtual file for an -@@ -132,10 +153,10 @@ int kbase_debugfs_helper_seq_read( - * - * Return: 0 if success, negative error code otherwise. - */ --int kbase_debugfs_helper_seq_write(struct file *const file, -- const char __user *const ubuf, size_t const count, -- size_t const nelems, -- kbase_debugfs_helper_set_attr_fn const set_attr_fn); -+int kbase_debugfs_helper_seq_write(struct file *file, -+ const char __user *ubuf, size_t count, -+ size_t nelems, -+ kbase_debugfs_helper_set_attr_fn *set_attr_fn); - - #endif /*_KBASE_DEBUGFS_HELPER_H_ */ - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h -index 7056d80..146695c 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,17 +17,11 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_defs.h -- * -- * Defintions (types, defines, etcs) common to Kbase. They are placed here to -- * allow the hierarchy of header files to work. -+ * DOC: Defintions (types, defines, etcs) common to Kbase. They are placed here -+ * to allow the hierarchy of header files to work. - */ - - #ifndef _KBASE_DEFS_H_ -@@ -37,10 +32,14 @@ - #include <mali_base_hwconfig_issues.h> - #include <mali_kbase_mem_lowlevel.h> - #include <mmu/mali_kbase_mmu_hw.h> --#include <mali_kbase_instr_defs.h> -+#include <backend/gpu/mali_kbase_instr_defs.h> - #include <mali_kbase_pm.h> - #include <mali_kbase_gpuprops_types.h> --#include <mali_kbase_hwcnt_backend_gpu.h> -+#if MALI_USE_CSF -+#include <mali_kbase_hwcnt_backend_csf.h> -+#else -+#include <mali_kbase_hwcnt_backend_jm.h> -+#endif - #include <protected_mode_switcher.h> - - #include <linux/atomic.h> -@@ -49,9 +48,6 @@ - #include <linux/file.h> - #include <linux/sizes.h> - --#ifdef CONFIG_MALI_BUSLOG --#include <linux/bus_logger.h> --#endif - - #if defined(CONFIG_SYNC) - #include <sync.h> -@@ -59,7 +55,7 @@ - #include "mali_kbase_fence_defs.h" - #endif - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - #include <linux/debugfs.h> - #endif /* CONFIG_DEBUG_FS */ - -@@ -75,8 +71,7 @@ - #include <linux/regulator/consumer.h> - #include <linux/memory_group_manager.h> - --#if defined(CONFIG_PM_RUNTIME) || \ -- (defined(CONFIG_PM) && LINUX_VERSION_CODE >= KERNEL_VERSION(3, 19, 0)) -+#if defined(CONFIG_PM_RUNTIME) || defined(CONFIG_PM) - #define KBASE_PM_RUNTIME 1 - #endif - -@@ -123,6 +118,11 @@ - */ - #define KBASE_LOCK_REGION_MIN_SIZE_LOG2 (15) - -+/** -+ * Maximum number of GPU memory region zones -+ */ -+#define KBASE_REG_ZONE_MAX 4ul -+ - #include "mali_kbase_hwaccess_defs.h" - - /* Maximum number of pages of memory that require a permanent mapping, per -@@ -138,24 +138,29 @@ - */ - #define KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS (200 * NSEC_PER_USEC) - -+#if MALI_USE_CSF -+/* The buffer count of CSF hwcnt backend ring buffer, which is used when CSF -+ * hwcnt backend allocate the ring buffer to communicate with CSF firmware for -+ * HWC dump samples. -+ * To meet the hardware requirement, this number MUST be power of 2, otherwise, -+ * CSF hwcnt backend creation will be failed. -+ */ -+#define KBASE_HWCNT_BACKEND_CSF_RING_BUFFER_COUNT (128) -+#endif -+ - /* Maximum number of clock/regulator pairs that may be referenced by - * the device node. - * This is dependent on support for of_property_read_u64_array() in the - * kernel. - */ --#if (KERNEL_VERSION(4, 0, 0) <= LINUX_VERSION_CODE) || \ -- defined(LSK_OPPV2_BACKPORT) - #define BASE_MAX_NR_CLOCKS_REGULATORS (2) --#else --#define BASE_MAX_NR_CLOCKS_REGULATORS (1) --#endif - - /* Forward declarations */ - struct kbase_context; - struct kbase_device; - struct kbase_as; - struct kbase_mmu_setup; --struct kbase_ipa_model_vinstr_data; -+struct kbase_kinstr_jm; - - /** - * struct kbase_io_access - holds information about 1 register access -@@ -178,11 +183,7 @@ struct kbase_io_access { - * @buf: array of kbase_io_access - */ - struct kbase_io_history { --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) - bool enabled; --#else -- u32 enabled; --#endif - - spinlock_t lock; - size_t count; -@@ -242,29 +243,6 @@ struct kbase_fault { - bool protected_mode; - }; - --/** -- * struct kbase_as - object representing an address space of GPU. -- * @number: Index at which this address space structure is present -- * in an array of address space structures embedded inside the -- * struct kbase_device. -- * @pf_wq: Workqueue for processing work items related to Bus fault -- * and Page fault handling. -- * @work_pagefault: Work item for the Page fault handling. -- * @work_busfault: Work item for the Bus fault handling. -- * @pf_data: Data relating to page fault. -- * @bf_data: Data relating to bus fault. -- * @current_setup: Stores the MMU configuration for this address space. -- */ --struct kbase_as { -- int number; -- struct workqueue_struct *pf_wq; -- struct work_struct work_pagefault; -- struct work_struct work_busfault; -- struct kbase_fault pf_data; -- struct kbase_fault bf_data; -- struct kbase_mmu_setup current_setup; --}; -- - /** - * struct kbase_mmu_table - object representing a set of GPU page tables - * @mmu_teardown_pages: Buffer of 4 Pages in size, used to cache the entries -@@ -291,7 +269,11 @@ struct kbase_mmu_table { - struct kbase_context *kctx; - }; - -+#if MALI_USE_CSF -+#include "csf/mali_kbase_csf_defs.h" -+#else - #include "jm/mali_kbase_jm_defs.h" -+#endif - - static inline int kbase_as_has_bus_fault(struct kbase_as *as, - struct kbase_fault *fault) -@@ -320,71 +302,115 @@ struct kbasep_mem_device { - atomic_t ir_threshold; - }; - -+struct kbase_clk_rate_listener; -+ -+/** -+ * typedef kbase_clk_rate_listener_on_change_t() - Frequency change callback -+ * -+ * @listener: Clock frequency change listener. -+ * @clk_index: Index of the clock for which the change has occurred. -+ * @clk_rate_hz: Clock frequency(Hz). -+ * -+ * A callback to call when clock rate changes. The function must not -+ * sleep. No clock rate manager functions must be called from here, as -+ * its lock is taken. -+ */ -+typedef void -+kbase_clk_rate_listener_on_change_t(struct kbase_clk_rate_listener *listener, -+ u32 clk_index, u32 clk_rate_hz); -+ -+/** -+ * struct kbase_clk_rate_listener - Clock frequency listener -+ * -+ * @node: List node. -+ * @notify: Callback to be called when GPU frequency changes. -+ */ -+struct kbase_clk_rate_listener { -+ struct list_head node; -+ kbase_clk_rate_listener_on_change_t *notify; -+}; -+ - /** -- * Data stored per device for power management. -+ * struct kbase_clk_rate_trace_manager - Data stored per device for GPU clock -+ * rate trace manager. - * -- * This structure contains data for the power management framework. There is one -- * instance of this structure per device in the system. -+ * @gpu_idle: Tracks the idle state of GPU. -+ * @clks: Array of pointer to structures storing data for every -+ * enumerated GPU clock. -+ * @clk_rate_trace_ops: Pointer to the platform specific GPU clock rate trace -+ * operations. -+ * @gpu_clk_rate_trace_write: Pointer to the function that would emit the -+ * tracepoint for the clock rate change. -+ * @listeners: List of listener attached. -+ * @lock: Lock to serialize the actions of GPU clock rate trace -+ * manager. -+ */ -+struct kbase_clk_rate_trace_manager { -+ bool gpu_idle; -+ struct kbase_clk_data *clks[BASE_MAX_NR_CLOCKS_REGULATORS]; -+ struct kbase_clk_rate_trace_op_conf *clk_rate_trace_ops; -+ struct list_head listeners; -+ spinlock_t lock; -+}; -+ -+/** -+ * struct kbase_pm_device_data - Data stored per device for power management. -+ * @lock: The lock protecting Power Management structures accessed outside of -+ * IRQ. -+ * This lock must also be held whenever the GPU is being powered on or -+ * off. -+ * @active_count: The reference count of active contexts on this device. Note -+ * that some code paths keep shaders/the tiler powered whilst this is 0. -+ * Use kbase_pm_is_active() instead to check for such cases. -+ * @suspending: Flag indicating suspending/suspended -+ * @gpu_lost: Flag indicating gpu lost -+ * This structure contains data for the power management framework. There -+ * is one instance of this structure per device in the system. -+ * @zero_active_count_wait: Wait queue set when active_count == 0 -+ * @resume_wait: system resume of GPU device. -+ * @debug_core_mask: Bit masks identifying the available shader cores that are -+ * specified via sysfs. One mask per job slot. -+ * @debug_core_mask_all: Bit masks identifying the available shader cores that -+ * are specified via sysfs. -+ * @callback_power_runtime_init: Callback for initializing the runtime power -+ * management. Return 0 on success, else error code -+ * @callback_power_runtime_term: Callback for terminating the runtime power -+ * management. -+ * @dvfs_period: Time in milliseconds between each dvfs sample -+ * @backend: KBase PM backend data -+ * @arb_vm_state: The state of the arbiter VM machine -+ * @gpu_users_waiting: Used by virtualization to notify the arbiter that there -+ * are users waiting for the GPU so that it can request and resume the -+ * driver. -+ * @clk_rtm: The state of the GPU clock rate trace manager - */ - struct kbase_pm_device_data { -- /** -- * The lock protecting Power Management structures accessed outside of -- * IRQ. -- * -- * This lock must also be held whenever the GPU is being powered on or -- * off. -- */ - struct mutex lock; -- -- /** -- * The reference count of active contexts on this device. Note that -- * some code paths keep shaders/the tiler powered whilst this is 0. Use -- * kbase_pm_is_active() instead to check for such cases. -- */ - int active_count; -- /** Flag indicating suspending/suspended */ - bool suspending; - #ifdef CONFIG_MALI_ARBITER_SUPPORT -- /* Flag indicating gpu lost */ -- bool gpu_lost; -+ atomic_t gpu_lost; - #endif /* CONFIG_MALI_ARBITER_SUPPORT */ -- /* Wait queue set when active_count == 0 */ - wait_queue_head_t zero_active_count_wait; -+ wait_queue_head_t resume_wait; - -- /** -- * Bit masks identifying the available shader cores that are specified -- * via sysfs. One mask per job slot. -- */ -+#if MALI_USE_CSF -+ u64 debug_core_mask; -+#else -+ /* One mask per job slot. */ - u64 debug_core_mask[BASE_JM_MAX_NR_SLOTS]; - u64 debug_core_mask_all; -+#endif /* MALI_USE_CSF */ - -- /** -- * Callback for initializing the runtime power management. -- * -- * @param kbdev The kbase device -- * -- * @return 0 on success, else error code -- */ -- int (*callback_power_runtime_init)(struct kbase_device *kbdev); -- -- /** -- * Callback for terminating the runtime power management. -- * -- * @param kbdev The kbase device -- */ -+ int (*callback_power_runtime_init)(struct kbase_device *kbdev); - void (*callback_power_runtime_term)(struct kbase_device *kbdev); -- -- /* Time in milliseconds between each dvfs sample */ - u32 dvfs_period; -- - struct kbase_pm_backend_data backend; -- - #ifdef CONFIG_MALI_ARBITER_SUPPORT -- /** -- * The state of the arbiter VM machine -- */ - struct kbase_arbiter_vm_state *arb_vm_state; -+ atomic_t gpu_users_waiting; - #endif /* CONFIG_MALI_ARBITER_SUPPORT */ -+ struct kbase_clk_rate_trace_manager clk_rtm; - }; - - /** -@@ -524,7 +550,6 @@ struct kbase_mmu_mode { - unsigned long flags; - }; - --struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void); - struct kbase_mmu_mode const *kbase_mmu_mode_get_aarch64(void); - - #define DEVNAME_SIZE 16 -@@ -559,6 +584,32 @@ struct kbase_devfreq_queue_info { - enum kbase_devfreq_work_type acted_type; - }; - -+/** -+ * struct kbase_process - Representing an object of a kbase process instantiated -+ * when the first kbase context is created under it. -+ * @tgid: Thread group ID. -+ * @total_gpu_pages: Total gpu pages allocated across all the contexts -+ * of this process, it accounts for both native allocations -+ * and dma_buf imported allocations. -+ * @kctx_list: List of kbase contexts created for the process. -+ * @kprcs_node: Node to a rb_tree, kbase_device will maintain a rb_tree -+ * based on key tgid, kprcs_node is the node link to -+ * &struct_kbase_device.process_root. -+ * @dma_buf_root: RB tree of the dma-buf imported allocations, imported -+ * across all the contexts created for this process. -+ * Used to ensure that pages of allocation are accounted -+ * only once for the process, even if the allocation gets -+ * imported multiple times for the process. -+ */ -+struct kbase_process { -+ pid_t tgid; -+ size_t total_gpu_pages; -+ struct list_head kctx_list; -+ -+ struct rb_node kprcs_node; -+ struct rb_root dma_buf_root; -+}; -+ - /** - * struct kbase_device - Object representing an instance of GPU platform device, - * allocated from the probe method of mali driver. -@@ -568,8 +619,8 @@ struct kbase_devfreq_queue_info { - * issues present in the GPU. - * @hw_quirks_mmu: Configuration to be used for the MMU as per the HW - * issues present in the GPU. -- * @hw_quirks_jm: Configuration to be used for the Job Manager as per -- * the HW issues present in the GPU. -+ * @hw_quirks_gpu: Configuration to be used for the Job Manager or CSF/MCU -+ * subsystems as per the HW issues present in the GPU. - * @entry: Links the device instance to the global list of GPU - * devices. The list would have as many entries as there - * are GPU device instances. -@@ -586,6 +637,8 @@ struct kbase_devfreq_queue_info { - * @irqs: Array containing IRQ resource info for 3 types of - * interrupts : Job scheduling, MMU & GPU events (like - * power management, cache etc.) -+ * @irqs.irq: irq number -+ * @irqs.flags: irq flags - * @clocks: Pointer to the input clock resources referenced by - * the GPU device node. - * @nr_clocks: Number of clocks set in the clocks array. -@@ -619,6 +672,8 @@ struct kbase_devfreq_queue_info { - * accesses made by the driver. - * @pm: Per device object for storing data for power management - * framework. -+ * @fw_load_lock: Mutex to protect firmware loading in @ref kbase_open. -+ * @csf: CSF object for the GPU device. - * @js_data: Per device object encapsulating the current context of - * Job Scheduler, which is global to the device and is not - * tied to any particular struct kbase_context running on -@@ -646,11 +701,21 @@ struct kbase_devfreq_queue_info { - * @disjoint_event: struct for keeping track of the disjoint information, - * that whether the GPU is in a disjoint state and the - * number of disjoint events that have occurred on GPU. -+ * @disjoint_event.count: disjoint event count -+ * @disjoint_event.state: disjoint event state - * @nr_hw_address_spaces: Number of address spaces actually available in the - * GPU, remains constant after driver initialisation. - * @nr_user_address_spaces: Number of address spaces available to user contexts -+ * @hwcnt_backend_csf_if_fw: Firmware interface to access CSF GPU performance -+ * counters. - * @hwcnt: Structure used for instrumentation and HW counters - * dumping -+ * @hwcnt.lock: The lock should be used when accessing any of the -+ * following members -+ * @hwcnt.kctx: kbase context -+ * @hwcnt.addr: HW counter address -+ * @hwcnt.addr_bytes: HW counter size in bytes -+ * @hwcnt.backend: Kbase instrumentation backend - * @hwcnt_gpu_iface: Backend interface for GPU hardware counter access. - * @hwcnt_gpu_ctx: Context for GPU hardware counter access. - * @hwaccess_lock must be held when calling -@@ -661,6 +726,7 @@ struct kbase_devfreq_queue_info { - * are enabled. If zero, there is no timeline client and - * therefore timeline is disabled. - * @timeline: Timeline context created per device. -+ * @ktrace: kbase device's ktrace - * @trace_lock: Lock to serialize the access to trace buffer. - * @trace_first_out: Index/offset in the trace buffer at which the first - * unread message is present. -@@ -686,6 +752,8 @@ struct kbase_devfreq_queue_info { - * including any contexts that might be created for - * hardware counters. - * @kctx_list_lock: Lock protecting concurrent accesses to @kctx_list. -+ * @group_max_uid_in_devices: Max value of any queue group UID in any kernel -+ * context in the kbase device. - * @devfreq_profile: Describes devfreq profile for the Mali GPU device, passed - * to devfreq_add_device() to add devfreq feature to Mali - * GPU device. -@@ -714,6 +782,7 @@ struct kbase_devfreq_queue_info { - * table in devicetree. - * @num_opps: Number of operating performance points available for the Mali - * GPU device. -+ * @last_devfreq_metrics: last PM metrics - * @devfreq_queue: Per device object for storing data that manages devfreq - * suspend & resume request queue and the related items. - * @devfreq_cooling: Pointer returned on registering devfreq cooling device -@@ -724,6 +793,17 @@ struct kbase_devfreq_queue_info { - * previously entered protected mode. - * @ipa: Top level structure for IPA, containing pointers to both - * configured & fallback models. -+ * @ipa.lock: Access to this struct must be with ipa.lock held -+ * @ipa.configured_model: ipa model to use -+ * @ipa.fallback_model: ipa fallback model -+ * @ipa.last_metrics: Values of the PM utilization metrics from last time -+ * the power model was invoked. The utilization is -+ * calculated as the difference between last_metrics -+ * and the current values. -+ * @ipa.force_fallback_model: true if use of fallback model has been forced by -+ * the User -+ * @ipa.last_sample_time: Records the time when counters, used for dynamic -+ * energy estimation, were last sampled. - * @previous_frequency: Previous frequency of GPU clock used for - * BASE_HW_ISSUE_GPU2017_1336 workaround, This clock is - * restored when L2 is powered on. -@@ -732,6 +812,7 @@ struct kbase_devfreq_queue_info { - * @mali_debugfs_directory: Root directory for the debugfs files created by the driver - * @debugfs_ctx_directory: Directory inside the @mali_debugfs_directory containing - * a sub-directory for every context. -+ * @debugfs_instr_directory: Instrumentation debugfs directory - * @debugfs_as_read_bitmap: bitmap of address spaces for which the bus or page fault - * has occurred. - * @job_fault_wq: Waitqueue to block the job fault dumping daemon till the -@@ -748,6 +829,8 @@ struct kbase_devfreq_queue_info { - * @job_fault_event_lock: Lock to protect concurrent accesses to @job_fault_event_list - * @regs_dump_debugfs_data: Contains the offset of register to be read through debugfs - * file "read_register". -+ * @regs_dump_debugfs_data.reg_offset: Contains the offset of register to be -+ * read through debugfs file "read_register". - * @ctx_num: Total number of contexts created for the device. - * @io_history: Pointer to an object keeping a track of all recent - * register accesses. The history of register accesses -@@ -806,12 +889,40 @@ struct kbase_devfreq_queue_info { - * Job Scheduler - * @l2_size_override: Used to set L2 cache size via device tree blob - * @l2_hash_override: Used to set L2 cache hash via device tree blob -+ * @l2_hash_values_override: true if @l2_hash_values is valid. -+ * @l2_hash_values: Used to set L2 asn_hash via device tree blob -+ * @process_root: rb_tree root node for maintaining a rb_tree of -+ * kbase_process based on key tgid(thread group ID). -+ * @dma_buf_root: rb_tree root node for maintaining a rb_tree of -+ * &struct kbase_dma_buf based on key dma_buf. -+ * We maintain a rb_tree of dma_buf mappings under -+ * kbase_device and kbase_process, one indicates a -+ * mapping and gpu memory usage at device level and -+ * other one at process level. -+ * @total_gpu_pages: Total GPU pages used for the complete GPU device. -+ * @dma_buf_lock: This mutex should be held while accounting for -+ * @total_gpu_pages from imported dma buffers. -+ * @gpu_mem_usage_lock: This spinlock should be held while accounting -+ * @total_gpu_pages for both native and dma-buf imported -+ * allocations. -+ * @dummy_job_wa: struct for dummy job execution workaround for the -+ * GPU hang issue -+ * @dummy_job_wa.ctx: dummy job workaround context -+ * @dummy_job_wa.jc: dummy job workaround job -+ * @dummy_job_wa.slot: dummy job workaround slot -+ * @dummy_job_wa.flags: dummy job workaround flags -+ * @dummy_job_wa_loaded: Flag for indicating that the workaround blob has -+ * been loaded. Protected by @fw_load_lock. -+ * @arb: Pointer to the arbiter device -+ * @pcm_dev: The priority control manager device. -+ * @oom_notifier_block: notifier_block containing kernel-registered out-of- -+ * memory handler. - */ - struct kbase_device { - u32 hw_quirks_sc; - u32 hw_quirks_tiler; - u32 hw_quirks_mmu; -- u32 hw_quirks_jm; -+ u32 hw_quirks_gpu; - - struct list_head entry; - struct device *dev; -@@ -827,7 +938,7 @@ struct kbase_device { - - struct clk *clocks[BASE_MAX_NR_CLOCKS_REGULATORS]; - unsigned int nr_clocks; --#ifdef CONFIG_REGULATOR -+#if IS_ENABLED(CONFIG_REGULATOR) - struct regulator *regulators[BASE_MAX_NR_CLOCKS_REGULATORS]; - unsigned int nr_regulators; - #if (KERNEL_VERSION(4, 10, 0) <= LINUX_VERSION_CODE) -@@ -837,16 +948,6 @@ struct kbase_device { - char devname[DEVNAME_SIZE]; - u32 id; - --#ifdef CONFIG_MALI_NO_MALI -- void *model; -- struct kmem_cache *irq_slab; -- struct workqueue_struct *irq_workq; -- atomic_t serving_job_irq; -- atomic_t serving_gpu_irq; -- atomic_t serving_mmu_irq; -- spinlock_t reg_op_lock; --#endif /* CONFIG_MALI_NO_MALI */ -- - struct kbase_pm_device_data pm; - - struct kbase_mem_pool_group mem_pools; -@@ -874,8 +975,10 @@ struct kbase_device { - s8 nr_hw_address_spaces; - s8 nr_user_address_spaces; - -+#if MALI_USE_CSF -+ struct kbase_hwcnt_backend_csf_if hwcnt_backend_csf_if_fw; -+#else - struct kbase_hwcnt { -- /* The lock should be used when accessing any of the following members */ - spinlock_t lock; - - struct kbase_context *kctx; -@@ -884,6 +987,7 @@ struct kbase_device { - - struct kbase_instr_backend backend; - } hwcnt; -+#endif - - struct kbase_hwcnt_backend_interface hwcnt_gpu_iface; - struct kbase_hwcnt_context *hwcnt_gpu_ctx; -@@ -906,6 +1010,7 @@ struct kbase_device { - - struct list_head kctx_list; - struct mutex kctx_list_lock; -+ atomic_t group_max_uid_in_devices; - - #ifdef CONFIG_MALI_DEVFREQ - struct devfreq_dev_profile devfreq_profile; -@@ -917,17 +1022,10 @@ struct kbase_device { - struct kbase_devfreq_opp *devfreq_table; - int num_opps; - struct kbasep_pm_metrics last_devfreq_metrics; -- --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 8, 0) - struct kbase_devfreq_queue_info devfreq_queue; --#endif - --#ifdef CONFIG_DEVFREQ_THERMAL --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 4, 0) -- struct devfreq_cooling_device *devfreq_cooling; --#else -+#if IS_ENABLED(CONFIG_DEVFREQ_THERMAL) - struct thermal_cooling_device *devfreq_cooling; --#endif - bool ipa_protection_mode_switched; - struct { - /* Access to this struct must be with ipa.lock held */ -@@ -940,11 +1038,13 @@ struct kbase_device { - * the difference between last_metrics and the current values. - */ - struct kbasep_pm_metrics last_metrics; -- /* Model data to pass to ipa_gpu_active/idle() */ -- struct kbase_ipa_model_vinstr_data *model_data; - - /* true if use of fallback model has been forced by the User */ - bool force_fallback_model; -+ /* Records the time when counters, used for dynamic energy -+ * estimation, were last sampled. -+ */ -+ ktime_t last_sample_time; - } ipa; - #endif /* CONFIG_DEVFREQ_THERMAL */ - #endif /* CONFIG_MALI_DEVFREQ */ -@@ -952,9 +1052,10 @@ struct kbase_device { - - atomic_t job_fault_debug; - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - struct dentry *mali_debugfs_directory; - struct dentry *debugfs_ctx_directory; -+ struct dentry *debugfs_instr_directory; - - #ifdef CONFIG_MALI_DEBUG - u64 debugfs_as_read_bitmap; -@@ -968,14 +1069,14 @@ struct kbase_device { - - #if !MALI_CUSTOMER_RELEASE - struct { -- u16 reg_offset; -+ u32 reg_offset; - } regs_dump_debugfs_data; - #endif /* !MALI_CUSTOMER_RELEASE */ - #endif /* CONFIG_DEBUG_FS */ - - atomic_t ctx_num; - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - struct kbase_io_history io_history; - #endif /* CONFIG_DEBUG_FS */ - -@@ -985,8 +1086,7 @@ struct kbase_device { - - bool poweroff_pending; - -- --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) -+#if (KERNEL_VERSION(4, 4, 0) <= LINUX_VERSION_CODE) - bool infinite_cache_active_default; - #else - u32 infinite_cache_active_default; -@@ -1015,9 +1115,6 @@ struct kbase_device { - - struct work_struct protected_mode_hwcnt_disable_work; - --#ifdef CONFIG_MALI_BUSLOG -- struct bus_logger_client *buslogger; --#endif - - bool irq_reset_flush; - -@@ -1029,7 +1126,14 @@ struct kbase_device { - - u8 l2_size_override; - u8 l2_hash_override; -+ bool l2_hash_values_override; -+ u32 l2_hash_values[ASN_HASH_COUNT]; - -+ struct mutex fw_load_lock; -+#if MALI_USE_CSF -+ /* CSF object for the GPU device. */ -+ struct kbase_csf_device csf; -+#else - struct kbasep_js_device_data js_data; - - /* See KBASE_JS_*_PRIORITY_MODE for details. */ -@@ -1042,6 +1146,14 @@ struct kbase_device { - u8 backup_serialize_jobs; - #endif /* CONFIG_MALI_CINSTR_GWT */ - -+#endif /* MALI_USE_CSF */ -+ -+ struct rb_root process_root; -+ struct rb_root dma_buf_root; -+ -+ size_t total_gpu_pages; -+ struct mutex dma_buf_lock; -+ spinlock_t gpu_mem_usage_lock; - - struct { - struct kbase_context *ctx; -@@ -1049,16 +1161,16 @@ struct kbase_device { - int slot; - u64 flags; - } dummy_job_wa; -+ bool dummy_job_wa_loaded; - - #ifdef CONFIG_MALI_ARBITER_SUPPORT -- /* Pointer to the arbiter device */ - struct kbase_arbiter_device arb; - #endif --}; -+ /* Priority Control Manager device */ -+ struct priority_control_manager_device *pcm_dev; - --#define KBASE_API_VERSION(major, minor) ((((major) & 0xFFF) << 20) | \ -- (((minor) & 0xFFF) << 8) | \ -- ((0 & 0xFF) << 0)) -+ struct notifier_block oom_notifier_block; -+}; - - /** - * enum kbase_file_state - Initialization state of a file opened by @kbase_open -@@ -1109,7 +1221,92 @@ struct kbase_file { - unsigned long api_version; - atomic_t setup_state; - }; -- -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+/** -+ * enum kbase_context_flags - Flags for kbase contexts -+ * -+ * @KCTX_COMPAT: Set when the context process is a compat process, 32-bit -+ * process on a 64-bit kernel. -+ * -+ * @KCTX_RUNNABLE_REF: Set when context is counted in -+ * kbdev->js_data.nr_contexts_runnable. Must hold queue_mutex when accessing. -+ * -+ * @KCTX_ACTIVE: Set when the context is active. -+ * -+ * @KCTX_PULLED: Set when last kick() caused atoms to be pulled from this -+ * context. -+ * -+ * @KCTX_MEM_PROFILE_INITIALIZED: Set when the context's memory profile has been -+ * initialized. -+ * -+ * @KCTX_INFINITE_CACHE: Set when infinite cache is to be enabled for new -+ * allocations. Existing allocations will not change. -+ * -+ * @KCTX_SUBMIT_DISABLED: Set to prevent context from submitting any jobs. -+ * -+ * @KCTX_PRIVILEGED:Set if the context uses an address space and should be kept -+ * scheduled in. -+ * -+ * @KCTX_SCHEDULED: Set when the context is scheduled on the Run Pool. -+ * This is only ever updated whilst the jsctx_mutex is held. -+ * -+ * @KCTX_DYING: Set when the context process is in the process of being evicted. -+ * -+ * @KCTX_NO_IMPLICIT_SYNC: Set when explicit Android fences are in use on this -+ * context, to disable use of implicit dma-buf fences. This is used to avoid -+ * potential synchronization deadlocks. -+ * -+ * @KCTX_FORCE_SAME_VA: Set when BASE_MEM_SAME_VA should be forced on memory -+ * allocations. For 64-bit clients it is enabled by default, and disabled by -+ * default on 32-bit clients. Being able to clear this flag is only used for -+ * testing purposes of the custom zone allocation on 64-bit user-space builds, -+ * where we also require more control than is available through e.g. the JIT -+ * allocation mechanism. However, the 64-bit user-space client must still -+ * reserve a JIT region using KBASE_IOCTL_MEM_JIT_INIT -+ * -+ * @KCTX_PULLED_SINCE_ACTIVE_JS0: Set when the context has had an atom pulled -+ * from it for job slot 0. This is reset when the context first goes active or -+ * is re-activated on that slot. -+ * -+ * @KCTX_PULLED_SINCE_ACTIVE_JS1: Set when the context has had an atom pulled -+ * from it for job slot 1. This is reset when the context first goes active or -+ * is re-activated on that slot. -+ * -+ * @KCTX_PULLED_SINCE_ACTIVE_JS2: Set when the context has had an atom pulled -+ * from it for job slot 2. This is reset when the context first goes active or -+ * is re-activated on that slot. -+ * -+ * @KCTX_AS_DISABLED_ON_FAULT: Set when the GPU address space is disabled for -+ * the context due to unhandled page(or bus) fault. It is cleared when the -+ * refcount for the context drops to 0 or on when the address spaces are -+ * re-enabled on GPU reset or power cycle. -+ * -+ * @KCTX_JPL_ENABLED: Set when JIT physical page limit is less than JIT virtual -+ * address page limit, so we must take care to not exceed the physical limit -+ * -+ * All members need to be separate bits. This enum is intended for use in a -+ * bitmask where multiple values get OR-ed together. -+ */ -+enum kbase_context_flags { -+ KCTX_COMPAT = 1U << 0, -+ KCTX_RUNNABLE_REF = 1U << 1, -+ KCTX_ACTIVE = 1U << 2, -+ KCTX_PULLED = 1U << 3, -+ KCTX_MEM_PROFILE_INITIALIZED = 1U << 4, -+ KCTX_INFINITE_CACHE = 1U << 5, -+ KCTX_SUBMIT_DISABLED = 1U << 6, -+ KCTX_PRIVILEGED = 1U << 7, -+ KCTX_SCHEDULED = 1U << 8, -+ KCTX_DYING = 1U << 9, -+ KCTX_NO_IMPLICIT_SYNC = 1U << 10, -+ KCTX_FORCE_SAME_VA = 1U << 11, -+ KCTX_PULLED_SINCE_ACTIVE_JS0 = 1U << 12, -+ KCTX_PULLED_SINCE_ACTIVE_JS1 = 1U << 13, -+ KCTX_PULLED_SINCE_ACTIVE_JS2 = 1U << 14, -+ KCTX_AS_DISABLED_ON_FAULT = 1U << 15, -+ KCTX_JPL_ENABLED = 1U << 16, -+}; -+#else - /** - * enum kbase_context_flags - Flags for kbase contexts - * -@@ -1190,6 +1387,7 @@ enum kbase_context_flags { - KCTX_PULLED_SINCE_ACTIVE_JS2 = 1U << 14, - KCTX_AS_DISABLED_ON_FAULT = 1U << 15, - }; -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - struct kbase_sub_alloc { - struct list_head link; -@@ -1197,6 +1395,21 @@ struct kbase_sub_alloc { - DECLARE_BITMAP(sub_pages, SZ_2M / SZ_4K); - }; - -+/** -+ * struct kbase_reg_zone - Information about GPU memory region zones -+ * @base_pfn: Page Frame Number in GPU virtual address space for the start of -+ * the Zone -+ * @va_size_pages: Size of the Zone in pages -+ * -+ * Track information about a zone KBASE_REG_ZONE() and related macros. -+ * In future, this could also store the &rb_root that are currently in -+ * &kbase_context -+ */ -+struct kbase_reg_zone { -+ u64 base_pfn; -+ u64 va_size_pages; -+}; -+ - /** - * struct kbase_context - Kernel base context - * -@@ -1247,6 +1460,7 @@ struct kbase_sub_alloc { - * @reg_rbtree_exec: RB tree of the memory regions allocated from the EXEC_VA - * zone of the GPU virtual address space. Used for GPU-executable - * allocations which don't need the SAME_VA property. -+ * @reg_zone: Zone information for the reg_rbtree_<...> members. - * @cookies: Bitmask containing of BITS_PER_LONG bits, used mainly for - * SAME_VA allocations to defer the reservation of memory region - * (from the GPU virtual address space) from base_mem_alloc -@@ -1271,6 +1485,7 @@ struct kbase_sub_alloc { - * which actually created the context. This is usually, - * but not necessarily, the same as the thread which - * opened the device file /dev/malixx instance. -+ * @csf: kbase csf context - * @jctx: object encapsulating all the Job dispatcher related state, - * including the array of atoms. - * @used_pages: Keeps a track of the number of 4KB physical pages in use -@@ -1287,6 +1502,8 @@ struct kbase_sub_alloc { - * evictable/reclaimable. - * @evict_list: List head for the list containing the allocations which - * can be evicted or freed up in the shrinker callback. -+ * @evict_nents: Total number of pages allocated by the allocations within -+ * @evict_list (atomic). - * @waiting_soft_jobs: List head for the list containing softjob atoms, which - * are either waiting for the event set operation, or waiting - * for the signaling of input fence or waiting for the GPU -@@ -1297,6 +1514,8 @@ struct kbase_sub_alloc { - * waiting atoms and the waitqueue to process the work item - * queued for the atoms blocked on the signaling of dma-buf - * fences. -+ * @dma_fence.waiting_resource: list head for the list of dma-buf fence -+ * @dma_fence.wq: waitqueue to process the work item queued - * @as_nr: id of the address space being used for the scheduled in - * context. This is effectively part of the Run Pool, because - * it only has a valid setting (!=KBASEP_AS_NR_INVALID) whilst -@@ -1315,15 +1534,13 @@ struct kbase_sub_alloc { - * at any point. - * Generally the reference count is incremented when the context - * is scheduled in and an atom is pulled from the context's per -- * slot runnable tree. -+ * slot runnable tree in JM GPU or GPU command queue -+ * group is programmed on CSG slot in CSF GPU. - * @mm_update_lock: lock used for handling of special tracking page. - * @process_mm: Pointer to the memory descriptor of the process which - * created the context. Used for accounting the physical - * pages used for GPU allocations, done for the context, - * to the memory consumed by the process. -- * @same_va_end: End address of the SAME_VA zone (in 4KB page units) -- * @exec_va_start: Start address of the EXEC_VA zone (in 4KB page units) -- * or U64_MAX if the EXEC_VA zone is uninitialized. - * @gpu_va_end: End address of the GPU va space (in 4KB page units) - * @jit_va: Indicates if a JIT_VA zone has been created. - * @mem_profile_data: Buffer containing the profiling information provided by -@@ -1399,6 +1616,16 @@ struct kbase_sub_alloc { - * that were used (i.e. the - * &struct_kbase_va_region.used_pages for regions - * that have had a usage report). -+ * @jit_phys_pages_to_be_allocated: Count of the physical pages that are being -+ * now allocated for just-in-time memory -+ * allocations of a context (across all the -+ * threads). This is supposed to be updated -+ * with @reg_lock held before allocating -+ * the backing pages. This helps ensure that -+ * total physical memory usage for just in -+ * time memory allocation remains within the -+ * @jit_phys_pages_limit in multi-threaded -+ * scenarios. - * @jit_active_head: List containing the just-in-time memory allocations - * which are in use. - * @jit_pool_head: List containing the just-in-time memory allocations -@@ -1425,6 +1652,10 @@ struct kbase_sub_alloc { - * is used to determine the atom's age when it is added to - * the runnable RB-tree. - * @trim_level: Level of JIT allocation trimming to perform on free (0-100%) -+ * @kprcs: Reference to @struct kbase_process that the current -+ * kbase_context belongs to. -+ * @kprcs_link: List link for the list of kbase context maintained -+ * under kbase_process. - * @gwt_enabled: Indicates if tracking of GPU writes is enabled, protected by - * kbase_context.reg_lock. - * @gwt_was_enabled: Simple sticky bit flag to know if GWT was ever enabled. -@@ -1435,6 +1666,12 @@ struct kbase_sub_alloc { - * for context scheduling, protected by hwaccess_lock. - * @atoms_count: Number of GPU atoms currently in use, per priority - * @create_flags: Flags used in context creation. -+ * @kinstr_jm: Kernel job manager instrumentation context handle -+ * @tl_kctx_list_node: List item into the device timeline's list of -+ * contexts, for timeline summarization. -+ * @limited_core_mask: The mask that is applied to the affinity in case of atoms -+ * marked with BASE_JD_REQ_LIMITED_CORE_MASK. -+ * @platform_data: Pointer to platform specific per-context data. - * - * A kernel base context is an entity among which the GPU is scheduled. - * Each context has its own GPU address space. -@@ -1453,7 +1690,9 @@ struct kbase_context { - struct list_head event_list; - struct list_head event_coalesce_list; - struct mutex event_mutex; -+#if !MALI_USE_CSF - atomic_t event_closed; -+#endif - struct workqueue_struct *event_workq; - atomic_t event_count; - int event_coalesce_count; -@@ -1470,7 +1709,11 @@ struct kbase_context { - struct rb_root reg_rbtree_same; - struct rb_root reg_rbtree_custom; - struct rb_root reg_rbtree_exec; -+ struct kbase_reg_zone reg_zone[KBASE_REG_ZONE_MAX]; - -+#if MALI_USE_CSF -+ struct kbase_csf_context csf; -+#else - struct kbase_jd_context jctx; - struct jsctx_queue jsctx_queue - [KBASE_JS_ATOM_SCHED_PRIO_COUNT][BASE_JM_MAX_NR_SLOTS]; -@@ -1488,6 +1731,7 @@ struct kbase_context { - s16 atoms_count[KBASE_JS_ATOM_SCHED_PRIO_COUNT]; - u32 slots_pullable; - u32 age_count; -+#endif /* MALI_USE_CSF */ - - DECLARE_BITMAP(cookies, BITS_PER_LONG); - struct kbase_va_region *pending_regions[BITS_PER_LONG]; -@@ -1503,6 +1747,7 @@ struct kbase_context { - - struct shrinker reclaim; - struct list_head evict_list; -+ atomic_t evict_nents; - - struct list_head waiting_soft_jobs; - spinlock_t waiting_soft_jobs_lock; -@@ -1519,12 +1764,10 @@ struct kbase_context { - - spinlock_t mm_update_lock; - struct mm_struct __rcu *process_mm; -- u64 same_va_end; -- u64 exec_va_start; - u64 gpu_va_end; - bool jit_va; - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - char *mem_profile_data; - size_t mem_profile_size; - struct mutex mem_profile_lock; -@@ -1545,10 +1788,11 @@ struct kbase_context { - u8 jit_current_allocations_per_bin[256]; - u8 jit_version; - u8 jit_group_id; --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - u64 jit_phys_pages_limit; - u64 jit_current_phys_pressure; --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+ u64 jit_phys_pages_to_be_allocated; -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - struct list_head jit_active_head; - struct list_head jit_pool_head; - struct list_head jit_destroy_head; -@@ -1559,6 +1803,9 @@ struct kbase_context { - - u8 trim_level; - -+ struct kbase_process *kprcs; -+ struct list_head kprcs_link; -+ - #ifdef CONFIG_MALI_CINSTR_GWT - bool gwt_enabled; - bool gwt_was_enabled; -@@ -1567,6 +1814,17 @@ struct kbase_context { - #endif - - base_context_create_flags create_flags; -+ -+#if !MALI_USE_CSF -+ struct kbase_kinstr_jm *kinstr_jm; -+#endif -+ struct list_head tl_kctx_list_node; -+ -+ u64 limited_core_mask; -+ -+#if !MALI_USE_CSF -+ void *platform_data; -+#endif - }; - - #ifdef CONFIG_MALI_CINSTR_GWT -@@ -1653,29 +1911,4 @@ static inline bool kbase_device_is_cpu_coherent(struct kbase_device *kbdev) - /* Maximum number of loops polling the GPU for an AS command to complete before we assume the GPU has hung */ - #define KBASE_AS_INACTIVE_MAX_LOOPS 100000000 - --/* JobDescriptorHeader - taken from the architecture specifications, the layout -- * is currently identical for all GPU archs. */ --struct job_descriptor_header { -- u32 exception_status; -- u32 first_incomplete_task; -- u64 fault_pointer; -- u8 job_descriptor_size : 1; -- u8 job_type : 7; -- u8 job_barrier : 1; -- u8 _reserved_01 : 1; -- u8 _reserved_1 : 1; -- u8 _reserved_02 : 1; -- u8 _reserved_03 : 1; -- u8 _reserved_2 : 1; -- u8 _reserved_04 : 1; -- u8 _reserved_05 : 1; -- u16 job_index; -- u16 job_dependency_index_1; -- u16 job_dependency_index_2; -- union { -- u64 _64; -- u32 _32; -- } next_job; --}; -- - #endif /* _KBASE_DEFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c -index b5ac414..7d6e475 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_disjoint_events.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c -index 25acbcb..69ff8cc 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dma_fence.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2017,2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2016, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,26 +17,21 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* Include mali_kbase_dma_fence.h before checking for CONFIG_MALI_DMA_FENCE as - * it will be set there. - */ - #include "mali_kbase_dma_fence.h" -- - #include <linux/atomic.h> - #include <linux/list.h> - #include <linux/lockdep.h> - #include <linux/mutex.h> --#include <linux/reservation.h> -+#include <linux/version.h> - #include <linux/slab.h> - #include <linux/spinlock.h> - #include <linux/workqueue.h> - #include <linux/ww_mutex.h> -- - #include <mali_kbase.h> - - static void -@@ -59,7 +55,11 @@ static int - kbase_dma_fence_lock_reservations(struct kbase_dma_fence_resv_info *info, - struct ww_acquire_ctx *ctx) - { -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - struct reservation_object *content_res = NULL; -+#else -+ struct dma_resv *content_res = NULL; -+#endif - unsigned int content_res_idx = 0; - unsigned int r; - int err = 0; -@@ -205,7 +205,7 @@ out: - } - - static void --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - kbase_dma_fence_cb(struct fence *fence, struct fence_cb *cb) - #else - kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) -@@ -225,12 +225,19 @@ kbase_dma_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) - kbase_dma_fence_queue_work(katom); - } - -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - static int - kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, - struct reservation_object *resv, - bool exclusive) -+#else -+static int -+kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, -+ struct dma_resv *resv, -+ bool exclusive) -+#endif - { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *excl_fence = NULL; - struct fence **shared_fences = NULL; - #else -@@ -240,7 +247,12 @@ kbase_dma_fence_add_reservation_callback(struct kbase_jd_atom *katom, - unsigned int shared_count = 0; - int err, i; - -- err = reservation_object_get_fences_rcu(resv, -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) -+ err = reservation_object_get_fences_rcu( -+#else -+ err = dma_resv_get_fences_rcu( -+#endif -+ resv, - &excl_fence, - &shared_count, - &shared_fences); -@@ -294,9 +306,15 @@ out: - return err; - } - -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - void kbase_dma_fence_add_reservation(struct reservation_object *resv, - struct kbase_dma_fence_resv_info *info, - bool exclusive) -+#else -+void kbase_dma_fence_add_reservation(struct dma_resv *resv, -+ struct kbase_dma_fence_resv_info *info, -+ bool exclusive) -+#endif - { - unsigned int i; - -@@ -317,7 +335,7 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, - struct kbase_dma_fence_resv_info *info) - { - int err, i; --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; - #else - struct dma_fence *fence; -@@ -346,10 +364,17 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, - } - - for (i = 0; i < info->dma_fence_resv_count; i++) { -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - struct reservation_object *obj = info->resv_objs[i]; -- -+#else -+ struct dma_resv *obj = info->resv_objs[i]; -+#endif - if (!test_bit(i, info->dma_fence_excl_bitmap)) { -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - err = reservation_object_reserve_shared(obj); -+#else -+ err = dma_resv_reserve_shared(obj, 0); -+#endif - if (err) { - dev_err(katom->kctx->kbdev->dev, - "Error %d reserving space for shared fence.\n", err); -@@ -363,7 +388,11 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, - goto end; - } - -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - reservation_object_add_shared_fence(obj, fence); -+#else -+ dma_resv_add_shared_fence(obj, fence); -+#endif - } else { - err = kbase_dma_fence_add_reservation_callback(katom, obj, true); - if (err) { -@@ -372,7 +401,11 @@ int kbase_dma_fence_wait(struct kbase_jd_atom *katom, - goto end; - } - -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - reservation_object_add_excl_fence(obj, fence); -+#else -+ dma_resv_add_excl_fence(obj, fence); -+#endif - } - } - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h -index 2a4d6fc..38d3581 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dma_fence.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2016, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_DMA_FENCE_H_ -@@ -26,10 +25,14 @@ - #ifdef CONFIG_MALI_DMA_FENCE - - #include <linux/list.h> -+#include <linux/version.h> -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - #include <linux/reservation.h> -+#else -+#include <linux/dma-resv.h> -+#endif - #include <mali_kbase_fence.h> - -- - /* Forward declaration from mali_kbase_defs.h */ - struct kbase_jd_atom; - struct kbase_context; -@@ -45,7 +48,11 @@ struct kbase_context; - * reservation objects. - */ - struct kbase_dma_fence_resv_info { -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - struct reservation_object **resv_objs; -+#else -+ struct dma_resv **resv_objs; -+#endif - unsigned int dma_fence_resv_count; - unsigned long *dma_fence_excl_bitmap; - }; -@@ -60,9 +67,15 @@ struct kbase_dma_fence_resv_info { - * reservation_objects. At the same time keeps track of which objects require - * exclusive access in dma_fence_excl_bitmap. - */ -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - void kbase_dma_fence_add_reservation(struct reservation_object *resv, - struct kbase_dma_fence_resv_info *info, - bool exclusive); -+#else -+void kbase_dma_fence_add_reservation(struct dma_resv *resv, -+ struct kbase_dma_fence_resv_info *info, -+ bool exclusive); -+#endif - - /** - * kbase_dma_fence_wait() - Creates a new fence and attaches it to the resv_objs -@@ -122,8 +135,7 @@ void kbase_dma_fence_term(struct kbase_context *kctx); - */ - int kbase_dma_fence_init(struct kbase_context *kctx); - -- --#else /* CONFIG_MALI_DMA_FENCE */ -+#else /* !CONFIG_MALI_DMA_FENCE */ - /* Dummy functions for when dma-buf fence isn't enabled. */ - - static inline int kbase_dma_fence_init(struct kbase_context *kctx) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dummy_job_wa.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dummy_job_wa.c -index 188e53b..1e91ba0 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dummy_job_wa.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dummy_job_wa.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -25,7 +24,7 @@ - */ - - #include <mali_kbase.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include <mali_kbase_dummy_job_wa.h> - - #include <linux/firmware.h> -@@ -282,6 +281,8 @@ int kbase_dummy_job_wa_load(struct kbase_device *kbdev) - int err; - struct kbase_context *kctx; - -+ lockdep_assert_held(&kbdev->fw_load_lock); -+ - if (!wa_blob_load_needed(kbdev)) - return 0; - -@@ -427,6 +428,10 @@ void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) - { - struct kbase_context *wa_ctx; - -+ /* return if the dummy job has not been loaded */ -+ if (kbdev->dummy_job_wa_loaded == false) -+ return; -+ - /* Can be safely called even if the file wasn't created on probe */ - sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr); - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dummy_job_wa.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dummy_job_wa.h -index 5bbe37d..8713ba1 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dummy_job_wa.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dummy_job_wa.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_DUMMY_JOB_WORKAROUND_ -@@ -31,6 +30,34 @@ - KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP | \ - KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) - -+#if MALI_USE_CSF -+ -+static inline int kbase_dummy_job_wa_load(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+ return 0; -+} -+ -+static inline void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+} -+ -+static inline int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, -+ u64 cores) -+{ -+ CSTD_UNUSED(kbdev); -+ CSTD_UNUSED(cores); -+ return 0; -+} -+ -+static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) -+{ -+ CSTD_UNUSED(kbdev); -+ return false; -+} -+ -+#else - - int kbase_dummy_job_wa_load(struct kbase_device *kbdev); - void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev); -@@ -41,5 +68,6 @@ static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev) - return (kbdev->dummy_job_wa.ctx != NULL); - } - -+#endif /* MALI_USE_CSF */ - - #endif /* _KBASE_DUMMY_JOB_WORKAROUND_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dvfs_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dvfs_debugfs.c -new file mode 100644 -index 0000000..1e584de ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dvfs_debugfs.c -@@ -0,0 +1,98 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_dvfs_debugfs.h" -+#include <mali_kbase.h> -+#include <linux/seq_file.h> -+ -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+ -+/** -+ * kbasep_dvfs_utilization_debugfs_show() - Print the DVFS utilization info -+ * -+ * @file: The seq_file for printing to -+ * @data: The debugfs dentry private data, a pointer to kbase_context -+ * -+ * Return: Negative error code or 0 on success. -+ */ -+static int kbasep_dvfs_utilization_debugfs_show(struct seq_file *file, void *data) -+{ -+ struct kbase_device *kbdev = file->private; -+ -+#if MALI_USE_CSF -+ seq_printf(file, "busy_time: %u idle_time: %u protm_time: %u\n", -+ kbdev->pm.backend.metrics.values.time_busy, -+ kbdev->pm.backend.metrics.values.time_idle, -+ kbdev->pm.backend.metrics.values.time_in_protm); -+#else -+ seq_printf(file, "busy_time: %u idle_time: %u\n", -+ kbdev->pm.backend.metrics.values.time_busy, -+ kbdev->pm.backend.metrics.values.time_idle); -+#endif -+ -+ return 0; -+} -+ -+static int kbasep_dvfs_utilization_debugfs_open(struct inode *in, -+ struct file *file) -+{ -+ return single_open(file, kbasep_dvfs_utilization_debugfs_show, -+ in->i_private); -+} -+ -+static const struct file_operations kbasep_dvfs_utilization_debugfs_fops = { -+ .open = kbasep_dvfs_utilization_debugfs_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+void kbase_dvfs_status_debugfs_init(struct kbase_device *kbdev) -+{ -+ struct dentry *file; -+#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) -+ const mode_t mode = 0444; -+#else -+ const mode_t mode = 0400; -+#endif -+ -+ if (WARN_ON(!kbdev || IS_ERR_OR_NULL(kbdev->mali_debugfs_directory))) -+ return; -+ -+ file = debugfs_create_file("dvfs_utilization", mode, -+ kbdev->mali_debugfs_directory, kbdev, -+ &kbasep_dvfs_utilization_debugfs_fops); -+ -+ if (IS_ERR_OR_NULL(file)) { -+ dev_warn(kbdev->dev, -+ "Unable to create dvfs debugfs entry"); -+ } -+} -+ -+#else -+/* -+ * Stub functions for when debugfs is disabled -+ */ -+void kbase_dvfs_status_debugfs_init(struct kbase_device *kbdev) -+{ -+} -+ -+#endif /* CONFIG_DEBUG_FS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dvfs_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dvfs_debugfs.h -new file mode 100644 -index 0000000..8334db7 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_dvfs_debugfs.h -@@ -0,0 +1,35 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_DVFS_DEBUGFS_H_ -+#define _KBASE_DVFS_DEBUGFS_H_ -+ -+/* Forward declaration */ -+struct kbase_device; -+ -+/** -+ * kbase_dvfs_status_debugfs_init() - Create a debugfs entry for DVFS queries -+ * -+ * @kbdev: Pointer to the GPU device for which to create the debugfs entry -+ */ -+void kbase_dvfs_status_debugfs_init(struct kbase_device *kbdev); -+ -+#endif /* _KBASE_DVFS_DEBUGFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_event.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_event.c -index c8b8f22..910c511 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_event.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_event.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2016,2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #include <mali_kbase.h> - #include <mali_kbase_debug.h> - #include <tl/mali_kbase_tracepoints.h> -@@ -45,7 +42,7 @@ static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, stru - KBASE_TLSTREAM_TL_DEL_ATOM(kbdev, katom); - - katom->status = KBASE_JD_ATOM_STATE_UNUSED; -- dev_dbg(kbdev->dev, "Atom %p status to unused\n", (void *)katom); -+ dev_dbg(kbdev->dev, "Atom %pK status to unused\n", (void *)katom); - wake_up(&katom->completed); - - return data; -@@ -82,7 +79,7 @@ int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *ueve - - mutex_unlock(&ctx->event_mutex); - -- dev_dbg(ctx->kbdev->dev, "event dequeuing %p\n", (void *)atom); -+ dev_dbg(ctx->kbdev->dev, "event dequeuing %pK\n", (void *)atom); - uevent->event_code = atom->event_code; - - uevent->atom_number = (atom - ctx->jctx.atoms); -@@ -154,7 +151,8 @@ static int kbase_event_coalesce(struct kbase_context *kctx) - const int event_count = kctx->event_coalesce_count; - - /* Join the list of pending events onto the tail of the main list -- and reset it */ -+ * and reset it -+ */ - list_splice_tail_init(&kctx->event_coalesce_list, &kctx->event_list); - kctx->event_coalesce_count = 0; - -@@ -166,7 +164,17 @@ void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom) - { - struct kbase_device *kbdev = ctx->kbdev; - -- dev_dbg(kbdev->dev, "Posting event for atom %p\n", (void *)atom); -+ dev_dbg(kbdev->dev, "Posting event for atom %pK\n", (void *)atom); -+ -+ if (WARN_ON(atom->status != KBASE_JD_ATOM_STATE_COMPLETED)) { -+ dev_warn(kbdev->dev, -+ "%s: Atom %d (%pK) not completed (status %d)\n", -+ __func__, -+ kbase_jd_atom_id(atom->kctx, atom), -+ atom->kctx, -+ atom->status); -+ return; -+ } - - if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) { - if (atom->event_code == BASE_JD_EVENT_DONE) { -@@ -227,7 +235,7 @@ int kbase_event_init(struct kbase_context *kctx) - kctx->event_coalesce_count = 0; - kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1); - -- if (NULL == kctx->event_workq) -+ if (kctx->event_workq == NULL) - return -EINVAL; - - return 0; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence.c -index 7a715b3..01557cd 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,79 +17,18 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/atomic.h> - #include <linux/list.h> - #include <linux/spinlock.h> --#include <mali_kbase_fence_defs.h> - #include <mali_kbase_fence.h> - #include <mali_kbase.h> - - /* Spin lock protecting all Mali fences as fence->lock. */ - static DEFINE_SPINLOCK(kbase_fence_lock); - --static const char * --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) --kbase_fence_get_driver_name(struct fence *fence) --#else --kbase_fence_get_driver_name(struct dma_fence *fence) --#endif --{ -- return kbase_drv_name; --} -- --static const char * --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) --kbase_fence_get_timeline_name(struct fence *fence) --#else --kbase_fence_get_timeline_name(struct dma_fence *fence) --#endif --{ -- return kbase_timeline_name; --} -- --static bool --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) --kbase_fence_enable_signaling(struct fence *fence) --#else --kbase_fence_enable_signaling(struct dma_fence *fence) --#endif --{ -- return true; --} -- --static void --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) --kbase_fence_fence_value_str(struct fence *fence, char *str, int size) --#else --kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) --#endif --{ --#if (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) -- snprintf(str, size, "%u", fence->seqno); --#else -- snprintf(str, size, "%llu", fence->seqno); --#endif --} -- --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) --const struct fence_ops kbase_fence_ops = { -- .wait = fence_default_wait, --#else --const struct dma_fence_ops kbase_fence_ops = { -- .wait = dma_fence_default_wait, --#endif -- .get_driver_name = kbase_fence_get_driver_name, -- .get_timeline_name = kbase_fence_get_timeline_name, -- .enable_signaling = kbase_fence_enable_signaling, -- .fence_value_str = kbase_fence_fence_value_str --}; -- -- --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence * - kbase_fence_out_new(struct kbase_jd_atom *katom) - #else -@@ -96,7 +36,7 @@ struct dma_fence * - kbase_fence_out_new(struct kbase_jd_atom *katom) - #endif - { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; - #else - struct dma_fence *fence; -@@ -157,7 +97,7 @@ kbase_fence_free_callbacks(struct kbase_jd_atom *katom) - return res; - } - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - int - kbase_fence_add_callback(struct kbase_jd_atom *katom, - struct fence *fence, -@@ -211,4 +151,3 @@ kbase_fence_add_callback(struct kbase_jd_atom *katom, - - return err; - } -- -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence.h -index 8e7024e..37823d5 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_FENCE_H_ -@@ -35,7 +34,7 @@ - #include "mali_kbase_fence_defs.h" - #include "mali_kbase.h" - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - extern const struct fence_ops kbase_fence_ops; - #else - extern const struct dma_fence_ops kbase_fence_ops; -@@ -49,7 +48,7 @@ extern const struct dma_fence_ops kbase_fence_ops; - * @node: List head for linking this callback to the katom - */ - struct kbase_fence_cb { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence_cb fence_cb; - struct fence *fence; - #else -@@ -66,7 +65,7 @@ struct kbase_fence_cb { - * - * return: A new fence object on success, NULL on failure. - */ --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *kbase_fence_out_new(struct kbase_jd_atom *katom); - #else - struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); -@@ -88,6 +87,7 @@ struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom); - #endif - - -+#if !MALI_USE_CSF - /** - * kbase_fence_out_remove() - Removes the output fence from atom - * @katom: Atom to remove output fence for -@@ -168,7 +168,7 @@ static inline int kbase_fence_out_signal(struct kbase_jd_atom *katom, - * Return: 0 on success: fence was either already signaled, or callback was - * set up. Negative error code is returned on error. - */ --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - int kbase_fence_add_callback(struct kbase_jd_atom *katom, - struct fence *fence, - fence_func_t callback); -@@ -269,6 +269,7 @@ bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom); - */ - #define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence) - -+#endif /* !MALI_USE_CSF */ - - /** - * kbase_fence_put() - Releases a reference to a fence -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h -index 607a95c..7a150bd 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence_defs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_FENCE_DEFS_H_ -@@ -28,11 +27,9 @@ - * This file hides the compatibility issues with this for the rest the driver - */ - --#if defined(CONFIG_MALI_DMA_FENCE) || defined(CONFIG_SYNC_FILE) -- - #include <linux/version.h> - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - - #include <linux/fence.h> - -@@ -55,7 +52,7 @@ - - #include <linux/dma-fence.h> - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 11, 0)) -+#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) - #define dma_fence_get_status(a) (dma_fence_is_signaled(a) ? \ - (a)->status ?: 1 \ - : 0) -@@ -63,6 +60,4 @@ - - #endif /* < 4.10.0 */ - --#endif /* CONFIG_MALI_DMA_FENCE || CONFIG_SYNC_FILE */ -- - #endif /* _KBASE_FENCE_DEFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence_ops.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence_ops.c -new file mode 100644 -index 0000000..14ddf03 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_fence_ops.c -@@ -0,0 +1,83 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <linux/atomic.h> -+#include <linux/list.h> -+#include <mali_kbase_fence_defs.h> -+#include <mali_kbase.h> -+ -+static const char * -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+kbase_fence_get_driver_name(struct fence *fence) -+#else -+kbase_fence_get_driver_name(struct dma_fence *fence) -+#endif -+{ -+ return kbase_drv_name; -+} -+ -+static const char * -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+kbase_fence_get_timeline_name(struct fence *fence) -+#else -+kbase_fence_get_timeline_name(struct dma_fence *fence) -+#endif -+{ -+ return kbase_timeline_name; -+} -+ -+static bool -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+kbase_fence_enable_signaling(struct fence *fence) -+#else -+kbase_fence_enable_signaling(struct dma_fence *fence) -+#endif -+{ -+ return true; -+} -+ -+static void -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+kbase_fence_fence_value_str(struct fence *fence, char *str, int size) -+#else -+kbase_fence_fence_value_str(struct dma_fence *fence, char *str, int size) -+#endif -+{ -+#if (KERNEL_VERSION(5, 1, 0) > LINUX_VERSION_CODE) -+ snprintf(str, size, "%u", fence->seqno); -+#else -+ snprintf(str, size, "%llu", fence->seqno); -+#endif -+} -+ -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -+const struct fence_ops kbase_fence_ops = { -+ .wait = fence_default_wait, -+#else -+const struct dma_fence_ops kbase_fence_ops = { -+ .wait = dma_fence_default_wait, -+#endif -+ .get_driver_name = kbase_fence_get_driver_name, -+ .get_timeline_name = kbase_fence_get_timeline_name, -+ .enable_signaling = kbase_fence_enable_signaling, -+ .fence_value_str = kbase_fence_fence_value_str -+}; -+ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gator.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gator.h -index 6428f08..88c96e0 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gator.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gator.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* NB taken from gator */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c -index 93f1565..6eaae83 100755 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2017, 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2017, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,24 +17,23 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> - #include <device/mali_kbase_device.h> - --#ifdef CONFIG_DEBUG_FS --/** Show callback for the @c gpu_memory debugfs file. -+#if IS_ENABLED(CONFIG_DEBUG_FS) -+/** -+ * kbasep_gpu_memory_seq_show - Show callback for the @c gpu_memory debugfs file -+ * @sfile: The debugfs entry -+ * @data: Data associated with the entry - * - * This function is called to get the contents of the @c gpu_memory debugfs - * file. This is a report of current gpu memory usage. - * -- * @param sfile The debugfs entry -- * @param data Data associated with the entry -- * -- * @return 0 if successfully prints data in debugfs entry file -- * -1 if it encountered an error -+ * Return: -+ * * 0 if successfully prints data in debugfs entry file -+ * * -1 if it encountered an error - */ - - static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h -index 28a871a..6d5423f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpu_memory_debugfs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2014, 2016 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2014, 2016, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_gpu_memory_debugfs.h -- * Header file for gpu_memory entry in debugfs -+ * DOC: Header file for gpu_memory entry in debugfs - * - */ - -@@ -34,8 +30,20 @@ - #include <linux/debugfs.h> - #include <linux/seq_file.h> - -+/* kbase_io_history_add - add new entry to the register access history -+ * -+ * @h: Pointer to the history data structure -+ * @addr: Register address -+ * @value: The value that is either read from or written to the register -+ * @write: 1 if it's a register write, 0 if it's a read -+ */ -+void kbase_io_history_add(struct kbase_io_history *h, void __iomem const *addr, -+ u32 value, u8 write); -+ - /** -- * @brief Initialize gpu_memory debugfs entry -+ * kbasep_gpu_memory_debugfs_init - Initialize gpu_memory debugfs entry -+ * -+ * @kbdev: Device pointer - */ - void kbasep_gpu_memory_debugfs_init(struct kbase_device *kbdev); - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c -index ae2458f..e4d52c9 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops.c -@@ -1,12 +1,12 @@ --// SPDX-License-Identifier: GPL-2.0 -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -17,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Base kernel property query APIs - */ -@@ -32,9 +28,9 @@ - #include <mali_kbase_gpuprops.h> - #include <mali_kbase_hwaccess_gpuprops.h> - #include <mali_kbase_config_defaults.h> --#include "mali_kbase_ioctl.h" -+#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h> - #include <linux/clk.h> --#include <mali_kbase_pm_internal.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> - #include <linux/of_platform.h> - #include <linux/moduleparam.h> - -@@ -48,7 +44,7 @@ static void kbase_gpuprops_construct_coherent_groups( - u64 first_set, first_set_prev; - u32 num_groups = 0; - -- KBASE_DEBUG_ASSERT(NULL != props); -+ KBASE_DEBUG_ASSERT(props != NULL); - - props->coherency_info.coherency = props->raw_props.mem_features; - props->coherency_info.num_core_groups = hweight64(props->raw_props.l2_present); -@@ -107,6 +103,71 @@ static void kbase_gpuprops_construct_coherent_groups( - props->coherency_info.num_groups = num_groups; - } - -+/** -+ * kbase_gpuprops_get_curr_config_props - Get the current allocated resources -+ * @kbdev: The &struct kbase_device structure for the device -+ * @curr_config: The &struct curr_config_props structure to receive the result -+ * -+ * Fill the &struct curr_config_props structure with values from the GPU -+ * configuration registers. -+ * -+ * Return: Zero on success, Linux error code on failure -+ */ -+int kbase_gpuprops_get_curr_config_props(struct kbase_device *kbdev, -+ struct curr_config_props * const curr_config) -+{ -+ struct kbase_current_config_regdump curr_config_regdump; -+ int err; -+ -+ if (WARN_ON(!kbdev) || WARN_ON(!curr_config)) -+ return -EINVAL; -+ -+ /* If update not needed just return. */ -+ if (!curr_config->update_needed) -+ return 0; -+ -+ /* Dump relevant registers */ -+ err = kbase_backend_gpuprops_get_curr_config(kbdev, -+ &curr_config_regdump); -+ if (err) -+ return err; -+ -+ curr_config->l2_slices = -+ KBASE_UBFX32(curr_config_regdump.mem_features, 8U, 4) + 1; -+ -+ curr_config->l2_present = -+ ((u64) curr_config_regdump.l2_present_hi << 32) + -+ curr_config_regdump.l2_present_lo; -+ -+ curr_config->shader_present = -+ ((u64) curr_config_regdump.shader_present_hi << 32) + -+ curr_config_regdump.shader_present_lo; -+ -+ curr_config->num_cores = hweight64(curr_config->shader_present); -+ -+ curr_config->update_needed = false; -+ -+ return 0; -+} -+ -+/** -+ * kbase_gpuprops_req_curr_config_update - Request Current Config Update -+ * @kbdev: The &struct kbase_device structure for the device -+ * -+ * Requests the current configuration to be updated next time the -+ * kbase_gpuprops_get_curr_config_props() is called. -+ * -+ * Return: Zero on success, Linux error code on failure -+ */ -+int kbase_gpuprops_req_curr_config_update(struct kbase_device *kbdev) -+{ -+ if (WARN_ON(!kbdev)) -+ return -EINVAL; -+ -+ kbdev->gpu_props.curr_config.update_needed = true; -+ return 0; -+} -+ - /** - * kbase_gpuprops_get_props - Get the GPU configuration - * @gpu_props: The &struct base_gpu_props structure -@@ -124,8 +185,8 @@ static int kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props, - int i; - int err; - -- KBASE_DEBUG_ASSERT(NULL != kbdev); -- KBASE_DEBUG_ASSERT(NULL != gpu_props); -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(gpu_props != NULL); - - /* Dump relevant registers */ - err = kbase_backend_gpuprops_get(kbdev, ®dump); -@@ -166,6 +227,10 @@ static int kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props, - gpu_props->raw_props.thread_features = regdump.thread_features; - gpu_props->raw_props.thread_tls_alloc = regdump.thread_tls_alloc; - -+ gpu_props->raw_props.gpu_features = -+ ((u64) regdump.gpu_features_hi << 32) + -+ regdump.gpu_features_lo; -+ - return 0; - } - -@@ -182,6 +247,59 @@ void kbase_gpuprops_update_core_props_gpu_id( - KBASE_UBFX32(gpu_props->raw_props.gpu_id, 16U, 16); - } - -+/** -+ * kbase_gpuprops_update_max_config_props - Updates the max config properties in -+ * the base_gpu_props. -+ * @base_props: The &struct base_gpu_props structure -+ * @kbdev: The &struct kbase_device structure for the device -+ * -+ * Updates the &struct base_gpu_props structure with the max config properties. -+ */ -+static void kbase_gpuprops_update_max_config_props( -+ struct base_gpu_props * const base_props, struct kbase_device *kbdev) -+{ -+ int l2_n = 0; -+ -+ if (WARN_ON(!kbdev) || WARN_ON(!base_props)) -+ return; -+ -+ /* return if the max_config is not set during arbif initialization */ -+ if (kbdev->gpu_props.max_config.core_mask == 0) -+ return; -+ -+ /* -+ * Set the base_props with the maximum config values to ensure that the -+ * user space will always be based on the maximum resources available. -+ */ -+ base_props->l2_props.num_l2_slices = -+ kbdev->gpu_props.max_config.l2_slices; -+ base_props->raw_props.shader_present = -+ kbdev->gpu_props.max_config.core_mask; -+ /* -+ * Update l2_present in the raw data to be consistent with the -+ * max_config.l2_slices number. -+ */ -+ base_props->raw_props.l2_present = 0; -+ for (l2_n = 0; l2_n < base_props->l2_props.num_l2_slices; l2_n++) { -+ base_props->raw_props.l2_present <<= 1; -+ base_props->raw_props.l2_present |= 0x1; -+ } -+ /* -+ * Update the coherency_info data using just one core group. For -+ * architectures where the max_config is provided by the arbiter it is -+ * not necessary to split the shader core groups in different coherent -+ * groups. -+ */ -+ base_props->coherency_info.coherency = -+ base_props->raw_props.mem_features; -+ base_props->coherency_info.num_core_groups = 1; -+ base_props->coherency_info.num_groups = 1; -+ base_props->coherency_info.group[0].core_mask = -+ kbdev->gpu_props.max_config.core_mask; -+ base_props->coherency_info.group[0].num_cores = -+ hweight32(kbdev->gpu_props.max_config.core_mask); -+} -+ - /** - * kbase_gpuprops_calculate_props - Calculate the derived properties - * @gpu_props: The &struct base_gpu_props structure -@@ -195,7 +313,6 @@ static void kbase_gpuprops_calculate_props( - { - int i; - u32 gpu_id; -- u32 product_id; - - /* Populate the base_gpu_props structure */ - kbase_gpuprops_update_core_props_gpu_id(gpu_props); -@@ -218,7 +335,8 @@ static void kbase_gpuprops_calculate_props( - - /* Field with number of l2 slices is added to MEM_FEATURES register - * since t76x. Below code assumes that for older GPU reserved bits will -- * be read as zero. */ -+ * be read as zero. -+ */ - gpu_props->l2_props.num_l2_slices = - KBASE_UBFX32(gpu_props->raw_props.mem_features, 8U, 4) + 1; - -@@ -251,9 +369,19 @@ static void kbase_gpuprops_calculate_props( - * Workaround for the incorrectly applied THREAD_FEATURES to tDUx. - */ - gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -- product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID; -- product_id >>= GPU_ID_VERSION_PRODUCT_ID_SHIFT; - -+#if MALI_USE_CSF -+ gpu_props->thread_props.max_registers = -+ KBASE_UBFX32(gpu_props->raw_props.thread_features, -+ 0U, 22); -+ gpu_props->thread_props.impl_tech = -+ KBASE_UBFX32(gpu_props->raw_props.thread_features, -+ 22U, 2); -+ gpu_props->thread_props.max_task_queue = -+ KBASE_UBFX32(gpu_props->raw_props.thread_features, -+ 24U, 8); -+ gpu_props->thread_props.max_thread_group_split = 0; -+#else - if ((gpu_id & GPU_ID2_PRODUCT_MODEL) == GPU_ID2_PRODUCT_TDUX) { - gpu_props->thread_props.max_registers = - KBASE_UBFX32(gpu_props->raw_props.thread_features, -@@ -279,6 +407,7 @@ static void kbase_gpuprops_calculate_props( - KBASE_UBFX32(gpu_props->raw_props.thread_features, - 30U, 2); - } -+#endif - - /* If values are not specified, then use defaults */ - if (gpu_props->thread_props.max_registers == 0) { -@@ -286,8 +415,30 @@ static void kbase_gpuprops_calculate_props( - gpu_props->thread_props.max_task_queue = THREAD_MTQ_DEFAULT; - gpu_props->thread_props.max_thread_group_split = THREAD_MTGS_DEFAULT; - } -- /* Initialize the coherent_group structure for each group */ -- kbase_gpuprops_construct_coherent_groups(gpu_props); -+ -+ /* -+ * If the maximum resources allocated information is available it is -+ * necessary to update the base_gpu_props with the max_config info to -+ * the userspace. This is applicable to systems that receive this -+ * information from the arbiter. -+ */ -+ if (kbdev->gpu_props.max_config.core_mask) -+ /* Update the max config properties in the base_gpu_props */ -+ kbase_gpuprops_update_max_config_props(gpu_props, -+ kbdev); -+ else -+ /* Initialize the coherent_group structure for each group */ -+ kbase_gpuprops_construct_coherent_groups(gpu_props); -+} -+ -+void kbase_gpuprops_set_max_config(struct kbase_device *kbdev, -+ const struct max_config_props *max_config) -+{ -+ if (WARN_ON(!kbdev) || WARN_ON(!max_config)) -+ return; -+ -+ kbdev->gpu_props.max_config.l2_slices = max_config->l2_slices; -+ kbdev->gpu_props.max_config.core_mask = max_config->core_mask; - } - - void kbase_gpuprops_set(struct kbase_device *kbdev) -@@ -295,7 +446,8 @@ void kbase_gpuprops_set(struct kbase_device *kbdev) - struct kbase_gpu_props *gpu_props; - struct gpu_raw_gpu_props *raw; - -- KBASE_DEBUG_ASSERT(NULL != kbdev); -+ if (WARN_ON(!kbdev)) -+ return; - gpu_props = &kbdev->gpu_props; - raw = &gpu_props->props.raw_props; - -@@ -315,9 +467,19 @@ void kbase_gpuprops_set(struct kbase_device *kbdev) - gpu_props->mmu.pa_bits = KBASE_UBFX32(raw->mmu_features, 8U, 8); - - gpu_props->num_cores = hweight64(raw->shader_present); -- gpu_props->num_core_groups = hweight64(raw->l2_present); -+ gpu_props->num_core_groups = -+ gpu_props->props.coherency_info.num_core_groups; - gpu_props->num_address_spaces = hweight32(raw->as_present); - gpu_props->num_job_slots = hweight32(raw->js_present); -+ -+ /* -+ * Current configuration is used on HW interactions so that the maximum -+ * config is just used for user space avoiding interactions with parts -+ * of the hardware that might not be allocated to the kbase instance at -+ * that moment. -+ */ -+ kbase_gpuprops_req_curr_config_update(kbdev); -+ kbase_gpuprops_get_curr_config_props(kbdev, &gpu_props->curr_config); - } - - int kbase_gpuprops_set_features(struct kbase_device *kbdev) -@@ -355,13 +517,34 @@ int kbase_gpuprops_set_features(struct kbase_device *kbdev) - * in sysfs. - */ - static u8 override_l2_size; --module_param(override_l2_size, byte, 0); -+module_param(override_l2_size, byte, 0000); - MODULE_PARM_DESC(override_l2_size, "Override L2 size config for testing"); - - static u8 override_l2_hash; --module_param(override_l2_hash, byte, 0); -+module_param(override_l2_hash, byte, 0000); - MODULE_PARM_DESC(override_l2_hash, "Override L2 hash config for testing"); - -+static u32 l2_hash_values[ASN_HASH_COUNT] = { -+ 0, -+}; -+static int num_override_l2_hash_values; -+module_param_array(l2_hash_values, uint, &num_override_l2_hash_values, 0000); -+MODULE_PARM_DESC(l2_hash_values, "Override L2 hash values config for testing"); -+ -+/* Definitions for range of supported user defined hash functions for GPUs -+ * that support L2_CONFIG and not ASN_HASH features. Supported hash function -+ * range from 0b1000-0b1111 inclusive. Selection of any other values will -+ * lead to undefined behavior. -+ */ -+#define USER_DEFINED_HASH_LO ((u8)0x08) -+#define USER_DEFINED_HASH_HI ((u8)0x0F) -+ -+enum l2_config_override_result { -+ L2_CONFIG_OVERRIDE_FAIL = -1, -+ L2_CONFIG_OVERRIDE_NONE, -+ L2_CONFIG_OVERRIDE_OK, -+}; -+ - /** - * kbase_read_l2_config_from_dt - Read L2 configuration - * @kbdev: The kbase device for which to get the L2 configuration. -@@ -370,30 +553,67 @@ MODULE_PARM_DESC(override_l2_hash, "Override L2 hash config for testing"); - * Override values in module parameters take priority over override values in - * device tree. - * -- * Return: true if either size or hash was overridden, false if no overrides -- * were found. -+ * Return: L2_CONFIG_OVERRIDE_OK if either size or hash, or both was properly -+ * overridden, L2_CONFIG_OVERRIDE_NONE if no overrides are provided. -+ * L2_CONFIG_OVERRIDE_FAIL otherwise. - */ --static bool kbase_read_l2_config_from_dt(struct kbase_device * const kbdev) -+static enum l2_config_override_result -+kbase_read_l2_config_from_dt(struct kbase_device *const kbdev) - { - struct device_node *np = kbdev->dev->of_node; - - if (!np) -- return false; -+ return L2_CONFIG_OVERRIDE_NONE; - - if (override_l2_size) - kbdev->l2_size_override = override_l2_size; - else if (of_property_read_u8(np, "l2-size", &kbdev->l2_size_override)) - kbdev->l2_size_override = 0; - -- if (override_l2_hash) -+ /* Check overriding value is supported, if not will result in -+ * undefined behavior. -+ */ -+ if (override_l2_hash >= USER_DEFINED_HASH_LO && -+ override_l2_hash <= USER_DEFINED_HASH_HI) - kbdev->l2_hash_override = override_l2_hash; - else if (of_property_read_u8(np, "l2-hash", &kbdev->l2_hash_override)) - kbdev->l2_hash_override = 0; - -- if (kbdev->l2_size_override || kbdev->l2_hash_override) -- return true; -+ kbdev->l2_hash_values_override = false; -+ if (num_override_l2_hash_values) { -+ int i; -+ -+ kbdev->l2_hash_values_override = true; -+ for (i = 0; i < num_override_l2_hash_values; i++) -+ kbdev->l2_hash_values[i] = l2_hash_values[i]; -+ } else if (!of_property_read_u32_array(np, "l2-hash-values", -+ kbdev->l2_hash_values, -+ ASN_HASH_COUNT)) -+ kbdev->l2_hash_values_override = true; -+ -+ if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH) && -+ (kbdev->l2_hash_override)) { -+ dev_err(kbdev->dev, "l2-hash not supported\n"); -+ return L2_CONFIG_OVERRIDE_FAIL; -+ } -+ -+ if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_ASN_HASH) && -+ (kbdev->l2_hash_values_override)) { -+ dev_err(kbdev->dev, "l2-hash-values not supported\n"); -+ return L2_CONFIG_OVERRIDE_FAIL; -+ } -+ -+ if (kbdev->l2_hash_override && kbdev->l2_hash_values_override) { -+ dev_err(kbdev->dev, -+ "both l2-hash & l2-hash-values not supported\n"); -+ return L2_CONFIG_OVERRIDE_FAIL; -+ } -+ -+ if (kbdev->l2_size_override || kbdev->l2_hash_override || -+ kbdev->l2_hash_values_override) -+ return L2_CONFIG_OVERRIDE_OK; - -- return false; -+ return L2_CONFIG_OVERRIDE_NONE; - } - - int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) -@@ -405,8 +625,25 @@ int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) - struct base_gpu_props *gpu_props = &kbdev->gpu_props.props; - - /* Check for L2 cache size & hash overrides */ -- if (!kbase_read_l2_config_from_dt(kbdev)) -- return 0; -+ switch (kbase_read_l2_config_from_dt(kbdev)) { -+ case L2_CONFIG_OVERRIDE_FAIL: -+ err = -EIO; -+ goto exit; -+ case L2_CONFIG_OVERRIDE_NONE: -+ goto exit; -+ default: -+ break; -+ } -+ -+ /* pm.active_count is expected to be 1 here, which is set in -+ * kbase_hwaccess_pm_powerup(). -+ */ -+ WARN_ON(kbdev->pm.active_count != 1); -+ /* The new settings for L2 cache can only be applied when it is -+ * off, so first do the power down. -+ */ -+ kbase_pm_context_idle(kbdev); -+ kbase_pm_wait_for_desired_state(kbdev); - - /* Need L2 to get powered to reflect to L2_FEATURES */ - kbase_pm_context_active(kbdev); -@@ -417,21 +654,21 @@ int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev) - /* Dump L2_FEATURES register */ - err = kbase_backend_gpuprops_get_l2_features(kbdev, ®dump); - if (err) -- goto idle_gpu; -+ goto exit; - - dev_info(kbdev->dev, "Reflected L2_FEATURES is 0x%x\n", -- regdump.l2_features); -+ regdump.l2_features); -+ dev_info(kbdev->dev, "Reflected L2_CONFIG is 0x%08x\n", -+ regdump.l2_config); -+ - - /* Update gpuprops with reflected L2_FEATURES */ - gpu_props->raw_props.l2_features = regdump.l2_features; - gpu_props->l2_props.log2_cache_size = - KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8); -- --idle_gpu: -- /* Let GPU idle */ -- kbase_pm_context_idle(kbdev); - } - -+exit: - return err; - } - -@@ -511,7 +748,7 @@ static struct { - PROP(RAW_THREAD_FEATURES, raw_props.thread_features), - PROP(RAW_THREAD_TLS_ALLOC, raw_props.thread_tls_alloc), - PROP(RAW_COHERENCY_MODE, raw_props.coherency_mode), -- -+ PROP(RAW_GPU_FEATURES, raw_props.gpu_features), - PROP(COHERENCY_NUM_GROUPS, coherency_info.num_groups), - PROP(COHERENCY_NUM_CORE_GROUPS, coherency_info.num_core_groups), - PROP(COHERENCY_COHERENCY, coherency_info.coherency), -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h -index 5eee794..b20b99b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2015, 2017, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,29 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ --/* -- * -- * (C) COPYRIGHT 2011-2015, 2017, 2019-2020 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * A copy of the licence is included with the program, and can also be obtained -- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- * Boston, MA 02110-1301, USA. -- * - */ - -- -- - /** -- * @file mali_kbase_gpuprops.h -- * Base kernel property query APIs -+ * DOC: Base kernel property query APIs - */ - - #ifndef _KBASE_GPUPROPS_H_ -@@ -64,11 +46,10 @@ struct kbase_device; - (((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1)) - - /** -- * @brief Set up Kbase GPU properties. -+ * kbase_gpuprops_set - Set up Kbase GPU properties. -+ * @kbdev: The struct kbase_device structure for the device - * - * Set up Kbase GPU properties with information from the GPU registers -- * -- * @param kbdev The struct kbase_device structure for the device - */ - void kbase_gpuprops_set(struct kbase_device *kbdev); - -@@ -89,6 +70,8 @@ int kbase_gpuprops_set_features(struct kbase_device *kbdev); - * @kbdev: Device pointer - * - * This function updates l2_features and the log2 cache size. -+ * The function expects GPU to be powered up and value of pm.active_count -+ * to be 1. - * - * Return: Zero on success, Linux error code for failure - */ -@@ -132,4 +115,38 @@ int kbase_device_populate_max_freq(struct kbase_device *kbdev); - void kbase_gpuprops_update_core_props_gpu_id( - struct base_gpu_props * const gpu_props); - -+/** -+ * kbase_gpuprops_set_max_config - Set the max config information -+ * @kbdev: Device pointer -+ * @max_config: Maximum configuration data to be updated -+ * -+ * This function sets max_config in the kbase_gpu_props. -+ */ -+void kbase_gpuprops_set_max_config(struct kbase_device *kbdev, -+ const struct max_config_props *max_config); -+ -+/** -+ * kbase_gpuprops_get_curr_config_props - Get the current allocated resources -+ * @kbdev: The &struct kbase_device structure for the device -+ * @curr_config: The &struct curr_config_props structure to receive the result -+ * -+ * Fill the &struct curr_config_props structure with values from the GPU -+ * configuration registers. -+ * -+ * Return: Zero on success, Linux error code on failure -+ */ -+int kbase_gpuprops_get_curr_config_props(struct kbase_device *kbdev, -+ struct curr_config_props * const curr_config); -+ -+/** -+ * kbase_gpuprops_req_curr_config_update - Request Current Config Update -+ * @kbdev: The &struct kbase_device structure for the device -+ * -+ * Requests the current configuration to be updated next time the -+ * kbase_gpuprops_get_curr_config_props() is called. -+ * -+ * Return: Zero on success, Linux error code on failure -+ */ -+int kbase_gpuprops_req_curr_config_update(struct kbase_device *kbdev); -+ - #endif /* _KBASE_GPUPROPS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h -index ec6f1c3..02705a0 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gpuprops_types.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,21 +17,16 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_gpuprops_types.h -- * Base kernel property query APIs -+ * DOC: Base kernel property query APIs - */ - - #ifndef _KBASE_GPUPROPS_TYPES_H_ - #define _KBASE_GPUPROPS_TYPES_H_ - --#include "mali_base_kernel.h" -+#include <uapi/gpu/arm/midgard/mali_base_kernel.h> - - #define KBASE_GPU_SPEED_MHZ 123 - #define KBASE_GPU_PC_SIZE_LOG2 24U -@@ -38,6 +34,7 @@ - struct kbase_gpuprops_regdump { - u32 gpu_id; - u32 l2_features; -+ u32 l2_config; - u32 core_features; - u32 tiler_features; - u32 mem_features; -@@ -60,6 +57,30 @@ struct kbase_gpuprops_regdump { - u32 stack_present_lo; - u32 stack_present_hi; - u32 coherency_features; -+ u32 gpu_features_lo; -+ u32 gpu_features_hi; -+}; -+ -+/** -+ * struct kbase_current_config_regdump - Register dump for current resources -+ * allocated to the GPU. -+ * @mem_features: Memory system features. Contains information about the -+ * features of the memory system. Used here to get the L2 slice -+ * count. -+ * @shader_present_lo: Shader core present bitmap. Low word. -+ * @shader_present_hi: Shader core present bitmap. High word. -+ * @l2_present_lo: L2 cache present bitmap. Low word. -+ * @l2_present_hi: L2 cache present bitmap. High word. -+ * -+ * Register dump structure used to store the resgisters data realated to the -+ * current resources allocated to the GPU. -+ */ -+struct kbase_current_config_regdump { -+ u32 mem_features; -+ u32 shader_present_lo; -+ u32 shader_present_hi; -+ u32 l2_present_lo; -+ u32 l2_present_hi; - }; - - struct kbase_gpu_cache_props { -@@ -76,6 +97,50 @@ struct kbase_gpu_mmu_props { - u8 pa_bits; - }; - -+/** -+ * struct max_config_props - Properties based on the maximum resources -+ * available. -+ * @l2_slices: Maximum number of L2 slices that can be assinged to the GPU -+ * during runtime. -+ * @padding: Padding to a multiple of 64 bits. -+ * @core_mask: Largest core mask bitmap that can be assigned to the GPU during -+ * runtime. -+ * -+ * Properties based on the maximum resources available (not necessarly -+ * allocated at that moment). Used to provide the maximum configuration to the -+ * userspace allowing the applications to allocate enough resources in case the -+ * real allocated resources change. -+ */ -+struct max_config_props { -+ u8 l2_slices; -+ u8 padding[3]; -+ u32 core_mask; -+}; -+ -+/** -+ * struct curr_config_props - Properties based on the current resources -+ * allocated to the GPU. -+ * @l2_present: Current L2 present bitmap that is allocated to the GPU. -+ * @shader_present: Current shader present bitmap that is allocated to the GPU. -+ * @num_cores: Current number of shader cores allocated to the GPU. -+ * @l2_slices: Current number of L2 slices allocated to the GPU. -+ * @update_needed: Defines if it is necessary to re-read the registers to -+ * update the current allocated resources. -+ * @padding: Padding to a multiple of 64 bits. -+ * -+ * Properties based on the current resource available. Used for operations with -+ * hardware interactions to avoid using userspace data that can be based on -+ * the maximum resource available. -+ */ -+struct curr_config_props { -+ u64 l2_present; -+ u64 shader_present; -+ u16 num_cores; -+ u8 l2_slices; -+ bool update_needed; -+ u8 padding[4]; -+}; -+ - struct kbase_gpu_props { - /* kernel-only properties */ - u8 num_cores; -@@ -88,6 +153,12 @@ struct kbase_gpu_props { - struct kbase_gpu_mem_props mem; - struct kbase_gpu_mmu_props mmu; - -+ /* Properties based on the current resource available */ -+ struct curr_config_props curr_config; -+ -+ /* Properties based on the maximum resource available */ -+ struct max_config_props max_config; -+ - /* Properties shared with userspace */ - struct base_gpu_props props; - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gwt.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gwt.c -index 6a47c9d..2a20a3d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gwt.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gwt.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase_gwt.h" -@@ -71,6 +70,7 @@ int kbase_gpu_gwt_start(struct kbase_context *kctx) - INIT_LIST_HEAD(&kctx->gwt_current_list); - INIT_LIST_HEAD(&kctx->gwt_snapshot_list); - -+#if !MALI_USE_CSF - /* If GWT is enabled using new vector dumping format - * from user space, back up status of the job serialization flag and - * use full serialisation of jobs for dumping. -@@ -80,6 +80,7 @@ int kbase_gpu_gwt_start(struct kbase_context *kctx) - kctx->kbdev->serialize_jobs = KBASE_SERIALIZE_INTRA_SLOT | - KBASE_SERIALIZE_INTER_SLOT; - -+#endif - /* Mark gwt enabled before making pages read only in case a - write page fault is triggered while we're still in this loop. - (kbase_gpu_vm_lock() doesn't prevent this!) -@@ -113,7 +114,9 @@ int kbase_gpu_gwt_stop(struct kbase_context *kctx) - kfree(pos); - } - -+#if !MALI_USE_CSF - kctx->kbdev->serialize_jobs = kctx->kbdev->backup_serialize_jobs; -+#endif - - kbase_gpu_gwt_setup_pages(kctx, ~0UL); - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gwt.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gwt.h -index 7e7746e..30de43d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gwt.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_gwt.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,13 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #if !defined(_KBASE_GWT_H) - #define _KBASE_GWT_H - - #include <mali_kbase.h> --#include <mali_kbase_ioctl.h> -+#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h> - - /** - * kbase_gpu_gwt_start - Start the GPU write tracking -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hw.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hw.c -index f8a9248..7ad583c 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hw.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hw.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Run-time work-arounds helpers - */ -@@ -68,6 +65,9 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) - case GPU_ID2_PRODUCT_TBEX: - features = base_hw_features_tBEx; - break; -+ case GPU_ID2_PRODUCT_TBAX: -+ features = base_hw_features_tBAx; -+ break; - case GPU_ID2_PRODUCT_TDUX: - features = base_hw_features_tDUx; - break; -@@ -81,14 +81,6 @@ void kbase_hw_set_features_mask(struct kbase_device *kbdev) - case GPU_ID2_PRODUCT_TVAX: - features = base_hw_features_tVAx; - break; -- case GPU_ID2_PRODUCT_TTUX: -- /* Fallthrough */ -- case GPU_ID2_PRODUCT_LTUX: -- features = base_hw_features_tTUx; -- break; -- case GPU_ID2_PRODUCT_TE2X: -- features = base_hw_features_tE2x; -- break; - default: - features = base_hw_features_generic; - break; -@@ -140,103 +132,99 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( - }; - - static const struct base_hw_product base_hw_products[] = { -- {GPU_ID2_PRODUCT_TMIX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 1), -- base_hw_issues_tMIx_r0p0_05dev0}, -- {GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tMIx_r0p1}, -- {U32_MAX /* sentinel value */, NULL} } }, -- -- {GPU_ID2_PRODUCT_THEX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1}, -- {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tHEx_r0p1}, -- {GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tHEx_r0p2}, -- {GPU_ID2_VERSION_MAKE(0, 3, 0), base_hw_issues_tHEx_r0p3}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TSIX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1}, -- {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0}, -- {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_tSIx_r1p1}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TDVX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDVx_r0p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TNOX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tNOx_r0p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TGOX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tGOx_r0p0}, -- {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tGOx_r1p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TTRX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTRx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tTRx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tTRx_r0p1}, -- {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tTRx_r0p1}, -- {GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tTRx_r0p2}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TNAX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tNAx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tNAx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 0, 4), base_hw_issues_tNAx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 0, 5), base_hw_issues_tNAx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tNAx_r0p1}, -- {GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tNAx_r0p1}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_LBEX, -- {{GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_lBEx_r1p0}, -- {GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_lBEx_r1p1}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TBEX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tBEx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tBEx_r0p0}, -- {GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tBEx_r0p1}, -- {GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBEx_r1p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TDUX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDUx_r0p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TODX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tODx_r0p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_LODX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tODx_r0p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TGRX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tGRx_r0p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TVAX, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tVAx_r0p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TTUX, -- {{GPU_ID2_VERSION_MAKE(2, 0, 0), base_hw_issues_tTUx_r0p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_LTUX, -- {{GPU_ID2_VERSION_MAKE(3, 0, 0), base_hw_issues_tTUx_r0p0}, -- {U32_MAX, NULL} } }, -- -- {GPU_ID2_PRODUCT_TE2X, -- {{GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tE2x_r0p0}, -- {U32_MAX, NULL} } }, -+ { GPU_ID2_PRODUCT_TMIX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 1), -+ base_hw_issues_tMIx_r0p0_05dev0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tMIx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tMIx_r0p1 }, -+ { U32_MAX /* sentinel value */, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_THEX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tHEx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tHEx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tHEx_r0p1 }, -+ { GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tHEx_r0p1 }, -+ { GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tHEx_r0p2 }, -+ { GPU_ID2_VERSION_MAKE(0, 3, 0), base_hw_issues_tHEx_r0p3 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TSIX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tSIx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tSIx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tSIx_r0p1 }, -+ { GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tSIx_r1p0 }, -+ { GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_tSIx_r1p1 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TDVX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDVx_r0p0 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TNOX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tNOx_r0p0 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TGOX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tGOx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tGOx_r1p0 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TTRX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tTRx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tTRx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tTRx_r0p1 }, -+ { GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tTRx_r0p1 }, -+ { GPU_ID2_VERSION_MAKE(0, 2, 0), base_hw_issues_tTRx_r0p2 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TNAX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tNAx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tNAx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 4), base_hw_issues_tNAx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 5), base_hw_issues_tNAx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tNAx_r0p1 }, -+ { GPU_ID2_VERSION_MAKE(0, 1, 1), base_hw_issues_tNAx_r0p1 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_LBEX, -+ { { GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_lBEx_r1p0 }, -+ { GPU_ID2_VERSION_MAKE(1, 1, 0), base_hw_issues_lBEx_r1p1 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TBEX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tBEx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 3), base_hw_issues_tBEx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 1, 0), base_hw_issues_tBEx_r0p1 }, -+ { GPU_ID2_VERSION_MAKE(1, 0, 0), base_hw_issues_tBEx_r1p0 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TBAX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tBAx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 1), base_hw_issues_tBAx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 2), base_hw_issues_tBAx_r0p0 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TDUX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tDUx_r0p0 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TODX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tODx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 4), base_hw_issues_tODx_r0p0 }, -+ { GPU_ID2_VERSION_MAKE(0, 0, 5), base_hw_issues_tODx_r0p0 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_LODX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tODx_r0p0 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TGRX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tGRx_r0p0 }, -+ { U32_MAX, NULL } } }, -+ -+ { GPU_ID2_PRODUCT_TVAX, -+ { { GPU_ID2_VERSION_MAKE(0, 0, 0), base_hw_issues_tVAx_r0p0 }, -+ { U32_MAX, NULL } } }, - }; - - u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; -@@ -269,8 +257,8 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( - } - - /* Check whether this is a candidate for most recent -- known version not later than the actual -- version. */ -+ * known version not later than the actual version. -+ */ - if ((version > product->map[v].version) && - (product->map[v].version >= fallback_version)) { - #if MALI_CUSTOMER_RELEASE -@@ -287,7 +275,8 @@ static const enum base_hw_issue *kbase_hw_get_issues_for_new_id( - - if ((issues == NULL) && (fallback_issues != NULL)) { - /* Fall back to the issue set of the most recent known -- version not later than the actual version. */ -+ * version not later than the actual version. -+ */ - issues = fallback_issues; - - #if MALI_CUSTOMER_RELEASE -@@ -340,7 +329,8 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev) - - #if !MALI_CUSTOMER_RELEASE - /* The GPU ID might have been replaced with the last -- known version of the same GPU. */ -+ * known version of the same GPU. -+ */ - gpu_id = kbdev->gpu_props.props.raw_props.gpu_id; - #endif - } else { -@@ -374,6 +364,9 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev) - case GPU_ID2_PRODUCT_TBEX: - issues = base_hw_issues_model_tBEx; - break; -+ case GPU_ID2_PRODUCT_TBAX: -+ issues = base_hw_issues_model_tBAx; -+ break; - case GPU_ID2_PRODUCT_TDUX: - issues = base_hw_issues_model_tDUx; - break; -@@ -387,13 +380,6 @@ int kbase_hw_set_issues_mask(struct kbase_device *kbdev) - case GPU_ID2_PRODUCT_TVAX: - issues = base_hw_issues_model_tVAx; - break; -- case GPU_ID2_PRODUCT_TTUX: -- case GPU_ID2_PRODUCT_LTUX: -- issues = base_hw_issues_model_tTUx; -- break; -- case GPU_ID2_PRODUCT_TE2X: -- issues = base_hw_issues_model_tE2x; -- break; - default: - dev_err(kbdev->dev, - "Unknown GPU ID %x", gpu_id); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hw.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hw.h -index f386b16..6c04a23 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hw.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hw.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file -- * Run-time work-arounds helpers -+ * DOC: Run-time work-arounds helpers - */ - - #ifndef _KBASE_HW_H_ -@@ -33,13 +29,17 @@ - #include "mali_kbase_defs.h" - - /** -- * @brief Tell whether a work-around should be enabled -+ * Tell whether a work-around should be enabled -+ * @kbdev: Device pointer -+ * @issue: issue to be checked - */ - #define kbase_hw_has_issue(kbdev, issue)\ - test_bit(issue, &(kbdev)->hw_issues_mask[0]) - - /** -- * @brief Tell whether a feature is supported -+ * Tell whether a feature is supported -+ * @kbdev: Device pointer -+ * @feature: feature to be checked - */ - #define kbase_hw_has_feature(kbdev, feature)\ - test_bit(feature, &(kbdev)->hw_features_mask[0]) -@@ -63,7 +63,8 @@ - int kbase_hw_set_issues_mask(struct kbase_device *kbdev); - - /** -- * @brief Set the features mask depending on the GPU ID -+ * Set the features mask depending on the GPU ID -+ * @kbdev: Device pointer - */ - void kbase_hw_set_features_mask(struct kbase_device *kbdev); - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h -index 89df251..0da4eb2 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_backend.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2015, 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2015, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * HW access backend common APIs - */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h -index 124a2d9..62a6ec5 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_defs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2016, 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2016-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,20 +17,16 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /** -- * @file mali_kbase_hwaccess_gpu_defs.h -- * HW access common definitions -+ * DOC: HW access common definitions - */ - - #ifndef _KBASE_HWACCESS_DEFS_H_ - #define _KBASE_HWACCESS_DEFS_H_ - --#include <mali_kbase_jm_defs.h> -+#include <backend/gpu/mali_kbase_jm_defs.h> - - /** - * struct kbase_hwaccess_data - object encapsulating the GPU backend specific -@@ -43,7 +40,9 @@ - * @backend: GPU backend specific data for HW access layer - */ - struct kbase_hwaccess_data { -+#if !MALI_USE_CSF - struct kbase_context *active_kctx[BASE_JM_MAX_NR_SLOTS]; -+#endif - - struct kbase_backend_data backend; - }; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h -index 3ae0dbe..71ccc91 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_gpuprops.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2015, 2018, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,25 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* SPDX-License-Identifier: GPL-2.0 */ --/* -- * -- * (C) COPYRIGHT 2014-2015, 2018, 2019-2020 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * A copy of the licence is included with the program, and can also be obtained -- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- * Boston, MA 02110-1301, USA. -- * - */ - -- - /** - * Base kernel property query backend APIs - */ -@@ -55,6 +39,23 @@ - int kbase_backend_gpuprops_get(struct kbase_device *kbdev, - struct kbase_gpuprops_regdump *regdump); - -+/** -+ * kbase_backend_gpuprops_get_curr_config() - Fill @curr_config_regdump with -+ * relevant GPU properties read from -+ * the GPU registers. -+ * @kbdev: Device pointer. -+ * @curr_config_regdump: Pointer to struct kbase_current_config_regdump -+ * structure. -+ * -+ * The caller should ensure that GPU remains powered-on during this function and -+ * the caller must ensure this function returns success before using the values -+ * returned in the curr_config_regdump in any part of the kernel. -+ * -+ * Return: Zero for succeess or a Linux error code -+ */ -+int kbase_backend_gpuprops_get_curr_config(struct kbase_device *kbdev, -+ struct kbase_current_config_regdump *curr_config_regdump); -+ - /** - * kbase_backend_gpuprops_get_features - Fill @regdump with GPU properties read - * from GPU -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h -index be85491..f836953 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_instr.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2015, 2017-2018, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2015, 2017-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * HW Access instrumentation common APIs - */ -@@ -29,27 +26,26 @@ - #ifndef _KBASE_HWACCESS_INSTR_H_ - #define _KBASE_HWACCESS_INSTR_H_ - --#include <mali_kbase_instr_defs.h> -+#include <backend/gpu/mali_kbase_instr_defs.h> - - /** - * struct kbase_instr_hwcnt_enable - Enable hardware counter collection. - * @dump_buffer: GPU address to write counters to. - * @dump_buffer_bytes: Size in bytes of the buffer pointed to by dump_buffer. -- * @jm_bm: counters selection bitmask (JM). -+ * @fe_bm: counters selection bitmask (Front End). - * @shader_bm: counters selection bitmask (Shader). - * @tiler_bm: counters selection bitmask (Tiler). - * @mmu_l2_bm: counters selection bitmask (MMU_L2). -- * @use_secondary: use secondary performance counters set for applicable -- * counter blocks. -+ * @counter_set: the performance counter set to use. - */ - struct kbase_instr_hwcnt_enable { - u64 dump_buffer; - u64 dump_buffer_bytes; -- u32 jm_bm; -+ u32 fe_bm; - u32 shader_bm; - u32 tiler_bm; - u32 mmu_l2_bm; -- bool use_secondary; -+ u8 counter_set; - }; - - /** -@@ -139,7 +135,7 @@ int kbase_instr_backend_init(struct kbase_device *kbdev); - */ - void kbase_instr_backend_term(struct kbase_device *kbdev); - --#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS -+#ifdef CONFIG_MALI_PRFCNT_SET_SELECT_VIA_DEBUG_FS - /** - * kbase_instr_backend_debugfs_init() - Add a debugfs entry for the - * hardware counter set. -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h -index 3d5934e..8689647 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_jm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * HW access job manager common APIs - */ -@@ -31,7 +29,7 @@ - /** - * kbase_backend_run_atom() - Run an atom on the GPU - * @kbdev: Device pointer -- * @atom: Atom to run -+ * @katom: Atom to run - * - * Caller must hold the HW access lock - */ -@@ -154,6 +152,7 @@ void kbase_backend_cache_clean(struct kbase_device *kbdev, - void kbase_backend_complete_wq(struct kbase_device *kbdev, - struct kbase_jd_atom *katom); - -+#if !MALI_USE_CSF - /** - * kbase_backend_complete_wq_post_sched - Perform backend-specific actions - * required on completing an atom, after -@@ -166,6 +165,7 @@ void kbase_backend_complete_wq(struct kbase_device *kbdev, - */ - void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev, - base_jd_core_req core_req); -+#endif /* !MALI_USE_CSF */ - - /** - * kbase_backend_reset() - The GPU is being reset. Cancel all jobs on the GPU -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h -index bbaf6ea..36bbe2d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_pm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2015, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2015, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,14 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /** -- * @file mali_kbase_hwaccess_pm.h -- * HW access power manager common APIs -+ * DOC: HW access power manager common APIs - */ - - #ifndef _KBASE_HWACCESS_PM_H_ -@@ -32,7 +29,7 @@ - #include <gpu/mali_kbase_gpu_regmap.h> - #include <linux/atomic.h> - --#include <mali_kbase_pm_defs.h> -+#include <backend/gpu/mali_kbase_pm_defs.h> - - /* Forward definition - see mali_kbase.h */ - struct kbase_device; -@@ -80,24 +77,21 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev, - * the time this function returns, regardless of whether or not the active power - * policy asks for the GPU to be powered off. - * -- * @param kbdev The kbase device structure for the device (must be a valid -- * pointer) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - */ - void kbase_hwaccess_pm_halt(struct kbase_device *kbdev); - - /** - * Perform any backend-specific actions to suspend the GPU - * -- * @param kbdev The kbase device structure for the device (must be a valid -- * pointer) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - */ - void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev); - - /** - * Perform any backend-specific actions to resume the GPU from a suspend - * -- * @param kbdev The kbase device structure for the device (must be a valid -- * pointer) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - */ - void kbase_hwaccess_pm_resume(struct kbase_device *kbdev); - -@@ -105,8 +99,7 @@ void kbase_hwaccess_pm_resume(struct kbase_device *kbdev); - * Perform any required actions for activating the GPU. Called when the first - * context goes active. - * -- * @param kbdev The kbase device structure for the device (must be a valid -- * pointer) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - */ - void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); - -@@ -114,35 +107,43 @@ void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev); - * Perform any required actions for idling the GPU. Called when the last - * context goes idle. - * -- * @param kbdev The kbase device structure for the device (must be a valid -- * pointer) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - */ - void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev); - -- -+#if MALI_USE_CSF -+/** -+ * Set the debug core mask. -+ * -+ * This determines which cores the power manager is allowed to use. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @new_core_mask: The core mask to use -+ */ -+void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, -+ u64 new_core_mask); -+#else - /** - * Set the debug core mask. - * - * This determines which cores the power manager is allowed to use. - * -- * @param kbdev The kbase device structure for the device (must be a -- * valid pointer) -- * @param new_core_mask_js0 The core mask to use for job slot 0 -- * @param new_core_mask_js0 The core mask to use for job slot 1 -- * @param new_core_mask_js0 The core mask to use for job slot 2 -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @new_core_mask_js0: The core mask to use for job slot 0 -+ * @new_core_mask_js1: The core mask to use for job slot 1 -+ * @new_core_mask_js2: The core mask to use for job slot 2 - */ - void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, - u64 new_core_mask_js0, u64 new_core_mask_js1, - u64 new_core_mask_js2); -- -+#endif /* MALI_USE_CSF */ - - /** - * Get the current policy. - * - * Returns the policy that is currently active. - * -- * @param kbdev The kbase device structure for the device (must be a valid -- * pointer) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * @return The current policy - */ -@@ -152,10 +153,9 @@ const struct kbase_pm_ca_policy - /** - * Change the policy to the one specified. - * -- * @param kbdev The kbase device structure for the device (must be a valid -- * pointer) -- * @param policy The policy to change to (valid pointer returned from -- * @ref kbase_pm_ca_list_policies) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @policy: The policy to change to (valid pointer returned from -+ * @ref kbase_pm_ca_list_policies) - */ - void kbase_pm_ca_set_policy(struct kbase_device *kbdev, - const struct kbase_pm_ca_policy *policy); -@@ -163,23 +163,20 @@ void kbase_pm_ca_set_policy(struct kbase_device *kbdev, - /** - * Retrieve a static list of the available policies. - * -- * @param[out] policies An array pointer to take the list of policies. This may -- * be NULL. The contents of this array must not be -- * modified. -+ * @policies: An array pointer to take the list of policies. This may be NULL. -+ * The contents of this array must not be modified. - * - * @return The number of policies - */ - int - kbase_pm_ca_list_policies(const struct kbase_pm_ca_policy * const **policies); - -- - /** - * Get the current policy. - * - * Returns the policy that is currently active. - * -- * @param kbdev The kbase device structure for the device (must be a valid -- * pointer) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * @return The current policy - */ -@@ -188,9 +185,9 @@ const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev); - /** - * Change the policy to the one specified. - * -- * @param kbdev The kbase device structure for the device (must be a valid -+ * @kbdev: The kbase device structure for the device (must be a valid - * pointer) -- * @param policy The policy to change to (valid pointer returned from -+ * @policy: The policy to change to (valid pointer returned from - * @ref kbase_pm_list_policies) - */ - void kbase_pm_set_policy(struct kbase_device *kbdev, -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h -index a61e5b9..8a4ece4 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwaccess_time.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014,2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,13 +16,6 @@ - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- */ -- -- --/** - * - */ - -@@ -30,13 +24,27 @@ - - /** - * kbase_backend_get_gpu_time() - Get current GPU time -+ * @kbdev: Device pointer -+ * @cycle_counter: Pointer to u64 to store cycle counter in. -+ * @system_time: Pointer to u64 to store system time in -+ * @ts: Pointer to struct timespec to store current monotonic -+ * time in -+ */ -+void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, -+ u64 *system_time, struct timespec64 *ts); -+ -+/** -+ * kbase_backend_get_gpu_time_norequest() - Get current GPU time without -+ * request/release cycle counter - * @kbdev: Device pointer - * @cycle_counter: Pointer to u64 to store cycle counter in - * @system_time: Pointer to u64 to store system time in - * @ts: Pointer to struct timespec to store current monotonic - * time in - */ --void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter, -- u64 *system_time, struct timespec64 *ts); -+void kbase_backend_get_gpu_time_norequest(struct kbase_device *kbdev, -+ u64 *cycle_counter, -+ u64 *system_time, -+ struct timespec64 *ts); - - #endif /* _KBASE_BACKEND_TIME_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt.c -index 14ec5cb..ea4893d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2018, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -28,9 +27,6 @@ - #include "mali_kbase_hwcnt_accumulator.h" - #include "mali_kbase_hwcnt_backend.h" - #include "mali_kbase_hwcnt_types.h" --#include "mali_malisw.h" --#include "mali_kbase_debug.h" --#include "mali_kbase_linux.h" - - #include <linux/mutex.h> - #include <linux/spinlock.h> -@@ -51,6 +47,7 @@ enum kbase_hwcnt_accum_state { - - /** - * struct kbase_hwcnt_accumulator - Hardware counter accumulator structure. -+ * @metadata: Pointer to immutable hwcnt metadata. - * @backend: Pointer to created counter backend. - * @state: The current state of the accumulator. - * - State transition from disabled->enabled or -@@ -89,6 +86,7 @@ enum kbase_hwcnt_accum_state { - * accum_lock. - */ - struct kbase_hwcnt_accumulator { -+ const struct kbase_hwcnt_metadata *metadata; - struct kbase_hwcnt_backend *backend; - enum kbase_hwcnt_accum_state state; - struct kbase_hwcnt_enable_map enable_map; -@@ -117,6 +115,10 @@ struct kbase_hwcnt_accumulator { - * state_lock. - * - Can be read while holding either lock. - * @accum: Hardware counter accumulator structure. -+ * @wq: Centralized workqueue for users of hardware counters to -+ * submit async hardware counter related work. Never directly -+ * called, but it's expected that a lot of the functions in this -+ * API will end up called from the enqueued async work. - */ - struct kbase_hwcnt_context { - const struct kbase_hwcnt_backend_interface *iface; -@@ -125,6 +127,7 @@ struct kbase_hwcnt_context { - struct mutex accum_lock; - bool accum_inited; - struct kbase_hwcnt_accumulator accum; -+ struct workqueue_struct *wq; - }; - - int kbase_hwcnt_context_init( -@@ -138,7 +141,7 @@ int kbase_hwcnt_context_init( - - hctx = kzalloc(sizeof(*hctx), GFP_KERNEL); - if (!hctx) -- return -ENOMEM; -+ goto err_alloc_hctx; - - hctx->iface = iface; - spin_lock_init(&hctx->state_lock); -@@ -146,11 +149,21 @@ int kbase_hwcnt_context_init( - mutex_init(&hctx->accum_lock); - hctx->accum_inited = false; - -+ hctx->wq = -+ alloc_workqueue("mali_kbase_hwcnt", WQ_HIGHPRI | WQ_UNBOUND, 0); -+ if (!hctx->wq) -+ goto err_alloc_workqueue; -+ - *out_hctx = hctx; - - return 0; -+ -+ destroy_workqueue(hctx->wq); -+err_alloc_workqueue: -+ kfree(hctx); -+err_alloc_hctx: -+ return -ENOMEM; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_context_init); - - void kbase_hwcnt_context_term(struct kbase_hwcnt_context *hctx) - { -@@ -159,9 +172,13 @@ void kbase_hwcnt_context_term(struct kbase_hwcnt_context *hctx) - - /* Make sure we didn't leak the accumulator */ - WARN_ON(hctx->accum_inited); -+ -+ /* We don't expect any work to be pending on this workqueue. -+ * Regardless, this will safely drain and complete the work. -+ */ -+ destroy_workqueue(hctx->wq); - kfree(hctx); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_context_term); - - /** - * kbasep_hwcnt_accumulator_term() - Terminate the accumulator for the context. -@@ -197,22 +214,23 @@ static int kbasep_hwcnt_accumulator_init(struct kbase_hwcnt_context *hctx) - if (errcode) - goto error; - -+ hctx->accum.metadata = hctx->iface->metadata(hctx->iface->info); - hctx->accum.state = ACCUM_STATE_ERROR; - -- errcode = kbase_hwcnt_enable_map_alloc( -- hctx->iface->metadata, &hctx->accum.enable_map); -+ errcode = kbase_hwcnt_enable_map_alloc(hctx->accum.metadata, -+ &hctx->accum.enable_map); - if (errcode) - goto error; - - hctx->accum.enable_map_any_enabled = false; - -- errcode = kbase_hwcnt_dump_buffer_alloc( -- hctx->iface->metadata, &hctx->accum.accum_buf); -+ errcode = kbase_hwcnt_dump_buffer_alloc(hctx->accum.metadata, -+ &hctx->accum.accum_buf); - if (errcode) - goto error; - -- errcode = kbase_hwcnt_enable_map_alloc( -- hctx->iface->metadata, &hctx->accum.scratch_map); -+ errcode = kbase_hwcnt_enable_map_alloc(hctx->accum.metadata, -+ &hctx->accum.scratch_map); - if (errcode) - goto error; - -@@ -242,6 +260,7 @@ static void kbasep_hwcnt_accumulator_disable( - bool backend_enabled = false; - struct kbase_hwcnt_accumulator *accum; - unsigned long flags; -+ u64 dump_time_ns; - - WARN_ON(!hctx); - lockdep_assert_held(&hctx->accum_lock); -@@ -271,7 +290,7 @@ static void kbasep_hwcnt_accumulator_disable( - goto disable; - - /* Try and accumulate before disabling */ -- errcode = hctx->iface->dump_request(accum->backend); -+ errcode = hctx->iface->dump_request(accum->backend, &dump_time_ns); - if (errcode) - goto disable; - -@@ -365,8 +384,8 @@ static int kbasep_hwcnt_accumulator_dump( - WARN_ON(!hctx); - WARN_ON(!ts_start_ns); - WARN_ON(!ts_end_ns); -- WARN_ON(dump_buf && (dump_buf->metadata != hctx->iface->metadata)); -- WARN_ON(new_map && (new_map->metadata != hctx->iface->metadata)); -+ WARN_ON(dump_buf && (dump_buf->metadata != hctx->accum.metadata)); -+ WARN_ON(new_map && (new_map->metadata != hctx->accum.metadata)); - WARN_ON(!hctx->accum_inited); - lockdep_assert_held(&hctx->accum_lock); - -@@ -419,23 +438,16 @@ static int kbasep_hwcnt_accumulator_dump( - - /* Initiate the dump if the backend is enabled. */ - if ((state == ACCUM_STATE_ENABLED) && cur_map_any_enabled) { -- /* Disable pre-emption, to make the timestamp as accurate as -- * possible. -- */ -- preempt_disable(); -- { -+ if (dump_buf) { -+ errcode = hctx->iface->dump_request( -+ accum->backend, &dump_time_ns); -+ dump_requested = true; -+ } else { - dump_time_ns = hctx->iface->timestamp_ns( -- accum->backend); -- if (dump_buf) { -- errcode = hctx->iface->dump_request( - accum->backend); -- dump_requested = true; -- } else { -- errcode = hctx->iface->dump_clear( -- accum->backend); -- } -+ errcode = hctx->iface->dump_clear(accum->backend); - } -- preempt_enable(); -+ - if (errcode) - goto error; - } else { -@@ -615,7 +627,6 @@ int kbase_hwcnt_accumulator_acquire( - - return 0; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_acquire); - - void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum) - { -@@ -650,7 +661,6 @@ void kbase_hwcnt_accumulator_release(struct kbase_hwcnt_accumulator *accum) - spin_unlock_irqrestore(&hctx->state_lock, flags); - mutex_unlock(&hctx->accum_lock); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_release); - - void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx) - { -@@ -669,7 +679,6 @@ void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx) - - mutex_unlock(&hctx->accum_lock); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_context_disable); - - bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx) - { -@@ -698,7 +707,6 @@ bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx) - - return atomic_disabled; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_context_disable_atomic); - - void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx) - { -@@ -718,7 +726,6 @@ void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx) - - spin_unlock_irqrestore(&hctx->state_lock, flags); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_context_enable); - - const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( - struct kbase_hwcnt_context *hctx) -@@ -726,9 +733,17 @@ const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( - if (!hctx) - return NULL; - -- return hctx->iface->metadata; -+ return hctx->iface->metadata(hctx->iface->info); -+} -+ -+bool kbase_hwcnt_context_queue_work(struct kbase_hwcnt_context *hctx, -+ struct work_struct *work) -+{ -+ if (WARN_ON(!hctx) || WARN_ON(!work)) -+ return false; -+ -+ return queue_work(hctx->wq, work); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_context_metadata); - - int kbase_hwcnt_accumulator_set_counters( - struct kbase_hwcnt_accumulator *accum, -@@ -745,8 +760,8 @@ int kbase_hwcnt_accumulator_set_counters( - - hctx = container_of(accum, struct kbase_hwcnt_context, accum); - -- if ((new_map->metadata != hctx->iface->metadata) || -- (dump_buf && (dump_buf->metadata != hctx->iface->metadata))) -+ if ((new_map->metadata != hctx->accum.metadata) || -+ (dump_buf && (dump_buf->metadata != hctx->accum.metadata))) - return -EINVAL; - - mutex_lock(&hctx->accum_lock); -@@ -758,7 +773,6 @@ int kbase_hwcnt_accumulator_set_counters( - - return errcode; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_set_counters); - - int kbase_hwcnt_accumulator_dump( - struct kbase_hwcnt_accumulator *accum, -@@ -774,7 +788,7 @@ int kbase_hwcnt_accumulator_dump( - - hctx = container_of(accum, struct kbase_hwcnt_context, accum); - -- if (dump_buf && (dump_buf->metadata != hctx->iface->metadata)) -+ if (dump_buf && (dump_buf->metadata != hctx->accum.metadata)) - return -EINVAL; - - mutex_lock(&hctx->accum_lock); -@@ -786,7 +800,6 @@ int kbase_hwcnt_accumulator_dump( - - return errcode; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_accumulator_dump); - - u64 kbase_hwcnt_accumulator_timestamp_ns(struct kbase_hwcnt_accumulator *accum) - { -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_accumulator.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_accumulator.h -index eb82ea4..4887eaa 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_accumulator.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_accumulator.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend.h -index b7aa0e1..0b5a188 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -41,11 +40,25 @@ struct kbase_hwcnt_dump_buffer; - struct kbase_hwcnt_backend_info; - - /* -- * struct kbase_hwcnt_backend_info - Opaque pointer to a hardware counter -- * backend, used to perform dumps. -+ * struct kbase_hwcnt_backend - Opaque pointer to a hardware counter -+ * backend, used to perform dumps. - */ - struct kbase_hwcnt_backend; - -+/* -+ * typedef kbase_hwcnt_backend_metadata_fn - Get the immutable hardware counter -+ * metadata that describes the layout -+ * of the counter data structures. -+ * @info: Non-NULL pointer to backend info. -+ * -+ * Multiple calls to this function with the same info are guaranteed to return -+ * the same metadata object each time. -+ * -+ * Return: Non-NULL pointer to immutable hardware counter metadata. -+ */ -+typedef const struct kbase_hwcnt_metadata * -+kbase_hwcnt_backend_metadata_fn(const struct kbase_hwcnt_backend_info *info); -+ - /** - * typedef kbase_hwcnt_backend_init_fn - Initialise a counter backend. - * @info: Non-NULL pointer to backend info. -@@ -56,16 +69,15 @@ struct kbase_hwcnt_backend; - * - * Return: 0 on success, else error code. - */ --typedef int (*kbase_hwcnt_backend_init_fn)( -- const struct kbase_hwcnt_backend_info *info, -- struct kbase_hwcnt_backend **out_backend); -+typedef int -+kbase_hwcnt_backend_init_fn(const struct kbase_hwcnt_backend_info *info, -+ struct kbase_hwcnt_backend **out_backend); - - /** - * typedef kbase_hwcnt_backend_term_fn - Terminate a counter backend. - * @backend: Pointer to backend to be terminated. - */ --typedef void (*kbase_hwcnt_backend_term_fn)( -- struct kbase_hwcnt_backend *backend); -+typedef void kbase_hwcnt_backend_term_fn(struct kbase_hwcnt_backend *backend); - - /** - * typedef kbase_hwcnt_backend_timestamp_ns_fn - Get the current backend -@@ -74,8 +86,8 @@ typedef void (*kbase_hwcnt_backend_term_fn)( - * - * Return: Backend timestamp in nanoseconds. - */ --typedef u64 (*kbase_hwcnt_backend_timestamp_ns_fn)( -- struct kbase_hwcnt_backend *backend); -+typedef u64 -+kbase_hwcnt_backend_timestamp_ns_fn(struct kbase_hwcnt_backend *backend); - - /** - * typedef kbase_hwcnt_backend_dump_enable_fn - Start counter dumping with the -@@ -90,7 +102,7 @@ typedef u64 (*kbase_hwcnt_backend_timestamp_ns_fn)( - * - * Return: 0 on success, else error code. - */ --typedef int (*kbase_hwcnt_backend_dump_enable_fn)( -+typedef int kbase_hwcnt_backend_dump_enable_fn( - struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map); - -@@ -106,7 +118,7 @@ typedef int (*kbase_hwcnt_backend_dump_enable_fn)( - * - * Return: 0 on success, else error code. - */ --typedef int (*kbase_hwcnt_backend_dump_enable_nolock_fn)( -+typedef int kbase_hwcnt_backend_dump_enable_nolock_fn( - struct kbase_hwcnt_backend *backend, - const struct kbase_hwcnt_enable_map *enable_map); - -@@ -118,8 +130,8 @@ typedef int (*kbase_hwcnt_backend_dump_enable_nolock_fn)( - * If the backend is already disabled, does nothing. - * Any undumped counter values since the last dump get will be lost. - */ --typedef void (*kbase_hwcnt_backend_dump_disable_fn)( -- struct kbase_hwcnt_backend *backend); -+typedef void -+kbase_hwcnt_backend_dump_disable_fn(struct kbase_hwcnt_backend *backend); - - /** - * typedef kbase_hwcnt_backend_dump_clear_fn - Reset all the current undumped -@@ -130,21 +142,24 @@ typedef void (*kbase_hwcnt_backend_dump_disable_fn)( - * - * Return: 0 on success, else error code. - */ --typedef int (*kbase_hwcnt_backend_dump_clear_fn)( -- struct kbase_hwcnt_backend *backend); -+typedef int -+kbase_hwcnt_backend_dump_clear_fn(struct kbase_hwcnt_backend *backend); - - /** - * typedef kbase_hwcnt_backend_dump_request_fn - Request an asynchronous counter - * dump. - * @backend: Non-NULL pointer to backend. -+ * @dump_time_ns: Non-NULL pointer where the timestamp of when the dump was -+ * requested will be written out to on success. - * - * If the backend is not enabled or another dump is already in progress, - * returns an error. - * - * Return: 0 on success, else error code. - */ --typedef int (*kbase_hwcnt_backend_dump_request_fn)( -- struct kbase_hwcnt_backend *backend); -+typedef int -+kbase_hwcnt_backend_dump_request_fn(struct kbase_hwcnt_backend *backend, -+ u64 *dump_time_ns); - - /** - * typedef kbase_hwcnt_backend_dump_wait_fn - Wait until the last requested -@@ -155,8 +170,8 @@ typedef int (*kbase_hwcnt_backend_dump_request_fn)( - * - * Return: 0 on success, else error code. - */ --typedef int (*kbase_hwcnt_backend_dump_wait_fn)( -- struct kbase_hwcnt_backend *backend); -+typedef int -+kbase_hwcnt_backend_dump_wait_fn(struct kbase_hwcnt_backend *backend); - - /** - * typedef kbase_hwcnt_backend_dump_get_fn - Copy or accumulate enable the -@@ -168,24 +183,25 @@ typedef int (*kbase_hwcnt_backend_dump_wait_fn)( - * @accumulate: True if counters should be accumulated into dump_buffer, rather - * than copied. - * -- * If the backend is not enabled, returns an error. -- * If a dump is in progress (i.e. dump_wait has not yet returned successfully) -- * then the resultant contents of the dump buffer will be undefined. -+ * The resultant contents of the dump buffer are only well defined if a prior -+ * call to dump_wait returned successfully, and a new dump has not yet been -+ * requested by a call to dump_request. - * - * Return: 0 on success, else error code. - */ --typedef int (*kbase_hwcnt_backend_dump_get_fn)( -- struct kbase_hwcnt_backend *backend, -- struct kbase_hwcnt_dump_buffer *dump_buffer, -- const struct kbase_hwcnt_enable_map *enable_map, -- bool accumulate); -+typedef int -+kbase_hwcnt_backend_dump_get_fn(struct kbase_hwcnt_backend *backend, -+ struct kbase_hwcnt_dump_buffer *dump_buffer, -+ const struct kbase_hwcnt_enable_map *enable_map, -+ bool accumulate); - - /** - * struct kbase_hwcnt_backend_interface - Hardware counter backend virtual - * interface. -- * @metadata: Immutable hardware counter metadata. - * @info: Immutable info used to initialise an instance of the - * backend. -+ * @metadata: Function ptr to get the immutable hardware counter -+ * metadata. - * @init: Function ptr to initialise an instance of the backend. - * @term: Function ptr to terminate an instance of the backend. - * @timestamp_ns: Function ptr to get the current backend timestamp. -@@ -200,18 +216,18 @@ typedef int (*kbase_hwcnt_backend_dump_get_fn)( - * buffer. - */ - struct kbase_hwcnt_backend_interface { -- const struct kbase_hwcnt_metadata *metadata; - const struct kbase_hwcnt_backend_info *info; -- kbase_hwcnt_backend_init_fn init; -- kbase_hwcnt_backend_term_fn term; -- kbase_hwcnt_backend_timestamp_ns_fn timestamp_ns; -- kbase_hwcnt_backend_dump_enable_fn dump_enable; -- kbase_hwcnt_backend_dump_enable_nolock_fn dump_enable_nolock; -- kbase_hwcnt_backend_dump_disable_fn dump_disable; -- kbase_hwcnt_backend_dump_clear_fn dump_clear; -- kbase_hwcnt_backend_dump_request_fn dump_request; -- kbase_hwcnt_backend_dump_wait_fn dump_wait; -- kbase_hwcnt_backend_dump_get_fn dump_get; -+ kbase_hwcnt_backend_metadata_fn *metadata; -+ kbase_hwcnt_backend_init_fn *init; -+ kbase_hwcnt_backend_term_fn *term; -+ kbase_hwcnt_backend_timestamp_ns_fn *timestamp_ns; -+ kbase_hwcnt_backend_dump_enable_fn *dump_enable; -+ kbase_hwcnt_backend_dump_enable_nolock_fn *dump_enable_nolock; -+ kbase_hwcnt_backend_dump_disable_fn *dump_disable; -+ kbase_hwcnt_backend_dump_clear_fn *dump_clear; -+ kbase_hwcnt_backend_dump_request_fn *dump_request; -+ kbase_hwcnt_backend_dump_wait_fn *dump_wait; -+ kbase_hwcnt_backend_dump_get_fn *dump_get; - }; - - #endif /* _KBASE_HWCNT_BACKEND_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf.c -new file mode 100644 -index 0000000..58b5e72 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf.c -@@ -0,0 +1,1864 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_hwcnt_backend_csf.h" -+#include "mali_kbase_hwcnt_gpu.h" -+#include "mali_kbase_hwcnt_types.h" -+ -+#include <linux/log2.h> -+#include <linux/kernel.h> -+#include <linux/sched.h> -+#include <linux/slab.h> -+#include <linux/spinlock.h> -+#include <linux/wait.h> -+#include <linux/workqueue.h> -+#include <linux/completion.h> -+ -+#ifndef BASE_MAX_NR_CLOCKS_REGULATORS -+#define BASE_MAX_NR_CLOCKS_REGULATORS 2 -+#endif -+ -+/** -+ * enum kbase_hwcnt_backend_csf_dump_state - HWC CSF backend dumping states. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE: Initial state, or the state if there is -+ * an error. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED: A dump has been requested and we are -+ * waiting for an ACK, this ACK could come from either PRFCNT_ACK, -+ * PROTMODE_ENTER_ACK, or if an error occurs. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT: Checking the insert -+ * immediately after receiving the ACK, so we know which index corresponds to -+ * the buffer we requested. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_DUMP_WORKER_LAUNCHED: The insert has been saved and -+ * now we have kicked off the worker. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_DUMP_ACCUMULATING: The insert has been saved and now -+ * we have kicked off the worker to accumulate up to that insert and then copy -+ * the delta to the user buffer to prepare for dump_get(). -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED: The dump completed successfully. -+ * -+ * Valid state transitions: -+ * IDLE -> REQUESTED (on dump request) -+ * REQUESTED -> QUERYING_INSERT (on dump ack) -+ * QUERYING_INSERT -> WORKER_LAUNCHED (on worker submission) -+ * WORKER_LAUNCHED -> ACCUMULATING (while the worker is accumulating) -+ * ACCUMULATING -> COMPLETED (on accumulation completion) -+ * COMPLETED -> REQUESTED (on dump request) -+ * COMPLETED -> IDLE (on disable) -+ * ANY -> IDLE (on error) -+ */ -+enum kbase_hwcnt_backend_csf_dump_state { -+ KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE, -+ KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED, -+ KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT, -+ KBASE_HWCNT_BACKEND_CSF_DUMP_WORKER_LAUNCHED, -+ KBASE_HWCNT_BACKEND_CSF_DUMP_ACCUMULATING, -+ KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED, -+}; -+ -+/** -+ * enum kbase_hwcnt_backend_csf_enable_state - HWC CSF backend enable states. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_DISABLED: Initial state, and the state when backend -+ * is disabled. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: Enable request is in -+ * progress, waiting for firmware acknowledgment. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_ENABLED: Enable request has been acknowledged, -+ * enable is done. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: Disable request is in -+ * progress, waiting for firmware acknowledgment. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: Disable request has been -+ * acknowledged, waiting for dump workers to be finished. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: An -+ * unrecoverable error happened, waiting for dump workers to be finished. -+ * -+ * @KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR: An unrecoverable error -+ * happened, and dump workers have finished, waiting for reset. -+ * -+ * Valid state transitions: -+ * DISABLED -> TRANSITIONING_TO_ENABLED (on enable) -+ * TRANSITIONING_TO_ENABLED -> ENABLED (on enable ack) -+ * ENABLED -> TRANSITIONING_TO_DISABLED (on disable) -+ * TRANSITIONING_TO_DISABLED -> DISABLED_WAIT_FOR_WORKER (on disable ack) -+ * DISABLED_WAIT_FOR_WORKER -> DISABLED (after workers are flushed) -+ * DISABLED -> UNRECOVERABLE_ERROR (on unrecoverable error) -+ * ANY but DISABLED -> UNRECOVERABLE_ERROR_WAIT_FOR_WORKER (on unrecoverable -+ * error) -+ * UNRECOVERABLE_ERROR -> DISABLED (on before reset) -+ */ -+enum kbase_hwcnt_backend_csf_enable_state { -+ KBASE_HWCNT_BACKEND_CSF_DISABLED, -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED, -+ KBASE_HWCNT_BACKEND_CSF_ENABLED, -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED, -+ KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER, -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER, -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR, -+}; -+ -+/** -+ * struct kbase_hwcnt_backend_csf_info - Information used to create an instance -+ * of a CSF hardware counter backend. -+ * @backend: Pointer to access CSF backend. -+ * @fw_in_protected_mode: True if FW is running in protected mode, else -+ * false. -+ * @unrecoverable_error_happened: True if an recoverable error happened, else -+ * false. -+ * @csf_if: CSF interface object pointer. -+ * @ring_buf_cnt: Dump buffer count in the ring buffer. -+ * @counter_set: The performance counter set to use. -+ * @metadata: Hardware counter metadata. -+ * @prfcnt_info: Performance counter information. -+ */ -+struct kbase_hwcnt_backend_csf_info { -+ struct kbase_hwcnt_backend_csf *backend; -+ bool fw_in_protected_mode; -+ bool unrecoverable_error_happened; -+ struct kbase_hwcnt_backend_csf_if *csf_if; -+ u32 ring_buf_cnt; -+ enum kbase_hwcnt_set counter_set; -+ const struct kbase_hwcnt_metadata *metadata; -+ struct kbase_hwcnt_backend_csf_if_prfcnt_info prfcnt_info; -+}; -+ -+/** -+ * struct kbase_hwcnt_csf_physical_layout - HWC sample memory physical layout -+ * information. -+ * @fe_cnt: Front end block count. -+ * @tiler_cnt: Tiler block count. -+ * @mmu_l2_cnt: Memory system(MMU and L2 cache) block count. -+ * @shader_cnt: Shader Core block count. -+ * @block_cnt: Total block count (sum of all other block counts). -+ * @shader_avail_mask: Bitmap of all shader cores in the system. -+ * @offset_enable_mask: Offset of enable mask in the block. -+ * @headers_per_block: Header size per block. -+ * @counters_per_block: Counters size per block. -+ * @values_per_block: Total size per block. -+ */ -+struct kbase_hwcnt_csf_physical_layout { -+ size_t fe_cnt; -+ size_t tiler_cnt; -+ size_t mmu_l2_cnt; -+ size_t shader_cnt; -+ size_t block_cnt; -+ u64 shader_avail_mask; -+ size_t offset_enable_mask; -+ size_t headers_per_block; -+ size_t counters_per_block; -+ size_t values_per_block; -+}; -+ -+/** -+ * struct kbase_hwcnt_backend_csf - Instance of a CSF hardware counter backend. -+ * @info: CSF Info used to create the backend. -+ * @dump_state: The dumping state of the backend. -+ * @enable_state: The CSF backend internal enabled state. -+ * @insert_index_to_accumulate: The insert index in the ring buffer which need -+ * to accumulate up to. -+ * @enable_state_waitq: Wait queue object used to notify the enable -+ * changing flag is done. -+ * @to_user_buf: HWC sample buffer for client user. -+ * @accum_buf: HWC sample buffer used as an internal -+ * accumulator. -+ * @old_sample_buf: HWC sample buffer to save the previous values -+ * for delta calculation. -+ * @ring_buf: Opaque pointer for ring buffer object. -+ * @ring_buf_cpu_base: CPU base address of the allocated ring buffer. -+ * @clk_enable_map: The enable map specifying enabled clock domains. -+ * @cycle_count_elapsed: Cycle count elapsed for a given sample period. -+ * @prev_cycle_count: Previous cycle count to calculate the cycle -+ * count for sample period. -+ * @phys_layout: Physical memory layout information of HWC -+ * sample buffer. -+ * @dump_completed: Completion signaled by the dump worker when -+ * it is completed accumulating up to the -+ * insert_index_to_accumulate. -+ * Should be initialized to the "complete" state. -+ * @hwc_dump_workq: Single threaded work queue for HWC workers -+ * execution. -+ * @hwc_dump_work: Worker to accumulate samples. -+ * @hwc_threshold_work: Worker for consuming available samples when -+ * threshold interrupt raised. -+ */ -+struct kbase_hwcnt_backend_csf { -+ struct kbase_hwcnt_backend_csf_info *info; -+ enum kbase_hwcnt_backend_csf_dump_state dump_state; -+ enum kbase_hwcnt_backend_csf_enable_state enable_state; -+ u32 insert_index_to_accumulate; -+ wait_queue_head_t enable_state_waitq; -+ u32 *to_user_buf; -+ u32 *accum_buf; -+ u32 *old_sample_buf; -+ struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf; -+ void *ring_buf_cpu_base; -+ u64 clk_enable_map; -+ u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; -+ u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; -+ struct kbase_hwcnt_csf_physical_layout phys_layout; -+ struct completion dump_completed; -+ struct workqueue_struct *hwc_dump_workq; -+ struct work_struct hwc_dump_work; -+ struct work_struct hwc_threshold_work; -+}; -+ -+static bool kbasep_hwcnt_backend_csf_backend_exists( -+ struct kbase_hwcnt_backend_csf_info *csf_info) -+{ -+ WARN_ON(!csf_info); -+ csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); -+ return (csf_info->backend != NULL); -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_cc_initial_sample() - Initialize cycle count -+ * tracking. -+ * -+ * @backend_csf: Non-NULL pointer to backend. -+ * @enable_map: Non-NULL pointer to enable map specifying enabled counters. -+ */ -+static void kbasep_hwcnt_backend_csf_cc_initial_sample( -+ struct kbase_hwcnt_backend_csf *backend_csf, -+ const struct kbase_hwcnt_enable_map *enable_map) -+{ -+ u64 clk_enable_map = enable_map->clk_enable_map; -+ u64 cycle_counts[BASE_MAX_NR_CLOCKS_REGULATORS]; -+ size_t clk; -+ -+ /* Read cycle count from CSF interface for both clock domains. */ -+ backend_csf->info->csf_if->get_gpu_cycle_count( -+ backend_csf->info->csf_if->ctx, cycle_counts, clk_enable_map); -+ -+ kbase_hwcnt_metadata_for_each_clock(enable_map->metadata, clk) { -+ if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, clk)) -+ backend_csf->prev_cycle_count[clk] = cycle_counts[clk]; -+ } -+ -+ /* Keep clk_enable_map for dump_request. */ -+ backend_csf->clk_enable_map = clk_enable_map; -+} -+ -+static void -+kbasep_hwcnt_backend_csf_cc_update(struct kbase_hwcnt_backend_csf *backend_csf) -+{ -+ u64 cycle_counts[BASE_MAX_NR_CLOCKS_REGULATORS]; -+ size_t clk; -+ -+ backend_csf->info->csf_if->assert_lock_held( -+ backend_csf->info->csf_if->ctx); -+ -+ backend_csf->info->csf_if->get_gpu_cycle_count( -+ backend_csf->info->csf_if->ctx, cycle_counts, -+ backend_csf->clk_enable_map); -+ -+ kbase_hwcnt_metadata_for_each_clock(backend_csf->info->metadata, clk) { -+ if (kbase_hwcnt_clk_enable_map_enabled( -+ backend_csf->clk_enable_map, clk)) { -+ backend_csf->cycle_count_elapsed[clk] = -+ cycle_counts[clk] - -+ backend_csf->prev_cycle_count[clk]; -+ backend_csf->prev_cycle_count[clk] = cycle_counts[clk]; -+ } -+ } -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_timestamp_ns_fn */ -+static u64 -+kbasep_hwcnt_backend_csf_timestamp_ns(struct kbase_hwcnt_backend *backend) -+{ -+ struct kbase_hwcnt_backend_csf *backend_csf = -+ (struct kbase_hwcnt_backend_csf *)backend; -+ -+ if (!backend_csf || !backend_csf->info || !backend_csf->info->csf_if) -+ return 0; -+ -+ return backend_csf->info->csf_if->timestamp_ns( -+ backend_csf->info->csf_if->ctx); -+} -+ -+/** kbasep_hwcnt_backend_csf_process_enable_map() - Process the enable_map to -+ * guarantee headers are -+ * enabled if any counter is -+ * required. -+ *@phys_enable_map: HWC physical enable map to be processed. -+ */ -+static void kbasep_hwcnt_backend_csf_process_enable_map( -+ struct kbase_hwcnt_physical_enable_map *phys_enable_map) -+{ -+ WARN_ON(!phys_enable_map); -+ -+ /* Enable header if any counter is required from user, the header is -+ * controlled by bit 0 of the enable mask. -+ */ -+ if (phys_enable_map->fe_bm) -+ phys_enable_map->fe_bm |= 1; -+ -+ if (phys_enable_map->tiler_bm) -+ phys_enable_map->tiler_bm |= 1; -+ -+ if (phys_enable_map->mmu_l2_bm) -+ phys_enable_map->mmu_l2_bm |= 1; -+ -+ if (phys_enable_map->shader_bm) -+ phys_enable_map->shader_bm |= 1; -+} -+ -+static void kbasep_hwcnt_backend_csf_init_layout( -+ const struct kbase_hwcnt_backend_csf_if_prfcnt_info *prfcnt_info, -+ struct kbase_hwcnt_csf_physical_layout *phys_layout) -+{ -+ WARN_ON(!prfcnt_info); -+ WARN_ON(!phys_layout); -+ -+ phys_layout->fe_cnt = 1; -+ phys_layout->tiler_cnt = 1; -+ phys_layout->mmu_l2_cnt = prfcnt_info->l2_count; -+ phys_layout->shader_cnt = fls64(prfcnt_info->core_mask); -+ phys_layout->block_cnt = phys_layout->fe_cnt + phys_layout->tiler_cnt + -+ phys_layout->mmu_l2_cnt + -+ phys_layout->shader_cnt; -+ -+ phys_layout->shader_avail_mask = prfcnt_info->core_mask; -+ -+ phys_layout->headers_per_block = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; -+ phys_layout->values_per_block = -+ prfcnt_info->prfcnt_block_size / KBASE_HWCNT_VALUE_BYTES; -+ phys_layout->counters_per_block = -+ phys_layout->values_per_block - phys_layout->headers_per_block; -+ phys_layout->offset_enable_mask = KBASE_HWCNT_V5_PRFCNT_EN_HEADER; -+} -+ -+static void kbasep_hwcnt_backend_csf_reset_internal_buffers( -+ struct kbase_hwcnt_backend_csf *backend_csf) -+{ -+ memset(backend_csf->to_user_buf, 0, -+ backend_csf->info->prfcnt_info.dump_bytes); -+ memset(backend_csf->accum_buf, 0, -+ backend_csf->info->prfcnt_info.dump_bytes); -+ memset(backend_csf->old_sample_buf, 0, -+ backend_csf->info->prfcnt_info.dump_bytes); -+} -+ -+static void kbasep_hwcnt_backend_csf_zero_sample_prfcnt_en_header( -+ struct kbase_hwcnt_backend_csf *backend_csf, u32 *sample) -+{ -+ u32 block_idx; -+ const struct kbase_hwcnt_csf_physical_layout *phys_layout; -+ u32 *block_buf; -+ -+ phys_layout = &backend_csf->phys_layout; -+ -+ for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { -+ block_buf = sample + block_idx * phys_layout->values_per_block; -+ block_buf[phys_layout->offset_enable_mask] = 0; -+ } -+} -+ -+static void kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header( -+ struct kbase_hwcnt_backend_csf *backend_csf) -+{ -+ u32 idx; -+ u32 *sample; -+ char *cpu_dump_base; -+ size_t dump_bytes = backend_csf->info->prfcnt_info.dump_bytes; -+ -+ cpu_dump_base = (char *)backend_csf->ring_buf_cpu_base; -+ -+ for (idx = 0; idx < backend_csf->info->ring_buf_cnt; idx++) { -+ sample = (u32 *)&cpu_dump_base[idx * dump_bytes]; -+ kbasep_hwcnt_backend_csf_zero_sample_prfcnt_en_header( -+ backend_csf, sample); -+ } -+} -+ -+static void kbasep_hwcnt_backend_csf_update_user_sample( -+ struct kbase_hwcnt_backend_csf *backend_csf) -+{ -+ /* Copy the data into the sample and wait for the user to get it. */ -+ memcpy(backend_csf->to_user_buf, backend_csf->accum_buf, -+ backend_csf->info->prfcnt_info.dump_bytes); -+ -+ /* After copied data into user sample, clear the accumulator values to -+ * prepare for the next accumulator, such as the next request or -+ * threshold. -+ */ -+ memset(backend_csf->accum_buf, 0, -+ backend_csf->info->prfcnt_info.dump_bytes); -+} -+ -+static void kbasep_hwcnt_backend_csf_accumulate_sample( -+ const struct kbase_hwcnt_csf_physical_layout *phys_layout, -+ size_t dump_bytes, u32 *accum_buf, const u32 *old_sample_buf, -+ const u32 *new_sample_buf, bool clearing_samples) -+{ -+ size_t block_idx, ctr_idx; -+ const u32 *old_block = old_sample_buf; -+ const u32 *new_block = new_sample_buf; -+ u32 *acc_block = accum_buf; -+ -+ for (block_idx = 0; block_idx < phys_layout->block_cnt; block_idx++) { -+ const u32 old_enable_mask = -+ old_block[phys_layout->offset_enable_mask]; -+ const u32 new_enable_mask = -+ new_block[phys_layout->offset_enable_mask]; -+ -+ if (new_enable_mask == 0) { -+ /* Hardware block was unavailable or we didn't turn on -+ * any counters. Do nothing. -+ */ -+ } else { -+ /* Hardware block was available and it had some counters -+ * enabled. We need to update the accumulation buffer. -+ */ -+ -+ /* Unconditionally copy the headers. */ -+ memcpy(acc_block, new_block, -+ phys_layout->headers_per_block * -+ KBASE_HWCNT_VALUE_BYTES); -+ -+ /* Accumulate counter samples -+ * -+ * When accumulating samples we need to take into -+ * account whether the counter sampling method involves -+ * clearing counters back to zero after each sample is -+ * taken. -+ * -+ * The intention for CSF was that all HW should use -+ * counters which wrap to zero when their maximum value -+ * is reached. This, combined with non-clearing -+ * sampling, enables multiple concurrent users to -+ * request samples without interfering with each other. -+ * -+ * However some early HW may not support wrapping -+ * counters, for these GPUs counters must be cleared on -+ * sample to avoid loss of data due to counters -+ * saturating at their maximum value. -+ */ -+ if (!clearing_samples) { -+ if (old_enable_mask == 0) { -+ /* Hardware block was previously -+ * unavailable. Accumulate the new -+ * counters only, as we know previous -+ * values are zeroes. -+ */ -+ for (ctr_idx = -+ phys_layout -+ ->headers_per_block; -+ ctr_idx < -+ phys_layout->values_per_block; -+ ctr_idx++) { -+ acc_block[ctr_idx] += -+ new_block[ctr_idx]; -+ } -+ } else { -+ /* Hardware block was previously -+ * available. Accumulate the delta -+ * between old and new counter values. -+ */ -+ for (ctr_idx = -+ phys_layout -+ ->headers_per_block; -+ ctr_idx < -+ phys_layout->values_per_block; -+ ctr_idx++) { -+ acc_block[ctr_idx] += -+ new_block[ctr_idx] - -+ old_block[ctr_idx]; -+ } -+ } -+ } else { -+ for (ctr_idx = phys_layout->headers_per_block; -+ ctr_idx < phys_layout->values_per_block; -+ ctr_idx++) { -+ acc_block[ctr_idx] += -+ new_block[ctr_idx]; -+ } -+ } -+ } -+ old_block += phys_layout->values_per_block; -+ new_block += phys_layout->values_per_block; -+ acc_block += phys_layout->values_per_block; -+ } -+ -+ WARN_ON(old_block != -+ old_sample_buf + dump_bytes / KBASE_HWCNT_VALUE_BYTES); -+ WARN_ON(new_block != -+ new_sample_buf + dump_bytes / KBASE_HWCNT_VALUE_BYTES); -+ WARN_ON(acc_block != accum_buf + dump_bytes / KBASE_HWCNT_VALUE_BYTES); -+ (void)dump_bytes; -+} -+ -+static void kbasep_hwcnt_backend_csf_accumulate_samples( -+ struct kbase_hwcnt_backend_csf *backend_csf, u32 extract_index_to_start, -+ u32 insert_index_to_stop) -+{ -+ u32 raw_idx; -+ unsigned long flags; -+ u8 *cpu_dump_base = (u8 *)backend_csf->ring_buf_cpu_base; -+ const size_t ring_buf_cnt = backend_csf->info->ring_buf_cnt; -+ const size_t buf_dump_bytes = backend_csf->info->prfcnt_info.dump_bytes; -+ bool clearing_samples = backend_csf->info->prfcnt_info.clearing_samples; -+ u32 *old_sample_buf = backend_csf->old_sample_buf; -+ u32 *new_sample_buf; -+ -+ if (extract_index_to_start == insert_index_to_stop) -+ /* No samples to accumulate. Early out. */ -+ return; -+ -+ /* Sync all the buffers to CPU side before read the data. */ -+ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, -+ backend_csf->ring_buf, -+ extract_index_to_start, -+ insert_index_to_stop, true); -+ -+ /* Consider u32 wrap case, '!=' is used here instead of '<' operator */ -+ for (raw_idx = extract_index_to_start; raw_idx != insert_index_to_stop; -+ raw_idx++) { -+ /* The logical "&" acts as a modulo operation since buf_count -+ * must be a power of two. -+ */ -+ const u32 buf_idx = raw_idx & (ring_buf_cnt - 1); -+ -+ new_sample_buf = -+ (u32 *)&cpu_dump_base[buf_idx * buf_dump_bytes]; -+ -+ kbasep_hwcnt_backend_csf_accumulate_sample( -+ &backend_csf->phys_layout, buf_dump_bytes, -+ backend_csf->accum_buf, old_sample_buf, new_sample_buf, -+ clearing_samples); -+ -+ old_sample_buf = new_sample_buf; -+ } -+ -+ /* Save the newest buffer as the old buffer for next time. */ -+ memcpy(backend_csf->old_sample_buf, new_sample_buf, buf_dump_bytes); -+ -+ /* Reset the prfcnt_en header on each sample before releasing them. */ -+ for (raw_idx = extract_index_to_start; raw_idx != insert_index_to_stop; -+ raw_idx++) { -+ const u32 buf_idx = raw_idx & (ring_buf_cnt - 1); -+ u32 *sample = (u32 *)&cpu_dump_base[buf_idx * buf_dump_bytes]; -+ -+ kbasep_hwcnt_backend_csf_zero_sample_prfcnt_en_header( -+ backend_csf, sample); -+ } -+ -+ /* Sync zeroed buffers to avoid coherency issues on future use. */ -+ backend_csf->info->csf_if->ring_buf_sync(backend_csf->info->csf_if->ctx, -+ backend_csf->ring_buf, -+ extract_index_to_start, -+ insert_index_to_stop, false); -+ -+ /* After consuming all samples between extract_idx and insert_idx, -+ * set the raw extract index to insert_idx so that the sample buffers -+ * can be released back to the ring buffer pool. -+ */ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ backend_csf->info->csf_if->set_extract_index( -+ backend_csf->info->csf_if->ctx, insert_index_to_stop); -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+} -+ -+static void kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ struct kbase_hwcnt_backend_csf *backend_csf, -+ enum kbase_hwcnt_backend_csf_enable_state new_state) -+{ -+ backend_csf->info->csf_if->assert_lock_held( -+ backend_csf->info->csf_if->ctx); -+ -+ if (backend_csf->enable_state != new_state) { -+ backend_csf->enable_state = new_state; -+ -+ wake_up(&backend_csf->enable_state_waitq); -+ } -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_dump_worker() - HWC dump worker. -+ * @work: Work structure. -+ * -+ * To accumulate all available samples in the ring buffer when a request has -+ * been done. -+ * -+ */ -+static void kbasep_hwcnt_backend_csf_dump_worker(struct work_struct *work) -+{ -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf *backend_csf; -+ u32 insert_index_to_acc; -+ u32 extract_index; -+ u32 insert_index; -+ -+ WARN_ON(!work); -+ backend_csf = container_of(work, struct kbase_hwcnt_backend_csf, -+ hwc_dump_work); -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ /* Assert the backend is not destroyed. */ -+ WARN_ON(backend_csf != backend_csf->info->backend); -+ -+ /* The backend was disabled or had an error while the worker was being -+ * launched. -+ */ -+ if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) { -+ WARN_ON(backend_csf->dump_state != -+ KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE); -+ WARN_ON(!completion_done(&backend_csf->dump_completed)); -+ backend_csf->info->csf_if->unlock( -+ backend_csf->info->csf_if->ctx, flags); -+ return; -+ } -+ -+ WARN_ON(backend_csf->dump_state != -+ KBASE_HWCNT_BACKEND_CSF_DUMP_WORKER_LAUNCHED); -+ -+ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_ACCUMULATING; -+ insert_index_to_acc = backend_csf->insert_index_to_accumulate; -+ -+ /* Read the raw extract and insert indexes from the CSF interface. */ -+ backend_csf->info->csf_if->get_indexes(backend_csf->info->csf_if->ctx, -+ &extract_index, &insert_index); -+ -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+ -+ /* Accumulate up to the insert we grabbed at the prfcnt request -+ * interrupt. -+ */ -+ kbasep_hwcnt_backend_csf_accumulate_samples(backend_csf, extract_index, -+ insert_index_to_acc); -+ -+ /* Copy to the user buffer so if a threshold interrupt fires -+ * between now and get(), the accumulations are untouched. -+ */ -+ kbasep_hwcnt_backend_csf_update_user_sample(backend_csf); -+ -+ /* Dump done, set state back to COMPLETED for next request. */ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ /* Assert the backend is not destroyed. */ -+ WARN_ON(backend_csf != backend_csf->info->backend); -+ -+ /* The backend was disabled or had an error while we were accumulating. -+ */ -+ if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) { -+ WARN_ON(backend_csf->dump_state != -+ KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE); -+ WARN_ON(!completion_done(&backend_csf->dump_completed)); -+ backend_csf->info->csf_if->unlock( -+ backend_csf->info->csf_if->ctx, flags); -+ return; -+ } -+ -+ WARN_ON(backend_csf->dump_state != -+ KBASE_HWCNT_BACKEND_CSF_DUMP_ACCUMULATING); -+ -+ /* Our work here is done - set the wait object and unblock waiters. */ -+ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED; -+ complete_all(&backend_csf->dump_completed); -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_threshold_worker() - Threshold worker. -+ * -+ * @work: Work structure. -+ * -+ * Called when a HWC threshold interrupt raised to consume all available samples -+ * in the ring buffer. -+ */ -+static void kbasep_hwcnt_backend_csf_threshold_worker(struct work_struct *work) -+{ -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf *backend_csf; -+ u32 extract_index; -+ u32 insert_index; -+ -+ WARN_ON(!work); -+ -+ backend_csf = container_of(work, struct kbase_hwcnt_backend_csf, -+ hwc_threshold_work); -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ -+ /* Assert the backend is not destroyed. */ -+ WARN_ON(backend_csf != backend_csf->info->backend); -+ -+ /* Read the raw extract and insert indexes from the CSF interface. */ -+ backend_csf->info->csf_if->get_indexes(backend_csf->info->csf_if->ctx, -+ &extract_index, &insert_index); -+ -+ /* The backend was disabled or had an error while the worker was being -+ * launched. -+ */ -+ if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) { -+ backend_csf->info->csf_if->unlock( -+ backend_csf->info->csf_if->ctx, flags); -+ return; -+ } -+ -+ /* Early out if we are not in the IDLE state or COMPLETED state, as this -+ * means a concurrent dump is in progress and we don't want to -+ * interfere. -+ */ -+ if ((backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE) && -+ (backend_csf->dump_state != -+ KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED)) { -+ backend_csf->info->csf_if->unlock( -+ backend_csf->info->csf_if->ctx, flags); -+ return; -+ } -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+ -+ /* Accumulate everything we possibly can. We grabbed the insert index -+ * immediately after we acquired the lock but before we checked whether -+ * a concurrent dump was triggered. This ensures that if a concurrent -+ * dump was triggered between releasing the lock and now, we know for a -+ * fact that our insert will not exceed the concurrent dump's -+ * insert_to_accumulate, so we don't risk accumulating too much data. -+ */ -+ kbasep_hwcnt_backend_csf_accumulate_samples(backend_csf, extract_index, -+ insert_index); -+ -+ /* No need to wake up anything since it is not a user dump request. */ -+} -+ -+static void kbase_hwcnt_backend_csf_submit_dump_worker( -+ struct kbase_hwcnt_backend_csf_info *csf_info) -+{ -+ u32 extract_index; -+ -+ WARN_ON(!csf_info); -+ csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); -+ -+ WARN_ON(!kbasep_hwcnt_backend_csf_backend_exists(csf_info)); -+ WARN_ON(csf_info->backend->enable_state != -+ KBASE_HWCNT_BACKEND_CSF_ENABLED); -+ WARN_ON(csf_info->backend->dump_state != -+ KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT); -+ -+ /* Save insert index now so that the dump worker only accumulates the -+ * HWC data associated with this request. Extract index is not stored -+ * as that needs to be checked when accumulating to prevent re-reading -+ * buffers that have already been read and returned to the GPU. -+ */ -+ csf_info->csf_if->get_indexes( -+ csf_info->csf_if->ctx, &extract_index, -+ &csf_info->backend->insert_index_to_accumulate); -+ csf_info->backend->dump_state = -+ KBASE_HWCNT_BACKEND_CSF_DUMP_WORKER_LAUNCHED; -+ -+ /* Submit the accumulator task into the work queue. */ -+ queue_work(csf_info->backend->hwc_dump_workq, -+ &csf_info->backend->hwc_dump_work); -+} -+ -+static void kbasep_hwcnt_backend_csf_get_physical_enable( -+ struct kbase_hwcnt_backend_csf *backend_csf, -+ const struct kbase_hwcnt_enable_map *enable_map, -+ struct kbase_hwcnt_backend_csf_if_enable *enable) -+{ -+ enum kbase_hwcnt_physical_set phys_counter_set; -+ struct kbase_hwcnt_physical_enable_map phys_enable_map; -+ -+ kbase_hwcnt_gpu_enable_map_to_physical(&phys_enable_map, enable_map); -+ -+ /* process the enable_map to guarantee the block header is enabled which -+ * is needed for delta calculation. -+ */ -+ kbasep_hwcnt_backend_csf_process_enable_map(&phys_enable_map); -+ -+ kbase_hwcnt_gpu_set_to_physical(&phys_counter_set, -+ backend_csf->info->counter_set); -+ -+ /* Use processed enable_map to enable HWC in HW level. */ -+ enable->fe_bm = phys_enable_map.fe_bm; -+ enable->shader_bm = phys_enable_map.shader_bm; -+ enable->tiler_bm = phys_enable_map.tiler_bm; -+ enable->mmu_l2_bm = phys_enable_map.mmu_l2_bm; -+ enable->counter_set = phys_counter_set; -+ enable->clk_enable_map = enable_map->clk_enable_map; -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ -+static int kbasep_hwcnt_backend_csf_dump_enable_nolock( -+ struct kbase_hwcnt_backend *backend, -+ const struct kbase_hwcnt_enable_map *enable_map) -+{ -+ struct kbase_hwcnt_backend_csf *backend_csf = -+ (struct kbase_hwcnt_backend_csf *)backend; -+ struct kbase_hwcnt_backend_csf_if_enable enable; -+ -+ if (!backend_csf || !enable_map || -+ (enable_map->metadata != backend_csf->info->metadata)) -+ return -EINVAL; -+ -+ backend_csf->info->csf_if->assert_lock_held( -+ backend_csf->info->csf_if->ctx); -+ -+ kbasep_hwcnt_backend_csf_get_physical_enable(backend_csf, enable_map, -+ &enable); -+ -+ /* enable_state should be DISABLED before we transfer it to enabled */ -+ if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) -+ return -EIO; -+ -+ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; -+ WARN_ON(!completion_done(&backend_csf->dump_completed)); -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED); -+ -+ backend_csf->info->csf_if->dump_enable(backend_csf->info->csf_if->ctx, -+ backend_csf->ring_buf, &enable); -+ -+ kbasep_hwcnt_backend_csf_cc_initial_sample(backend_csf, enable_map); -+ -+ return 0; -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_dump_enable_fn */ -+static int kbasep_hwcnt_backend_csf_dump_enable( -+ struct kbase_hwcnt_backend *backend, -+ const struct kbase_hwcnt_enable_map *enable_map) -+{ -+ int errcode; -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf *backend_csf = -+ (struct kbase_hwcnt_backend_csf *)backend; -+ -+ if (!backend_csf) -+ return -EINVAL; -+ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ errcode = kbasep_hwcnt_backend_csf_dump_enable_nolock(backend, -+ enable_map); -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+ return errcode; -+} -+ -+static void kbasep_hwcnt_backend_csf_wait_enable_transition_complete( -+ struct kbase_hwcnt_backend_csf *backend_csf, unsigned long *lock_flags) -+{ -+ backend_csf->info->csf_if->assert_lock_held( -+ backend_csf->info->csf_if->ctx); -+ -+ while ((backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) || -+ (backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED)) { -+ backend_csf->info->csf_if->unlock( -+ backend_csf->info->csf_if->ctx, *lock_flags); -+ -+ wait_event( -+ backend_csf->enable_state_waitq, -+ (backend_csf->enable_state != -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) && -+ (backend_csf->enable_state != -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED)); -+ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, -+ lock_flags); -+ } -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_dump_disable_fn */ -+static void -+kbasep_hwcnt_backend_csf_dump_disable(struct kbase_hwcnt_backend *backend) -+{ -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf *backend_csf = -+ (struct kbase_hwcnt_backend_csf *)backend; -+ bool do_disable = false; -+ -+ WARN_ON(!backend_csf); -+ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ -+ /* Make sure we wait until any previous enable or disable have completed -+ * before doing anything. -+ */ -+ kbasep_hwcnt_backend_csf_wait_enable_transition_complete(backend_csf, -+ &flags); -+ -+ if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED || -+ backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) { -+ /* If we are already disabled or in an unrecoverable error -+ * state, there is nothing for us to do. -+ */ -+ backend_csf->info->csf_if->unlock( -+ backend_csf->info->csf_if->ctx, flags); -+ return; -+ } -+ -+ if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) { -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); -+ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; -+ complete_all(&backend_csf->dump_completed); -+ /* Only disable if we were previously enabled - in all other -+ * cases the call to disable will have already been made. -+ */ -+ do_disable = true; -+ } -+ -+ WARN_ON(backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE); -+ WARN_ON(!completion_done(&backend_csf->dump_completed)); -+ -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+ -+ /* Block until any async work has completed. We have transitioned out of -+ * the ENABLED state so we can guarantee no new work will concurrently -+ * be submitted. -+ */ -+ flush_workqueue(backend_csf->hwc_dump_workq); -+ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ -+ if (do_disable) -+ backend_csf->info->csf_if->dump_disable( -+ backend_csf->info->csf_if->ctx); -+ -+ kbasep_hwcnt_backend_csf_wait_enable_transition_complete(backend_csf, -+ &flags); -+ -+ switch (backend_csf->enable_state) { -+ case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); -+ break; -+ case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); -+ break; -+ default: -+ WARN_ON(true); -+ break; -+ } -+ -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+ -+ /* After disable, zero the header of all buffers in the ring buffer back -+ * to 0 to prepare for the next enable. -+ */ -+ kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(backend_csf); -+ -+ /* Sync zeroed buffers to avoid coherency issues on future use. */ -+ backend_csf->info->csf_if->ring_buf_sync( -+ backend_csf->info->csf_if->ctx, backend_csf->ring_buf, 0, -+ backend_csf->info->ring_buf_cnt, false); -+ -+ /* Reset accumulator, old_sample_buf and user_sample to all-0 to prepare -+ * for next enable. -+ */ -+ kbasep_hwcnt_backend_csf_reset_internal_buffers(backend_csf); -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_dump_request_fn */ -+static int -+kbasep_hwcnt_backend_csf_dump_request(struct kbase_hwcnt_backend *backend, -+ u64 *dump_time_ns) -+{ -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf *backend_csf = -+ (struct kbase_hwcnt_backend_csf *)backend; -+ bool do_request = false; -+ -+ if (!backend_csf) -+ return -EINVAL; -+ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ -+ /* If we're transitioning to enabled there's nothing to accumulate, and -+ * the user dump buffer is already zeroed. We can just short circuit to -+ * the DUMP_COMPLETED state. -+ */ -+ if (backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) { -+ backend_csf->dump_state = -+ KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED; -+ *dump_time_ns = kbasep_hwcnt_backend_csf_timestamp_ns(backend); -+ kbasep_hwcnt_backend_csf_cc_update(backend_csf); -+ backend_csf->info->csf_if->unlock( -+ backend_csf->info->csf_if->ctx, flags); -+ return 0; -+ } -+ -+ /* Otherwise, make sure we're already enabled. */ -+ if (backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_ENABLED) { -+ backend_csf->info->csf_if->unlock( -+ backend_csf->info->csf_if->ctx, flags); -+ return -EIO; -+ } -+ -+ /* Make sure that this is either the first request since enable or the -+ * previous dump has completed, so we can avoid midway through a dump. -+ */ -+ if ((backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE) && -+ (backend_csf->dump_state != -+ KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED)) { -+ backend_csf->info->csf_if->unlock( -+ backend_csf->info->csf_if->ctx, flags); -+ /* HWC is disabled or another dump is ongoing, or we are on -+ * fault. -+ */ -+ return -EIO; -+ } -+ -+ /* Reset the completion so dump_wait() has something to wait on. */ -+ reinit_completion(&backend_csf->dump_completed); -+ -+ if ((backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) && -+ !backend_csf->info->fw_in_protected_mode) { -+ /* Only do the request if we are fully enabled and not in -+ * protected mode. -+ */ -+ backend_csf->dump_state = -+ KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED; -+ do_request = true; -+ } else { -+ /* Skip the request and waiting for ack and go straight to -+ * checking the insert and kicking off the worker to do the dump -+ */ -+ backend_csf->dump_state = -+ KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT; -+ } -+ -+ /* CSF firmware might enter protected mode now, but still call request. -+ * That is fine, as we changed state while holding the lock, so the -+ * protected mode enter function will query the insert and launch the -+ * dumping worker. -+ * At some point we will get the dump request ACK saying a dump is done, -+ * but we can ignore it if we are not in the REQUESTED state and process -+ * it in next round dumping worker. -+ */ -+ -+ *dump_time_ns = kbasep_hwcnt_backend_csf_timestamp_ns(backend); -+ kbasep_hwcnt_backend_csf_cc_update(backend_csf); -+ -+ if (do_request) -+ backend_csf->info->csf_if->dump_request( -+ backend_csf->info->csf_if->ctx); -+ else -+ kbase_hwcnt_backend_csf_submit_dump_worker(backend_csf->info); -+ -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+ return 0; -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_dump_wait_fn */ -+static int -+kbasep_hwcnt_backend_csf_dump_wait(struct kbase_hwcnt_backend *backend) -+{ -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf *backend_csf = -+ (struct kbase_hwcnt_backend_csf *)backend; -+ int errcode; -+ -+ if (!backend_csf) -+ return -EINVAL; -+ -+ wait_for_completion(&backend_csf->dump_completed); -+ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ /* Make sure the last dump actually succeeded. */ -+ errcode = (backend_csf->dump_state == -+ KBASE_HWCNT_BACKEND_CSF_DUMP_COMPLETED) ? -+ 0 : -+ -EIO; -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+ -+ return errcode; -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_dump_clear_fn */ -+static int -+kbasep_hwcnt_backend_csf_dump_clear(struct kbase_hwcnt_backend *backend) -+{ -+ struct kbase_hwcnt_backend_csf *backend_csf = -+ (struct kbase_hwcnt_backend_csf *)backend; -+ int errcode; -+ u64 ts; -+ -+ if (!backend_csf) -+ return -EINVAL; -+ -+ /* Request a dump so we can clear all current counters. */ -+ errcode = kbasep_hwcnt_backend_csf_dump_request(backend, &ts); -+ if (!errcode) -+ /* Wait for the manual dump or auto dump to be done and -+ * accumulator to be updated. -+ */ -+ errcode = kbasep_hwcnt_backend_csf_dump_wait(backend); -+ -+ return errcode; -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_dump_get_fn */ -+static int kbasep_hwcnt_backend_csf_dump_get( -+ struct kbase_hwcnt_backend *backend, -+ struct kbase_hwcnt_dump_buffer *dst, -+ const struct kbase_hwcnt_enable_map *dst_enable_map, bool accumulate) -+{ -+ struct kbase_hwcnt_backend_csf *backend_csf = -+ (struct kbase_hwcnt_backend_csf *)backend; -+ int ret; -+ size_t clk; -+ -+ if (!backend_csf || !dst || !dst_enable_map || -+ (backend_csf->info->metadata != dst->metadata) || -+ (dst_enable_map->metadata != dst->metadata)) -+ return -EINVAL; -+ -+ kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) { -+ if (!kbase_hwcnt_clk_enable_map_enabled( -+ dst_enable_map->clk_enable_map, clk)) -+ continue; -+ -+ /* Extract elapsed cycle count for each clock domain. */ -+ dst->clk_cnt_buf[clk] = backend_csf->cycle_count_elapsed[clk]; -+ } -+ -+ /* We just return the user buffer without checking the current state, -+ * as it is undefined to call this function without a prior succeeding -+ * one to dump_wait(). -+ */ -+ ret = kbase_hwcnt_csf_dump_get(dst, backend_csf->to_user_buf, -+ dst_enable_map, accumulate); -+ -+ return ret; -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_destroy() - Destroy CSF backend. -+ * @backend_csf: Pointer to CSF backend to destroy. -+ * -+ * Can be safely called on a backend in any state of partial construction. -+ * -+ */ -+static void -+kbasep_hwcnt_backend_csf_destroy(struct kbase_hwcnt_backend_csf *backend_csf) -+{ -+ if (!backend_csf) -+ return; -+ -+ destroy_workqueue(backend_csf->hwc_dump_workq); -+ -+ backend_csf->info->csf_if->ring_buf_free(backend_csf->info->csf_if->ctx, -+ backend_csf->ring_buf); -+ -+ kfree(backend_csf->accum_buf); -+ backend_csf->accum_buf = NULL; -+ -+ kfree(backend_csf->old_sample_buf); -+ backend_csf->old_sample_buf = NULL; -+ -+ kfree(backend_csf->to_user_buf); -+ backend_csf->to_user_buf = NULL; -+ -+ kfree(backend_csf); -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_create() - Create a CSF backend instance. -+ * -+ * @csf_info: Non-NULL pointer to backend info. -+ * @out_backend: Non-NULL pointer to where backend is stored on success. -+ * Return: 0 on success, else error code. -+ */ -+static int -+kbasep_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_info *csf_info, -+ struct kbase_hwcnt_backend_csf **out_backend) -+{ -+ struct kbase_hwcnt_backend_csf *backend_csf = NULL; -+ int errcode = -ENOMEM; -+ -+ WARN_ON(!csf_info); -+ WARN_ON(!out_backend); -+ -+ backend_csf = kzalloc(sizeof(*backend_csf), GFP_KERNEL); -+ if (!backend_csf) -+ goto alloc_error; -+ -+ backend_csf->info = csf_info; -+ kbasep_hwcnt_backend_csf_init_layout(&csf_info->prfcnt_info, -+ &backend_csf->phys_layout); -+ -+ backend_csf->accum_buf = -+ kzalloc(csf_info->prfcnt_info.dump_bytes, GFP_KERNEL); -+ if (!backend_csf->accum_buf) -+ goto err_alloc_acc_buf; -+ -+ backend_csf->old_sample_buf = -+ kzalloc(csf_info->prfcnt_info.dump_bytes, GFP_KERNEL); -+ if (!backend_csf->old_sample_buf) -+ goto err_alloc_pre_sample_buf; -+ -+ backend_csf->to_user_buf = -+ kzalloc(csf_info->prfcnt_info.dump_bytes, GFP_KERNEL); -+ if (!backend_csf->to_user_buf) -+ goto err_alloc_user_sample_buf; -+ -+ errcode = csf_info->csf_if->ring_buf_alloc( -+ csf_info->csf_if->ctx, csf_info->ring_buf_cnt, -+ &backend_csf->ring_buf_cpu_base, &backend_csf->ring_buf); -+ if (errcode) -+ goto err_ring_buf_alloc; -+ -+ /* Zero all performance enable header to prepare for first enable. */ -+ kbasep_hwcnt_backend_csf_zero_all_prfcnt_en_header(backend_csf); -+ -+ /* Sync zeroed buffers to avoid coherency issues on use. */ -+ backend_csf->info->csf_if->ring_buf_sync( -+ backend_csf->info->csf_if->ctx, backend_csf->ring_buf, 0, -+ backend_csf->info->ring_buf_cnt, false); -+ -+ init_completion(&backend_csf->dump_completed); -+ -+ init_waitqueue_head(&backend_csf->enable_state_waitq); -+ -+ /* Allocate a single threaded work queue for dump worker and threshold -+ * worker. -+ */ -+ backend_csf->hwc_dump_workq = -+ alloc_workqueue("mali_hwc_dump_wq", WQ_HIGHPRI | WQ_UNBOUND, 1); -+ if (!backend_csf->hwc_dump_workq) -+ goto err_alloc_workqueue; -+ -+ INIT_WORK(&backend_csf->hwc_dump_work, -+ kbasep_hwcnt_backend_csf_dump_worker); -+ INIT_WORK(&backend_csf->hwc_threshold_work, -+ kbasep_hwcnt_backend_csf_threshold_worker); -+ -+ backend_csf->enable_state = KBASE_HWCNT_BACKEND_CSF_DISABLED; -+ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; -+ complete_all(&backend_csf->dump_completed); -+ -+ *out_backend = backend_csf; -+ return 0; -+ -+ destroy_workqueue(backend_csf->hwc_dump_workq); -+err_alloc_workqueue: -+ backend_csf->info->csf_if->ring_buf_free(backend_csf->info->csf_if->ctx, -+ backend_csf->ring_buf); -+err_ring_buf_alloc: -+ kfree(backend_csf->to_user_buf); -+ backend_csf->to_user_buf = NULL; -+err_alloc_user_sample_buf: -+ kfree(backend_csf->old_sample_buf); -+ backend_csf->old_sample_buf = NULL; -+err_alloc_pre_sample_buf: -+ kfree(backend_csf->accum_buf); -+ backend_csf->accum_buf = NULL; -+err_alloc_acc_buf: -+ kfree(backend_csf); -+alloc_error: -+ return errcode; -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_init_fn */ -+static int -+kbasep_hwcnt_backend_csf_init(const struct kbase_hwcnt_backend_info *info, -+ struct kbase_hwcnt_backend **out_backend) -+{ -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf *backend_csf = NULL; -+ struct kbase_hwcnt_backend_csf_info *csf_info = -+ (struct kbase_hwcnt_backend_csf_info *)info; -+ int errcode; -+ bool success = false; -+ -+ if (!info || !out_backend) -+ return -EINVAL; -+ -+ /* Create the backend. */ -+ errcode = kbasep_hwcnt_backend_csf_create(csf_info, &backend_csf); -+ if (errcode) -+ return errcode; -+ -+ /* If it was not created before, attach it to csf_info. -+ * Use spin lock to avoid concurrent initialization. -+ */ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ if (csf_info->backend == NULL) { -+ csf_info->backend = backend_csf; -+ *out_backend = (struct kbase_hwcnt_backend *)backend_csf; -+ success = true; -+ if (csf_info->unrecoverable_error_happened) -+ backend_csf->enable_state = -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR; -+ } -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+ -+ /* Destroy the new created backend if the backend has already created -+ * before. In normal case, this won't happen if the client call init() -+ * function properly. -+ */ -+ if (!success) { -+ kbasep_hwcnt_backend_csf_destroy(backend_csf); -+ return -EBUSY; -+ } -+ -+ return 0; -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_term_fn */ -+static void kbasep_hwcnt_backend_csf_term(struct kbase_hwcnt_backend *backend) -+{ -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf *backend_csf = -+ (struct kbase_hwcnt_backend_csf *)backend; -+ -+ if (!backend) -+ return; -+ -+ kbasep_hwcnt_backend_csf_dump_disable(backend); -+ -+ /* Set the backend in csf_info to NULL so we won't handle any external -+ * notification anymore since we are terminating. -+ */ -+ backend_csf->info->csf_if->lock(backend_csf->info->csf_if->ctx, &flags); -+ backend_csf->info->backend = NULL; -+ backend_csf->info->csf_if->unlock(backend_csf->info->csf_if->ctx, -+ flags); -+ -+ kbasep_hwcnt_backend_csf_destroy(backend_csf); -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_info_destroy() - Destroy a CSF backend info. -+ * @info: Pointer to info to destroy. -+ * -+ * Can be safely called on a backend info in any state of partial construction. -+ * -+ */ -+static void kbasep_hwcnt_backend_csf_info_destroy( -+ const struct kbase_hwcnt_backend_csf_info *info) -+{ -+ if (!info) -+ return; -+ -+ /* The backend should be destroyed before the info object destroy. */ -+ WARN_ON(info->backend != NULL); -+ -+ /* The metadata should be destroyed before the info object destroy. */ -+ WARN_ON(info->metadata != NULL); -+ -+ kfree(info); -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_info_create() - Create a CSF backend info. -+ * -+ * @csf_if: Non-NULL pointer to a hwcnt backend CSF interface structure -+ * used to create backend interface. -+ * @ring_buf_cnt: The buffer count of the CSF hwcnt backend ring buffer. -+ * MUST be power of 2. -+ * @out_info: Non-NULL pointer to where info is stored on success. -+ * @return 0 on success, else error code. -+ */ -+static int kbasep_hwcnt_backend_csf_info_create( -+ struct kbase_hwcnt_backend_csf_if *csf_if, u32 ring_buf_cnt, -+ const struct kbase_hwcnt_backend_csf_info **out_info) -+{ -+ struct kbase_hwcnt_backend_csf_info *info = NULL; -+ -+ WARN_ON(!csf_if); -+ WARN_ON(!out_info); -+ WARN_ON(!is_power_of_2(ring_buf_cnt)); -+ -+ info = kzalloc(sizeof(*info), GFP_KERNEL); -+ if (!info) -+ return -ENOMEM; -+ -+#if defined(CONFIG_MALI_PRFCNT_SET_SECONDARY) -+ info->counter_set = KBASE_HWCNT_SET_SECONDARY; -+#elif defined(CONFIG_MALI_PRFCNT_SET_TERTIARY) -+ info->counter_set = KBASE_HWCNT_SET_TERTIARY; -+#else -+ /* Default to primary */ -+ info->counter_set = KBASE_HWCNT_SET_PRIMARY; -+#endif -+ -+ info->backend = NULL; -+ info->csf_if = csf_if; -+ info->ring_buf_cnt = ring_buf_cnt; -+ info->fw_in_protected_mode = false; -+ info->unrecoverable_error_happened = false; -+ -+ *out_info = info; -+ -+ return 0; -+} -+ -+/* CSF backend implementation of kbase_hwcnt_backend_metadata_fn */ -+static const struct kbase_hwcnt_metadata * -+kbasep_hwcnt_backend_csf_metadata(const struct kbase_hwcnt_backend_info *info) -+{ -+ if (!info) -+ return NULL; -+ -+ WARN_ON(!((const struct kbase_hwcnt_backend_csf_info *)info)->metadata); -+ -+ return ((const struct kbase_hwcnt_backend_csf_info *)info)->metadata; -+} -+ -+static void kbasep_hwcnt_backend_csf_handle_unrecoverable_error( -+ struct kbase_hwcnt_backend_csf *backend_csf) -+{ -+ bool do_disable = false; -+ -+ backend_csf->info->csf_if->assert_lock_held( -+ backend_csf->info->csf_if->ctx); -+ -+ /* We are already in or transitioning to the unrecoverable error state. -+ * Early out. -+ */ -+ if ((backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) || -+ (backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER)) -+ return; -+ -+ /* If we are disabled, we know we have no pending workers, so skip the -+ * waiting state. -+ */ -+ if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_DISABLED) { -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR); -+ return; -+ } -+ -+ /* Trigger a disable only if we are not already transitioning to -+ * disabled, we don't want to disable twice if an unrecoverable error -+ * happens while we are disabling. -+ */ -+ do_disable = (backend_csf->enable_state != -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); -+ -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER); -+ -+ /* Transition the dump to the IDLE state and unblock any waiters. The -+ * IDLE state signifies an error. -+ */ -+ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; -+ complete_all(&backend_csf->dump_completed); -+ -+ /* Trigger a disable only if we are not already transitioning to -+ * disabled, - we don't want to disable twice if an unrecoverable error -+ * happens while we are disabling. -+ */ -+ if (do_disable) -+ backend_csf->info->csf_if->dump_disable( -+ backend_csf->info->csf_if->ctx); -+} -+ -+static void kbasep_hwcnt_backend_csf_handle_recoverable_error( -+ struct kbase_hwcnt_backend_csf *backend_csf) -+{ -+ backend_csf->info->csf_if->assert_lock_held( -+ backend_csf->info->csf_if->ctx); -+ -+ switch (backend_csf->enable_state) { -+ case KBASE_HWCNT_BACKEND_CSF_DISABLED: -+ case KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER: -+ case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED: -+ case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR: -+ case KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR_WAIT_FOR_WORKER: -+ /* Already disabled or disabling, or in an unrecoverable error. -+ * Nothing to be done to handle the error. -+ */ -+ return; -+ case KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED: -+ /* A seemingly recoverable error that occurs while we are -+ * transitioning to enabled is probably unrecoverable. -+ */ -+ kbasep_hwcnt_backend_csf_handle_unrecoverable_error( -+ backend_csf); -+ return; -+ case KBASE_HWCNT_BACKEND_CSF_ENABLED: -+ /* Start transitioning to the disabled state. We can't wait for -+ * it as this recoverable error might be triggered from an -+ * interrupt. The wait will be done in the eventual call to -+ * disable(). -+ */ -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED); -+ /* Transition the dump to the IDLE state and unblock any -+ * waiters. The IDLE state signifies an error. -+ */ -+ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_IDLE; -+ complete_all(&backend_csf->dump_completed); -+ -+ backend_csf->info->csf_if->dump_disable( -+ backend_csf->info->csf_if->ctx); -+ return; -+ } -+} -+ -+void kbase_hwcnt_backend_csf_protm_entered( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ struct kbase_hwcnt_backend_csf_info *csf_info = -+ (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ -+ csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); -+ csf_info->fw_in_protected_mode = true; -+ -+ /* Call on_prfcnt_sample() to trigger collection of the protected mode -+ * entry auto-sample if there is currently a pending dump request. -+ */ -+ kbase_hwcnt_backend_csf_on_prfcnt_sample(iface); -+} -+ -+void kbase_hwcnt_backend_csf_protm_exited( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ -+ csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); -+ csf_info->fw_in_protected_mode = false; -+} -+ -+void kbase_hwcnt_backend_csf_on_unrecoverable_error( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ -+ csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); -+ csf_info->unrecoverable_error_happened = true; -+ /* Early out if the backend does not exist. */ -+ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { -+ csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); -+ return; -+ } -+ -+ kbasep_hwcnt_backend_csf_handle_unrecoverable_error(csf_info->backend); -+ -+ csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); -+} -+ -+void kbase_hwcnt_backend_csf_on_before_reset( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ unsigned long flags; -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ struct kbase_hwcnt_backend_csf *backend_csf; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ -+ csf_info->csf_if->lock(csf_info->csf_if->ctx, &flags); -+ csf_info->unrecoverable_error_happened = false; -+ /* Early out if the backend does not exist. */ -+ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) { -+ csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); -+ return; -+ } -+ backend_csf = csf_info->backend; -+ -+ if ((backend_csf->enable_state != KBASE_HWCNT_BACKEND_CSF_DISABLED) && -+ (backend_csf->enable_state != -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR)) { -+ /* Before a reset occurs, we must either have been disabled -+ * (else we lose data) or we should have encountered an -+ * unrecoverable error. Either way, we will have disabled the -+ * interface and waited for any workers that might have still -+ * been in flight. -+ * If not in these states, fire off one more disable to make -+ * sure everything is turned off before the power is pulled. -+ * We can't wait for this disable to complete, but it doesn't -+ * really matter, the power is being pulled. -+ */ -+ kbasep_hwcnt_backend_csf_handle_unrecoverable_error( -+ csf_info->backend); -+ } -+ -+ /* A reset is the only way to exit the unrecoverable error state */ -+ if (backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_UNRECOVERABLE_ERROR) { -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, KBASE_HWCNT_BACKEND_CSF_DISABLED); -+ } -+ -+ csf_info->csf_if->unlock(csf_info->csf_if->ctx, flags); -+} -+ -+void kbase_hwcnt_backend_csf_on_prfcnt_sample( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ struct kbase_hwcnt_backend_csf *backend_csf; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); -+ -+ /* Early out if the backend does not exist. */ -+ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) -+ return; -+ backend_csf = csf_info->backend; -+ -+ /* If the current state is not REQUESTED, this HWC sample will be -+ * skipped and processed in next dump_request. -+ */ -+ if (backend_csf->dump_state != KBASE_HWCNT_BACKEND_CSF_DUMP_REQUESTED) -+ return; -+ backend_csf->dump_state = KBASE_HWCNT_BACKEND_CSF_DUMP_QUERYING_INSERT; -+ -+ kbase_hwcnt_backend_csf_submit_dump_worker(csf_info); -+} -+ -+void kbase_hwcnt_backend_csf_on_prfcnt_threshold( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ struct kbase_hwcnt_backend_csf *backend_csf; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); -+ -+ /* Early out if the backend does not exist. */ -+ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) -+ return; -+ backend_csf = csf_info->backend; -+ -+ if (backend_csf->enable_state == KBASE_HWCNT_BACKEND_CSF_ENABLED) -+ /* Submit the threshold work into the work queue to consume the -+ * available samples. -+ */ -+ queue_work(backend_csf->hwc_dump_workq, -+ &backend_csf->hwc_threshold_work); -+} -+ -+void kbase_hwcnt_backend_csf_on_prfcnt_overflow( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); -+ -+ /* Early out if the backend does not exist. */ -+ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) -+ return; -+ -+ /* Called when an overflow occurs. We treat this as a recoverable error, -+ * so we start transitioning to the disabled state. -+ * We could try and handle it while enabled, but in a real system we -+ * never expect an overflow to occur so there is no point implementing -+ * complex recovery code when we can just turn ourselves off instead for -+ * a while. -+ */ -+ kbasep_hwcnt_backend_csf_handle_recoverable_error(csf_info->backend); -+} -+ -+void kbase_hwcnt_backend_csf_on_prfcnt_enable( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ struct kbase_hwcnt_backend_csf *backend_csf; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); -+ -+ /* Early out if the backend does not exist. */ -+ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) -+ return; -+ backend_csf = csf_info->backend; -+ -+ if (backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_ENABLED) { -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, KBASE_HWCNT_BACKEND_CSF_ENABLED); -+ } else if (backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_ENABLED) { -+ /* Unexpected, but we are already in the right state so just -+ * ignore it. -+ */ -+ } else { -+ /* Unexpected state change, assume everything is broken until -+ * we reset. -+ */ -+ kbasep_hwcnt_backend_csf_handle_unrecoverable_error( -+ csf_info->backend); -+ } -+} -+ -+void kbase_hwcnt_backend_csf_on_prfcnt_disable( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ struct kbase_hwcnt_backend_csf *backend_csf; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ csf_info->csf_if->assert_lock_held(csf_info->csf_if->ctx); -+ -+ /* Early out if the backend does not exist. */ -+ if (!kbasep_hwcnt_backend_csf_backend_exists(csf_info)) -+ return; -+ backend_csf = csf_info->backend; -+ -+ if (backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_TRANSITIONING_TO_DISABLED) { -+ kbasep_hwcnt_backend_csf_change_es_and_wake_waiters( -+ backend_csf, -+ KBASE_HWCNT_BACKEND_CSF_DISABLED_WAIT_FOR_WORKER); -+ } else if (backend_csf->enable_state == -+ KBASE_HWCNT_BACKEND_CSF_DISABLED) { -+ /* Unexpected, but we are already in the right state so just -+ * ignore it. -+ */ -+ } else { -+ /* Unexpected state change, assume everything is broken until -+ * we reset. -+ */ -+ kbasep_hwcnt_backend_csf_handle_unrecoverable_error( -+ csf_info->backend); -+ } -+} -+ -+int kbase_hwcnt_backend_csf_metadata_init( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ int errcode; -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ struct kbase_hwcnt_gpu_info gpu_info; -+ -+ if (!iface) -+ return -EINVAL; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ -+ WARN_ON(!csf_info->csf_if->get_prfcnt_info); -+ -+ csf_info->csf_if->get_prfcnt_info(csf_info->csf_if->ctx, -+ &csf_info->prfcnt_info); -+ -+ /* The clock domain counts should not exceed the number of maximum -+ * number of clock regulators. -+ */ -+ if (csf_info->prfcnt_info.clk_cnt > BASE_MAX_NR_CLOCKS_REGULATORS) -+ return -EIO; -+ -+ gpu_info.l2_count = csf_info->prfcnt_info.l2_count; -+ gpu_info.core_mask = csf_info->prfcnt_info.core_mask; -+ gpu_info.clk_cnt = csf_info->prfcnt_info.clk_cnt; -+ gpu_info.prfcnt_values_per_block = -+ csf_info->prfcnt_info.prfcnt_block_size / -+ KBASE_HWCNT_VALUE_BYTES; -+ errcode = kbase_hwcnt_csf_metadata_create( -+ &gpu_info, csf_info->counter_set, &csf_info->metadata); -+ if (errcode) -+ return errcode; -+ -+ /* -+ * Dump abstraction size should be exactly the same size and layout as -+ * the physical dump size, for backwards compatibility. -+ */ -+ WARN_ON(csf_info->prfcnt_info.dump_bytes != -+ csf_info->metadata->dump_buf_bytes); -+ -+ return 0; -+} -+ -+void kbase_hwcnt_backend_csf_metadata_term( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ struct kbase_hwcnt_backend_csf_info *csf_info; -+ -+ if (!iface) -+ return; -+ -+ csf_info = (struct kbase_hwcnt_backend_csf_info *)iface->info; -+ if (csf_info->metadata) { -+ kbase_hwcnt_csf_metadata_destroy(csf_info->metadata); -+ csf_info->metadata = NULL; -+ } -+} -+ -+int kbase_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_if *csf_if, -+ u32 ring_buf_cnt, -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ int errcode; -+ const struct kbase_hwcnt_backend_csf_info *info = NULL; -+ -+ if (!iface || !csf_if) -+ return -EINVAL; -+ -+ /* The buffer count must be power of 2 */ -+ if (!is_power_of_2(ring_buf_cnt)) -+ return -EINVAL; -+ -+ errcode = kbasep_hwcnt_backend_csf_info_create(csf_if, ring_buf_cnt, -+ &info); -+ if (errcode) -+ return errcode; -+ -+ iface->info = (struct kbase_hwcnt_backend_info *)info; -+ iface->metadata = kbasep_hwcnt_backend_csf_metadata; -+ iface->init = kbasep_hwcnt_backend_csf_init; -+ iface->term = kbasep_hwcnt_backend_csf_term; -+ iface->timestamp_ns = kbasep_hwcnt_backend_csf_timestamp_ns; -+ iface->dump_enable = kbasep_hwcnt_backend_csf_dump_enable; -+ iface->dump_enable_nolock = kbasep_hwcnt_backend_csf_dump_enable_nolock; -+ iface->dump_disable = kbasep_hwcnt_backend_csf_dump_disable; -+ iface->dump_clear = kbasep_hwcnt_backend_csf_dump_clear; -+ iface->dump_request = kbasep_hwcnt_backend_csf_dump_request; -+ iface->dump_wait = kbasep_hwcnt_backend_csf_dump_wait; -+ iface->dump_get = kbasep_hwcnt_backend_csf_dump_get; -+ -+ return 0; -+} -+ -+void kbase_hwcnt_backend_csf_destroy(struct kbase_hwcnt_backend_interface *iface) -+{ -+ if (!iface) -+ return; -+ -+ kbasep_hwcnt_backend_csf_info_destroy( -+ (const struct kbase_hwcnt_backend_csf_info *)iface->info); -+ memset(iface, 0, sizeof(*iface)); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf.h -new file mode 100644 -index 0000000..ce1af9a ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf.h -@@ -0,0 +1,162 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/** -+ * Concrete implementation of mali_kbase_hwcnt_backend interface for CSF -+ * backend. -+ */ -+ -+#ifndef _KBASE_HWCNT_BACKEND_CSF_H_ -+#define _KBASE_HWCNT_BACKEND_CSF_H_ -+ -+#include "mali_kbase_hwcnt_backend.h" -+#include "mali_kbase_hwcnt_backend_csf_if.h" -+ -+/** -+ * kbase_hwcnt_backend_csf_create() - Create a CSF hardware counter backend -+ * interface. -+ * @csf_if: Non-NULL pointer to a hwcnt backend CSF interface structure -+ * used to create backend interface. -+ * @ring_buf_cnt: The buffer count of CSF hwcnt backend, used when allocate ring -+ * buffer, MUST be power of 2. -+ * @iface: Non-NULL pointer to backend interface structure that is filled -+ * in on creation success. -+ * -+ * Calls to iface->dump_enable_nolock() require the CSF Scheduler IRQ lock. -+ * -+ * Return: 0 on success, else error code. -+ */ -+int kbase_hwcnt_backend_csf_create(struct kbase_hwcnt_backend_csf_if *csf_if, -+ u32 ring_buf_cnt, -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_metadata_init() - Initialize the metadata for a CSF -+ * hardware counter backend. -+ * @iface: Non-NULL pointer to backend interface structure -+ * Return: 0 on success, else error code. -+ */ -+int kbase_hwcnt_backend_csf_metadata_init( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_metadata_term() - Terminate the metadata for a CSF -+ * hardware counter backend. -+ * @iface: Non-NULL pointer to backend interface structure. -+ */ -+void kbase_hwcnt_backend_csf_metadata_term( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_destroy() - Destroy a CSF hardware counter backend -+ * interface. -+ * @iface: Pointer to interface to destroy. -+ * -+ * Can be safely called on an all-zeroed interface, or on an already destroyed -+ * interface. -+ */ -+void kbase_hwcnt_backend_csf_destroy( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_protm_entered() - CSF HWC backend function to receive -+ * notification that protected mode -+ * has been entered. -+ * @iface: Non-NULL pointer to HWC backend interface. -+ */ -+void kbase_hwcnt_backend_csf_protm_entered( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_protm_exited() - CSF HWC backend function to receive -+ * notification that protected mode has -+ * been exited. -+ * @iface: Non-NULL pointer to HWC backend interface. -+ */ -+void kbase_hwcnt_backend_csf_protm_exited( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_on_unrecoverable_error() - CSF HWC backend function -+ * called when unrecoverable -+ * errors are detected. -+ * @iface: Non-NULL pointer to HWC backend interface. -+ * -+ * This should be called on encountering errors that can only be recovered from -+ * with reset, or that may put HWC logic in state that could result in hang. For -+ * example, on bus error, or when FW becomes unresponsive. -+ */ -+void kbase_hwcnt_backend_csf_on_unrecoverable_error( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_on_before_reset() - CSF HWC backend function to be -+ * called immediately before a -+ * reset. Takes us out of the -+ * unrecoverable error state, if we -+ * were in it. -+ * @iface: Non-NULL pointer to HWC backend interface. -+ */ -+void kbase_hwcnt_backend_csf_on_before_reset( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_on_prfcnt_sample() - CSF performance counter sample -+ * complete interrupt handler. -+ * @iface: Non-NULL pointer to HWC backend interface. -+ */ -+void kbase_hwcnt_backend_csf_on_prfcnt_sample( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_on_prfcnt_threshold() - CSF performance counter -+ * buffer reach threshold -+ * interrupt handler. -+ * @iface: Non-NULL pointer to HWC backend interface. -+ */ -+void kbase_hwcnt_backend_csf_on_prfcnt_threshold( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_on_prfcnt_overflow() - CSF performance counter buffer -+ * overflow interrupt handler. -+ * @iface: Non-NULL pointer to HWC backend interface. -+ */ -+void kbase_hwcnt_backend_csf_on_prfcnt_overflow( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_on_prfcnt_enable() - CSF performance counter enabled -+ * interrupt handler. -+ * @iface: Non-NULL pointer to HWC backend interface. -+ */ -+void kbase_hwcnt_backend_csf_on_prfcnt_enable( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+/** -+ * kbase_hwcnt_backend_csf_on_prfcnt_disable() - CSF performance counter -+ * disabled interrupt handler. -+ * @iface: Non-NULL pointer to HWC backend interface. -+ */ -+void kbase_hwcnt_backend_csf_on_prfcnt_disable( -+ struct kbase_hwcnt_backend_interface *iface); -+ -+#endif /* _KBASE_HWCNT_BACKEND_CSF_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if.h -new file mode 100644 -index 0000000..f6387c2 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if.h -@@ -0,0 +1,311 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * Virtual interface for CSF hardware counter backend. -+ */ -+ -+#ifndef _KBASE_HWCNT_BACKEND_CSF_IF_H_ -+#define _KBASE_HWCNT_BACKEND_CSF_IF_H_ -+ -+#include <linux/types.h> -+ -+/** -+ * struct kbase_hwcnt_backend_csf_if_ctx - Opaque pointer to a CSF interface -+ * context. -+ */ -+struct kbase_hwcnt_backend_csf_if_ctx; -+ -+/** -+ * struct kbase_hwcnt_backend_csf_if_ring_buf - Opaque pointer to a CSF -+ * interface ring buffer. -+ */ -+struct kbase_hwcnt_backend_csf_if_ring_buf; -+ -+/** -+ * struct kbase_hwcnt_backend_csf_if_enable - enable hardware counter collection -+ * structure. -+ * @fe_bm: Front End counters selection bitmask. -+ * @shader_bm: Shader counters selection bitmask. -+ * @tiler_bm: Tiler counters selection bitmask. -+ * @mmu_l2_bm: MMU_L2 counters selection bitmask. -+ * @counter_set: The performance counter set to enable. -+ * @clk_enable_map: An array of u64 bitfields, each bit of which enables cycle -+ * counter for a given clock domain. -+ */ -+struct kbase_hwcnt_backend_csf_if_enable { -+ u32 fe_bm; -+ u32 shader_bm; -+ u32 tiler_bm; -+ u32 mmu_l2_bm; -+ u8 counter_set; -+ u64 clk_enable_map; -+}; -+ -+/** -+ * struct kbase_hwcnt_backend_csf_if_prfcnt_info - Performance counter -+ * information. -+ * @dump_bytes: Bytes of GPU memory required to perform a performance -+ * counter dump. -+ * @prfcnt_block_size Bytes of each performance counter block. -+ * @l2_count: The MMU L2 cache count. -+ * @core_mask: Shader core mask. -+ * @clk_cnt: Clock domain count in the system. -+ * @clearing_samples: Indicates whether counters are cleared after each sample -+ * is taken. -+ */ -+struct kbase_hwcnt_backend_csf_if_prfcnt_info { -+ size_t dump_bytes; -+ size_t prfcnt_block_size; -+ size_t l2_count; -+ u64 core_mask; -+ u8 clk_cnt; -+ bool clearing_samples; -+}; -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_assert_lock_held_fn - Assert that the -+ * backend spinlock is -+ * held. -+ * @ctx: Non-NULL pointer to a CSF context. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_assert_lock_held_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_lock_fn - Acquire backend spinlock. -+ * -+ * @ctx: Non-NULL pointer to a CSF context. -+ * @flags: Pointer to the memory location that would store the previous -+ * interrupt state. -+ */ -+typedef void -+kbase_hwcnt_backend_csf_if_lock_fn(struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ unsigned long *flags); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_unlock_fn - Release backend spinlock. -+ * -+ * @ctx: Non-NULL pointer to a CSF context. -+ * @flags: Previously stored interrupt state when Scheduler interrupt -+ * spinlock was acquired. -+ */ -+typedef void -+kbase_hwcnt_backend_csf_if_unlock_fn(struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ unsigned long flags); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_get_prfcnt_info_fn - Get performance -+ * counter information. -+ * @ctx: Non-NULL pointer to a CSF context. -+ * @prfcnt_info: Non-NULL pointer to struct where performance counter -+ * information should be stored. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_get_prfcnt_info_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ struct kbase_hwcnt_backend_csf_if_prfcnt_info *prfcnt_info); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_ring_buf_alloc_fn - Allocate a ring buffer -+ * for CSF interface. -+ * @ctx: Non-NULL pointer to a CSF context. -+ * @buf_count: The buffer count in the ring buffer to be allocated, -+ * MUST be power of 2. -+ * @cpu_dump_base: Non-NULL pointer to where ring buffer CPU base address is -+ * stored when success. -+ * @ring_buf: Non-NULL pointer to where ring buffer is stored when success. -+ * -+ * A ring buffer is needed by the CSF interface to do manual HWC sample and -+ * automatic HWC samples, the buffer count in the ring buffer MUST be power -+ * of 2 to meet the hardware requirement. -+ * -+ * Return: 0 on success, else error code. -+ */ -+typedef int kbase_hwcnt_backend_csf_if_ring_buf_alloc_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 buf_count, -+ void **cpu_dump_base, -+ struct kbase_hwcnt_backend_csf_if_ring_buf **ring_buf); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_ring_buf_sync_fn - Sync HWC dump buffers -+ * memory. -+ * @ctx: Non-NULL pointer to a CSF context. -+ * @ring_buf: Non-NULL pointer to the ring buffer. -+ * @buf_index_first: The first buffer index in the ring buffer to be synced, -+ * inclusive. -+ * @buf_index_last: The last buffer index in the ring buffer to be synced, -+ * exclusive. -+ * @for_cpu: The direction of sync to be applied, set to true when CPU -+ * cache needs invalidating before reading the buffer, and set -+ * to false after CPU writes to flush these before this memory -+ * is overwritten by the GPU. -+ * -+ * Flush cached HWC dump buffer data to ensure that all writes from GPU and CPU -+ * are correctly observed. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_ring_buf_sync_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, -+ u32 buf_index_first, u32 buf_index_last, bool for_cpu); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_ring_buf_free_fn - Free a ring buffer for -+ * the CSF interface. -+ * -+ * @ctx: Non-NULL pointer to a CSF interface context. -+ * @ring_buf: Non-NULL pointer to the ring buffer which to be freed. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_ring_buf_free_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_timestamp_ns_fn - Get the current -+ * timestamp of the CSF -+ * interface. -+ * @ctx: Non-NULL pointer to a CSF interface context. -+ * -+ * Return: CSF interface timestamp in nanoseconds. -+ */ -+typedef u64 kbase_hwcnt_backend_csf_if_timestamp_ns_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_dump_enable_fn - Setup and enable hardware -+ * counter in CSF interface. -+ * @ctx: Non-NULL pointer to a CSF interface context. -+ * @ring_buf: Non-NULL pointer to the ring buffer which used to setup the HWC. -+ * @enable: Non-NULL pointer to the enable map of HWC. -+ * -+ * Requires lock to be taken before calling. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_dump_enable_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, -+ struct kbase_hwcnt_backend_csf_if_enable *enable); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_dump_disable_fn - Disable hardware counter -+ * in CSF interface. -+ * @ctx: Non-NULL pointer to a CSF interface context. -+ * -+ * Requires lock to be taken before calling. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_dump_disable_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_dump_request_fn - Request a HWC dump. -+ * -+ * @ctx: Non-NULL pointer to the interface context. -+ * -+ * Requires lock to be taken before calling. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_dump_request_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_get_indexes_fn - Get current extract and -+ * insert indexes of the -+ * ring buffer. -+ * -+ * @ctx: Non-NULL pointer to a CSF interface context. -+ * @extract_index: Non-NULL pointer where current extract index to be saved. -+ * @insert_index: Non-NULL pointer where current insert index to be saved. -+ * -+ * Requires lock to be taken before calling. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_get_indexes_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 *extract_index, -+ u32 *insert_index); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_set_extract_index_fn - Update the extract -+ * index of the ring -+ * buffer. -+ * -+ * @ctx: Non-NULL pointer to a CSF interface context. -+ * @extract_index: New extract index to be set. -+ * -+ * Requires lock to be taken before calling. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_set_extract_index_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 extract_index); -+ -+/** -+ * typedef kbase_hwcnt_backend_csf_if_get_gpu_cycle_count_fn - Get the current -+ * GPU cycle count. -+ * @ctx: Non-NULL pointer to a CSF interface context. -+ * @cycle_counts: Non-NULL pointer to an array where cycle counts to be saved, -+ * the array size should be at least as big as the number of -+ * clock domains returned by get_prfcnt_info interface. -+ * @clk_enable_map: An array of bitfields, each bit specifies an enabled clock -+ * domain. -+ * -+ * Requires lock to be taken before calling. -+ */ -+typedef void kbase_hwcnt_backend_csf_if_get_gpu_cycle_count_fn( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, u64 *cycle_counts, -+ u64 clk_enable_map); -+ -+/** -+ * struct kbase_hwcnt_backend_csf_if - Hardware counter backend CSF virtual -+ * interface. -+ * @ctx: CSF interface context. -+ * @assert_lock_held: Function ptr to assert backend spinlock is held. -+ * @lock: Function ptr to acquire backend spinlock. -+ * @unlock: Function ptr to release backend spinlock. -+ * @get_prfcnt_info: Function ptr to get performance counter related -+ * information. -+ * @ring_buf_alloc: Function ptr to allocate ring buffer for CSF HWC. -+ * @ring_buf_sync: Function ptr to sync ring buffer to CPU. -+ * @ring_buf_free: Function ptr to free ring buffer for CSF HWC. -+ * @timestamp_ns: Function ptr to get the current CSF interface -+ * timestamp. -+ * @dump_enable: Function ptr to enable dumping. -+ * @dump_enable_nolock: Function ptr to enable dumping while the -+ * backend-specific spinlock is already held. -+ * @dump_disable: Function ptr to disable dumping. -+ * @dump_request: Function ptr to request a dump. -+ * @get_indexes: Function ptr to get extract and insert indexes of the -+ * ring buffer. -+ * @set_extract_index: Function ptr to set extract index of ring buffer. -+ * @get_gpu_cycle_count: Function ptr to get the GPU cycle count. -+ */ -+struct kbase_hwcnt_backend_csf_if { -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx; -+ kbase_hwcnt_backend_csf_if_assert_lock_held_fn *assert_lock_held; -+ kbase_hwcnt_backend_csf_if_lock_fn *lock; -+ kbase_hwcnt_backend_csf_if_unlock_fn *unlock; -+ kbase_hwcnt_backend_csf_if_get_prfcnt_info_fn *get_prfcnt_info; -+ kbase_hwcnt_backend_csf_if_ring_buf_alloc_fn *ring_buf_alloc; -+ kbase_hwcnt_backend_csf_if_ring_buf_sync_fn *ring_buf_sync; -+ kbase_hwcnt_backend_csf_if_ring_buf_free_fn *ring_buf_free; -+ kbase_hwcnt_backend_csf_if_timestamp_ns_fn *timestamp_ns; -+ kbase_hwcnt_backend_csf_if_dump_enable_fn *dump_enable; -+ kbase_hwcnt_backend_csf_if_dump_disable_fn *dump_disable; -+ kbase_hwcnt_backend_csf_if_dump_request_fn *dump_request; -+ kbase_hwcnt_backend_csf_if_get_indexes_fn *get_indexes; -+ kbase_hwcnt_backend_csf_if_set_extract_index_fn *set_extract_index; -+ kbase_hwcnt_backend_csf_if_get_gpu_cycle_count_fn *get_gpu_cycle_count; -+}; -+ -+#endif /* #define _KBASE_HWCNT_BACKEND_CSF_IF_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if_fw.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if_fw.c -new file mode 100644 -index 0000000..979299f ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if_fw.c -@@ -0,0 +1,787 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * CSF GPU HWC backend firmware interface APIs. -+ */ -+ -+#include <mali_kbase.h> -+#include <gpu/mali_kbase_gpu_regmap.h> -+#include <device/mali_kbase_device.h> -+#include "mali_kbase_hwcnt_gpu.h" -+#include "mali_kbase_hwcnt_types.h" -+#include <uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h> -+ -+#include "csf/mali_kbase_csf_firmware.h" -+#include "mali_kbase_hwcnt_backend_csf_if_fw.h" -+#include "mali_kbase_hwaccess_time.h" -+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" -+ -+#include <linux/log2.h> -+#include "mali_kbase_ccswe.h" -+ -+ -+/** The number of nanoseconds in a second. */ -+#define NSECS_IN_SEC 1000000000ull /* ns */ -+ -+/* Ring buffer virtual address start at 4GB */ -+#define KBASE_HWC_CSF_RING_BUFFER_VA_START (1ull << 32) -+ -+/** -+ * struct kbase_hwcnt_backend_csf_if_fw_ring_buf - ring buffer for CSF interface -+ * used to save the manual and -+ * auto HWC samples from -+ * firmware. -+ * @gpu_dump_base: Starting GPU base address of the ring buffer. -+ * @cpu_dump_base: Starting CPU address for the mapping. -+ * @buf_count: Buffer count in the ring buffer, MUST be power of 2. -+ * @as_nr: Address space number for the memory mapping. -+ * @phys: Physical memory allocation used by the mapping. -+ * @num_pages: Size of the mapping, in memory pages. -+ */ -+struct kbase_hwcnt_backend_csf_if_fw_ring_buf { -+ u64 gpu_dump_base; -+ void *cpu_dump_base; -+ size_t buf_count; -+ u32 as_nr; -+ struct tagged_addr *phys; -+ size_t num_pages; -+}; -+ -+/** -+ * struct kbase_hwcnt_backend_csf_if_fw_ctx - Firmware context for the CSF -+ * interface, used to communicate -+ * with firmware. -+ * @kbdev: KBase device. -+ * @buf_bytes: The size in bytes for each buffer in the ring buffer. -+ * @clk_cnt: The number of clock domains in the system. -+ * The maximum is 64. -+ * @clk_enable_map: Bitmask of enabled clocks -+ * @rate_listener: Clock rate listener callback state. -+ * @ccswe_shader_cores: Shader cores cycle count software estimator. -+ */ -+struct kbase_hwcnt_backend_csf_if_fw_ctx { -+ struct kbase_device *kbdev; -+ size_t buf_bytes; -+ u8 clk_cnt; -+ u64 clk_enable_map; -+ struct kbase_clk_rate_listener rate_listener; -+ struct kbase_ccswe ccswe_shader_cores; -+}; -+ -+static void kbasep_hwcnt_backend_csf_if_fw_assert_lock_held( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx; -+ struct kbase_device *kbdev; -+ -+ WARN_ON(!ctx); -+ -+ fw_ctx = (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ kbdev = fw_ctx->kbdev; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+} -+ -+static void -+kbasep_hwcnt_backend_csf_if_fw_lock(struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ unsigned long *flags) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx; -+ struct kbase_device *kbdev; -+ -+ WARN_ON(!ctx); -+ -+ fw_ctx = (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ kbdev = fw_ctx->kbdev; -+ -+ kbase_csf_scheduler_spin_lock(kbdev, flags); -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_unlock( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, unsigned long flags) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx; -+ struct kbase_device *kbdev; -+ -+ WARN_ON(!ctx); -+ -+ fw_ctx = (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ kbdev = fw_ctx->kbdev; -+ -+ kbase_csf_scheduler_spin_lock_assert_held(kbdev); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_if_fw_on_freq_change() - On freq change callback -+ * -+ * @rate_listener: Callback state -+ * @clk_index: Clock index -+ * @clk_rate_hz: Clock frequency(hz) -+ */ -+static void kbasep_hwcnt_backend_csf_if_fw_on_freq_change( -+ struct kbase_clk_rate_listener *rate_listener, u32 clk_index, -+ u32 clk_rate_hz) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ container_of(rate_listener, -+ struct kbase_hwcnt_backend_csf_if_fw_ctx, -+ rate_listener); -+ u64 timestamp_ns; -+ -+ if (clk_index != KBASE_CLOCK_DOMAIN_SHADER_CORES) -+ return; -+ -+ timestamp_ns = ktime_get_raw_ns(); -+ kbase_ccswe_freq_change(&fw_ctx->ccswe_shader_cores, timestamp_ns, -+ clk_rate_hz); -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_if_fw_cc_enable() - Enable cycle count tracking -+ * -+ * @fw_ctx: Non-NULL pointer to CSF firmware interface context. -+ * @clk_enable_map: Non-NULL pointer to enable map specifying enabled counters. -+ */ -+static void kbasep_hwcnt_backend_csf_if_fw_cc_enable( -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx, u64 clk_enable_map) -+{ -+ struct kbase_device *kbdev = fw_ctx->kbdev; -+ -+ if (kbase_hwcnt_clk_enable_map_enabled( -+ clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { -+ /* software estimation for non-top clock domains */ -+ struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; -+ const struct kbase_clk_data *clk_data = -+ rtm->clks[KBASE_CLOCK_DOMAIN_SHADER_CORES]; -+ u32 cur_freq; -+ unsigned long flags; -+ u64 timestamp_ns; -+ -+ timestamp_ns = ktime_get_raw_ns(); -+ -+ spin_lock_irqsave(&rtm->lock, flags); -+ -+ cur_freq = (u32)clk_data->clock_val; -+ kbase_ccswe_reset(&fw_ctx->ccswe_shader_cores); -+ kbase_ccswe_freq_change(&fw_ctx->ccswe_shader_cores, -+ timestamp_ns, cur_freq); -+ -+ kbase_clk_rate_trace_manager_subscribe_no_lock( -+ rtm, &fw_ctx->rate_listener); -+ -+ spin_unlock_irqrestore(&rtm->lock, flags); -+ } -+ -+ fw_ctx->clk_enable_map = clk_enable_map; -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_if_fw_cc_disable() - Disable cycle count tracking -+ * -+ * @fw_ctx: Non-NULL pointer to CSF firmware interface context. -+ */ -+static void kbasep_hwcnt_backend_csf_if_fw_cc_disable( -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx) -+{ -+ struct kbase_device *kbdev = fw_ctx->kbdev; -+ struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; -+ u64 clk_enable_map = fw_ctx->clk_enable_map; -+ -+ if (kbase_hwcnt_clk_enable_map_enabled(clk_enable_map, -+ KBASE_CLOCK_DOMAIN_SHADER_CORES)) -+ kbase_clk_rate_trace_manager_unsubscribe( -+ rtm, &fw_ctx->rate_listener); -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ struct kbase_hwcnt_backend_csf_if_prfcnt_info *prfcnt_info) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx; -+ struct kbase_device *kbdev; -+ u32 prfcnt_size; -+ u32 prfcnt_hw_size = 0; -+ u32 prfcnt_fw_size = 0; -+ u32 prfcnt_block_size = KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK * -+ KBASE_HWCNT_VALUE_BYTES; -+ -+ WARN_ON(!ctx); -+ WARN_ON(!prfcnt_info); -+ -+ fw_ctx = (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ kbdev = fw_ctx->kbdev; -+ prfcnt_size = kbdev->csf.global_iface.prfcnt_size; -+ prfcnt_hw_size = (prfcnt_size & 0xFF) << 8; -+ prfcnt_fw_size = (prfcnt_size >> 16) << 8; -+ fw_ctx->buf_bytes = prfcnt_hw_size + prfcnt_fw_size; -+ -+ -+ prfcnt_info->dump_bytes = fw_ctx->buf_bytes; -+ prfcnt_info->prfcnt_block_size = prfcnt_block_size; -+ prfcnt_info->l2_count = kbdev->gpu_props.props.l2_props.num_l2_slices; -+ prfcnt_info->core_mask = -+ kbdev->gpu_props.props.coherency_info.group[0].core_mask; -+ -+ prfcnt_info->clk_cnt = fw_ctx->clk_cnt; -+ prfcnt_info->clearing_samples = true; -+ -+ /* Block size must be multiple of counter size. */ -+ WARN_ON((prfcnt_info->prfcnt_block_size % KBASE_HWCNT_VALUE_BYTES) != -+ 0); -+ /* Total size must be multiple of block size. */ -+ WARN_ON((prfcnt_info->dump_bytes % prfcnt_info->prfcnt_block_size) != -+ 0); -+} -+ -+static int kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 buf_count, -+ void **cpu_dump_base, -+ struct kbase_hwcnt_backend_csf_if_ring_buf **out_ring_buf) -+{ -+ struct kbase_device *kbdev; -+ struct tagged_addr *phys; -+ struct page **page_list; -+ void *cpu_addr; -+ int ret; -+ int i; -+ size_t num_pages; -+ u64 flags; -+ struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf; -+ -+ pgprot_t cpu_map_prot = PAGE_KERNEL; -+ u64 gpu_va_base = KBASE_HWC_CSF_RING_BUFFER_VA_START; -+ -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ -+ WARN_ON(!ctx); -+ WARN_ON(!cpu_dump_base); -+ WARN_ON(!out_ring_buf); -+ -+ kbdev = fw_ctx->kbdev; -+ -+ /* The buffer count must be power of 2 */ -+ if (!is_power_of_2(buf_count)) -+ return -EINVAL; -+ -+ /* alignment failure */ -+ if (gpu_va_base & (2048 - 1)) -+ return -EINVAL; -+ -+ fw_ring_buf = kzalloc(sizeof(*fw_ring_buf), GFP_KERNEL); -+ if (!fw_ring_buf) -+ return -ENOMEM; -+ -+ num_pages = PFN_UP(fw_ctx->buf_bytes * buf_count); -+ phys = kmalloc_array(num_pages, sizeof(*phys), GFP_KERNEL); -+ if (!phys) -+ goto phys_alloc_error; -+ -+ page_list = kmalloc_array(num_pages, sizeof(*page_list), GFP_KERNEL); -+ if (!page_list) -+ goto page_list_alloc_error; -+ -+ /* Get physical page for the buffer */ -+ ret = kbase_mem_pool_alloc_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, -+ phys, false); -+ if (ret != num_pages) -+ goto phys_mem_pool_alloc_error; -+ -+ /* Get the CPU virtual address */ -+ for (i = 0; i < num_pages; i++) -+ page_list[i] = as_page(phys[i]); -+ -+ cpu_addr = vmap(page_list, num_pages, VM_MAP, cpu_map_prot); -+ if (!cpu_addr) -+ goto vmap_error; -+ -+ flags = KBASE_REG_GPU_WR | KBASE_REG_GPU_NX | -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_NON_CACHEABLE); -+ -+ /* Update MMU table */ -+ ret = kbase_mmu_insert_pages(kbdev, &kbdev->csf.mcu_mmu, -+ gpu_va_base >> PAGE_SHIFT, phys, num_pages, -+ flags, MCU_AS_NR, KBASE_MEM_GROUP_CSF_FW); -+ if (ret) -+ goto mmu_insert_failed; -+ -+ kfree(page_list); -+ -+ fw_ring_buf->gpu_dump_base = gpu_va_base; -+ fw_ring_buf->cpu_dump_base = cpu_addr; -+ fw_ring_buf->phys = phys; -+ fw_ring_buf->num_pages = num_pages; -+ fw_ring_buf->buf_count = buf_count; -+ fw_ring_buf->as_nr = MCU_AS_NR; -+ -+ *cpu_dump_base = fw_ring_buf->cpu_dump_base; -+ *out_ring_buf = -+ (struct kbase_hwcnt_backend_csf_if_ring_buf *)fw_ring_buf; -+ -+ -+ return 0; -+ -+mmu_insert_failed: -+ vunmap(cpu_addr); -+vmap_error: -+ kbase_mem_pool_free_pages( -+ &kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], num_pages, -+ phys, false, false); -+phys_mem_pool_alloc_error: -+ kfree(page_list); -+page_list_alloc_error: -+ kfree(phys); -+phys_alloc_error: -+ kfree(fw_ring_buf); -+ return -ENOMEM; -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_ring_buf_sync( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, -+ u32 buf_index_first, u32 buf_index_last, bool for_cpu) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf = -+ (struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf; -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ size_t i; -+ size_t pg_first; -+ size_t pg_last; -+ u64 start_address; -+ u64 stop_address; -+ u32 ring_buf_index_first; -+ u32 ring_buf_index_last; -+ -+ WARN_ON(!ctx); -+ WARN_ON(!ring_buf); -+ -+ /* The index arguments for this function form an inclusive, exclusive -+ * range. -+ * However, when masking back to the available buffers we will make this -+ * inclusive at both ends so full flushes are not 0 -> 0. -+ */ -+ ring_buf_index_first = buf_index_first & (fw_ring_buf->buf_count - 1); -+ ring_buf_index_last = -+ (buf_index_last - 1) & (fw_ring_buf->buf_count - 1); -+ -+ /* The start address is the offset of the first buffer. */ -+ start_address = fw_ctx->buf_bytes * ring_buf_index_first; -+ pg_first = start_address >> PAGE_SHIFT; -+ -+ /* The stop address is the last byte in the final buffer. */ -+ stop_address = (fw_ctx->buf_bytes * (ring_buf_index_last + 1)) - 1; -+ pg_last = stop_address >> PAGE_SHIFT; -+ -+ /* Check whether the buffer range wraps. */ -+ if (start_address > stop_address) { -+ /* sync the first part to the end of ring buffer. */ -+ for (i = pg_first; i < fw_ring_buf->num_pages; i++) { -+ struct page *pg = as_page(fw_ring_buf->phys[i]); -+ -+ if (for_cpu) { -+ kbase_sync_single_for_cpu(fw_ctx->kbdev, -+ kbase_dma_addr(pg), -+ PAGE_SIZE, -+ DMA_BIDIRECTIONAL); -+ } else { -+ kbase_sync_single_for_device(fw_ctx->kbdev, -+ kbase_dma_addr(pg), -+ PAGE_SIZE, -+ DMA_BIDIRECTIONAL); -+ } -+ } -+ -+ /* second part starts from page 0. */ -+ pg_first = 0; -+ } -+ -+ for (i = pg_first; i <= pg_last; i++) { -+ struct page *pg = as_page(fw_ring_buf->phys[i]); -+ -+ if (for_cpu) { -+ kbase_sync_single_for_cpu(fw_ctx->kbdev, -+ kbase_dma_addr(pg), PAGE_SIZE, -+ DMA_BIDIRECTIONAL); -+ } else { -+ kbase_sync_single_for_device(fw_ctx->kbdev, -+ kbase_dma_addr(pg), -+ PAGE_SIZE, -+ DMA_BIDIRECTIONAL); -+ } -+ } -+} -+ -+static u64 kbasep_hwcnt_backend_csf_if_fw_timestamp_ns( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx) -+{ -+ CSTD_UNUSED(ctx); -+ return ktime_get_raw_ns(); -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_ring_buf_free( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf = -+ (struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf; -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ -+ if (!fw_ring_buf) -+ return; -+ -+ if (fw_ring_buf->phys) { -+ u64 gpu_va_base = KBASE_HWC_CSF_RING_BUFFER_VA_START; -+ -+ WARN_ON(kbase_mmu_teardown_pages( -+ fw_ctx->kbdev, &fw_ctx->kbdev->csf.mcu_mmu, -+ gpu_va_base >> PAGE_SHIFT, fw_ring_buf->num_pages, -+ MCU_AS_NR)); -+ -+ vunmap(fw_ring_buf->cpu_dump_base); -+ -+ kbase_mem_pool_free_pages( -+ &fw_ctx->kbdev->mem_pools.small[KBASE_MEM_GROUP_CSF_FW], -+ fw_ring_buf->num_pages, fw_ring_buf->phys, false, -+ false); -+ -+ kfree(fw_ring_buf->phys); -+ -+ kfree(fw_ring_buf); -+ } -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_dump_enable( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, -+ struct kbase_hwcnt_backend_csf_if_ring_buf *ring_buf, -+ struct kbase_hwcnt_backend_csf_if_enable *enable) -+{ -+ u32 prfcnt_config; -+ struct kbase_device *kbdev; -+ struct kbase_csf_global_iface *global_iface; -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ struct kbase_hwcnt_backend_csf_if_fw_ring_buf *fw_ring_buf = -+ (struct kbase_hwcnt_backend_csf_if_fw_ring_buf *)ring_buf; -+ -+ WARN_ON(!ctx); -+ WARN_ON(!ring_buf); -+ WARN_ON(!enable); -+ kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx); -+ -+ kbdev = fw_ctx->kbdev; -+ global_iface = &kbdev->csf.global_iface; -+ -+ /* Configure */ -+ prfcnt_config = fw_ring_buf->buf_count; -+ prfcnt_config |= enable->counter_set << PRFCNT_CONFIG_SETSELECT_SHIFT; -+ -+ /* Configure the ring buffer base address */ -+ kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_JASID, -+ fw_ring_buf->as_nr); -+ kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_BASE_LO, -+ fw_ring_buf->gpu_dump_base & U32_MAX); -+ kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_BASE_HI, -+ fw_ring_buf->gpu_dump_base >> 32); -+ -+ /* Set extract position to 0 */ -+ kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_EXTRACT, 0); -+ -+ /* Configure the enable bitmap */ -+ kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_CSF_EN, -+ enable->fe_bm); -+ kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_SHADER_EN, -+ enable->shader_bm); -+ kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_MMU_L2_EN, -+ enable->mmu_l2_bm); -+ kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_TILER_EN, -+ enable->tiler_bm); -+ -+ /* Configure the HWC set and buffer size */ -+ kbase_csf_firmware_global_input(global_iface, GLB_PRFCNT_CONFIG, -+ prfcnt_config); -+ -+ kbdev->csf.hwcnt.enable_pending = true; -+ -+ /* Unmask the interrupts */ -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_ACK_IRQ_MASK, -+ GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK, -+ GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK); -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_ACK_IRQ_MASK, -+ GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK, -+ GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK); -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_ACK_IRQ_MASK, -+ GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK, -+ GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK); -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_ACK_IRQ_MASK, -+ GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK, -+ GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK); -+ -+ /* Enable the HWC */ -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, -+ (1 << GLB_REQ_PRFCNT_ENABLE_SHIFT), -+ GLB_REQ_PRFCNT_ENABLE_MASK); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ -+ prfcnt_config = kbase_csf_firmware_global_input_read(global_iface, -+ GLB_PRFCNT_CONFIG); -+ -+ kbasep_hwcnt_backend_csf_if_fw_cc_enable(fw_ctx, -+ enable->clk_enable_map); -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_dump_disable( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx) -+{ -+ struct kbase_device *kbdev; -+ struct kbase_csf_global_iface *global_iface; -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ -+ WARN_ON(!ctx); -+ kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx); -+ -+ kbdev = fw_ctx->kbdev; -+ global_iface = &kbdev->csf.global_iface; -+ -+ /* Disable the HWC */ -+ kbdev->csf.hwcnt.enable_pending = true; -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, 0, -+ GLB_REQ_PRFCNT_ENABLE_MASK); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+ -+ /* mask the interrupts */ -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_ACK_IRQ_MASK, 0, -+ GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK); -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_ACK_IRQ_MASK, 0, -+ GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK); -+ kbase_csf_firmware_global_input_mask( -+ global_iface, GLB_ACK_IRQ_MASK, 0, -+ GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK); -+ -+ /* In case we have a previous request in flight when the disable -+ * happens. -+ */ -+ kbdev->csf.hwcnt.request_pending = false; -+ -+ kbasep_hwcnt_backend_csf_if_fw_cc_disable(fw_ctx); -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_dump_request( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx) -+{ -+ u32 glb_req; -+ struct kbase_device *kbdev; -+ struct kbase_csf_global_iface *global_iface; -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ -+ WARN_ON(!ctx); -+ kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx); -+ -+ kbdev = fw_ctx->kbdev; -+ global_iface = &kbdev->csf.global_iface; -+ -+ /* Trigger dumping */ -+ kbdev->csf.hwcnt.request_pending = true; -+ glb_req = kbase_csf_firmware_global_input_read(global_iface, GLB_REQ); -+ glb_req ^= GLB_REQ_PRFCNT_SAMPLE_MASK; -+ kbase_csf_firmware_global_input_mask(global_iface, GLB_REQ, glb_req, -+ GLB_REQ_PRFCNT_SAMPLE_MASK); -+ kbase_csf_ring_doorbell(kbdev, CSF_KERNEL_DOORBELL_NR); -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_get_indexes( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 *extract_index, -+ u32 *insert_index) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ -+ WARN_ON(!ctx); -+ WARN_ON(!extract_index); -+ WARN_ON(!insert_index); -+ kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx); -+ -+ *extract_index = kbase_csf_firmware_global_input_read( -+ &fw_ctx->kbdev->csf.global_iface, GLB_PRFCNT_EXTRACT); -+ *insert_index = kbase_csf_firmware_global_output( -+ &fw_ctx->kbdev->csf.global_iface, GLB_PRFCNT_INSERT); -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_set_extract_index( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, u32 extract_idx) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ -+ WARN_ON(!ctx); -+ kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx); -+ -+ /* Set the raw extract index to release the buffer back to the ring -+ * buffer. -+ */ -+ kbase_csf_firmware_global_input(&fw_ctx->kbdev->csf.global_iface, -+ GLB_PRFCNT_EXTRACT, extract_idx); -+} -+ -+static void kbasep_hwcnt_backend_csf_if_fw_get_gpu_cycle_count( -+ struct kbase_hwcnt_backend_csf_if_ctx *ctx, u64 *cycle_counts, -+ u64 clk_enable_map) -+{ -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx = -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)ctx; -+ u8 clk; -+ u64 timestamp_ns = ktime_get_raw_ns(); -+ -+ WARN_ON(!ctx); -+ WARN_ON(!cycle_counts); -+ kbasep_hwcnt_backend_csf_if_fw_assert_lock_held(ctx); -+ -+ for (clk = 0; clk < fw_ctx->clk_cnt; clk++) { -+ if (!(clk_enable_map & (1ull << clk))) -+ continue; -+ -+ if (clk == KBASE_CLOCK_DOMAIN_TOP) { -+ /* Read cycle count for top clock domain. */ -+ kbase_backend_get_gpu_time_norequest( -+ fw_ctx->kbdev, &cycle_counts[clk], NULL, NULL); -+ } else { -+ /* Estimate cycle count for non-top clock domain. */ -+ cycle_counts[clk] = kbase_ccswe_cycle_at( -+ &fw_ctx->ccswe_shader_cores, timestamp_ns); -+ } -+ } -+} -+ -+/** -+ * kbasep_hwcnt_backedn_csf_if_fw_cts_destroy() - Destroy a CSF FW interface context. -+ * -+ * @fw_ctx: Pointer to context to destroy. -+ */ -+static void kbasep_hwcnt_backend_csf_if_fw_ctx_destroy( -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *fw_ctx) -+{ -+ if (!fw_ctx) -+ return; -+ -+ kfree(fw_ctx); -+} -+ -+/** -+ * kbasep_hwcnt_backend_csf_if_fw_ctx_create() - Create a CSF Firmware context. -+ * -+ * @kbdev: Non_NULL pointer to kbase device. -+ * @out_ctx: Non-NULL pointer to where info is stored on success. -+ * Return: 0 on success, else error code. -+ */ -+static int kbasep_hwcnt_backend_csf_if_fw_ctx_create( -+ struct kbase_device *kbdev, -+ struct kbase_hwcnt_backend_csf_if_fw_ctx **out_ctx) -+{ -+ u8 clk; -+ int errcode = -ENOMEM; -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *ctx = NULL; -+ -+ WARN_ON(!kbdev); -+ WARN_ON(!out_ctx); -+ -+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); -+ if (!ctx) -+ goto error; -+ -+ ctx->kbdev = kbdev; -+ -+ /* Determine the number of available clock domains. */ -+ for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) { -+ if (kbdev->pm.clk_rtm.clks[clk] == NULL) -+ break; -+ } -+ ctx->clk_cnt = clk; -+ -+ ctx->clk_enable_map = 0; -+ kbase_ccswe_init(&ctx->ccswe_shader_cores); -+ ctx->rate_listener.notify = -+ kbasep_hwcnt_backend_csf_if_fw_on_freq_change; -+ -+ *out_ctx = ctx; -+ -+ return 0; -+error: -+ kbasep_hwcnt_backend_csf_if_fw_ctx_destroy(ctx); -+ return errcode; -+} -+ -+void kbase_hwcnt_backend_csf_if_fw_destroy( -+ struct kbase_hwcnt_backend_csf_if *if_fw) -+{ -+ if (!if_fw) -+ return; -+ -+ kbasep_hwcnt_backend_csf_if_fw_ctx_destroy( -+ (struct kbase_hwcnt_backend_csf_if_fw_ctx *)if_fw->ctx); -+ memset(if_fw, 0, sizeof(*if_fw)); -+} -+ -+int kbase_hwcnt_backend_csf_if_fw_create( -+ struct kbase_device *kbdev, struct kbase_hwcnt_backend_csf_if *if_fw) -+{ -+ int errcode; -+ struct kbase_hwcnt_backend_csf_if_fw_ctx *ctx = NULL; -+ -+ if (!kbdev || !if_fw) -+ return -EINVAL; -+ -+ errcode = kbasep_hwcnt_backend_csf_if_fw_ctx_create(kbdev, &ctx); -+ if (errcode) -+ return errcode; -+ -+ if_fw->ctx = (struct kbase_hwcnt_backend_csf_if_ctx *)ctx; -+ if_fw->assert_lock_held = -+ kbasep_hwcnt_backend_csf_if_fw_assert_lock_held; -+ if_fw->lock = kbasep_hwcnt_backend_csf_if_fw_lock; -+ if_fw->unlock = kbasep_hwcnt_backend_csf_if_fw_unlock; -+ if_fw->get_prfcnt_info = kbasep_hwcnt_backend_csf_if_fw_get_prfcnt_info; -+ if_fw->ring_buf_alloc = kbasep_hwcnt_backend_csf_if_fw_ring_buf_alloc; -+ if_fw->ring_buf_sync = kbasep_hwcnt_backend_csf_if_fw_ring_buf_sync; -+ if_fw->ring_buf_free = kbasep_hwcnt_backend_csf_if_fw_ring_buf_free; -+ if_fw->timestamp_ns = kbasep_hwcnt_backend_csf_if_fw_timestamp_ns; -+ if_fw->dump_enable = kbasep_hwcnt_backend_csf_if_fw_dump_enable; -+ if_fw->dump_disable = kbasep_hwcnt_backend_csf_if_fw_dump_disable; -+ if_fw->dump_request = kbasep_hwcnt_backend_csf_if_fw_dump_request; -+ if_fw->get_gpu_cycle_count = -+ kbasep_hwcnt_backend_csf_if_fw_get_gpu_cycle_count; -+ if_fw->get_indexes = kbasep_hwcnt_backend_csf_if_fw_get_indexes; -+ if_fw->set_extract_index = -+ kbasep_hwcnt_backend_csf_if_fw_set_extract_index; -+ -+ return 0; -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if_fw.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if_fw.h -new file mode 100644 -index 0000000..b69668b ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_csf_if_fw.h -@@ -0,0 +1,50 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * Concrete implementation of kbase_hwcnt_backend_csf_if interface for CSF FW -+ */ -+ -+#ifndef _KBASE_HWCNT_BACKEND_CSF_IF_FW_H_ -+#define _KBASE_HWCNT_BACKEND_CSF_IF_FW_H_ -+ -+#include "mali_kbase_hwcnt_backend_csf_if.h" -+ -+/** -+ * kbase_hwcnt_backend_csf_if_fw_create() - Create a firmware CSF interface -+ * of hardware counter backend. -+ * @kbdev: Non-NULL pointer to Kbase device. -+ * @if_fw: Non-NULL pointer to backend interface structure that is filled in on -+ * creation success. -+ * Return: 0 on success, else error code. -+ */ -+int kbase_hwcnt_backend_csf_if_fw_create( -+ struct kbase_device *kbdev, struct kbase_hwcnt_backend_csf_if *if_fw); -+ -+/** -+ * kbase_hwcnt_backend_csf_if_fw_destroy() - Destroy a firmware CSF interface of -+ * hardware counter backend. -+ * @if_fw: Pointer to a CSF interface to destroy. -+ */ -+void kbase_hwcnt_backend_csf_if_fw_destroy( -+ struct kbase_hwcnt_backend_csf_if *if_fw); -+ -+#endif /* _KBASE_HWCNT_BACKEND_CSF_IF_FW_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_gpu.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_gpu.c -deleted file mode 100644 -index 407c768..0000000 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_gpu.c -+++ /dev/null -@@ -1,510 +0,0 @@ --/* -- * -- * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- */ -- --#include "mali_kbase_hwcnt_backend_gpu.h" --#include "mali_kbase_hwcnt_gpu.h" --#include "mali_kbase_hwcnt_types.h" --#include "mali_kbase.h" --#include "mali_kbase_pm_ca.h" --#include "mali_kbase_hwaccess_instr.h" --#ifdef CONFIG_MALI_NO_MALI --#include "backend/gpu/mali_kbase_model_dummy.h" --#endif -- -- --/** -- * struct kbase_hwcnt_backend_gpu_info - Information used to create an instance -- * of a GPU hardware counter backend. -- * @kbdev: KBase device. -- * @use_secondary: True if secondary performance counters should be used, -- * else false. Ignored if secondary counters are not supported. -- * @metadata: Hardware counter metadata. -- * @dump_bytes: Bytes of GPU memory required to perform a -- * hardware counter dump. -- */ --struct kbase_hwcnt_backend_gpu_info { -- struct kbase_device *kbdev; -- bool use_secondary; -- const struct kbase_hwcnt_metadata *metadata; -- size_t dump_bytes; --}; -- --/** -- * struct kbase_hwcnt_backend_gpu - Instance of a GPU hardware counter backend. -- * @info: Info used to create the backend. -- * @kctx: KBase context used for GPU memory allocation and -- * counter dumping. -- * @gpu_dump_va: GPU hardware counter dump buffer virtual address. -- * @cpu_dump_va: CPU mapping of gpu_dump_va. -- * @vmap: Dump buffer vmap. -- * @enabled: True if dumping has been enabled, else false. -- * @pm_core_mask: PM state sync-ed shaders core mask for the enabled dumping. -- */ --struct kbase_hwcnt_backend_gpu { -- const struct kbase_hwcnt_backend_gpu_info *info; -- struct kbase_context *kctx; -- u64 gpu_dump_va; -- void *cpu_dump_va; -- struct kbase_vmap_struct *vmap; -- bool enabled; -- u64 pm_core_mask; --}; -- --/* GPU backend implementation of kbase_hwcnt_backend_timestamp_ns_fn */ --static u64 kbasep_hwcnt_backend_gpu_timestamp_ns( -- struct kbase_hwcnt_backend *backend) --{ -- (void)backend; -- return ktime_get_raw_ns(); --} -- --/* GPU backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ --static int kbasep_hwcnt_backend_gpu_dump_enable_nolock( -- struct kbase_hwcnt_backend *backend, -- const struct kbase_hwcnt_enable_map *enable_map) --{ -- int errcode; -- struct kbase_hwcnt_backend_gpu *backend_gpu = -- (struct kbase_hwcnt_backend_gpu *)backend; -- struct kbase_context *kctx; -- struct kbase_device *kbdev; -- struct kbase_hwcnt_physical_enable_map phys; -- struct kbase_instr_hwcnt_enable enable; -- -- if (!backend_gpu || !enable_map || backend_gpu->enabled || -- (enable_map->metadata != backend_gpu->info->metadata)) -- return -EINVAL; -- -- kctx = backend_gpu->kctx; -- kbdev = backend_gpu->kctx->kbdev; -- -- lockdep_assert_held(&kbdev->hwaccess_lock); -- -- kbase_hwcnt_gpu_enable_map_to_physical(&phys, enable_map); -- -- enable.jm_bm = phys.jm_bm; -- enable.shader_bm = phys.shader_bm; -- enable.tiler_bm = phys.tiler_bm; -- enable.mmu_l2_bm = phys.mmu_l2_bm; -- enable.use_secondary = backend_gpu->info->use_secondary; -- enable.dump_buffer = backend_gpu->gpu_dump_va; -- enable.dump_buffer_bytes = backend_gpu->info->dump_bytes; -- -- errcode = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &enable); -- if (errcode) -- goto error; -- -- backend_gpu->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev); -- backend_gpu->enabled = true; -- -- return 0; --error: -- return errcode; --} -- --/* GPU backend implementation of kbase_hwcnt_backend_dump_enable_fn */ --static int kbasep_hwcnt_backend_gpu_dump_enable( -- struct kbase_hwcnt_backend *backend, -- const struct kbase_hwcnt_enable_map *enable_map) --{ -- unsigned long flags; -- int errcode; -- struct kbase_hwcnt_backend_gpu *backend_gpu = -- (struct kbase_hwcnt_backend_gpu *)backend; -- struct kbase_device *kbdev; -- -- if (!backend_gpu) -- return -EINVAL; -- -- kbdev = backend_gpu->kctx->kbdev; -- -- spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- -- errcode = kbasep_hwcnt_backend_gpu_dump_enable_nolock( -- backend, enable_map); -- -- spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -- -- return errcode; --} -- --/* GPU backend implementation of kbase_hwcnt_backend_dump_disable_fn */ --static void kbasep_hwcnt_backend_gpu_dump_disable( -- struct kbase_hwcnt_backend *backend) --{ -- int errcode; -- struct kbase_hwcnt_backend_gpu *backend_gpu = -- (struct kbase_hwcnt_backend_gpu *)backend; -- -- if (WARN_ON(!backend_gpu) || !backend_gpu->enabled) -- return; -- -- errcode = kbase_instr_hwcnt_disable_internal(backend_gpu->kctx); -- WARN_ON(errcode); -- -- backend_gpu->enabled = false; --} -- --/* GPU backend implementation of kbase_hwcnt_backend_dump_clear_fn */ --static int kbasep_hwcnt_backend_gpu_dump_clear( -- struct kbase_hwcnt_backend *backend) --{ -- struct kbase_hwcnt_backend_gpu *backend_gpu = -- (struct kbase_hwcnt_backend_gpu *)backend; -- -- if (!backend_gpu || !backend_gpu->enabled) -- return -EINVAL; -- -- return kbase_instr_hwcnt_clear(backend_gpu->kctx); --} -- --/* GPU backend implementation of kbase_hwcnt_backend_dump_request_fn */ --static int kbasep_hwcnt_backend_gpu_dump_request( -- struct kbase_hwcnt_backend *backend) --{ -- struct kbase_hwcnt_backend_gpu *backend_gpu = -- (struct kbase_hwcnt_backend_gpu *)backend; -- -- if (!backend_gpu || !backend_gpu->enabled) -- return -EINVAL; -- -- return kbase_instr_hwcnt_request_dump(backend_gpu->kctx); --} -- --/* GPU backend implementation of kbase_hwcnt_backend_dump_wait_fn */ --static int kbasep_hwcnt_backend_gpu_dump_wait( -- struct kbase_hwcnt_backend *backend) --{ -- struct kbase_hwcnt_backend_gpu *backend_gpu = -- (struct kbase_hwcnt_backend_gpu *)backend; -- -- if (!backend_gpu || !backend_gpu->enabled) -- return -EINVAL; -- -- return kbase_instr_hwcnt_wait_for_dump(backend_gpu->kctx); --} -- --/* GPU backend implementation of kbase_hwcnt_backend_dump_get_fn */ --static int kbasep_hwcnt_backend_gpu_dump_get( -- struct kbase_hwcnt_backend *backend, -- struct kbase_hwcnt_dump_buffer *dst, -- const struct kbase_hwcnt_enable_map *dst_enable_map, -- bool accumulate) --{ -- struct kbase_hwcnt_backend_gpu *backend_gpu = -- (struct kbase_hwcnt_backend_gpu *)backend; -- -- if (!backend_gpu || !dst || !dst_enable_map || -- (backend_gpu->info->metadata != dst->metadata) || -- (dst_enable_map->metadata != dst->metadata)) -- return -EINVAL; -- -- /* Invalidate the kernel buffer before reading from it. */ -- kbase_sync_mem_regions( -- backend_gpu->kctx, backend_gpu->vmap, KBASE_SYNC_TO_CPU); -- -- return kbase_hwcnt_gpu_dump_get( -- dst, backend_gpu->cpu_dump_va, dst_enable_map, -- backend_gpu->pm_core_mask, accumulate); --} -- --/** -- * kbasep_hwcnt_backend_gpu_dump_alloc() - Allocate a GPU dump buffer. -- * @info: Non-NULL pointer to GPU backend info. -- * @kctx: Non-NULL pointer to kbase context. -- * @gpu_dump_va: Non-NULL pointer to where GPU dump buffer virtual address -- * is stored on success. -- * -- * Return: 0 on success, else error code. -- */ --static int kbasep_hwcnt_backend_gpu_dump_alloc( -- const struct kbase_hwcnt_backend_gpu_info *info, -- struct kbase_context *kctx, -- u64 *gpu_dump_va) --{ -- struct kbase_va_region *reg; -- u64 flags; -- u64 nr_pages; -- -- WARN_ON(!info); -- WARN_ON(!kctx); -- WARN_ON(!gpu_dump_va); -- -- flags = BASE_MEM_PROT_CPU_RD | -- BASE_MEM_PROT_GPU_WR | -- BASEP_MEM_PERMANENT_KERNEL_MAPPING | -- BASE_MEM_CACHED_CPU; -- -- if (kctx->kbdev->mmu_mode->flags & KBASE_MMU_MODE_HAS_NON_CACHEABLE) -- flags |= BASE_MEM_UNCACHED_GPU; -- -- nr_pages = PFN_UP(info->dump_bytes); -- -- reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, gpu_dump_va); -- -- if (!reg) -- return -ENOMEM; -- -- return 0; --} -- --/** -- * kbasep_hwcnt_backend_gpu_dump_free() - Free an allocated GPU dump buffer. -- * @kctx: Non-NULL pointer to kbase context. -- * @gpu_dump_va: GPU dump buffer virtual address. -- */ --static void kbasep_hwcnt_backend_gpu_dump_free( -- struct kbase_context *kctx, -- u64 gpu_dump_va) --{ -- WARN_ON(!kctx); -- if (gpu_dump_va) -- kbase_mem_free(kctx, gpu_dump_va); --} -- --/** -- * kbasep_hwcnt_backend_gpu_destroy() - Destroy a GPU backend. -- * @backend: Pointer to GPU backend to destroy. -- * -- * Can be safely called on a backend in any state of partial construction. -- */ --static void kbasep_hwcnt_backend_gpu_destroy( -- struct kbase_hwcnt_backend_gpu *backend) --{ -- if (!backend) -- return; -- -- if (backend->kctx) { -- struct kbase_context *kctx = backend->kctx; -- struct kbase_device *kbdev = kctx->kbdev; -- -- if (backend->cpu_dump_va) -- kbase_phy_alloc_mapping_put(kctx, backend->vmap); -- -- if (backend->gpu_dump_va) -- kbasep_hwcnt_backend_gpu_dump_free( -- kctx, backend->gpu_dump_va); -- -- kbasep_js_release_privileged_ctx(kbdev, kctx); -- kbase_destroy_context(kctx); -- } -- -- kfree(backend); --} -- --/** -- * kbasep_hwcnt_backend_gpu_create() - Create a GPU backend. -- * @info: Non-NULL pointer to backend info. -- * @out_backend: Non-NULL pointer to where backend is stored on success. -- * -- * Return: 0 on success, else error code. -- */ --static int kbasep_hwcnt_backend_gpu_create( -- const struct kbase_hwcnt_backend_gpu_info *info, -- struct kbase_hwcnt_backend_gpu **out_backend) --{ -- -- int errcode; -- struct kbase_device *kbdev; -- struct kbase_hwcnt_backend_gpu *backend = NULL; -- -- WARN_ON(!info); -- WARN_ON(!out_backend); -- -- kbdev = info->kbdev; -- -- backend = kzalloc(sizeof(*backend), GFP_KERNEL); -- if (!backend) -- goto alloc_error; -- -- backend->info = info; -- -- backend->kctx = kbase_create_context(kbdev, true, -- BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED, 0, NULL); -- if (!backend->kctx) -- goto alloc_error; -- -- kbasep_js_schedule_privileged_ctx(kbdev, backend->kctx); -- -- errcode = kbasep_hwcnt_backend_gpu_dump_alloc( -- info, backend->kctx, &backend->gpu_dump_va); -- if (errcode) -- goto error; -- -- backend->cpu_dump_va = kbase_phy_alloc_mapping_get(backend->kctx, -- backend->gpu_dump_va, &backend->vmap); -- if (!backend->cpu_dump_va) -- goto alloc_error; -- --#ifdef CONFIG_MALI_NO_MALI -- /* The dummy model needs the CPU mapping. */ -- gpu_model_set_dummy_prfcnt_base_cpu(backend->cpu_dump_va); --#endif -- -- *out_backend = backend; -- return 0; -- --alloc_error: -- errcode = -ENOMEM; --error: -- kbasep_hwcnt_backend_gpu_destroy(backend); -- return errcode; --} -- --/* GPU backend implementation of kbase_hwcnt_backend_init_fn */ --static int kbasep_hwcnt_backend_gpu_init( -- const struct kbase_hwcnt_backend_info *info, -- struct kbase_hwcnt_backend **out_backend) --{ -- int errcode; -- struct kbase_hwcnt_backend_gpu *backend = NULL; -- -- if (!info || !out_backend) -- return -EINVAL; -- -- errcode = kbasep_hwcnt_backend_gpu_create( -- (const struct kbase_hwcnt_backend_gpu_info *) info, &backend); -- if (errcode) -- return errcode; -- -- *out_backend = (struct kbase_hwcnt_backend *)backend; -- -- return 0; --} -- --/* GPU backend implementation of kbase_hwcnt_backend_term_fn */ --static void kbasep_hwcnt_backend_gpu_term(struct kbase_hwcnt_backend *backend) --{ -- if (!backend) -- return; -- -- kbasep_hwcnt_backend_gpu_dump_disable(backend); -- kbasep_hwcnt_backend_gpu_destroy( -- (struct kbase_hwcnt_backend_gpu *)backend); --} -- --/** -- * kbasep_hwcnt_backend_gpu_info_destroy() - Destroy a GPU backend info. -- * @info: Pointer to info to destroy. -- * -- * Can be safely called on a backend info in any state of partial construction. -- */ --static void kbasep_hwcnt_backend_gpu_info_destroy( -- const struct kbase_hwcnt_backend_gpu_info *info) --{ -- if (!info) -- return; -- -- kbase_hwcnt_gpu_metadata_destroy(info->metadata); -- kfree(info); --} -- --/** -- * kbasep_hwcnt_backend_gpu_info_create() - Create a GPU backend info. -- * @kbdev: Non_NULL pointer to kbase device. -- * @out_info: Non-NULL pointer to where info is stored on success. -- * -- * Return 0 on success, else error code. -- */ --static int kbasep_hwcnt_backend_gpu_info_create( -- struct kbase_device *kbdev, -- const struct kbase_hwcnt_backend_gpu_info **out_info) --{ -- int errcode = -ENOMEM; -- struct kbase_hwcnt_gpu_info hwcnt_gpu_info; -- struct kbase_hwcnt_backend_gpu_info *info = NULL; -- -- WARN_ON(!kbdev); -- WARN_ON(!out_info); -- -- errcode = kbase_hwcnt_gpu_info_init(kbdev, &hwcnt_gpu_info); -- if (errcode) -- return errcode; -- -- info = kzalloc(sizeof(*info), GFP_KERNEL); -- if (!info) -- goto error; -- -- info->kbdev = kbdev; -- --#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY -- info->use_secondary = true; --#else -- info->use_secondary = false; --#endif -- -- errcode = kbase_hwcnt_gpu_metadata_create( -- &hwcnt_gpu_info, info->use_secondary, -- &info->metadata, -- &info->dump_bytes); -- if (errcode) -- goto error; -- -- *out_info = info; -- -- return 0; --error: -- kbasep_hwcnt_backend_gpu_info_destroy(info); -- return errcode; --} -- --int kbase_hwcnt_backend_gpu_create( -- struct kbase_device *kbdev, -- struct kbase_hwcnt_backend_interface *iface) --{ -- int errcode; -- const struct kbase_hwcnt_backend_gpu_info *info = NULL; -- -- if (!kbdev || !iface) -- return -EINVAL; -- -- errcode = kbasep_hwcnt_backend_gpu_info_create(kbdev, &info); -- -- if (errcode) -- return errcode; -- -- iface->metadata = info->metadata; -- iface->info = (struct kbase_hwcnt_backend_info *)info; -- iface->init = kbasep_hwcnt_backend_gpu_init; -- iface->term = kbasep_hwcnt_backend_gpu_term; -- iface->timestamp_ns = kbasep_hwcnt_backend_gpu_timestamp_ns; -- iface->dump_enable = kbasep_hwcnt_backend_gpu_dump_enable; -- iface->dump_enable_nolock = kbasep_hwcnt_backend_gpu_dump_enable_nolock; -- iface->dump_disable = kbasep_hwcnt_backend_gpu_dump_disable; -- iface->dump_clear = kbasep_hwcnt_backend_gpu_dump_clear; -- iface->dump_request = kbasep_hwcnt_backend_gpu_dump_request; -- iface->dump_wait = kbasep_hwcnt_backend_gpu_dump_wait; -- iface->dump_get = kbasep_hwcnt_backend_gpu_dump_get; -- -- return 0; --} -- --void kbase_hwcnt_backend_gpu_destroy( -- struct kbase_hwcnt_backend_interface *iface) --{ -- if (!iface) -- return; -- -- kbasep_hwcnt_backend_gpu_info_destroy( -- (const struct kbase_hwcnt_backend_gpu_info *)iface->info); -- memset(iface, 0, sizeof(*iface)); --} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_jm.c -new file mode 100644 -index 0000000..64001b1 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_jm.c -@@ -0,0 +1,793 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include "mali_kbase_hwcnt_backend_jm.h" -+#include "mali_kbase_hwcnt_gpu.h" -+#include "mali_kbase_hwcnt_types.h" -+#include "mali_kbase.h" -+#include "backend/gpu/mali_kbase_pm_ca.h" -+#include "mali_kbase_hwaccess_instr.h" -+#include "mali_kbase_hwaccess_time.h" -+#include "mali_kbase_ccswe.h" -+ -+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" -+ -+#include "backend/gpu/mali_kbase_pm_internal.h" -+ -+/** -+ * struct kbase_hwcnt_backend_jm_info - Information used to create an instance -+ * of a JM hardware counter backend. -+ * @kbdev: KBase device. -+ * @counter_set: The performance counter set to use. -+ * @metadata: Hardware counter metadata. -+ * @dump_bytes: Bytes of GPU memory required to perform a -+ * hardware counter dump. -+ */ -+struct kbase_hwcnt_backend_jm_info { -+ struct kbase_device *kbdev; -+ enum kbase_hwcnt_set counter_set; -+ const struct kbase_hwcnt_metadata *metadata; -+ size_t dump_bytes; -+}; -+ -+/** -+ * struct kbase_hwcnt_backend_jm - Instance of a JM hardware counter backend. -+ * @info: Info used to create the backend. -+ * @kctx: KBase context used for GPU memory allocation and -+ * counter dumping. -+ * @gpu_dump_va: GPU hardware counter dump buffer virtual address. -+ * @cpu_dump_va: CPU mapping of gpu_dump_va. -+ * @vmap: Dump buffer vmap. -+ * @enabled: True if dumping has been enabled, else false. -+ * @pm_core_mask: PM state sync-ed shaders core mask for the enabled -+ * dumping. -+ * @curr_config: Current allocated hardware resources to correctly map the src -+ * raw dump buffer to the dst dump buffer. -+ * @clk_enable_map: The enable map specifying enabled clock domains. -+ * @cycle_count_elapsed: -+ * Cycle count elapsed for a given sample period. -+ * The top clock cycle, index 0, is read directly from -+ * hardware, but the other clock domains need to be -+ * calculated with software estimation. -+ * @prev_cycle_count: Previous cycle count to calculate the cycle count for -+ * sample period. -+ * @rate_listener: Clock rate listener callback state. -+ * @ccswe_shader_cores: Shader cores cycle count software estimator. -+ */ -+struct kbase_hwcnt_backend_jm { -+ const struct kbase_hwcnt_backend_jm_info *info; -+ struct kbase_context *kctx; -+ u64 gpu_dump_va; -+ void *cpu_dump_va; -+ struct kbase_vmap_struct *vmap; -+ bool enabled; -+ u64 pm_core_mask; -+ struct kbase_hwcnt_curr_config curr_config; -+ u64 clk_enable_map; -+ u64 cycle_count_elapsed[BASE_MAX_NR_CLOCKS_REGULATORS]; -+ u64 prev_cycle_count[BASE_MAX_NR_CLOCKS_REGULATORS]; -+ struct kbase_clk_rate_listener rate_listener; -+ struct kbase_ccswe ccswe_shader_cores; -+}; -+ -+/** -+ * kbasep_hwcnt_backend_jm_gpu_info_init() - Initialise an info structure used -+ * to create the hwcnt metadata. -+ * @kbdev: Non-NULL pointer to kbase device. -+ * @info: Non-NULL pointer to data structure to be filled in. -+ * -+ * The initialised info struct will only be valid for use while kbdev is valid. -+ */ -+static int -+kbasep_hwcnt_backend_jm_gpu_info_init(struct kbase_device *kbdev, -+ struct kbase_hwcnt_gpu_info *info) -+{ -+ size_t clk; -+ -+ if (!kbdev || !info) -+ return -EINVAL; -+ -+ { -+ const struct base_gpu_props *props = &kbdev->gpu_props.props; -+ const size_t l2_count = props->l2_props.num_l2_slices; -+ const size_t core_mask = -+ props->coherency_info.group[0].core_mask; -+ -+ info->l2_count = l2_count; -+ info->core_mask = core_mask; -+ info->prfcnt_values_per_block = -+ KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK; -+ } -+ -+ /* Determine the number of available clock domains. */ -+ for (clk = 0; clk < BASE_MAX_NR_CLOCKS_REGULATORS; clk++) { -+ if (kbdev->pm.clk_rtm.clks[clk] == NULL) -+ break; -+ } -+ info->clk_cnt = clk; -+ -+ return 0; -+} -+ -+/** -+ * kbasep_hwcnt_backend_jm_on_freq_change() - On freq change callback -+ * -+ * @rate_listener: Callback state -+ * @clk_index: Clock index -+ * @clk_rate_hz: Clock frequency(hz) -+ */ -+static void kbasep_hwcnt_backend_jm_on_freq_change( -+ struct kbase_clk_rate_listener *rate_listener, -+ u32 clk_index, -+ u32 clk_rate_hz) -+{ -+ struct kbase_hwcnt_backend_jm *backend_jm = container_of( -+ rate_listener, struct kbase_hwcnt_backend_jm, rate_listener); -+ u64 timestamp_ns; -+ -+ if (clk_index != KBASE_CLOCK_DOMAIN_SHADER_CORES) -+ return; -+ -+ timestamp_ns = ktime_get_raw_ns(); -+ kbase_ccswe_freq_change( -+ &backend_jm->ccswe_shader_cores, timestamp_ns, clk_rate_hz); -+} -+ -+/** -+ * kbasep_hwcnt_backend_jm_cc_enable() - Enable cycle count tracking -+ * -+ * @backend_jm: Non-NULL pointer to backend. -+ * @enable_map: Non-NULL pointer to enable map specifying enabled counters. -+ * @timestamp_ns: Timestamp(ns) when HWCNT were enabled. -+ */ -+static void kbasep_hwcnt_backend_jm_cc_enable( -+ struct kbase_hwcnt_backend_jm *backend_jm, -+ const struct kbase_hwcnt_enable_map *enable_map, -+ u64 timestamp_ns) -+{ -+ struct kbase_device *kbdev = backend_jm->kctx->kbdev; -+ u64 clk_enable_map = enable_map->clk_enable_map; -+ u64 cycle_count; -+ -+ if (kbase_hwcnt_clk_enable_map_enabled( -+ clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { -+ /* turn on the cycle counter */ -+ kbase_pm_request_gpu_cycle_counter_l2_is_on(kbdev); -+ /* Read cycle count for top clock domain. */ -+ kbase_backend_get_gpu_time_norequest( -+ kbdev, &cycle_count, NULL, NULL); -+ -+ backend_jm->prev_cycle_count[KBASE_CLOCK_DOMAIN_TOP] = -+ cycle_count; -+ } -+ -+ if (kbase_hwcnt_clk_enable_map_enabled( -+ clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { -+ /* software estimation for non-top clock domains */ -+ struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; -+ const struct kbase_clk_data *clk_data = -+ rtm->clks[KBASE_CLOCK_DOMAIN_SHADER_CORES]; -+ u32 cur_freq; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&rtm->lock, flags); -+ -+ cur_freq = (u32) clk_data->clock_val; -+ kbase_ccswe_reset(&backend_jm->ccswe_shader_cores); -+ kbase_ccswe_freq_change( -+ &backend_jm->ccswe_shader_cores, -+ timestamp_ns, -+ cur_freq); -+ -+ kbase_clk_rate_trace_manager_subscribe_no_lock( -+ rtm, &backend_jm->rate_listener); -+ -+ spin_unlock_irqrestore(&rtm->lock, flags); -+ -+ /* ccswe was reset. The estimated cycle is zero. */ -+ backend_jm->prev_cycle_count[ -+ KBASE_CLOCK_DOMAIN_SHADER_CORES] = 0; -+ } -+ -+ /* Keep clk_enable_map for dump_request. */ -+ backend_jm->clk_enable_map = clk_enable_map; -+} -+ -+/** -+ * kbasep_hwcnt_backend_jm_cc_disable() - Disable cycle count tracking -+ * -+ * @backend_jm: Non-NULL pointer to backend. -+ */ -+static void kbasep_hwcnt_backend_jm_cc_disable( -+ struct kbase_hwcnt_backend_jm *backend_jm) -+{ -+ struct kbase_device *kbdev = backend_jm->kctx->kbdev; -+ struct kbase_clk_rate_trace_manager *rtm = &kbdev->pm.clk_rtm; -+ u64 clk_enable_map = backend_jm->clk_enable_map; -+ -+ if (kbase_hwcnt_clk_enable_map_enabled( -+ clk_enable_map, KBASE_CLOCK_DOMAIN_TOP)) { -+ /* turn off the cycle counter */ -+ kbase_pm_release_gpu_cycle_counter(kbdev); -+ } -+ -+ if (kbase_hwcnt_clk_enable_map_enabled( -+ clk_enable_map, KBASE_CLOCK_DOMAIN_SHADER_CORES)) { -+ -+ kbase_clk_rate_trace_manager_unsubscribe( -+ rtm, &backend_jm->rate_listener); -+ } -+} -+ -+ -+/** -+ * kbasep_hwcnt_gpu_update_curr_config() - Update the destination buffer with -+ * current config information. -+ * @kbdev: Non-NULL pointer to kbase device. -+ * @curr_config: Non-NULL pointer to return the current configuration of -+ * hardware allocated to the GPU. -+ * -+ * The current configuration information is used for architectures where the -+ * max_config interface is available from the Arbiter. In this case the current -+ * allocated hardware is not always the same, so the current config information -+ * is used to correctly map the current allocated resources to the memory layout -+ * that is copied to the user space. -+ * -+ * Return: 0 on success, else error code. -+ */ -+static int kbasep_hwcnt_gpu_update_curr_config( -+ struct kbase_device *kbdev, -+ struct kbase_hwcnt_curr_config *curr_config) -+{ -+ if (WARN_ON(!kbdev) || WARN_ON(!curr_config)) -+ return -EINVAL; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ curr_config->num_l2_slices = -+ kbdev->gpu_props.curr_config.l2_slices; -+ curr_config->shader_present = -+ kbdev->gpu_props.curr_config.shader_present; -+ return 0; -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_timestamp_ns_fn */ -+static u64 kbasep_hwcnt_backend_jm_timestamp_ns( -+ struct kbase_hwcnt_backend *backend) -+{ -+ (void)backend; -+ return ktime_get_raw_ns(); -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_dump_enable_nolock_fn */ -+static int kbasep_hwcnt_backend_jm_dump_enable_nolock( -+ struct kbase_hwcnt_backend *backend, -+ const struct kbase_hwcnt_enable_map *enable_map) -+{ -+ int errcode; -+ struct kbase_hwcnt_backend_jm *backend_jm = -+ (struct kbase_hwcnt_backend_jm *)backend; -+ struct kbase_context *kctx; -+ struct kbase_device *kbdev; -+ struct kbase_hwcnt_physical_enable_map phys_enable_map; -+ enum kbase_hwcnt_physical_set phys_counter_set; -+ struct kbase_instr_hwcnt_enable enable; -+ u64 timestamp_ns; -+ -+ if (!backend_jm || !enable_map || backend_jm->enabled || -+ (enable_map->metadata != backend_jm->info->metadata)) -+ return -EINVAL; -+ -+ kctx = backend_jm->kctx; -+ kbdev = backend_jm->kctx->kbdev; -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ kbase_hwcnt_gpu_enable_map_to_physical(&phys_enable_map, enable_map); -+ -+ kbase_hwcnt_gpu_set_to_physical(&phys_counter_set, -+ backend_jm->info->counter_set); -+ -+ enable.fe_bm = phys_enable_map.fe_bm; -+ enable.shader_bm = phys_enable_map.shader_bm; -+ enable.tiler_bm = phys_enable_map.tiler_bm; -+ enable.mmu_l2_bm = phys_enable_map.mmu_l2_bm; -+ enable.counter_set = phys_counter_set; -+ enable.dump_buffer = backend_jm->gpu_dump_va; -+ enable.dump_buffer_bytes = backend_jm->info->dump_bytes; -+ -+ timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); -+ -+ /* Update the current configuration information. */ -+ errcode = kbasep_hwcnt_gpu_update_curr_config(kbdev, -+ &backend_jm->curr_config); -+ if (errcode) -+ goto error; -+ -+ errcode = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &enable); -+ if (errcode) -+ goto error; -+ -+ backend_jm->pm_core_mask = kbase_pm_ca_get_instr_core_mask(kbdev); -+ -+ backend_jm->enabled = true; -+ -+ kbasep_hwcnt_backend_jm_cc_enable(backend_jm, enable_map, timestamp_ns); -+ -+ return 0; -+error: -+ return errcode; -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_dump_enable_fn */ -+static int kbasep_hwcnt_backend_jm_dump_enable( -+ struct kbase_hwcnt_backend *backend, -+ const struct kbase_hwcnt_enable_map *enable_map) -+{ -+ unsigned long flags; -+ int errcode; -+ struct kbase_hwcnt_backend_jm *backend_jm = -+ (struct kbase_hwcnt_backend_jm *)backend; -+ struct kbase_device *kbdev; -+ -+ if (!backend_jm) -+ return -EINVAL; -+ -+ kbdev = backend_jm->kctx->kbdev; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ -+ errcode = kbasep_hwcnt_backend_jm_dump_enable_nolock( -+ backend, enable_map); -+ -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return errcode; -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_dump_disable_fn */ -+static void kbasep_hwcnt_backend_jm_dump_disable( -+ struct kbase_hwcnt_backend *backend) -+{ -+ int errcode; -+ struct kbase_hwcnt_backend_jm *backend_jm = -+ (struct kbase_hwcnt_backend_jm *)backend; -+ -+ if (WARN_ON(!backend_jm) || !backend_jm->enabled) -+ return; -+ -+ kbasep_hwcnt_backend_jm_cc_disable(backend_jm); -+ -+ errcode = kbase_instr_hwcnt_disable_internal(backend_jm->kctx); -+ WARN_ON(errcode); -+ -+ backend_jm->enabled = false; -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_dump_clear_fn */ -+static int kbasep_hwcnt_backend_jm_dump_clear( -+ struct kbase_hwcnt_backend *backend) -+{ -+ struct kbase_hwcnt_backend_jm *backend_jm = -+ (struct kbase_hwcnt_backend_jm *)backend; -+ -+ if (!backend_jm || !backend_jm->enabled) -+ return -EINVAL; -+ -+ return kbase_instr_hwcnt_clear(backend_jm->kctx); -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_dump_request_fn */ -+static int kbasep_hwcnt_backend_jm_dump_request( -+ struct kbase_hwcnt_backend *backend, -+ u64 *dump_time_ns) -+{ -+ struct kbase_hwcnt_backend_jm *backend_jm = -+ (struct kbase_hwcnt_backend_jm *)backend; -+ struct kbase_device *kbdev; -+ const struct kbase_hwcnt_metadata *metadata; -+ u64 current_cycle_count; -+ size_t clk; -+ int ret; -+ -+ if (!backend_jm || !backend_jm->enabled || !dump_time_ns) -+ return -EINVAL; -+ -+ kbdev = backend_jm->kctx->kbdev; -+ metadata = backend_jm->info->metadata; -+ -+ /* Disable pre-emption, to make the timestamp as accurate as possible */ -+ preempt_disable(); -+ { -+ *dump_time_ns = kbasep_hwcnt_backend_jm_timestamp_ns(backend); -+ ret = kbase_instr_hwcnt_request_dump(backend_jm->kctx); -+ -+ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { -+ if (!kbase_hwcnt_clk_enable_map_enabled( -+ backend_jm->clk_enable_map, clk)) -+ continue; -+ -+ if (clk == KBASE_CLOCK_DOMAIN_TOP) { -+ /* Read cycle count for top clock domain. */ -+ kbase_backend_get_gpu_time_norequest( -+ kbdev, ¤t_cycle_count, -+ NULL, NULL); -+ } else { -+ /* -+ * Estimate cycle count for non-top clock -+ * domain. -+ */ -+ current_cycle_count = kbase_ccswe_cycle_at( -+ &backend_jm->ccswe_shader_cores, -+ *dump_time_ns); -+ } -+ backend_jm->cycle_count_elapsed[clk] = -+ current_cycle_count - -+ backend_jm->prev_cycle_count[clk]; -+ -+ /* -+ * Keep the current cycle count for later calculation. -+ */ -+ backend_jm->prev_cycle_count[clk] = current_cycle_count; -+ } -+ } -+ preempt_enable(); -+ -+ return ret; -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_dump_wait_fn */ -+static int kbasep_hwcnt_backend_jm_dump_wait( -+ struct kbase_hwcnt_backend *backend) -+{ -+ struct kbase_hwcnt_backend_jm *backend_jm = -+ (struct kbase_hwcnt_backend_jm *)backend; -+ -+ if (!backend_jm || !backend_jm->enabled) -+ return -EINVAL; -+ -+ return kbase_instr_hwcnt_wait_for_dump(backend_jm->kctx); -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_dump_get_fn */ -+static int kbasep_hwcnt_backend_jm_dump_get( -+ struct kbase_hwcnt_backend *backend, -+ struct kbase_hwcnt_dump_buffer *dst, -+ const struct kbase_hwcnt_enable_map *dst_enable_map, -+ bool accumulate) -+{ -+ struct kbase_hwcnt_backend_jm *backend_jm = -+ (struct kbase_hwcnt_backend_jm *)backend; -+ size_t clk; -+ -+ if (!backend_jm || !dst || !dst_enable_map || -+ (backend_jm->info->metadata != dst->metadata) || -+ (dst_enable_map->metadata != dst->metadata)) -+ return -EINVAL; -+ -+ /* Invalidate the kernel buffer before reading from it. */ -+ kbase_sync_mem_regions( -+ backend_jm->kctx, backend_jm->vmap, KBASE_SYNC_TO_CPU); -+ -+ kbase_hwcnt_metadata_for_each_clock(dst_enable_map->metadata, clk) { -+ if (!kbase_hwcnt_clk_enable_map_enabled( -+ dst_enable_map->clk_enable_map, clk)) -+ continue; -+ -+ /* Extract elapsed cycle count for each clock domain. */ -+ dst->clk_cnt_buf[clk] = backend_jm->cycle_count_elapsed[clk]; -+ } -+ -+ return kbase_hwcnt_jm_dump_get(dst, backend_jm->cpu_dump_va, -+ dst_enable_map, backend_jm->pm_core_mask, -+ &backend_jm->curr_config, accumulate); -+} -+ -+/** -+ * kbasep_hwcnt_backend_jm_dump_alloc() - Allocate a GPU dump buffer. -+ * @info: Non-NULL pointer to JM backend info. -+ * @kctx: Non-NULL pointer to kbase context. -+ * @gpu_dump_va: Non-NULL pointer to where GPU dump buffer virtual address -+ * is stored on success. -+ * -+ * Return: 0 on success, else error code. -+ */ -+static int kbasep_hwcnt_backend_jm_dump_alloc( -+ const struct kbase_hwcnt_backend_jm_info *info, -+ struct kbase_context *kctx, -+ u64 *gpu_dump_va) -+{ -+ struct kbase_va_region *reg; -+ u64 flags; -+ u64 nr_pages; -+ -+ WARN_ON(!info); -+ WARN_ON(!kctx); -+ WARN_ON(!gpu_dump_va); -+ -+ flags = BASE_MEM_PROT_CPU_RD | -+ BASE_MEM_PROT_GPU_WR | -+ BASEP_MEM_PERMANENT_KERNEL_MAPPING | -+ BASE_MEM_CACHED_CPU | -+ BASE_MEM_UNCACHED_GPU; -+ -+ nr_pages = PFN_UP(info->dump_bytes); -+ -+ reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags, gpu_dump_va); -+ -+ if (!reg) -+ return -ENOMEM; -+ -+ return 0; -+} -+ -+/** -+ * kbasep_hwcnt_backend_jm_dump_free() - Free an allocated GPU dump buffer. -+ * @kctx: Non-NULL pointer to kbase context. -+ * @gpu_dump_va: GPU dump buffer virtual address. -+ */ -+static void kbasep_hwcnt_backend_jm_dump_free( -+ struct kbase_context *kctx, -+ u64 gpu_dump_va) -+{ -+ WARN_ON(!kctx); -+ if (gpu_dump_va) -+ kbase_mem_free(kctx, gpu_dump_va); -+} -+ -+/** -+ * kbasep_hwcnt_backend_jm_destroy() - Destroy a JM backend. -+ * @backend: Pointer to JM backend to destroy. -+ * -+ * Can be safely called on a backend in any state of partial construction. -+ */ -+static void kbasep_hwcnt_backend_jm_destroy( -+ struct kbase_hwcnt_backend_jm *backend) -+{ -+ if (!backend) -+ return; -+ -+ if (backend->kctx) { -+ struct kbase_context *kctx = backend->kctx; -+ struct kbase_device *kbdev = kctx->kbdev; -+ -+ if (backend->cpu_dump_va) -+ kbase_phy_alloc_mapping_put(kctx, backend->vmap); -+ -+ if (backend->gpu_dump_va) -+ kbasep_hwcnt_backend_jm_dump_free( -+ kctx, backend->gpu_dump_va); -+ -+ kbasep_js_release_privileged_ctx(kbdev, kctx); -+ kbase_destroy_context(kctx); -+ } -+ -+ kfree(backend); -+} -+ -+/** -+ * kbasep_hwcnt_backend_jm_create() - Create a JM backend. -+ * @info: Non-NULL pointer to backend info. -+ * @out_backend: Non-NULL pointer to where backend is stored on success. -+ * -+ * Return: 0 on success, else error code. -+ */ -+static int kbasep_hwcnt_backend_jm_create( -+ const struct kbase_hwcnt_backend_jm_info *info, -+ struct kbase_hwcnt_backend_jm **out_backend) -+{ -+ int errcode; -+ struct kbase_device *kbdev; -+ struct kbase_hwcnt_backend_jm *backend = NULL; -+ -+ WARN_ON(!info); -+ WARN_ON(!out_backend); -+ -+ kbdev = info->kbdev; -+ -+ backend = kzalloc(sizeof(*backend), GFP_KERNEL); -+ if (!backend) -+ goto alloc_error; -+ -+ backend->info = info; -+ -+ backend->kctx = kbase_create_context(kbdev, true, -+ BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED, 0, NULL); -+ if (!backend->kctx) -+ goto alloc_error; -+ -+ kbasep_js_schedule_privileged_ctx(kbdev, backend->kctx); -+ -+ errcode = kbasep_hwcnt_backend_jm_dump_alloc( -+ info, backend->kctx, &backend->gpu_dump_va); -+ if (errcode) -+ goto error; -+ -+ backend->cpu_dump_va = kbase_phy_alloc_mapping_get(backend->kctx, -+ backend->gpu_dump_va, &backend->vmap); -+ if (!backend->cpu_dump_va) -+ goto alloc_error; -+ -+ kbase_ccswe_init(&backend->ccswe_shader_cores); -+ backend->rate_listener.notify = kbasep_hwcnt_backend_jm_on_freq_change; -+ -+ -+ *out_backend = backend; -+ return 0; -+ -+alloc_error: -+ errcode = -ENOMEM; -+error: -+ kbasep_hwcnt_backend_jm_destroy(backend); -+ return errcode; -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_metadata_fn */ -+static const struct kbase_hwcnt_metadata * -+kbasep_hwcnt_backend_jm_metadata(const struct kbase_hwcnt_backend_info *info) -+{ -+ if (!info) -+ return NULL; -+ -+ return ((const struct kbase_hwcnt_backend_jm_info *)info)->metadata; -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_init_fn */ -+static int kbasep_hwcnt_backend_jm_init( -+ const struct kbase_hwcnt_backend_info *info, -+ struct kbase_hwcnt_backend **out_backend) -+{ -+ int errcode; -+ struct kbase_hwcnt_backend_jm *backend = NULL; -+ -+ if (!info || !out_backend) -+ return -EINVAL; -+ -+ errcode = kbasep_hwcnt_backend_jm_create( -+ (const struct kbase_hwcnt_backend_jm_info *) info, &backend); -+ if (errcode) -+ return errcode; -+ -+ *out_backend = (struct kbase_hwcnt_backend *)backend; -+ -+ return 0; -+} -+ -+/* JM backend implementation of kbase_hwcnt_backend_term_fn */ -+static void kbasep_hwcnt_backend_jm_term(struct kbase_hwcnt_backend *backend) -+{ -+ if (!backend) -+ return; -+ -+ kbasep_hwcnt_backend_jm_dump_disable(backend); -+ kbasep_hwcnt_backend_jm_destroy( -+ (struct kbase_hwcnt_backend_jm *)backend); -+} -+ -+/** -+ * kbasep_hwcnt_backend_jm_info_destroy() - Destroy a JM backend info. -+ * @info: Pointer to info to destroy. -+ * -+ * Can be safely called on a backend info in any state of partial construction. -+ */ -+static void kbasep_hwcnt_backend_jm_info_destroy( -+ const struct kbase_hwcnt_backend_jm_info *info) -+{ -+ if (!info) -+ return; -+ -+ kbase_hwcnt_jm_metadata_destroy(info->metadata); -+ kfree(info); -+} -+ -+/** -+ * kbasep_hwcnt_backend_jm_info_create() - Create a JM backend info. -+ * @kbdev: Non_NULL pointer to kbase device. -+ * @out_info: Non-NULL pointer to where info is stored on success. -+ * -+ * Return 0 on success, else error code. -+ */ -+static int kbasep_hwcnt_backend_jm_info_create( -+ struct kbase_device *kbdev, -+ const struct kbase_hwcnt_backend_jm_info **out_info) -+{ -+ int errcode = -ENOMEM; -+ struct kbase_hwcnt_gpu_info hwcnt_gpu_info; -+ struct kbase_hwcnt_backend_jm_info *info = NULL; -+ -+ WARN_ON(!kbdev); -+ WARN_ON(!out_info); -+ -+ errcode = kbasep_hwcnt_backend_jm_gpu_info_init(kbdev, &hwcnt_gpu_info); -+ if (errcode) -+ return errcode; -+ -+ info = kzalloc(sizeof(*info), GFP_KERNEL); -+ if (!info) -+ goto error; -+ -+ info->kbdev = kbdev; -+ -+#if defined(CONFIG_MALI_PRFCNT_SET_SECONDARY) -+ info->counter_set = KBASE_HWCNT_SET_SECONDARY; -+#elif defined(CONFIG_MALI_PRFCNT_SET_TERTIARY) -+ info->counter_set = KBASE_HWCNT_SET_TERTIARY; -+#else -+ /* Default to primary */ -+ info->counter_set = KBASE_HWCNT_SET_PRIMARY; -+#endif -+ -+ errcode = kbase_hwcnt_jm_metadata_create(&hwcnt_gpu_info, -+ info->counter_set, -+ &info->metadata, -+ &info->dump_bytes); -+ if (errcode) -+ goto error; -+ -+ *out_info = info; -+ -+ return 0; -+error: -+ kbasep_hwcnt_backend_jm_info_destroy(info); -+ return errcode; -+} -+ -+int kbase_hwcnt_backend_jm_create( -+ struct kbase_device *kbdev, -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ int errcode; -+ const struct kbase_hwcnt_backend_jm_info *info = NULL; -+ -+ if (!kbdev || !iface) -+ return -EINVAL; -+ -+ errcode = kbasep_hwcnt_backend_jm_info_create(kbdev, &info); -+ -+ if (errcode) -+ return errcode; -+ -+ iface->info = (struct kbase_hwcnt_backend_info *)info; -+ iface->metadata = kbasep_hwcnt_backend_jm_metadata; -+ iface->init = kbasep_hwcnt_backend_jm_init; -+ iface->term = kbasep_hwcnt_backend_jm_term; -+ iface->timestamp_ns = kbasep_hwcnt_backend_jm_timestamp_ns; -+ iface->dump_enable = kbasep_hwcnt_backend_jm_dump_enable; -+ iface->dump_enable_nolock = kbasep_hwcnt_backend_jm_dump_enable_nolock; -+ iface->dump_disable = kbasep_hwcnt_backend_jm_dump_disable; -+ iface->dump_clear = kbasep_hwcnt_backend_jm_dump_clear; -+ iface->dump_request = kbasep_hwcnt_backend_jm_dump_request; -+ iface->dump_wait = kbasep_hwcnt_backend_jm_dump_wait; -+ iface->dump_get = kbasep_hwcnt_backend_jm_dump_get; -+ -+ return 0; -+} -+ -+void kbase_hwcnt_backend_jm_destroy( -+ struct kbase_hwcnt_backend_interface *iface) -+{ -+ if (!iface) -+ return; -+ -+ kbasep_hwcnt_backend_jm_info_destroy( -+ (const struct kbase_hwcnt_backend_jm_info *)iface->info); -+ memset(iface, 0, sizeof(*iface)); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_gpu.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_jm.h -similarity index 75% -rename from dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_gpu.h -rename to dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_jm.h -index 7712f14..5319516 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_gpu.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_backend_jm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,24 +17,22 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -- * Concrete implementation of mali_kbase_hwcnt_backend interface for GPU -+ * Concrete implementation of mali_kbase_hwcnt_backend interface for JM - * backend. - */ - --#ifndef _KBASE_HWCNT_BACKEND_GPU_H_ --#define _KBASE_HWCNT_BACKEND_GPU_H_ -+#ifndef _KBASE_HWCNT_BACKEND_JM_H_ -+#define _KBASE_HWCNT_BACKEND_JM_H_ - - #include "mali_kbase_hwcnt_backend.h" - - struct kbase_device; - - /** -- * kbase_hwcnt_backend_gpu_create() - Create a GPU hardware counter backend -+ * kbase_hwcnt_backend_jm_create() - Create a JM hardware counter backend - * interface. - * @kbdev: Non-NULL pointer to kbase device. - * @iface: Non-NULL pointer to backend interface structure that is filled in -@@ -43,19 +42,19 @@ struct kbase_device; - * - * Return: 0 on success, else error code. - */ --int kbase_hwcnt_backend_gpu_create( -+int kbase_hwcnt_backend_jm_create( - struct kbase_device *kbdev, - struct kbase_hwcnt_backend_interface *iface); - - /** -- * kbase_hwcnt_backend_gpu_destroy() - Destroy a GPU hardware counter backend -+ * kbase_hwcnt_backend_jm_destroy() - Destroy a JM hardware counter backend - * interface. - * @iface: Pointer to interface to destroy. - * - * Can be safely called on an all-zeroed interface, or on an already destroyed - * interface. - */ --void kbase_hwcnt_backend_gpu_destroy( -+void kbase_hwcnt_backend_jm_destroy( - struct kbase_hwcnt_backend_interface *iface); - --#endif /* _KBASE_HWCNT_BACKEND_GPU_H_ */ -+#endif /* _KBASE_HWCNT_BACKEND_JM_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_context.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_context.h -index bc50ad1..1adf2ef 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_context.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_context.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -@@ -28,6 +27,7 @@ - #define _KBASE_HWCNT_CONTEXT_H_ - - #include <linux/types.h> -+#include <linux/workqueue.h> - - struct kbase_hwcnt_backend_interface; - struct kbase_hwcnt_context; -@@ -66,7 +66,7 @@ const struct kbase_hwcnt_metadata *kbase_hwcnt_context_metadata( - - /** - * kbase_hwcnt_context_disable() - Increment the disable count of the context. -- * @hctx: Pointer to the hardware counter context. -+ * @hctx: Non-NULL pointer to the hardware counter context. - * - * If a call to this function increments the disable count from 0 to 1, and - * an accumulator has been acquired, then a counter dump will be performed -@@ -84,7 +84,7 @@ void kbase_hwcnt_context_disable(struct kbase_hwcnt_context *hctx); - * kbase_hwcnt_context_disable_atomic() - Increment the disable count of the - * context if possible in an atomic - * context. -- * @hctx: Pointer to the hardware counter context. -+ * @hctx: Non-NULL pointer to the hardware counter context. - * - * This function will only succeed if hardware counters are effectively already - * disabled, i.e. there is no accumulator, the disable count is already -@@ -99,7 +99,7 @@ bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx); - - /** - * kbase_hwcnt_context_enable() - Decrement the disable count of the context. -- * @hctx: Pointer to the hardware counter context. -+ * @hctx: Non-NULL pointer to the hardware counter context. - * - * If a call to this function decrements the disable count from 1 to 0, and - * an accumulator has been acquired, then counters will be re-enabled via the -@@ -116,4 +116,36 @@ bool kbase_hwcnt_context_disable_atomic(struct kbase_hwcnt_context *hctx); - */ - void kbase_hwcnt_context_enable(struct kbase_hwcnt_context *hctx); - -+/** -+ * kbase_hwcnt_context_queue_work() - Queue hardware counter related async -+ * work on a workqueue specialized for -+ * hardware counters. -+ * @hctx: Non-NULL pointer to the hardware counter context. -+ * @work: Non-NULL pointer to work to queue. -+ * -+ * Return: false if work was already on a queue, true otherwise. -+ * -+ * Performance counter related work is high priority, short running, and -+ * generally CPU locality is unimportant. There is no standard workqueue that -+ * can service this flavor of work. -+ * -+ * Rather than have each user of counters define their own workqueue, we have -+ * a centralized one in here that anybody using this hardware counter API -+ * should use. -+ * -+ * Before the context is destroyed, all work submitted must have been completed. -+ * Given that the work enqueued via this function is likely to be hardware -+ * counter related and will therefore use the context object, this is likely -+ * to be behavior that will occur naturally. -+ * -+ * Historical note: prior to this centralized workqueue, the system_highpri_wq -+ * was used. This was generally fine, except when a particularly long running, -+ * higher priority thread ended up scheduled on the enqueuing CPU core. Given -+ * that hardware counters requires tight integration with power management, -+ * this meant progress through the power management states could be stalled -+ * for however long that higher priority thread took. -+ */ -+bool kbase_hwcnt_context_queue_work(struct kbase_hwcnt_context *hctx, -+ struct work_struct *work); -+ - #endif /* _KBASE_HWCNT_CONTEXT_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_gpu.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_gpu.c -index 095c765..2975269 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_gpu.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_gpu.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,170 +17,111 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase_hwcnt_gpu.h" - #include "mali_kbase_hwcnt_types.h" --#include "mali_kbase.h" --#ifdef CONFIG_MALI_NO_MALI --#include "backend/gpu/mali_kbase_model_dummy.h" --#endif -- --#define KBASE_HWCNT_V4_BLOCKS_PER_GROUP 8 --#define KBASE_HWCNT_V4_SC_BLOCKS_PER_GROUP 4 --#define KBASE_HWCNT_V4_MAX_GROUPS \ -- (KBASE_HWCNT_AVAIL_MASK_BITS / KBASE_HWCNT_V4_BLOCKS_PER_GROUP) --#define KBASE_HWCNT_V4_HEADERS_PER_BLOCK 4 --#define KBASE_HWCNT_V4_COUNTERS_PER_BLOCK 60 --#define KBASE_HWCNT_V4_VALUES_PER_BLOCK \ -- (KBASE_HWCNT_V4_HEADERS_PER_BLOCK + KBASE_HWCNT_V4_COUNTERS_PER_BLOCK) --/* Index of the PRFCNT_EN header into a V4 counter block */ --#define KBASE_HWCNT_V4_PRFCNT_EN_HEADER 2 -- --#define KBASE_HWCNT_V5_BLOCK_TYPE_COUNT 4 --#define KBASE_HWCNT_V5_HEADERS_PER_BLOCK 4 --#define KBASE_HWCNT_V5_COUNTERS_PER_BLOCK 60 --#define KBASE_HWCNT_V5_VALUES_PER_BLOCK \ -- (KBASE_HWCNT_V5_HEADERS_PER_BLOCK + KBASE_HWCNT_V5_COUNTERS_PER_BLOCK) --/* Index of the PRFCNT_EN header into a V5 counter block */ --#define KBASE_HWCNT_V5_PRFCNT_EN_HEADER 2 -- --/** -- * kbasep_hwcnt_backend_gpu_metadata_v4_create() - Create hardware counter -- * metadata for a v4 GPU. -- * @v4_info: Non-NULL pointer to hwcnt info for a v4 GPU. -- * @metadata: Non-NULL pointer to where created metadata is stored on success. -- * -- * Return: 0 on success, else error code. -- */ --static int kbasep_hwcnt_backend_gpu_metadata_v4_create( -- const struct kbase_hwcnt_gpu_v4_info *v4_info, -- const struct kbase_hwcnt_metadata **metadata) --{ -- size_t grp; -- int errcode = -ENOMEM; -- struct kbase_hwcnt_description desc; -- struct kbase_hwcnt_group_description *grps; -- size_t avail_mask_bit; - -- WARN_ON(!v4_info); -- WARN_ON(!metadata); -+#include <linux/bug.h> -+#include <linux/err.h> - -- /* Check if there are enough bits in the availability mask to represent -- * all the hardware counter blocks in the system. -- */ -- if (v4_info->cg_count > KBASE_HWCNT_V4_MAX_GROUPS) -- return -EINVAL; - -- grps = kcalloc(v4_info->cg_count, sizeof(*grps), GFP_KERNEL); -- if (!grps) -- goto clean_up; -- -- desc.grp_cnt = v4_info->cg_count; -- desc.grps = grps; -- -- for (grp = 0; grp < v4_info->cg_count; grp++) { -- size_t blk; -- size_t sc; -- const u64 core_mask = v4_info->cgs[grp].core_mask; -- struct kbase_hwcnt_block_description *blks = kcalloc( -- KBASE_HWCNT_V4_BLOCKS_PER_GROUP, -- sizeof(*blks), -- GFP_KERNEL); -- -- if (!blks) -- goto clean_up; -- -- grps[grp].type = KBASE_HWCNT_GPU_GROUP_TYPE_V4; -- grps[grp].blk_cnt = KBASE_HWCNT_V4_BLOCKS_PER_GROUP; -- grps[grp].blks = blks; -- -- for (blk = 0; blk < KBASE_HWCNT_V4_BLOCKS_PER_GROUP; blk++) { -- blks[blk].inst_cnt = 1; -- blks[blk].hdr_cnt = -- KBASE_HWCNT_V4_HEADERS_PER_BLOCK; -- blks[blk].ctr_cnt = -- KBASE_HWCNT_V4_COUNTERS_PER_BLOCK; -+static void kbasep_get_fe_block_type(u64 *dst, enum kbase_hwcnt_set counter_set, -+ bool is_csf) -+{ -+ switch (counter_set) { -+ case KBASE_HWCNT_SET_PRIMARY: -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE; -+ break; -+ case KBASE_HWCNT_SET_SECONDARY: -+ if (is_csf) { -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE2; -+ } else { -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED; - } -- -- for (sc = 0; sc < KBASE_HWCNT_V4_SC_BLOCKS_PER_GROUP; sc++) { -- blks[sc].type = core_mask & (1ull << sc) ? -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_SHADER : -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_RESERVED; -+ break; -+ case KBASE_HWCNT_SET_TERTIARY: -+ if (is_csf) { -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE3; -+ } else { -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED; - } -- -- blks[4].type = KBASE_HWCNT_GPU_V4_BLOCK_TYPE_TILER; -- blks[5].type = KBASE_HWCNT_GPU_V4_BLOCK_TYPE_MMU_L2; -- blks[6].type = KBASE_HWCNT_GPU_V4_BLOCK_TYPE_RESERVED; -- blks[7].type = (grp == 0) ? -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_JM : -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_RESERVED; -- -- WARN_ON(KBASE_HWCNT_V4_BLOCKS_PER_GROUP != 8); -+ break; -+ default: -+ WARN_ON(true); - } -+} - -- /* Initialise the availability mask */ -- desc.avail_mask = 0; -- avail_mask_bit = 0; -- -- for (grp = 0; grp < desc.grp_cnt; grp++) { -- size_t blk; -- const struct kbase_hwcnt_block_description *blks = -- desc.grps[grp].blks; -- for (blk = 0; blk < desc.grps[grp].blk_cnt; blk++) { -- WARN_ON(blks[blk].inst_cnt != 1); -- if (blks[blk].type != -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_RESERVED) -- desc.avail_mask |= (1ull << avail_mask_bit); -- -- avail_mask_bit++; -- } -+static void kbasep_get_tiler_block_type(u64 *dst, -+ enum kbase_hwcnt_set counter_set) -+{ -+ switch (counter_set) { -+ case KBASE_HWCNT_SET_PRIMARY: -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER; -+ break; -+ case KBASE_HWCNT_SET_SECONDARY: -+ case KBASE_HWCNT_SET_TERTIARY: -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED; -+ break; -+ default: -+ WARN_ON(true); - } -+} - -- errcode = kbase_hwcnt_metadata_create(&desc, metadata); -- -- /* Always clean up, as metadata will make a copy of the input args */ --clean_up: -- if (grps) { -- for (grp = 0; grp < v4_info->cg_count; grp++) -- kfree(grps[grp].blks); -- kfree(grps); -+static void kbasep_get_sc_block_type(u64 *dst, enum kbase_hwcnt_set counter_set, -+ bool is_csf) -+{ -+ switch (counter_set) { -+ case KBASE_HWCNT_SET_PRIMARY: -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC; -+ break; -+ case KBASE_HWCNT_SET_SECONDARY: -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2; -+ break; -+ case KBASE_HWCNT_SET_TERTIARY: -+ if (is_csf) { -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC3; -+ } else { -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED; -+ } -+ break; -+ default: -+ WARN_ON(true); - } -- return errcode; - } - --/** -- * kbasep_hwcnt_backend_gpu_v4_dump_bytes() - Get the raw dump buffer size for a -- * V4 GPU. -- * @v4_info: Non-NULL pointer to hwcnt info for a v4 GPU. -- * -- * Return: Size of buffer the V4 GPU needs to perform a counter dump. -- */ --static size_t kbasep_hwcnt_backend_gpu_v4_dump_bytes( -- const struct kbase_hwcnt_gpu_v4_info *v4_info) -+static void kbasep_get_memsys_block_type(u64 *dst, -+ enum kbase_hwcnt_set counter_set) - { -- return v4_info->cg_count * -- KBASE_HWCNT_V4_BLOCKS_PER_GROUP * -- KBASE_HWCNT_V4_VALUES_PER_BLOCK * -- KBASE_HWCNT_VALUE_BYTES; -+ switch (counter_set) { -+ case KBASE_HWCNT_SET_PRIMARY: -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS; -+ break; -+ case KBASE_HWCNT_SET_SECONDARY: -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2; -+ break; -+ case KBASE_HWCNT_SET_TERTIARY: -+ *dst = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED; -+ break; -+ default: -+ WARN_ON(true); -+ } - } - - /** -- * kbasep_hwcnt_backend_gpu_metadata_v5_create() - Create hardware counter -- * metadata for a v5 GPU. -- * @v5_info: Non-NULL pointer to hwcnt info for a v5 GPU. -- * @use_secondary: True if secondary performance counters should be used, else -- * false. Ignored if secondary counters are not supported. -+ * kbasep_hwcnt_backend_gpu_metadata_create() - Create hardware counter metadata -+ * for the GPU. -+ * @gpu_info: Non-NULL pointer to hwcnt info for current GPU. -+ * @is_csf: true for CSF GPU, otherwise false. -+ * @counter_set: The performance counter set to use. - * @metadata: Non-NULL pointer to where created metadata is stored - * on success. - * - * Return: 0 on success, else error code. - */ --static int kbasep_hwcnt_backend_gpu_metadata_v5_create( -- const struct kbase_hwcnt_gpu_v5_info *v5_info, -- bool use_secondary, -+static int kbasep_hwcnt_backend_gpu_metadata_create( -+ const struct kbase_hwcnt_gpu_info *gpu_info, const bool is_csf, -+ enum kbase_hwcnt_set counter_set, - const struct kbase_hwcnt_metadata **metadata) - { - struct kbase_hwcnt_description desc; -@@ -189,13 +131,13 @@ static int kbasep_hwcnt_backend_gpu_metadata_v5_create( - size_t non_sc_block_count; - size_t sc_block_count; - -- WARN_ON(!v5_info); -+ WARN_ON(!gpu_info); - WARN_ON(!metadata); - - /* Calculate number of block instances that aren't shader cores */ -- non_sc_block_count = 2 + v5_info->l2_count; -+ non_sc_block_count = 2 + gpu_info->l2_count; - /* Calculate number of block instances that are shader cores */ -- sc_block_count = fls64(v5_info->core_mask); -+ sc_block_count = fls64(gpu_info->core_mask); - - /* - * A system can have up to 64 shader cores, but the 64-bit -@@ -207,25 +149,26 @@ static int kbasep_hwcnt_backend_gpu_metadata_v5_create( - if ((sc_block_count + non_sc_block_count) > KBASE_HWCNT_AVAIL_MASK_BITS) - return -EINVAL; - -- /* One Job Manager block */ -- blks[0].type = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM; -+ /* One Front End block */ -+ kbasep_get_fe_block_type(&blks[0].type, counter_set, is_csf); - blks[0].inst_cnt = 1; - blks[0].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; -- blks[0].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; -+ blks[0].ctr_cnt = gpu_info->prfcnt_values_per_block - -+ KBASE_HWCNT_V5_HEADERS_PER_BLOCK; - - /* One Tiler block */ -- blks[1].type = KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER; -+ kbasep_get_tiler_block_type(&blks[1].type, counter_set); - blks[1].inst_cnt = 1; - blks[1].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; -- blks[1].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; -+ blks[1].ctr_cnt = gpu_info->prfcnt_values_per_block - -+ KBASE_HWCNT_V5_HEADERS_PER_BLOCK; - - /* l2_count memsys blks */ -- blks[2].type = use_secondary ? -- KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2 : -- KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS; -- blks[2].inst_cnt = v5_info->l2_count; -+ kbasep_get_memsys_block_type(&blks[2].type, counter_set); -+ blks[2].inst_cnt = gpu_info->l2_count; - blks[2].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; -- blks[2].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; -+ blks[2].ctr_cnt = gpu_info->prfcnt_values_per_block - -+ KBASE_HWCNT_V5_HEADERS_PER_BLOCK; - - /* - * There are as many shader cores in the system as there are bits set in -@@ -243,12 +186,11 @@ static int kbasep_hwcnt_backend_gpu_metadata_v5_create( - * requirements, and embed the core mask into the availability mask so - * we can determine later which shader cores physically exist. - */ -- blks[3].type = use_secondary ? -- KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2 : -- KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC; -+ kbasep_get_sc_block_type(&blks[3].type, counter_set, is_csf); - blks[3].inst_cnt = sc_block_count; - blks[3].hdr_cnt = KBASE_HWCNT_V5_HEADERS_PER_BLOCK; -- blks[3].ctr_cnt = KBASE_HWCNT_V5_COUNTERS_PER_BLOCK; -+ blks[3].ctr_cnt = gpu_info->prfcnt_values_per_block - -+ KBASE_HWCNT_V5_HEADERS_PER_BLOCK; - - WARN_ON(KBASE_HWCNT_V5_BLOCK_TYPE_COUNT != 4); - -@@ -258,61 +200,35 @@ static int kbasep_hwcnt_backend_gpu_metadata_v5_create( - - desc.grp_cnt = 1; - desc.grps = &group; -+ desc.clk_cnt = gpu_info->clk_cnt; - - /* The JM, Tiler, and L2s are always available, and are before cores */ - desc.avail_mask = (1ull << non_sc_block_count) - 1; - /* Embed the core mask directly in the availability mask */ -- desc.avail_mask |= (v5_info->core_mask << non_sc_block_count); -+ desc.avail_mask |= (gpu_info->core_mask << non_sc_block_count); - - return kbase_hwcnt_metadata_create(&desc, metadata); - } - - /** -- * kbasep_hwcnt_backend_gpu_v5_dump_bytes() - Get the raw dump buffer size for a -- * V5 GPU. -- * @v5_info: Non-NULL pointer to hwcnt info for a v5 GPU. -+ * kbasep_hwcnt_backend_jm_dump_bytes() - Get the raw dump buffer size for the -+ * GPU. -+ * @gpu_info: Non-NULL pointer to hwcnt info for the GPU. - * -- * Return: Size of buffer the V5 GPU needs to perform a counter dump. -+ * Return: Size of buffer the GPU needs to perform a counter dump. - */ --static size_t kbasep_hwcnt_backend_gpu_v5_dump_bytes( -- const struct kbase_hwcnt_gpu_v5_info *v5_info) -+static size_t -+kbasep_hwcnt_backend_jm_dump_bytes(const struct kbase_hwcnt_gpu_info *gpu_info) - { -- WARN_ON(!v5_info); -- return (2 + v5_info->l2_count + fls64(v5_info->core_mask)) * -- KBASE_HWCNT_V5_VALUES_PER_BLOCK * -- KBASE_HWCNT_VALUE_BYTES; --} -+ WARN_ON(!gpu_info); - --int kbase_hwcnt_gpu_info_init( -- struct kbase_device *kbdev, -- struct kbase_hwcnt_gpu_info *info) --{ -- if (!kbdev || !info) -- return -EINVAL; -- --#ifdef CONFIG_MALI_NO_MALI -- /* NO_MALI uses V5 layout, regardless of the underlying platform. */ -- info->type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; -- info->v5.l2_count = KBASE_DUMMY_MODEL_MAX_MEMSYS_BLOCKS; -- info->v5.core_mask = (1ull << KBASE_DUMMY_MODEL_MAX_SHADER_CORES) - 1; --#else -- { -- const struct base_gpu_props *props = &kbdev->gpu_props.props; -- const size_t l2_count = props->l2_props.num_l2_slices; -- const size_t core_mask = -- props->coherency_info.group[0].core_mask; -- -- info->type = KBASE_HWCNT_GPU_GROUP_TYPE_V5; -- info->v5.l2_count = l2_count; -- info->v5.core_mask = core_mask; -- } --#endif -- return 0; -+ return (2 + gpu_info->l2_count + fls64(gpu_info->core_mask)) * -+ gpu_info->prfcnt_values_per_block * KBASE_HWCNT_VALUE_BYTES; - } - --int kbase_hwcnt_gpu_metadata_create( -- const struct kbase_hwcnt_gpu_info *info, -- bool use_secondary, -+int kbase_hwcnt_jm_metadata_create( -+ const struct kbase_hwcnt_gpu_info *gpu_info, -+ enum kbase_hwcnt_set counter_set, - const struct kbase_hwcnt_metadata **out_metadata, - size_t *out_dump_bytes) - { -@@ -320,23 +236,19 @@ int kbase_hwcnt_gpu_metadata_create( - const struct kbase_hwcnt_metadata *metadata; - size_t dump_bytes; - -- if (!info || !out_metadata || !out_dump_bytes) -+ if (!gpu_info || !out_metadata || !out_dump_bytes) - return -EINVAL; - -- switch (info->type) { -- case KBASE_HWCNT_GPU_GROUP_TYPE_V4: -- dump_bytes = kbasep_hwcnt_backend_gpu_v4_dump_bytes(&info->v4); -- errcode = kbasep_hwcnt_backend_gpu_metadata_v4_create( -- &info->v4, &metadata); -- break; -- case KBASE_HWCNT_GPU_GROUP_TYPE_V5: -- dump_bytes = kbasep_hwcnt_backend_gpu_v5_dump_bytes(&info->v5); -- errcode = kbasep_hwcnt_backend_gpu_metadata_v5_create( -- &info->v5, use_secondary, &metadata); -- break; -- default: -- return -EINVAL; -- } -+ /* -+ * For architectures where a max_config interface is available -+ * from the arbiter, the v5 dump bytes and the metadata v5 are -+ * based on the maximum possible allocation of the HW in the -+ * GPU cause it needs to be prepared for the worst case where -+ * all the available L2 cache and Shader cores are allocated. -+ */ -+ dump_bytes = kbasep_hwcnt_backend_jm_dump_bytes(gpu_info); -+ errcode = kbasep_hwcnt_backend_gpu_metadata_create( -+ gpu_info, false, counter_set, &metadata); - if (errcode) - return errcode; - -@@ -351,9 +263,37 @@ int kbase_hwcnt_gpu_metadata_create( - - return 0; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_metadata_create); - --void kbase_hwcnt_gpu_metadata_destroy( -+void kbase_hwcnt_jm_metadata_destroy(const struct kbase_hwcnt_metadata *metadata) -+{ -+ if (!metadata) -+ return; -+ -+ kbase_hwcnt_metadata_destroy(metadata); -+} -+ -+int kbase_hwcnt_csf_metadata_create( -+ const struct kbase_hwcnt_gpu_info *gpu_info, -+ enum kbase_hwcnt_set counter_set, -+ const struct kbase_hwcnt_metadata **out_metadata) -+{ -+ int errcode; -+ const struct kbase_hwcnt_metadata *metadata; -+ -+ if (!gpu_info || !out_metadata) -+ return -EINVAL; -+ -+ errcode = kbasep_hwcnt_backend_gpu_metadata_create( -+ gpu_info, true, counter_set, &metadata); -+ if (errcode) -+ return errcode; -+ -+ *out_metadata = metadata; -+ -+ return 0; -+} -+ -+void kbase_hwcnt_csf_metadata_destroy( - const struct kbase_hwcnt_metadata *metadata) - { - if (!metadata) -@@ -361,7 +301,127 @@ void kbase_hwcnt_gpu_metadata_destroy( - - kbase_hwcnt_metadata_destroy(metadata); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_metadata_destroy); -+ -+int kbase_hwcnt_gpu_metadata_create_truncate_64( -+ const struct kbase_hwcnt_metadata **dst_md, -+ const struct kbase_hwcnt_metadata *src_md) -+{ -+ struct kbase_hwcnt_description desc; -+ struct kbase_hwcnt_group_description group; -+ struct kbase_hwcnt_block_description -+ blks[KBASE_HWCNT_V5_BLOCK_TYPE_COUNT]; -+ size_t prfcnt_values_per_block; -+ size_t blk; -+ -+ if (!dst_md || !src_md || !src_md->grp_metadata || -+ !src_md->grp_metadata[0].blk_metadata) -+ return -EINVAL; -+ -+ /* Only support 1 group count and KBASE_HWCNT_V5_BLOCK_TYPE_COUNT block -+ * count in the metadata. -+ */ -+ if ((kbase_hwcnt_metadata_group_count(src_md) != 1) || -+ (kbase_hwcnt_metadata_block_count(src_md, 0) != -+ KBASE_HWCNT_V5_BLOCK_TYPE_COUNT)) -+ return -EINVAL; -+ -+ /* Get the values count in the first block. */ -+ prfcnt_values_per_block = -+ kbase_hwcnt_metadata_block_values_count(src_md, 0, 0); -+ -+ /* check all blocks should have same values count. */ -+ for (blk = 0; blk < KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; blk++) { -+ size_t val_cnt = -+ kbase_hwcnt_metadata_block_values_count(src_md, 0, blk); -+ if (val_cnt != prfcnt_values_per_block) -+ return -EINVAL; -+ } -+ -+ /* Only support 64 and 128 entries per block. */ -+ if ((prfcnt_values_per_block != 64) && (prfcnt_values_per_block != 128)) -+ return -EINVAL; -+ -+ if (prfcnt_values_per_block == 64) { -+ /* If the values per block is 64, no need to truncate. */ -+ *dst_md = NULL; -+ return 0; -+ } -+ -+ /* Truncate from 128 to 64 entries per block to keep API backward -+ * compatibility. -+ */ -+ prfcnt_values_per_block = 64; -+ -+ for (blk = 0; blk < KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; blk++) { -+ blks[blk].type = -+ kbase_hwcnt_metadata_block_type(src_md, 0, blk); -+ blks[blk].inst_cnt = kbase_hwcnt_metadata_block_instance_count( -+ src_md, 0, blk); -+ blks[blk].hdr_cnt = kbase_hwcnt_metadata_block_headers_count( -+ src_md, 0, blk); -+ blks[blk].ctr_cnt = prfcnt_values_per_block - blks[blk].hdr_cnt; -+ } -+ -+ group.type = kbase_hwcnt_metadata_group_type(src_md, 0); -+ group.blk_cnt = KBASE_HWCNT_V5_BLOCK_TYPE_COUNT; -+ group.blks = blks; -+ -+ desc.grp_cnt = kbase_hwcnt_metadata_group_count(src_md); -+ desc.avail_mask = src_md->avail_mask; -+ desc.clk_cnt = src_md->clk_cnt; -+ desc.grps = &group; -+ -+ return kbase_hwcnt_metadata_create(&desc, dst_md); -+} -+ -+void kbase_hwcnt_dump_buffer_copy_strict_narrow( -+ struct kbase_hwcnt_dump_buffer *dst, -+ const struct kbase_hwcnt_dump_buffer *src, -+ const struct kbase_hwcnt_enable_map *dst_enable_map) -+{ -+ const struct kbase_hwcnt_metadata *metadata; -+ size_t grp, blk, blk_inst; -+ size_t clk; -+ -+ if (WARN_ON(!dst) || WARN_ON(!src) || WARN_ON(!dst_enable_map) || -+ WARN_ON(dst == src) || WARN_ON(dst->metadata == src->metadata) || -+ WARN_ON(dst->metadata->grp_cnt != src->metadata->grp_cnt) || -+ WARN_ON(src->metadata->grp_cnt != 1) || -+ WARN_ON(dst->metadata->grp_metadata[0].blk_cnt != -+ src->metadata->grp_metadata[0].blk_cnt) || -+ WARN_ON(dst->metadata->grp_metadata[0].blk_cnt != 4) || -+ WARN_ON(dst->metadata->grp_metadata[0].blk_metadata[0].ctr_cnt > -+ src->metadata->grp_metadata[0].blk_metadata[0].ctr_cnt)) -+ return; -+ -+ /* Don't use src metadata since src buffer is bigger than dst buffer. */ -+ metadata = dst->metadata; -+ -+ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { -+ u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( -+ dst, grp, blk, blk_inst); -+ const u32 *src_blk = kbase_hwcnt_dump_buffer_block_instance( -+ src, grp, blk, blk_inst); -+ const u64 *blk_em = kbase_hwcnt_enable_map_block_instance( -+ dst_enable_map, grp, blk, blk_inst); -+ size_t val_cnt = kbase_hwcnt_metadata_block_values_count( -+ metadata, grp, blk); -+ /* Align upwards to include padding bytes */ -+ val_cnt = KBASE_HWCNT_ALIGN_UPWARDS( -+ val_cnt, (KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT / -+ KBASE_HWCNT_VALUE_BYTES)); -+ -+ kbase_hwcnt_dump_buffer_block_copy_strict(dst_blk, src_blk, -+ blk_em, val_cnt); -+ } -+ -+ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { -+ bool clk_enabled = kbase_hwcnt_clk_enable_map_enabled( -+ dst_enable_map->clk_enable_map, clk); -+ -+ dst->clk_cnt_buf[clk] = clk_enabled ? src->clk_cnt_buf[clk] : 0; -+ } -+} - - static bool is_block_type_shader( - const u64 grp_type, -@@ -370,44 +430,53 @@ static bool is_block_type_shader( - { - bool is_shader = false; - -+ /* Warn on unknown group type */ -+ if (WARN_ON(grp_type != KBASE_HWCNT_GPU_GROUP_TYPE_V5)) -+ return false; -+ -+ if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC || -+ blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2 || -+ blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC3) -+ is_shader = true; -+ -+ return is_shader; -+} -+ -+static bool is_block_type_l2_cache( -+ const u64 grp_type, -+ const u64 blk_type) -+{ -+ bool is_l2_cache = false; -+ - switch (grp_type) { -- case KBASE_HWCNT_GPU_GROUP_TYPE_V4: -- /* blk-value in [0, KBASE_HWCNT_V4_SC_BLOCKS_PER_GROUP-1] -- * corresponds to a shader, or its implementation -- * reserved. As such, here we use the blk index value to -- * tell the reserved case. -- */ -- if (blk_type == KBASE_HWCNT_GPU_V4_BLOCK_TYPE_SHADER || -- (blk < KBASE_HWCNT_V4_SC_BLOCKS_PER_GROUP && -- blk_type == KBASE_HWCNT_GPU_V4_BLOCK_TYPE_RESERVED)) -- is_shader = true; -- break; - case KBASE_HWCNT_GPU_GROUP_TYPE_V5: -- if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC || -- blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2) -- is_shader = true; -+ if (blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS || -+ blk_type == KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2) -+ is_l2_cache = true; - break; - default: - /* Warn on unknown group type */ - WARN_ON(true); - } - -- return is_shader; -+ return is_l2_cache; - } - --int kbase_hwcnt_gpu_dump_get( -- struct kbase_hwcnt_dump_buffer *dst, -- void *src, -- const struct kbase_hwcnt_enable_map *dst_enable_map, -- u64 pm_core_mask, -- bool accumulate) -+int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, void *src, -+ const struct kbase_hwcnt_enable_map *dst_enable_map, -+ u64 pm_core_mask, -+ const struct kbase_hwcnt_curr_config *curr_config, -+ bool accumulate) - { - const struct kbase_hwcnt_metadata *metadata; - const u32 *dump_src; - size_t src_offset, grp, blk, blk_inst; -- size_t grp_prev = 0; - u64 core_mask = pm_core_mask; - -+ /* Variables to deal with the current configuration */ -+ int l2_count = 0; -+ bool hw_res_available = true; -+ - if (!dst || !src || !dst_enable_map || - (dst_enable_map->metadata != dst->metadata)) - return -EINVAL; -@@ -429,27 +498,43 @@ int kbase_hwcnt_gpu_dump_get( - const bool is_shader_core = is_block_type_shader( - kbase_hwcnt_metadata_group_type(metadata, grp), - blk_type, blk); -+ const bool is_l2_cache = is_block_type_l2_cache( -+ kbase_hwcnt_metadata_group_type(metadata, grp), -+ blk_type); - -- if (grp != grp_prev) { -- /* grp change would only happen with V4. V5 and -- * further are envisaged to be single group -- * scenario only. Here needs to drop the lower -- * group core-mask by shifting right with -- * KBASE_HWCNT_V4_SC_BLOCKS_PER_GROUP. -- */ -- core_mask = pm_core_mask >> -- KBASE_HWCNT_V4_SC_BLOCKS_PER_GROUP; -- grp_prev = grp; -+ /* -+ * If l2 blocks is greater than the current allocated number of -+ * L2 slices, there is no hw allocated to that block. -+ */ -+ if (is_l2_cache) { -+ l2_count++; -+ if (l2_count > curr_config->num_l2_slices) -+ hw_res_available = false; -+ else -+ hw_res_available = true; -+ } -+ /* -+ * For the shader cores, the current shader_mask allocated is -+ * always a subgroup of the maximum shader_mask, so after -+ * jumping any L2 cache not available the available shader cores -+ * will always have a matching set of blk instances available to -+ * accumulate them. -+ */ -+ else { -+ hw_res_available = true; - } - -- /* Early out if no values in the dest block are enabled */ -+ /* -+ * Early out if no values in the dest block are enabled or if -+ * the resource target of the block is not available in the HW. -+ */ - if (kbase_hwcnt_enable_map_block_enabled( - dst_enable_map, grp, blk, blk_inst)) { - u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( - dst, grp, blk, blk_inst); - const u32 *src_blk = dump_src + src_offset; - -- if (!is_shader_core || (core_mask & 1)) { -+ if ((!is_shader_core || (core_mask & 1)) && hw_res_available) { - if (accumulate) { - kbase_hwcnt_dump_buffer_block_accumulate( - dst_blk, src_blk, hdr_cnt, -@@ -465,14 +550,60 @@ int kbase_hwcnt_gpu_dump_get( - } - } - -- src_offset += (hdr_cnt + ctr_cnt); -+ /* Just increase the src_offset if the HW is available */ -+ if (hw_res_available) -+ src_offset += (hdr_cnt + ctr_cnt); - if (is_shader_core) - core_mask = core_mask >> 1; - } - - return 0; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_dump_get); -+ -+int kbase_hwcnt_csf_dump_get(struct kbase_hwcnt_dump_buffer *dst, void *src, -+ const struct kbase_hwcnt_enable_map *dst_enable_map, -+ bool accumulate) -+{ -+ const struct kbase_hwcnt_metadata *metadata; -+ const u32 *dump_src; -+ size_t src_offset, grp, blk, blk_inst; -+ -+ if (!dst || !src || !dst_enable_map || -+ (dst_enable_map->metadata != dst->metadata)) -+ return -EINVAL; -+ -+ metadata = dst->metadata; -+ dump_src = (const u32 *)src; -+ src_offset = 0; -+ -+ kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) { -+ const size_t hdr_cnt = kbase_hwcnt_metadata_block_headers_count( -+ metadata, grp, blk); -+ const size_t ctr_cnt = -+ kbase_hwcnt_metadata_block_counters_count(metadata, grp, -+ blk); -+ -+ /* Early out if no values in the dest block are enabled */ -+ if (kbase_hwcnt_enable_map_block_enabled(dst_enable_map, grp, -+ blk, blk_inst)) { -+ u32 *dst_blk = kbase_hwcnt_dump_buffer_block_instance( -+ dst, grp, blk, blk_inst); -+ const u32 *src_blk = dump_src + src_offset; -+ -+ if (accumulate) { -+ kbase_hwcnt_dump_buffer_block_accumulate( -+ dst_blk, src_blk, hdr_cnt, ctr_cnt); -+ } else { -+ kbase_hwcnt_dump_buffer_block_copy( -+ dst_blk, src_blk, (hdr_cnt + ctr_cnt)); -+ } -+ } -+ -+ src_offset += (hdr_cnt + ctr_cnt); -+ } -+ -+ return 0; -+} - - /** - * kbasep_hwcnt_backend_gpu_block_map_to_physical() - Convert from a block -@@ -563,7 +694,7 @@ void kbase_hwcnt_gpu_enable_map_to_physical( - { - const struct kbase_hwcnt_metadata *metadata; - -- u64 jm_bm = 0; -+ u64 fe_bm = 0; - u64 shader_bm = 0; - u64 tiler_bm = 0; - u64 mmu_l2_bm = 0; -@@ -581,45 +712,26 @@ void kbase_hwcnt_gpu_enable_map_to_physical( - metadata, grp); - const u64 blk_type = kbase_hwcnt_metadata_block_type( - metadata, grp, blk); -- const size_t blk_val_cnt = -- kbase_hwcnt_metadata_block_values_count( -- metadata, grp, blk); - const u64 *blk_map = kbase_hwcnt_enable_map_block_instance( - src, grp, blk, blk_inst); - -- switch ((enum kbase_hwcnt_gpu_group_type)grp_type) { -- case KBASE_HWCNT_GPU_GROUP_TYPE_V4: -- WARN_ON(blk_val_cnt != KBASE_HWCNT_V4_VALUES_PER_BLOCK); -- switch ((enum kbase_hwcnt_gpu_v4_block_type)blk_type) { -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_SHADER: -- shader_bm |= *blk_map; -- break; -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_TILER: -- tiler_bm |= *blk_map; -- break; -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_MMU_L2: -- mmu_l2_bm |= *blk_map; -- break; -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_JM: -- jm_bm |= *blk_map; -- break; -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_RESERVED: -- break; -- default: -- WARN_ON(true); -- } -- break; -- case KBASE_HWCNT_GPU_GROUP_TYPE_V5: -- WARN_ON(blk_val_cnt != KBASE_HWCNT_V5_VALUES_PER_BLOCK); -+ if ((enum kbase_hwcnt_gpu_group_type)grp_type == -+ KBASE_HWCNT_GPU_GROUP_TYPE_V5) { - switch ((enum kbase_hwcnt_gpu_v5_block_type)blk_type) { -- case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM: -- jm_bm |= *blk_map; -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED: -+ /* Nothing to do in this case. */ -+ break; -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE: -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE2: -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE3: -+ fe_bm |= *blk_map; - break; - case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: - tiler_bm |= *blk_map; - break; - case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: - case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC3: - shader_bm |= *blk_map; - break; - case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: -@@ -629,14 +741,13 @@ void kbase_hwcnt_gpu_enable_map_to_physical( - default: - WARN_ON(true); - } -- break; -- default: -+ } else { - WARN_ON(true); - } - } - -- dst->jm_bm = -- kbasep_hwcnt_backend_gpu_block_map_to_physical(jm_bm, 0); -+ dst->fe_bm = -+ kbasep_hwcnt_backend_gpu_block_map_to_physical(fe_bm, 0); - dst->shader_bm = - kbasep_hwcnt_backend_gpu_block_map_to_physical(shader_bm, 0); - dst->tiler_bm = -@@ -644,7 +755,24 @@ void kbase_hwcnt_gpu_enable_map_to_physical( - dst->mmu_l2_bm = - kbasep_hwcnt_backend_gpu_block_map_to_physical(mmu_l2_bm, 0); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_enable_map_to_physical); -+ -+void kbase_hwcnt_gpu_set_to_physical(enum kbase_hwcnt_physical_set *dst, -+ enum kbase_hwcnt_set src) -+{ -+ switch (src) { -+ case KBASE_HWCNT_SET_PRIMARY: -+ *dst = KBASE_HWCNT_PHYSICAL_SET_PRIMARY; -+ break; -+ case KBASE_HWCNT_SET_SECONDARY: -+ *dst = KBASE_HWCNT_PHYSICAL_SET_SECONDARY; -+ break; -+ case KBASE_HWCNT_SET_TERTIARY: -+ *dst = KBASE_HWCNT_PHYSICAL_SET_TERTIARY; -+ break; -+ default: -+ WARN_ON(true); -+ } -+} - - void kbase_hwcnt_gpu_enable_map_from_physical( - struct kbase_hwcnt_enable_map *dst, -@@ -653,7 +781,7 @@ void kbase_hwcnt_gpu_enable_map_from_physical( - const struct kbase_hwcnt_metadata *metadata; - - u64 ignored_hi; -- u64 jm_bm; -+ u64 fe_bm; - u64 shader_bm; - u64 tiler_bm; - u64 mmu_l2_bm; -@@ -665,7 +793,7 @@ void kbase_hwcnt_gpu_enable_map_from_physical( - metadata = dst->metadata; - - kbasep_hwcnt_backend_gpu_block_map_from_physical( -- src->jm_bm, &jm_bm, &ignored_hi); -+ src->fe_bm, &fe_bm, &ignored_hi); - kbasep_hwcnt_backend_gpu_block_map_from_physical( - src->shader_bm, &shader_bm, &ignored_hi); - kbasep_hwcnt_backend_gpu_block_map_from_physical( -@@ -678,45 +806,26 @@ void kbase_hwcnt_gpu_enable_map_from_physical( - metadata, grp); - const u64 blk_type = kbase_hwcnt_metadata_block_type( - metadata, grp, blk); -- const size_t blk_val_cnt = -- kbase_hwcnt_metadata_block_values_count( -- metadata, grp, blk); - u64 *blk_map = kbase_hwcnt_enable_map_block_instance( - dst, grp, blk, blk_inst); - -- switch ((enum kbase_hwcnt_gpu_group_type)grp_type) { -- case KBASE_HWCNT_GPU_GROUP_TYPE_V4: -- WARN_ON(blk_val_cnt != KBASE_HWCNT_V4_VALUES_PER_BLOCK); -- switch ((enum kbase_hwcnt_gpu_v4_block_type)blk_type) { -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_SHADER: -- *blk_map = shader_bm; -- break; -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_TILER: -- *blk_map = tiler_bm; -- break; -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_MMU_L2: -- *blk_map = mmu_l2_bm; -- break; -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_JM: -- *blk_map = jm_bm; -- break; -- case KBASE_HWCNT_GPU_V4_BLOCK_TYPE_RESERVED: -- break; -- default: -- WARN_ON(true); -- } -- break; -- case KBASE_HWCNT_GPU_GROUP_TYPE_V5: -- WARN_ON(blk_val_cnt != KBASE_HWCNT_V5_VALUES_PER_BLOCK); -+ if ((enum kbase_hwcnt_gpu_group_type)grp_type == -+ KBASE_HWCNT_GPU_GROUP_TYPE_V5) { - switch ((enum kbase_hwcnt_gpu_v5_block_type)blk_type) { -- case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM: -- *blk_map = jm_bm; -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED: -+ /* Nothing to do in this case. */ -+ break; -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE: -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE2: -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE3: -+ *blk_map = fe_bm; - break; - case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: - *blk_map = tiler_bm; - break; - case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: - case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: -+ case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC3: - *blk_map = shader_bm; - break; - case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: -@@ -726,13 +835,11 @@ void kbase_hwcnt_gpu_enable_map_from_physical( - default: - WARN_ON(true); - } -- break; -- default: -+ } else { - WARN_ON(true); - } - } - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_enable_map_from_physical); - - void kbase_hwcnt_gpu_patch_dump_headers( - struct kbase_hwcnt_dump_buffer *buf, -@@ -758,16 +865,11 @@ void kbase_hwcnt_gpu_patch_dump_headers( - kbasep_hwcnt_backend_gpu_block_map_to_physical( - blk_map[0], 0); - -- switch ((enum kbase_hwcnt_gpu_group_type)grp_type) { -- case KBASE_HWCNT_GPU_GROUP_TYPE_V4: -- buf_blk[KBASE_HWCNT_V4_PRFCNT_EN_HEADER] = prfcnt_en; -- break; -- case KBASE_HWCNT_GPU_GROUP_TYPE_V5: -+ if ((enum kbase_hwcnt_gpu_group_type)grp_type == -+ KBASE_HWCNT_GPU_GROUP_TYPE_V5) { - buf_blk[KBASE_HWCNT_V5_PRFCNT_EN_HEADER] = prfcnt_en; -- break; -- default: -+ } else { - WARN_ON(true); - } - } - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_gpu_patch_dump_headers); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_gpu.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_gpu.h -index 12891e0..50ae80d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_gpu.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_gpu.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_HWCNT_GPU_H_ -@@ -30,125 +29,157 @@ struct kbase_hwcnt_metadata; - struct kbase_hwcnt_enable_map; - struct kbase_hwcnt_dump_buffer; - -+#define KBASE_HWCNT_V5_BLOCK_TYPE_COUNT 4 -+#define KBASE_HWCNT_V5_HEADERS_PER_BLOCK 4 -+#define KBASE_HWCNT_V5_DEFAULT_COUNTERS_PER_BLOCK 60 -+#define KBASE_HWCNT_V5_DEFAULT_VALUES_PER_BLOCK \ -+ (KBASE_HWCNT_V5_HEADERS_PER_BLOCK + \ -+ KBASE_HWCNT_V5_DEFAULT_COUNTERS_PER_BLOCK) -+/** Index of the PRFCNT_EN header into a V5 counter block */ -+#define KBASE_HWCNT_V5_PRFCNT_EN_HEADER 2 -+ - /** - * enum kbase_hwcnt_gpu_group_type - GPU hardware counter group types, used to - * identify metadata groups. -- * @KBASE_HWCNT_GPU_GROUP_TYPE_V4: GPU V4 group type. - * @KBASE_HWCNT_GPU_GROUP_TYPE_V5: GPU V5 group type. - */ - enum kbase_hwcnt_gpu_group_type { -- KBASE_HWCNT_GPU_GROUP_TYPE_V4 = 0x10, - KBASE_HWCNT_GPU_GROUP_TYPE_V5, - }; - --/** -- * enum kbase_hwcnt_gpu_v4_block_type - GPU V4 hardware counter block types, -- * used to identify metadata blocks. -- * @KBASE_HWCNT_GPU_V4_BLOCK_TYPE_SHADER: Shader block. -- * @KBASE_HWCNT_GPU_V4_BLOCK_TYPE_TILER: Tiler block. -- * @KBASE_HWCNT_GPU_V4_BLOCK_TYPE_MMU_L2: MMU/L2 block. -- * @KBASE_HWCNT_GPU_V4_BLOCK_TYPE_JM: Job Manager block. -- * @KBASE_HWCNT_GPU_V4_BLOCK_TYPE_RESERVED: Reserved block. -- */ --enum kbase_hwcnt_gpu_v4_block_type { -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_SHADER = 0x20, -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_TILER, -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_MMU_L2, -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_JM, -- KBASE_HWCNT_GPU_V4_BLOCK_TYPE_RESERVED, --}; -- - /** - * enum kbase_hwcnt_gpu_v5_block_type - GPU V5 hardware counter block types, - * used to identify metadata blocks. -- * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM: Job Manager block. -- * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: Tiler block. -- * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: Shader Core block. -- * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: Secondary Shader Core block. -- * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: Memsys block. -- * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: Secondary Memsys block. -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED: Undefined block (e.g. if a -+ * counter set that a block -+ * doesn't support is used). -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE: Front End block (Job manager -+ * or CSF HW). -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE2: Secondary Front End block (Job -+ * manager or CSF HW). -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE3: Tertiary Front End block (Job -+ * manager or CSF HW). -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: Tiler block. -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: Shader Core block. -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: Secondary Shader Core block. -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC3: Tertiary Shader Core block. -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: Memsys block. -+ * @KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: Secondary Memsys block. - */ - enum kbase_hwcnt_gpu_v5_block_type { -- KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_JM = 0x40, -+ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED, -+ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE, -+ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE2, -+ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE3, - KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER, - KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC, - KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2, -+ KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC3, - KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS, - KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2, - }; - -+/** -+ * enum kbase_hwcnt_set - GPU hardware counter sets -+ * @KBASE_HWCNT_SET_PRIMARY: The Primary set of counters -+ * @KBASE_HWCNT_SET_SECONDARY: The Secondary set of counters -+ * @KBASE_HWCNT_SET_TERTIARY: The Tertiary set of counters -+ */ -+enum kbase_hwcnt_set { -+ KBASE_HWCNT_SET_PRIMARY, -+ KBASE_HWCNT_SET_SECONDARY, -+ KBASE_HWCNT_SET_TERTIARY, -+}; -+ - /** - * struct kbase_hwcnt_physical_enable_map - Representation of enable map - * directly used by GPU. -- * @jm_bm: Job Manager counters selection bitmask. -+ * @fe_bm: Front end (JM/CSHW) counters selection bitmask. - * @shader_bm: Shader counters selection bitmask. - * @tiler_bm: Tiler counters selection bitmask. - * @mmu_l2_bm: MMU_L2 counters selection bitmask. - */ - struct kbase_hwcnt_physical_enable_map { -- u32 jm_bm; -+ u32 fe_bm; - u32 shader_bm; - u32 tiler_bm; - u32 mmu_l2_bm; - }; - --/** -- * struct kbase_hwcnt_gpu_v4_info - Information about hwcnt blocks on v4 GPUs. -- * @cg_count: Core group count. -- * @cgs: Non-NULL pointer to array of cg_count coherent group structures. -- * -- * V4 devices are Mali-T6xx or Mali-T72x, and have one or more core groups, -- * where each core group may have a physically different layout. -+/* -+ * Values for Hardware Counter SET_SELECT value. -+ * Directly passed to HW. - */ --struct kbase_hwcnt_gpu_v4_info { -- size_t cg_count; -- const struct mali_base_gpu_coherent_group *cgs; -+enum kbase_hwcnt_physical_set { -+ KBASE_HWCNT_PHYSICAL_SET_PRIMARY = 0, -+ KBASE_HWCNT_PHYSICAL_SET_SECONDARY = 1, -+ KBASE_HWCNT_PHYSICAL_SET_TERTIARY = 2, - }; - - /** -- * struct kbase_hwcnt_gpu_v5_info - Information about hwcnt blocks on v5 GPUs. -- * @l2_count: L2 cache count. -- * @core_mask: Shader core mask. May be sparse. -+ * struct kbase_hwcnt_gpu_info - Information about hwcnt blocks on the GPUs. -+ * @l2_count: L2 cache count. -+ * @core_mask: Shader core mask. May be sparse. -+ * @clk_cnt: Number of clock domains available. -+ * @prfcnt_values_per_block: Total entries (header + counters) of performance -+ * counter per block. - */ --struct kbase_hwcnt_gpu_v5_info { -+struct kbase_hwcnt_gpu_info { - size_t l2_count; - u64 core_mask; -+ u8 clk_cnt; -+ size_t prfcnt_values_per_block; - }; - - /** -- * struct kbase_hwcnt_gpu_info - Tagged union with information about the current -- * GPU's hwcnt blocks. -- * @type: GPU type. -- * @v4: Info filled in if a v4 GPU. -- * @v5: Info filled in if a v5 GPU. -- */ --struct kbase_hwcnt_gpu_info { -- enum kbase_hwcnt_gpu_group_type type; -- union { -- struct kbase_hwcnt_gpu_v4_info v4; -- struct kbase_hwcnt_gpu_v5_info v5; -- }; --}; -- --/** -- * kbase_hwcnt_gpu_info_init() - Initialise an info structure used to create the -- * hwcnt metadata. -- * @kbdev: Non-NULL pointer to kbase device. -- * @info: Non-NULL pointer to data structure to be filled in. -+ * struct kbase_hwcnt_curr_config - Current Configuration of HW allocated to the -+ * GPU. -+ * @num_l2_slices: Current number of L2 slices allocated to the GPU. -+ * @shader_present: Current shader present bitmap that is allocated to the GPU. - * -- * The initialised info struct will only be valid for use while kbdev is valid. -+ * For architectures with the max_config interface available from the Arbiter, -+ * the current resources allocated may change during runtime due to a -+ * re-partitioning (possible with partition manager). Thus, the HWC needs to be -+ * prepared to report any possible set of counters. For this reason the memory -+ * layout in the userspace is based on the maximum possible allocation. On the -+ * other hand, each partition has just the view of its currently allocated -+ * resources. Therefore, it is necessary to correctly map the dumped HWC values -+ * from the registers into this maximum memory layout so that it can be exposed -+ * to the userspace side correctly. -+ * -+ * For L2 cache just the number is enough once the allocated ones will be -+ * accumulated on the first L2 slots available in the destination buffer. -+ * -+ * For the correct mapping of the shader cores it is necessary to jump all the -+ * L2 cache slots in the destination buffer that are not allocated. But, it is -+ * not necessary to add any logic to map the shader cores bitmap into the memory -+ * layout because the shader_present allocated will always be a subset of the -+ * maximum shader_present. It is possible because: -+ * 1 - Partitions are made of slices and they are always ordered from the ones -+ * with more shader cores to the ones with less. -+ * 2 - The shader cores in a slice are always contiguous. -+ * 3 - A partition can only have a contiguous set of slices allocated to it. -+ * So, for example, if 4 slices are available in total, 1 with 4 cores, 2 with -+ * 3 cores and 1 with 2 cores. The maximum possible shader_present would be: -+ * 0x0011|0111|0111|1111 -> note the order and that the shader cores are -+ * contiguous in any slice. -+ * Supposing that a partition takes the two slices in the middle, the current -+ * config shader_present for this partition would be: -+ * 0x0111|0111 -> note that this is a subset of the maximum above and the slices -+ * are contiguous. -+ * Therefore, by directly copying any subset of the maximum possible -+ * shader_present the mapping is already achieved. - */ --int kbase_hwcnt_gpu_info_init( -- struct kbase_device *kbdev, -- struct kbase_hwcnt_gpu_info *info); -+struct kbase_hwcnt_curr_config { -+ size_t num_l2_slices; -+ u64 shader_present; -+}; - - /** -- * kbase_hwcnt_gpu_metadata_create() - Create hardware counter metadata for the -- * current GPU. -- * @info: Non-NULL pointer to info struct initialised by -- * kbase_hwcnt_gpu_info_init. -- * @use_secondary: True if secondary performance counters should be used, else -- * false. Ignored if secondary counters are not supported. -+ * kbase_hwcnt_jm_metadata_create() - Create hardware counter metadata for the -+ * JM GPUs. -+ * @info: Non-NULL pointer to info struct. -+ * @counter_set: The performance counter set used. - * @out_metadata: Non-NULL pointer to where created metadata is stored on - * success. - * @out_dump_bytes: Non-NULL pointer to where the size of the GPU counter dump -@@ -156,44 +187,133 @@ int kbase_hwcnt_gpu_info_init( - * - * Return: 0 on success, else error code. - */ --int kbase_hwcnt_gpu_metadata_create( -+int kbase_hwcnt_jm_metadata_create( - const struct kbase_hwcnt_gpu_info *info, -- bool use_secondary, -+ enum kbase_hwcnt_set counter_set, - const struct kbase_hwcnt_metadata **out_metadata, - size_t *out_dump_bytes); - - /** -- * kbase_hwcnt_gpu_metadata_destroy() - Destroy GPU hardware counter metadata. -+ * kbase_hwcnt_jm_metadata_destroy() - Destroy JM GPU hardware counter metadata. -+ * -+ * @metadata: Pointer to metadata to destroy. -+ */ -+void kbase_hwcnt_jm_metadata_destroy( -+ const struct kbase_hwcnt_metadata *metadata); -+ -+/** -+ * kbase_hwcnt_csf_metadata_create() - Create hardware counter metadata for the -+ * CSF GPUs. -+ * @info: Non-NULL pointer to info struct. -+ * @counter_set: The performance counter set used. -+ * @out_metadata: Non-NULL pointer to where created metadata is stored on -+ * success. -+ * -+ * Return: 0 on success, else error code. -+ */ -+int kbase_hwcnt_csf_metadata_create( -+ const struct kbase_hwcnt_gpu_info *info, -+ enum kbase_hwcnt_set counter_set, -+ const struct kbase_hwcnt_metadata **out_metadata); -+ -+/** -+ * kbase_hwcnt_csf_metadata_destroy() - Destroy CSF GPU hardware counter -+ * metadata. - * @metadata: Pointer to metadata to destroy. - */ --void kbase_hwcnt_gpu_metadata_destroy( -+void kbase_hwcnt_csf_metadata_destroy( - const struct kbase_hwcnt_metadata *metadata); - - /** -- * kbase_hwcnt_gpu_dump_get() - Copy or accumulate enabled counters from the raw -+ * kbase_hwcnt_gpu_metadata_create_truncate_64() - Create HWC metadata with HWC -+ * block entries truncated -+ * to 64. -+ * -+ * @dst_md: Non-NULL pointer to where created metadata is stored on success. -+ * @src_md: Non-NULL pointer to the HWC metadata used as the source to create -+ * dst_md. -+ * -+ * If the total block entries in src_md is 64, metadata dst_md returns NULL -+ * since no need to truncate. -+ * if the total block entries in src_md is 128, then a new metadata with block -+ * entries truncated to 64 will be created for dst_md, which keeps the interface -+ * to user clients backward compatible. -+ * If the total block entries in src_md is other values, function returns error -+ * since it's not supported. -+ * -+ * Return: 0 on success, else error code. -+ */ -+int kbase_hwcnt_gpu_metadata_create_truncate_64( -+ const struct kbase_hwcnt_metadata **dst_md, -+ const struct kbase_hwcnt_metadata *src_md); -+ -+/** -+ * kbase_hwcnt_dump_buffer_copy_strict_narrow() - Copy all enabled values from -+ * src to dst. -+ * -+ * @dst: Non-NULL pointer to dst dump buffer. -+ * @src: Non-NULL pointer to src dump buffer. -+ * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. -+ * -+ * After the operation, all non-enabled values (including padding bytes) will be -+ * zero. -+ * -+ * The dst and src have different metadata, and the dst metadata is narrower -+ * than src metadata. -+ */ -+void kbase_hwcnt_dump_buffer_copy_strict_narrow( -+ struct kbase_hwcnt_dump_buffer *dst, -+ const struct kbase_hwcnt_dump_buffer *src, -+ const struct kbase_hwcnt_enable_map *dst_enable_map); -+ -+/** -+ * kbase_hwcnt_jm_dump_get() - Copy or accumulate enabled counters from the raw -+ * dump buffer in src into the dump buffer -+ * abstraction in dst. -+ * @dst: Non-NULL pointer to dst dump buffer. -+ * @src: Non-NULL pointer to src raw dump buffer, of same length -+ * as returned in out_dump_bytes parameter of -+ * kbase_hwcnt_jm_metadata_create. -+ * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. -+ * @pm_core_mask: PM state synchronized shaders core mask with the dump. -+ * @curr_config: Current allocated hardware resources to correctly map the -+ * src raw dump buffer to the dst dump buffer. -+ * @accumulate: True if counters in src should be accumulated into dst, -+ * rather than copied. -+ * -+ * The dst and dst_enable_map MUST have been created from the same metadata as -+ * returned from the call to kbase_hwcnt_jm_metadata_create as was used to get -+ * the length of src. -+ * -+ * Return: 0 on success, else error code. -+ */ -+int kbase_hwcnt_jm_dump_get(struct kbase_hwcnt_dump_buffer *dst, void *src, -+ const struct kbase_hwcnt_enable_map *dst_enable_map, -+ const u64 pm_core_mask, -+ const struct kbase_hwcnt_curr_config *curr_config, -+ bool accumulate); -+ -+/** -+ * kbase_hwcnt_csf_dump_get() - Copy or accumulate enabled counters from the raw - * dump buffer in src into the dump buffer - * abstraction in dst. - * @dst: Non-NULL pointer to dst dump buffer. - * @src: Non-NULL pointer to src raw dump buffer, of same length - * as returned in out_dump_bytes parameter of -- * kbase_hwcnt_gpu_metadata_create. -+ * kbase_hwcnt_csf_metadata_create. - * @dst_enable_map: Non-NULL pointer to enable map specifying enabled values. -- * @pm_core_mask: PM state synchronized shaders core mask with the dump. - * @accumulate: True if counters in src should be accumulated into dst, - * rather than copied. - * - * The dst and dst_enable_map MUST have been created from the same metadata as -- * returned from the call to kbase_hwcnt_gpu_metadata_create as was used to get -+ * returned from the call to kbase_hwcnt_csf_metadata_create as was used to get - * the length of src. - * - * Return: 0 on success, else error code. - */ --int kbase_hwcnt_gpu_dump_get( -- struct kbase_hwcnt_dump_buffer *dst, -- void *src, -- const struct kbase_hwcnt_enable_map *dst_enable_map, -- const u64 pm_core_mask, -- bool accumulate); -+int kbase_hwcnt_csf_dump_get(struct kbase_hwcnt_dump_buffer *dst, void *src, -+ const struct kbase_hwcnt_enable_map *dst_enable_map, -+ bool accumulate); - - /** - * kbase_hwcnt_gpu_enable_map_to_physical() - Convert an enable map abstraction -@@ -202,7 +322,7 @@ int kbase_hwcnt_gpu_dump_get( - * @src: Non-NULL pointer to src enable map abstraction. - * - * The src must have been created from a metadata returned from a call to -- * kbase_hwcnt_gpu_metadata_create. -+ * kbase_hwcnt_jm_metadata_create or kbase_hwcnt_csf_metadata_create. - * - * This is a lossy conversion, as the enable map abstraction has one bit per - * individual counter block value, but the physical enable map uses 1 bit for -@@ -212,6 +332,16 @@ void kbase_hwcnt_gpu_enable_map_to_physical( - struct kbase_hwcnt_physical_enable_map *dst, - const struct kbase_hwcnt_enable_map *src); - -+/** -+ * kbase_hwcnt_gpu_set_to_physical() - Map counter set selection to physical -+ * SET_SELECT value. -+ * -+ * @dst: Non-NULL pointer to dst physical SET_SELECT value. -+ * @src: Non-NULL pointer to src counter set selection. -+ */ -+void kbase_hwcnt_gpu_set_to_physical(enum kbase_hwcnt_physical_set *dst, -+ enum kbase_hwcnt_set src); -+ - /** - * kbase_hwcnt_gpu_enable_map_from_physical() - Convert a physical enable map to - * an enable map abstraction. -@@ -219,7 +349,7 @@ void kbase_hwcnt_gpu_enable_map_to_physical( - * @src: Non-NULL pointer to src physical enable map. - * - * The dst must have been created from a metadata returned from a call to -- * kbase_hwcnt_gpu_metadata_create. -+ * kbase_hwcnt_jm_metadata_create or kbase_hwcnt_csf_metadata_create. - * - * This is a lossy conversion, as the physical enable map can technically - * support counter blocks with 128 counters each, but no hardware actually uses -@@ -238,7 +368,7 @@ void kbase_hwcnt_gpu_enable_map_from_physical( - * @enable_map: Non-NULL pointer to enable map. - * - * The buf and enable_map must have been created from a metadata returned from -- * a call to kbase_hwcnt_gpu_metadata_create. -+ * a call to kbase_hwcnt_jm_metadata_create or kbase_hwcnt_csf_metadata_create. - * - * This function should be used before handing off a dump buffer over the - * kernel-user boundary, to ensure the header is accurate for the enable map -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_legacy.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_legacy.c -index b0e6aee..0687253 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_legacy.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_legacy.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,13 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase_hwcnt_legacy.h" - #include "mali_kbase_hwcnt_virtualizer.h" - #include "mali_kbase_hwcnt_types.h" - #include "mali_kbase_hwcnt_gpu.h" --#include "mali_kbase_ioctl.h" -+#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h> - - #include <linux/slab.h> - #include <linux/uaccess.h> -@@ -69,7 +68,7 @@ int kbase_hwcnt_legacy_client_create( - goto error; - - /* Translate from the ioctl enable map to the internal one */ -- phys_em.jm_bm = enable->jm_bm; -+ phys_em.fe_bm = enable->fe_bm; - phys_em.shader_bm = enable->shader_bm; - phys_em.tiler_bm = enable->tiler_bm; - phys_em.mmu_l2_bm = enable->mmu_l2_bm; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_legacy.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_legacy.h -index 7a610ae..36ff44d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_legacy.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_legacy.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_types.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_types.c -index 1e9efde..492f572 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_types.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_types.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,26 +17,11 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase_hwcnt_types.h" --#include "mali_kbase.h" -- --/* Minimum alignment of each block of hardware counters */ --#define KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT \ -- (KBASE_HWCNT_BITFIELD_BITS * KBASE_HWCNT_VALUE_BYTES) - --/** -- * KBASE_HWCNT_ALIGN_UPWARDS() - Align a value to an alignment. -- * @value: The value to align upwards. -- * @alignment: The alignment. -- * -- * Return: A number greater than or equal to value that is aligned to alignment. -- */ --#define KBASE_HWCNT_ALIGN_UPWARDS(value, alignment) \ -- (value + ((alignment - (value % alignment)) % alignment)) -+#include <linux/slab.h> - - int kbase_hwcnt_metadata_create( - const struct kbase_hwcnt_description *desc, -@@ -55,6 +41,10 @@ int kbase_hwcnt_metadata_create( - if (!desc || !out_metadata) - return -EINVAL; - -+ /* The maximum number of clock domains is 64. */ -+ if (desc->clk_cnt > (sizeof(u64) * BITS_PER_BYTE)) -+ return -EINVAL; -+ - /* Calculate the bytes needed to tightly pack the metadata */ - - /* Top level metadata */ -@@ -158,6 +148,7 @@ int kbase_hwcnt_metadata_create( - enable_map_count * KBASE_HWCNT_BITFIELD_BYTES; - metadata->dump_buf_bytes = dump_buf_count * KBASE_HWCNT_VALUE_BYTES; - metadata->avail_mask = desc->avail_mask; -+ metadata->clk_cnt = desc->clk_cnt; - - WARN_ON(size != offset); - /* Due to the block alignment, there should be exactly one enable map -@@ -170,13 +161,11 @@ int kbase_hwcnt_metadata_create( - *out_metadata = metadata; - return 0; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_metadata_create); - - void kbase_hwcnt_metadata_destroy(const struct kbase_hwcnt_metadata *metadata) - { - kfree(metadata); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_metadata_destroy); - - int kbase_hwcnt_enable_map_alloc( - const struct kbase_hwcnt_metadata *metadata, -@@ -187,45 +176,55 @@ int kbase_hwcnt_enable_map_alloc( - if (!metadata || !enable_map) - return -EINVAL; - -- enable_map_buf = kzalloc(metadata->enable_map_bytes, GFP_KERNEL); -- if (!enable_map_buf) -- return -ENOMEM; -+ if (metadata->enable_map_bytes > 0) { -+ enable_map_buf = -+ kzalloc(metadata->enable_map_bytes, GFP_KERNEL); -+ if (!enable_map_buf) -+ return -ENOMEM; -+ } else { -+ enable_map_buf = NULL; -+ } - - enable_map->metadata = metadata; -- enable_map->enable_map = enable_map_buf; -+ enable_map->hwcnt_enable_map = enable_map_buf; - return 0; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_enable_map_alloc); - - void kbase_hwcnt_enable_map_free(struct kbase_hwcnt_enable_map *enable_map) - { - if (!enable_map) - return; - -- kfree(enable_map->enable_map); -- enable_map->enable_map = NULL; -+ kfree(enable_map->hwcnt_enable_map); -+ enable_map->hwcnt_enable_map = NULL; - enable_map->metadata = NULL; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_enable_map_free); - - int kbase_hwcnt_dump_buffer_alloc( - const struct kbase_hwcnt_metadata *metadata, - struct kbase_hwcnt_dump_buffer *dump_buf) - { -- u32 *buf; -+ size_t dump_buf_bytes; -+ size_t clk_cnt_buf_bytes; -+ u8 *buf; - - if (!metadata || !dump_buf) - return -EINVAL; - -- buf = kmalloc(metadata->dump_buf_bytes, GFP_KERNEL); -+ dump_buf_bytes = metadata->dump_buf_bytes; -+ clk_cnt_buf_bytes = sizeof(*dump_buf->clk_cnt_buf) * metadata->clk_cnt; -+ -+ /* Make a single allocation for both dump_buf and clk_cnt_buf. */ -+ buf = kmalloc(dump_buf_bytes + clk_cnt_buf_bytes, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - dump_buf->metadata = metadata; -- dump_buf->dump_buf = buf; -+ dump_buf->dump_buf = (u32 *)buf; -+ dump_buf->clk_cnt_buf = (u64 *)(buf + dump_buf_bytes); -+ - return 0; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_alloc); - - void kbase_hwcnt_dump_buffer_free(struct kbase_hwcnt_dump_buffer *dump_buf) - { -@@ -235,7 +234,6 @@ void kbase_hwcnt_dump_buffer_free(struct kbase_hwcnt_dump_buffer *dump_buf) - kfree(dump_buf->dump_buf); - memset(dump_buf, 0, sizeof(*dump_buf)); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_free); - - int kbase_hwcnt_dump_buffer_array_alloc( - const struct kbase_hwcnt_metadata *metadata, -@@ -246,10 +244,16 @@ int kbase_hwcnt_dump_buffer_array_alloc( - size_t buf_idx; - unsigned int order; - unsigned long addr; -+ size_t dump_buf_bytes; -+ size_t clk_cnt_buf_bytes; - - if (!metadata || !dump_bufs) - return -EINVAL; - -+ dump_buf_bytes = metadata->dump_buf_bytes; -+ clk_cnt_buf_bytes = -+ sizeof(*dump_bufs->bufs->clk_cnt_buf) * metadata->clk_cnt; -+ - /* Allocate memory for the dump buffer struct array */ - buffers = kmalloc_array(n, sizeof(*buffers), GFP_KERNEL); - if (!buffers) -@@ -258,8 +262,8 @@ int kbase_hwcnt_dump_buffer_array_alloc( - /* Allocate pages for the actual dump buffers, as they tend to be fairly - * large. - */ -- order = get_order(metadata->dump_buf_bytes * n); -- addr = __get_free_pages(GFP_KERNEL, order); -+ order = get_order((dump_buf_bytes + clk_cnt_buf_bytes) * n); -+ addr = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); - - if (!addr) { - kfree(buffers); -@@ -273,15 +277,18 @@ int kbase_hwcnt_dump_buffer_array_alloc( - - /* Set the buffer of each dump buf */ - for (buf_idx = 0; buf_idx < n; buf_idx++) { -- const size_t offset = metadata->dump_buf_bytes * buf_idx; -+ const size_t dump_buf_offset = dump_buf_bytes * buf_idx; -+ const size_t clk_cnt_buf_offset = -+ (dump_buf_bytes * n) + (clk_cnt_buf_bytes * buf_idx); - - buffers[buf_idx].metadata = metadata; -- buffers[buf_idx].dump_buf = (u32 *)(addr + offset); -+ buffers[buf_idx].dump_buf = (u32 *)(addr + dump_buf_offset); -+ buffers[buf_idx].clk_cnt_buf = -+ (u64 *)(addr + clk_cnt_buf_offset); - } - - return 0; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_array_alloc); - - void kbase_hwcnt_dump_buffer_array_free( - struct kbase_hwcnt_dump_buffer_array *dump_bufs) -@@ -293,7 +300,6 @@ void kbase_hwcnt_dump_buffer_array_free( - free_pages(dump_bufs->page_addr, dump_bufs->page_order); - memset(dump_bufs, 0, sizeof(*dump_bufs)); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_array_free); - - void kbase_hwcnt_dump_buffer_zero( - struct kbase_hwcnt_dump_buffer *dst, -@@ -324,8 +330,10 @@ void kbase_hwcnt_dump_buffer_zero( - - kbase_hwcnt_dump_buffer_block_zero(dst_blk, val_cnt); - } -+ -+ memset(dst->clk_cnt_buf, 0, -+ sizeof(*dst->clk_cnt_buf) * metadata->clk_cnt); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero); - - void kbase_hwcnt_dump_buffer_zero_strict( - struct kbase_hwcnt_dump_buffer *dst) -@@ -334,8 +342,10 @@ void kbase_hwcnt_dump_buffer_zero_strict( - return; - - memset(dst->dump_buf, 0, dst->metadata->dump_buf_bytes); -+ -+ memset(dst->clk_cnt_buf, 0, -+ sizeof(*dst->clk_cnt_buf) * dst->metadata->clk_cnt); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero_strict); - - void kbase_hwcnt_dump_buffer_zero_non_enabled( - struct kbase_hwcnt_dump_buffer *dst, -@@ -375,7 +385,6 @@ void kbase_hwcnt_dump_buffer_zero_non_enabled( - } - } - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_zero_non_enabled); - - void kbase_hwcnt_dump_buffer_copy( - struct kbase_hwcnt_dump_buffer *dst, -@@ -384,6 +393,7 @@ void kbase_hwcnt_dump_buffer_copy( - { - const struct kbase_hwcnt_metadata *metadata; - size_t grp, blk, blk_inst; -+ size_t clk; - - if (WARN_ON(!dst) || - WARN_ON(!src) || -@@ -413,8 +423,13 @@ void kbase_hwcnt_dump_buffer_copy( - - kbase_hwcnt_dump_buffer_block_copy(dst_blk, src_blk, val_cnt); - } -+ -+ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { -+ if (kbase_hwcnt_clk_enable_map_enabled( -+ dst_enable_map->clk_enable_map, clk)) -+ dst->clk_cnt_buf[clk] = src->clk_cnt_buf[clk]; -+ } - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_copy); - - void kbase_hwcnt_dump_buffer_copy_strict( - struct kbase_hwcnt_dump_buffer *dst, -@@ -423,6 +438,7 @@ void kbase_hwcnt_dump_buffer_copy_strict( - { - const struct kbase_hwcnt_metadata *metadata; - size_t grp, blk, blk_inst; -+ size_t clk; - - if (WARN_ON(!dst) || - WARN_ON(!src) || -@@ -451,8 +467,15 @@ void kbase_hwcnt_dump_buffer_copy_strict( - kbase_hwcnt_dump_buffer_block_copy_strict( - dst_blk, src_blk, blk_em, val_cnt); - } -+ -+ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { -+ bool clk_enabled = -+ kbase_hwcnt_clk_enable_map_enabled( -+ dst_enable_map->clk_enable_map, clk); -+ -+ dst->clk_cnt_buf[clk] = clk_enabled ? src->clk_cnt_buf[clk] : 0; -+ } - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_copy_strict); - - void kbase_hwcnt_dump_buffer_accumulate( - struct kbase_hwcnt_dump_buffer *dst, -@@ -461,6 +484,7 @@ void kbase_hwcnt_dump_buffer_accumulate( - { - const struct kbase_hwcnt_metadata *metadata; - size_t grp, blk, blk_inst; -+ size_t clk; - - if (WARN_ON(!dst) || - WARN_ON(!src) || -@@ -494,8 +518,13 @@ void kbase_hwcnt_dump_buffer_accumulate( - kbase_hwcnt_dump_buffer_block_accumulate( - dst_blk, src_blk, hdr_cnt, ctr_cnt); - } -+ -+ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { -+ if (kbase_hwcnt_clk_enable_map_enabled( -+ dst_enable_map->clk_enable_map, clk)) -+ dst->clk_cnt_buf[clk] += src->clk_cnt_buf[clk]; -+ } - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_accumulate); - - void kbase_hwcnt_dump_buffer_accumulate_strict( - struct kbase_hwcnt_dump_buffer *dst, -@@ -504,6 +533,7 @@ void kbase_hwcnt_dump_buffer_accumulate_strict( - { - const struct kbase_hwcnt_metadata *metadata; - size_t grp, blk, blk_inst; -+ size_t clk; - - if (WARN_ON(!dst) || - WARN_ON(!src) || -@@ -534,5 +564,12 @@ void kbase_hwcnt_dump_buffer_accumulate_strict( - kbase_hwcnt_dump_buffer_block_accumulate_strict( - dst_blk, src_blk, blk_em, hdr_cnt, ctr_cnt); - } -+ -+ kbase_hwcnt_metadata_for_each_clock(metadata, clk) { -+ if (kbase_hwcnt_clk_enable_map_enabled( -+ dst_enable_map->clk_enable_map, clk)) -+ dst->clk_cnt_buf[clk] += src->clk_cnt_buf[clk]; -+ else -+ dst->clk_cnt_buf[clk] = 0; -+ } - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_dump_buffer_accumulate_strict); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_types.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_types.h -index 4d78c84..da12952 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_types.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_types.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -@@ -85,7 +84,6 @@ - #include <linux/kernel.h> - #include <linux/string.h> - #include <linux/types.h> --#include "mali_malisw.h" - - /* Number of bytes in each bitfield */ - #define KBASE_HWCNT_BITFIELD_BYTES (sizeof(u64)) -@@ -101,6 +99,21 @@ - */ - #define KBASE_HWCNT_AVAIL_MASK_BITS (sizeof(u64) * BITS_PER_BYTE) - -+/* Minimum alignment of each block of hardware counters */ -+#define KBASE_HWCNT_BLOCK_BYTE_ALIGNMENT \ -+ (KBASE_HWCNT_BITFIELD_BITS * KBASE_HWCNT_VALUE_BYTES) -+ -+/** -+ * KBASE_HWCNT_ALIGN_UPWARDS() - Calculate next aligned value. -+ * @value: The value to align upwards. -+ * @alignment: The alignment boundary. -+ * -+ * Return: Input value if already aligned to the specified boundary, or next -+ * (incrementing upwards) aligned value. -+ */ -+#define KBASE_HWCNT_ALIGN_UPWARDS(value, alignment) \ -+ (value + ((alignment - (value % alignment)) % alignment)) -+ - /** - * struct kbase_hwcnt_block_description - Description of one or more identical, - * contiguous, Hardware Counter Blocks. -@@ -136,11 +149,13 @@ struct kbase_hwcnt_group_description { - * @grps: Non-NULL pointer to an array of grp_cnt group descriptions, - * describing each Hardware Counter Group in the system. - * @avail_mask: Flat Availability Mask for all block instances in the system. -+ * @clk_cnt: The number of clock domains in the system. The maximum is 64. - */ - struct kbase_hwcnt_description { - size_t grp_cnt; - const struct kbase_hwcnt_group_description *grps; - u64 avail_mask; -+ u8 clk_cnt; - }; - - /** -@@ -220,6 +235,7 @@ struct kbase_hwcnt_group_metadata { - * @enable_map_bytes: The size in bytes of an Enable Map needed for the system. - * @dump_buf_bytes: The size in bytes of a Dump Buffer needed for the system. - * @avail_mask: The Availability Mask for the system. -+ * @clk_cnt: The number of clock domains in the system. - */ - struct kbase_hwcnt_metadata { - size_t grp_cnt; -@@ -227,6 +243,7 @@ struct kbase_hwcnt_metadata { - size_t enable_map_bytes; - size_t dump_buf_bytes; - u64 avail_mask; -+ u8 clk_cnt; - }; - - /** -@@ -234,13 +251,16 @@ struct kbase_hwcnt_metadata { - * bitfields. - * @metadata: Non-NULL pointer to metadata used to identify, and to describe - * the layout of the enable map. -- * @enable_map: Non-NULL pointer of size metadata->enable_map_bytes to an array -- * of u64 bitfields, each bit of which enables one hardware -+ * @hwcnt_enable_map: Non-NULL pointer of size metadata->enable_map_bytes to an -+ * array of u64 bitfields, each bit of which enables one hardware - * counter. -+ * @clk_enable_map: An array of u64 bitfields, each bit of which enables cycle -+ * counter for a given clock domain. - */ - struct kbase_hwcnt_enable_map { - const struct kbase_hwcnt_metadata *metadata; -- u64 *enable_map; -+ u64 *hwcnt_enable_map; -+ u64 clk_enable_map; - }; - - /** -@@ -250,10 +270,13 @@ struct kbase_hwcnt_enable_map { - * the layout of the Dump Buffer. - * @dump_buf: Non-NULL pointer of size metadata->dump_buf_bytes to an array - * of u32 values. -+ * @clk_cnt_buf: A pointer to an array of u64 values for cycle count elapsed -+ * for each clock domain. - */ - struct kbase_hwcnt_dump_buffer { - const struct kbase_hwcnt_metadata *metadata; - u32 *dump_buf; -+ u64 *clk_cnt_buf; - }; - - /** -@@ -349,8 +372,7 @@ void kbase_hwcnt_metadata_destroy(const struct kbase_hwcnt_metadata *metadata); - * @grp: Index of the group in the metadata. - * @blk: Index of the block in the group. - * -- * Return: Number of u32 counter headers in each instance of block blk in -- * group grp. -+ * Return: Number of counter headers in each instance of block blk in group grp. - */ - #define kbase_hwcnt_metadata_block_headers_count(metadata, grp, blk) \ - ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].hdr_cnt) -@@ -361,19 +383,29 @@ void kbase_hwcnt_metadata_destroy(const struct kbase_hwcnt_metadata *metadata); - * @grp: Index of the group in the metadata. - * @blk: Index of the block in the group. - * -- * Return: Number of u32 counters in each instance of block blk in group -- * grp. -+ * Return: Number of counters in each instance of block blk in group grp. - */ - #define kbase_hwcnt_metadata_block_counters_count(metadata, grp, blk) \ - ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].ctr_cnt) - -+/** -+ * kbase_hwcnt_metadata_block_enable_map_stride() - Get the enable map stride. -+ * @metadata: Non-NULL pointer to metadata. -+ * @grp: Index of the group in the metadata. -+ * @blk: Index of the block in the group. -+ * -+ * Return: enable map stride in each instance of block blk in group grp. -+ */ -+#define kbase_hwcnt_metadata_block_enable_map_stride(metadata, grp, blk) \ -+ ((metadata)->grp_metadata[(grp)].blk_metadata[(blk)].enable_map_stride) -+ - /** - * kbase_hwcnt_metadata_block_values_count() - Get the number of values. - * @metadata: Non-NULL pointer to metadata. - * @grp: Index of the group in the metadata. - * @blk: Index of the block in the group. - * -- * Return: Number of u32 headers plus counters in each instance of block blk -+ * Return: Number of headers plus counters in each instance of block blk - * in group grp. - */ - #define kbase_hwcnt_metadata_block_values_count(metadata, grp, blk) \ -@@ -473,7 +505,7 @@ void kbase_hwcnt_enable_map_free(struct kbase_hwcnt_enable_map *enable_map); - * block instance. - */ - #define kbase_hwcnt_enable_map_block_instance(map, grp, blk, blk_inst) \ -- ((map)->enable_map + \ -+ ((map)->hwcnt_enable_map + \ - (map)->metadata->grp_metadata[(grp)].enable_map_index + \ - (map)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].enable_map_index + \ - (map)->metadata->grp_metadata[(grp)].blk_metadata[(blk)].enable_map_stride * (blk_inst)) -@@ -520,7 +552,11 @@ static inline void kbase_hwcnt_enable_map_block_disable_all( - static inline void kbase_hwcnt_enable_map_disable_all( - struct kbase_hwcnt_enable_map *dst) - { -- memset(dst->enable_map, 0, dst->metadata->enable_map_bytes); -+ if (dst->hwcnt_enable_map != NULL) -+ memset(dst->hwcnt_enable_map, 0, -+ dst->metadata->enable_map_bytes); -+ -+ dst->clk_enable_map = 0; - } - - /** -@@ -569,6 +605,8 @@ static inline void kbase_hwcnt_enable_map_enable_all( - kbase_hwcnt_metadata_for_each_block(dst->metadata, grp, blk, blk_inst) - kbase_hwcnt_enable_map_block_enable_all( - dst, grp, blk, blk_inst); -+ -+ dst->clk_enable_map = (1ull << dst->metadata->clk_cnt) - 1; - } - - /** -@@ -582,9 +620,13 @@ static inline void kbase_hwcnt_enable_map_copy( - struct kbase_hwcnt_enable_map *dst, - const struct kbase_hwcnt_enable_map *src) - { -- memcpy(dst->enable_map, -- src->enable_map, -- dst->metadata->enable_map_bytes); -+ if (dst->hwcnt_enable_map != NULL) { -+ memcpy(dst->hwcnt_enable_map, -+ src->hwcnt_enable_map, -+ dst->metadata->enable_map_bytes); -+ } -+ -+ dst->clk_enable_map = src->clk_enable_map; - } - - /** -@@ -602,8 +644,12 @@ static inline void kbase_hwcnt_enable_map_union( - dst->metadata->enable_map_bytes / KBASE_HWCNT_BITFIELD_BYTES; - size_t i; - -- for (i = 0; i < bitfld_count; i++) -- dst->enable_map[i] |= src->enable_map[i]; -+ if (dst->hwcnt_enable_map != NULL) { -+ for (i = 0; i < bitfld_count; i++) -+ dst->hwcnt_enable_map[i] |= src->hwcnt_enable_map[i]; -+ } -+ -+ dst->clk_enable_map |= src->clk_enable_map; - } - - /** -@@ -656,6 +702,12 @@ static inline bool kbase_hwcnt_enable_map_any_enabled( - const struct kbase_hwcnt_enable_map *enable_map) - { - size_t grp, blk, blk_inst; -+ const u64 clk_enable_map_mask = -+ (1ull << enable_map->metadata->clk_cnt) - 1; -+ -+ if (enable_map->metadata->clk_cnt > 0 && -+ (enable_map->clk_enable_map & clk_enable_map_mask)) -+ return true; - - kbase_hwcnt_metadata_for_each_block( - enable_map->metadata, grp, blk, blk_inst) { -@@ -749,12 +801,10 @@ void kbase_hwcnt_dump_buffer_free(struct kbase_hwcnt_dump_buffer *dump_buf); - * kbase_hwcnt_dump_buffer_array_alloc() - Allocate an array of dump buffers. - * @metadata: Non-NULL pointer to metadata describing the system. - * @n: Number of dump buffers to allocate -- * @dump_bufs: Non-NULL pointer to dump buffer array to be initialised. Each -- * dump buffer in the array will be initialised to undefined values, -- * so must be used as a copy dest, or cleared before use. -+ * @dump_bufs: Non-NULL pointer to dump buffer array to be initialised. - * -- * A single contiguous page allocation will be used for all of the buffers -- * inside the array, where: -+ * A single zeroed contiguous page allocation will be used for all of the -+ * buffers inside the array, where: - * dump_bufs[n].dump_buf == page_addr + n * metadata.dump_buf_bytes - * - * Return: 0 on success, else error code. -@@ -1084,4 +1134,29 @@ static inline void kbase_hwcnt_dump_buffer_block_accumulate_strict( - } - } - -+/** -+ * Iterate over each clock domain in the metadata. -+ * -+ * @md: Non-NULL pointer to metadata. -+ * @clk: size_t variable used as clock iterator. -+ */ -+#define kbase_hwcnt_metadata_for_each_clock(md, clk) \ -+ for ((clk) = 0; (clk) < (md)->clk_cnt; (clk)++) -+ -+/** -+ * kbase_hwcnt_clk_enable_map_enabled() - Check if the given index is enabled -+ * in clk_enable_map. -+ * @clk_enable_map: An enable map for clock domains. -+ * @index: Index of the enable map for clock domain. -+ * -+ * Return: true if the index of the clock domain is enabled, else false. -+ */ -+static inline bool kbase_hwcnt_clk_enable_map_enabled( -+ const u64 clk_enable_map, const size_t index) -+{ -+ if (clk_enable_map & (1ull << index)) -+ return true; -+ return false; -+} -+ - #endif /* _KBASE_HWCNT_TYPES_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_virtualizer.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_virtualizer.c -index 917e47c..52ecb7b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_virtualizer.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_virtualizer.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,17 +17,12 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase_hwcnt_virtualizer.h" - #include "mali_kbase_hwcnt_accumulator.h" - #include "mali_kbase_hwcnt_context.h" - #include "mali_kbase_hwcnt_types.h" --#include "mali_malisw.h" --#include "mali_kbase_debug.h" --#include "mali_kbase_linux.h" - - #include <linux/mutex.h> - #include <linux/slab.h> -@@ -87,7 +83,6 @@ const struct kbase_hwcnt_metadata *kbase_hwcnt_virtualizer_metadata( - - return hvirt->metadata; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_metadata); - - /** - * kbasep_hwcnt_virtualizer_client_free - Free a virtualizer client's memory. -@@ -496,7 +491,6 @@ int kbase_hwcnt_virtualizer_client_set_counters( - - return errcode; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_set_counters); - - /** - * kbasep_hwcnt_virtualizer_client_dump - Perform a dump of the client's -@@ -686,7 +680,6 @@ int kbase_hwcnt_virtualizer_client_dump( - - return errcode; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_dump); - - int kbase_hwcnt_virtualizer_client_create( - struct kbase_hwcnt_virtualizer *hvirt, -@@ -719,7 +712,6 @@ int kbase_hwcnt_virtualizer_client_create( - *out_hvcli = hvcli; - return 0; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_create); - - void kbase_hwcnt_virtualizer_client_destroy( - struct kbase_hwcnt_virtualizer_client *hvcli) -@@ -735,7 +727,6 @@ void kbase_hwcnt_virtualizer_client_destroy( - - kbasep_hwcnt_virtualizer_client_free(hvcli); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_client_destroy); - - int kbase_hwcnt_virtualizer_init( - struct kbase_hwcnt_context *hctx, -@@ -766,7 +757,6 @@ int kbase_hwcnt_virtualizer_init( - *out_hvirt = virt; - return 0; - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_init); - - void kbase_hwcnt_virtualizer_term( - struct kbase_hwcnt_virtualizer *hvirt) -@@ -787,4 +777,12 @@ void kbase_hwcnt_virtualizer_term( - - kfree(hvirt); - } --KBASE_EXPORT_TEST_API(kbase_hwcnt_virtualizer_term); -+ -+bool kbase_hwcnt_virtualizer_queue_work(struct kbase_hwcnt_virtualizer *hvirt, -+ struct work_struct *work) -+{ -+ if (WARN_ON(!hvirt) || WARN_ON(!work)) -+ return false; -+ -+ return kbase_hwcnt_context_queue_work(hvirt->hctx, work); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_virtualizer.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_virtualizer.h -index 8f628c3..e3a8137 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_virtualizer.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_virtualizer.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -@@ -31,6 +30,7 @@ - #define _KBASE_HWCNT_VIRTUALIZER_H_ - - #include <linux/types.h> -+#include <linux/workqueue.h> - - struct kbase_hwcnt_context; - struct kbase_hwcnt_virtualizer; -@@ -142,4 +142,19 @@ int kbase_hwcnt_virtualizer_client_dump( - u64 *ts_end_ns, - struct kbase_hwcnt_dump_buffer *dump_buf); - -+/** -+ * kbase_hwcnt_virtualizer_queue_work() - Queue hardware counter related async -+ * work on a workqueue specialized for -+ * hardware counters. -+ * @hvirt: Non-NULL pointer to the hardware counter virtualizer. -+ * @work: Non-NULL pointer to work to queue. -+ * -+ * Return: false if work was already on a queue, true otherwise. -+ * -+ * This is a convenience function that directly calls the underlying -+ * kbase_hwcnt_context's kbase_hwcnt_context_queue_work. -+ */ -+bool kbase_hwcnt_virtualizer_queue_work(struct kbase_hwcnt_virtualizer *hvirt, -+ struct work_struct *work); -+ - #endif /* _KBASE_HWCNT_VIRTUALIZER_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd.c -index 43f4b4d..83977f8 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,22 +17,20 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #include <linux/dma-buf.h> --#ifdef CONFIG_COMPAT -+#if IS_ENABLED(CONFIG_COMPAT) - #include <linux/compat.h> - #endif - #include <mali_kbase.h> - #include <linux/random.h> - #include <linux/version.h> - #include <linux/ratelimit.h> -+#include <linux/priority_control_manager.h> - - #include <mali_kbase_jm.h> -+#include <mali_kbase_kinstr_jm.h> - #include <mali_kbase_hwaccess_jm.h> - #include <tl/mali_kbase_tracepoints.h> - #include <mali_linux_trace.h> -@@ -39,24 +38,17 @@ - #include "mali_kbase_dma_fence.h" - #include <mali_kbase_cs_experimental.h> - --#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) -+#include <mali_kbase_caps.h> - --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0) --/* random32 was renamed to prandom_u32 in 3.8 */ --#define prandom_u32 random32 --#endif -+#define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a) - - /* Return whether katom will run on the GPU or not. Currently only soft jobs and -- * dependency-only atoms do not run on the GPU */ -+ * dependency-only atoms do not run on the GPU -+ */ - #define IS_GPU_ATOM(katom) (!((katom->core_req & BASE_JD_REQ_SOFT_JOB) || \ - ((katom->core_req & BASE_JD_REQ_ATOM_TYPE) == \ - BASE_JD_REQ_DEP))) - --/* Minimum API version that supports the just-in-time memory allocation pressure -- * limit feature. -- */ --#define MIN_API_VERSION_WITH_JPL KBASE_API_VERSION(11, 20) -- - /* - * This is the kernel side of the API. Only entry points are: - * - kbase_jd_submit(): Called from userspace to submit a single bag -@@ -70,13 +62,22 @@ - static void __user * - get_compat_pointer(struct kbase_context *kctx, const u64 p) - { --#ifdef CONFIG_COMPAT -+#if IS_ENABLED(CONFIG_COMPAT) - if (kbase_ctx_flag(kctx, KCTX_COMPAT)) - return compat_ptr(p); - #endif - return u64_to_user_ptr(p); - } - -+/* Mark an atom as complete, and trace it in kinstr_jm */ -+static void jd_mark_atom_complete(struct kbase_jd_atom *katom) -+{ -+ katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -+ kbase_kinstr_jm_atom_complete(katom); -+ dev_dbg(katom->kctx->kbdev->dev, "Atom %pK status to completed\n", -+ (void *)katom); -+} -+ - /* Runs an atom, either by handing to the JS or by immediately running it in the case of soft-jobs - * - * Returns whether the JS needs a reschedule. -@@ -88,7 +89,7 @@ static bool jd_run_atom(struct kbase_jd_atom *katom) - { - struct kbase_context *kctx = katom->kctx; - -- dev_dbg(kctx->kbdev->dev, "JD run atom %p in kctx %p\n", -+ dev_dbg(kctx->kbdev->dev, "JD run atom %pK in kctx %pK\n", - (void *)katom, (void *)kctx); - - KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); -@@ -97,30 +98,24 @@ static bool jd_run_atom(struct kbase_jd_atom *katom) - /* Dependency only atom */ - trace_sysgraph(SGR_SUBMIT, kctx->id, - kbase_jd_atom_id(katom->kctx, katom)); -- katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -- dev_dbg(kctx->kbdev->dev, "Atom %p status to completed\n", -- (void *)katom); -- return 0; -+ jd_mark_atom_complete(katom); -+ return false; - } else if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { - /* Soft-job */ - if (katom->will_fail_event_code) { - kbase_finish_soft_job(katom); -- katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -- dev_dbg(kctx->kbdev->dev, -- "Atom %p status to completed\n", (void *)katom); -- return 0; -+ jd_mark_atom_complete(katom); -+ return false; - } - if (kbase_process_soft_job(katom) == 0) { - kbase_finish_soft_job(katom); -- katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -- dev_dbg(kctx->kbdev->dev, -- "Atom %p status to completed\n", (void *)katom); -+ jd_mark_atom_complete(katom); - } -- return 0; -+ return false; - } - - katom->status = KBASE_JD_ATOM_STATE_IN_JS; -- dev_dbg(kctx->kbdev->dev, "Atom %p status to in JS\n", (void *)katom); -+ dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", (void *)katom); - /* Queue an action about whether we should try scheduling a context */ - return kbasep_js_add_job(kctx, katom); - } -@@ -205,7 +200,7 @@ static void kbase_jd_post_external_resources(struct kbase_jd_atom *katom) - * jctx.lock must be held when this is called. - */ - --static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom_v2 *user_atom) -+static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const struct base_jd_atom *user_atom) - { - int err_ret_val = -EINVAL; - u32 res_no; -@@ -242,7 +237,8 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st - - /* copy user buffer to the end of our real buffer. - * Make sure the struct sizes haven't changed in a way -- * we don't support */ -+ * we don't support -+ */ - BUILD_BUG_ON(sizeof(*input_extres) > sizeof(*katom->extres)); - input_extres = (struct base_external_resource *) - (((unsigned char *)katom->extres) + -@@ -258,9 +254,14 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st - - #ifdef CONFIG_MALI_DMA_FENCE - if (implicit_sync) { -- info.resv_objs = kmalloc_array(katom->nr_extres, -- sizeof(struct reservation_object *), -- GFP_KERNEL); -+ info.resv_objs = -+ kmalloc_array(katom->nr_extres, -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) -+ sizeof(struct reservation_object *), -+#else -+ sizeof(struct dma_resv *), -+#endif -+ GFP_KERNEL); - if (!info.resv_objs) { - err_ret_val = -ENOMEM; - goto early_err_out; -@@ -277,7 +278,7 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st - #endif /* CONFIG_MALI_DMA_FENCE */ - - /* Take the processes mmap lock */ -- down_read(¤t->mm->mmap_sem); -+ down_read(kbase_mem_get_process_mmap_lock()); - - /* need to keep the GPU VM locked while we set up UMM buffers */ - kbase_gpu_vm_lock(katom->kctx); -@@ -314,8 +315,11 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st - #ifdef CONFIG_MALI_DMA_FENCE - if (implicit_sync && - reg->gpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { -+#if (KERNEL_VERSION(5, 4, 0) > LINUX_VERSION_CODE) - struct reservation_object *resv; -- -+#else -+ struct dma_resv *resv; -+#endif - resv = reg->gpu_alloc->imported.umm.dma_buf->resv; - if (resv) - kbase_dma_fence_add_reservation(resv, &info, -@@ -328,7 +332,7 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st - * at least not before the first write) as we overwrite elements - * as we loop and could be overwriting ourself, so no writes - * until the last read for an element. -- * */ -+ */ - katom->extres[res_no].gpu_address = reg->start_pfn << PAGE_SHIFT; /* save the start_pfn (as an address, not pfn) to use fast lookup later */ - katom->extres[res_no].alloc = alloc; - } -@@ -337,7 +341,7 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st - kbase_gpu_vm_unlock(katom->kctx); - - /* Release the processes mmap lock */ -- up_read(¤t->mm->mmap_sem); -+ up_read(kbase_mem_get_process_mmap_lock()); - - #ifdef CONFIG_MALI_DMA_FENCE - if (implicit_sync) { -@@ -362,7 +366,7 @@ static int kbase_jd_pre_external_resources(struct kbase_jd_atom *katom, const st - #ifdef CONFIG_MALI_DMA_FENCE - failed_dma_fence_setup: - /* Lock the processes mmap lock */ -- down_read(¤t->mm->mmap_sem); -+ down_read(kbase_mem_get_process_mmap_lock()); - - /* lock before we unmap */ - kbase_gpu_vm_lock(katom->kctx); -@@ -378,7 +382,7 @@ failed_dma_fence_setup: - kbase_gpu_vm_unlock(katom->kctx); - - /* Release the processes mmap lock */ -- up_read(¤t->mm->mmap_sem); -+ up_read(kbase_mem_get_process_mmap_lock()); - - early_err_out: - kfree(katom->extres); -@@ -455,9 +459,6 @@ static inline void jd_resolve_dep(struct list_head *out_list, - #endif /* CONFIG_MALI_DMA_FENCE */ - - if (dep_satisfied) { -- trace_sysgraph(SGR_DEP_RES, -- dep_atom->kctx->id, -- kbase_jd_atom_id(katom->kctx, dep_atom)); - dep_atom->in_jd_list = true; - list_add_tail(&dep_atom->jd_item, out_list); - } -@@ -481,7 +482,8 @@ static inline void jd_resolve_dep(struct list_head *out_list, - static bool is_dep_valid(struct kbase_jd_atom *katom) - { - /* If there's no dependency then this is 'valid' from the perspective of -- * early dependency submission */ -+ * early dependency submission -+ */ - if (!katom) - return true; - -@@ -490,7 +492,8 @@ static bool is_dep_valid(struct kbase_jd_atom *katom) - return false; - - /* If dependency has completed and has failed or will fail then it is -- * not valid */ -+ * not valid -+ */ - if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED && - (katom->event_code != BASE_JD_EVENT_DONE || - katom->will_fail_event_code)) -@@ -544,10 +547,6 @@ static void jd_try_submitting_deps(struct list_head *out_list, - #endif /* CONFIG_MALI_DMA_FENCE */ - - if (dep0_valid && dep1_valid && dep_satisfied) { -- trace_sysgraph(SGR_DEP_RES, -- dep_atom->kctx->id, -- kbase_jd_atom_id(dep_atom->kctx, -- dep_atom)); - dep_atom->in_jd_list = true; - list_add(&dep_atom->jd_item, out_list); - } -@@ -556,7 +555,7 @@ static void jd_try_submitting_deps(struct list_head *out_list, - } - } - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /** - * jd_update_jit_usage - Update just-in-time physical memory usage for an atom. - * -@@ -632,8 +631,8 @@ static void jd_update_jit_usage(struct kbase_jd_atom *katom) - u64 addr_end; - - if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { -- const unsigned long extent_bytes = reg->extent -- << PAGE_SHIFT; -+ const unsigned long extension_bytes = -+ reg->extension << PAGE_SHIFT; - const u64 low_ptr = ptr[LOW]; - const u64 high_ptr = ptr[HIGH]; - -@@ -654,8 +653,8 @@ static void jd_update_jit_usage(struct kbase_jd_atom *katom) - * this, but here to avoid future maintenance - * hazards - */ -- WARN_ON(!is_power_of_2(extent_bytes)); -- addr_end = ALIGN(read_val, extent_bytes); -+ WARN_ON(!is_power_of_2(extension_bytes)); -+ addr_end = ALIGN(read_val, extension_bytes); - } else { - addr_end = read_val = READ_ONCE(*ptr); - } -@@ -696,7 +695,7 @@ static void jd_update_jit_usage(struct kbase_jd_atom *katom) - - kbase_jit_retry_pending_alloc(kctx); - } --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /* - * Perform the necessary handling of an atom that has finished running -@@ -721,12 +720,14 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, - - KBASE_DEBUG_ASSERT(katom->status != KBASE_JD_ATOM_STATE_UNUSED); - --#if MALI_JIT_PRESSURE_LIMIT -- jd_update_jit_usage(katom); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (kbase_ctx_flag(kctx, KCTX_JPL_ENABLED)) -+ jd_update_jit_usage(katom); -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /* This is needed in case an atom is failed due to being invalid, this -- * can happen *before* the jobs that the atom depends on have completed */ -+ * can happen *before* the jobs that the atom depends on have completed -+ */ - for (i = 0; i < 2; i++) { - if (kbase_jd_katom_dep_atom(&katom->dep[i])) { - list_del(&katom->dep_item[i]); -@@ -734,9 +735,7 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, - } - } - -- katom->status = KBASE_JD_ATOM_STATE_COMPLETED; -- dev_dbg(kctx->kbdev->dev, "Atom %p status to completed\n", -- (void *)katom); -+ jd_mark_atom_complete(katom); - list_add_tail(&katom->jd_item, &completed_jobs); - - while (!list_empty(&completed_jobs)) { -@@ -759,7 +758,7 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, - list_del(runnable_jobs.next); - node->in_jd_list = false; - -- dev_dbg(kctx->kbdev->dev, "List node %p has status %d\n", -+ dev_dbg(kctx->kbdev->dev, "List node %pK has status %d\n", - node, node->status); - - KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED); -@@ -786,7 +785,8 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, - !node->will_fail_event_code) { - /* Node successfully submitted, try submitting - * dependencies as they may now be representable -- * in JS */ -+ * in JS -+ */ - jd_try_submitting_deps(&runnable_jobs, node); - } - } -@@ -802,10 +802,14 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, - - /* Decrement and check the TOTAL number of jobs. This includes - * those not tracked by the scheduler: 'not ready to run' and -- * 'dependency-only' jobs. */ -+ * 'dependency-only' jobs. -+ */ - if (--kctx->jctx.job_nr == 0) -- wake_up(&kctx->jctx.zero_jobs_wait); /* All events are safely queued now, and we can signal any waiter -- * that we've got no more jobs (so we can be safely terminated) */ -+ /* All events are safely queued now, and we can signal -+ * any waiter that we've got no more jobs (so we can be -+ * safely terminated) -+ */ -+ wake_up(&kctx->jctx.zero_jobs_wait); - } - - return need_to_try_schedule_context; -@@ -813,7 +817,7 @@ bool jd_done_nolock(struct kbase_jd_atom *katom, - - KBASE_EXPORT_TEST_API(jd_done_nolock); - --#ifdef CONFIG_GPU_TRACEPOINTS -+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) - enum { - CORE_REQ_DEP_ONLY, - CORE_REQ_SOFT, -@@ -868,8 +872,23 @@ static const char *kbasep_map_core_reqs_to_string(base_jd_core_req core_req) - } - #endif - -+/* Trace an atom submission. */ -+static void jd_trace_atom_submit(struct kbase_context *const kctx, -+ struct kbase_jd_atom *const katom, -+ int *priority) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ KBASE_TLSTREAM_TL_NEW_ATOM(kbdev, katom, kbase_jd_atom_id(kctx, katom)); -+ KBASE_TLSTREAM_TL_RET_ATOM_CTX(kbdev, katom, kctx); -+ if (priority) -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(kbdev, katom, *priority); -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_IDLE); -+ kbase_kinstr_jm_atom_queue(katom); -+} -+ - static bool jd_submit_atom(struct kbase_context *const kctx, -- const struct base_jd_atom_v2 *const user_atom, -+ const struct base_jd_atom *const user_atom, - const struct base_jd_fragment *const user_jc_incr, - struct kbase_jd_atom *const katom) - { -@@ -879,14 +898,17 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - int i; - int sched_prio; - bool will_fail = false; -+ unsigned long flags; -+ enum kbase_jd_atom_state status; - -- dev_dbg(kbdev->dev, "User did JD submit atom %p\n", (void *)katom); -+ dev_dbg(kbdev->dev, "User did JD submit atom %pK\n", (void *)katom); - - /* Update the TOTAL number of jobs. This includes those not tracked by -- * the scheduler: 'not ready to run' and 'dependency-only' jobs. */ -+ * the scheduler: 'not ready to run' and 'dependency-only' jobs. -+ */ - jctx->job_nr++; - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+#if KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE - katom->start_timestamp.tv64 = 0; - #else - katom->start_timestamp = 0; -@@ -899,6 +921,7 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - katom->jc = user_atom->jc; - katom->core_req = user_atom->core_req; - katom->jobslot = user_atom->jobslot; -+ katom->seq_nr = user_atom->seq_nr; - katom->atom_flags = 0; - katom->retry_count = 0; - katom->need_cache_flush_cores_retained = 0; -@@ -911,19 +934,19 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - - trace_sysgraph(SGR_ARRIVE, kctx->id, user_atom->atom_number); - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /* Older API version atoms might have random values where jit_id now - * lives, but we must maintain backwards compatibility - handle the - * issue. - */ -- if (kctx->api_version < MIN_API_VERSION_WITH_JPL) { -+ if (!mali_kbase_supports_jit_pressure_limit(kctx->api_version)) { - katom->jit_ids[0] = 0; - katom->jit_ids[1] = 0; - } else { - katom->jit_ids[0] = user_atom->jit_id[0]; - katom->jit_ids[1] = user_atom->jit_id[1]; - } --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - katom->renderpass_id = user_atom->renderpass_id; - -@@ -953,23 +976,14 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - katom->event_code = BASE_JD_EVENT_JOB_CONFIG_FAULT; - katom->status = KBASE_JD_ATOM_STATE_COMPLETED; - dev_dbg(kbdev->dev, -- "Atom %p status to completed\n", -+ "Atom %pK status to completed\n", - (void *)katom); - - /* Wrong dependency setup. Atom will be sent - * back to user space. Do not record any -- * dependencies. */ -- KBASE_TLSTREAM_TL_NEW_ATOM( -- kbdev, -- katom, -- kbase_jd_atom_id(kctx, katom)); -- KBASE_TLSTREAM_TL_RET_ATOM_CTX( -- kbdev, -- katom, kctx); -- KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE( -- kbdev, -- katom, -- TL_ATOM_STATE_IDLE); -+ * dependencies. -+ */ -+ jd_trace_atom_submit(kctx, katom, NULL); - - return jd_done_nolock(katom, NULL); - } -@@ -1005,19 +1019,13 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - /* Atom has completed, propagate the error code if any */ - katom->event_code = dep_atom->event_code; - katom->status = KBASE_JD_ATOM_STATE_QUEUED; -- dev_dbg(kbdev->dev, "Atom %p status to queued\n", -+ dev_dbg(kbdev->dev, "Atom %pK status to queued\n", - (void *)katom); - - /* This atom will be sent back to user space. - * Do not record any dependencies. - */ -- KBASE_TLSTREAM_TL_NEW_ATOM( -- kbdev, -- katom, -- kbase_jd_atom_id(kctx, katom)); -- KBASE_TLSTREAM_TL_RET_ATOM_CTX(kbdev, katom, kctx); -- KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, -- TL_ATOM_STATE_IDLE); -+ jd_trace_atom_submit(kctx, katom, NULL); - - will_fail = true; - -@@ -1045,20 +1053,7 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - return jd_done_nolock(katom, NULL); - } - -- if (katom->core_req & BASE_JD_REQ_SOFT_JOB) { -- /* This softjob has failed due to a previous -- * dependency, however we should still run the -- * prepare & finish functions -- */ -- if (kbase_prepare_soft_job(katom) != 0) { -- katom->event_code = -- BASE_JD_EVENT_JOB_INVALID; -- return jd_done_nolock(katom, NULL); -- } -- } -- - katom->will_fail_event_code = katom->event_code; -- return false; - } - - /* These must occur after the above loop to ensure that an atom -@@ -1067,22 +1062,19 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - */ - katom->event_code = BASE_JD_EVENT_DONE; - katom->status = KBASE_JD_ATOM_STATE_QUEUED; -- dev_dbg(kbdev->dev, "Atom %p status to queued\n", (void *)katom); -+ dev_dbg(kbdev->dev, "Atom %pK status to queued\n", (void *)katom); - - /* For invalid priority, be most lenient and choose the default */ - sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio); - if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID) - sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT; -- katom->sched_priority = sched_prio; -+ -+ /* Cap the priority to jctx.max_priority */ -+ katom->sched_priority = (sched_prio < kctx->jctx.max_priority) ? -+ kctx->jctx.max_priority : sched_prio; - - /* Create a new atom. */ -- KBASE_TLSTREAM_TL_NEW_ATOM( -- kbdev, -- katom, -- kbase_jd_atom_id(kctx, katom)); -- KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, katom, TL_ATOM_STATE_IDLE); -- KBASE_TLSTREAM_TL_ATTRIB_ATOM_PRIORITY(kbdev, katom, katom->sched_priority); -- KBASE_TLSTREAM_TL_RET_ATOM_CTX(kbdev, katom, kctx); -+ jd_trace_atom_submit(kctx, katom, &katom->sched_priority); - - #if !MALI_INCREMENTAL_RENDERING - /* Reject atoms for incremental rendering if not supported */ -@@ -1149,8 +1141,8 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - } - } - --#if !MALI_JIT_PRESSURE_LIMIT -- if ((kctx->api_version >= MIN_API_VERSION_WITH_JPL) && -+#if !MALI_JIT_PRESSURE_LIMIT_BASE -+ if (mali_kbase_supports_jit_pressure_limit(kctx->api_version) && - (user_atom->jit_id[0] || user_atom->jit_id[1])) { - /* JIT pressure limit is disabled, but we are receiving non-0 - * JIT IDs - atom is invalid. -@@ -1158,7 +1150,7 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - katom->event_code = BASE_JD_EVENT_JOB_INVALID; - return jd_done_nolock(katom, NULL); - } --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /* Validate the atom. Function will return error if the atom is - * malformed. -@@ -1180,7 +1172,7 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - } - } - --#ifdef CONFIG_GPU_TRACEPOINTS -+#if IS_ENABLED(CONFIG_GPU_TRACEPOINTS) - katom->work_id = atomic_inc_return(&jctx->work_id); - trace_gpu_job_enqueue(kctx->id, katom->work_id, - kbasep_map_core_reqs_to_string(katom->core_req)); -@@ -1207,13 +1199,24 @@ static bool jd_submit_atom(struct kbase_context *const kctx, - bool need_to_try_schedule_context; - - katom->status = KBASE_JD_ATOM_STATE_IN_JS; -- dev_dbg(kctx->kbdev->dev, "Atom %p status to in JS\n", -+ dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", - (void *)katom); - - need_to_try_schedule_context = kbasep_js_add_job(kctx, katom); - /* If job was cancelled then resolve immediately */ - if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED) - return need_to_try_schedule_context; -+ -+ /* Synchronize with backend reset */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ status = katom->status; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ if (status == KBASE_JD_ATOM_STATE_HW_COMPLETED) { -+ dev_dbg(kctx->kbdev->dev, -+ "Atom %d cancelled on HW\n", -+ kbase_jd_atom_id(katom->kctx, katom)); -+ return need_to_try_schedule_context; -+ } - } - - /* This is a pure dependency. Resolve it immediately */ -@@ -1231,6 +1234,9 @@ int kbase_jd_submit(struct kbase_context *kctx, - struct kbase_device *kbdev; - u32 latest_flush; - -+ bool jd_atom_is_v2 = (stride == sizeof(struct base_jd_atom_v2) || -+ stride == offsetof(struct base_jd_atom_v2, renderpass_id)); -+ - /* - * kbase_jd_submit isn't expected to fail and so all errors with the - * jobs are reported by immediately failing them (through event system) -@@ -1245,7 +1251,9 @@ int kbase_jd_submit(struct kbase_context *kctx, - } - - if (stride != offsetof(struct base_jd_atom_v2, renderpass_id) && -- stride != sizeof(struct base_jd_atom_v2)) { -+ stride != sizeof(struct base_jd_atom_v2) && -+ stride != offsetof(struct base_jd_atom, renderpass_id) && -+ stride != sizeof(struct base_jd_atom)) { - dev_err(kbdev->dev, - "Stride %u passed to job_submit isn't supported by the kernel\n", - stride); -@@ -1256,16 +1264,29 @@ int kbase_jd_submit(struct kbase_context *kctx, - latest_flush = kbase_backend_get_current_flush_id(kbdev); - - for (i = 0; i < nr_atoms; i++) { -- struct base_jd_atom_v2 user_atom; -+ struct base_jd_atom user_atom; - struct base_jd_fragment user_jc_incr; - struct kbase_jd_atom *katom; - -- if (copy_from_user(&user_atom, user_addr, stride) != 0) { -- dev_err(kbdev->dev, -- "Invalid atom address %p passed to job_submit\n", -- user_addr); -- err = -EFAULT; -- break; -+ if (unlikely(jd_atom_is_v2)) { -+ if (copy_from_user(&user_atom.jc, user_addr, sizeof(struct base_jd_atom_v2)) != 0) { -+ dev_dbg(kbdev->dev, -+ "Invalid atom address %p passed to job_submit\n", -+ user_addr); -+ err = -EFAULT; -+ break; -+ } -+ -+ /* no seq_nr in v2 */ -+ user_atom.seq_nr = 0; -+ } else { -+ if (copy_from_user(&user_atom, user_addr, stride) != 0) { -+ dev_dbg(kbdev->dev, -+ "Invalid atom address %p passed to job_submit\n", -+ user_addr); -+ err = -EFAULT; -+ break; -+ } - } - - if (stride == offsetof(struct base_jd_atom_v2, renderpass_id)) { -@@ -1399,7 +1420,7 @@ void kbase_jd_done_worker(struct work_struct *data) - js_kctx_info = &kctx->jctx.sched_info; - js_devdata = &kbdev->js_data; - -- dev_dbg(kbdev->dev, "Enter atom %p done worker for kctx %p\n", -+ dev_dbg(kbdev->dev, "Enter atom %pK done worker for kctx %pK\n", - (void *)katom, (void *)kctx); - - KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER, kctx, katom, katom->jc, 0); -@@ -1423,7 +1444,7 @@ void kbase_jd_done_worker(struct work_struct *data) - if (katom->event_code == BASE_JD_EVENT_STOPPED) { - unsigned long flags; - -- dev_dbg(kbdev->dev, "Atom %p has been promoted to stopped\n", -+ dev_dbg(kbdev->dev, "Atom %pK has been promoted to stopped\n", - (void *)katom); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); -@@ -1431,7 +1452,7 @@ void kbase_jd_done_worker(struct work_struct *data) - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - katom->status = KBASE_JD_ATOM_STATE_IN_JS; -- dev_dbg(kctx->kbdev->dev, "Atom %p status to in JS\n", -+ dev_dbg(kctx->kbdev->dev, "Atom %pK status to in JS\n", - (void *)katom); - kbase_js_unpull(kctx, katom); - -@@ -1518,7 +1539,9 @@ void kbase_jd_done_worker(struct work_struct *data) - mutex_unlock(&jctx->lock); - - /* Job is now no longer running, so can now safely release the context -- * reference, and handle any actions that were logged against the atom's retained state */ -+ * reference, and handle any actions that were logged against the -+ * atom's retained state -+ */ - - kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state); - -@@ -1526,7 +1549,8 @@ void kbase_jd_done_worker(struct work_struct *data) - - if (!atomic_dec_return(&kctx->work_count)) { - /* If worker now idle then post all events that jd_done_nolock() -- * has queued */ -+ * has queued -+ */ - mutex_lock(&jctx->lock); - while (!list_empty(&kctx->completed_jobs)) { - struct kbase_jd_atom *atom = list_entry( -@@ -1546,7 +1570,7 @@ void kbase_jd_done_worker(struct work_struct *data) - - KBASE_KTRACE_ADD_JM(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0); - -- dev_dbg(kbdev->dev, "Leave atom %p done worker for kctx %p\n", -+ dev_dbg(kbdev->dev, "Leave atom %pK done worker for kctx %pK\n", - (void *)katom, (void *)kctx); - } - -@@ -1601,7 +1625,8 @@ static void jd_cancel_worker(struct work_struct *data) - need_to_try_schedule_context = jd_done_nolock(katom, NULL); - /* Because we're zapping, we're not adding any more jobs to this ctx, so no need to - * schedule the context. There's also no need for the jsctx_mutex to have been taken -- * around this too. */ -+ * around this too. -+ */ - KBASE_DEBUG_ASSERT(!need_to_try_schedule_context); - - /* katom may have been freed now, do not use! */ -@@ -1652,7 +1677,7 @@ void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr, - - atomic_inc(&kctx->work_count); - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - /* a failed job happened and is waiting for dumping*/ - if (!katom->will_fail_event_code && - kbase_debug_job_fault_process(katom, katom->event_code)) -@@ -1670,12 +1695,12 @@ void kbase_jd_cancel(struct kbase_device *kbdev, struct kbase_jd_atom *katom) - { - struct kbase_context *kctx; - -- KBASE_DEBUG_ASSERT(NULL != kbdev); -- KBASE_DEBUG_ASSERT(NULL != katom); -+ KBASE_DEBUG_ASSERT(kbdev != NULL); -+ KBASE_DEBUG_ASSERT(katom != NULL); - kctx = katom->kctx; -- KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(kctx != NULL); - -- dev_dbg(kbdev->dev, "JD: cancelling atom %p\n", (void *)katom); -+ dev_dbg(kbdev->dev, "JD: cancelling atom %pK\n", (void *)katom); - KBASE_KTRACE_ADD_JM(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0); - - /* This should only be done from a context that is not scheduled */ -@@ -1731,7 +1756,7 @@ void kbase_jd_zap_context(struct kbase_context *kctx) - flush_workqueue(kctx->dma_fence.wq); - #endif - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - kbase_debug_job_fault_kctx_unblock(kctx); - #endif - -@@ -1744,12 +1769,15 @@ int kbase_jd_init(struct kbase_context *kctx) - { - int i; - int mali_err = 0; -+ struct priority_control_manager_device *pcm_device = NULL; - - KBASE_DEBUG_ASSERT(kctx); -+ pcm_device = kctx->kbdev->pcm_dev; -+ kctx->jctx.max_priority = KBASE_JS_ATOM_SCHED_PRIO_REALTIME; - - kctx->jctx.job_done_wq = alloc_workqueue("mali_jd", - WQ_HIGHPRI | WQ_UNBOUND, 1); -- if (NULL == kctx->jctx.job_done_wq) { -+ if (kctx->jctx.job_done_wq == NULL) { - mali_err = -ENOMEM; - goto out1; - } -@@ -1785,6 +1813,11 @@ int kbase_jd_init(struct kbase_context *kctx) - INIT_LIST_HEAD(&kctx->completed_jobs); - atomic_set(&kctx->work_count, 0); - -+ /* Check if there are platform rules for maximum priority */ -+ if (pcm_device) -+ kctx->jctx.max_priority = pcm_device->ops.pcm_scheduler_priority_check( -+ pcm_device, current, KBASE_JS_ATOM_SCHED_PRIO_REALTIME); -+ - return 0; - - out1: -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c -index e9a161f..67d1bd9 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,9 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - - #include <linux/seq_file.h> - #include <mali_kbase.h> -@@ -29,7 +28,7 @@ - #if defined(CONFIG_SYNC) || defined(CONFIG_SYNC_FILE) - #include <mali_kbase_sync.h> - #endif --#include <mali_kbase_ioctl.h> -+#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h> - - struct kbase_jd_debugfs_depinfo { - u8 id; -@@ -47,13 +46,13 @@ static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, - case BASE_JD_REQ_SOFT_FENCE_TRIGGER: - res = kbase_sync_fence_out_info_get(atom, &info); - if (res == 0) -- seq_printf(sfile, "Sa([%p]%d) ", -+ seq_printf(sfile, "Sa([%pK]%d) ", - info.fence, info.status); - break; - case BASE_JD_REQ_SOFT_FENCE_WAIT: - res = kbase_sync_fence_in_info_get(atom, &info); - if (res == 0) -- seq_printf(sfile, "Wa([%p]%d) ", -+ seq_printf(sfile, "Wa([%pK]%d) ", - info.fence, info.status); - break; - default: -@@ -66,42 +65,40 @@ static void kbase_jd_debugfs_fence_info(struct kbase_jd_atom *atom, - struct kbase_fence_cb *cb; - - if (atom->dma_fence.fence) { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence = atom->dma_fence.fence; - #else - struct dma_fence *fence = atom->dma_fence.fence; - #endif - - seq_printf(sfile, --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) -- "Sd(%u#%u: %s) ", -+#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE) -+ "Sd(%u#%u: %s) ", - #else -- "Sd(%llu#%u: %s) ", -+ "Sd(%llu#%u: %s) ", - #endif -- fence->context, -- fence->seqno, -- dma_fence_is_signaled(fence) ? -- "signaled" : "active"); -+ fence->context, fence->seqno, -+ dma_fence_is_signaled(fence) ? "signaled" : -+ "active"); - } - - list_for_each_entry(cb, &atom->dma_fence.callbacks, - node) { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence = cb->fence; - #else - struct dma_fence *fence = cb->fence; - #endif - - seq_printf(sfile, --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) -- "Wd(%u#%u: %s) ", -+#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE) -+ "Wd(%u#%u: %s) ", - #else -- "Wd(%llu#%u: %s) ", -+ "Wd(%llu#%u: %s) ", - #endif -- fence->context, -- fence->seqno, -- dma_fence_is_signaled(fence) ? -- "signaled" : "active"); -+ fence->context, fence->seqno, -+ dma_fence_is_signaled(fence) ? "signaled" : -+ "active"); - } - } - #endif /* CONFIG_MALI_DMA_FENCE */ -@@ -180,7 +177,8 @@ static int kbasep_jd_debugfs_atoms_show(struct seq_file *sfile, void *data) - - /* start_timestamp is cleared as soon as the atom leaves UNUSED state - * and set before a job is submitted to the h/w, a non-zero value means -- * it is valid */ -+ * it is valid -+ */ - if (ktime_to_ns(atom->start_timestamp)) - start_timestamp = ktime_to_ns( - ktime_sub(ktime_get(), atom->start_timestamp)); -@@ -228,6 +226,12 @@ static const struct file_operations kbasep_jd_debugfs_atoms_fops = { - - void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) - { -+#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) -+ const mode_t mode = S_IRUGO; -+#else -+ const mode_t mode = S_IRUSR; -+#endif -+ - /* Caller already ensures this, but we keep the pattern for - * maintenance safety. - */ -@@ -236,7 +240,7 @@ void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx) - return; - - /* Expose all atoms */ -- debugfs_create_file("atoms", S_IRUGO, kctx->kctx_dentry, kctx, -+ debugfs_create_file("atoms", mode, kctx->kctx_dentry, kctx, - &kbasep_jd_debugfs_atoms_fops); - - } -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h -index 697bdef..8e6140c 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jd_debugfs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,13 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -- * @file mali_kbase_jd_debugfs.h -- * Header file for job dispatcher-related entries in debugfs -+ * DOC: Header file for job dispatcher-related entries in debugfs - */ - - #ifndef _KBASE_JD_DEBUGFS_H -@@ -38,7 +36,7 @@ struct kbase_context; - /** - * kbasep_jd_debugfs_ctx_init() - Add debugfs entries for JD system - * -- * @kctx Pointer to kbase_context -+ * @kctx: Pointer to kbase_context - */ - void kbasep_jd_debugfs_ctx_init(struct kbase_context *kctx); - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jm.c -index 3f17dd7..6995050 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jm.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * HW access job manager common APIs - */ -@@ -29,6 +27,7 @@ - #include "mali_kbase_hwaccess_jm.h" - #include "mali_kbase_jm.h" - -+#if !MALI_USE_CSF - /** - * kbase_jm_next_job() - Attempt to run the next @nr_jobs_to_submit jobs on slot - * @js on the active context. -@@ -46,7 +45,7 @@ static bool kbase_jm_next_job(struct kbase_device *kbdev, int js, - - kctx = kbdev->hwaccess.active_kctx[js]; - dev_dbg(kbdev->dev, -- "Trying to run the next %d jobs in kctx %p (s:%d)\n", -+ "Trying to run the next %d jobs in kctx %pK (s:%d)\n", - nr_jobs_to_submit, (void *)kctx, js); - - if (!kctx) -@@ -118,7 +117,7 @@ void kbase_jm_idle_ctx(struct kbase_device *kbdev, struct kbase_context *kctx) - - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - if (kbdev->hwaccess.active_kctx[js] == kctx) { -- dev_dbg(kbdev->dev, "Marking kctx %p as inactive (s:%d)\n", -+ dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%d)\n", - (void *)kctx, js); - kbdev->hwaccess.active_kctx[js] = NULL; - } -@@ -130,7 +129,7 @@ struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, - { - lockdep_assert_held(&kbdev->hwaccess_lock); - -- dev_dbg(kbdev->dev, "Atom %p is returning with event code 0x%x\n", -+ dev_dbg(kbdev->dev, "Atom %pK is returning with event code 0x%x\n", - (void *)katom, katom->event_code); - - if (katom->event_code != BASE_JD_EVENT_STOPPED && -@@ -149,3 +148,4 @@ struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, - - return kbase_js_complete_atom(katom, end_timestamp); - } -+#endif /* !MALI_USE_CSF */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jm.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jm.h -index a3c7744..c6b28f3 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_jm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2016, 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2013-2014, 2016, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - /* - * Job manager common APIs - */ -@@ -28,6 +26,7 @@ - #ifndef _KBASE_JM_H_ - #define _KBASE_JM_H_ - -+#if !MALI_USE_CSF - /** - * kbase_jm_kick() - Indicate that there are jobs ready to run. - * @kbdev: Device pointer -@@ -73,7 +72,9 @@ void kbase_jm_try_kick(struct kbase_device *kbdev, u32 js_mask); - * kbase_jm_kick_all() otherwise it will do nothing. - */ - void kbase_jm_try_kick_all(struct kbase_device *kbdev); -+#endif /* !MALI_USE_CSF */ - -+#if !MALI_USE_CSF - /** - * kbase_jm_idle_ctx() - Mark a context as idle. - * @kbdev: Device pointer -@@ -111,5 +112,6 @@ struct kbase_jd_atom *kbase_jm_return_atom_to_js(struct kbase_device *kbdev, - */ - struct kbase_jd_atom *kbase_jm_complete(struct kbase_device *kbdev, - struct kbase_jd_atom *katom, ktime_t *end_timestamp); -+#endif /* !MALI_USE_CSF */ - - #endif /* _KBASE_JM_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js.c -index 0b0c5bf..3682486 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Job Scheduler Implementation - */ -@@ -37,6 +34,7 @@ - - #include "mali_kbase_jm.h" - #include "mali_kbase_hwaccess_jm.h" -+#include <linux/priority_control_manager.h> - - /* - * Private types -@@ -45,26 +43,30 @@ - /* Bitpattern indicating the result of releasing a context */ - enum { - /* The context was descheduled - caller should try scheduling in a new -- * one to keep the runpool full */ -+ * one to keep the runpool full -+ */ - KBASEP_JS_RELEASE_RESULT_WAS_DESCHEDULED = (1u << 0), - /* Ctx attributes were changed - caller should try scheduling all -- * contexts */ -+ * contexts -+ */ - KBASEP_JS_RELEASE_RESULT_SCHED_ALL = (1u << 1) - }; - - typedef u32 kbasep_js_release_result; - - const int kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS] = { -- KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ -- KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ -- KBASE_JS_ATOM_SCHED_PRIO_LOW /* BASE_JD_PRIO_LOW */ -+ KBASE_JS_ATOM_SCHED_PRIO_MED, /* BASE_JD_PRIO_MEDIUM */ -+ KBASE_JS_ATOM_SCHED_PRIO_HIGH, /* BASE_JD_PRIO_HIGH */ -+ KBASE_JS_ATOM_SCHED_PRIO_LOW, /* BASE_JD_PRIO_LOW */ -+ KBASE_JS_ATOM_SCHED_PRIO_REALTIME /* BASE_JD_PRIO_REALTIME */ - }; - - const base_jd_prio - kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT] = { -- BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ -- BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ -- BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ -+ BASE_JD_PRIO_REALTIME, /* KBASE_JS_ATOM_SCHED_PRIO_REALTIME */ -+ BASE_JD_PRIO_HIGH, /* KBASE_JS_ATOM_SCHED_PRIO_HIGH */ -+ BASE_JD_PRIO_MEDIUM, /* KBASE_JS_ATOM_SCHED_PRIO_MED */ -+ BASE_JD_PRIO_LOW /* KBASE_JS_ATOM_SCHED_PRIO_LOW */ - }; - - -@@ -79,7 +81,7 @@ static int kbase_js_get_slot(struct kbase_device *kbdev, - struct kbase_jd_atom *katom); - - static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, -- kbasep_js_ctx_job_cb callback); -+ kbasep_js_ctx_job_cb *callback); - - /* Helper for ktrace */ - #if KBASE_KTRACE_ENABLE -@@ -160,7 +162,7 @@ jsctx_rb_none_to_pull_prio(struct kbase_context *kctx, int js, int prio) - none_to_pull = RB_EMPTY_ROOT(&rb->runnable_tree); - - dev_dbg(kctx->kbdev->dev, -- "Slot %d (prio %d) is %spullable in kctx %p\n", -+ "Slot %d (prio %d) is %spullable in kctx %pK\n", - js, prio, none_to_pull ? "not " : "", kctx); - - return none_to_pull; -@@ -184,7 +186,7 @@ jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - -- for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; -+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; - prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { - if (!jsctx_rb_none_to_pull_prio(kctx, js, prio)) - return false; -@@ -210,9 +212,8 @@ jsctx_rb_none_to_pull(struct kbase_context *kctx, int js) - * - * The HW access lock must always be held when calling this function. - */ --static void --jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, -- kbasep_js_ctx_job_cb callback) -+static void jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, -+ int prio, kbasep_js_ctx_job_cb *callback) - { - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - -@@ -234,7 +235,7 @@ jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, - WARN_ON(!(entry->core_req & - BASE_JD_REQ_END_RENDERPASS)); - dev_dbg(kctx->kbdev->dev, -- "Del runnable atom %p from X_DEP list\n", -+ "Del runnable atom %pK from X_DEP list\n", - (void *)entry); - - list_del(&entry->queue); -@@ -250,7 +251,7 @@ jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, - WARN_ON(!(entry->atom_flags & - KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST)); - dev_dbg(kctx->kbdev->dev, -- "Del blocked atom %p from X_DEP list\n", -+ "Del blocked atom %pK from X_DEP list\n", - (void *)entry); - - list_del(queue->x_dep_head.next); -@@ -271,13 +272,12 @@ jsctx_queue_foreach_prio(struct kbase_context *kctx, int js, int prio, - * jsctx_queue_foreach_prio() to iterate over the queue and invoke @callback - * for each entry, and remove the entry from the queue. - */ --static inline void --jsctx_queue_foreach(struct kbase_context *kctx, int js, -- kbasep_js_ctx_job_cb callback) -+static inline void jsctx_queue_foreach(struct kbase_context *kctx, int js, -+ kbasep_js_ctx_job_cb *callback) - { - int prio; - -- for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; -+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; - prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) - jsctx_queue_foreach_prio(kctx, js, prio, callback); - } -@@ -301,7 +301,7 @@ jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - dev_dbg(kctx->kbdev->dev, -- "Peeking runnable tree of kctx %p for prio %d (s:%d)\n", -+ "Peeking runnable tree of kctx %pK for prio %d (s:%d)\n", - (void *)kctx, prio, js); - - node = rb_first(&rb->runnable_tree); -@@ -319,7 +319,7 @@ jsctx_rb_peek_prio(struct kbase_context *kctx, int js, int prio) - * @js: Job slot id to check. - * - * Check the ring buffers for all priorities, starting from -- * KBASE_JS_ATOM_SCHED_PRIO_HIGH, for the specified @js and @prio and return a -+ * KBASE_JS_ATOM_SCHED_PRIO_REALTIME, for the specified @js and @prio and return a - * pointer to the next atom, unless all the priority's ring buffers are empty. - * - * Caller must hold the hwaccess_lock. -@@ -333,7 +333,7 @@ jsctx_rb_peek(struct kbase_context *kctx, int js) - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - -- for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; -+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; - prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { - struct kbase_jd_atom *katom; - -@@ -363,7 +363,7 @@ jsctx_rb_pull(struct kbase_context *kctx, struct kbase_jd_atom *katom) - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - -- dev_dbg(kctx->kbdev->dev, "Erasing atom %p from runnable tree of kctx %p\n", -+ dev_dbg(kctx->kbdev->dev, "Erasing atom %pK from runnable tree of kctx %pK\n", - (void *)katom, (void *)kctx); - - /* Atoms must be pulled in the correct order. */ -@@ -385,7 +385,7 @@ jsctx_tree_add(struct kbase_context *kctx, struct kbase_jd_atom *katom) - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); - -- dev_dbg(kbdev->dev, "Adding atom %p to runnable tree of kctx %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "Adding atom %pK to runnable tree of kctx %pK (s:%d)\n", - (void *)katom, (void *)kctx, js); - - while (*new) { -@@ -448,7 +448,8 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) - - #ifdef CONFIG_MALI_DEBUG - /* Soft-stop will be disabled on a single context by default unless -- * softstop_always is set */ -+ * softstop_always is set -+ */ - jsdd->softstop_always = false; - #endif /* CONFIG_MALI_DEBUG */ - jsdd->nr_all_contexts_running = 0; -@@ -531,14 +532,15 @@ int kbasep_js_devdata_init(struct kbase_device * const kbdev) - kbdev->gpu_props.props.raw_props.js_features[i]); - - /* On error, we could continue on: providing none of the below resources -- * rely on the ones above */ -+ * rely on the ones above -+ */ - - mutex_init(&jsdd->runpool_mutex); - mutex_init(&jsdd->queue_mutex); - sema_init(&jsdd->schedule_sem, 1); - - for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) { -- for (j = 0; j < KBASE_JS_ATOM_SCHED_PRIO_COUNT; ++j) { -+ for (j = KBASE_JS_ATOM_SCHED_PRIO_FIRST; j < KBASE_JS_ATOM_SCHED_PRIO_COUNT; ++j) { - INIT_LIST_HEAD(&jsdd->ctx_list_pullable[i][j]); - INIT_LIST_HEAD(&jsdd->ctx_list_unpullable[i][j]); - } -@@ -595,16 +597,18 @@ int kbasep_js_kctx_init(struct kbase_context *const kctx) - sizeof(js_kctx_info->ctx.ctx_attr_ref_count)); - - /* Initially, the context is disabled from submission until the create -- * flags are set */ -+ * flags are set -+ */ - kbase_ctx_flag_set(kctx, KCTX_SUBMIT_DISABLED); - - /* On error, we could continue on: providing none of the below resources -- * rely on the ones above */ -+ * rely on the ones above -+ */ - mutex_init(&js_kctx_info->ctx.jsctx_mutex); - - init_waitqueue_head(&js_kctx_info->ctx.is_scheduled_wait); - -- for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { -+ for (i = KBASE_JS_ATOM_SCHED_PRIO_FIRST; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { - for (j = 0; j < BASE_JM_MAX_NR_SLOTS; j++) { - INIT_LIST_HEAD(&kctx->jsctx_queue[i][j].x_dep_head); - kctx->jsctx_queue[i][j].runnable_tree = RB_ROOT; -@@ -678,7 +682,7 @@ static bool kbase_js_ctx_list_add_pullable_nolock(struct kbase_device *kbdev, - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); -- dev_dbg(kbdev->dev, "Add pullable tail kctx %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "Add pullable tail kctx %pK (s:%d)\n", - (void *)kctx, js); - - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) -@@ -720,7 +724,7 @@ static bool kbase_js_ctx_list_add_pullable_head_nolock( - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); -- dev_dbg(kbdev->dev, "Add pullable head kctx %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "Add pullable head kctx %pK (s:%d)\n", - (void *)kctx, js); - - if (!list_empty(&kctx->jctx.sched_info.ctx.ctx_list_entry[js])) -@@ -796,7 +800,7 @@ static bool kbase_js_ctx_list_add_unpullable_nolock(struct kbase_device *kbdev, - bool ret = false; - - lockdep_assert_held(&kbdev->hwaccess_lock); -- dev_dbg(kbdev->dev, "Add unpullable tail kctx %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "Add unpullable tail kctx %pK (s:%d)\n", - (void *)kctx, js); - - list_move_tail(&kctx->jctx.sched_info.ctx.ctx_list_entry[js], -@@ -879,7 +883,7 @@ static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( - - lockdep_assert_held(&kbdev->hwaccess_lock); - -- for (i = 0; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { -+ for (i = KBASE_JS_ATOM_SCHED_PRIO_FIRST; i < KBASE_JS_ATOM_SCHED_PRIO_COUNT; i++) { - if (list_empty(&kbdev->js_data.ctx_list_pullable[js][i])) - continue; - -@@ -889,7 +893,7 @@ static struct kbase_context *kbase_js_ctx_list_pop_head_nolock( - - list_del_init(&kctx->jctx.sched_info.ctx.ctx_list_entry[js]); - dev_dbg(kbdev->dev, -- "Popped %p from the pullable queue (s:%d)\n", -+ "Popped %pK from the pullable queue (s:%d)\n", - (void *)kctx, js); - return kctx; - } -@@ -943,25 +947,25 @@ static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, - - if (is_scheduled) { - if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { -- dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %p\n", -+ dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %pK\n", - (void *)kctx); - return false; - } - } - katom = jsctx_rb_peek(kctx, js); - if (!katom) { -- dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%d)\n", - (void *)kctx, js); - return false; /* No pullable atoms */ - } - if (kctx->blocked_js[js][katom->sched_priority]) { - dev_dbg(kbdev->dev, -- "JS: kctx %p is blocked from submitting atoms at priority %d (s:%d)\n", -+ "JS: kctx %pK is blocked from submitting atoms at priority %d (s:%d)\n", - (void *)kctx, katom->sched_priority, js); - return false; - } - if (atomic_read(&katom->blocked)) { -- dev_dbg(kbdev->dev, "JS: Atom %p is blocked in js_ctx_pullable\n", -+ dev_dbg(kbdev->dev, "JS: Atom %pK is blocked in js_ctx_pullable\n", - (void *)katom); - return false; /* next atom blocked */ - } -@@ -970,20 +974,20 @@ static bool kbase_js_ctx_pullable(struct kbase_context *kctx, int js, - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || - katom->x_pre_dep->will_fail_event_code) { - dev_dbg(kbdev->dev, -- "JS: X pre-dep %p is not present in slot FIFO or will fail\n", -+ "JS: X pre-dep %pK is not present in slot FIFO or will fail\n", - (void *)katom->x_pre_dep); - return false; - } - if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && - kbase_backend_nr_atoms_on_slot(kctx->kbdev, js)) { - dev_dbg(kbdev->dev, -- "JS: Atom %p has cross-slot fail dependency and atoms on slot (s:%d)\n", -+ "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%d)\n", - (void *)katom, js); - return false; - } - } - -- dev_dbg(kbdev->dev, "JS: Atom %p is pullable in kctx %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "JS: Atom %pK is pullable in kctx %pK (s:%d)\n", - (void *)katom, (void *)kctx, js); - - return true; -@@ -1007,7 +1011,7 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - int dep_prio = dep_atom->sched_priority; - - dev_dbg(kbdev->dev, -- "Checking dep %d of atom %p (s:%d) on %p (s:%d)\n", -+ "Checking dep %d of atom %pK (s:%d) on %pK (s:%d)\n", - i, (void *)katom, js, (void *)dep_atom, dep_js); - - /* Dependent atom must already have been submitted */ -@@ -1020,7 +1024,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - } - - /* Dependencies with different priorities can't -- be represented in the ringbuffer */ -+ * be represented in the ringbuffer -+ */ - if (prio != dep_prio) { - dev_dbg(kbdev->dev, - "Different atom priorities\n"); -@@ -1030,7 +1035,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - - if (js == dep_js) { - /* Only one same-slot dependency can be -- * represented in the ringbuffer */ -+ * represented in the ringbuffer -+ */ - if (has_dep) { - dev_dbg(kbdev->dev, - "Too many same-slot deps\n"); -@@ -1038,7 +1044,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - break; - } - /* Each dependee atom can only have one -- * same-slot dependency */ -+ * same-slot dependency -+ */ - if (dep_atom->post_dep) { - dev_dbg(kbdev->dev, - "Too many same-slot successors\n"); -@@ -1048,7 +1055,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - has_dep = true; - } else { - /* Only one cross-slot dependency can be -- * represented in the ringbuffer */ -+ * represented in the ringbuffer -+ */ - if (has_x_dep) { - dev_dbg(kbdev->dev, - "Too many cross-slot deps\n"); -@@ -1056,7 +1064,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - break; - } - /* Each dependee atom can only have one -- * cross-slot dependency */ -+ * cross-slot dependency -+ */ - if (dep_atom->x_post_dep) { - dev_dbg(kbdev->dev, - "Too many cross-slot successors\n"); -@@ -1064,7 +1073,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - break; - } - /* The dependee atom can not already be in the -- * HW access ringbuffer */ -+ * HW access ringbuffer -+ */ - if (dep_atom->gpu_rb_state != - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) { - dev_dbg(kbdev->dev, -@@ -1074,7 +1084,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - break; - } - /* The dependee atom can not already have -- * completed */ -+ * completed -+ */ - if (dep_atom->status != - KBASE_JD_ATOM_STATE_IN_JS) { - dev_dbg(kbdev->dev, -@@ -1092,7 +1103,8 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - } - - /* If dependencies can be represented by ringbuffer then clear them from -- * atom structure */ -+ * atom structure -+ */ - if (ret) { - for (i = 0; i < 2; i++) { - struct kbase_jd_atom *dep_atom = katom->dep[i].atom; -@@ -1101,7 +1113,7 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - int dep_js = kbase_js_get_slot(kbdev, dep_atom); - - dev_dbg(kbdev->dev, -- "Clearing dep %d of atom %p (s:%d) on %p (s:%d)\n", -+ "Clearing dep %d of atom %pK (s:%d) on %pK (s:%d)\n", - i, (void *)katom, js, (void *)dep_atom, - dep_js); - -@@ -1116,7 +1128,7 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - katom->atom_flags |= - KBASE_KATOM_FLAG_X_DEP_BLOCKED; - -- dev_dbg(kbdev->dev, "Set X_DEP flag on atom %p\n", -+ dev_dbg(kbdev->dev, "Set X_DEP flag on atom %pK\n", - (void *)katom); - - katom->x_pre_dep = dep_atom; -@@ -1140,7 +1152,7 @@ static bool kbase_js_dep_validate(struct kbase_context *kctx, - } - } else { - dev_dbg(kbdev->dev, -- "Deps of atom %p (s:%d) could not be represented\n", -+ "Deps of atom %pK (s:%d) could not be represented\n", - (void *)katom, js); - } - -@@ -1181,7 +1193,7 @@ void kbase_js_update_ctx_priority(struct kbase_context *kctx) - /* Determine the new priority for context, as per the priority - * of currently in-use atoms. - */ -- for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; -+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; - prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { - if (kctx->atoms_count[prio]) { - new_priority = prio; -@@ -1192,6 +1204,7 @@ void kbase_js_update_ctx_priority(struct kbase_context *kctx) - - kbase_js_set_ctx_priority(kctx, new_priority); - } -+KBASE_EXPORT_TEST_API(kbase_js_update_ctx_priority); - - /** - * js_add_start_rp() - Add an atom that starts a renderpass to the job scheduler -@@ -1222,7 +1235,7 @@ static int js_add_start_rp(struct kbase_jd_atom *const start_katom) - if (rp->state != KBASE_JD_RP_COMPLETE) - return -EINVAL; - -- dev_dbg(kctx->kbdev->dev, "JS add start atom %p of RP %d\n", -+ dev_dbg(kctx->kbdev->dev, "JS add start atom %pK of RP %d\n", - (void *)start_katom, start_katom->renderpass_id); - - /* The following members are read when updating the job slot -@@ -1265,7 +1278,7 @@ static int js_add_end_rp(struct kbase_jd_atom *const end_katom) - - rp = &kctx->jctx.renderpasses[end_katom->renderpass_id]; - -- dev_dbg(kbdev->dev, "JS add end atom %p in state %d of RP %d\n", -+ dev_dbg(kbdev->dev, "JS add end atom %pK in state %d of RP %d\n", - (void *)end_katom, (int)rp->state, end_katom->renderpass_id); - - if (rp->state == KBASE_JD_RP_COMPLETE) -@@ -1332,7 +1345,7 @@ bool kbasep_js_add_job(struct kbase_context *kctx, - /* Refcount ctx.nr_jobs */ - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs < U32_MAX); - ++(js_kctx_info->ctx.nr_jobs); -- dev_dbg(kbdev->dev, "Add atom %p to kctx %p; now %d in ctx\n", -+ dev_dbg(kbdev->dev, "Add atom %pK to kctx %pK; now %d in ctx\n", - (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); - - /* Lock for state available during IRQ */ -@@ -1345,13 +1358,14 @@ bool kbasep_js_add_job(struct kbase_context *kctx, - /* Dependencies could not be represented */ - --(js_kctx_info->ctx.nr_jobs); - dev_dbg(kbdev->dev, -- "Remove atom %p from kctx %p; now %d in ctx\n", -+ "Remove atom %pK from kctx %pK; now %d in ctx\n", - (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); - - /* Setting atom status back to queued as it still has unresolved -- * dependencies */ -+ * dependencies -+ */ - atom->status = KBASE_JD_ATOM_STATE_QUEUED; -- dev_dbg(kbdev->dev, "Atom %p status to queued\n", (void *)atom); -+ dev_dbg(kbdev->dev, "Atom %pK status to queued\n", (void *)atom); - - /* Undo the count, as the atom will get added again later but - * leave the context priority adjusted or boosted, in case if -@@ -1389,7 +1403,8 @@ bool kbasep_js_add_job(struct kbase_context *kctx, - kbdev, kctx, atom->slot_nr); - } - /* If this context is active and the atom is the first on its slot, -- * kick the job manager to attempt to fast-start the atom */ -+ * kick the job manager to attempt to fast-start the atom -+ */ - if (enqueue_required && kctx == - kbdev->hwaccess.active_kctx[atom->slot_nr]) - kbase_jm_try_kick(kbdev, 1 << atom->slot_nr); -@@ -1404,22 +1419,25 @@ bool kbasep_js_add_job(struct kbase_context *kctx, - if (kbase_ctx_flag(kctx, KCTX_DYING)) { - /* A job got added while/after kbase_job_zap_context() - * was called on a non-scheduled context. Kill that job -- * by killing the context. */ -+ * by killing the context. -+ */ - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, - false); - } else if (js_kctx_info->ctx.nr_jobs == 1) { - /* Handle Refcount going from 0 to 1: schedule the -- * context on the Queue */ -+ * context on the Queue -+ */ - KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -- dev_dbg(kbdev->dev, "JS: Enqueue Context %p", kctx); -+ dev_dbg(kbdev->dev, "JS: Enqueue Context %pK", kctx); - -- /* Queue was updated - caller must try to -- * schedule the head context */ -+ /* Queue was updated - caller must try to schedule the -+ * head context -+ */ - WARN_ON(!enqueue_required); - } - } - out_unlock: -- dev_dbg(kbdev->dev, "Enqueue of kctx %p is %srequired\n", -+ dev_dbg(kbdev->dev, "Enqueue of kctx %pK is %srequired\n", - kctx, enqueue_required ? "" : "not "); - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -@@ -1448,7 +1466,7 @@ void kbasep_js_remove_job(struct kbase_device *kbdev, - KBASE_DEBUG_ASSERT(js_kctx_info->ctx.nr_jobs > 0); - --(js_kctx_info->ctx.nr_jobs); - dev_dbg(kbdev->dev, -- "Remove atom %p from kctx %p; now %d in ctx\n", -+ "Remove atom %pK from kctx %pK; now %d in ctx\n", - (void *)atom, (void *)kctx, js_kctx_info->ctx.nr_jobs); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -@@ -1478,7 +1496,8 @@ bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev, - * - * This is because it returns false for soft-stopped atoms, but we - * want to override that, because we're cancelling an atom regardless of -- * whether it was soft-stopped or not */ -+ * whether it was soft-stopped or not -+ */ - attr_state_changed = kbasep_js_ctx_attr_ctx_release_atom(kbdev, kctx, - &katom_retained_state); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -@@ -1525,7 +1544,8 @@ static kbasep_js_release_result kbasep_js_run_jobs_after_ctx_and_atom_release( - - if (js_devdata->nr_user_contexts_running != 0 && runpool_ctx_attr_change) { - /* A change in runpool ctx attributes might mean we can -- * run more jobs than before */ -+ * run more jobs than before -+ */ - result = KBASEP_JS_RELEASE_RESULT_SCHED_ALL; - - KBASE_KTRACE_ADD_JM_SLOT(kbdev, JD_DONE_TRY_RUN_NEXT_JOB, -@@ -1624,7 +1644,8 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( - - /* Make a set of checks to see if the context should be scheduled out. - * Note that there'll always be at least 1 reference to the context -- * which was previously acquired by kbasep_js_schedule_ctx(). */ -+ * which was previously acquired by kbasep_js_schedule_ctx(). -+ */ - if (new_ref_count == 1 && - (!kbasep_js_is_submit_allowed(js_devdata, kctx) || - #ifdef CONFIG_MALI_ARBITER_SUPPORT -@@ -1635,8 +1656,9 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( - int slot; - - /* Last reference, and we've been told to remove this context -- * from the Run Pool */ -- dev_dbg(kbdev->dev, "JS: RunPool Remove Context %p because refcount=%d, jobs=%d, allowed=%d", -+ * from the Run Pool -+ */ -+ dev_dbg(kbdev->dev, "JS: RunPool Remove Context %pK because refcount=%d, jobs=%d, allowed=%d", - kctx, new_ref_count, js_kctx_info->ctx.nr_jobs, - kbasep_js_is_submit_allowed(js_devdata, kctx)); - -@@ -1646,7 +1668,7 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( - - for (slot = 0; slot < num_slots; slot++) { - if (kbdev->hwaccess.active_kctx[slot] == kctx) { -- dev_dbg(kbdev->dev, "Marking kctx %p as inactive (s:%d)\n", -+ dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%d)\n", - (void *)kctx, slot); - kbdev->hwaccess.active_kctx[slot] = NULL; - } -@@ -1662,7 +1684,8 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( - kbasep_js_ctx_attr_runpool_release_ctx(kbdev, kctx); - - /* Releasing the context and katom retained state can allow -- * more jobs to run */ -+ * more jobs to run -+ */ - release_result |= - kbasep_js_run_jobs_after_ctx_and_atom_release(kbdev, - kctx, katom_retained_state, -@@ -1702,7 +1725,8 @@ static kbasep_js_release_result kbasep_js_runpool_release_ctx_internal( - kbase_ctx_flag_clear(kctx, KCTX_SCHEDULED); - /* Signal any waiter that the context is not scheduled, so is - * safe for termination - once the jsctx_mutex is also dropped, -- * and jobs have finished. */ -+ * and jobs have finished. -+ */ - wake_up(&js_kctx_info->ctx.is_scheduled_wait); - - /* Queue an action to occur after we've dropped the lock */ -@@ -1744,9 +1768,10 @@ void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev, - - if (kbase_ctx_flag(kctx, KCTX_DYING)) { - /* Dying: don't requeue, but kill all jobs on the context. This -- * happens asynchronously */ -+ * happens asynchronously -+ */ - dev_dbg(kbdev->dev, -- "JS: ** Killing Context %p on RunPool Remove **", kctx); -+ "JS: ** Killing Context %pK on RunPool Remove **", kctx); - kbase_js_foreach_ctx_job(kctx, &kbase_jd_cancel); - } - } -@@ -1798,7 +1823,8 @@ void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev, - } - - /* Variant of kbasep_js_runpool_release_ctx() that doesn't call into -- * kbase_js_sched_all() */ -+ * kbase_js_sched_all() -+ */ - static void kbasep_js_runpool_release_ctx_no_schedule( - struct kbase_device *kbdev, struct kbase_context *kctx) - { -@@ -1851,7 +1877,7 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, - bool kctx_suspended = false; - int as_nr; - -- dev_dbg(kbdev->dev, "Scheduling kctx %p (s:%d)\n", kctx, js); -+ dev_dbg(kbdev->dev, "Scheduling kctx %pK (s:%d)\n", kctx, js); - - js_devdata = &kbdev->js_data; - js_kctx_info = &kctx->jctx.sched_info; -@@ -1867,7 +1893,8 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, - kbdev, kctx); - if (as_nr != KBASEP_AS_NR_INVALID) { - /* Attempt to retain the context again, this should -- * succeed */ -+ * succeed -+ */ - mutex_lock(&kbdev->mmu_hw_mutex); - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - as_nr = kbase_ctx_sched_retain_ctx(kctx); -@@ -1926,7 +1953,8 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, - KBASE_TLSTREAM_TL_RET_AS_CTX(kbdev, &kbdev->as[kctx->as_nr], kctx); - - /* Cause any future waiter-on-termination to wait until the context is -- * descheduled */ -+ * descheduled -+ */ - wake_up(&js_kctx_info->ctx.is_scheduled_wait); - - /* Re-check for suspending: a suspend could've occurred, and all the -@@ -1939,7 +1967,8 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, - * was taken (i.e. this condition doesn't execute), then the - * kbasep_js_suspend() code will cleanup this context instead (by virtue - * of it being called strictly after the suspend flag is set, and will -- * wait for this lock to drop) */ -+ * wait for this lock to drop) -+ */ - #ifdef CONFIG_MALI_ARBITER_SUPPORT - if (kbase_pm_is_suspending(kbdev) || kbase_pm_is_gpu_lost(kbdev)) { - #else -@@ -1967,13 +1996,15 @@ static bool kbasep_js_schedule_ctx(struct kbase_device *kbdev, - mutex_unlock(&js_devdata->runpool_mutex); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - /* Note: after this point, the context could potentially get scheduled -- * out immediately */ -+ * out immediately -+ */ - - if (kctx_suspended) { - /* Finishing forcing out the context due to a suspend. Use a - * variant of kbasep_js_runpool_release_ctx() that doesn't - * schedule a new context, to prevent a risk of recursion back -- * into this function */ -+ * into this function -+ */ - kbasep_js_runpool_release_ctx_no_schedule(kbdev, kctx); - return false; - } -@@ -1992,7 +2023,7 @@ static bool kbase_js_use_ctx(struct kbase_device *kbdev, - kbase_backend_use_ctx_sched(kbdev, kctx, js)) { - - dev_dbg(kbdev->dev, -- "kctx %p already has ASID - mark as active (s:%d)\n", -+ "kctx %pK already has ASID - mark as active (s:%d)\n", - (void *)kctx, js); - - if (kbdev->hwaccess.active_kctx[js] != kctx) { -@@ -2059,7 +2090,8 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, - kbase_js_sync_timers(kbdev); - - /* Fast-starting requires the jsctx_mutex to be dropped, -- * because it works on multiple ctxs */ -+ * because it works on multiple ctxs -+ */ - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); - -@@ -2071,7 +2103,8 @@ void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev, - kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - } else { - /* Already scheduled in - We need to retain it to keep the -- * corresponding address space */ -+ * corresponding address space -+ */ - WARN_ON(!kbase_ctx_sched_inc_refcount(kctx)); - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); - mutex_unlock(&js_devdata->queue_mutex); -@@ -2116,7 +2149,8 @@ void kbasep_js_suspend(struct kbase_device *kbdev) - js_devdata->runpool_irq.submit_allowed = 0; - - /* Retain each of the contexts, so we can cause it to leave even if it -- * had no refcount to begin with */ -+ * had no refcount to begin with -+ */ - for (i = BASE_MAX_NR_AS - 1; i >= 0; --i) { - struct kbase_context *kctx = kbdev->as_to_kctx[i]; - -@@ -2137,7 +2171,8 @@ void kbasep_js_suspend(struct kbase_device *kbdev) - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - /* De-ref the previous retain to ensure each context gets pulled out -- * sometime later. */ -+ * sometime later. -+ */ - for (i = 0; - i < BASE_MAX_NR_AS; - ++i, retained = retained >> 1) { -@@ -2148,7 +2183,8 @@ void kbasep_js_suspend(struct kbase_device *kbdev) - } - - /* Caller must wait for all Power Manager active references to be -- * dropped */ -+ * dropped -+ */ - } - - void kbasep_js_resume(struct kbase_device *kbdev) -@@ -2162,7 +2198,7 @@ void kbasep_js_resume(struct kbase_device *kbdev) - - mutex_lock(&js_devdata->queue_mutex); - for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) { -- for (prio = KBASE_JS_ATOM_SCHED_PRIO_HIGH; -+ for (prio = KBASE_JS_ATOM_SCHED_PRIO_FIRST; - prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT; prio++) { - struct kbase_context *kctx, *n; - unsigned long flags; -@@ -2283,7 +2319,8 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, - lockdep_assert_held(&kctx->jctx.lock); - - /* If slot will transition from unpullable to pullable then add to -- * pullable list */ -+ * pullable list -+ */ - if (jsctx_rb_none_to_pull(kctx, katom->slot_nr)) { - enqueue_required = true; - } else { -@@ -2297,7 +2334,7 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, - int js = katom->slot_nr; - struct jsctx_queue *queue = &kctx->jsctx_queue[prio][js]; - -- dev_dbg(kctx->kbdev->dev, "Add atom %p to X_DEP list (s:%d)\n", -+ dev_dbg(kctx->kbdev->dev, "Add atom %pK to X_DEP list (s:%d)\n", - (void *)katom, js); - - list_add_tail(&katom->queue, &queue->x_dep_head); -@@ -2307,7 +2344,7 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, - add_required = false; - } - } else { -- dev_dbg(kctx->kbdev->dev, "Atom %p not added to X_DEP list\n", -+ dev_dbg(kctx->kbdev->dev, "Atom %pK not added to X_DEP list\n", - (void *)katom); - } - -@@ -2321,7 +2358,7 @@ bool kbase_js_dep_resolved_submit(struct kbase_context *kctx, - } - - dev_dbg(kctx->kbdev->dev, -- "Enqueue of kctx %p is %srequired to submit atom %p\n", -+ "Enqueue of kctx %pK is %srequired to submit atom %pK\n", - kctx, enqueue_required ? "" : "not ", katom); - - return enqueue_required; -@@ -2348,7 +2385,7 @@ static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) - - if (!kbase_js_atom_blocked_on_x_dep(katom)) { - dev_dbg(kctx->kbdev->dev, -- "Del atom %p from X_DEP list in js_move_to_tree\n", -+ "Del atom %pK from X_DEP list in js_move_to_tree\n", - (void *)katom); - - list_del(&katom->queue); -@@ -2366,7 +2403,7 @@ static void kbase_js_move_to_tree(struct kbase_jd_atom *katom) - } - } else { - dev_dbg(kctx->kbdev->dev, -- "Atom %p blocked on x-dep in js_move_to_tree\n", -+ "Atom %pK blocked on x-dep in js_move_to_tree\n", - (void *)katom); - break; - } -@@ -2409,10 +2446,8 @@ static void kbase_js_evict_deps(struct kbase_context *kctx, - KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST))) { - /* Remove dependency.*/ - x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; -- trace_sysgraph(SGR_DEP_RES, kctx->id, -- kbase_jd_atom_id(kctx, x_dep)); - -- dev_dbg(kctx->kbdev->dev, "Cleared X_DEP flag on atom %p\n", -+ dev_dbg(kctx->kbdev->dev, "Cleared X_DEP flag on atom %pK\n", - (void *)x_dep); - - /* Fail if it had a data dependency. */ -@@ -2434,14 +2469,14 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) - KBASE_DEBUG_ASSERT(kctx); - - kbdev = kctx->kbdev; -- dev_dbg(kbdev->dev, "JS: pulling an atom from kctx %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "JS: pulling an atom from kctx %pK (s:%d)\n", - (void *)kctx, js); - - js_devdata = &kbdev->js_data; - lockdep_assert_held(&kbdev->hwaccess_lock); - - if (!kbasep_js_is_submit_allowed(js_devdata, kctx)) { -- dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %p\n", -+ dev_dbg(kbdev->dev, "JS: No submit allowed for kctx %pK\n", - (void *)kctx); - return NULL; - } -@@ -2454,25 +2489,26 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) - - katom = jsctx_rb_peek(kctx, js); - if (!katom) { -- dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "JS: No pullable atom in kctx %pK (s:%d)\n", - (void *)kctx, js); - return NULL; - } - if (kctx->blocked_js[js][katom->sched_priority]) { - dev_dbg(kbdev->dev, -- "JS: kctx %p is blocked from submitting atoms at priority %d (s:%d)\n", -+ "JS: kctx %pK is blocked from submitting atoms at priority %d (s:%d)\n", - (void *)kctx, katom->sched_priority, js); - return NULL; - } - if (atomic_read(&katom->blocked)) { -- dev_dbg(kbdev->dev, "JS: Atom %p is blocked in js_pull\n", -+ dev_dbg(kbdev->dev, "JS: Atom %pK is blocked in js_pull\n", - (void *)katom); - return NULL; - } - - /* Due to ordering restrictions when unpulling atoms on failure, we do - * not allow multiple runs of fail-dep atoms from the same context to be -- * present on the same slot */ -+ * present on the same slot -+ */ - if (katom->pre_dep && atomic_read(&kctx->atoms_pulled_slot[js])) { - struct kbase_jd_atom *prev_atom = - kbase_backend_inspect_tail(kbdev, js); -@@ -2486,14 +2522,14 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) - KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB || - katom->x_pre_dep->will_fail_event_code) { - dev_dbg(kbdev->dev, -- "JS: X pre-dep %p is not present in slot FIFO or will fail\n", -+ "JS: X pre-dep %pK is not present in slot FIFO or will fail\n", - (void *)katom->x_pre_dep); - return NULL; - } - if ((katom->atom_flags & KBASE_KATOM_FLAG_FAIL_BLOCKER) && - kbase_backend_nr_atoms_on_slot(kbdev, js)) { - dev_dbg(kbdev->dev, -- "JS: Atom %p has cross-slot fail dependency and atoms on slot (s:%d)\n", -+ "JS: Atom %pK has cross-slot fail dependency and atoms on slot (s:%d)\n", - (void *)katom, js); - return NULL; - } -@@ -2518,7 +2554,7 @@ struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js) - - katom->ticks = 0; - -- dev_dbg(kbdev->dev, "JS: successfully pulled atom %p from kctx %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "JS: successfully pulled atom %pK from kctx %pK (s:%d)\n", - (void *)katom, (void *)kctx, js); - - return katom; -@@ -2561,7 +2597,7 @@ static void js_return_of_start_rp(struct kbase_jd_atom *const start_katom) - return; - - dev_dbg(kctx->kbdev->dev, -- "JS return start atom %p in state %d of RP %d\n", -+ "JS return start atom %pK in state %d of RP %d\n", - (void *)start_katom, (int)rp->state, - start_katom->renderpass_id); - -@@ -2589,7 +2625,7 @@ static void js_return_of_start_rp(struct kbase_jd_atom *const start_katom) - /* Prevent the tiler job being pulled for execution in the - * job scheduler again. - */ -- dev_dbg(kbdev->dev, "Blocking start atom %p\n", -+ dev_dbg(kbdev->dev, "Blocking start atom %pK\n", - (void *)start_katom); - atomic_inc(&start_katom->blocked); - -@@ -2601,14 +2637,14 @@ static void js_return_of_start_rp(struct kbase_jd_atom *const start_katom) - /* Was the fragment job chain submitted to kbase yet? */ - end_katom = rp->end_katom; - if (end_katom) { -- dev_dbg(kctx->kbdev->dev, "JS return add end atom %p\n", -+ dev_dbg(kctx->kbdev->dev, "JS return add end atom %pK\n", - (void *)end_katom); - - if (rp->state == KBASE_JD_RP_RETRY_OOM) { - /* Allow the end of the renderpass to be pulled for - * execution again to continue incremental rendering. - */ -- dev_dbg(kbdev->dev, "Unblocking end atom %p\n", -+ dev_dbg(kbdev->dev, "Unblocking end atom %pK\n", - (void *)end_katom); - atomic_dec(&end_katom->blocked); - WARN_ON(!(end_katom->atom_flags & -@@ -2670,7 +2706,7 @@ static void js_return_of_end_rp(struct kbase_jd_atom *const end_katom) - return; - - dev_dbg(kctx->kbdev->dev, -- "JS return end atom %p in state %d of RP %d\n", -+ "JS return end atom %pK in state %d of RP %d\n", - (void *)end_katom, (int)rp->state, end_katom->renderpass_id); - - if (WARN_ON(rp->state != KBASE_JD_RP_OOM && -@@ -2692,14 +2728,14 @@ static void js_return_of_end_rp(struct kbase_jd_atom *const end_katom) - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - - dev_dbg(kbdev->dev, -- "Reset backing to %zu pages for region %p\n", -+ "Reset backing to %zu pages for region %pK\n", - reg->threshold_pages, (void *)reg); - - if (!WARN_ON(reg->flags & KBASE_REG_VA_FREED)) - kbase_mem_shrink(kctx, reg, reg->threshold_pages); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -- dev_dbg(kbdev->dev, "Deleting region %p from list\n", -+ dev_dbg(kbdev->dev, "Deleting region %pK from list\n", - (void *)reg); - list_del_init(®->link); - kbase_va_region_alloc_put(kctx, reg); -@@ -2717,7 +2753,7 @@ static void js_return_of_end_rp(struct kbase_jd_atom *const end_katom) - */ - start_katom = rp->start_katom; - if (!WARN_ON(!start_katom)) { -- dev_dbg(kbdev->dev, "Unblocking start atom %p\n", -+ dev_dbg(kbdev->dev, "Unblocking start atom %pK\n", - (void *)start_katom); - atomic_dec(&start_katom->blocked); - (void)kbase_js_ctx_list_add_pullable_head_nolock(kbdev, kctx, -@@ -2743,7 +2779,7 @@ static void js_return_worker(struct work_struct *data) - unsigned long flags; - base_jd_core_req core_req = katom->core_req; - -- dev_dbg(kbdev->dev, "%s for atom %p with event code 0x%x\n", -+ dev_dbg(kbdev->dev, "%s for atom %pK with event code 0x%x\n", - __func__, (void *)katom, katom->event_code); - - if (katom->event_code != BASE_JD_EVENT_END_RP_DONE) -@@ -2771,13 +2807,15 @@ static void js_return_worker(struct work_struct *data) - timer_sync |= kbase_js_ctx_list_remove_nolock(kbdev, kctx, js); - - /* If this slot has been blocked due to soft-stopped atoms, and all -- * atoms have now been processed, then unblock the slot */ -+ * atoms have now been processed, then unblock the slot -+ */ - if (!kctx->atoms_pulled_slot_pri[js][prio] && - kctx->blocked_js[js][prio]) { - kctx->blocked_js[js][prio] = false; - - /* Only mark the slot as pullable if the context is not idle - -- * that case is handled below */ -+ * that case is handled below -+ */ - if (atomic_read(&kctx->atoms_pulled) && - kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= kbase_js_ctx_list_add_pullable_nolock( -@@ -2786,12 +2824,12 @@ static void js_return_worker(struct work_struct *data) - - if (!atomic_read(&kctx->atoms_pulled)) { - dev_dbg(kbdev->dev, -- "No atoms currently pulled from context %p\n", -+ "No atoms currently pulled from context %pK\n", - (void *)kctx); - - if (!kctx->slots_pullable) { - dev_dbg(kbdev->dev, -- "Context %p %s counted as runnable\n", -+ "Context %pK %s counted as runnable\n", - (void *)kctx, - kbase_ctx_flag(kctx, KCTX_RUNNABLE_REF) ? - "is" : "isn't"); -@@ -2827,7 +2865,7 @@ static void js_return_worker(struct work_struct *data) - - if (context_idle) { - dev_dbg(kbdev->dev, -- "Context %p %s counted as active\n", -+ "Context %pK %s counted as active\n", - (void *)kctx, - kbase_ctx_flag(kctx, KCTX_ACTIVE) ? - "is" : "isn't"); -@@ -2866,13 +2904,13 @@ static void js_return_worker(struct work_struct *data) - - kbase_backend_complete_wq_post_sched(kbdev, core_req); - -- dev_dbg(kbdev->dev, "Leaving %s for atom %p\n", -+ dev_dbg(kbdev->dev, "Leaving %s for atom %pK\n", - __func__, (void *)katom); - } - - void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom) - { -- dev_dbg(kctx->kbdev->dev, "Unpulling atom %p in kctx %p\n", -+ dev_dbg(kctx->kbdev->dev, "Unpulling atom %pK in kctx %pK\n", - (void *)katom, (void *)kctx); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -@@ -2927,7 +2965,7 @@ static bool js_complete_start_rp(struct kbase_context *kctx, - return false; - - dev_dbg(kctx->kbdev->dev, -- "Start atom %p is done in state %d of RP %d\n", -+ "Start atom %pK is done in state %d of RP %d\n", - (void *)start_katom, (int)rp->state, - start_katom->renderpass_id); - -@@ -2939,7 +2977,7 @@ static bool js_complete_start_rp(struct kbase_context *kctx, - unsigned long flags; - - dev_dbg(kctx->kbdev->dev, -- "Start atom %p completed before soft-stop\n", -+ "Start atom %pK completed before soft-stop\n", - (void *)start_katom); - - kbase_gpu_vm_lock(kctx); -@@ -2951,7 +2989,7 @@ static bool js_complete_start_rp(struct kbase_context *kctx, - struct kbase_va_region, link); - - WARN_ON(reg->flags & KBASE_REG_VA_FREED); -- dev_dbg(kctx->kbdev->dev, "Deleting region %p from list\n", -+ dev_dbg(kctx->kbdev->dev, "Deleting region %pK from list\n", - (void *)reg); - list_del_init(®->link); - kbase_va_region_alloc_put(kctx, reg); -@@ -2961,7 +2999,7 @@ static bool js_complete_start_rp(struct kbase_context *kctx, - kbase_gpu_vm_unlock(kctx); - } else { - dev_dbg(kctx->kbdev->dev, -- "Start atom %p did not exceed memory threshold\n", -+ "Start atom %pK did not exceed memory threshold\n", - (void *)start_katom); - - WARN_ON(rp->state != KBASE_JD_RP_START && -@@ -2978,7 +3016,7 @@ static bool js_complete_start_rp(struct kbase_context *kctx, - /* Allow the end of the renderpass to be pulled for - * execution again to continue incremental rendering. - */ -- dev_dbg(kbdev->dev, "Unblocking end atom %p!\n", -+ dev_dbg(kbdev->dev, "Unblocking end atom %pK!\n", - (void *)end_katom); - atomic_dec(&end_katom->blocked); - -@@ -3022,7 +3060,7 @@ static void js_complete_end_rp(struct kbase_context *kctx, - if (WARN_ON(rp->end_katom != end_katom)) - return; - -- dev_dbg(kbdev->dev, "End atom %p is done in state %d of RP %d\n", -+ dev_dbg(kbdev->dev, "End atom %pK is done in state %d of RP %d\n", - (void *)end_katom, (int)rp->state, end_katom->renderpass_id); - - if (WARN_ON(rp->state == KBASE_JD_RP_COMPLETE) || -@@ -3056,7 +3094,7 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, - kbdev = kctx->kbdev; - atom_slot = katom->slot_nr; - -- dev_dbg(kbdev->dev, "%s for atom %p (s:%d)\n", -+ dev_dbg(kbdev->dev, "%s for atom %pK (s:%d)\n", - __func__, (void *)katom, atom_slot); - - /* Update the incremental rendering state machine. -@@ -3075,7 +3113,7 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - - if (katom->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { -- dev_dbg(kbdev->dev, "Atom %p is in runnable_tree\n", -+ dev_dbg(kbdev->dev, "Atom %pK is in runnable_tree\n", - (void *)katom); - - context_idle = !atomic_dec_return(&kctx->atoms_pulled); -@@ -3091,11 +3129,12 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, - } - - /* If this slot has been blocked due to soft-stopped atoms, and -- * all atoms have now been processed, then unblock the slot */ -+ * all atoms have now been processed, then unblock the slot -+ */ - if (!kctx->atoms_pulled_slot_pri[atom_slot][prio] - && kctx->blocked_js[atom_slot][prio]) { - dev_dbg(kbdev->dev, -- "kctx %p is no longer blocked from submitting on slot %d at priority %d\n", -+ "kctx %pK is no longer blocked from submitting on slot %d at priority %d\n", - (void *)kctx, atom_slot, prio); - - kctx->blocked_js[atom_slot][prio] = false; -@@ -3149,7 +3188,7 @@ bool kbase_js_complete_atom_wq(struct kbase_context *kctx, - * jd_done_worker(). - */ - if (context_idle) { -- dev_dbg(kbdev->dev, "kctx %p is no longer active\n", -+ dev_dbg(kbdev->dev, "kctx %pK is no longer active\n", - (void *)kctx); - kbase_ctx_flag_clear(kctx, KCTX_ACTIVE); - } -@@ -3200,7 +3239,7 @@ static bool js_end_rp_is_complete(struct kbase_jd_atom *const end_katom) - return true; - - dev_dbg(kbdev->dev, -- "JS complete end atom %p in state %d of RP %d\n", -+ "JS complete end atom %pK in state %d of RP %d\n", - (void *)end_katom, (int)rp->state, - end_katom->renderpass_id); - -@@ -3229,7 +3268,7 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, - struct kbase_jd_atom *x_dep = katom->x_post_dep; - - kbdev = kctx->kbdev; -- dev_dbg(kbdev->dev, "Atom %p complete in kctx %p (post-dep %p)\n", -+ dev_dbg(kbdev->dev, "Atom %pK complete in kctx %pK (post-dep %pK)\n", - (void *)katom, (void *)kctx, (void *)x_dep); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -@@ -3245,7 +3284,7 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, - katom->event_code = katom->will_fail_event_code; - - katom->status = KBASE_JD_ATOM_STATE_HW_COMPLETED; -- dev_dbg(kbdev->dev, "Atom %p status to HW completed\n", (void *)katom); -+ dev_dbg(kbdev->dev, "Atom %pK status to HW completed\n", (void *)katom); - - if (katom->event_code != BASE_JD_EVENT_DONE) { - kbase_js_evict_deps(kctx, katom, katom->slot_nr, -@@ -3267,9 +3306,7 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, - bool was_pullable = kbase_js_ctx_pullable(kctx, x_dep->slot_nr, - false); - x_dep->atom_flags &= ~KBASE_KATOM_FLAG_X_DEP_BLOCKED; -- trace_sysgraph(SGR_DEP_RES, kctx->id, -- kbase_jd_atom_id(katom->kctx, x_dep)); -- dev_dbg(kbdev->dev, "Cleared X_DEP flag on atom %p\n", -+ dev_dbg(kbdev->dev, "Cleared X_DEP flag on atom %pK\n", - (void *)x_dep); - - kbase_js_move_to_tree(x_dep); -@@ -3280,13 +3317,13 @@ struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom, - x_dep->slot_nr); - - if (x_dep->atom_flags & KBASE_KATOM_FLAG_JSCTX_IN_TREE) { -- dev_dbg(kbdev->dev, "Atom %p is in runnable tree\n", -+ dev_dbg(kbdev->dev, "Atom %pK is in runnable tree\n", - (void *)x_dep); - return x_dep; - } - } else { - dev_dbg(kbdev->dev, -- "No cross-slot dep to unblock for atom %p\n", -+ "No cross-slot dep to unblock for atom %pK\n", - (void *)katom); - } - -@@ -3317,13 +3354,13 @@ bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *const katom) - - if (!(katom->atom_flags & - KBASE_KATOM_FLAG_X_DEP_BLOCKED)) { -- dev_dbg(kbdev->dev, "Atom %p is not blocked on a cross-slot dependency", -+ dev_dbg(kbdev->dev, "Atom %pK is not blocked on a cross-slot dependency", - (void *)katom); - return false; - } - - if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS)) { -- dev_dbg(kbdev->dev, "Atom %p is blocked on a cross-slot dependency", -+ dev_dbg(kbdev->dev, "Atom %pK is blocked on a cross-slot dependency", - (void *)katom); - return true; - } -@@ -3349,12 +3386,12 @@ bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *const katom) - * if it only depends on the tiler job chain. - */ - if (katom->x_pre_dep != rp->start_katom) { -- dev_dbg(kbdev->dev, "Dependency is on %p not start atom %p\n", -+ dev_dbg(kbdev->dev, "Dependency is on %pK not start atom %pK\n", - (void *)katom->x_pre_dep, (void *)rp->start_katom); - return true; - } - -- dev_dbg(kbdev->dev, "Ignoring cross-slot dep on atom %p\n", -+ dev_dbg(kbdev->dev, "Ignoring cross-slot dep on atom %pK\n", - (void *)katom->x_pre_dep); - - return false; -@@ -3368,7 +3405,7 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) - bool ctx_waiting[BASE_JM_MAX_NR_SLOTS]; - int js; - -- dev_dbg(kbdev->dev, "%s kbdev %p mask 0x%x\n", -+ dev_dbg(kbdev->dev, "%s kbdev %pK mask 0x%x\n", - __func__, (void *)kbdev, (unsigned int)js_mask); - - js_devdata = &kbdev->js_data; -@@ -3403,7 +3440,7 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) - context_idle = true; - - dev_dbg(kbdev->dev, -- "kctx %p is not active (s:%d)\n", -+ "kctx %pK is not active (s:%d)\n", - (void *)kctx, js); - - if (kbase_pm_context_active_handle_suspend( -@@ -3412,7 +3449,8 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) - dev_dbg(kbdev->dev, - "Suspend pending (s:%d)\n", js); - /* Suspend pending - return context to -- * queue and stop scheduling */ -+ * queue and stop scheduling -+ */ - mutex_lock( - &kctx->jctx.sched_info.ctx.jsctx_mutex); - if (kbase_js_ctx_list_add_pullable_head( -@@ -3432,7 +3470,7 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) - &kctx->jctx.sched_info.ctx.jsctx_mutex); - - dev_dbg(kbdev->dev, -- "kctx %p cannot be used at this time\n", -+ "kctx %pK cannot be used at this time\n", - kctx); - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -@@ -3474,7 +3512,7 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) - bool pullable; - - dev_dbg(kbdev->dev, -- "No atoms pulled from kctx %p (s:%d)\n", -+ "No atoms pulled from kctx %pK (s:%d)\n", - (void *)kctx, js); - - pullable = kbase_js_ctx_pullable(kctx, js, -@@ -3483,7 +3521,8 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) - /* Failed to pull jobs - push to head of list. - * Unless this context is already 'active', in - * which case it's effectively already scheduled -- * so push it to the back of the list. */ -+ * so push it to the back of the list. -+ */ - if (pullable && kctx == last_active[js] && - kbase_ctx_flag(kctx, - (KCTX_PULLED_SINCE_ACTIVE_JS0 << -@@ -3508,7 +3547,8 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) - * slot, then we need to remove the active - * marker to prevent it from submitting atoms in - * the IRQ handler, which would prevent this -- * context from making progress. */ -+ * context from making progress. -+ */ - if (last_active[js] && kctx != last_active[js] - && kbase_js_ctx_pullable( - last_active[js], js, true)) -@@ -3534,7 +3574,7 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) - break; /* Could not run atoms on this slot */ - } - -- dev_dbg(kbdev->dev, "Push kctx %p to back of list\n", -+ dev_dbg(kbdev->dev, "Push kctx %pK to back of list\n", - (void *)kctx); - if (kbase_js_ctx_pullable(kctx, js, true)) - timer_sync |= -@@ -3556,7 +3596,7 @@ void kbase_js_sched(struct kbase_device *kbdev, int js_mask) - for (js = 0; js < BASE_JM_MAX_NR_SLOTS; js++) { - if (kbdev->hwaccess.active_kctx[js] == last_active[js] && - ctx_waiting[js]) { -- dev_dbg(kbdev->dev, "Marking kctx %p as inactive (s:%d)\n", -+ dev_dbg(kbdev->dev, "Marking kctx %pK as inactive (s:%d)\n", - (void *)last_active[js], js); - kbdev->hwaccess.active_kctx[js] = NULL; - } -@@ -3580,13 +3620,14 @@ void kbase_js_zap_context(struct kbase_context *kctx) - - /* First, atomically do the following: - * - mark the context as dying -- * - try to evict it from the queue */ -+ * - try to evict it from the queue -+ */ - mutex_lock(&kctx->jctx.lock); - mutex_lock(&js_devdata->queue_mutex); - mutex_lock(&js_kctx_info->ctx.jsctx_mutex); - kbase_ctx_flag_set(kctx, KCTX_DYING); - -- dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx); -+ dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %pK", kctx); - - /* - * At this point we know: -@@ -3650,13 +3691,14 @@ void kbase_js_zap_context(struct kbase_context *kctx) - - KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); - -- dev_dbg(kbdev->dev, "Zap: Ctx %p scheduled=0", kctx); -+ dev_dbg(kbdev->dev, "Zap: Ctx %pK scheduled=0", kctx); - - /* Only cancel jobs when we evicted from the - * queue. No Power Manager active reference was held. - * -- * Having is_dying set ensures that this kills, and -- * doesn't requeue */ -+ * Having is_dying set ensures that this kills, and doesn't -+ * requeue -+ */ - kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, false); - - mutex_unlock(&js_kctx_info->ctx.jsctx_mutex); -@@ -3667,9 +3709,10 @@ void kbase_js_zap_context(struct kbase_context *kctx) - bool was_retained; - - /* Case c: didn't evict, but it is scheduled - it's in the Run -- * Pool */ -+ * Pool -+ */ - KBASE_KTRACE_ADD_JM(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, kbase_ctx_flag(kctx, KCTX_SCHEDULED)); -- dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx); -+ dev_dbg(kbdev->dev, "Zap: Ctx %pK is in RunPool", kctx); - - /* Disable the ctx from submitting any more jobs */ - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -@@ -3678,18 +3721,21 @@ void kbase_js_zap_context(struct kbase_context *kctx) - - /* Retain and (later) release the context whilst it is is now - * disallowed from submitting jobs - ensures that someone -- * somewhere will be removing the context later on */ -+ * somewhere will be removing the context later on -+ */ - was_retained = kbase_ctx_sched_inc_refcount_nolock(kctx); - - /* Since it's scheduled and we have the jsctx_mutex, it must be -- * retained successfully */ -+ * retained successfully -+ */ - KBASE_DEBUG_ASSERT(was_retained); - -- dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx); -+ dev_dbg(kbdev->dev, "Zap: Ctx %pK Kill Any Running jobs", kctx); - - /* Cancel any remaining running jobs for this kctx - if any. - * Submit is disallowed which takes effect immediately, so no -- * more new jobs will appear after we do this. */ -+ * more new jobs will appear after we do this. -+ */ - kbase_backend_jm_kill_running_jobs_from_kctx(kctx); - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -@@ -3697,7 +3743,7 @@ void kbase_js_zap_context(struct kbase_context *kctx) - mutex_unlock(&js_devdata->queue_mutex); - mutex_unlock(&kctx->jctx.lock); - -- dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", -+ dev_dbg(kbdev->dev, "Zap: Ctx %pK Release (may or may not schedule out immediately)", - kctx); - - kbasep_js_runpool_release_ctx(kbdev, kctx); -@@ -3711,7 +3757,8 @@ void kbase_js_zap_context(struct kbase_context *kctx) - * to be destroyed, and the context to be de-scheduled (if it was on the - * runpool). - * -- * kbase_jd_zap_context() will do this. */ -+ * kbase_jd_zap_context() will do this. -+ */ - } - - static inline int trace_get_refcnt(struct kbase_device *kbdev, -@@ -3739,7 +3786,7 @@ static inline int trace_get_refcnt(struct kbase_device *kbdev, - * - it will be holding kbasep_js_kctx_info::ctx::jsctx_mutex. - */ - static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, -- kbasep_js_ctx_job_cb callback) -+ kbasep_js_ctx_job_cb *callback) - { - struct kbase_device *kbdev; - unsigned long flags; -@@ -3758,3 +3805,18 @@ static void kbase_js_foreach_ctx_job(struct kbase_context *kctx, - - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); - } -+ -+base_jd_prio kbase_js_priority_check(struct kbase_device *kbdev, base_jd_prio priority) -+{ -+ struct priority_control_manager_device *pcm_device = kbdev->pcm_dev; -+ int req_priority, out_priority; -+ base_jd_prio out_jd_priority = priority; -+ -+ if (pcm_device) { -+ req_priority = kbasep_js_atom_prio_to_sched_prio(priority); -+ out_priority = pcm_device->ops.pcm_scheduler_priority_check(pcm_device, current, req_priority); -+ out_jd_priority = kbasep_js_sched_prio_to_atom_prio(out_priority); -+ } -+ return out_jd_priority; -+} -+ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js.h -index 541acd4..96974c8 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_js.h -- * Job Scheduler APIs. -+ * DOC: Job Scheduler APIs. - */ - - #ifndef _KBASE_JS_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c -index 141d04a..7775648 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2016, 2018, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2016, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - #include <mali_kbase.h> - #include <mali_kbase_config.h> - -@@ -29,8 +27,11 @@ - */ - - /** -- * @brief Check whether a ctx has a certain attribute, and if so, retain that -+ * Check whether a ctx has a certain attribute, and if so, retain that - * attribute on the runpool. -+ * @kbdev: Device pointer -+ * @kctx: KBase context -+ * @attribute: Atribute to check/retain - * - * Requires: - * - jsctx mutex -@@ -75,8 +76,11 @@ static bool kbasep_js_ctx_attr_runpool_retain_attr(struct kbase_device *kbdev, s - } - - /** -- * @brief Check whether a ctx has a certain attribute, and if so, release that -+ * Check whether a ctx has a certain attribute, and if so, release that - * attribute on the runpool. -+ * @kbdev: Device pointer -+ * @kctx: KBase context -+ * @attribute: Atribute to release - * - * Requires: - * - jsctx mutex -@@ -120,8 +124,11 @@ static bool kbasep_js_ctx_attr_runpool_release_attr(struct kbase_device *kbdev, - } - - /** -- * @brief Retain a certain attribute on a ctx, also retaining it on the runpool -+ * Retain a certain attribute on a ctx, also retaining it on the runpool - * if the context is scheduled. -+ * @kbdev: Device pointer -+ * @kctx: KBase context -+ * @attribute: Atribute to retain - * - * Requires: - * - jsctx mutex -@@ -156,9 +163,12 @@ static bool kbasep_js_ctx_attr_ctx_retain_attr(struct kbase_device *kbdev, struc - return runpool_state_changed; - } - --/* -- * @brief Release a certain attribute on a ctx, also releasing it from the runpool -+/** -+ * Release a certain attribute on a ctx, also releasing it from the runpool - * if the context is scheduled. -+ * @kbdev: Device pointer -+ * @kctx: KBase context -+ * @attribute: Atribute to release - * - * Requires: - * - jsctx mutex -@@ -211,7 +221,8 @@ void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kb - - /* We don't need to know about state changed, because retaining a - * context occurs on scheduling it, and that itself will also try -- * to run new atoms */ -+ * to run new atoms -+ */ - CSTD_UNUSED(runpool_state_changed); - } - } -@@ -251,9 +262,9 @@ void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase - runpool_state_changed |= kbasep_js_ctx_attr_ctx_retain_attr(kbdev, kctx, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES); - } - -- /* We don't need to know about state changed, because retaining an -- * atom occurs on adding it, and that itself will also try to run -- * new atoms */ -+ /* We don't need to know about state changed, because retaining an atom -+ * occurs on adding it, and that itself will also try to run new atoms -+ */ - CSTD_UNUSED(runpool_state_changed); - } - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h -index 25fd397..6f29241 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_js_ctx_attr.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2015, 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2015, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,37 +17,19 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_js_ctx_attr.h -- * Job Scheduler Context Attribute APIs -+ * DOC: Job Scheduler Context Attribute APIs - */ - - #ifndef _KBASE_JS_CTX_ATTR_H_ - #define _KBASE_JS_CTX_ATTR_H_ - --/** -- * @addtogroup base_api -- * @{ -- */ -- --/** -- * @addtogroup base_kbase_api -- * @{ -- */ -- --/** -- * @addtogroup kbase_js -- * @{ -- */ -- - /** - * Retain all attributes of a context -+ * @kbdev: KBase device -+ * @kctx: KBase context - * - * This occurs on scheduling in the context on the runpool (but after - * is_scheduled is set) -@@ -60,6 +43,8 @@ void kbasep_js_ctx_attr_runpool_retain_ctx(struct kbase_device *kbdev, struct kb - - /** - * Release all attributes of a context -+ * @kbdev: KBase device -+ * @kctx: KBase context - * - * This occurs on scheduling out the context from the runpool (but before - * is_scheduled is cleared) -@@ -79,6 +64,9 @@ bool kbasep_js_ctx_attr_runpool_release_ctx(struct kbase_device *kbdev, struct k - - /** - * Retain all attributes of an atom -+ * @kbdev: KBase device -+ * @kctx: KBase context -+ * @katom: Atom - * - * This occurs on adding an atom to a context - * -@@ -90,6 +78,9 @@ void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase - - /** - * Release all attributes of an atom, given its retained state. -+ * @kbdev: KBase device -+ * @kctx: KBase context -+ * @katom_retained_state: Retained state - * - * This occurs after (permanently) removing an atom from a context - * -@@ -107,7 +98,7 @@ void kbasep_js_ctx_attr_ctx_retain_atom(struct kbase_device *kbdev, struct kbase - */ - bool kbasep_js_ctx_attr_ctx_release_atom(struct kbase_device *kbdev, struct kbase_context *kctx, struct kbasep_js_atom_retained_state *katom_retained_state); - --/** -+/* - * Requires: - * - runpool_irq spinlock - */ -@@ -122,7 +113,7 @@ static inline s8 kbasep_js_ctx_attr_count_on_runpool(struct kbase_device *kbdev, - return js_devdata->runpool_irq.ctx_attr_ref_count[attribute]; - } - --/** -+/* - * Requires: - * - runpool_irq spinlock - */ -@@ -132,7 +123,7 @@ static inline bool kbasep_js_ctx_attr_is_attr_on_runpool(struct kbase_device *kb - return (bool) kbasep_js_ctx_attr_count_on_runpool(kbdev, attribute); - } - --/** -+/* - * Requires: - * - jsctx mutex - */ -@@ -148,8 +139,4 @@ static inline bool kbasep_js_ctx_attr_is_attr_on_ctx(struct kbase_context *kctx, - return (bool) (js_kctx_info->ctx.ctx_attr_ref_count[attribute]); - } - -- /** @} *//* end group kbase_js */ -- /** @} *//* end group base_kbase_api */ -- /** @} *//* end group base_api */ -- - #endif /* _KBASE_JS_DEFS_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_kinstr_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_kinstr_jm.c -new file mode 100644 -index 0000000..1b23b41 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_kinstr_jm.c -@@ -0,0 +1,894 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * mali_kbase_kinstr_jm.c -+ * Kernel driver public interface to job manager atom tracing -+ */ -+ -+#include "mali_kbase_kinstr_jm.h" -+#include <uapi/gpu/arm/midgard/mali_kbase_kinstr_jm_reader.h> -+ -+#include "mali_kbase.h" -+#include "mali_kbase_linux.h" -+ -+#include <backend/gpu/mali_kbase_jm_rb.h> -+ -+#include <asm/barrier.h> -+#include <linux/anon_inodes.h> -+#include <linux/circ_buf.h> -+#include <linux/fs.h> -+#include <linux/kref.h> -+#include <linux/ktime.h> -+#include <linux/log2.h> -+#include <linux/mutex.h> -+#include <linux/rculist_bl.h> -+#include <linux/poll.h> -+#include <linux/slab.h> -+#include <linux/spinlock.h> -+#include <linux/version.h> -+#include <linux/wait.h> -+ -+#if KERNEL_VERSION(5, 1, 0) <= LINUX_VERSION_CODE -+#include <linux/build_bug.h> -+#else -+// Stringify the expression if no message is given. -+#define static_assert(e, ...) __static_assert(e, #__VA_ARGS__, #e) -+#define __static_assert(e, msg, ...) _Static_assert(e, msg) -+#endif -+ -+#if KERNEL_VERSION(4, 16, 0) >= LINUX_VERSION_CODE -+typedef unsigned int __poll_t; -+#endif -+ -+#ifndef ENOTSUP -+#define ENOTSUP EOPNOTSUPP -+#endif -+ -+/* The module printing prefix */ -+#define PR_ "mali_kbase_kinstr_jm: " -+ -+/* Allows us to perform ASM goto for the tracing -+ * https://www.kernel.org/doc/Documentation/static-keys.txt -+ */ -+DEFINE_STATIC_KEY_FALSE(basep_kinstr_jm_reader_static_key); -+ -+#define KBASE_KINSTR_JM_VERSION 2 -+ -+/** -+ * struct kbase_kinstr_jm - The context for the kernel job manager atom tracing -+ * @readers: a bitlocked list of opened readers. Readers are attached to the -+ * private data of a file descriptor that the user opens with the -+ * KBASE_IOCTL_KINSTR_JM_FD IO control call. -+ * @refcount: reference count for the context. Any reader will have a link -+ * back to the context so that they can remove themselves from the -+ * list. -+ * -+ * This is opaque outside this compilation unit -+ */ -+struct kbase_kinstr_jm { -+ struct hlist_bl_head readers; -+ struct kref refcount; -+}; -+ -+/** -+ * struct kbase_kinstr_jm_atom_state_change - Represents an atom changing to a -+ * new state -+ * @timestamp: Raw monotonic nanoseconds of the state change -+ * @state: The state that the atom has moved to -+ * @atom: The atom number that has changed state -+ * @flags: Flags associated with the state change. See -+ * KBASE_KINSTR_JM_ATOM_STATE_FLAG_* defines. -+ * @reserved: Reserved for future use. -+ * @data: Extra data for the state change. Active member depends on state. -+ * @data.start: Extra data for the state change. Active member depends on -+ * state. -+ * @data.start.slot: Extra data for the state change. Active member depends on -+ * state. -+ * @data.padding: Padding -+ * -+ * We can add new fields to the structure and old user code will gracefully -+ * ignore the new fields. -+ * -+ * We can change the size of the structure and old user code will gracefully -+ * skip over the new size via `struct kbase_kinstr_jm_fd_out->size`. -+ * -+ * If we remove fields, the version field in `struct -+ * kbase_kinstr_jm_fd_out->version` will be incremented and old user code will -+ * gracefully fail and tell the user that the kernel API is too new and has -+ * backwards-incompatible changes. Note that one userspace can opt to handle -+ * multiple kernel major versions of the structure. -+ * -+ * If we need to change the _meaning_ of one of the fields, i.e. the state -+ * machine has had a incompatible change, we can keep the same members in the -+ * structure and update the version as above. User code will no longer -+ * recognise that it has the supported field and can gracefully explain to the -+ * user that the kernel API is no longer supported. -+ * -+ * When making changes to this structure, make sure they are either: -+ * - additions to the end (for minor version bumps (i.e. only a size increase)) -+ * such that the layout of existing fields doesn't change, or; -+ * - update the version reported to userspace so that it can fail explicitly. -+ */ -+struct kbase_kinstr_jm_atom_state_change { -+ u64 timestamp; -+ s8 state; /* enum kbase_kinstr_jm_reader_atom_state */ -+ u8 atom; -+ u8 flags; -+ u8 reserved[1]; -+ /* Tagged union based on state. Ensure members are aligned correctly! */ -+ union { -+ struct { -+ u8 slot; -+ } start; -+ u8 padding[4]; -+ } data; -+}; -+static_assert( -+ ((1 << 8 * sizeof(((struct kbase_kinstr_jm_atom_state_change *)0)->state)) - 1) >= -+ KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT); -+ -+#define KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW BIT(0) -+ -+/** -+ * struct reader_changes - The circular buffer of kernel atom state changes -+ * @data: The allocated buffer. This is allocated when the user requests -+ * the reader file descriptor. It is released when the user calls -+ * close() on the fd. When accessing this, lock the producer spin -+ * lock to prevent races on the allocated memory. The consume lock -+ * does not need to be held because newly-inserted data will always -+ * be outside the currenly-read range. -+ * @producer: The producing spinlock which allows us to push changes into the -+ * buffer at the same time as a user read occurring. This needs to -+ * be locked when saving/restoring the IRQ because we can receive an -+ * interrupt from the GPU when an atom completes. The CPU could have -+ * a task preempted that is holding this lock. -+ * @consumer: The consuming mutex which locks around the user read(). -+ * Must be held when updating the tail of the circular buffer. -+ * @head: The head of the circular buffer. Can be used with Linux @c CIRC_ -+ * helpers. The producer should lock and update this with an SMP -+ * store when a new change lands. The consumer can read with an -+ * SMP load. This allows the producer to safely insert new changes -+ * into the circular buffer. -+ * @tail: The tail of the circular buffer. Can be used with Linux @c CIRC_ -+ * helpers. The producer should do a READ_ONCE load and the consumer -+ * should SMP store. -+ * @size: The number of changes that are allowed in @c data. Can be used -+ * with Linux @c CIRC_ helpers. Will always be a power of two. The -+ * producer lock should be held when updating this and stored with -+ * an SMP release memory barrier. This means that the consumer can -+ * do an SMP load. -+ * @threshold: The number of changes above which threads polling on the reader -+ * file descriptor will be woken up. -+ */ -+struct reader_changes { -+ struct kbase_kinstr_jm_atom_state_change *data; -+ spinlock_t producer; -+ struct mutex consumer; -+ u32 head; -+ u32 tail; -+ u32 size; -+ u32 threshold; -+}; -+ -+/** -+ * reader_changes_is_valid_size() - Determines if requested changes buffer size -+ * is valid. -+ * @size: The requested memory size -+ * -+ * We have a constraint that the underlying physical buffer must be a -+ * power of two so that we can use the efficient circular buffer helpers that -+ * the kernel provides. It also needs to be representable within a u32. -+ * -+ * Return: -+ * * true - the size is valid -+ * * false - the size is invalid -+ */ -+static inline bool reader_changes_is_valid_size(const size_t size) -+{ -+ typedef struct reader_changes changes_t; -+ const size_t elem_size = sizeof(*((changes_t *)0)->data); -+ const size_t size_size = sizeof(((changes_t *)0)->size); -+ const size_t size_max = (1ull << (size_size * 8)) - 1; -+ -+ return is_power_of_2(size) && /* Is a power of two */ -+ ((size / elem_size) <= size_max); /* Small enough */ -+} -+ -+/** -+ * reader_changes_init() - Initializes the reader changes and allocates the -+ * changes buffer -+ * @changes: The context pointer, must point to a zero-inited allocated reader -+ * changes structure. We may support allocating the structure in the -+ * future. -+ * @size: The requested changes buffer size -+ * -+ * Return: -+ * (0, U16_MAX] - the number of data elements allocated -+ * -EINVAL - a pointer was invalid -+ * -ENOTSUP - we do not support allocation of the context -+ * -ERANGE - the requested memory size was invalid -+ * -ENOMEM - could not allocate the memory -+ * -EADDRINUSE - the buffer memory was already allocated -+ */ -+static int reader_changes_init(struct reader_changes *const changes, -+ const size_t size) -+{ -+ BUILD_BUG_ON((PAGE_SIZE % sizeof(*changes->data)) != 0); -+ -+ if (!reader_changes_is_valid_size(size)) { -+ pr_warn(PR_ "invalid size %zu\n", size); -+ return -ERANGE; -+ } -+ -+ changes->data = vmalloc(size); -+ if (!changes->data) -+ return -ENOMEM; -+ -+ spin_lock_init(&changes->producer); -+ mutex_init(&changes->consumer); -+ -+ changes->size = size / sizeof(*changes->data); -+ changes->threshold = min(((size_t)(changes->size)) / 4, -+ ((size_t)(PAGE_SIZE)) / sizeof(*changes->data)); -+ -+ return changes->size; -+} -+ -+/** -+ * reader_changes_term() - Cleans up a reader changes structure -+ * @changes: The context to clean up -+ * -+ * Releases the allocated state changes memory -+ */ -+static void reader_changes_term(struct reader_changes *const changes) -+{ -+ struct kbase_kinstr_jm_atom_state_change *data = NULL; -+ unsigned long irq; -+ -+ /* -+ * Although changes->data is used on the consumer side, too, no active -+ * consumer is possible by the time we clean up the reader changes, so -+ * no need to take the consumer lock. However, we do need the producer -+ * lock because the list removal can race with list traversal. -+ */ -+ spin_lock_irqsave(&changes->producer, irq); -+ swap(changes->data, data); -+ spin_unlock_irqrestore(&changes->producer, irq); -+ -+ mutex_destroy(&changes->consumer); -+ vfree(data); -+} -+ -+/** -+ * reader_changes_count_locked() - Retrieves the count of state changes from the -+ * tail to the physical end of the buffer -+ * @changes: The state changes context -+ * -+ * The consumer mutex must be held. Uses the CIRC_CNT_TO_END macro to -+ * determine the count, so there may be more items. However, that's the maximum -+ * number that can be read in one contiguous read. -+ * -+ * Return: the number of changes in the circular buffer until the end of the -+ * allocation -+ */ -+static u32 reader_changes_count_locked(struct reader_changes *const changes) -+{ -+ u32 head; -+ -+ lockdep_assert_held_once(&changes->consumer); -+ -+ head = smp_load_acquire(&changes->head); -+ -+ return CIRC_CNT_TO_END(head, changes->tail, changes->size); -+} -+ -+/** -+ * reader_changes_count() - Retrieves the count of state changes from the -+ * tail to the physical end of the buffer -+ * @changes: The state changes context -+ * -+ * Return: the number of changes in the circular buffer until the end of the -+ * allocation -+ */ -+static u32 reader_changes_count(struct reader_changes *const changes) -+{ -+ u32 ret; -+ -+ mutex_lock(&changes->consumer); -+ ret = reader_changes_count_locked(changes); -+ mutex_unlock(&changes->consumer); -+ return ret; -+} -+ -+/** -+ * reader_changes_push() - Pushes a change into the reader circular buffer. -+ * @changes: The buffer to insert the change into -+ * @change: Kernel atom change to insert -+ * @wait_queue: The queue to be kicked when changes should be read from -+ * userspace. Kicked when a threshold is reached or there is -+ * overflow. -+ */ -+static void reader_changes_push( -+ struct reader_changes *const changes, -+ const struct kbase_kinstr_jm_atom_state_change *const change, -+ wait_queue_head_t *const wait_queue) -+{ -+ u32 head, tail, size, space; -+ unsigned long irq; -+ struct kbase_kinstr_jm_atom_state_change *data; -+ -+ spin_lock_irqsave(&changes->producer, irq); -+ -+ /* We may be called for a reader_changes that's awaiting cleanup. */ -+ data = changes->data; -+ if (!data) -+ goto unlock; -+ -+ size = changes->size; -+ head = changes->head; -+ tail = smp_load_acquire(&changes->tail); -+ -+ space = CIRC_SPACE(head, tail, size); -+ if (space >= 1) { -+ data[head] = *change; -+ if (space == 1) { -+ data[head].flags |= -+ KBASE_KINSTR_JM_ATOM_STATE_FLAG_OVERFLOW; -+ pr_warn(PR_ "overflow of circular buffer\n"); -+ } -+ smp_store_release(&changes->head, (head + 1) & (size - 1)); -+ } -+ -+ /* Wake for either overflow or over-threshold cases. */ -+ if (CIRC_CNT(head + 1, tail, size) >= changes->threshold) -+ wake_up_interruptible(wait_queue); -+ -+unlock: -+ spin_unlock_irqrestore(&changes->producer, irq); -+} -+ -+/** -+ * struct reader - Allows the kernel state changes to be read by user space. -+ * @node: The node in the @c readers locked list -+ * @rcu_head: storage for the RCU callback to free this reader (see kfree_rcu) -+ * @changes: The circular buffer of user changes -+ * @wait_queue: A wait queue for poll -+ * @context: a pointer to the parent context that created this reader. Can be -+ * used to remove the reader from the list of readers. Reference -+ * counted. -+ * -+ * The reader is a circular buffer in kernel space. State changes are pushed -+ * into the buffer. The flow from user space is: -+ * -+ * * Request file descriptor with KBASE_IOCTL_KINSTR_JM_FD. This will -+ * allocate the kernel side circular buffer with a size specified in the -+ * ioctl argument. -+ * * The user will then poll the file descriptor for data -+ * * Upon receiving POLLIN, perform a read() on the file descriptor to get -+ * the data out. -+ * * The buffer memory will be freed when the file descriptor is closed -+ */ -+struct reader { -+ struct hlist_bl_node node; -+ struct rcu_head rcu_head; -+ struct reader_changes changes; -+ wait_queue_head_t wait_queue; -+ struct kbase_kinstr_jm *context; -+}; -+ -+static struct kbase_kinstr_jm * -+kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx); -+static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx); -+static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, -+ struct reader *const reader); -+static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, -+ struct reader *const reader); -+ -+/** -+ * reader_term() - Terminate a instrumentation job manager reader context. -+ * @reader: Pointer to context to be terminated. -+ */ -+static void reader_term(struct reader *const reader) -+{ -+ if (!reader) -+ return; -+ -+ kbase_kinstr_jm_readers_del(reader->context, reader); -+ reader_changes_term(&reader->changes); -+ kbase_kinstr_jm_ref_put(reader->context); -+ -+ kfree_rcu(reader, rcu_head); -+} -+ -+/** -+ * reader_init() - Initialise a instrumentation job manager reader context. -+ * @out_reader: Non-NULL pointer to where the pointer to the created context -+ * will be stored on success. -+ * @ctx: the pointer to the parent context. Reference count will be -+ * increased if initialization is successful -+ * @num_changes: The number of changes to allocate a buffer for -+ * -+ * Return: 0 on success, else error code. -+ */ -+static int reader_init(struct reader **const out_reader, -+ struct kbase_kinstr_jm *const ctx, -+ size_t const num_changes) -+{ -+ struct reader *reader = NULL; -+ const size_t change_size = sizeof(struct kbase_kinstr_jm_atom_state_change); -+ int status; -+ -+ if (!out_reader || !ctx || !num_changes) -+ return -EINVAL; -+ -+ reader = kzalloc(sizeof(*reader), GFP_KERNEL); -+ if (!reader) -+ return -ENOMEM; -+ -+ INIT_HLIST_BL_NODE(&reader->node); -+ init_waitqueue_head(&reader->wait_queue); -+ -+ reader->context = kbase_kinstr_jm_ref_get(ctx); -+ -+ status = reader_changes_init(&reader->changes, num_changes * change_size); -+ if (status < 0) -+ goto fail; -+ -+ status = kbase_kinstr_jm_readers_add(ctx, reader); -+ if (status < 0) -+ goto fail; -+ -+ *out_reader = reader; -+ -+ return 0; -+ -+fail: -+ kbase_kinstr_jm_ref_put(reader->context); -+ kfree(reader); -+ return status; -+} -+ -+/** -+ * reader_release() - Invoked when the reader file descriptor is released -+ * @node: The inode that the file descriptor that the file corresponds to. In -+ * our case our reader file descriptor is backed by an anonymous node so -+ * not much is in this. -+ * @file: the file data. Our reader context is held in the private data -+ * Return: zero on success -+ */ -+static int reader_release(struct inode *const node, struct file *const file) -+{ -+ struct reader *const reader = file->private_data; -+ -+ reader_term(reader); -+ file->private_data = NULL; -+ -+ return 0; -+} -+ -+/** -+ * reader_changes_copy_to_user() - Copy any changes from a changes structure to -+ * the user-provided buffer. -+ * @changes: The changes structure from which to copy. -+ * @buffer: The user buffer to copy the data to. -+ * @buffer_size: The number of bytes in the buffer. -+ * Return: The number of bytes copied or negative errno on failure. -+ */ -+static ssize_t reader_changes_copy_to_user(struct reader_changes *const changes, -+ char __user *buffer, -+ size_t buffer_size) -+{ -+ ssize_t ret = 0; -+ struct kbase_kinstr_jm_atom_state_change const *src_buf = READ_ONCE( -+ changes->data); -+ size_t const entry_size = sizeof(*src_buf); -+ size_t changes_tail, changes_count, read_size; -+ -+ /* Needed for the quick buffer capacity calculation below. -+ * Note that we can't use is_power_of_2() since old compilers don't -+ * understand it's a constant expression. -+ */ -+#define is_power_of_two(x) ((x) && !((x) & ((x) - 1))) -+ static_assert(is_power_of_two( -+ sizeof(struct kbase_kinstr_jm_atom_state_change))); -+#undef is_power_of_two -+ -+ lockdep_assert_held_once(&changes->consumer); -+ -+ /* Read continuously until either: -+ * - we've filled the output buffer, or -+ * - there are no changes when we check. -+ * -+ * If more changes arrive while we're copying to the user, we can copy -+ * those as well, space permitting. -+ */ -+ do { -+ changes_tail = changes->tail; -+ changes_count = reader_changes_count_locked(changes); -+ read_size = min(changes_count * entry_size, -+ buffer_size & ~(entry_size - 1)); -+ -+ if (!read_size) -+ break; -+ -+ if (copy_to_user(buffer, &(src_buf[changes_tail]), read_size)) -+ return -EFAULT; -+ -+ buffer += read_size; -+ buffer_size -= read_size; -+ ret += read_size; -+ changes_tail = (changes_tail + read_size / entry_size) & -+ (changes->size - 1); -+ smp_store_release(&changes->tail, changes_tail); -+ } while (read_size); -+ -+ return ret; -+} -+ -+/** -+ * reader_read() - Handles a read call on the reader file descriptor -+ * -+ * @filp: The file that the read was performed on -+ * @buffer: The destination buffer -+ * @buffer_size: The maximum number of bytes to read -+ * @offset: The offset into the 'file' to read from. -+ * -+ * Note the destination buffer needs to be fully mapped in userspace or the read -+ * will fault. -+ * -+ * Return: -+ * * The number of bytes read or: -+ * * -EBADF - the file descriptor did not have an attached reader -+ * * -EFAULT - memory access fault -+ * * -EAGAIN - if the file is set to nonblocking reads with O_NONBLOCK and there -+ * is no data available -+ * -+ * Note: The number of bytes read will always be a multiple of the size of an -+ * entry. -+ */ -+static ssize_t reader_read(struct file *const filp, -+ char __user *const buffer, -+ size_t const buffer_size, -+ loff_t *const offset) -+{ -+ struct reader *const reader = filp->private_data; -+ struct reader_changes *changes; -+ ssize_t ret; -+ -+ if (!reader) -+ return -EBADF; -+ -+ if (buffer_size < sizeof(struct kbase_kinstr_jm_atom_state_change)) -+ return -ENOBUFS; -+ -+#if KERNEL_VERSION(5, 0, 0) <= LINUX_VERSION_CODE -+ if (!access_ok(buffer, buffer_size)) -+ return -EIO; -+#else -+ if (!access_ok(VERIFY_WRITE, buffer, buffer_size)) -+ return -EIO; -+#endif -+ -+ changes = &reader->changes; -+ -+ mutex_lock(&changes->consumer); -+ if (!reader_changes_count_locked(changes)) { -+ if (filp->f_flags & O_NONBLOCK) { -+ ret = -EAGAIN; -+ goto exit; -+ } -+ -+ if (wait_event_interruptible( -+ reader->wait_queue, -+ !!reader_changes_count_locked(changes))) { -+ ret = -EINTR; -+ goto exit; -+ } -+ } -+ -+ ret = reader_changes_copy_to_user(changes, buffer, buffer_size); -+ -+exit: -+ mutex_unlock(&changes->consumer); -+ return ret; -+} -+ -+/** -+ * reader_poll() - Handles a poll call on the reader file descriptor -+ * @file: The file that the poll was performed on -+ * @wait: The poll table -+ * -+ * The results of the poll will be unreliable if there is no mapped memory as -+ * there is no circular buffer to push atom state changes into. -+ * -+ * Return: -+ * * 0 - no data ready -+ * * POLLIN - state changes have been buffered -+ * * -EBADF - the file descriptor did not have an attached reader -+ * * -EINVAL - the IO control arguments were invalid -+ */ -+static __poll_t reader_poll(struct file *const file, -+ struct poll_table_struct *const wait) -+{ -+ struct reader *reader; -+ struct reader_changes *changes; -+ -+ if (unlikely(!file || !wait)) -+ return -EINVAL; -+ -+ reader = file->private_data; -+ if (unlikely(!reader)) -+ return -EBADF; -+ -+ changes = &reader->changes; -+ -+ if (reader_changes_count(changes) >= changes->threshold) -+ return POLLIN; -+ -+ poll_wait(file, &reader->wait_queue, wait); -+ -+ return (reader_changes_count(changes) > 0) ? POLLIN : 0; -+} -+ -+/* The file operations virtual function table */ -+static const struct file_operations file_operations = { -+ .owner = THIS_MODULE, -+ .llseek = no_llseek, -+ .read = reader_read, -+ .poll = reader_poll, -+ .release = reader_release -+}; -+ -+/* The maximum amount of readers that can be created on a context. */ -+static const size_t kbase_kinstr_jm_readers_max = 16; -+ -+/** -+ * kbasep_kinstr_jm_release() - Invoked when the reference count is dropped -+ * @ref: the context reference count -+ */ -+static void kbase_kinstr_jm_release(struct kref *const ref) -+{ -+ struct kbase_kinstr_jm *const ctx = -+ container_of(ref, struct kbase_kinstr_jm, refcount); -+ -+ kfree(ctx); -+} -+ -+/** -+ * kbase_kinstr_jm_ref_get() - Reference counts the instrumentation context -+ * @ctx: the context to reference count -+ * Return: the reference counted context -+ */ -+static struct kbase_kinstr_jm * -+kbase_kinstr_jm_ref_get(struct kbase_kinstr_jm *const ctx) -+{ -+ if (likely(ctx)) -+ kref_get(&ctx->refcount); -+ return ctx; -+} -+ -+/** -+ * kbase_kinstr_jm_ref_put() - Dereferences the instrumentation context -+ * @ctx: the context to lower the reference count on -+ */ -+static void kbase_kinstr_jm_ref_put(struct kbase_kinstr_jm *const ctx) -+{ -+ if (likely(ctx)) -+ kref_put(&ctx->refcount, kbase_kinstr_jm_release); -+} -+ -+/** -+ * kbase_kinstr_jm_readers_add() - Adds a reader to the list of readers -+ * @ctx: the instrumentation context -+ * @reader: the reader to add -+ * -+ * Return: -+ * 0 - success -+ * -ENOMEM - too many readers already added. -+ */ -+static int kbase_kinstr_jm_readers_add(struct kbase_kinstr_jm *const ctx, -+ struct reader *const reader) -+{ -+ struct hlist_bl_head *const readers = &ctx->readers; -+ struct hlist_bl_node *node; -+ struct reader *temp; -+ size_t count = 0; -+ -+ hlist_bl_lock(readers); -+ -+ hlist_bl_for_each_entry_rcu(temp, node, readers, node) -+ ++count; -+ -+ if (kbase_kinstr_jm_readers_max < count) { -+ hlist_bl_unlock(readers); -+ return -ENOMEM; -+ } -+ -+ hlist_bl_add_head_rcu(&reader->node, readers); -+ -+ hlist_bl_unlock(readers); -+ -+ static_branch_inc(&basep_kinstr_jm_reader_static_key); -+ -+ return 0; -+} -+ -+/** -+ * readers_del() - Deletes a reader from the list of readers -+ * @ctx: the instrumentation context -+ * @reader: the reader to delete -+ */ -+static void kbase_kinstr_jm_readers_del(struct kbase_kinstr_jm *const ctx, -+ struct reader *const reader) -+{ -+ struct hlist_bl_head *const readers = &ctx->readers; -+ -+ hlist_bl_lock(readers); -+ hlist_bl_del_rcu(&reader->node); -+ hlist_bl_unlock(readers); -+ -+ static_branch_dec(&basep_kinstr_jm_reader_static_key); -+} -+ -+int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, -+ union kbase_kinstr_jm_fd *jm_fd_arg) -+{ -+ struct kbase_kinstr_jm_fd_in const *in; -+ struct reader *reader; -+ size_t const change_size = sizeof(struct -+ kbase_kinstr_jm_atom_state_change); -+ int status; -+ int fd; -+ int i; -+ -+ if (!ctx || !jm_fd_arg) -+ return -EINVAL; -+ -+ in = &jm_fd_arg->in; -+ -+ if (!is_power_of_2(in->count)) -+ return -EINVAL; -+ -+ for (i = 0; i < sizeof(in->padding); ++i) -+ if (in->padding[i]) -+ return -EINVAL; -+ -+ status = reader_init(&reader, ctx, in->count); -+ if (status < 0) -+ return status; -+ -+ jm_fd_arg->out.version = KBASE_KINSTR_JM_VERSION; -+ jm_fd_arg->out.size = change_size; -+ memset(&jm_fd_arg->out.padding, 0, sizeof(jm_fd_arg->out.padding)); -+ -+ fd = anon_inode_getfd("[mali_kinstr_jm]", &file_operations, reader, -+ O_CLOEXEC); -+ if (fd < 0) -+ reader_term(reader); -+ -+ return fd; -+} -+ -+int kbase_kinstr_jm_init(struct kbase_kinstr_jm **const out_ctx) -+{ -+ struct kbase_kinstr_jm *ctx = NULL; -+ -+ if (!out_ctx) -+ return -EINVAL; -+ -+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); -+ if (!ctx) -+ return -ENOMEM; -+ -+ INIT_HLIST_BL_HEAD(&ctx->readers); -+ kref_init(&ctx->refcount); -+ -+ *out_ctx = ctx; -+ -+ return 0; -+} -+ -+void kbase_kinstr_jm_term(struct kbase_kinstr_jm *const ctx) -+{ -+ kbase_kinstr_jm_ref_put(ctx); -+} -+ -+void kbasep_kinstr_jm_atom_state( -+ struct kbase_jd_atom *const katom, -+ const enum kbase_kinstr_jm_reader_atom_state state) -+{ -+ struct kbase_context *const kctx = katom->kctx; -+ struct kbase_kinstr_jm *const ctx = kctx->kinstr_jm; -+ const u8 id = kbase_jd_atom_id(kctx, katom); -+ struct kbase_kinstr_jm_atom_state_change change = { -+ .timestamp = ktime_get_raw_ns(), .atom = id, .state = state -+ }; -+ struct reader *reader; -+ struct hlist_bl_node *node; -+ -+ WARN(KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT < state || 0 > state, -+ PR_ "unsupported katom (%u) state (%i)", id, state); -+ -+ switch (state) { -+ case KBASE_KINSTR_JM_READER_ATOM_STATE_START: -+ change.data.start.slot = katom->slot_nr; -+ break; -+ default: -+ break; -+ } -+ -+ rcu_read_lock(); -+ hlist_bl_for_each_entry_rcu(reader, node, &ctx->readers, node) -+ reader_changes_push( -+ &reader->changes, &change, &reader->wait_queue); -+ rcu_read_unlock(); -+} -+ -+KBASE_EXPORT_TEST_API(kbasep_kinstr_jm_atom_state); -+ -+void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const katom) -+{ -+ struct kbase_context *const kctx = katom->kctx; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ const int slot = katom->slot_nr; -+ struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); -+ -+ BUILD_BUG_ON(SLOT_RB_SIZE != 2); -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) -+ return; -+ if (WARN_ON(!submitted)) -+ return; -+ -+ if (submitted == katom) -+ kbase_kinstr_jm_atom_state_start(katom); -+} -+ -+void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const katom) -+{ -+ struct kbase_context *const kctx = katom->kctx; -+ struct kbase_device *const kbdev = kctx->kbdev; -+ const int slot = katom->slot_nr; -+ struct kbase_jd_atom *const submitted = kbase_gpu_inspect(kbdev, slot, 0); -+ struct kbase_jd_atom *const queued = kbase_gpu_inspect(kbdev, slot, 1); -+ -+ BUILD_BUG_ON(SLOT_RB_SIZE != 2); -+ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (WARN_ON(slot < 0 || slot >= GPU_MAX_JOB_SLOTS)) -+ return; -+ if (WARN_ON(!submitted)) -+ return; -+ if (WARN_ON((submitted != katom) && (queued != katom))) -+ return; -+ -+ if (queued == katom) -+ return; -+ -+ if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) -+ kbase_kinstr_jm_atom_state_stop(katom); -+ if (queued && queued->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED) -+ kbase_kinstr_jm_atom_state_start(queued); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_kinstr_jm.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_kinstr_jm.h -new file mode 100644 -index 0000000..2c904e5 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_kinstr_jm.h -@@ -0,0 +1,275 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * mali_kbase_kinstr_jm.h -+ * Kernel driver public interface to job manager atom tracing. This API provides -+ * a method to get the atom state changes into user space. -+ * -+ * The flow of operation is: -+ * -+ * | kernel | user | -+ * | ----------------------------------- | ----------------------------------- | -+ * | Initialize API with | | -+ * | kbase_kinstr_jm_init() | | -+ * | | | -+ * | Kernel code injects states with | | -+ * | kbase_kinstr_jm_atom_state_*() APIs | | -+ * | | Call ioctl() to get file descriptor | -+ * | | via KBASE_IOCTL_KINSTR_JM_FD | -+ * | Allocates a reader attached to FD | | -+ * | Allocates circular buffer and | | -+ * | patches, via ASM goto, the | | -+ * | kbase_kinstr_jm_atom_state_*() | | -+ * | | loop: | -+ * | | Call poll() on FD for POLLIN | -+ * | When threshold of changes is hit, | | -+ * | the poll is interrupted with | | -+ * | POLLIN. If circular buffer is | | -+ * | full then store the missed count | | -+ * | and interrupt poll | Call read() to get data from | -+ * | | circular buffer via the fd | -+ * | Kernel advances tail of circular | | -+ * | buffer | | -+ * | | Close file descriptor | -+ * | Deallocates circular buffer | | -+ * | | | -+ * | Terminate API with | | -+ * | kbase_kinstr_jm_term() | | -+ * -+ * All tracepoints are guarded on a static key. The static key is activated when -+ * a user space reader gets created. This means that there is negligible cost -+ * inserting the tracepoints into code when there are no readers. -+ */ -+ -+#ifndef _KBASE_KINSTR_JM_H_ -+#define _KBASE_KINSTR_JM_H_ -+ -+#include <uapi/gpu/arm/midgard/mali_kbase_kinstr_jm_reader.h> -+ -+#ifdef __KERNEL__ -+#include <linux/version.h> -+#include <linux/static_key.h> -+#else -+/* empty wrapper macros for userspace */ -+#define static_branch_unlikely(key) (1) -+#define KERNEL_VERSION(a, b, c) (0) -+#define LINUX_VERSION_CODE (1) -+#endif /* __KERNEL__ */ -+ -+/* Forward declarations */ -+struct kbase_context; -+struct kbase_kinstr_jm; -+struct kbase_jd_atom; -+union kbase_kinstr_jm_fd; -+ -+/** -+ * kbase_kinstr_jm_init() - Initialise an instrumentation job manager context. -+ * @ctx: Non-NULL pointer to where the pointer to the created context will -+ * be stored on success. -+ * -+ * Return: 0 on success, else error code. -+ */ -+int kbase_kinstr_jm_init(struct kbase_kinstr_jm **ctx); -+ -+/** -+ * kbase_kinstr_jm_term() - Terminate an instrumentation job manager context. -+ * @ctx: Pointer to context to be terminated. -+ */ -+void kbase_kinstr_jm_term(struct kbase_kinstr_jm *ctx); -+ -+/** -+ * kbase_kinstr_jm_get_fd() - Retrieves a file descriptor that can be used to -+ * read the atom state changes from userspace -+ * -+ * @ctx: Pointer to the initialized context -+ * @jm_fd_arg: Pointer to the union containing the in/out params -+ * Return: -1 on failure, valid file descriptor on success -+ */ -+int kbase_kinstr_jm_get_fd(struct kbase_kinstr_jm *const ctx, -+ union kbase_kinstr_jm_fd *jm_fd_arg); -+ -+/** -+ * kbasep_kinstr_jm_atom_state() - Signifies that an atom has changed state -+ * @atom: The atom that has changed state -+ * @state: The new state of the atom -+ * -+ * This performs the actual storage of the state ready for user space to -+ * read the data. It is only called when the static key is enabled from -+ * kbase_kinstr_jm_atom_state(). There is almost never a need to invoke this -+ * function directly. -+ */ -+void kbasep_kinstr_jm_atom_state( -+ struct kbase_jd_atom *const atom, -+ const enum kbase_kinstr_jm_reader_atom_state state); -+ -+/* Allows ASM goto patching to reduce tracing overhead. This is -+ * incremented/decremented when readers are created and terminated. This really -+ * shouldn't be changed externally, but if you do, make sure you use -+ * a static_key_inc()/static_key_dec() pair. -+ */ -+extern struct static_key_false basep_kinstr_jm_reader_static_key; -+ -+/** -+ * kbase_kinstr_jm_atom_state() - Signifies that an atom has changed state -+ * @atom: The atom that has changed state -+ * @state: The new state of the atom -+ * -+ * This uses a static key to reduce overhead when tracing is disabled -+ */ -+static inline void kbase_kinstr_jm_atom_state( -+ struct kbase_jd_atom *const atom, -+ const enum kbase_kinstr_jm_reader_atom_state state) -+{ -+ if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) -+ kbasep_kinstr_jm_atom_state(atom, state); -+} -+ -+/** -+ * kbase_kinstr_jm_atom_state_queue() - Signifies that an atom has entered a -+ * hardware or software queue. -+ * @atom: The atom that has changed state -+ */ -+static inline void kbase_kinstr_jm_atom_state_queue( -+ struct kbase_jd_atom *const atom) -+{ -+ kbase_kinstr_jm_atom_state( -+ atom, KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE); -+} -+ -+/** -+ * kbase_kinstr_jm_atom_state_start() - Signifies that work has started on an -+ * atom -+ * @atom: The atom that has changed state -+ */ -+static inline void kbase_kinstr_jm_atom_state_start( -+ struct kbase_jd_atom *const atom) -+{ -+ kbase_kinstr_jm_atom_state( -+ atom, KBASE_KINSTR_JM_READER_ATOM_STATE_START); -+} -+ -+/** -+ * kbase_kinstr_jm_atom_state_stop() - Signifies that work has stopped on an -+ * atom -+ * @atom: The atom that has changed state -+ */ -+static inline void kbase_kinstr_jm_atom_state_stop( -+ struct kbase_jd_atom *const atom) -+{ -+ kbase_kinstr_jm_atom_state( -+ atom, KBASE_KINSTR_JM_READER_ATOM_STATE_STOP); -+} -+ -+/** -+ * kbase_kinstr_jm_atom_state_complete() - Signifies that all work has completed -+ * on an atom -+ * @atom: The atom that has changed state -+ */ -+static inline void kbase_kinstr_jm_atom_state_complete( -+ struct kbase_jd_atom *const atom) -+{ -+ kbase_kinstr_jm_atom_state( -+ atom, KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE); -+} -+ -+/** -+ * kbase_kinstr_jm_atom_queue() - A software *or* hardware atom is queued for -+ * execution -+ * @atom: The atom that has changed state -+ */ -+static inline void kbase_kinstr_jm_atom_queue(struct kbase_jd_atom *const atom) -+{ -+ kbase_kinstr_jm_atom_state_queue(atom); -+} -+ -+/** -+ * kbase_kinstr_jm_atom_complete() - A software *or* hardware atom is fully -+ * completed -+ * @atom: The atom that has changed state -+ */ -+static inline void kbase_kinstr_jm_atom_complete( -+ struct kbase_jd_atom *const atom) -+{ -+ kbase_kinstr_jm_atom_state_complete(atom); -+} -+ -+/** -+ * kbase_kinstr_jm_atom_sw_start() - A software atom has started work -+ * @atom: The atom that has changed state -+ */ -+static inline void kbase_kinstr_jm_atom_sw_start( -+ struct kbase_jd_atom *const atom) -+{ -+ kbase_kinstr_jm_atom_state_start(atom); -+} -+ -+/** -+ * kbase_kinstr_jm_atom_sw_stop() - A software atom has stopped work -+ * @atom: The atom that has changed state -+ */ -+static inline void kbase_kinstr_jm_atom_sw_stop( -+ struct kbase_jd_atom *const atom) -+{ -+ kbase_kinstr_jm_atom_state_stop(atom); -+} -+ -+/** -+ * kbasep_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted -+ * @atom: The atom that has been submitted -+ * -+ * This private implementation should not be called directly, it is protected -+ * by a static key in kbase_kinstr_jm_atom_hw_submit(). Use that instead. -+ */ -+void kbasep_kinstr_jm_atom_hw_submit(struct kbase_jd_atom *const atom); -+ -+/** -+ * kbase_kinstr_jm_atom_hw_submit() - A hardware atom has been submitted -+ * @atom: The atom that has been submitted -+ */ -+static inline void kbase_kinstr_jm_atom_hw_submit( -+ struct kbase_jd_atom *const atom) -+{ -+ if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) -+ kbasep_kinstr_jm_atom_hw_submit(atom); -+} -+ -+/** -+ * kbasep_kinstr_jm_atom_hw_release() - A hardware atom has been released -+ * @atom: The atom that has been released -+ * -+ * This private implementation should not be called directly, it is protected -+ * by a static key in kbase_kinstr_jm_atom_hw_release(). Use that instead. -+ */ -+void kbasep_kinstr_jm_atom_hw_release(struct kbase_jd_atom *const atom); -+ -+/** -+ * kbase_kinstr_jm_atom_hw_release() - A hardware atom has been released -+ * @atom: The atom that has been released -+ */ -+static inline void kbase_kinstr_jm_atom_hw_release( -+ struct kbase_jd_atom *const atom) -+{ -+ if (static_branch_unlikely(&basep_kinstr_jm_reader_static_key)) -+ kbasep_kinstr_jm_atom_hw_release(atom); -+} -+ -+#endif /* _KBASE_KINSTR_JM_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_linux.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_linux.h -index 003ac9e..1d8d196 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_linux.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_linux.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2014, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_linux.h -- * Base kernel APIs, Linux implementation. -+ * DOC: Base kernel APIs, Linux implementation. - */ - - #ifndef _KBASE_LINUX_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c -index 4a1004b..a68e4ea 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -@@ -29,7 +28,7 @@ - #include <linux/compat.h> - #include <linux/version.h> - #include <linux/log2.h> --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - #include <linux/of_platform.h> - #endif - -@@ -43,6 +42,7 @@ - #include <mali_kbase_mem_pool_group.h> - #include <mmu/mali_kbase_mmu.h> - #include <mali_kbase_config_defaults.h> -+#include <mali_kbase_trace_gpu_mem.h> - - /* - * Alignment of objects allocated by the GPU inside a just-in-time memory -@@ -89,7 +89,7 @@ static size_t kbase_get_num_cpu_va_bits(struct kbase_context *kctx) - #error "Unknown CPU VA width for this architecture" - #endif - --#ifdef CONFIG_64BIT -+#if IS_ENABLED(CONFIG_64BIT) - if (kbase_ctx_flag(kctx, KCTX_COMPAT)) - cpu_va_bits = 32; - #endif -@@ -98,27 +98,34 @@ static size_t kbase_get_num_cpu_va_bits(struct kbase_context *kctx) - } - - /* This function finds out which RB tree the given pfn from the GPU VA belongs -- * to based on the memory zone the pfn refers to */ -+ * to based on the memory zone the pfn refers to -+ */ - static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, - u64 gpu_pfn) - { - struct rb_root *rbtree = NULL; -+ struct kbase_reg_zone *exec_va_zone = -+ kbase_ctx_reg_zone_get(kctx, KBASE_REG_ZONE_EXEC_VA); - - /* The gpu_pfn can only be greater than the starting pfn of the EXEC_VA - * zone if this has been initialized. - */ -- if (gpu_pfn >= kctx->exec_va_start) -+ if (gpu_pfn >= exec_va_zone->base_pfn) - rbtree = &kctx->reg_rbtree_exec; - else { - u64 same_va_end; - --#ifdef CONFIG_64BIT -- if (kbase_ctx_flag(kctx, KCTX_COMPAT)) -+#if IS_ENABLED(CONFIG_64BIT) -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { - #endif /* CONFIG_64BIT */ - same_va_end = KBASE_REG_ZONE_CUSTOM_VA_BASE; --#ifdef CONFIG_64BIT -- else -- same_va_end = kctx->same_va_end; -+#if IS_ENABLED(CONFIG_64BIT) -+ } else { -+ struct kbase_reg_zone *same_va_zone = -+ kbase_ctx_reg_zone_get(kctx, -+ KBASE_REG_ZONE_SAME_VA); -+ same_va_end = kbase_reg_zone_end_pfn(same_va_zone); -+ } - #endif /* CONFIG_64BIT */ - - if (gpu_pfn >= same_va_end) -@@ -228,7 +235,7 @@ struct kbase_va_region *kbase_region_tracker_find_region_enclosing_address( - u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; - struct rb_root *rbtree = NULL; - -- KBASE_DEBUG_ASSERT(NULL != kctx); -+ KBASE_DEBUG_ASSERT(kctx != NULL); - - lockdep_assert_held(&kctx->reg_lock); - -@@ -288,7 +295,8 @@ static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs( - struct rb_root *rbtree = NULL; - - /* Note that this search is a linear search, as we do not have a target -- address in mind, so does not benefit from the rbtree search */ -+ * address in mind, so does not benefit from the rbtree search -+ */ - rbtree = reg_reqs->rbtree; - - for (rbnode = rb_first(rbtree); rbnode; rbnode = rb_next(rbnode)) { -@@ -303,7 +311,8 @@ static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs( - * (start_pfn + align_mask) & ~(align_mask) - * - * Otherwise, it aligns to n*align + offset, for the -- * lowest value n that makes this still >start_pfn */ -+ * lowest value n that makes this still >start_pfn -+ */ - start_pfn += align_mask; - start_pfn -= (start_pfn - align_offset) & (align_mask); - -@@ -341,7 +350,8 @@ static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs( - } - - /** -- * @brief Remove a region object from the global list. -+ * Remove a region object from the global list. -+ * @reg: Region object to remove - * - * The region reg is removed, possibly by merging with other free and - * compatible adjacent regions. It must be called with the context -@@ -367,8 +377,9 @@ int kbase_remove_va_region(struct kbase_va_region *reg) - if (rbprev) { - prev = rb_entry(rbprev, struct kbase_va_region, rblink); - if (prev->flags & KBASE_REG_FREE) { -- /* We're compatible with the previous VMA, -- * merge with it */ -+ /* We're compatible with the previous VMA, merge with -+ * it -+ */ - WARN_ON((prev->flags & KBASE_REG_ZONE_MASK) != - (reg->flags & KBASE_REG_ZONE_MASK)); - prev->nr_pages += reg->nr_pages; -@@ -511,8 +522,8 @@ int kbase_add_va_region(struct kbase_context *kctx, - int gpu_pc_bits = - kbdev->gpu_props.props.core_props.log2_program_counter_size; - -- KBASE_DEBUG_ASSERT(NULL != kctx); -- KBASE_DEBUG_ASSERT(NULL != reg); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(reg != NULL); - - lockdep_assert_held(&kctx->reg_lock); - -@@ -614,13 +625,15 @@ int kbase_add_va_region_rbtree(struct kbase_device *kbdev, - size_t align_offset = align; - size_t align_mask = align - 1; - -+#if !MALI_USE_CSF - if ((reg->flags & KBASE_REG_TILER_ALIGN_TOP)) { - WARN(align > 1, "%s with align %lx might not be honored for KBASE_REG_TILER_ALIGN_TOP memory", - __func__, - (unsigned long)align); -- align_mask = reg->extent - 1; -- align_offset = reg->extent - reg->initial_commit; -+ align_mask = reg->extension - 1; -+ align_offset = reg->extension - reg->initial_commit; - } -+#endif /* !MALI_USE_CSF */ - - tmp = kbase_region_tracker_find_region_meeting_reqs(reg, - nr_pages, align_offset, align_mask, -@@ -643,7 +656,7 @@ exit: - return err; - } - --/** -+/* - * @brief Initialize the internal region tracker data structure. - */ - static void kbase_region_tracker_ds_init(struct kbase_context *kctx, -@@ -698,6 +711,9 @@ void kbase_region_tracker_term(struct kbase_context *kctx) - kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same); - kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom); - kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec); -+#if MALI_USE_CSF -+ WARN_ON(!list_empty(&kctx->csf.event_pages_head)); -+#endif - kbase_gpu_vm_unlock(kctx); - } - -@@ -720,23 +736,26 @@ int kbase_region_tracker_init(struct kbase_context *kctx) - u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; - u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; - u64 same_va_pages; -+ u64 same_va_base = 1u; - int err; - - /* Take the lock as kbase_free_alloced_region requires it */ - kbase_gpu_vm_lock(kctx); - -- same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; -+ same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - same_va_base; - /* all have SAME_VA */ -- same_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 1, -- same_va_pages, -- KBASE_REG_ZONE_SAME_VA); -+ same_va_reg = -+ kbase_alloc_free_region(&kctx->reg_rbtree_same, same_va_base, -+ same_va_pages, KBASE_REG_ZONE_SAME_VA); - - if (!same_va_reg) { - err = -ENOMEM; - goto fail_unlock; - } -+ kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_SAME_VA, same_va_base, -+ same_va_pages); - --#ifdef CONFIG_64BIT -+#if IS_ENABLED(CONFIG_64BIT) - /* 32-bit clients have custom VA zones */ - if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { - #endif -@@ -760,19 +779,28 @@ int kbase_region_tracker_init(struct kbase_context *kctx) - err = -ENOMEM; - goto fail_free_same_va; - } --#ifdef CONFIG_64BIT -+ kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_CUSTOM_VA, -+ KBASE_REG_ZONE_CUSTOM_VA_BASE, -+ custom_va_size); -+#if IS_ENABLED(CONFIG_64BIT) - } else { - custom_va_size = 0; - } - #endif -+ /* EXEC_VA zone's codepaths are slightly easier when its base_pfn is -+ * initially U64_MAX -+ */ -+ kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_EXEC_VA, U64_MAX, 0u); -+ /* Other zones are 0: kbase_create_context() uses vzalloc */ - - kbase_region_tracker_ds_init(kctx, same_va_reg, custom_va_reg); - -- kctx->same_va_end = same_va_pages + 1; -- kctx->gpu_va_end = kctx->same_va_end + custom_va_size; -- kctx->exec_va_start = U64_MAX; -+ kctx->gpu_va_end = same_va_base + same_va_pages + custom_va_size; - kctx->jit_va = false; - -+#if MALI_USE_CSF -+ INIT_LIST_HEAD(&kctx->csf.event_pages_head); -+#endif - - kbase_gpu_vm_unlock(kctx); - return 0; -@@ -784,44 +812,147 @@ fail_unlock: - return err; - } - --#ifdef CONFIG_64BIT -+static bool kbase_has_exec_va_zone_locked(struct kbase_context *kctx) -+{ -+ struct kbase_reg_zone *exec_va_zone; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ exec_va_zone = kbase_ctx_reg_zone_get(kctx, KBASE_REG_ZONE_EXEC_VA); -+ -+ return (exec_va_zone->base_pfn != U64_MAX); -+} -+ -+bool kbase_has_exec_va_zone(struct kbase_context *kctx) -+{ -+ bool has_exec_va_zone; -+ -+ kbase_gpu_vm_lock(kctx); -+ has_exec_va_zone = kbase_has_exec_va_zone_locked(kctx); -+ kbase_gpu_vm_unlock(kctx); -+ -+ return has_exec_va_zone; -+} -+ -+/** -+ * Determine if any allocations have been made on a context's region tracker -+ * @kctx: KBase context -+ * -+ * Check the context to determine if any allocations have been made yet from -+ * any of its zones. This check should be done before resizing a zone, e.g. to -+ * make space to add a second zone. -+ * -+ * Whilst a zone without allocations can be resized whilst other zones have -+ * allocations, we still check all of @kctx 's zones anyway: this is a stronger -+ * guarantee and should be adhered to when creating new zones anyway. -+ * -+ * Allocations from kbdev zones are not counted. -+ * -+ * Return: true if any allocs exist on any zone, false otherwise -+ */ -+static bool kbase_region_tracker_has_allocs(struct kbase_context *kctx) -+{ -+ unsigned int zone_idx; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ for (zone_idx = 0; zone_idx < KBASE_REG_ZONE_MAX; ++zone_idx) { -+ struct kbase_reg_zone *zone; -+ struct kbase_va_region *reg; -+ u64 zone_base_addr; -+ unsigned long zone_bits = KBASE_REG_ZONE(zone_idx); -+ unsigned long reg_zone; -+ -+ zone = kbase_ctx_reg_zone_get(kctx, zone_bits); -+ zone_base_addr = zone->base_pfn << PAGE_SHIFT; -+ -+ reg = kbase_region_tracker_find_region_base_address( -+ kctx, zone_base_addr); -+ -+ if (!zone->va_size_pages) { -+ WARN(reg, -+ "Should not have found a region that starts at 0x%.16llx for zone 0x%lx", -+ (unsigned long long)zone_base_addr, zone_bits); -+ continue; -+ } -+ -+ if (WARN(!reg, -+ "There should always be a region that starts at 0x%.16llx for zone 0x%lx, couldn't find it", -+ (unsigned long long)zone_base_addr, zone_bits)) -+ return true; /* Safest return value */ -+ -+ reg_zone = reg->flags & KBASE_REG_ZONE_MASK; -+ if (WARN(reg_zone != zone_bits, -+ "The region that starts at 0x%.16llx should be in zone 0x%lx but was found in the wrong zone 0x%lx", -+ (unsigned long long)zone_base_addr, zone_bits, -+ reg_zone)) -+ return true; /* Safest return value */ -+ -+ /* Unless the region is completely free, of the same size as -+ * the original zone, then it has allocs -+ */ -+ if ((!(reg->flags & KBASE_REG_FREE)) || -+ (reg->nr_pages != zone->va_size_pages)) -+ return true; -+ } -+ -+ /* All zones are the same size as originally made, so there are no -+ * allocs -+ */ -+ return false; -+} -+ -+#if IS_ENABLED(CONFIG_64BIT) - static int kbase_region_tracker_init_jit_64(struct kbase_context *kctx, - u64 jit_va_pages) - { -- struct kbase_va_region *same_va; -+ struct kbase_va_region *same_va_reg; -+ struct kbase_reg_zone *same_va_zone; -+ u64 same_va_zone_base_addr; -+ const unsigned long same_va_zone_bits = KBASE_REG_ZONE_SAME_VA; - struct kbase_va_region *custom_va_reg; -+ u64 jit_va_start; - - lockdep_assert_held(&kctx->reg_lock); - -- /* First verify that a JIT_VA zone has not been created already. */ -- if (kctx->jit_va) -- return -EINVAL; -- - /* -- * Modify the same VA free region after creation. Be careful to ensure -- * that allocations haven't been made as they could cause an overlap -- * to happen with existing same VA allocations and the custom VA zone. -+ * Modify the same VA free region after creation. The caller has -+ * ensured that allocations haven't been made, as any allocations could -+ * cause an overlap to happen with existing same VA allocations and the -+ * custom VA zone. - */ -- same_va = kbase_region_tracker_find_region_base_address(kctx, -- PAGE_SIZE); -- if (!same_va) -+ same_va_zone = kbase_ctx_reg_zone_get(kctx, same_va_zone_bits); -+ same_va_zone_base_addr = same_va_zone->base_pfn << PAGE_SHIFT; -+ -+ same_va_reg = kbase_region_tracker_find_region_base_address( -+ kctx, same_va_zone_base_addr); -+ if (WARN(!same_va_reg, -+ "Already found a free region at the start of every zone, but now cannot find any region for zone base 0x%.16llx zone 0x%lx", -+ (unsigned long long)same_va_zone_base_addr, same_va_zone_bits)) - return -ENOMEM; - -- if (same_va->nr_pages < jit_va_pages || kctx->same_va_end < jit_va_pages) -+ /* kbase_region_tracker_has_allocs() in the caller has already ensured -+ * that all of the zones have no allocs, so no need to check that again -+ * on same_va_reg -+ */ -+ WARN_ON((!(same_va_reg->flags & KBASE_REG_FREE)) || -+ same_va_reg->nr_pages != same_va_zone->va_size_pages); -+ -+ if (same_va_reg->nr_pages < jit_va_pages || -+ same_va_zone->va_size_pages < jit_va_pages) - return -ENOMEM; - - /* It's safe to adjust the same VA zone now */ -- same_va->nr_pages -= jit_va_pages; -- kctx->same_va_end -= jit_va_pages; -+ same_va_reg->nr_pages -= jit_va_pages; -+ same_va_zone->va_size_pages -= jit_va_pages; -+ jit_va_start = kbase_reg_zone_end_pfn(same_va_zone); - - /* - * Create a custom VA zone at the end of the VA for allocations which - * JIT can use so it doesn't have to allocate VA from the kernel. - */ -- custom_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom, -- kctx->same_va_end, -- jit_va_pages, -- KBASE_REG_ZONE_CUSTOM_VA); -+ custom_va_reg = -+ kbase_alloc_free_region(&kctx->reg_rbtree_custom, jit_va_start, -+ jit_va_pages, KBASE_REG_ZONE_CUSTOM_VA); - - /* - * The context will be destroyed if we fail here so no point -@@ -829,6 +960,11 @@ static int kbase_region_tracker_init_jit_64(struct kbase_context *kctx, - */ - if (!custom_va_reg) - return -ENOMEM; -+ /* Since this is 64-bit, the custom zone will not have been -+ * initialized, so initialize it now -+ */ -+ kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_CUSTOM_VA, jit_va_start, -+ jit_va_pages); - - kbase_region_tracker_insert(custom_va_reg); - return 0; -@@ -847,16 +983,34 @@ int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, - if (group_id < 0 || group_id >= MEMORY_GROUP_MANAGER_NR_GROUPS) - return -EINVAL; - --#if MALI_JIT_PRESSURE_LIMIT - if (phys_pages_limit > jit_va_pages) --#else -- if (phys_pages_limit != jit_va_pages) --#endif /* MALI_JIT_PRESSURE_LIMIT */ - return -EINVAL; - -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (phys_pages_limit != jit_va_pages) -+ kbase_ctx_flag_set(kctx, KCTX_JPL_ENABLED); -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -+ - kbase_gpu_vm_lock(kctx); - --#ifdef CONFIG_64BIT -+ /* Verify that a JIT_VA zone has not been created already. */ -+ if (kctx->jit_va) { -+ err = -EINVAL; -+ goto exit_unlock; -+ } -+ -+ /* If in 64-bit, we always lookup the SAME_VA zone. To ensure it has no -+ * allocs, we can ensure there are no allocs anywhere. -+ * -+ * This check is also useful in 32-bit, just to make sure init of the -+ * zone is always done before any allocs. -+ */ -+ if (kbase_region_tracker_has_allocs(kctx)) { -+ err = -ENOMEM; -+ goto exit_unlock; -+ } -+ -+#if IS_ENABLED(CONFIG_64BIT) - if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) - err = kbase_region_tracker_init_jit_64(kctx, jit_va_pages); - #endif -@@ -870,13 +1024,14 @@ int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, - kctx->trim_level = trim_level; - kctx->jit_va = true; - kctx->jit_group_id = group_id; --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - kctx->jit_phys_pages_limit = phys_pages_limit; - dev_dbg(kctx->kbdev->dev, "phys_pages_limit set to %llu\n", - phys_pages_limit); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - } - -+exit_unlock: - kbase_gpu_vm_unlock(kctx); - - return err; -@@ -884,24 +1039,33 @@ int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, - - int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages) - { -- struct kbase_va_region *shrinking_va_reg; - struct kbase_va_region *exec_va_reg; -- u64 exec_va_start, exec_va_base_addr; -+ struct kbase_reg_zone *exec_va_zone; -+ struct kbase_reg_zone *target_zone; -+ struct kbase_va_region *target_reg; -+ u64 target_zone_base_addr; -+ unsigned long target_zone_bits; -+ u64 exec_va_start; - int err; - -- /* The EXEC_VA zone shall be created by making space at the end of the -- * address space. Firstly, verify that the number of EXEC_VA pages -- * requested by the client is reasonable and then make sure that it is -- * not greater than the address space itself before calculating the base -- * address of the new zone. -+ /* The EXEC_VA zone shall be created by making space either: -+ * - for 64-bit clients, at the end of the process's address space -+ * - for 32-bit clients, in the CUSTOM zone -+ * -+ * Firstly, verify that the number of EXEC_VA pages requested by the -+ * client is reasonable and then make sure that it is not greater than -+ * the address space itself before calculating the base address of the -+ * new zone. - */ - if (exec_va_pages == 0 || exec_va_pages > KBASE_REG_ZONE_EXEC_VA_MAX_PAGES) - return -EINVAL; - - kbase_gpu_vm_lock(kctx); - -- /* First verify that a JIT_VA zone has not been created already. */ -- if (kctx->jit_va) { -+ /* Verify that we've not already created a EXEC_VA zone, and that the -+ * EXEC_VA zone must come before JIT's CUSTOM_VA. -+ */ -+ if (kbase_has_exec_va_zone_locked(kctx) || kctx->jit_va) { - err = -EPERM; - goto exit_unlock; - } -@@ -911,28 +1075,50 @@ int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages - goto exit_unlock; - } - -- exec_va_start = kctx->gpu_va_end - exec_va_pages; -- exec_va_base_addr = exec_va_start << PAGE_SHIFT; -- -- shrinking_va_reg = kbase_region_tracker_find_region_enclosing_address(kctx, -- exec_va_base_addr); -- if (!shrinking_va_reg) { -+ /* Verify no allocations have already been made */ -+ if (kbase_region_tracker_has_allocs(kctx)) { - err = -ENOMEM; - goto exit_unlock; - } - -- /* Make sure that the EXEC_VA region is still uninitialized */ -- if ((shrinking_va_reg->flags & KBASE_REG_ZONE_MASK) == -- KBASE_REG_ZONE_EXEC_VA) { -- err = -EPERM; -+#if IS_ENABLED(CONFIG_64BIT) -+ if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { -+#endif -+ /* 32-bit client: take from CUSTOM_VA zone */ -+ target_zone_bits = KBASE_REG_ZONE_CUSTOM_VA; -+#if IS_ENABLED(CONFIG_64BIT) -+ } else { -+ /* 64-bit client: take from SAME_VA zone */ -+ target_zone_bits = KBASE_REG_ZONE_SAME_VA; -+ } -+#endif -+ target_zone = kbase_ctx_reg_zone_get(kctx, target_zone_bits); -+ target_zone_base_addr = target_zone->base_pfn << PAGE_SHIFT; -+ -+ target_reg = kbase_region_tracker_find_region_base_address( -+ kctx, target_zone_base_addr); -+ if (WARN(!target_reg, -+ "Already found a free region at the start of every zone, but now cannot find any region for zone base 0x%.16llx zone 0x%lx", -+ (unsigned long long)target_zone_base_addr, target_zone_bits)) { -+ err = -ENOMEM; - goto exit_unlock; - } -+ /* kbase_region_tracker_has_allocs() above has already ensured that all -+ * of the zones have no allocs, so no need to check that again on -+ * target_reg -+ */ -+ WARN_ON((!(target_reg->flags & KBASE_REG_FREE)) || -+ target_reg->nr_pages != target_zone->va_size_pages); - -- if (shrinking_va_reg->nr_pages <= exec_va_pages) { -+ if (target_reg->nr_pages <= exec_va_pages || -+ target_zone->va_size_pages <= exec_va_pages) { - err = -ENOMEM; - goto exit_unlock; - } - -+ /* Taken from the end of the target zone */ -+ exec_va_start = kbase_reg_zone_end_pfn(target_zone) - exec_va_pages; -+ - exec_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_exec, - exec_va_start, - exec_va_pages, -@@ -941,13 +1127,17 @@ int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages - err = -ENOMEM; - goto exit_unlock; - } -+ /* Update EXEC_VA zone -+ * -+ * not using kbase_ctx_reg_zone_init() - it was already initialized -+ */ -+ exec_va_zone = kbase_ctx_reg_zone_get(kctx, KBASE_REG_ZONE_EXEC_VA); -+ exec_va_zone->base_pfn = exec_va_start; -+ exec_va_zone->va_size_pages = exec_va_pages; - -- shrinking_va_reg->nr_pages -= exec_va_pages; --#ifdef CONFIG_64BIT -- if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) -- kctx->same_va_end -= exec_va_pages; --#endif -- kctx->exec_va_start = exec_va_start; -+ /* Update target zone and corresponding region */ -+ target_reg->nr_pages -= exec_va_pages; -+ target_zone->va_size_pages -= exec_va_pages; - - kbase_region_tracker_insert(exec_va_reg); - err = 0; -@@ -957,12 +1147,40 @@ exit_unlock: - return err; - } - -+#if MALI_USE_CSF -+void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev) -+{ -+ kbase_region_tracker_term_rbtree(&kbdev->csf.shared_reg_rbtree); -+} -+ -+int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev) -+{ -+ struct kbase_va_region *shared_reg; -+ u64 shared_reg_start_pfn; -+ u64 shared_reg_size; -+ -+ shared_reg_start_pfn = KBASE_REG_ZONE_MCU_SHARED_BASE; -+ shared_reg_size = KBASE_REG_ZONE_MCU_SHARED_SIZE; -+ -+ kbdev->csf.shared_reg_rbtree = RB_ROOT; -+ -+ shared_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree, -+ shared_reg_start_pfn, -+ shared_reg_size, -+ KBASE_REG_ZONE_MCU_SHARED); -+ if (!shared_reg) -+ return -ENOMEM; -+ -+ kbase_region_tracker_insert(shared_reg); -+ return 0; -+} -+#endif - - int kbase_mem_init(struct kbase_device *kbdev) - { - int err = 0; - struct kbasep_mem_device *memdev; --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - struct device_node *mgm_node = NULL; - #endif - -@@ -976,6 +1194,12 @@ int kbase_mem_init(struct kbase_device *kbdev) - /* Initialize memory usage */ - atomic_set(&memdev->used_pages, 0); - -+ spin_lock_init(&kbdev->gpu_mem_usage_lock); -+ kbdev->total_gpu_pages = 0; -+ kbdev->process_root = RB_ROOT; -+ kbdev->dma_buf_root = RB_ROOT; -+ mutex_init(&kbdev->dma_buf_lock); -+ - #ifdef IR_THRESHOLD - atomic_set(&memdev->ir_threshold, IR_THRESHOLD); - #else -@@ -984,7 +1208,7 @@ int kbase_mem_init(struct kbase_device *kbdev) - - kbdev->mgm_dev = &kbase_native_mgm_dev; - --#ifdef CONFIG_OF -+#if IS_ENABLED(CONFIG_OF) - /* Check to see whether or not a platform-specific memory group manager - * is configured and available. - */ -@@ -1053,13 +1277,22 @@ void kbase_mem_term(struct kbase_device *kbdev) - - kbase_mem_pool_group_term(&kbdev->mem_pools); - -+ WARN_ON(kbdev->total_gpu_pages); -+ WARN_ON(!RB_EMPTY_ROOT(&kbdev->process_root)); -+ WARN_ON(!RB_EMPTY_ROOT(&kbdev->dma_buf_root)); -+ mutex_destroy(&kbdev->dma_buf_lock); -+ - if (kbdev->mgm_dev) - module_put(kbdev->mgm_dev->owner); - } - KBASE_EXPORT_TEST_API(kbase_mem_term); - - /** -- * @brief Allocate a free region object. -+ * Allocate a free region object. -+ * @rbtree: Backlink to the red-black tree of memory regions. -+ * @start_pfn: The Page Frame Number in GPU virtual address space. -+ * @nr_pages: The size of the region in pages. -+ * @zone: KBASE_REG_ZONE_CUSTOM_VA or KBASE_REG_ZONE_SAME_VA - * - * The allocated object is not part of any list yet, and is flagged as - * KBASE_REG_FREE. No mapping is allocated yet. -@@ -1132,7 +1365,8 @@ static struct kbase_context *kbase_reg_flags_to_kctx( - } - - /** -- * @brief Free a region object. -+ * Free a region object. -+ * @reg: Region - * - * The described region must be freed of any mapping. - * -@@ -1143,6 +1377,13 @@ static struct kbase_context *kbase_reg_flags_to_kctx( - */ - void kbase_free_alloced_region(struct kbase_va_region *reg) - { -+#if MALI_USE_CSF -+ if ((reg->flags & KBASE_REG_ZONE_MASK) == -+ KBASE_REG_ZONE_MCU_SHARED) { -+ kfree(reg); -+ return; -+ } -+#endif - if (!(reg->flags & KBASE_REG_FREE)) { - struct kbase_context *kctx = kbase_reg_flags_to_kctx(reg); - -@@ -1152,8 +1393,12 @@ void kbase_free_alloced_region(struct kbase_va_region *reg) - if (WARN_ON(kbase_is_region_invalid(reg))) - return; - -- dev_dbg(kctx->kbdev->dev, "Freeing memory region %p\n", -+ dev_dbg(kctx->kbdev->dev, "Freeing memory region %pK\n", - (void *)reg); -+#if MALI_USE_CSF -+ if (reg->flags & KBASE_REG_CSF_EVENT) -+ kbase_unlink_event_mem_page(kctx, reg); -+#endif - - mutex_lock(&kctx->jit_evict_lock); - -@@ -1233,8 +1478,8 @@ int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 - else - attr = KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_WRITE_ALLOC); - -- KBASE_DEBUG_ASSERT(NULL != kctx); -- KBASE_DEBUG_ASSERT(NULL != reg); -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(reg != NULL); - - err = kbase_add_va_region(kctx, reg, addr, nr_pages, align); - if (err) -@@ -1260,7 +1505,9 @@ int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 - if (err) - goto bad_insert; - -- kbase_mem_phy_alloc_gpu_mapped(alloc->imported.alias.aliased[i].alloc); -+ /* Note: mapping count is tracked at alias -+ * creation time -+ */ - } else { - err = kbase_mmu_insert_single_page(kctx, - reg->start_pfn + i * stride, -@@ -1319,13 +1566,6 @@ bad_insert: - reg->start_pfn, reg->nr_pages, - kctx->as_nr); - -- if (alloc->type == KBASE_MEM_TYPE_ALIAS) { -- KBASE_DEBUG_ASSERT(alloc->imported.alias.aliased); -- while (i--) -- if (alloc->imported.alias.aliased[i].alloc) -- kbase_mem_phy_alloc_gpu_unmapped(alloc->imported.alias.aliased[i].alloc); -- } -- - kbase_remove_va_region(reg); - - return err; -@@ -1339,7 +1579,6 @@ static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, - int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) - { - int err = 0; -- size_t i; - - if (reg->start_pfn == 0) - return 0; -@@ -1364,10 +1603,9 @@ int kbase_gpu_munmap(struct kbase_context *kctx, struct kbase_va_region *reg) - /* Update tracking, and other cleanup, depending on memory type. */ - switch (reg->gpu_alloc->type) { - case KBASE_MEM_TYPE_ALIAS: -- KBASE_DEBUG_ASSERT(reg->gpu_alloc->imported.alias.aliased); -- for (i = 0; i < reg->gpu_alloc->imported.alias.nents; i++) -- if (reg->gpu_alloc->imported.alias.aliased[i].alloc) -- kbase_mem_phy_alloc_gpu_unmapped(reg->gpu_alloc->imported.alias.aliased[i].alloc); -+ /* We mark the source allocs as unmapped from the GPU when -+ * putting reg's allocs -+ */ - break; - case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { - struct kbase_alloc_import_user_buf *user_buf = -@@ -1404,7 +1642,7 @@ static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping( - unsigned long map_start; - size_t map_size; - -- lockdep_assert_held(¤t->mm->mmap_sem); -+ lockdep_assert_held(kbase_mem_get_process_mmap_lock()); - - if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */ - return NULL; -@@ -1676,9 +1914,9 @@ int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *re - { - int err; - -- KBASE_DEBUG_ASSERT(NULL != kctx); -- KBASE_DEBUG_ASSERT(NULL != reg); -- dev_dbg(kctx->kbdev->dev, "%s %p in kctx %p\n", -+ KBASE_DEBUG_ASSERT(kctx != NULL); -+ KBASE_DEBUG_ASSERT(reg != NULL); -+ dev_dbg(kctx->kbdev->dev, "%s %pK in kctx %pK\n", - __func__, (void *)reg, (void *)kctx); - lockdep_assert_held(&kctx->reg_lock); - -@@ -1724,7 +1962,9 @@ int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *re - KBASE_EXPORT_TEST_API(kbase_mem_free_region); - - /** -- * @brief Free the region from the GPU and unregister it. -+ * Free the region from the GPU and unregister it. -+ * @kctx: KBase context -+ * @gpu_addr: GPU address to free - * - * This function implements the free operation on a memory segment. - * It will loudly fail if called with outstanding mappings. -@@ -1735,7 +1975,7 @@ int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) - struct kbase_va_region *reg; - - KBASE_DEBUG_ASSERT(kctx != NULL); -- dev_dbg(kctx->kbdev->dev, "%s 0x%llx in kctx %p\n", -+ dev_dbg(kctx->kbdev->dev, "%s 0x%llx in kctx %pK\n", - __func__, gpu_addr, (void *)kctx); - - if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) { -@@ -1743,7 +1983,7 @@ int kbase_mem_free(struct kbase_context *kctx, u64 gpu_addr) - return -EINVAL; - } - -- if (0 == gpu_addr) { -+ if (gpu_addr == 0) { - dev_warn(kctx->kbdev->dev, "gpu_addr 0 is reserved for the ringbuffer and it's an error to try to free it using kbase_mem_free\n"); - return -EINVAL; - } -@@ -1796,7 +2036,7 @@ KBASE_EXPORT_TEST_API(kbase_mem_free); - int kbase_update_region_flags(struct kbase_context *kctx, - struct kbase_va_region *reg, unsigned long flags) - { -- KBASE_DEBUG_ASSERT(NULL != reg); -+ KBASE_DEBUG_ASSERT(reg != NULL); - KBASE_DEBUG_ASSERT((flags & ~((1ul << BASE_MEM_FLAGS_NR_BITS) - 1)) == 0); - - reg->flags |= kbase_cache_enabled(flags, reg->nr_pages); -@@ -1835,9 +2075,25 @@ int kbase_update_region_flags(struct kbase_context *kctx, - reg->flags |= KBASE_REG_SHARE_IN; - } - -+#if !MALI_USE_CSF - if (flags & BASE_MEM_TILER_ALIGN_TOP) - reg->flags |= KBASE_REG_TILER_ALIGN_TOP; -+#endif /* !MALI_USE_CSF */ -+ -+#if MALI_USE_CSF -+ if (flags & BASE_MEM_CSF_EVENT) { -+ reg->flags |= KBASE_REG_CSF_EVENT; -+ reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING; - -+ if (!(reg->flags & KBASE_REG_SHARE_BOTH)) { -+ /* On non coherent platforms need to map as uncached on -+ * both sides. -+ */ -+ reg->flags &= ~KBASE_REG_CPU_CACHED; -+ reg->flags &= ~KBASE_REG_GPU_CACHED; -+ } -+ } -+#endif - - /* Set up default MEMATTR usage */ - if (!(reg->flags & KBASE_REG_GPU_CACHED)) { -@@ -1851,6 +2107,13 @@ int kbase_update_region_flags(struct kbase_context *kctx, - "Can't allocate GPU uncached memory due to MMU in Legacy Mode\n"); - return -EINVAL; - } -+#if MALI_USE_CSF -+ } else if (reg->flags & KBASE_REG_CSF_EVENT) { -+ WARN_ON(!(reg->flags & KBASE_REG_SHARE_BOTH)); -+ -+ reg->flags |= -+ KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED); -+#endif - } else if (kctx->kbdev->system_coherency == COHERENCY_ACE && - (reg->flags & KBASE_REG_SHARE_BOTH)) { - reg->flags |= -@@ -1905,7 +2168,8 @@ int kbase_alloc_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, - &kctx->kbdev->memdev.used_pages); - - /* Increase mm counters before we allocate pages so that this -- * allocation is visible to the OOM killer */ -+ * allocation is visible to the OOM killer -+ */ - kbase_process_page_usage_inc(kctx, nr_pages_requested); - - tp = alloc->pages + alloc->nents; -@@ -2033,6 +2297,9 @@ no_new_partial: - (u64)new_page_count); - - alloc->nents += nr_pages_requested; -+ -+ kbase_trace_gpu_mem_usage_inc(kctx->kbdev, kctx, nr_pages_requested); -+ - done: - return 0; - -@@ -2209,6 +2476,9 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked( - (u64)new_page_count); - - alloc->nents += nr_pages_requested; -+ -+ kbase_trace_gpu_mem_usage_inc(kctx->kbdev, kctx, nr_pages_requested); -+ - done: - return new_pages; - -@@ -2303,7 +2573,7 @@ int kbase_free_phy_pages_helper( - } - - /* early out if nothing to do */ -- if (0 == nr_pages_to_free) -+ if (nr_pages_to_free == 0) - return 0; - - start_free = alloc->pages + alloc->nents - nr_pages_to_free; -@@ -2374,6 +2644,8 @@ int kbase_free_phy_pages_helper( - kbdev, - kctx->id, - (u64)new_page_count); -+ -+ kbase_trace_gpu_mem_usage_dec(kctx->kbdev, kctx, freed); - } - - return 0; -@@ -2496,9 +2768,19 @@ void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc, - kbdev, - kctx->id, - (u64)new_page_count); -+ -+ kbase_trace_gpu_mem_usage_dec(kctx->kbdev, kctx, freed); - } - } -+KBASE_EXPORT_TEST_API(kbase_free_phy_pages_helper_locked); - -+#if MALI_USE_CSF -+/** -+ * kbase_jd_user_buf_unpin_pages - Release the pinned pages of a user buffer. -+ * @alloc: The allocation for the imported user buffer. -+ */ -+static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc); -+#endif - - void kbase_mem_kref_free(struct kref *kref) - { -@@ -2540,8 +2822,10 @@ void kbase_mem_kref_free(struct kref *kref) - aliased = alloc->imported.alias.aliased; - if (aliased) { - for (i = 0; i < alloc->imported.alias.nents; i++) -- if (aliased[i].alloc) -+ if (aliased[i].alloc) { -+ kbase_mem_phy_alloc_gpu_unmapped(aliased[i].alloc); - kbase_mem_phy_alloc_put(aliased[i].alloc); -+ } - vfree(aliased); - } - break; -@@ -2558,12 +2842,17 @@ void kbase_mem_kref_free(struct kref *kref) - alloc->imported.umm.dma_attachment, - alloc->imported.umm.sgt, - DMA_BIDIRECTIONAL); -+ kbase_remove_dma_buf_usage(alloc->imported.umm.kctx, -+ alloc); - } - dma_buf_detach(alloc->imported.umm.dma_buf, - alloc->imported.umm.dma_attachment); - dma_buf_put(alloc->imported.umm.dma_buf); - break; - case KBASE_MEM_TYPE_IMPORTED_USER_BUF: -+#if MALI_USE_CSF -+ kbase_jd_user_buf_unpin_pages(alloc); -+#endif - if (alloc->imported.user_buf.mm) - mmdrop(alloc->imported.user_buf.mm); - if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) -@@ -2587,7 +2876,7 @@ KBASE_EXPORT_TEST_API(kbase_mem_kref_free); - - int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size) - { -- KBASE_DEBUG_ASSERT(NULL != reg); -+ KBASE_DEBUG_ASSERT(reg != NULL); - KBASE_DEBUG_ASSERT(vsize > 0); - - /* validate user provided arguments */ -@@ -2600,7 +2889,7 @@ int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size - if ((size_t) vsize > ((size_t) -1 / sizeof(*reg->cpu_alloc->pages))) - goto out_term; - -- KBASE_DEBUG_ASSERT(0 != vsize); -+ KBASE_DEBUG_ASSERT(vsize != 0); - - if (kbase_alloc_phy_pages_helper(reg->cpu_alloc, size) != 0) - goto out_term; -@@ -2643,22 +2932,37 @@ bool kbase_check_alloc_flags(unsigned long flags) - /* GPU executable memory cannot: - * - Be written by the GPU - * - Be grown on GPU page fault -- * - Have the top of its initial commit aligned to 'extent' */ -+ */ -+ if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & -+ (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF))) -+ return false; -+ -+#if !MALI_USE_CSF -+ /* GPU executable memory also cannot have the top of its initial -+ * commit aligned to 'extension' -+ */ - if ((flags & BASE_MEM_PROT_GPU_EX) && (flags & -- (BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF | -- BASE_MEM_TILER_ALIGN_TOP))) -+ BASE_MEM_TILER_ALIGN_TOP)) - return false; -+#endif /* !MALI_USE_CSF */ - - /* To have an allocation lie within a 4GB chunk is required only for -- * TLS memory, which will never be used to contain executable code -- * and also used for Tiler heap. -+ * TLS memory, which will never be used to contain executable code. - */ - if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & -- (BASE_MEM_PROT_GPU_EX | BASE_MEM_TILER_ALIGN_TOP))) -+ BASE_MEM_PROT_GPU_EX)) - return false; - -+#if !MALI_USE_CSF -+ /* TLS memory should also not be used for tiler heap */ -+ if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags & -+ BASE_MEM_TILER_ALIGN_TOP)) -+ return false; -+#endif /* !MALI_USE_CSF */ -+ - /* GPU should have at least read or write access otherwise there is no -- reason for allocating. */ -+ * reason for allocating. -+ */ - if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) - return false; - -@@ -2666,14 +2970,15 @@ bool kbase_check_alloc_flags(unsigned long flags) - if ((flags & BASE_MEM_IMPORT_SHARED) == BASE_MEM_IMPORT_SHARED) - return false; - -- /* BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP is only valid for imported -- * memory */ -+ /* BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP is only valid for imported memory -+ */ - if ((flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) == - BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) - return false; - - /* Should not combine BASE_MEM_COHERENT_LOCAL with -- * BASE_MEM_COHERENT_SYSTEM */ -+ * BASE_MEM_COHERENT_SYSTEM -+ */ - if ((flags & (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) == - (BASE_MEM_COHERENT_LOCAL | BASE_MEM_COHERENT_SYSTEM)) - return false; -@@ -2699,12 +3004,15 @@ bool kbase_check_import_flags(unsigned long flags) - if (flags & BASE_MEM_GROW_ON_GPF) - return false; - -+#if !MALI_USE_CSF - /* Imported memory cannot be aligned to the end of its initial commit */ - if (flags & BASE_MEM_TILER_ALIGN_TOP) - return false; -+#endif /* !MALI_USE_CSF */ - - /* GPU should have at least read or write access otherwise there is no -- reason for importing. */ -+ * reason for importing. -+ */ - if ((flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR)) == 0) - return false; - -@@ -2716,19 +3024,19 @@ bool kbase_check_import_flags(unsigned long flags) - } - - int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, -- u64 va_pages, u64 commit_pages, u64 large_extent) -+ u64 va_pages, u64 commit_pages, u64 large_extension) - { - struct device *dev = kctx->kbdev->dev; - int gpu_pc_bits = kctx->kbdev->gpu_props.props.core_props.log2_program_counter_size; - u64 gpu_pc_pages_max = 1ULL << gpu_pc_bits >> PAGE_SHIFT; - struct kbase_va_region test_reg; - -- /* kbase_va_region's extent member can be of variable size, so check against that type */ -- test_reg.extent = large_extent; -+ /* kbase_va_region's extension member can be of variable size, so check against that type */ -+ test_reg.extension = large_extension; - - #define KBASE_MSG_PRE "GPU allocation attempted with " - -- if (0 == va_pages) { -+ if (va_pages == 0) { - dev_warn(dev, KBASE_MSG_PRE "0 va_pages!"); - return -EINVAL; - } -@@ -2740,7 +3048,8 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, - } - - /* Note: commit_pages is checked against va_pages during -- * kbase_alloc_phy_pages() */ -+ * kbase_alloc_phy_pages() -+ */ - - /* Limit GPU executable allocs to GPU PC size */ - if ((flags & BASE_MEM_PROT_GPU_EX) && (va_pages > gpu_pc_pages_max)) { -@@ -2751,47 +3060,73 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, - return -EINVAL; - } - -- if ((flags & (BASE_MEM_GROW_ON_GPF | BASE_MEM_TILER_ALIGN_TOP)) && -- test_reg.extent == 0) { -- dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GROW_ON_GPF or BASE_MEM_TILER_ALIGN_TOP but extent == 0\n"); -+ if ((flags & BASE_MEM_GROW_ON_GPF) && (test_reg.extension == 0)) { -+ dev_warn(dev, KBASE_MSG_PRE -+ "BASE_MEM_GROW_ON_GPF but extension == 0\n"); -+ return -EINVAL; -+ } -+ -+#if !MALI_USE_CSF -+ if ((flags & BASE_MEM_TILER_ALIGN_TOP) && (test_reg.extension == 0)) { -+ dev_warn(dev, KBASE_MSG_PRE -+ "BASE_MEM_TILER_ALIGN_TOP but extension == 0\n"); - return -EINVAL; - } - - if (!(flags & (BASE_MEM_GROW_ON_GPF | BASE_MEM_TILER_ALIGN_TOP)) && -- test_reg.extent != 0) { -- dev_warn(dev, KBASE_MSG_PRE "neither BASE_MEM_GROW_ON_GPF nor BASE_MEM_TILER_ALIGN_TOP set but extent != 0\n"); -+ test_reg.extension != 0) { -+ dev_warn( -+ dev, KBASE_MSG_PRE -+ "neither BASE_MEM_GROW_ON_GPF nor BASE_MEM_TILER_ALIGN_TOP set but extension != 0\n"); -+ return -EINVAL; -+ } -+#else -+ if (!(flags & BASE_MEM_GROW_ON_GPF) && test_reg.extension != 0) { -+ dev_warn(dev, KBASE_MSG_PRE -+ "BASE_MEM_GROW_ON_GPF not set but extension != 0\n"); - return -EINVAL; - } -+#endif /* !MALI_USE_CSF */ - -+#if !MALI_USE_CSF - /* BASE_MEM_TILER_ALIGN_TOP memory has a number of restrictions */ - if (flags & BASE_MEM_TILER_ALIGN_TOP) { - #define KBASE_MSG_PRE_FLAG KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP and " -- unsigned long small_extent; -- -- if (large_extent > BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES) { -- dev_warn(dev, KBASE_MSG_PRE_FLAG "extent==%lld pages exceeds limit %lld", -- (unsigned long long)large_extent, -- BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES); -+ unsigned long small_extension; -+ -+ if (large_extension > -+ BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES) { -+ dev_warn(dev, -+ KBASE_MSG_PRE_FLAG -+ "extension==%lld pages exceeds limit %lld", -+ (unsigned long long)large_extension, -+ BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES); - return -EINVAL; - } - /* For use with is_power_of_2, which takes unsigned long, so -- * must ensure e.g. on 32-bit kernel it'll fit in that type */ -- small_extent = (unsigned long)large_extent; -+ * must ensure e.g. on 32-bit kernel it'll fit in that type -+ */ -+ small_extension = (unsigned long)large_extension; - -- if (!is_power_of_2(small_extent)) { -- dev_warn(dev, KBASE_MSG_PRE_FLAG "extent==%ld not a non-zero power of 2", -- small_extent); -+ if (!is_power_of_2(small_extension)) { -+ dev_warn(dev, -+ KBASE_MSG_PRE_FLAG -+ "extension==%ld not a non-zero power of 2", -+ small_extension); - return -EINVAL; - } - -- if (commit_pages > large_extent) { -- dev_warn(dev, KBASE_MSG_PRE_FLAG "commit_pages==%ld exceeds extent==%ld", -- (unsigned long)commit_pages, -- (unsigned long)large_extent); -+ if (commit_pages > large_extension) { -+ dev_warn(dev, -+ KBASE_MSG_PRE_FLAG -+ "commit_pages==%ld exceeds extension==%ld", -+ (unsigned long)commit_pages, -+ (unsigned long)large_extension); - return -EINVAL; - } - #undef KBASE_MSG_PRE_FLAG - } -+#endif /* !MALI_USE_CSF */ - - if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && - (va_pages > (BASE_MEM_PFN_MASK_4GB + 1))) { -@@ -2805,7 +3140,8 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, - } - - /** -- * @brief Acquire the per-context region list lock -+ * Acquire the per-context region list lock -+ * @kctx: KBase context - */ - void kbase_gpu_vm_lock(struct kbase_context *kctx) - { -@@ -2816,7 +3152,8 @@ void kbase_gpu_vm_lock(struct kbase_context *kctx) - KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); - - /** -- * @brief Release the per-context region list lock -+ * Release the per-context region list lock -+ * @kctx: KBase context - */ - void kbase_gpu_vm_unlock(struct kbase_context *kctx) - { -@@ -2826,7 +3163,7 @@ void kbase_gpu_vm_unlock(struct kbase_context *kctx) - - KBASE_EXPORT_TEST_API(kbase_gpu_vm_unlock); - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - struct kbase_jit_debugfs_data { - int (*func)(struct kbase_jit_debugfs_data *); - struct mutex lock; -@@ -2879,7 +3216,7 @@ static ssize_t kbase_jit_debugfs_common_read(struct file *file, - } - - size = scnprintf(data->buffer, sizeof(data->buffer), -- "%llu,%llu,%llu", data->active_value, -+ "%llu,%llu,%llu\n", data->active_value, - data->pool_value, data->destroy_value); - } - -@@ -2983,19 +3320,23 @@ static int kbase_jit_debugfs_phys_get(struct kbase_jit_debugfs_data *data) - KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_phys_fops, - kbase_jit_debugfs_phys_get); - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - static int kbase_jit_debugfs_used_get(struct kbase_jit_debugfs_data *data) - { - struct kbase_context *kctx = data->kctx; - struct kbase_va_region *reg; - -+#if !MALI_USE_CSF - mutex_lock(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ - mutex_lock(&kctx->jit_evict_lock); - list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { - data->active_value += reg->used_pages; - } - mutex_unlock(&kctx->jit_evict_lock); -+#if !MALI_USE_CSF - mutex_unlock(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ - - return 0; - } -@@ -3012,7 +3353,9 @@ static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data) - struct kbase_context *kctx = data->kctx; - struct kbase_va_region *reg; - -+#if !MALI_USE_CSF - mutex_lock(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ - kbase_gpu_vm_lock(kctx); - mutex_lock(&kctx->jit_evict_lock); - list_for_each_entry(reg, &kctx->jit_active_head, jit_node) { -@@ -3031,14 +3374,16 @@ static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data) - } - mutex_unlock(&kctx->jit_evict_lock); - kbase_gpu_vm_unlock(kctx); -+#if !MALI_USE_CSF - mutex_unlock(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ - - return 0; - } - - KBASE_JIT_DEBUGFS_DECLARE(kbase_jit_debugfs_trim_fops, - kbase_jit_debugfs_trim_get); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - void kbase_jit_debugfs_init(struct kbase_context *kctx) - { -@@ -3078,7 +3423,7 @@ void kbase_jit_debugfs_init(struct kbase_context *kctx) - */ - debugfs_create_file("mem_jit_phys", mode, kctx->kctx_dentry, - kctx, &kbase_jit_debugfs_phys_fops); --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /* - * Debugfs entry for getting the number of pages used - * by JIT allocations for estimating the physical pressure -@@ -3093,7 +3438,7 @@ void kbase_jit_debugfs_init(struct kbase_context *kctx) - */ - debugfs_create_file("mem_jit_trim", mode, kctx->kctx_dentry, - kctx, &kbase_jit_debugfs_trim_fops); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - } - #endif /* CONFIG_DEBUG_FS */ - -@@ -3138,8 +3483,13 @@ int kbase_jit_init(struct kbase_context *kctx) - INIT_LIST_HEAD(&kctx->jit_destroy_head); - INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker); - -+#if MALI_USE_CSF -+ INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_cmds_head); -+ INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_blocked_queues); -+#else /* !MALI_USE_CSF */ - INIT_LIST_HEAD(&kctx->jctx.jit_atoms_head); - INIT_LIST_HEAD(&kctx->jctx.jit_pending_alloc); -+#endif /* MALI_USE_CSF */ - mutex_unlock(&kctx->jit_evict_lock); - - kctx->jit_max_allocations = 0; -@@ -3153,25 +3503,29 @@ int kbase_jit_init(struct kbase_context *kctx) - * allocation and also, if BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP is set, meets - * the alignment requirements. - */ --static bool meet_size_and_tiler_align_top_requirements(struct kbase_context *kctx, -- struct kbase_va_region *walker, const struct base_jit_alloc_info *info) -+static bool meet_size_and_tiler_align_top_requirements( -+ const struct kbase_va_region *walker, -+ const struct base_jit_alloc_info *info) - { - bool meet_reqs = true; - - if (walker->nr_pages != info->va_pages) - meet_reqs = false; -- else if (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP) { -- size_t align = info->extent; -+ -+#if !MALI_USE_CSF -+ if (meet_reqs && (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP)) { -+ size_t align = info->extension; - size_t align_mask = align - 1; - - if ((walker->start_pfn + info->commit_pages) & align_mask) - meet_reqs = false; - } -+#endif /* !MALI_USE_CSF */ - - return meet_reqs; - } - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /* Function will guarantee *@freed will not exceed @pages_needed - */ - static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, -@@ -3185,7 +3539,9 @@ static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, - size_t to_free = 0u; - size_t max_allowed_pages = old_pages; - -+#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ - lockdep_assert_held(&kctx->reg_lock); - - /* Is this a JIT allocation that has been reported on? */ -@@ -3213,20 +3569,20 @@ static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx, - KBASE_GPU_ALLOCATED_OBJECT_ALIGN_BYTES); - } else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { - /* The GPU could report being ready to write to the next -- * 'extent' sized chunk, but didn't actually write to it, so we -- * can report up to 'extent' size pages more than the backed -+ * 'extension' sized chunk, but didn't actually write to it, so we -+ * can report up to 'extension' size pages more than the backed - * size. - * - * Note, this is allowed to exceed reg->nr_pages. - */ -- max_allowed_pages += reg->extent; -+ max_allowed_pages += reg->extension; - - /* Also note that in these GPUs, the GPU may make a large (>1 - * page) initial allocation but not actually write out to all - * of it. Hence it might report that a much higher amount of - * memory was used than actually was written to. This does not - * result in a real warning because on growing this memory we -- * round up the size of the allocation up to an 'extent' sized -+ * round up the size of the allocation up to an 'extension' sized - * chunk, hence automatically bringing the backed size up to - * the reported size. - */ -@@ -3308,8 +3664,12 @@ static size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, - struct kbase_va_region *reg, *tmp; - size_t total_freed = 0; - -- kbase_gpu_vm_lock(kctx); -- mutex_lock(&kctx->jit_evict_lock); -+#if !MALI_USE_CSF -+ lockdep_assert_held(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ -+ lockdep_assert_held(&kctx->reg_lock); -+ lockdep_assert_held(&kctx->jit_evict_lock); -+ - list_for_each_entry_safe(reg, tmp, &kctx->jit_active_head, jit_node) { - int err; - size_t freed = 0u; -@@ -3328,18 +3688,17 @@ static size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx, - if (!pages_needed) - break; - } -- mutex_unlock(&kctx->jit_evict_lock); -- kbase_gpu_vm_unlock(kctx); - - trace_mali_jit_trim(total_freed); - - return total_freed; - } --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - static int kbase_jit_grow(struct kbase_context *kctx, -- const struct base_jit_alloc_info *info, -- struct kbase_va_region *reg) -+ const struct base_jit_alloc_info *info, -+ struct kbase_va_region *reg, -+ struct kbase_sub_alloc **prealloc_sas) - { - size_t delta; - size_t pages_required; -@@ -3347,15 +3706,13 @@ static int kbase_jit_grow(struct kbase_context *kctx, - struct kbase_mem_pool *pool; - int ret = -ENOMEM; - struct tagged_addr *gpu_pages; -- struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; -- int i; - - if (info->commit_pages > reg->nr_pages) { - /* Attempted to grow larger than maximum size */ - return -EINVAL; - } - -- kbase_gpu_vm_lock(kctx); -+ lockdep_assert_held(&kctx->reg_lock); - - /* Make the physical backing no longer reclaimable */ - if (!kbase_mem_evictable_unmake(reg->gpu_alloc)) -@@ -3372,14 +3729,6 @@ static int kbase_jit_grow(struct kbase_context *kctx, - pages_required = delta; - - #ifdef CONFIG_MALI_2MB_ALLOC -- /* Preallocate memory for the sub-allocation structs */ -- for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { -- prealloc_sas[i] = kmalloc(sizeof(*prealloc_sas[i]), -- GFP_KERNEL); -- if (!prealloc_sas[i]) -- goto update_failed; -- } -- - if (pages_required >= (SZ_2M / SZ_4K)) { - pool = &kctx->mem_pools.large[kctx->jit_group_id]; - /* Round up to number of 2 MB pages required */ -@@ -3405,15 +3754,18 @@ static int kbase_jit_grow(struct kbase_context *kctx, - */ - while (kbase_mem_pool_size(pool) < pages_required) { - int pool_delta = pages_required - kbase_mem_pool_size(pool); -+ int ret; - - kbase_mem_pool_unlock(pool); - spin_unlock(&kctx->mem_partials_lock); -+ - kbase_gpu_vm_unlock(kctx); -+ ret = kbase_mem_pool_grow(pool, pool_delta); -+ kbase_gpu_vm_lock(kctx); - -- if (kbase_mem_pool_grow(pool, pool_delta)) -- goto update_failed_unlocked; -+ if (ret) -+ goto update_failed; - -- kbase_gpu_vm_lock(kctx); - spin_lock(&kctx->mem_partials_lock); - kbase_mem_pool_lock(pool); - } -@@ -3456,14 +3808,9 @@ done: - - /* Update attributes of JIT allocation taken from the pool */ - reg->initial_commit = info->commit_pages; -- reg->extent = info->extent; -+ reg->extension = info->extension; - - update_failed: -- kbase_gpu_vm_unlock(kctx); --update_failed_unlocked: -- for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) -- kfree(prealloc_sas[i]); -- - return ret; - } - -@@ -3492,9 +3839,9 @@ static void trace_jit_stats(struct kbase_context *kctx, - max_allocations, alloc_count, va_pages, ph_pages); - } - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /** -- * get_jit_backed_pressure() - calculate the physical backing of all JIT -+ * get_jit_phys_backing() - calculate the physical backing of all JIT - * allocations - * - * @kctx: Pointer to the kbase context whose active JIT allocations will be -@@ -3502,83 +3849,50 @@ static void trace_jit_stats(struct kbase_context *kctx, - * - * Return: number of pages that are committed by JIT allocations - */ --static size_t get_jit_backed_pressure(struct kbase_context *kctx) -+static size_t get_jit_phys_backing(struct kbase_context *kctx) - { -- size_t backed_pressure = 0; -- int jit_id; -- -- lockdep_assert_held(&kctx->jctx.lock); -+ struct kbase_va_region *walker; -+ size_t backing = 0; - -- kbase_gpu_vm_lock(kctx); -- for (jit_id = 0; jit_id <= BASE_JIT_ALLOC_COUNT; jit_id++) { -- struct kbase_va_region *reg = kctx->jit_alloc[jit_id]; -+ lockdep_assert_held(&kctx->jit_evict_lock); - -- if (reg && (reg != KBASE_RESERVED_REG_JIT_ALLOC)) { -- /* If region has no report, be pessimistic */ -- if (reg->used_pages == reg->nr_pages) { -- backed_pressure += reg->nr_pages; -- } else { -- backed_pressure += -- kbase_reg_current_backed_size(reg); -- } -- } -+ list_for_each_entry(walker, &kctx->jit_active_head, jit_node) { -+ backing += kbase_reg_current_backed_size(walker); - } -- kbase_gpu_vm_unlock(kctx); - -- return backed_pressure; -+ return backing; - } - --/** -- * jit_trim_necessary_pages() - calculate and trim the least pages possible to -- * satisfy a new JIT allocation -- * -- * @kctx: Pointer to the kbase context -- * @info: Pointer to JIT allocation information for the new allocation -- * -- * Before allocating a new just-in-time memory region or reusing a previous -- * one, ensure that the total JIT physical page usage also will not exceed the -- * pressure limit. -- * -- * If there are no reported-on allocations, then we already guarantee this will -- * be the case - because our current pressure then only comes from the va_pages -- * of each JIT region, hence JIT physical page usage is guaranteed to be -- * bounded by this. -- * -- * However as soon as JIT allocations become "reported on", the pressure is -- * lowered to allow new JIT regions to be allocated. It is after such a point -- * that the total JIT physical page usage could (either now or in the future on -- * a grow-on-GPU-page-fault) exceed the pressure limit, but only on newly -- * allocated JIT regions. Hence, trim any "reported on" regions. -- * -- * Any pages freed will go into the pool and be allocated from there in -- * kbase_mem_alloc(). -- */ --static void jit_trim_necessary_pages(struct kbase_context *kctx, -- const struct base_jit_alloc_info *info) -+void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, -+ size_t needed_pages) - { -- size_t backed_pressure = 0; -- size_t needed_pages = 0; -+ size_t jit_backing = 0; -+ size_t pages_to_trim = 0; - -- backed_pressure = get_jit_backed_pressure(kctx); -+#if !MALI_USE_CSF -+ lockdep_assert_held(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ -+ lockdep_assert_held(&kctx->reg_lock); -+ lockdep_assert_held(&kctx->jit_evict_lock); -+ -+ jit_backing = get_jit_phys_backing(kctx); - - /* It is possible that this is the case - if this is the first - * allocation after "ignore_pressure_limit" allocation. - */ -- if (backed_pressure > kctx->jit_phys_pages_limit) { -- needed_pages += -- (backed_pressure - kctx->jit_phys_pages_limit) -- + info->va_pages; -+ if (jit_backing > kctx->jit_phys_pages_limit) { -+ pages_to_trim += (jit_backing - kctx->jit_phys_pages_limit) + -+ needed_pages; - } else { -- size_t backed_diff = -- kctx->jit_phys_pages_limit - backed_pressure; -+ size_t backed_diff = kctx->jit_phys_pages_limit - jit_backing; - -- if (info->va_pages > backed_diff) -- needed_pages += info->va_pages - backed_diff; -+ if (needed_pages > backed_diff) -+ pages_to_trim += needed_pages - backed_diff; - } - -- if (needed_pages) { -- size_t trimmed_pages = kbase_mem_jit_trim_pages(kctx, -- needed_pages); -+ if (pages_to_trim) { -+ size_t trimmed_pages = -+ kbase_mem_jit_trim_pages(kctx, pages_to_trim); - - /* This should never happen - we already asserted that - * we are not violating JIT pressure limit in earlier -@@ -3586,10 +3900,10 @@ static void jit_trim_necessary_pages(struct kbase_context *kctx, - * must have enough unused pages to satisfy the new - * allocation - */ -- WARN_ON(trimmed_pages < needed_pages); -+ WARN_ON(trimmed_pages < pages_to_trim); - } - } --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /** - * jit_allow_allocate() - check whether basic conditions are satisfied to allow -@@ -3606,10 +3920,14 @@ static bool jit_allow_allocate(struct kbase_context *kctx, - const struct base_jit_alloc_info *info, - bool ignore_pressure_limit) - { -+#if MALI_USE_CSF -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+#else - lockdep_assert_held(&kctx->jctx.lock); -+#endif - --#if MALI_JIT_PRESSURE_LIMIT -- if (likely(!ignore_pressure_limit) && -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (!ignore_pressure_limit && - ((kctx->jit_phys_pages_limit <= kctx->jit_current_phys_pressure) || - (info->va_pages > (kctx->jit_phys_pages_limit - kctx->jit_current_phys_pressure)))) { - dev_dbg(kctx->kbdev->dev, -@@ -3618,7 +3936,7 @@ static bool jit_allow_allocate(struct kbase_context *kctx, - kctx->jit_phys_pages_limit); - return false; - } --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - if (kctx->jit_current_allocations >= kctx->jit_max_allocations) { - /* Too many current allocations */ -@@ -3644,123 +3962,156 @@ static bool jit_allow_allocate(struct kbase_context *kctx, - return true; - } - -+static struct kbase_va_region * -+find_reasonable_region(const struct base_jit_alloc_info *info, -+ struct list_head *pool_head, bool ignore_usage_id) -+{ -+ struct kbase_va_region *closest_reg = NULL; -+ struct kbase_va_region *walker; -+ size_t current_diff = SIZE_MAX; -+ -+ list_for_each_entry(walker, pool_head, jit_node) { -+ if ((ignore_usage_id || -+ walker->jit_usage_id == info->usage_id) && -+ walker->jit_bin_id == info->bin_id && -+ meet_size_and_tiler_align_top_requirements(walker, info)) { -+ size_t min_size, max_size, diff; -+ -+ /* -+ * The JIT allocations VA requirements have been met, -+ * it's suitable but other allocations might be a -+ * better fit. -+ */ -+ min_size = min_t(size_t, walker->gpu_alloc->nents, -+ info->commit_pages); -+ max_size = max_t(size_t, walker->gpu_alloc->nents, -+ info->commit_pages); -+ diff = max_size - min_size; -+ -+ if (current_diff > diff) { -+ current_diff = diff; -+ closest_reg = walker; -+ } -+ -+ /* The allocation is an exact match */ -+ if (current_diff == 0) -+ break; -+ } -+ } -+ -+ return closest_reg; -+} -+ - struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, - const struct base_jit_alloc_info *info, - bool ignore_pressure_limit) - { - struct kbase_va_region *reg = NULL; -+ struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; -+ int i; - -+#if MALI_USE_CSF -+ lockdep_assert_held(&kctx->csf.kcpu_queues.lock); -+#else - lockdep_assert_held(&kctx->jctx.lock); -+#endif - - if (!jit_allow_allocate(kctx, info, ignore_pressure_limit)) - return NULL; - --#if MALI_JIT_PRESSURE_LIMIT -- if (!ignore_pressure_limit) -- jit_trim_necessary_pages(kctx, info); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#ifdef CONFIG_MALI_2MB_ALLOC -+ /* Preallocate memory for the sub-allocation structs */ -+ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) { -+ prealloc_sas[i] = kmalloc(sizeof(*prealloc_sas[i]), GFP_KERNEL); -+ if (!prealloc_sas[i]) -+ goto end; -+ } -+#endif - -+ kbase_gpu_vm_lock(kctx); - mutex_lock(&kctx->jit_evict_lock); - - /* - * Scan the pool for an existing allocation which meets our - * requirements and remove it. - */ -- if (info->usage_id != 0) { -+ if (info->usage_id != 0) - /* First scan for an allocation with the same usage ID */ -- struct kbase_va_region *walker; -- size_t current_diff = SIZE_MAX; -- -- list_for_each_entry(walker, &kctx->jit_pool_head, jit_node) { -- -- if (walker->jit_usage_id == info->usage_id && -- walker->jit_bin_id == info->bin_id && -- meet_size_and_tiler_align_top_requirements( -- kctx, walker, info)) { -- size_t min_size, max_size, diff; -- -- /* -- * The JIT allocations VA requirements have been -- * met, it's suitable but other allocations -- * might be a better fit. -- */ -- min_size = min_t(size_t, -- walker->gpu_alloc->nents, -- info->commit_pages); -- max_size = max_t(size_t, -- walker->gpu_alloc->nents, -- info->commit_pages); -- diff = max_size - min_size; -- -- if (current_diff > diff) { -- current_diff = diff; -- reg = walker; -- } -- -- /* The allocation is an exact match */ -- if (current_diff == 0) -- break; -- } -- } -- } -+ reg = find_reasonable_region(info, &kctx->jit_pool_head, false); - -- if (!reg) { -+ if (!reg) - /* No allocation with the same usage ID, or usage IDs not in - * use. Search for an allocation we can reuse. - */ -- struct kbase_va_region *walker; -- size_t current_diff = SIZE_MAX; -- -- list_for_each_entry(walker, &kctx->jit_pool_head, jit_node) { -- -- if (walker->jit_bin_id == info->bin_id && -- meet_size_and_tiler_align_top_requirements( -- kctx, walker, info)) { -- size_t min_size, max_size, diff; -- -- /* -- * The JIT allocations VA requirements have been -- * met, it's suitable but other allocations -- * might be a better fit. -- */ -- min_size = min_t(size_t, -- walker->gpu_alloc->nents, -- info->commit_pages); -- max_size = max_t(size_t, -- walker->gpu_alloc->nents, -- info->commit_pages); -- diff = max_size - min_size; -- -- if (current_diff > diff) { -- current_diff = diff; -- reg = walker; -- } -- -- /* The allocation is an exact match, so stop -- * looking. -- */ -- if (current_diff == 0) -- break; -- } -- } -- } -+ reg = find_reasonable_region(info, &kctx->jit_pool_head, true); - - if (reg) { -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ size_t needed_pages = 0; -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -+ int ret; -+ - /* - * Remove the found region from the pool and add it to the - * active list. - */ - list_move(®->jit_node, &kctx->jit_active_head); - -+ WARN_ON(reg->gpu_alloc->evicted); -+ - /* - * Remove the allocation from the eviction list as it's no - * longer eligible for eviction. This must be done before - * dropping the jit_evict_lock - */ - list_del_init(®->gpu_alloc->evict_node); -+ -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (!ignore_pressure_limit) { -+ if (info->commit_pages > reg->gpu_alloc->nents) -+ needed_pages = info->commit_pages - -+ reg->gpu_alloc->nents; -+ -+ /* Update early the recycled JIT region's estimate of -+ * used_pages to ensure it doesn't get trimmed -+ * undesirably. This is needed as the recycled JIT -+ * region has been added to the active list but the -+ * number of used pages for it would be zero, so it -+ * could get trimmed instead of other allocations only -+ * to be regrown later resulting in a breach of the JIT -+ * physical pressure limit. -+ * Also that trimming would disturb the accounting of -+ * physical pages, i.e. the VM stats, as the number of -+ * backing pages would have changed when the call to -+ * kbase_mem_evictable_unmark_reclaim is made. -+ * -+ * The second call to update pressure at the end of -+ * this function would effectively be a nop. -+ */ -+ kbase_jit_report_update_pressure( -+ kctx, reg, info->va_pages, -+ KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); -+ -+ kbase_jit_request_phys_increase_locked(kctx, -+ needed_pages); -+ } -+#endif - mutex_unlock(&kctx->jit_evict_lock); - -- if (kbase_jit_grow(kctx, info, reg) < 0) { -+ /* kbase_jit_grow() can release & reacquire 'kctx->reg_lock', -+ * so any state protected by that lock might need to be -+ * re-evaluated if more code is added here in future. -+ */ -+ ret = kbase_jit_grow(kctx, info, reg, prealloc_sas); -+ -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (!ignore_pressure_limit) -+ kbase_jit_done_phys_increase(kctx, needed_pages); -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -+ -+ kbase_gpu_vm_unlock(kctx); -+ -+ if (ret < 0) { - /* - * An update to an allocation from the pool failed, - * chances are slim a new allocation would fair any -@@ -3770,10 +4121,21 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, - dev_dbg(kctx->kbdev->dev, - "JIT allocation resize failed: va_pages 0x%llx, commit_pages 0x%llx\n", - info->va_pages, info->commit_pages); -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ /* Undo the early change made to the recycled JIT -+ * region's estimate of used_pages. -+ */ -+ if (!ignore_pressure_limit) { -+ kbase_jit_report_update_pressure( -+ kctx, reg, 0, -+ KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); -+ } -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - mutex_lock(&kctx->jit_evict_lock); - list_move(®->jit_node, &kctx->jit_pool_head); - mutex_unlock(&kctx->jit_evict_lock); -- return NULL; -+ reg = NULL; -+ goto end; - } - } else { - /* No suitable JIT allocation was found so create a new one */ -@@ -3783,15 +4145,28 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, - BASEP_MEM_NO_USER_FREE; - u64 gpu_addr; - -- mutex_unlock(&kctx->jit_evict_lock); -- -+#if !MALI_USE_CSF - if (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP) - flags |= BASE_MEM_TILER_ALIGN_TOP; -+#endif /* !MALI_USE_CSF */ - - flags |= base_mem_group_id_set(kctx->jit_group_id); -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (!ignore_pressure_limit) { -+ flags |= BASEP_MEM_PERFORM_JIT_TRIM; -+ /* The corresponding call to 'done_phys_increase' would -+ * be made inside the kbase_mem_alloc(). -+ */ -+ kbase_jit_request_phys_increase_locked( -+ kctx, info->commit_pages); -+ } -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -+ -+ mutex_unlock(&kctx->jit_evict_lock); -+ kbase_gpu_vm_unlock(kctx); - - reg = kbase_mem_alloc(kctx, info->va_pages, info->commit_pages, -- info->extent, &flags, &gpu_addr); -+ info->extension, &flags, &gpu_addr); - if (!reg) { - /* Most likely not enough GPU virtual space left for - * the new JIT allocation. -@@ -3799,12 +4174,22 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, - dev_dbg(kctx->kbdev->dev, - "Failed to allocate JIT memory: va_pages 0x%llx, commit_pages 0x%llx\n", - info->va_pages, info->commit_pages); -- return NULL; -+ goto end; - } - -- mutex_lock(&kctx->jit_evict_lock); -- list_add(®->jit_node, &kctx->jit_active_head); -- mutex_unlock(&kctx->jit_evict_lock); -+ if (!ignore_pressure_limit) { -+ /* Due to enforcing of pressure limit, kbase_mem_alloc -+ * was instructed to perform the trimming which in turn -+ * would have ensured that the new JIT allocation is -+ * already in the jit_active_head list, so nothing to -+ * do here. -+ */ -+ WARN_ON(list_empty(®->jit_node)); -+ } else { -+ mutex_lock(&kctx->jit_evict_lock); -+ list_add(®->jit_node, &kctx->jit_active_head); -+ mutex_unlock(&kctx->jit_evict_lock); -+ } - } - - trace_mali_jit_alloc(reg, info->id); -@@ -3816,13 +4201,18 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx, - - reg->jit_usage_id = info->usage_id; - reg->jit_bin_id = info->bin_id; --#if MALI_JIT_PRESSURE_LIMIT -+ reg->flags |= KBASE_REG_ACTIVE_JIT_ALLOC; -+#if MALI_JIT_PRESSURE_LIMIT_BASE - if (info->flags & BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) - reg->flags = reg->flags | KBASE_REG_HEAP_INFO_IS_SIZE; - reg->heap_info_gpu_addr = info->heap_info_gpu_addr; - kbase_jit_report_update_pressure(kctx, reg, info->va_pages, - KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -+ -+end: -+ for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) -+ kfree(prealloc_sas[i]); - - return reg; - } -@@ -3844,15 +4234,18 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) - div_u64(old_pages * (100 - kctx->trim_level), 100)); - u64 delta = old_pages - new_size; - -- if (delta) -+ if (delta) { -+ mutex_lock(&kctx->reg_lock); - kbase_mem_shrink(kctx, reg, old_pages - delta); -+ mutex_unlock(&kctx->reg_lock); -+ } - } - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - reg->heap_info_gpu_addr = 0; - kbase_jit_report_update_pressure(kctx, reg, 0, - KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - kctx->jit_current_allocations--; - kctx->jit_current_allocations_per_bin[reg->jit_bin_id]--; -@@ -3863,6 +4256,7 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) - - kbase_gpu_vm_lock(kctx); - reg->flags |= KBASE_REG_DONT_NEED; -+ reg->flags &= ~KBASE_REG_ACTIVE_JIT_ALLOC; - kbase_mem_shrink_cpu_mapping(kctx, reg, 0, reg->gpu_alloc->nents); - kbase_gpu_vm_unlock(kctx); - -@@ -3875,6 +4269,7 @@ void kbase_jit_free(struct kbase_context *kctx, struct kbase_va_region *reg) - /* This allocation can't already be on a list. */ - WARN_ON(!list_empty(®->gpu_alloc->evict_node)); - list_add(®->gpu_alloc->evict_node, &kctx->evict_list); -+ atomic_add(reg->gpu_alloc->nents, &kctx->evict_nents); - - list_move(®->jit_node, &kctx->jit_pool_head); - -@@ -3962,6 +4357,9 @@ void kbase_jit_term(struct kbase_context *kctx) - kbase_mem_free_region(kctx, walker); - mutex_lock(&kctx->jit_evict_lock); - } -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ WARN_ON(kctx->jit_phys_pages_to_be_allocated); -+#endif - mutex_unlock(&kctx->jit_evict_lock); - kbase_gpu_vm_unlock(kctx); - -@@ -3972,7 +4370,7 @@ void kbase_jit_term(struct kbase_context *kctx) - cancel_work_sync(&kctx->jit_work); - } - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, - struct kbase_va_region *reg, unsigned int flags) - { -@@ -4015,16 +4413,18 @@ void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, - out: - return; - } --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - void kbase_jit_report_update_pressure(struct kbase_context *kctx, - struct kbase_va_region *reg, u64 new_used_pages, - unsigned int flags) - { - u64 diff; - -+#if !MALI_USE_CSF - lockdep_assert_held(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ - - trace_mali_jit_report_pressure(reg, new_used_pages, - kctx->jit_current_phys_pressure + new_used_pages - -@@ -4053,19 +4453,22 @@ void kbase_jit_report_update_pressure(struct kbase_context *kctx, - } - - } --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - --bool kbase_has_exec_va_zone(struct kbase_context *kctx) -+#if MALI_USE_CSF -+static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc) - { -- bool has_exec_va_zone; -+ if (alloc->nents) { -+ struct page **pages = alloc->imported.user_buf.pages; -+ long i; - -- kbase_gpu_vm_lock(kctx); -- has_exec_va_zone = (kctx->exec_va_start != U64_MAX); -- kbase_gpu_vm_unlock(kctx); -+ WARN_ON(alloc->nents != alloc->imported.user_buf.nr_pages); - -- return has_exec_va_zone; -+ for (i = 0; i < alloc->nents; i++) -+ put_page(pages[i]); -+ } - } -- -+#endif - - int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx, - struct kbase_va_region *reg) -@@ -4090,7 +4493,7 @@ int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx, - if (WARN_ON(reg->gpu_alloc->imported.user_buf.mm != current->mm)) - return -EINVAL; - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE - pinned_pages = get_user_pages(NULL, mm, - address, - alloc->imported.user_buf.nr_pages, -@@ -4102,24 +4505,30 @@ KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE - reg->flags & KBASE_REG_GPU_WR, - 0, pages, NULL); - #endif --#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+#elif KERNEL_VERSION(4, 9, 0) > LINUX_VERSION_CODE - pinned_pages = get_user_pages_remote(NULL, mm, - address, - alloc->imported.user_buf.nr_pages, - reg->flags & KBASE_REG_GPU_WR, - 0, pages, NULL); --#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0) -+#elif KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE - pinned_pages = get_user_pages_remote(NULL, mm, - address, - alloc->imported.user_buf.nr_pages, - reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, - pages, NULL); --#else -+#elif KERNEL_VERSION(5, 9, 0) > LINUX_VERSION_CODE - pinned_pages = get_user_pages_remote(NULL, mm, - address, - alloc->imported.user_buf.nr_pages, - reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, - pages, NULL, NULL); -+#else -+ pinned_pages = get_user_pages_remote(mm, -+ address, -+ alloc->imported.user_buf.nr_pages, -+ reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, -+ pages, NULL, NULL); - #endif - - if (pinned_pages <= 0) -@@ -4232,12 +4641,16 @@ static void kbase_jd_user_buf_unmap(struct kbase_context *kctx, - DMA_BIDIRECTIONAL); - if (writeable) - set_page_dirty_lock(pages[i]); -+#if !MALI_USE_CSF - put_page(pages[i]); - pages[i] = NULL; -+#endif - - size -= local_size; - } -+#if !MALI_USE_CSF - alloc->nents = 0; -+#endif - } - - int kbase_mem_copy_to_pinned_user_pages(struct page **dest_pages, -@@ -4296,7 +4709,8 @@ struct kbase_mem_phy_alloc *kbase_map_external_resource( - goto exit; - - reg->gpu_alloc->imported.user_buf.current_mapping_usage_count++; -- if (1 == reg->gpu_alloc->imported.user_buf.current_mapping_usage_count) { -+ if (reg->gpu_alloc->imported.user_buf -+ .current_mapping_usage_count == 1) { - err = kbase_jd_user_buf_map(kctx, reg); - if (err) { - reg->gpu_alloc->imported.user_buf.current_mapping_usage_count--; -@@ -4331,7 +4745,7 @@ void kbase_unmap_external_resource(struct kbase_context *kctx, - case KBASE_MEM_TYPE_IMPORTED_USER_BUF: { - alloc->imported.user_buf.current_mapping_usage_count--; - -- if (0 == alloc->imported.user_buf.current_mapping_usage_count) { -+ if (alloc->imported.user_buf.current_mapping_usage_count == 0) { - bool writeable = true; - - if (!kbase_is_region_invalid_or_free(reg) && -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h -index 6e921ec..e9ac809 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_mem.h -- * Base kernel memory APIs -+ * DOC: Base kernel memory APIs - */ - - #ifndef _KBASE_MEM_H_ -@@ -35,7 +31,7 @@ - #endif - - #include <linux/kref.h> --#include "mali_base_kernel.h" -+#include <uapi/gpu/arm/midgard/mali_base_kernel.h> - #include <mali_kbase_hw.h> - #include "mali_kbase_pm.h" - #include "mali_kbase_defs.h" -@@ -48,10 +44,13 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, - /* Part of the workaround for uTLB invalid pages is to ensure we grow/shrink tmem by 4 pages at a time */ - #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316 (2) /* round to 4 pages */ - --/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by 8 pages. --The MMU reads in 8 page table entries from memory at a time, if we have more than one page fault within the same 8 pages and --page tables are updated accordingly, the MMU does not re-read the page table entries from memory for the subsequent page table --updates and generates duplicate page faults as the page table information used by the MMU is not valid. */ -+/* Part of the workaround for PRLAM-9630 requires us to grow/shrink memory by -+ * 8 pages. The MMU reads in 8 page table entries from memory at a time, if we -+ * have more than one page fault within the same 8 pages and page tables are -+ * updated accordingly, the MMU does not re-read the page table entries from -+ * memory for the subsequent page table updates and generates duplicate page -+ * faults as the page table information used by the MMU is not valid. -+ */ - #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630 (3) /* round to 8 pages */ - - #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2 (0) /* round to 1 page */ -@@ -60,7 +59,8 @@ updates and generates duplicate page faults as the page table information used b - #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2) - #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_8316 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_8316) - #define KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_HW_ISSUE_9630 (1u << KBASEP_TMEM_GROWABLE_BLOCKSIZE_PAGES_LOG2_HW_ISSUE_9630) --/** -+ -+/* - * A CPU mapping - */ - struct kbase_cpu_mapping { -@@ -81,16 +81,15 @@ enum kbase_memory_type { - }; - - /* internal structure, mirroring base_mem_aliasing_info, -- * but with alloc instead of a gpu va (handle) */ -+ * but with alloc instead of a gpu va (handle) -+ */ - struct kbase_aliased { - struct kbase_mem_phy_alloc *alloc; /* NULL for special, non-NULL for native */ - u64 offset; /* in pages */ - u64 length; /* in pages */ - }; - --/** -- * @brief Physical pages tracking object properties -- */ -+/* Physical pages tracking object properties */ - #define KBASE_MEM_PHY_ALLOC_ACCESSED_CACHED (1u << 0) - #define KBASE_MEM_PHY_ALLOC_LARGE (1u << 1) - -@@ -105,7 +104,13 @@ struct kbase_aliased { - * updated as part of the change. - * - * @kref: number of users of this alloc -- * @gpu_mappings: count number of times mapped on the GPU -+ * @gpu_mappings: count number of times mapped on the GPU. Indicates the number -+ * of references there are to the physical pages from different -+ * GPU VA regions. -+ * @kernel_mappings: count number of times mapped on the CPU, specifically in -+ * the kernel. Indicates the number of references there are -+ * to the physical pages to prevent flag changes or shrink -+ * while maps are still held. - * @nents: 0..N - * @pages: N elements, only 0..nents are valid - * @mappings: List of CPU mappings of this physical memory allocation. -@@ -128,6 +133,7 @@ struct kbase_aliased { - struct kbase_mem_phy_alloc { - struct kref kref; - atomic_t gpu_mappings; -+ atomic_t kernel_mappings; - size_t nents; - struct tagged_addr *pages; - struct list_head mappings; -@@ -141,6 +147,7 @@ struct kbase_mem_phy_alloc { - - union { - struct { -+ struct kbase_context *kctx; - struct dma_buf *dma_buf; - struct dma_buf_attachment *dma_attachment; - unsigned int current_mapping_usage_count; -@@ -210,12 +217,36 @@ static inline void kbase_mem_phy_alloc_gpu_unmapped(struct kbase_mem_phy_alloc * - KBASE_DEBUG_ASSERT(alloc); - /* we only track mappings of NATIVE buffers */ - if (alloc->type == KBASE_MEM_TYPE_NATIVE) -- if (0 > atomic_dec_return(&alloc->gpu_mappings)) { -+ if (atomic_dec_return(&alloc->gpu_mappings) < 0) { - pr_err("Mismatched %s:\n", __func__); - dump_stack(); - } - } - -+/** -+ * kbase_mem_phy_alloc_kernel_mapped - Increment kernel_mappings -+ * counter for a memory region to prevent commit and flag changes -+ * -+ * @alloc: Pointer to physical pages tracking object -+ */ -+static inline void -+kbase_mem_phy_alloc_kernel_mapped(struct kbase_mem_phy_alloc *alloc) -+{ -+ atomic_inc(&alloc->kernel_mappings); -+} -+ -+/** -+ * kbase_mem_phy_alloc_kernel_unmapped - Decrement kernel_mappings -+ * counter for a memory region to allow commit and flag changes -+ * -+ * @alloc: Pointer to physical pages tracking object -+ */ -+static inline void -+kbase_mem_phy_alloc_kernel_unmapped(struct kbase_mem_phy_alloc *alloc) -+{ -+ WARN_ON(atomic_dec_return(&alloc->kernel_mappings) < 0); -+} -+ - /** - * kbase_mem_is_imported - Indicate whether a memory type is imported - * -@@ -248,7 +279,7 @@ static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_m - } - - /** -- * A GPU memory region, and attributes for CPU mappings. -+ * struct kbase_va_region - A GPU memory region, and attributes for CPU mappings - * - * @rblink: Node in a red-black tree of memory regions within the same zone of - * the GPU's virtual address space. -@@ -262,13 +293,31 @@ static inline struct kbase_mem_phy_alloc *kbase_mem_phy_alloc_put(struct kbase_m - * @threshold_pages: If non-zero and the amount of memory committed to a region - * that can grow on page fault exceeds this number of pages - * then the driver switches to incremental rendering. -- * @extent: Number of pages allocated on page fault. -+ * @flags: Flags -+ * @extension: Number of pages allocated on page fault. - * @cpu_alloc: The physical memory we mmap to the CPU when mapping this region. - * @gpu_alloc: The physical memory we mmap to the GPU when mapping this region. - * @jit_node: Links to neighboring regions in the just-in-time memory pool. - * @jit_usage_id: The last just-in-time memory usage ID for this region. - * @jit_bin_id: The just-in-time memory bin this region came from. - * @va_refcnt: Number of users of this region. Protected by reg_lock. -+ * @heap_info_gpu_addr: Pointer to an object in GPU memory defining an end of -+ * an allocated region -+ * The object can be one of: -+ * - u32 value defining the size of the region -+ * - u64 pointer first unused byte in the region -+ * The interpretation of the object depends on -+ * BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE flag in -+ * jit_info_flags - if it is set, the heap info object -+ * should be interpreted as size. -+ * @used_pages: The current estimate of the number of pages used, which in -+ * normal use is either: -+ * - the initial estimate == va_pages -+ * - the actual pages used, as found by a JIT usage report -+ * Note that since the value is calculated from GPU memory after a -+ * JIT usage report, at any point in time it is allowed to take a -+ * random value that is no greater than va_pages (e.g. it may be -+ * greater than gpu_alloc->nents) - */ - struct kbase_va_region { - struct rb_node rblink; -@@ -308,8 +357,13 @@ struct kbase_va_region { - #define KBASE_REG_SHARE_BOTH (1ul << 10) - - /* Space for 4 different zones */ --#define KBASE_REG_ZONE_MASK (3ul << 11) --#define KBASE_REG_ZONE(x) (((x) & 3) << 11) -+#define KBASE_REG_ZONE_MASK ((KBASE_REG_ZONE_MAX - 1ul) << 11) -+#define KBASE_REG_ZONE(x) (((x) & (KBASE_REG_ZONE_MAX - 1ul)) << 11) -+#define KBASE_REG_ZONE_IDX(x) (((x) & KBASE_REG_ZONE_MASK) >> 11) -+ -+#if ((KBASE_REG_ZONE_MAX - 1) & 0x3) != (KBASE_REG_ZONE_MAX - 1) -+#error KBASE_REG_ZONE_MAX too large for allocation of KBASE_REG_<...> bits -+#endif - - /* GPU read access */ - #define KBASE_REG_GPU_RD (1ul<<13) -@@ -328,14 +382,29 @@ struct kbase_va_region { - /* Imported buffer is padded? */ - #define KBASE_REG_IMPORT_PAD (1ul << 21) - -+#if MALI_USE_CSF -+/* CSF event memory */ -+#define KBASE_REG_CSF_EVENT (1ul << 22) -+#else - /* Bit 22 is reserved. - * -- * Do not remove, use the next unreserved bit for new flags */ -+ * Do not remove, use the next unreserved bit for new flags -+ */ - #define KBASE_REG_RESERVED_BIT_22 (1ul << 22) -+#endif - --/* The top of the initial commit is aligned to extent pages. -- * Extent must be a power of 2 */ -+#if !MALI_USE_CSF -+/* The top of the initial commit is aligned to extension pages. -+ * Extent must be a power of 2 -+ */ - #define KBASE_REG_TILER_ALIGN_TOP (1ul << 23) -+#else -+/* Bit 23 is reserved. -+ * -+ * Do not remove, use the next unreserved bit for new flags -+ */ -+#define KBASE_REG_RESERVED_BIT_23 (1ul << 23) -+#endif /* !MALI_USE_CSF */ - - /* Whilst this flag is set the GPU allocation is not supposed to be freed by - * user space. The flag will remain set for the lifetime of JIT allocations. -@@ -367,6 +436,9 @@ struct kbase_va_region { - */ - #define KBASE_REG_HEAP_INFO_IS_SIZE (1ul << 27) - -+/* Allocation is actively used for JIT memory */ -+#define KBASE_REG_ACTIVE_JIT_ALLOC (1ul << 28) -+ - #define KBASE_REG_ZONE_SAME_VA KBASE_REG_ZONE(0) - - /* only used with 32-bit clients */ -@@ -390,15 +462,21 @@ struct kbase_va_region { - #define KBASE_REG_ZONE_EXEC_VA KBASE_REG_ZONE(2) - #define KBASE_REG_ZONE_EXEC_VA_MAX_PAGES ((1ULL << 32) >> PAGE_SHIFT) /* 4 GB */ - -+#if MALI_USE_CSF -+#define KBASE_REG_ZONE_MCU_SHARED KBASE_REG_ZONE(3) -+#define KBASE_REG_ZONE_MCU_SHARED_BASE (0x04000000ULL >> PAGE_SHIFT) -+#define KBASE_REG_ZONE_MCU_SHARED_SIZE (((0x08000000ULL) >> PAGE_SHIFT) - \ -+ KBASE_REG_ZONE_MCU_SHARED_BASE) -+#endif - - unsigned long flags; -- size_t extent; -+ size_t extension; - struct kbase_mem_phy_alloc *cpu_alloc; - struct kbase_mem_phy_alloc *gpu_alloc; - struct list_head jit_node; - u16 jit_usage_id; - u8 jit_bin_id; --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /* Pointer to an object in GPU memory defining an end of an allocated - * region - * -@@ -423,7 +501,7 @@ struct kbase_va_region { - * gpu_alloc->nents) - */ - size_t used_pages; --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - int va_refcnt; - }; -@@ -471,7 +549,7 @@ static inline struct kbase_va_region *kbase_va_region_alloc_get( - WARN_ON(!region->va_refcnt); - - /* non-atomic as kctx->reg_lock is held */ -- dev_dbg(kctx->kbdev->dev, "va_refcnt %d before get %p\n", -+ dev_dbg(kctx->kbdev->dev, "va_refcnt %d before get %pK\n", - region->va_refcnt, (void *)region); - region->va_refcnt++; - -@@ -488,7 +566,7 @@ static inline struct kbase_va_region *kbase_va_region_alloc_put( - - /* non-atomic as kctx->reg_lock is held */ - region->va_refcnt--; -- dev_dbg(kctx->kbdev->dev, "va_refcnt %d after put %p\n", -+ dev_dbg(kctx->kbdev->dev, "va_refcnt %d after put %pK\n", - region->va_refcnt, (void *)region); - if (!region->va_refcnt) - kbase_region_refcnt_free(region); -@@ -580,6 +658,7 @@ static inline struct kbase_mem_phy_alloc *kbase_alloc_create( - - kref_init(&alloc->kref); - atomic_set(&alloc->gpu_mappings, 0); -+ atomic_set(&alloc->kernel_mappings, 0); - alloc->nents = 0; - alloc->pages = (void *)(alloc + 1); - INIT_LIST_HEAD(&alloc->mappings); -@@ -1019,7 +1098,9 @@ struct kbase_va_region *kbase_find_region_enclosing_address( - struct rb_root *rbtree, u64 gpu_addr); - - /** -- * @brief Check that a pointer is actually a valid region. -+ * Check that a pointer is actually a valid region. -+ * @kctx: kbase context containing the region -+ * @gpu_addr: pointer to check - * - * Must be called with context lock held. - */ -@@ -1048,7 +1129,7 @@ bool kbase_check_import_flags(unsigned long flags); - * @flags: The flags passed from user space - * @va_pages: The size of the requested region, in pages. - * @commit_pages: Number of pages to commit initially. -- * @extent: Number of pages to grow by on GPU page fault and/or alignment -+ * @extension: Number of pages to grow by on GPU page fault and/or alignment - * (depending on flags) - * - * Makes checks on the size parameters passed in from user space for a memory -@@ -1057,7 +1138,7 @@ bool kbase_check_import_flags(unsigned long flags); - * Return: 0 if sizes are valid for these flags, negative error code otherwise - */ - int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, -- u64 va_pages, u64 commit_pages, u64 extent); -+ u64 va_pages, u64 commit_pages, u64 extension); - - /** - * kbase_update_region_flags - Convert user space flags to kernel region flags -@@ -1080,14 +1161,21 @@ void kbase_gpu_vm_unlock(struct kbase_context *kctx); - int kbase_alloc_phy_pages(struct kbase_va_region *reg, size_t vsize, size_t size); - - /** -- * @brief Register region and map it on the GPU. -+ * Register region and map it on the GPU. -+ * @kctx: kbase context containing the region -+ * @reg: the region to add -+ * @addr: the address to insert the region at -+ * @nr_pages: the number of pages in the region -+ * @align: the minimum alignment in pages - * - * Call kbase_add_va_region() and map the region on the GPU. - */ - int kbase_gpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, u64 addr, size_t nr_pages, size_t align); - - /** -- * @brief Remove the region from the GPU and unregister it. -+ * Remove the region from the GPU and unregister it. -+ * @kctx: KBase context -+ * @reg: The region to remove - * - * Must be called with context lock held. - */ -@@ -1136,20 +1224,23 @@ void kbase_mmu_disable_as(struct kbase_device *kbdev, int as_nr); - - void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat); - --/** Dump the MMU tables to a buffer -+/** -+ * kbase_mmu_dump() - Dump the MMU tables to a buffer. - * -- * This function allocates a buffer (of @c nr_pages pages) to hold a dump of the MMU tables and fills it. If the -- * buffer is too small then the return value will be NULL. -+ * This function allocates a buffer (of @c nr_pages pages) to hold a dump -+ * of the MMU tables and fills it. If the buffer is too small -+ * then the return value will be NULL. - * - * The GPU vm lock must be held when calling this function. - * -- * The buffer returned should be freed with @ref vfree when it is no longer required. -+ * The buffer returned should be freed with @ref vfree when it is no longer -+ * required. - * -- * @param[in] kctx The kbase context to dump -- * @param[in] nr_pages The number of pages to allocate for the buffer. -+ * @kctx: The kbase context to dump -+ * @nr_pages: The number of pages to allocate for the buffer. - * -- * @return The address of the buffer containing the MMU dump or NULL on error (including if the @c nr_pages is too -- * small) -+ * Return: The address of the buffer containing the MMU dump or NULL on error -+ * (including if the @c nr_pages is too small) - */ - void *kbase_mmu_dump(struct kbase_context *kctx, int nr_pages); - -@@ -1174,25 +1265,27 @@ void kbase_os_mem_map_lock(struct kbase_context *kctx); - void kbase_os_mem_map_unlock(struct kbase_context *kctx); - - /** -- * @brief Update the memory allocation counters for the current process -+ * kbasep_os_process_page_usage_update() - Update the memory allocation -+ * counters for the current process. - * -- * OS specific call to updates the current memory allocation counters for the current process with -- * the supplied delta. -+ * OS specific call to updates the current memory allocation counters -+ * for the current process with the supplied delta. - * -- * @param[in] kctx The kbase context -- * @param[in] pages The desired delta to apply to the memory usage counters. -+ * @kctx: The kbase context -+ * @pages: The desired delta to apply to the memory usage counters. - */ - - void kbasep_os_process_page_usage_update(struct kbase_context *kctx, int pages); - - /** -- * @brief Add to the memory allocation counters for the current process -+ * kbase_process_page_usage_inc() - Add to the memory allocation counters for -+ * the current process - * -- * OS specific call to add to the current memory allocation counters for the current process by -- * the supplied amount. -+ * OS specific call to add to the current memory allocation counters for -+ * the current process by the supplied amount. - * -- * @param[in] kctx The kernel base context used for the allocation. -- * @param[in] pages The desired delta to apply to the memory usage counters. -+ * @kctx: The kernel base context used for the allocation. -+ * @pages: The desired delta to apply to the memory usage counters. - */ - - static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int pages) -@@ -1201,13 +1294,14 @@ static inline void kbase_process_page_usage_inc(struct kbase_context *kctx, int - } - - /** -- * @brief Subtract from the memory allocation counters for the current process -+ * kbase_process_page_usage_dec() - Subtract from the memory allocation -+ * counters for the current process. - * -- * OS specific call to subtract from the current memory allocation counters for the current process by -- * the supplied amount. -+ * OS specific call to subtract from the current memory allocation counters -+ * for the current process by the supplied amount. - * -- * @param[in] kctx The kernel base context used for the allocation. -- * @param[in] pages The desired delta to apply to the memory usage counters. -+ * @kctx: The kernel base context used for the allocation. -+ * @pages: The desired delta to apply to the memory usage counters. - */ - - static inline void kbase_process_page_usage_dec(struct kbase_context *kctx, int pages) -@@ -1332,15 +1426,15 @@ struct tagged_addr *kbase_alloc_phy_pages_helper_locked( - struct kbase_sub_alloc **prealloc_sa); - - /** --* @brief Free physical pages. --* --* Frees \a nr_pages and updates the alloc object. --* --* @param[in] alloc allocation object to free pages from --* @param[in] nr_pages_to_free number of physical pages to free --* --* Return: 0 on success, otherwise a negative error code --*/ -+ * kbase_free_phy_pages_helper() - Free physical pages. -+ * -+ * Frees \a nr_pages and updates the alloc object. -+ * -+ * @alloc: allocation object to free pages from -+ * @nr_pages_to_free: number of physical pages to free -+ * -+ * Return: 0 on success, otherwise a negative error code -+ */ - int kbase_free_phy_pages_helper(struct kbase_mem_phy_alloc *alloc, size_t nr_pages_to_free); - - /** -@@ -1370,7 +1464,8 @@ static inline void kbase_set_dma_addr(struct page *p, dma_addr_t dma_addr) - /* on 32-bit ARM with LPAE dma_addr_t becomes larger, but the - * private field stays the same. So we have to be clever and - * use the fact that we only store DMA addresses of whole pages, -- * so the low bits should be zero */ -+ * so the low bits should be zero -+ */ - KBASE_DEBUG_ASSERT(!(dma_addr & (PAGE_SIZE - 1))); - set_page_private(p, dma_addr >> PAGE_SHIFT); - } else { -@@ -1392,26 +1487,11 @@ static inline void kbase_clear_dma_addr(struct page *p) - } - - /** -- * @brief Process a page fault. -- * -- * @param[in] data work_struct passed by queue_work() -- */ --void page_fault_worker(struct work_struct *data); -- --/** -- * @brief Process a bus fault. -- * -- * @param[in] data work_struct passed by queue_work() -- */ --void bus_fault_worker(struct work_struct *data); -- --/** -- * @brief Flush MMU workqueues. -+ * kbase_flush_mmu_wqs() - Flush MMU workqueues. -+ * @kbdev: Device pointer. - * - * This function will cause any outstanding page or bus faults to be processed. - * It should be called prior to powering off the GPU. -- * -- * @param[in] kbdev Device pointer - */ - void kbase_flush_mmu_wqs(struct kbase_device *kbdev); - -@@ -1437,7 +1517,7 @@ void kbase_sync_single_for_device(struct kbase_device *kbdev, dma_addr_t handle, - void kbase_sync_single_for_cpu(struct kbase_device *kbdev, dma_addr_t handle, - size_t size, enum dma_data_direction dir); - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - /** - * kbase_jit_debugfs_init - Add per context debugfs entry for JIT. - * @kctx: kbase context -@@ -1497,7 +1577,7 @@ bool kbase_jit_evict(struct kbase_context *kctx); - */ - void kbase_jit_term(struct kbase_context *kctx); - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /** - * kbase_trace_jit_report_gpu_mem_trace_enabled - variant of - * kbase_trace_jit_report_gpu_mem() that should only be called once the -@@ -1508,7 +1588,7 @@ void kbase_jit_term(struct kbase_context *kctx); - */ - void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, - struct kbase_va_region *reg, unsigned int flags); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /** - * kbase_trace_jit_report_gpu_mem - Trace information about the GPU memory used -@@ -1530,7 +1610,7 @@ void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, - * been included. Also gives no opportunity for the compiler to mess up - * inlining it. - */ --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - #define kbase_trace_jit_report_gpu_mem(kctx, reg, flags) \ - do { \ - if (trace_mali_jit_report_gpu_mem_enabled()) \ -@@ -1540,9 +1620,9 @@ void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, - #else - #define kbase_trace_jit_report_gpu_mem(kctx, reg, flags) \ - CSTD_NOP(kctx, reg, flags) --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /** - * kbase_jit_report_update_pressure - safely update the JIT physical page - * pressure and JIT region's estimate of used_pages -@@ -1562,7 +1642,127 @@ void kbase_trace_jit_report_gpu_mem_trace_enabled(struct kbase_context *kctx, - void kbase_jit_report_update_pressure(struct kbase_context *kctx, - struct kbase_va_region *reg, u64 new_used_pages, - unsigned int flags); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+ -+/** -+ * jit_trim_necessary_pages() - calculate and trim the least pages possible to -+ * satisfy a new JIT allocation -+ * -+ * @kctx: Pointer to the kbase context -+ * @needed_pages: Number of JIT physical pages by which trimming is requested. -+ * The actual number of pages trimmed could differ. -+ * -+ * Before allocating a new just-in-time memory region or reusing a previous -+ * one, ensure that the total JIT physical page usage also will not exceed the -+ * pressure limit. -+ * -+ * If there are no reported-on allocations, then we already guarantee this will -+ * be the case - because our current pressure then only comes from the va_pages -+ * of each JIT region, hence JIT physical page usage is guaranteed to be -+ * bounded by this. -+ * -+ * However as soon as JIT allocations become "reported on", the pressure is -+ * lowered to allow new JIT regions to be allocated. It is after such a point -+ * that the total JIT physical page usage could (either now or in the future on -+ * a grow-on-GPU-page-fault) exceed the pressure limit, but only on newly -+ * allocated JIT regions. Hence, trim any "reported on" regions. -+ * -+ * Any pages freed will go into the pool and be allocated from there in -+ * kbase_mem_alloc(). -+ */ -+void kbase_jit_trim_necessary_pages(struct kbase_context *kctx, -+ size_t needed_pages); -+ -+/* -+ * Same as kbase_jit_request_phys_increase(), except that Caller is supposed -+ * to take jit_evict_lock also on @kctx before calling this function. -+ */ -+static inline void -+kbase_jit_request_phys_increase_locked(struct kbase_context *kctx, -+ size_t needed_pages) -+{ -+#if !MALI_USE_CSF -+ lockdep_assert_held(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ -+ lockdep_assert_held(&kctx->reg_lock); -+ lockdep_assert_held(&kctx->jit_evict_lock); -+ -+ kctx->jit_phys_pages_to_be_allocated += needed_pages; -+ -+ kbase_jit_trim_necessary_pages(kctx, -+ kctx->jit_phys_pages_to_be_allocated); -+} -+ -+/** -+ * kbase_jit_request_phys_increase() - Increment the backing pages count and do -+ * the required trimming before allocating pages for a JIT allocation. -+ * -+ * @kctx: Pointer to the kbase context -+ * @needed_pages: Number of pages to be allocated for the JIT allocation. -+ * -+ * This function needs to be called before allocating backing pages for a -+ * just-in-time memory region. The backing pages are currently allocated when, -+ * -+ * - A new JIT region is created. -+ * - An old JIT region is reused from the cached pool. -+ * - GPU page fault occurs for the active JIT region. -+ * - Backing is grown for the JIT region through the commit ioctl. -+ * -+ * This function would ensure that the total JIT physical page usage does not -+ * exceed the pressure limit even when the backing pages get allocated -+ * simultaneously for multiple JIT allocations from different threads. -+ * -+ * There should be a matching call to kbase_jit_done_phys_increase(), after -+ * the pages have been allocated and accounted against the active JIT -+ * allocation. -+ * -+ * Caller is supposed to take reg_lock on @kctx before calling this function. -+ */ -+static inline void kbase_jit_request_phys_increase(struct kbase_context *kctx, -+ size_t needed_pages) -+{ -+#if !MALI_USE_CSF -+ lockdep_assert_held(&kctx->jctx.lock); -+#endif /* !MALI_USE_CSF */ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ mutex_lock(&kctx->jit_evict_lock); -+ kbase_jit_request_phys_increase_locked(kctx, needed_pages); -+ mutex_unlock(&kctx->jit_evict_lock); -+} -+ -+/** -+ * kbase_jit_done_phys_increase() - Decrement the backing pages count after the -+ * allocation of pages for a JIT allocation. -+ * -+ * @kctx: Pointer to the kbase context -+ * @needed_pages: Number of pages that were allocated for the JIT allocation. -+ * -+ * This function should be called after backing pages have been allocated and -+ * accounted against the active JIT allocation. -+ * The call should be made when the following have been satisfied: -+ * when the allocation is on the jit_active_head. -+ * when additional needed_pages have been allocated. -+ * kctx->reg_lock was held during the above and has not yet been unlocked. -+ * Failure to call this function before unlocking the kctx->reg_lock when -+ * either the above have changed may result in over-accounting the memory. -+ * This ensures kbase_jit_trim_necessary_pages() gets a consistent count of -+ * the memory. -+ * -+ * A matching call to kbase_jit_request_phys_increase() should have been made, -+ * before the allocation of backing pages. -+ * -+ * Caller is supposed to take reg_lock on @kctx before calling this function. -+ */ -+static inline void kbase_jit_done_phys_increase(struct kbase_context *kctx, -+ size_t needed_pages) -+{ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ WARN_ON(kctx->jit_phys_pages_to_be_allocated < needed_pages); -+ -+ kctx->jit_phys_pages_to_be_allocated -= needed_pages; -+} -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - /** - * kbase_has_exec_va_zone - EXEC_VA zone predicate -@@ -1693,6 +1893,63 @@ static inline void kbase_mem_pool_unlock(struct kbase_mem_pool *pool) - */ - void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc); - -+#if MALI_USE_CSF -+/** -+ * kbase_link_event_mem_page - Add the new event memory region to the per -+ * context list of event pages. -+ * @kctx: Pointer to kbase context -+ * @reg: Pointer to the region allocated for event memory. -+ * -+ * The region being linked shouldn't have been marked as free and should -+ * have KBASE_REG_CSF_EVENT flag set for it. -+ */ -+static inline void kbase_link_event_mem_page(struct kbase_context *kctx, -+ struct kbase_va_region *reg) -+{ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ WARN_ON(reg->flags & KBASE_REG_FREE); -+ WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT)); -+ -+ list_add(®->link, &kctx->csf.event_pages_head); -+} -+ -+/** -+ * kbase_unlink_event_mem_page - Remove the event memory region from the per -+ * context list of event pages. -+ * @kctx: Pointer to kbase context -+ * @reg: Pointer to the region allocated for event memory. -+ * -+ * The region being un-linked shouldn't have been marked as free and should -+ * have KBASE_REG_CSF_EVENT flag set for it. -+ */ -+static inline void kbase_unlink_event_mem_page(struct kbase_context *kctx, -+ struct kbase_va_region *reg) -+{ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ WARN_ON(reg->flags & KBASE_REG_FREE); -+ WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT)); -+ -+ list_del(®->link); -+} -+ -+/** -+ * kbase_mcu_shared_interface_region_tracker_init - Initialize the rb tree to -+ * manage the shared interface segment of MCU firmware address space. -+ * @kbdev: Pointer to the kbase device -+ * -+ * Returns zero on success or negative error number on failure. -+ */ -+int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev); -+ -+/** -+ * kbase_mcu_shared_interface_region_tracker_term - Teardown the rb tree -+ * managing the shared interface segment of MCU firmware address space. -+ * @kbdev: Pointer to the kbase device -+ */ -+void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev); -+#endif - - /** - * kbase_mem_umm_map - Map dma-buf -@@ -1742,7 +1999,6 @@ void kbase_mem_umm_unmap(struct kbase_context *kctx, - int kbase_mem_do_sync_imported(struct kbase_context *kctx, - struct kbase_va_region *reg, enum kbase_sync_type sync_fn); - -- - /** - * kbase_mem_copy_to_pinned_user_pages - Memcpy from source input page to - * an unaligned address at a given offset from the start of a target page. -@@ -1768,4 +2024,76 @@ int kbase_mem_copy_to_pinned_user_pages(struct page **dest_pages, - void *src_page, size_t *to_copy, unsigned int nr_pages, - unsigned int *target_page_nr, size_t offset); - -+/** -+ * kbase_ctx_reg_zone_end_pfn - return the end Page Frame Number of @zone -+ * @zone: zone to query -+ * -+ * Return: The end of the zone corresponding to @zone -+ */ -+static inline u64 kbase_reg_zone_end_pfn(struct kbase_reg_zone *zone) -+{ -+ return zone->base_pfn + zone->va_size_pages; -+} -+ -+/** -+ * kbase_ctx_reg_zone_init - initialize a zone in @kctx -+ * @kctx: Pointer to kbase context -+ * @zone_bits: A KBASE_REG_ZONE_<...> to initialize -+ * @base_pfn: Page Frame Number in GPU virtual address space for the start of -+ * the Zone -+ * @va_size_pages: Size of the Zone in pages -+ */ -+static inline void kbase_ctx_reg_zone_init(struct kbase_context *kctx, -+ unsigned long zone_bits, -+ u64 base_pfn, u64 va_size_pages) -+{ -+ struct kbase_reg_zone *zone; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ WARN_ON((zone_bits & KBASE_REG_ZONE_MASK) != zone_bits); -+ -+ zone = &kctx->reg_zone[KBASE_REG_ZONE_IDX(zone_bits)]; -+ *zone = (struct kbase_reg_zone){ -+ .base_pfn = base_pfn, .va_size_pages = va_size_pages, -+ }; -+} -+ -+/** -+ * kbase_ctx_reg_zone_get_nolock - get a zone from @kctx where the caller does -+ * not have @kctx 's region lock -+ * @kctx: Pointer to kbase context -+ * @zone_bits: A KBASE_REG_ZONE_<...> to retrieve -+ * -+ * This should only be used in performance-critical paths where the code is -+ * resilient to a race with the zone changing. -+ * -+ * Return: The zone corresponding to @zone_bits -+ */ -+static inline struct kbase_reg_zone * -+kbase_ctx_reg_zone_get_nolock(struct kbase_context *kctx, -+ unsigned long zone_bits) -+{ -+ WARN_ON((zone_bits & KBASE_REG_ZONE_MASK) != zone_bits); -+ -+ return &kctx->reg_zone[KBASE_REG_ZONE_IDX(zone_bits)]; -+} -+ -+/** -+ * kbase_ctx_reg_zone_get - get a zone from @kctx -+ * @kctx: Pointer to kbase context -+ * @zone_bits: A KBASE_REG_ZONE_<...> to retrieve -+ * -+ * The get is not refcounted - there is no corresponding 'put' operation -+ * -+ * Return: The zone corresponding to @zone_bits -+ */ -+static inline struct kbase_reg_zone * -+kbase_ctx_reg_zone_get(struct kbase_context *kctx, unsigned long zone_bits) -+{ -+ lockdep_assert_held(&kctx->reg_lock); -+ WARN_ON((zone_bits & KBASE_REG_ZONE_MASK) != zone_bits); -+ -+ return &kctx->reg_zone[KBASE_REG_ZONE_IDX(zone_bits)]; -+} -+ - #endif /* _KBASE_MEM_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c -index b669f2a..21302c1 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_mem_linux.c -- * Base kernel memory APIs, Linux implementation. -+ * DOC: Base kernel memory APIs, Linux implementation. - */ - - #include <linux/compat.h> -@@ -35,10 +31,9 @@ - #include <linux/fs.h> - #include <linux/version.h> - #include <linux/dma-mapping.h> --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 5, 0)) && \ -- (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0)) -+#if (KERNEL_VERSION(4, 8, 0) > LINUX_VERSION_CODE) - #include <linux/dma-attrs.h> --#endif /* LINUX_VERSION_CODE >= 3.5.0 && < 4.8.0 */ -+#endif /* LINUX_VERSION_CODE < 4.8.0 */ - #include <linux/dma-buf.h> - #include <linux/shrinker.h> - #include <linux/cache.h> -@@ -47,8 +42,11 @@ - #include <mali_kbase.h> - #include <mali_kbase_mem_linux.h> - #include <tl/mali_kbase_tracepoints.h> --#include <mali_kbase_ioctl.h> -+#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h> - #include <mmu/mali_kbase_mmu.h> -+#include <mali_kbase_caps.h> -+#include <mali_kbase_trace_gpu_mem.h> -+#include <mali_kbase_reset_gpu.h> - - #if ((KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) || \ - (KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE)) -@@ -87,6 +85,12 @@ - - #define IR_THRESHOLD_STEPS (256u) - -+#if MALI_USE_CSF -+static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, -+ struct vm_area_struct *vma); -+static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, -+ struct vm_area_struct *vma); -+#endif - - static int kbase_vmap_phy_pages(struct kbase_context *kctx, - struct kbase_va_region *reg, u64 offset_bytes, size_t size, -@@ -107,6 +111,25 @@ static int kbase_mem_shrink_gpu_mapping(struct kbase_context *kctx, - static struct kbase_va_region *kbase_find_event_mem_region( - struct kbase_context *kctx, u64 gpu_addr) - { -+#if MALI_USE_CSF -+ u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; -+ struct kbase_va_region *reg; -+ -+ lockdep_assert_held(&kctx->reg_lock); -+ -+ list_for_each_entry(reg, &kctx->csf.event_pages_head, link) { -+ if ((reg->start_pfn <= gpu_pfn) && -+ (gpu_pfn < (reg->start_pfn + reg->nr_pages))) { -+ if (WARN_ON(reg->flags & KBASE_REG_FREE)) -+ return NULL; -+ -+ if (WARN_ON(!(reg->flags & KBASE_REG_CSF_EVENT))) -+ return NULL; -+ -+ return reg; -+ } -+ } -+#endif - - return NULL; - } -@@ -269,8 +292,8 @@ void kbase_phy_alloc_mapping_put(struct kbase_context *kctx, - } - - struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, -- u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, -- u64 *gpu_va) -+ u64 va_pages, u64 commit_pages, -+ u64 extension, u64 *flags, u64 *gpu_va) - { - int zone; - struct kbase_va_region *reg; -@@ -282,15 +305,20 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, - KBASE_DEBUG_ASSERT(gpu_va); - - dev = kctx->kbdev->dev; -- dev_dbg(dev, "Allocating %lld va_pages, %lld commit_pages, %lld extent, 0x%llX flags\n", -- va_pages, commit_pages, extent, *flags); -+ dev_dbg(dev, -+ "Allocating %lld va_pages, %lld commit_pages, %lld extension, 0x%llX flags\n", -+ va_pages, commit_pages, extension, *flags); - -+#if MALI_USE_CSF -+ *gpu_va = 0; /* return 0 on failure */ -+#else - if (!(*flags & BASE_MEM_FLAG_MAP_FIXED)) - *gpu_va = 0; /* return 0 on failure */ - else - dev_err(dev, - "Keeping requested GPU VA of 0x%llx\n", - (unsigned long long)*gpu_va); -+#endif - - if (!kbase_check_alloc_flags(*flags)) { - dev_warn(dev, -@@ -299,7 +327,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, - goto bad_flags; - } - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - if (unlikely(kbase_ctx_flag(kctx, KCTX_INFINITE_CACHE))) { - /* Mask coherency flags if infinite cache is enabled to prevent - * the skipping of syncs from BASE side. -@@ -325,7 +353,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, - *flags &= ~BASE_MEM_COHERENT_SYSTEM; - } - -- if (kbase_check_alloc_sizes(kctx, *flags, va_pages, commit_pages, extent)) -+ if (kbase_check_alloc_sizes(kctx, *flags, va_pages, commit_pages, -+ extension)) - goto bad_sizes; - - #ifdef CONFIG_MALI_MEMORY_FULLY_BACKED -@@ -363,6 +392,15 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, - goto prepare_failed; - } - -+ if (unlikely(reg->cpu_alloc != reg->gpu_alloc)) -+ *flags |= BASE_MEM_KERNEL_SYNC; -+ -+ /* make sure base knows if the memory is actually cached or not */ -+ if (reg->flags & KBASE_REG_CPU_CACHED) -+ *flags |= BASE_MEM_CACHED_CPU; -+ else -+ *flags &= ~BASE_MEM_CACHED_CPU; -+ - if (*flags & BASE_MEM_GROW_ON_GPF) { - unsigned int const ir_threshold = atomic_read( - &kctx->kbdev->memdev.ir_threshold); -@@ -372,12 +410,17 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, - } else - reg->threshold_pages = 0; - -- if (*flags & (BASE_MEM_GROW_ON_GPF|BASE_MEM_TILER_ALIGN_TOP)) { -- /* kbase_check_alloc_sizes() already checks extent is valid for -- * assigning to reg->extent */ -- reg->extent = extent; -+ if (*flags & BASE_MEM_GROW_ON_GPF) { -+ /* kbase_check_alloc_sizes() already checks extension is valid for -+ * assigning to reg->extension -+ */ -+ reg->extension = extension; -+#if !MALI_USE_CSF -+ } else if (*flags & BASE_MEM_TILER_ALIGN_TOP) { -+ reg->extension = extension; -+#endif /* !MALI_USE_CSF */ - } else { -- reg->extent = 0; -+ reg->extension = 0; - } - - if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) { -@@ -404,7 +447,6 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, - } - } - -- - /* mmap needed to setup VA? */ - if (*flags & BASE_MEM_SAME_VA) { - unsigned long cookie, cookie_nr; -@@ -436,6 +478,17 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, - *gpu_va = reg->start_pfn << PAGE_SHIFT; - } - -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (*flags & BASEP_MEM_PERFORM_JIT_TRIM) { -+ kbase_jit_done_phys_increase(kctx, commit_pages); -+ -+ mutex_lock(&kctx->jit_evict_lock); -+ WARN_ON(!list_empty(®->jit_node)); -+ list_add(®->jit_node, &kctx->jit_active_head); -+ mutex_unlock(&kctx->jit_evict_lock); -+ } -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -+ - kbase_gpu_vm_unlock(kctx); - return reg; - -@@ -443,6 +496,13 @@ no_mmap: - no_cookie: - no_kern_mapping: - no_mem: -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (*flags & BASEP_MEM_PERFORM_JIT_TRIM) { -+ kbase_gpu_vm_lock(kctx); -+ kbase_jit_done_phys_increase(kctx, commit_pages); -+ kbase_gpu_vm_unlock(kctx); -+ } -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - kbase_mem_phy_alloc_put(reg->cpu_alloc); - kbase_mem_phy_alloc_put(reg->gpu_alloc); - invalid_flags: -@@ -511,21 +571,36 @@ int kbase_mem_query(struct kbase_context *kctx, - *out |= BASE_MEM_COHERENT_SYSTEM; - if (KBASE_REG_SHARE_IN & reg->flags) - *out |= BASE_MEM_COHERENT_LOCAL; -- if (kctx->api_version >= KBASE_API_VERSION(11, 2)) { -- /* Prior to 11.2, these were known about by user-side -- * but we did not return them. Returning some of these -- * caused certain clients that were not expecting them -- * to fail, so we omit all of them as a special-case -- * for compatibility reasons */ -+ if (mali_kbase_supports_mem_grow_on_gpf(kctx->api_version)) { -+ /* Prior to this version, this was known about by -+ * user-side but we did not return them. Returning -+ * it caused certain clients that were not expecting -+ * it to fail, so we omit it as a special-case for -+ * compatibility reasons -+ */ - if (KBASE_REG_PF_GROW & reg->flags) - *out |= BASE_MEM_GROW_ON_GPF; -+ } -+ if (mali_kbase_supports_mem_protected(kctx->api_version)) { -+ /* Prior to this version, this was known about by -+ * user-side but we did not return them. Returning -+ * it caused certain clients that were not expecting -+ * it to fail, so we omit it as a special-case for -+ * compatibility reasons -+ */ - if (KBASE_REG_PROTECTED & reg->flags) - *out |= BASE_MEM_PROTECTED; - } -+#if !MALI_USE_CSF - if (KBASE_REG_TILER_ALIGN_TOP & reg->flags) - *out |= BASE_MEM_TILER_ALIGN_TOP; -+#endif /* !MALI_USE_CSF */ - if (!(KBASE_REG_GPU_CACHED & reg->flags)) - *out |= BASE_MEM_UNCACHED_GPU; -+#if MALI_USE_CSF -+ if (KBASE_REG_CSF_EVENT & reg->flags) -+ *out |= BASE_MEM_CSF_EVENT; -+#endif - if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags) - *out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE; - -@@ -561,18 +636,17 @@ unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, - struct shrink_control *sc) - { - struct kbase_context *kctx; -- struct kbase_mem_phy_alloc *alloc; -- unsigned long pages = 0; - - kctx = container_of(s, struct kbase_context, reclaim); - -- mutex_lock(&kctx->jit_evict_lock); -- -- list_for_each_entry(alloc, &kctx->evict_list, evict_node) -- pages += alloc->nents; -+ WARN((sc->gfp_mask & __GFP_ATOMIC), -+ "Shrinkers cannot be called for GFP_ATOMIC allocations. Check kernel mm for problems. gfp_mask==%x\n", -+ sc->gfp_mask); -+ WARN(in_atomic(), -+ "Shrinker called whilst in atomic context. The caller must switch to using GFP_ATOMIC or similar. gfp_mask==%x\n", -+ sc->gfp_mask); - -- mutex_unlock(&kctx->jit_evict_lock); -- return pages; -+ return atomic_read(&kctx->evict_nents); - } - - /** -@@ -604,6 +678,7 @@ unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, - unsigned long freed = 0; - - kctx = container_of(s, struct kbase_context, reclaim); -+ - mutex_lock(&kctx->jit_evict_lock); - - list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) { -@@ -630,6 +705,7 @@ unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, - - kbase_free_phy_pages_helper(alloc, alloc->evicted); - freed += alloc->evicted; -+ WARN_ON(atomic_sub_return(alloc->evicted, &kctx->evict_nents) < 0); - list_del_init(&alloc->evict_node); - - /* -@@ -648,35 +724,20 @@ out_unlock: - return freed; - } - --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) --static int kbase_mem_evictable_reclaim_shrink(struct shrinker *s, -- struct shrink_control *sc) --{ -- if (sc->nr_to_scan == 0) -- return kbase_mem_evictable_reclaim_count_objects(s, sc); -- -- return kbase_mem_evictable_reclaim_scan_objects(s, sc); --} --#endif -- - int kbase_mem_evictable_init(struct kbase_context *kctx) - { - INIT_LIST_HEAD(&kctx->evict_list); - mutex_init(&kctx->jit_evict_lock); - -- /* Register shrinker */ --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -- kctx->reclaim.shrink = kbase_mem_evictable_reclaim_shrink; --#else -+ atomic_set(&kctx->evict_nents, 0); -+ - kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; - kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; --#endif - kctx->reclaim.seeks = DEFAULT_SEEKS; - /* Kernel versions prior to 3.1 : -- * struct shrinker does not define batch */ --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) -+ * struct shrinker does not define batch -+ */ - kctx->reclaim.batch = 0; --#endif - register_shrinker(&kctx->reclaim); - return 0; - } -@@ -705,6 +766,7 @@ void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) - kbdev, - kctx->id, - (u64)new_page_count); -+ kbase_trace_gpu_mem_usage_dec(kbdev, kctx, alloc->nents); - } - - /** -@@ -731,6 +793,7 @@ void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) - kbdev, - kctx->id, - (u64)new_page_count); -+ kbase_trace_gpu_mem_usage_inc(kbdev, kctx, alloc->nents); - } - - int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) -@@ -751,6 +814,7 @@ int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) - * can reclaim it. - */ - list_add(&gpu_alloc->evict_node, &kctx->evict_list); -+ atomic_add(gpu_alloc->nents, &kctx->evict_nents); - mutex_unlock(&kctx->jit_evict_lock); - kbase_mem_evictable_mark_reclaim(gpu_alloc); - -@@ -770,6 +834,7 @@ bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) - * First remove the allocation from the eviction list as it's no - * longer eligible for eviction. - */ -+ WARN_ON(atomic_sub_return(gpu_alloc->nents, &kctx->evict_nents) < 0); - list_del_init(&gpu_alloc->evict_node); - mutex_unlock(&kctx->jit_evict_lock); - -@@ -840,7 +905,7 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in - real_flags |= KBASE_REG_SHARE_IN; - - /* now we can lock down the context, and find the region */ -- down_write(¤t->mm->mmap_sem); -+ down_write(kbase_mem_get_process_mmap_lock()); - kbase_gpu_vm_lock(kctx); - - /* Validate the region */ -@@ -852,10 +917,18 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in - prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; - new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; - if (prev_needed != new_needed) { -- /* Aliased allocations can't be made ephemeral */ -+ /* Aliased allocations can't be shrunk as the code doesn't -+ * support looking up: -+ * - all physical pages assigned to different GPU VAs -+ * - CPU mappings for the physical pages at different vm_pgoff -+ * (==GPU VA) locations. -+ */ - if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) - goto out_unlock; - -+ if (atomic_read(®->cpu_alloc->kernel_mappings) > 0) -+ goto out_unlock; -+ - if (new_needed) { - /* Only native allocations can be marked not needed */ - if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { -@@ -938,7 +1011,7 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in - - out_unlock: - kbase_gpu_vm_unlock(kctx); -- up_write(¤t->mm->mmap_sem); -+ up_write(kbase_mem_get_process_mmap_lock()); - out: - return ret; - } -@@ -1029,7 +1102,7 @@ int kbase_mem_do_sync_imported(struct kbase_context *kctx, - dir); - #endif /* KBASE_MEM_ION_SYNC_WORKAROUND */ - break; -- }; -+ } - - if (unlikely(ret)) - dev_warn(kctx->kbdev->dev, -@@ -1056,6 +1129,8 @@ static void kbase_mem_umm_unmap_attachment(struct kbase_context *kctx, - alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); - alloc->imported.umm.sgt = NULL; - -+ kbase_remove_dma_buf_usage(kctx, alloc); -+ - memset(pa, 0xff, sizeof(*pa) * alloc->nents); - alloc->nents = 0; - } -@@ -1123,6 +1198,7 @@ static int kbase_mem_umm_map_attachment(struct kbase_context *kctx, - - /* Update nents as we now have pages to map */ - alloc->nents = count; -+ kbase_add_dma_buf_usage(kctx, alloc); - - return 0; - -@@ -1327,7 +1403,7 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, - if (*flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) - need_sync = true; - --#ifdef CONFIG_64BIT -+#if IS_ENABLED(CONFIG_64BIT) - if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { - /* - * 64-bit tasks require us to reserve VA on the CPU that we use -@@ -1383,7 +1459,8 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, - reg->gpu_alloc->imported.umm.dma_attachment = dma_attachment; - reg->gpu_alloc->imported.umm.current_mapping_usage_count = 0; - reg->gpu_alloc->imported.umm.need_sync = need_sync; -- reg->extent = 0; -+ reg->gpu_alloc->imported.umm.kctx = kctx; -+ reg->extension = 0; - - if (!IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND)) { - int err; -@@ -1436,6 +1513,7 @@ static struct kbase_va_region *kbase_mem_from_user_buffer( - u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx->kbdev); - struct kbase_alloc_import_user_buf *user_buf; - struct page **pages = NULL; -+ int write; - - /* Flag supported only for dma-buf imported memory */ - if (*flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) -@@ -1478,7 +1556,7 @@ static struct kbase_va_region *kbase_mem_from_user_buffer( - if (*flags & BASE_MEM_IMPORT_SHARED) - shared_zone = true; - --#ifdef CONFIG_64BIT -+#if IS_ENABLED(CONFIG_64BIT) - if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { - /* - * 64-bit tasks require us to reserve VA on the CPU that we use -@@ -1547,33 +1625,33 @@ static struct kbase_va_region *kbase_mem_from_user_buffer( - *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; - } - -- down_read(¤t->mm->mmap_sem); -+ down_read(kbase_mem_get_process_mmap_lock()); -+ -+ write = reg->flags & (KBASE_REG_CPU_WR | KBASE_REG_GPU_WR); - --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) -+#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE - faulted_pages = get_user_pages(current, current->mm, address, *va_pages, - #if KERNEL_VERSION(4, 4, 168) <= LINUX_VERSION_CODE && \ - KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE -- reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, -- pages, NULL); -+ write ? FOLL_WRITE : 0, pages, NULL); - #else -- reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); -+ write, 0, pages, NULL); - #endif --#elif LINUX_VERSION_CODE < KERNEL_VERSION(4, 9, 0) -+#elif KERNEL_VERSION(4, 9, 0) > LINUX_VERSION_CODE - faulted_pages = get_user_pages(address, *va_pages, -- reg->flags & KBASE_REG_GPU_WR, 0, pages, NULL); -+ write, 0, pages, NULL); - #else - faulted_pages = get_user_pages(address, *va_pages, -- reg->flags & KBASE_REG_GPU_WR ? FOLL_WRITE : 0, -- pages, NULL); -+ write ? FOLL_WRITE : 0, pages, NULL); - #endif - -- up_read(¤t->mm->mmap_sem); -+ up_read(kbase_mem_get_process_mmap_lock()); - - if (faulted_pages != *va_pages) - goto fault_mismatch; - - reg->gpu_alloc->nents = 0; -- reg->extent = 0; -+ reg->extension = 0; - - if (pages) { - struct device *dev = kctx->kbdev->dev; -@@ -1672,10 +1750,11 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, - /* calculate the number of pages this alias will cover */ - *num_pages = nents * stride; - --#ifdef CONFIG_64BIT -+#if IS_ENABLED(CONFIG_64BIT) - if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { - /* 64-bit tasks must MMAP anyway, but not expose this address to -- * clients */ -+ * clients -+ */ - *flags |= BASE_MEM_NEED_MMAP; - reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 0, - *num_pages, -@@ -1721,7 +1800,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, - goto bad_handle; /* must be > 0 */ - if (ai[i].length > stride) - goto bad_handle; /* can't be larger than the -- stride */ -+ * stride -+ */ - reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; - } else { - struct kbase_va_region *aliasing_reg; -@@ -1736,6 +1816,15 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, - goto bad_handle; /* Not found/already free */ - if (aliasing_reg->flags & KBASE_REG_DONT_NEED) - goto bad_handle; /* Ephemeral region */ -+ if (aliasing_reg->flags & KBASE_REG_NO_USER_FREE) -+ goto bad_handle; /* JIT regions can't be -+ * aliased. NO_USER_FREE flag -+ * covers the entire lifetime -+ * of JIT regions. The other -+ * types of regions covered -+ * by this flag also shall -+ * not be aliased. -+ */ - if (!(aliasing_reg->flags & KBASE_REG_GPU_CACHED)) - goto bad_handle; /* GPU uncached memory */ - if (!aliasing_reg->gpu_alloc) -@@ -1743,16 +1832,18 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, - if (aliasing_reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) - goto bad_handle; /* Not a native alloc */ - if (coherent != ((aliasing_reg->flags & KBASE_REG_SHARE_BOTH) != 0)) -- goto bad_handle; -- /* Non-coherent memory cannot alias -- coherent memory, and vice versa.*/ -+ goto bad_handle; /* Non-coherent memory cannot -+ * alias coherent memory, and -+ * vice versa. -+ */ - - /* check size against stride */ - if (!ai[i].length) - goto bad_handle; /* must be > 0 */ - if (ai[i].length > stride) - goto bad_handle; /* can't be larger than the -- stride */ -+ * stride -+ */ - - alloc = aliasing_reg->gpu_alloc; - -@@ -1765,10 +1856,22 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, - reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); - reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; - reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; -+ -+ /* Ensure the underlying alloc is marked as being -+ * mapped at >1 different GPU VA immediately, even -+ * though mapping might not happen until later. -+ * -+ * Otherwise, we would (incorrectly) allow shrinking of -+ * the source region (aliasing_reg) and so freeing the -+ * physical pages (without freeing the entire alloc) -+ * whilst we still hold an implicit reference on those -+ * physical pages. -+ */ -+ kbase_mem_phy_alloc_gpu_mapped(alloc); - } - } - --#ifdef CONFIG_64BIT -+#if IS_ENABLED(CONFIG_64BIT) - if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { - /* Bind to a cookie */ - if (bitmap_empty(kctx->cookies, BITS_PER_LONG)) { -@@ -1803,11 +1906,15 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, - - return gpu_va; - --#ifdef CONFIG_64BIT -+#if IS_ENABLED(CONFIG_64BIT) - no_cookie: - #endif - no_mmap: - bad_handle: -+ /* Marking the source allocs as not being mapped on the GPU and putting -+ * them is handled by putting reg's allocs, so no rollback of those -+ * actions is done here. -+ */ - kbase_gpu_vm_unlock(kctx); - no_aliased_array: - invalid_flags: -@@ -1887,7 +1994,7 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, - sizeof(user_buffer))) { - reg = NULL; - } else { --#ifdef CONFIG_COMPAT -+#if IS_ENABLED(CONFIG_COMPAT) - if (kbase_ctx_flag(kctx, KCTX_COMPAT)) - uptr = compat_ptr(user_buffer.ptr); - else -@@ -2024,7 +2131,7 @@ static int kbase_mem_shrink_gpu_mapping(struct kbase_context *const kctx, - int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) - { - u64 old_pages; -- u64 delta; -+ u64 delta = 0; - int res = -EINVAL; - struct kbase_va_region *reg; - bool read_locked = false; -@@ -2037,7 +2144,7 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) - return -EINVAL; - } - -- down_write(¤t->mm->mmap_sem); -+ down_write(kbase_mem_get_process_mmap_lock()); - kbase_gpu_vm_lock(kctx); - - /* Validate the region */ -@@ -2054,13 +2161,27 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) - if (0 == (reg->flags & KBASE_REG_GROWABLE)) - goto out_unlock; - -+ if (reg->flags & KBASE_REG_ACTIVE_JIT_ALLOC) -+ goto out_unlock; -+ - /* Would overflow the VA region */ - if (new_pages > reg->nr_pages) - goto out_unlock; - -- /* can't be mapped more than once on the GPU */ -+ /* Can't shrink when physical pages are mapped to different GPU -+ * VAs. The code doesn't support looking up: -+ * - all physical pages assigned to different GPU VAs -+ * - CPU mappings for the physical pages at different vm_pgoff -+ * (==GPU VA) locations. -+ * -+ * Note that for Native allocs mapped at multiple GPU VAs, growth of -+ * such allocs is not a supported use-case. -+ */ - if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) - goto out_unlock; -+ -+ if (atomic_read(®->cpu_alloc->kernel_mappings) > 0) -+ goto out_unlock; - /* can't grow regions which are ephemeral */ - if (reg->flags & KBASE_REG_DONT_NEED) - goto out_unlock; -@@ -2085,7 +2206,7 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) - * No update to the mm so downgrade the writer lock to a read - * lock so other readers aren't blocked after this point. - */ -- downgrade_write(¤t->mm->mmap_sem); -+ downgrade_write(kbase_mem_get_process_mmap_lock()); - read_locked = true; - - /* Allocate some more pages */ -@@ -2127,9 +2248,9 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) - out_unlock: - kbase_gpu_vm_unlock(kctx); - if (read_locked) -- up_read(¤t->mm->mmap_sem); -+ up_read(kbase_mem_get_process_mmap_lock()); - else -- up_write(¤t->mm->mmap_sem); -+ up_write(kbase_mem_get_process_mmap_lock()); - - return res; - } -@@ -2360,11 +2481,7 @@ static int kbase_cpu_mmap(struct kbase_context *kctx, - * See MIDBASE-1057 - */ - --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) - vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; --#else -- vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; --#endif - vma->vm_ops = &kbase_vm_ops; - vma->vm_private_data = map; - -@@ -2510,16 +2627,14 @@ out: - - void kbase_os_mem_map_lock(struct kbase_context *kctx) - { -- struct mm_struct *mm = current->mm; - (void)kctx; -- down_read(&mm->mmap_sem); -+ down_read(kbase_mem_get_process_mmap_lock()); - } - - void kbase_os_mem_map_unlock(struct kbase_context *kctx) - { -- struct mm_struct *mm = current->mm; - (void)kctx; -- up_read(&mm->mmap_sem); -+ up_read(kbase_mem_get_process_mmap_lock()); - } - - static int kbasep_reg_mmap(struct kbase_context *kctx, -@@ -2547,7 +2662,8 @@ static int kbasep_reg_mmap(struct kbase_context *kctx, - /* incorrect mmap size */ - /* leave the cookie for a potential later - * mapping, or to be reclaimed later when the -- * context is freed */ -+ * context is freed -+ */ - err = -ENOMEM; - goto out; - } -@@ -2576,6 +2692,11 @@ static int kbasep_reg_mmap(struct kbase_context *kctx, - kctx->pending_regions[cookie] = NULL; - bitmap_set(kctx->cookies, cookie, 1); - -+#if MALI_USE_CSF -+ if (reg->flags & KBASE_REG_CSF_EVENT) -+ kbase_link_event_mem_page(kctx, reg); -+#endif -+ - /* - * Overwrite the offset with the region start_pfn, so we effectively - * map from offset 0 in the region. However subtract the aligned -@@ -2595,7 +2716,7 @@ int kbase_context_mmap(struct kbase_context *const kctx, - { - struct kbase_va_region *reg = NULL; - void *kaddr = NULL; -- size_t nr_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; -+ size_t nr_pages = vma_pages(vma); - int err = 0; - int free_on_close = 0; - struct device *dev = kctx->kbdev->dev; -@@ -2608,7 +2729,7 @@ int kbase_context_mmap(struct kbase_context *const kctx, - if (!(vma->vm_flags & VM_WRITE)) - vma->vm_flags &= ~VM_MAYWRITE; - -- if (0 == nr_pages) { -+ if (nr_pages == 0) { - err = -EINVAL; - goto out; - } -@@ -2629,7 +2750,8 @@ int kbase_context_mmap(struct kbase_context *const kctx, - /* if not the MTP, verify that the MTP has been mapped */ - rcu_read_lock(); - /* catches both when the special page isn't present or -- * when we've forked */ -+ * when we've forked -+ */ - if (rcu_dereference(kctx->process_mm) != current->mm) { - err = -EINVAL; - rcu_read_unlock(); -@@ -2646,16 +2768,30 @@ int kbase_context_mmap(struct kbase_context *const kctx, - case PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE): - /* MMU dump */ - err = kbase_mmu_dump_mmap(kctx, vma, ®, &kaddr); -- if (0 != err) -+ if (err != 0) - goto out_unlock; - /* free the region on munmap */ - free_on_close = 1; - break; -+#if MALI_USE_CSF -+ case PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE): -+ kbase_gpu_vm_unlock(kctx); -+ err = kbase_csf_cpu_mmap_user_reg_page(kctx, vma); -+ goto out; -+ case PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE) ... -+ PFN_DOWN(BASE_MEM_COOKIE_BASE) - 1: { -+ kbase_gpu_vm_unlock(kctx); -+ mutex_lock(&kctx->csf.lock); -+ err = kbase_csf_cpu_mmap_user_io_pages(kctx, vma); -+ mutex_unlock(&kctx->csf.lock); -+ goto out; -+ } -+#endif - case PFN_DOWN(BASE_MEM_COOKIE_BASE) ... - PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { - err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, - &aligned_offset); -- if (0 != err) -+ if (err != 0) - goto out_unlock; - /* free the region on munmap */ - free_on_close = 1; -@@ -2728,8 +2864,21 @@ int kbase_context_mmap(struct kbase_context *const kctx, - - if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { - /* MMU dump - userspace should now have a reference on -- * the pages, so we can now free the kernel mapping */ -+ * the pages, so we can now free the kernel mapping -+ */ - vfree(kaddr); -+ /* CPU mapping of GPU allocations have GPU VA as the vm_pgoff -+ * and that is used to shrink the mapping when the commit size -+ * is reduced. So vm_pgoff for CPU mapping created to get the -+ * snapshot of GPU page tables shall not match with any GPU VA. -+ * That can be ensured by setting vm_pgoff as vma->vm_start -+ * because, -+ * - GPU VA of any SAME_VA allocation cannot match with -+ * vma->vm_start, as CPU VAs are unique. -+ * - GPU VA of CUSTOM_VA allocations are outside the CPU -+ * virtual address space. -+ */ -+ vma->vm_pgoff = PFN_DOWN(vma->vm_start); - } - - out_unlock: -@@ -2824,8 +2973,8 @@ static int kbase_vmap_phy_pages(struct kbase_context *kctx, - - /* Note: enforcing a RO prot_request onto prot is not done, since: - * - CPU-arch-specific integration required -- * - kbase_vmap() requires no access checks to be made/enforced */ -- -+ * - kbase_vmap() requires no access checks to be made/enforced -+ */ - cpu_addr = vmap(pages, page_count, VM_MAP, prot); - - kfree(pages); -@@ -2846,6 +2995,7 @@ static int kbase_vmap_phy_pages(struct kbase_context *kctx, - if (map->sync_needed) - kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_CPU); - -+ kbase_mem_phy_alloc_kernel_mapped(reg->cpu_alloc); - return 0; - } - -@@ -2901,7 +3051,8 @@ void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, - * be made. - * - * As mentioned in kbase_vmap_prot() this means that a kernel-side -- * CPU-RO mapping is not enforced to allow this to work */ -+ * CPU-RO mapping is not enforced to allow this to work -+ */ - return kbase_vmap_prot(kctx, gpu_addr, size, 0u, map); - } - KBASE_EXPORT_TEST_API(kbase_vmap); -@@ -2915,6 +3066,7 @@ static void kbase_vunmap_phy_pages(struct kbase_context *kctx, - if (map->sync_needed) - kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_DEVICE); - -+ kbase_mem_phy_alloc_kernel_unmapped(map->cpu_alloc); - map->offset_in_page = 0; - map->cpu_pages = NULL; - map->gpu_pages = NULL; -@@ -2933,7 +3085,7 @@ KBASE_EXPORT_TEST_API(kbase_vunmap); - - static void kbasep_add_mm_counter(struct mm_struct *mm, int member, long value) - { --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 19, 0)) -+#if (KERNEL_VERSION(4, 19, 0) <= LINUX_VERSION_CODE) - /* To avoid the build breakage due to an unexported kernel symbol - * 'mm_trace_rss_stat' from later kernels, i.e. from V4.19.0 onwards, - * we inline here the equivalent of 'add_mm_counter()' from linux -@@ -3017,14 +3169,319 @@ static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_ - - /* no real access */ - vma->vm_flags &= ~(VM_READ | VM_MAYREAD | VM_WRITE | VM_MAYWRITE | VM_EXEC | VM_MAYEXEC); --#if (LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0)) - vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP | VM_IO; -+ vma->vm_ops = &kbase_vm_special_ops; -+ vma->vm_private_data = kctx; -+ -+ return 0; -+} -+ -+#if MALI_USE_CSF -+static unsigned long get_queue_doorbell_pfn(struct kbase_device *kbdev, -+ struct kbase_queue *queue) -+{ -+ lockdep_assert_held(&kbdev->csf.reg_lock); -+ -+ /* Return the real Hw doorbell page if queue has been -+ * assigned one, otherwise a dummy page. Always return the -+ * dummy page in no mali builds. -+ */ -+ if (queue->doorbell_nr == KBASEP_USER_DB_NR_INVALID) -+ return PFN_DOWN(as_phys_addr_t(kbdev->csf.dummy_db_page)); -+ return (PFN_DOWN(kbdev->reg_start + CSF_HW_DOORBELL_PAGE_OFFSET + -+ (u64)queue->doorbell_nr * CSF_HW_DOORBELL_PAGE_SIZE)); -+} -+ -+static void kbase_csf_user_io_pages_vm_open(struct vm_area_struct *vma) -+{ -+ WARN(1, "Unexpected attempt to clone private vma\n"); -+ vma->vm_private_data = NULL; -+} -+ -+static void kbase_csf_user_io_pages_vm_close(struct vm_area_struct *vma) -+{ -+ struct kbase_queue *queue = vma->vm_private_data; -+ struct kbase_context *kctx; -+ struct kbase_device *kbdev; -+ int err; -+ bool reset_prevented = false; -+ -+ if (WARN_ON(!queue)) -+ return; -+ -+ kctx = queue->kctx; -+ kbdev = kctx->kbdev; -+ -+ err = kbase_reset_gpu_prevent_and_wait(kbdev); -+ if (err) -+ dev_warn( -+ kbdev->dev, -+ "Unsuccessful GPU reset detected when unbinding queue (csi_index=%d), attempting to unbind regardless", -+ queue->csi_index); -+ else -+ reset_prevented = true; -+ -+ mutex_lock(&kctx->csf.lock); -+ kbase_csf_queue_unbind(queue); -+ mutex_unlock(&kctx->csf.lock); -+ -+ if (reset_prevented) -+ kbase_reset_gpu_allow(kbdev); -+ -+ /* Now as the vma is closed, drop the reference on mali device file */ -+ fput(kctx->filp); -+} -+ -+#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) -+static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_area_struct *vma, -+ struct vm_fault *vmf) -+{ - #else -- vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_RESERVED | VM_IO; -+static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf) -+{ -+ struct vm_area_struct *vma = vmf->vma; - #endif -- vma->vm_ops = &kbase_vm_special_ops; -+ struct kbase_queue *queue = vma->vm_private_data; -+ unsigned long doorbell_cpu_addr, input_cpu_addr, output_cpu_addr; -+ unsigned long doorbell_page_pfn, input_page_pfn, output_page_pfn; -+ pgprot_t doorbell_pgprot, input_page_pgprot, output_page_pgprot; -+ size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); -+ vm_fault_t ret; -+ struct kbase_device *kbdev; -+ struct memory_group_manager_device *mgm_dev; -+ -+ /* Few sanity checks up front */ -+ if ((nr_pages != BASEP_QUEUE_NR_MMAP_USER_PAGES) || -+ (vma->vm_pgoff != queue->db_file_offset)) -+ return VM_FAULT_SIGBUS; -+ -+ mutex_lock(&queue->kctx->csf.lock); -+ kbdev = queue->kctx->kbdev; -+ mgm_dev = kbdev->mgm_dev; -+ -+ /* Always map the doorbell page as uncached */ -+ doorbell_pgprot = pgprot_device(vma->vm_page_prot); -+ -+#if ((KERNEL_VERSION(4, 4, 147) >= LINUX_VERSION_CODE) || \ -+ ((KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE) && \ -+ (KERNEL_VERSION(4, 5, 0) <= LINUX_VERSION_CODE))) -+ vma->vm_page_prot = doorbell_pgprot; -+ input_page_pgprot = doorbell_pgprot; -+ output_page_pgprot = doorbell_pgprot; -+#else -+ if (kbdev->system_coherency == COHERENCY_NONE) { -+ input_page_pgprot = pgprot_writecombine(vma->vm_page_prot); -+ output_page_pgprot = pgprot_writecombine(vma->vm_page_prot); -+ } else { -+ input_page_pgprot = vma->vm_page_prot; -+ output_page_pgprot = vma->vm_page_prot; -+ } -+#endif -+ -+ doorbell_cpu_addr = vma->vm_start; -+ -+#if KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE -+ if ((unsigned long)vmf->virtual_address == doorbell_cpu_addr) { -+#else -+ if (vmf->address == doorbell_cpu_addr) { -+#endif -+ mutex_lock(&kbdev->csf.reg_lock); -+ doorbell_page_pfn = get_queue_doorbell_pfn(kbdev, queue); -+ ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, -+ KBASE_MEM_GROUP_CSF_IO, vma, doorbell_cpu_addr, -+ doorbell_page_pfn, doorbell_pgprot); -+ mutex_unlock(&kbdev->csf.reg_lock); -+ } else { -+ /* Map the Input page */ -+ input_cpu_addr = doorbell_cpu_addr + PAGE_SIZE; -+ input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[0])); -+ ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, -+ KBASE_MEM_GROUP_CSF_IO, vma, input_cpu_addr, -+ input_page_pfn, input_page_pgprot); -+ if (ret != VM_FAULT_NOPAGE) -+ goto exit; -+ -+ /* Map the Output page */ -+ output_cpu_addr = input_cpu_addr + PAGE_SIZE; -+ output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[1])); -+ ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, -+ KBASE_MEM_GROUP_CSF_IO, vma, output_cpu_addr, -+ output_page_pfn, output_page_pgprot); -+ } -+ -+exit: -+ mutex_unlock(&queue->kctx->csf.lock); -+ return ret; -+} -+ -+static const struct vm_operations_struct kbase_csf_user_io_pages_vm_ops = { -+ .open = kbase_csf_user_io_pages_vm_open, -+ .close = kbase_csf_user_io_pages_vm_close, -+ .fault = kbase_csf_user_io_pages_vm_fault -+}; -+ -+/* Program the client process's page table entries to map the pair of -+ * input/output pages & Hw doorbell page. The caller should have validated that -+ * vma->vm_pgoff maps to the range of csf cookies. -+ */ -+static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, -+ struct vm_area_struct *vma) -+{ -+ unsigned long cookie = -+ vma->vm_pgoff - PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); -+ size_t nr_pages = vma_pages(vma); -+ struct kbase_queue *queue; -+ int err = 0; -+ -+ lockdep_assert_held(&kctx->csf.lock); -+ -+ queue = kctx->csf.user_pages_info[cookie]; -+ -+ /* Looks like the bind has been aborted */ -+ if (!queue) -+ return -EINVAL; -+ -+ if (WARN_ON(test_bit(cookie, kctx->csf.cookies))) -+ return -EINVAL; -+ -+ /* no need for the cookie anymore */ -+ kctx->csf.user_pages_info[cookie] = NULL; -+ bitmap_set(kctx->csf.cookies, cookie, 1); -+ -+ /* Reset the handle to avoid (re)freeing the cookie (which can -+ * now get re-assigned) on unbind. -+ */ -+ queue->handle = BASEP_MEM_INVALID_HANDLE; -+ -+ if (nr_pages != BASEP_QUEUE_NR_MMAP_USER_PAGES) { -+ err = -EINVAL; -+ goto map_failed; -+ } -+ -+ err = kbase_csf_alloc_command_stream_user_pages(kctx, queue); -+ if (err) -+ goto map_failed; -+ -+ vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; -+ /* TODO use VM_MIXEDMAP, since it is more appropriate as both types of -+ * memory with and without "struct page" backing are being inserted here. -+ * Hw Doorbell pages comes from the device register area so kernel does -+ * not use "struct page" for them. -+ */ -+ vma->vm_flags |= VM_PFNMAP; -+ -+ vma->vm_ops = &kbase_csf_user_io_pages_vm_ops; -+ vma->vm_private_data = queue; -+ -+ /* Make vma point to the special internal file, but don't drop the -+ * reference on mali device file (that would be done later when the -+ * vma is closed). -+ */ -+ vma->vm_file = kctx->kbdev->csf.db_filp; -+ get_file(vma->vm_file); -+ /* Also adjust the vm_pgoff */ -+ vma->vm_pgoff = queue->db_file_offset; -+ -+ return 0; -+ -+map_failed: -+ /* The queue cannot have got to KBASE_CSF_QUEUE_BOUND state if we -+ * reached here, so safe to use a variant of unbind that only works on -+ * stopped queues -+ * -+ * This is so we don't enter the CSF scheduler from this path. -+ */ -+ kbase_csf_queue_unbind_stopped(queue); -+ -+ return err; -+} -+ -+static void kbase_csf_user_reg_vm_close(struct vm_area_struct *vma) -+{ -+ struct kbase_context *kctx = vma->vm_private_data; -+ -+ WARN_ON(!kctx->csf.user_reg_vma); -+ -+ kctx->csf.user_reg_vma = NULL; -+} -+ -+#if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) -+static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_area_struct *vma, -+ struct vm_fault *vmf) -+{ -+#else -+static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_fault *vmf) -+{ -+ struct vm_area_struct *vma = vmf->vma; -+#endif -+ struct kbase_context *kctx = vma->vm_private_data; -+ struct kbase_device *kbdev = kctx->kbdev; -+ struct memory_group_manager_device *mgm_dev = kbdev->mgm_dev; -+ unsigned long pfn = PFN_DOWN(kbdev->reg_start + USER_BASE); -+ size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); -+ vm_fault_t ret = VM_FAULT_SIGBUS; -+ -+ /* Few sanity checks up front */ -+ if (WARN_ON(nr_pages != 1) || -+ WARN_ON(vma != kctx->csf.user_reg_vma) || -+ WARN_ON(vma->vm_pgoff != -+ PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE))) -+ return VM_FAULT_SIGBUS; -+ -+ mutex_lock(&kbdev->pm.lock); -+ -+ /* Don't map in the actual register page if GPU is powered down. -+ * Always map in the dummy page in no mali builds. -+ */ -+ if (!kbdev->pm.backend.gpu_powered) -+ pfn = PFN_DOWN(as_phys_addr_t(kbdev->csf.dummy_user_reg_page)); -+ -+ ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, -+ KBASE_MEM_GROUP_CSF_FW, vma, -+ vma->vm_start, pfn, -+ vma->vm_page_prot); -+ -+ mutex_unlock(&kbdev->pm.lock); -+ -+ return ret; -+} -+ -+static const struct vm_operations_struct kbase_csf_user_reg_vm_ops = { -+ .close = kbase_csf_user_reg_vm_close, -+ .fault = kbase_csf_user_reg_vm_fault -+}; -+ -+static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, -+ struct vm_area_struct *vma) -+{ -+ size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); -+ -+ /* Few sanity checks */ -+ if (kctx->csf.user_reg_vma) -+ return -EBUSY; -+ -+ if (nr_pages != 1) -+ return -EINVAL; -+ -+ if (vma->vm_flags & (VM_WRITE | VM_MAYWRITE)) -+ return -EPERM; -+ -+ /* Map uncached */ -+ vma->vm_page_prot = pgprot_device(vma->vm_page_prot); -+ -+ vma->vm_flags |= VM_DONTCOPY | VM_DONTDUMP | VM_DONTEXPAND | VM_IO; -+ -+ /* User register page comes from the device register area so -+ * "struct page" isn't available for it. -+ */ -+ vma->vm_flags |= VM_PFNMAP; -+ -+ kctx->csf.user_reg_vma = vma; -+ -+ vma->vm_ops = &kbase_csf_user_reg_vm_ops; - vma->vm_private_data = kctx; - - return 0; - } - -+#endif /* MALI_USE_CSF */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h -index cd094b3..36159c1 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010, 2012-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010, 2012-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,21 +17,16 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_mem_linux.h - * Base kernel memory APIs, Linux implementation. - */ - - #ifndef _KBASE_MEM_LINUX_H_ - #define _KBASE_MEM_LINUX_H_ - --/** A HWC dump mapping */ -+/* A HWC dump mapping */ - struct kbase_hwc_dma_mapping { - void *cpu_va; - dma_addr_t dma_pa; -@@ -43,7 +39,7 @@ struct kbase_hwc_dma_mapping { - * @kctx: The kernel context - * @va_pages: The number of pages of virtual address space to reserve - * @commit_pages: The number of physical pages to allocate upfront -- * @extent: The number of extra pages to allocate on each GPU fault which -+ * @extension: The number of extra pages to allocate on each GPU fault which - * grows the region. - * @flags: bitmask of BASE_MEM_* flags to convey special requirements & - * properties for the new allocation. -@@ -53,8 +49,8 @@ struct kbase_hwc_dma_mapping { - * Return: 0 on success or error code - */ - struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, -- u64 va_pages, u64 commit_pages, u64 extent, u64 *flags, -- u64 *gpu_va); -+ u64 va_pages, u64 commit_pages, -+ u64 extension, u64 *flags, u64 *gpu_va); - - /** - * kbase_mem_query - Query properties of a GPU memory region -@@ -194,8 +190,8 @@ int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, - * Take the provided region and make all the physical pages within it - * reclaimable by the kernel, updating the per-process VM stats as well. - * Remove any CPU mappings (as these can't be removed in the shrinker callback -- * as mmap_sem might already be taken) but leave the GPU mapping intact as -- * and until the shrinker reclaims the allocation. -+ * as mmap_sem/mmap_lock might already be taken) but leave the GPU mapping -+ * intact as and until the shrinker reclaims the allocation. - * - * Note: Must be called with the region lock of the containing context. - */ -@@ -461,4 +457,18 @@ static inline vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, - } - #endif - -+/** -+ * kbase_mem_get_process_mmap_lock - Return the mmap lock for the current process -+ * -+ * Return: the mmap lock for the current process -+ */ -+static inline struct rw_semaphore *kbase_mem_get_process_mmap_lock(void) -+{ -+#if KERNEL_VERSION(5, 8, 0) > LINUX_VERSION_CODE -+ return ¤t->mm->mmap_sem; -+#else /* KERNEL_VERSION(5, 8, 0) > LINUX_VERSION_CODE */ -+ return ¤t->mm->mmap_lock; -+#endif /* KERNEL_VERSION(5, 8, 0) > LINUX_VERSION_CODE */ -+} -+ - #endif /* _KBASE_MEM_LINUX_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h -index 7011603..3f260bf 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_lowlevel.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2014,2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2014, 2016-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #ifndef _KBASE_MEM_LOWLEVEL_H - #define _KBASE_MEM_LOWLEVEL_H - -@@ -31,9 +28,7 @@ - - #include <linux/dma-mapping.h> - --/** -- * @brief Flags for kbase_phy_allocator_pages_alloc -- */ -+/* Flags for kbase_phy_allocator_pages_alloc */ - #define KBASE_PHY_PAGES_FLAG_DEFAULT (0) /** Default allocation flag */ - #define KBASE_PHY_PAGES_FLAG_CLEAR (1 << 0) /** Clear the pages after allocation */ - #define KBASE_PHY_PAGES_FLAG_POISON (1 << 1) /** Fill the memory with a poison value */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c -index 0723e32..a11da82 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2015-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> -@@ -154,20 +153,12 @@ static void kbase_mem_pool_spill(struct kbase_mem_pool *next_pool, - struct page *kbase_mem_alloc_page(struct kbase_mem_pool *pool) - { - struct page *p; -- gfp_t gfp; -+ gfp_t gfp = GFP_HIGHUSER | __GFP_ZERO; - struct kbase_device *const kbdev = pool->kbdev; - struct device *const dev = kbdev->dev; - dma_addr_t dma_addr; - int i; - --#if defined(CONFIG_ARM) && !defined(CONFIG_HAVE_DMA_ATTRS) && \ -- LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) -- /* DMA cache sync fails for HIGHMEM before 3.5 on ARM */ -- gfp = GFP_USER | __GFP_ZERO; --#else -- gfp = GFP_HIGHUSER | __GFP_ZERO; --#endif -- - /* don't warn on higher order failures */ - if (pool->order) - gfp |= __GFP_NOWARN; -@@ -318,7 +309,7 @@ void kbase_mem_pool_set_max_size(struct kbase_mem_pool *pool, size_t max_size) - - kbase_mem_pool_unlock(pool); - } -- -+KBASE_EXPORT_TEST_API(kbase_mem_pool_set_max_size); - - static unsigned long kbase_mem_pool_reclaim_count_objects(struct shrinker *s, - struct shrink_control *sc) -@@ -364,17 +355,6 @@ static unsigned long kbase_mem_pool_reclaim_scan_objects(struct shrinker *s, - return freed; - } - --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) --static int kbase_mem_pool_reclaim_shrink(struct shrinker *s, -- struct shrink_control *sc) --{ -- if (sc->nr_to_scan == 0) -- return kbase_mem_pool_reclaim_count_objects(s, sc); -- -- return kbase_mem_pool_reclaim_scan_objects(s, sc); --} --#endif -- - int kbase_mem_pool_init(struct kbase_mem_pool *pool, - const struct kbase_mem_pool_config *config, - unsigned int order, -@@ -398,19 +378,13 @@ int kbase_mem_pool_init(struct kbase_mem_pool *pool, - spin_lock_init(&pool->pool_lock); - INIT_LIST_HEAD(&pool->page_list); - -- /* Register shrinker */ --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) -- pool->reclaim.shrink = kbase_mem_pool_reclaim_shrink; --#else - pool->reclaim.count_objects = kbase_mem_pool_reclaim_count_objects; - pool->reclaim.scan_objects = kbase_mem_pool_reclaim_scan_objects; --#endif - pool->reclaim.seeks = DEFAULT_SEEKS; - /* Kernel versions prior to 3.1 : -- * struct shrinker does not define batch */ --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 1, 0) -+ * struct shrinker does not define batch -+ */ - pool->reclaim.batch = 0; --#endif - register_shrinker(&pool->reclaim); - - pool_dbg(pool, "initialized\n"); -@@ -830,8 +804,8 @@ void kbase_mem_pool_free_pages_locked(struct kbase_mem_pool *pool, - nr_to_pool = kbase_mem_pool_capacity(pool); - nr_to_pool = min(nr_pages, nr_to_pool); - -- kbase_mem_pool_add_array_locked(pool, nr_pages, pages, false, -- dirty); -+ kbase_mem_pool_add_array_locked(pool, nr_to_pool, pages, false, -+ dirty); - - i += nr_to_pool; - } -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c -index 5879fdf..cfb43b0 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/debugfs.h> -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h -index 2932945..207b585 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_debugfs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_MEM_POOL_DEBUGFS_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_group.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_group.c -index aa25548..8d7bb4d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_group.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_group.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_group.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_group.h -index 0484f59..38fd4ca 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_group.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_pool_group.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_MEM_POOL_GROUP_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c -index 5752d4a..ea8e34b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2017, 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2017, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,23 +17,23 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <mali_kbase.h> - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - --/** Show callback for the @c mem_profile debugfs file. -+/** -+ * Show callback for the @c mem_profile debugfs file. - * - * This function is called to get the contents of the @c mem_profile debugfs - * file. This is a report of current memory usage and distribution in userspace. - * -- * @param sfile The debugfs entry -- * @param data Data associated with the entry -+ * @sfile: The debugfs entry -+ * @data: Data associated with the entry - * -- * @return 0 if it successfully prints data in debugfs entry file, non-zero otherwise -+ * Return: 0 if it successfully prints data in debugfs entry file, non-zero -+ * otherwise - */ - static int kbasep_mem_profile_seq_show(struct seq_file *sfile, void *data) - { -@@ -71,6 +72,11 @@ static const struct file_operations kbasep_mem_profile_debugfs_fops = { - int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, - size_t size) - { -+#if (KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE) -+ const mode_t mode = 0444; -+#else -+ const mode_t mode = 0400; -+#endif - int err = 0; - - if (IS_ERR_OR_NULL(kctx->kctx_dentry)) /* not initialized */ -@@ -84,7 +90,7 @@ int kbasep_mem_profile_debugfs_insert(struct kbase_context *kctx, char *data, - if (!kbase_ctx_flag(kctx, KCTX_MEM_PROFILE_INITIALIZED)) { - if (IS_ERR_OR_NULL(kctx->kctx_dentry)) { - err = -ENOMEM; -- } else if (!debugfs_create_file("mem_profile", 0444, -+ } else if (!debugfs_create_file("mem_profile", mode, - kctx->kctx_dentry, kctx, - &kbasep_mem_profile_debugfs_fops)) { - err = -EAGAIN; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h -index 1462247..093a65e 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2016 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2016, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,14 +17,9 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_mem_profile_debugfs.h - * Header file for mem profiles entries in debugfs - * - */ -@@ -35,12 +31,17 @@ - #include <linux/seq_file.h> - - /** -- * @brief Remove entry from Mali memory profile debugfs -+ * Remove entry from Mali memory profile debugfs -+ * @kctx: The context whose debugfs file @p data should be removed from - */ - void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx); - - /** -- * @brief Insert @p data to the debugfs file so it can be read by userspace -+ * Insert @p data to the debugfs file so it can be read by userspace -+ * @kctx: The context whose debugfs file @p data should be inserted to -+ * @data: A NULL-terminated string to be inserted to the debugfs file, -+ * without the trailing new line character -+ * @size: The length of the @p data string - * - * The function takes ownership of @p data and frees it later when new data - * is inserted. -@@ -48,10 +49,6 @@ void kbasep_mem_profile_debugfs_remove(struct kbase_context *kctx); - * If the debugfs entry corresponding to the @p kctx doesn't exist, - * an attempt will be made to create it. - * -- * @param kctx The context whose debugfs file @p data should be inserted to -- * @param data A NULL-terminated string to be inserted to the debugfs file, -- * without the trailing new line character -- * @param size The length of the @p data string - * @return 0 if @p data inserted correctly - * -EAGAIN in case of error - * @post @ref mem_profile_initialized will be set to @c true -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h -index d55cc85..3184a98 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_profile_debugfs_buf_size.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2017-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,9 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -- * @file mali_kbase_mem_profile_debugfs_buf_size.h - * Header file for the size of the buffer to accumulate the histogram report text in - */ - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mipe_gen_header.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mipe_gen_header.h -index ec52122..f0b385e 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mipe_gen_header.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mipe_gen_header.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. -@@ -40,14 +39,14 @@ - * defined. See documentation below: - */ - --/** -+/* - * The name of the variable where the result BLOB will be stored. - */ - #if !defined(MIPE_HEADER_BLOB_VAR_NAME) - #error "MIPE_HEADER_BLOB_VAR_NAME must be defined!" - #endif - --/** -+/* - * A compiler attribute for the BLOB variable. - * - * e.g. __attribute__((section("my_section"))) -@@ -58,6 +57,17 @@ - #define MIPE_HEADER_BLOB_VAR_ATTRIBUTE - #endif - -+/** -+ * A compiler attribute for packing structures -+ * -+ * e.g. __packed -+ * -+ * Default value is __attribute__((__packed__)) -+ */ -+#if !defined(MIPE_HEADER_PACKED_ATTRIBUTE) -+#define MIPE_HEADER_PACKED_ATTRIBUTE __attribute__((__packed__)) -+#endif -+ - /** - * MIPE stream id. - * -@@ -67,7 +77,7 @@ - #error "MIPE_HEADER_STREAM_ID must be defined!" - #endif - --/** -+/* - * MIPE packet class. - * - * See enum tl_packet_class. -@@ -76,10 +86,11 @@ - #error "MIPE_HEADER_PKT_CLASS must be defined!" - #endif - --/** -+/* - * The list of tracepoints to process. - * - * It should be defined as follows: -+ * - * #define MIPE_HEADER_TRACEPOINT_LIST \ - * TRACEPOINT_DESC(FIRST_TRACEPOINT, "Some description", "@II", "first_arg,second_arg") \ - * TRACEPOINT_DESC(SECOND_TRACEPOINT, "Some description", "@II", "first_arg,second_arg") \ -@@ -94,17 +105,18 @@ - #error "MIPE_HEADER_TRACEPOINT_LIST must be defined!" - #endif - --/** -+/* - * The number of entries in MIPE_HEADER_TRACEPOINT_LIST. - */ - #if !defined(MIPE_HEADER_TRACEPOINT_LIST_SIZE) - #error "MIPE_HEADER_TRACEPOINT_LIST_SIZE must be defined!" - #endif - --/** -+/* - * The list of enums to process. - * - * It should be defined as follows: -+ * - * #define MIPE_HEADER_ENUM_LIST \ - * ENUM_DESC(enum_arg_name, enum_value) \ - * ENUM_DESC(enum_arg_name, enum_value) \ -@@ -117,7 +129,7 @@ - */ - #if defined(MIPE_HEADER_ENUM_LIST) - --/** -+/* - * Tracepoint message ID used for enums declaration. - */ - #if !defined(MIPE_HEADER_ENUM_MSG_ID) -@@ -149,7 +161,7 @@ const struct - char _arg_types[sizeof(arg_types)]; \ - u32 _size_arg_names; \ - char _arg_names[sizeof(arg_names)]; \ -- } __attribute__ ((__packed__)) __ ## name; -+ } MIPE_HEADER_PACKED_ATTRIBUTE __ ## name; - - #define ENUM_DESC(arg_name, value) \ - struct { \ -@@ -159,13 +171,13 @@ const struct - u32 _value; \ - u32 _value_str_len; \ - char _value_str[sizeof(#value)]; \ -- } __attribute__ ((__packed__)) __ ## arg_name ## _ ## value; -+ } MIPE_HEADER_PACKED_ATTRIBUTE __ ## arg_name ## _ ## value; - - MIPE_HEADER_TRACEPOINT_LIST - MIPE_HEADER_ENUM_LIST - #undef TRACEPOINT_DESC - #undef ENUM_DESC --} __attribute__((packed)) MIPE_HEADER_BLOB_VAR_NAME MIPE_HEADER_BLOB_VAR_ATTRIBUTE = { -+} MIPE_HEADER_PACKED_ATTRIBUTE MIPE_HEADER_BLOB_VAR_NAME MIPE_HEADER_BLOB_VAR_ATTRIBUTE = { - ._mipe_w0 = MIPE_PACKET_HEADER_W0( - TL_PACKET_FAMILY_TL, - MIPE_HEADER_PKT_CLASS, -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mipe_proto.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mipe_proto.h -index 54667cf..c35ee61 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mipe_proto.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_mipe_proto.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* THIS FILE IS AUTOGENERATED BY mali_trace_generator.py. -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_native_mgm.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_native_mgm.c -index 38ae46e..4554bee 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_native_mgm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_native_mgm.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/gfp.h> -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_native_mgm.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_native_mgm.h -index 431b1f4..1eae2fc 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_native_mgm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_native_mgm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_NATIVE_MGM_H_ -@@ -25,7 +24,7 @@ - - #include <linux/memory_group_manager.h> - --/** -+/* - * kbase_native_mgm_dev - Native memory group manager device - * - * An implementation of the memory group manager interface that is intended for -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c -index fbb090e..bf525ed 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_platform_fake.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2014, 2016-2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2014, 2016-2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/errno.h> -@@ -26,7 +25,6 @@ - #include <linux/platform_device.h> - #include <linux/string.h> - -- - /* - * This file is included only for type definitions and functions belonging to - * specific platform folders. Do not add dependencies with symbols that are -@@ -41,14 +39,13 @@ static struct platform_device *mali_device; - - #ifndef CONFIG_OF - /** -- * @brief Convert data in struct kbase_io_resources struct to Linux-specific resources -+ * Convert data in struct kbase_io_resources struct to Linux-specific resources -+ * @io_resources: Input IO resource data -+ * @linux_resources: Pointer to output array of Linux resource structures - * - * Function converts data in struct kbase_io_resources struct to an array of Linux resource structures. Note that function - * assumes that size of linux_resource array is at least PLATFORM_CONFIG_RESOURCE_COUNT. - * Resources are put in fixed order: I/O memory region, job IRQ, MMU IRQ, GPU IRQ. -- * -- * @param[in] io_resource Input IO resource data -- * @param[out] linux_resources Pointer to output array of Linux resource structures - */ - static void kbasep_config_parse_io_resources(const struct kbase_io_resources *io_resources, struct resource *const linux_resources) - { -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_pm.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_pm.c -index b9ed8c3..de100dd 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_pm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_pm.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,15 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_pm.c -- * Base kernel power management APIs -+ * DOC: Base kernel power management APIs - */ - - #include <mali_kbase.h> -@@ -33,12 +29,14 @@ - #include <mali_kbase_hwcnt_context.h> - - #include <mali_kbase_pm.h> --#include <mali_kbase_pm_internal.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> - - #ifdef CONFIG_MALI_ARBITER_SUPPORT - #include <arbiter/mali_kbase_arbiter_pm.h> - #endif /* CONFIG_MALI_ARBITER_SUPPORT */ - -+#include <backend/gpu/mali_kbase_clk_rate_trace_mgr.h> -+ - int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags) - { - return kbase_hwaccess_pm_powerup(kbdev, flags); -@@ -66,14 +64,14 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, - kbase_pm_lock(kbdev); - - #ifdef CONFIG_MALI_ARBITER_SUPPORT -- if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, suspend_handler)) -+ if (kbase_arbiter_pm_ctx_active_handle_suspend(kbdev, -+ suspend_handler)) { -+ kbase_pm_unlock(kbdev); - return 1; -+ } -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ - -- if (kbase_pm_is_suspending(kbdev) || -- kbase_pm_is_gpu_lost(kbdev)) { --#else - if (kbase_pm_is_suspending(kbdev)) { --#endif /* CONFIG_MALI_ARBITER_SUPPORT */ - switch (suspend_handler) { - case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE: - if (kbdev->pm.active_count != 0) -@@ -101,6 +99,7 @@ int kbase_pm_context_active_handle_suspend(struct kbase_device *kbdev, - #ifdef CONFIG_MALI_ARBITER_SUPPORT - kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_REF_EVENT); - #endif /* CONFIG_MALI_ARBITER_SUPPORT */ -+ kbase_clk_rate_trace_manager_gpu_active(kbdev); - } - - kbase_pm_unlock(kbdev); -@@ -128,6 +127,7 @@ void kbase_pm_context_idle(struct kbase_device *kbdev) - if (c == 0) { - /* Last context has gone idle */ - kbase_hwaccess_pm_gpu_idle(kbdev); -+ kbase_clk_rate_trace_manager_gpu_idle(kbdev); - - /* Wake up anyone waiting for this to become 0 (e.g. suspend). - * The waiters must synchronize with us by locking the pm.lock -@@ -171,6 +171,7 @@ void kbase_pm_driver_suspend(struct kbase_device *kbdev) - unsigned long flags; - - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbdev->js_data.runpool_irq.submit_allowed = 0; - kbase_disjoint_state_up(kbdev); - for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) - kbase_job_slot_softstop(kbdev, i, NULL); -@@ -184,9 +185,14 @@ void kbase_pm_driver_suspend(struct kbase_device *kbdev) - * all pm references - */ - -+#if !MALI_USE_CSF - /* Suspend job scheduler and associated components, so that it releases all -- * the PM active count references */ -+ * the PM active count references -+ */ - kbasep_js_suspend(kbdev); -+#else -+ kbase_csf_scheduler_pm_suspend(kbdev); -+#endif - - /* Wait for the active count to reach zero. This is not the same as - * waiting for a power down, since not all policies power down when this -@@ -221,14 +227,16 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) - - /* Initial active call, to power on the GPU/cores if needed */ - #ifdef CONFIG_MALI_ARBITER_SUPPORT -- (void)kbase_pm_context_active_handle_suspend(kbdev, -- (arb_gpu_start ? -- KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED : -- KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE)); -+ if (kbase_pm_context_active_handle_suspend(kbdev, -+ (arb_gpu_start ? -+ KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED : -+ KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE))) -+ return; - #else - kbase_pm_context_active(kbdev); - #endif - -+#if !MALI_USE_CSF - /* Resume any blocked atoms (which may cause contexts to be scheduled in - * and dependent atoms to run) - */ -@@ -238,6 +246,9 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) - * atoms - */ - kbasep_js_resume(kbdev); -+#else -+ kbase_csf_scheduler_pm_resume(kbdev); -+#endif - - /* Matching idle call, to power off the GPU/cores if we didn't actually - * need it and the policy doesn't want it on -@@ -245,9 +256,15 @@ void kbase_pm_driver_resume(struct kbase_device *kbdev, bool arb_gpu_start) - kbase_pm_context_idle(kbdev); - - /* Re-enable GPU hardware counters */ -+#if MALI_USE_CSF -+ kbase_csf_scheduler_spin_lock(kbdev, &flags); -+ kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); -+ kbase_csf_scheduler_spin_unlock(kbdev, flags); -+#else - spin_lock_irqsave(&kbdev->hwaccess_lock, flags); - kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx); - spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+#endif - - /* Resume vinstr */ - kbase_vinstr_resume(kbdev->vinstr_ctx); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_pm.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_pm.h -index 257f959..980a8d1 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_pm.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_pm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,14 +17,9 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_kbase_pm.h - * Power management API definitions - */ - -@@ -35,6 +31,13 @@ - #define PM_ENABLE_IRQS 0x01 - #define PM_HW_ISSUES_DETECT 0x02 - -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+/* In the case that the GPU was granted by the Arbiter, it will have -+ * already been reset. The following flag ensures it is not reset -+ * twice. -+ */ -+#define PM_NO_RESET 0x04 -+#endif - - /** Initialize the power management framework. - * -@@ -59,12 +62,12 @@ int kbase_pm_powerup(struct kbase_device *kbdev, unsigned int flags); - - /** - * Halt the power management framework. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * - * Should ensure that no new interrupts are generated, - * but allow any currently running interrupt handlers to complete successfully. - * The GPU is forced off by the time this function returns, regardless of - * whether or not the active power policy asks for the GPU to be powered off. -- * -- * @param kbdev The kbase device structure for the device (must be a valid pointer) - */ - void kbase_pm_halt(struct kbase_device *kbdev); - -@@ -154,6 +157,7 @@ void kbase_pm_context_idle(struct kbase_device *kbdev); - /** - * Suspend the GPU and prevent any further register accesses to it from Kernel - * threads. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * This is called in response to an OS suspend event, and calls into the various - * kbase components to complete the suspend. -@@ -161,21 +165,18 @@ void kbase_pm_context_idle(struct kbase_device *kbdev); - * @note the mechanisms used here rely on all user-space threads being frozen - * by the OS before we suspend. Otherwise, an IOCTL could occur that powers up - * the GPU e.g. via atom submission. -- * -- * @param kbdev The kbase device structure for the device (must be a valid pointer) - */ - void kbase_pm_suspend(struct kbase_device *kbdev); - - /** - * Resume the GPU, allow register accesses to it, and resume running atoms on - * the GPU. -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * This is called in response to an OS resume event, and calls into the various - * kbase components to complete the resume. - * - * Also called when using VM arbiter, when GPU access has been granted. -- * -- * @param kbdev The kbase device structure for the device (must be a valid pointer) - */ - void kbase_pm_resume(struct kbase_device *kbdev); - -@@ -192,8 +193,7 @@ void kbase_pm_vsync_callback(int buffer_updated, void *data); - - /** - * kbase_pm_driver_suspend() - Put GPU and driver in suspend state -- * @param kbdev The kbase device structure for the device -- * (must be a valid pointer) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) - * - * Suspend the GPU and prevent any further register accesses to it from Kernel - * threads. -@@ -212,8 +212,8 @@ void kbase_pm_driver_suspend(struct kbase_device *kbdev); - - /** - * kbase_pm_driver_resume() - Put GPU and driver in resume -- * @param kbdev The kbase device structure for the device -- * (must be a valid pointer) -+ * @kbdev: The kbase device structure for the device (must be a valid pointer) -+ * @arb_gpu_start: Arbiter has notified we can use GPU - * - * Resume the GPU, allow register accesses to it, and resume running atoms on - * the GPU. -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c -index 1d114a6..1e807d7 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2016, 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2016, 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,18 +17,130 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase.h" -- - #include "mali_kbase_regs_history_debugfs.h" - --#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) -+#if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_NO_MALI) - - #include <linux/debugfs.h> - -+/** -+ * kbase_io_history_resize - resize the register access history buffer. -+ * -+ * @h: Pointer to a valid register history to resize -+ * @new_size: Number of accesses the buffer could hold -+ * -+ * A successful resize will clear all recent register accesses. -+ * If resizing fails for any reason (e.g., could not allocate memory, invalid -+ * buffer size) then the original buffer will be kept intact. -+ * -+ * @return 0 if the buffer was resized, failure otherwise -+ */ -+static int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size) -+{ -+ struct kbase_io_access *old_buf; -+ struct kbase_io_access *new_buf; -+ unsigned long flags; -+ -+ if (!new_size) -+ goto out_err; /* The new size must not be 0 */ -+ -+ new_buf = vmalloc(new_size * sizeof(*h->buf)); -+ if (!new_buf) -+ goto out_err; -+ -+ spin_lock_irqsave(&h->lock, flags); -+ -+ old_buf = h->buf; -+ -+ /* Note: we won't bother with copying the old data over. The dumping -+ * logic wouldn't work properly as it relies on 'count' both as a -+ * counter and as an index to the buffer which would have changed with -+ * the new array. This is a corner case that we don't need to support. -+ */ -+ h->count = 0; -+ h->size = new_size; -+ h->buf = new_buf; -+ -+ spin_unlock_irqrestore(&h->lock, flags); -+ -+ vfree(old_buf); -+ -+ return 0; -+ -+out_err: -+ return -1; -+} -+ -+int kbase_io_history_init(struct kbase_io_history *h, u16 n) -+{ -+ h->enabled = false; -+ spin_lock_init(&h->lock); -+ h->count = 0; -+ h->size = 0; -+ h->buf = NULL; -+ if (kbase_io_history_resize(h, n)) -+ return -1; -+ -+ return 0; -+} -+ -+void kbase_io_history_term(struct kbase_io_history *h) -+{ -+ vfree(h->buf); -+ h->buf = NULL; -+} -+ -+void kbase_io_history_add(struct kbase_io_history *h, -+ void __iomem const *addr, u32 value, u8 write) -+{ -+ struct kbase_io_access *io; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&h->lock, flags); -+ -+ io = &h->buf[h->count % h->size]; -+ io->addr = (uintptr_t)addr | write; -+ io->value = value; -+ ++h->count; -+ /* If count overflows, move the index by the buffer size so the entire -+ * buffer will still be dumped later -+ */ -+ if (unlikely(!h->count)) -+ h->count = h->size; -+ -+ spin_unlock_irqrestore(&h->lock, flags); -+} -+ -+void kbase_io_history_dump(struct kbase_device *kbdev) -+{ -+ struct kbase_io_history *const h = &kbdev->io_history; -+ size_t i; -+ size_t iters; -+ unsigned long flags; -+ -+ if (!unlikely(h->enabled)) -+ return; -+ -+ spin_lock_irqsave(&h->lock, flags); -+ -+ dev_err(kbdev->dev, "Register IO History:"); -+ iters = (h->size > h->count) ? h->count : h->size; -+ dev_err(kbdev->dev, "Last %zu register accesses of %zu total:\n", iters, -+ h->count); -+ for (i = 0; i < iters; ++i) { -+ struct kbase_io_access *io = -+ &h->buf[(h->count - iters + i) % h->size]; -+ char const access = (io->addr & 1) ? 'w' : 'r'; -+ -+ dev_err(kbdev->dev, "%6zu: %c: reg 0x%016lx val %08x\n", i, -+ access, (unsigned long)(io->addr & ~0x1), io->value); -+ } -+ -+ spin_unlock_irqrestore(&h->lock, flags); -+} - - static int regs_history_size_get(void *data, u64 *val) - { -@@ -66,7 +179,7 @@ DEFINE_SIMPLE_ATTRIBUTE(regs_history_size_fops, - static int regs_history_show(struct seq_file *sfile, void *data) - { - struct kbase_io_history *const h = sfile->private; -- u16 i; -+ size_t i; - size_t iters; - unsigned long flags; - -@@ -85,8 +198,8 @@ static int regs_history_show(struct seq_file *sfile, void *data) - &h->buf[(h->count - iters + i) % h->size]; - char const access = (io->addr & 1) ? 'w' : 'r'; - -- seq_printf(sfile, "%6i: %c: reg 0x%016lx val %08x\n", i, access, -- (unsigned long)(io->addr & ~0x1), io->value); -+ seq_printf(sfile, "%6zu: %c: reg 0x%016lx val %08x\n", i, -+ access, (unsigned long)(io->addr & ~0x1), io->value); - } - - spin_unlock_irqrestore(&h->lock, flags); -@@ -95,7 +208,6 @@ out: - return 0; - } - -- - /** - * regs_history_open - open operation for regs_history debugfs file - * -@@ -109,7 +221,6 @@ static int regs_history_open(struct inode *in, struct file *file) - return single_open(file, ®s_history_show, in->i_private); - } - -- - static const struct file_operations regs_history_fops = { - .owner = THIS_MODULE, - .open = ®s_history_open, -@@ -118,7 +229,6 @@ static const struct file_operations regs_history_fops = { - .release = single_release, - }; - -- - void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev) - { - debugfs_create_bool("regs_history_enabled", S_IRUGO | S_IWUSR, -@@ -131,6 +241,4 @@ void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev) - kbdev->mali_debugfs_directory, &kbdev->io_history, - ®s_history_fops); - } -- -- --#endif /* CONFIG_DEBUG_FS */ -+#endif /* defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_NO_MALI) */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h -index a0078cb..3b181d3 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2016 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2016, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -@@ -37,7 +36,31 @@ - - struct kbase_device; - --#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_NO_MALI) -+#if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_NO_MALI) -+ -+/** -+ * kbase_io_history_init - initialize data struct for register access history -+ * -+ * @h: The register history to initialize -+ * @n: The number of register accesses that the buffer could hold -+ * -+ * @return 0 if successfully initialized, failure otherwise -+ */ -+int kbase_io_history_init(struct kbase_io_history *h, u16 n); -+ -+/** -+ * kbase_io_history_term - uninit all resources for the register access history -+ * -+ * @h: The register history to terminate -+ */ -+void kbase_io_history_term(struct kbase_io_history *h); -+ -+/** -+ * kbase_io_history_dump - print the register history to the kernel ring buffer -+ * -+ * @kbdev: Pointer to kbase_device containing the register history to dump -+ */ -+void kbase_io_history_dump(struct kbase_device *kbdev); - - /** - * kbasep_regs_history_debugfs_init - add debugfs entries for register history -@@ -46,10 +69,7 @@ struct kbase_device; - */ - void kbasep_regs_history_debugfs_init(struct kbase_device *kbdev); - --#else /* CONFIG_DEBUG_FS */ -- --#define kbasep_regs_history_debugfs_init CSTD_NOP -- --#endif /* CONFIG_DEBUG_FS */ -+#else /* defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_NO_MALI) */ -+#endif /* defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_NO_MALI) */ - - #endif /*_KBASE_REGS_HISTORY_DEBUGFS_H*/ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_reset_gpu.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_reset_gpu.h -index df72eec..292a29c 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_reset_gpu.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_reset_gpu.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,16 +17,142 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_RESET_GPU_H_ - #define _KBASE_RESET_GPU_H_ - -+/** -+ * kbase_reset_gpu_prevent_and_wait - Prevent GPU resets from starting whilst -+ * the current thread is accessing the GPU, -+ * and wait for any in-flight reset to -+ * finish. -+ * @kbdev: Device pointer -+ * -+ * This should be used when a potential access to the HW is going to be made -+ * from a non-atomic context. -+ * -+ * It will wait for any in-flight reset to finish before returning. Hence, -+ * correct lock ordering must be observed with respect to the calling thread -+ * and the reset worker thread. -+ * -+ * This does not synchronize general access to the HW, and so multiple threads -+ * can prevent GPU reset concurrently, whilst not being serialized. This is -+ * advantageous as the threads can make this call at points where they do not -+ * know for sure yet whether they will indeed access the GPU (for example, to -+ * respect lock ordering), without unnecessarily blocking others. -+ * -+ * Threads must still use other synchronization to ensure they access the HW -+ * consistently, at a point where they are certain it needs to be accessed. -+ * -+ * On success, ensure that when access to the GPU by the caller thread has -+ * finished, that it calls kbase_reset_gpu_allow() again to allow resets to -+ * happen. -+ * -+ * This may return a failure in cases such as a previous failure to reset the -+ * GPU within a reasonable time. If that happens, the GPU might be -+ * non-operational and the caller should not attempt any further access. -+ * -+ * Note: -+ * For atomic context, instead check kbase_reset_gpu_is_active(). -+ * -+ * Return: 0 on success, or negative error code on failure. -+ */ -+int kbase_reset_gpu_prevent_and_wait(struct kbase_device *kbdev); -+ -+/** -+ * kbase_reset_gpu_try_prevent - Attempt to prevent GPU resets from starting -+ * whilst the current thread is accessing the -+ * GPU, unless a reset is already in progress. -+ * @kbdev: Device pointer -+ * -+ * Similar to kbase_reset_gpu_prevent_and_wait(), but it does not wait for an -+ * existing reset to complete. This can be used on codepaths that the Reset -+ * worker waits on, where use of kbase_reset_gpu_prevent_and_wait() would -+ * otherwise deadlock. -+ * -+ * Instead, a reset that is currently happening will cause this function to -+ * return an error code indicating that, and further resets will not have been -+ * prevented. -+ * -+ * In such cases, the caller must check for -EAGAIN, and take similar actions -+ * as for handling reset in atomic context. That is, they must cancel any -+ * actions that depended on reset being prevented, possibly deferring them -+ * until after the reset. -+ * -+ * Otherwise a successful return means that the caller can continue its actions -+ * safely in the knowledge that reset is prevented, and the reset worker will -+ * correctly wait instead of deadlocking against this thread. -+ * -+ * On success, ensure that when access to the GPU by the caller thread has -+ * finished, that it calls kbase_reset_gpu_allow() again to allow resets to -+ * happen. -+ * -+ * Refer to kbase_reset_gpu_prevent_and_wait() for more information. -+ * -+ * Return: 0 on success. -EAGAIN if a reset is currently happening. Other -+ * negative error codes on failure. -+ */ -+int kbase_reset_gpu_try_prevent(struct kbase_device *kbdev); -+ -+/** -+ * kbase_reset_gpu_allow - Allow GPU resets to happen again after having been -+ * previously prevented. -+ * @kbdev: Device pointer -+ * -+ * This should be used when a potential access to the HW has finished from a -+ * non-atomic context. -+ * -+ * It must be used from the same thread that originally made a previously call -+ * to kbase_reset_gpu_prevent_and_wait(). It must not be deferred to another -+ * thread. -+ */ -+void kbase_reset_gpu_allow(struct kbase_device *kbdev); -+ -+/** -+ * kbase_reset_gpu_assert_prevented - Make debugging checks that GPU reset is -+ * currently prevented by the current -+ * thread. -+ * @kbdev: Device pointer -+ * -+ * Make debugging checks that the current thread has made a call to -+ * kbase_reset_gpu_prevent_and_wait(), but has yet to make a subsequent call to -+ * kbase_reset_gpu_allow(). -+ * -+ * CONFIG_LOCKDEP is required to prove that reset is indeed -+ * prevented. Otherwise only limited debugging checks can be made. -+ */ -+void kbase_reset_gpu_assert_prevented(struct kbase_device *kbdev); -+ -+/** -+ * kbase_reset_gpu_assert_failed_or_prevented - Make debugging checks that -+ * either GPU reset previously -+ * failed, or is currently -+ * prevented. -+ * -+ * @kbdev: Device pointer -+ * -+ * As with kbase_reset_gpu_assert_prevented(), but also allow for paths where -+ * reset was not prevented due to a failure, yet we still need to execute the -+ * cleanup code following. -+ * -+ * Cleanup code following this call must handle any inconsistent state modified -+ * by the failed GPU reset, and must timeout any blocking operations instead of -+ * waiting forever. -+ */ -+void kbase_reset_gpu_assert_failed_or_prevented(struct kbase_device *kbdev); -+ -+/** -+ * Flags for kbase_prepare_to_reset_gpu -+ */ -+#define RESET_FLAGS_NONE ((unsigned int)0) -+/* This reset should be treated as an unrecoverable error by HW counter logic */ -+#define RESET_FLAGS_HWC_UNRECOVERABLE_ERROR ((unsigned int)(1 << 0)) -+ - /** - * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU. - * @kbdev: Device pointer -+ * @flags: Bitfield indicating impact of reset (see flag defines) - * - * Caller is expected to hold the kbdev->hwaccess_lock. - * -@@ -34,18 +161,20 @@ - * - false - Another thread is performing a reset, kbase_reset_gpu should - * not be called. - */ --bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev); -+bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev, -+ unsigned int flags); - - /** - * kbase_prepare_to_reset_gpu - Prepare for resetting the GPU. - * @kbdev: Device pointer -- * -+ * @flags: Bitfield indicating impact of reset (see flag defines) -+ - * Return: a boolean which should be interpreted as follows: - * - true - Prepared for reset, kbase_reset_gpu should be called. - * - false - Another thread is performing a reset, kbase_reset_gpu should - * not be called. - */ --bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev); -+bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev, unsigned int flags); - - /** - * kbase_reset_gpu - Reset the GPU -@@ -95,8 +224,13 @@ int kbase_reset_gpu_silent(struct kbase_device *kbdev); - * kbase_reset_gpu_is_active - Reports if the GPU is being reset - * @kbdev: Device pointer - * -- * Return: True if the GPU is in the process of being reset (or if the reset of -- * GPU failed, not applicable to Job Manager GPUs). -+ * Any changes made to the HW when this returns true may be lost, overwritten -+ * or corrupted. -+ * -+ * Note that unless appropriate locks are held when using this function, the -+ * state could change immediately afterwards. -+ * -+ * Return: True if the GPU is in the process of being reset. - */ - bool kbase_reset_gpu_is_active(struct kbase_device *kbdev); - -@@ -126,14 +260,4 @@ int kbase_reset_gpu_init(struct kbase_device *kbdev); - */ - void kbase_reset_gpu_term(struct kbase_device *kbdev); - --/** -- * kbase_reset_gpu_register_complete_cb - Register the callback function to be -- * invoked on completion of GPU reset. -- * -- * @kbdev: Device pointer -- * @complete_callback: Pointer to the callback function -- */ --void kbase_reset_gpu_register_complete_cb(struct kbase_device *kbdev, -- int (*complete_callback)(struct kbase_device *kbdev)); -- - #endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_smc.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_smc.c -index b5c7b12..abbe8d5 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_smc.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_smc.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2015, 2018, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,9 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifdef CONFIG_ARM64 -+#if IS_ENABLED(CONFIG_ARM64) - - #include <mali_kbase.h> - #include <mali_kbase_smc.h> -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_smc.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_smc.h -index 221eb21..d0086db 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_smc.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_smc.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,16 +17,12 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #ifndef _KBASE_SMC_H_ - #define _KBASE_SMC_H_ - --#ifdef CONFIG_ARM64 -+#if IS_ENABLED(CONFIG_ARM64) - - #include <mali_kbase.h> - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_softjobs.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_softjobs.c -index cbb0c76..bee3513 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_softjobs.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_softjobs.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #include <mali_kbase.h> - - #include <linux/dma-buf.h> -@@ -30,8 +27,9 @@ - #include <mali_kbase_sync.h> - #endif - #include <linux/dma-mapping.h> --#include <mali_base_kernel.h> -+#include <uapi/gpu/arm/midgard/mali_base_kernel.h> - #include <mali_kbase_hwaccess_time.h> -+#include <mali_kbase_kinstr_jm.h> - #include <mali_kbase_mem_linux.h> - #include <tl/mali_kbase_tracepoints.h> - #include <mali_linux_trace.h> -@@ -42,10 +40,9 @@ - #include <linux/kernel.h> - #include <linux/cache.h> - -+#if !MALI_USE_CSF - /** -- * @file mali_kbase_softjobs.c -- * -- * This file implements the logic behind software only jobs that are -+ * DOC: This file implements the logic behind software only jobs that are - * executed within the driver rather than being handed over to the GPU. - */ - -@@ -136,7 +133,7 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) - void *user_result; - struct timespec64 ts; - struct base_dump_cpu_gpu_counters data; -- u64 system_time; -+ u64 system_time = 0ULL; - u64 cycle_counter; - u64 jc = katom->jc; - struct kbase_context *kctx = katom->kctx; -@@ -146,7 +143,11 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) - - /* Take the PM active reference as late as possible - otherwise, it could - * delay suspend until we process the atom (which may be at the end of a -- * long chain of dependencies */ -+ * long chain of dependencies -+ */ -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ atomic_inc(&kctx->kbdev->pm.gpu_users_waiting); -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ - pm_active_err = kbase_pm_context_active_handle_suspend(kctx->kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE); - if (pm_active_err) { - struct kbasep_js_device_data *js_devdata = &kctx->kbdev->js_data; -@@ -164,6 +165,10 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) - - return pm_active_err; - } -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ else -+ atomic_dec(&kctx->kbdev->pm.gpu_users_waiting); -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ - - kbase_backend_get_gpu_time(kctx->kbdev, &cycle_counter, &system_time, - &ts); -@@ -181,7 +186,8 @@ static int kbase_dump_cpu_gpu_time(struct kbase_jd_atom *katom) - /* GPU_WR access is checked on the range for returning the result to - * userspace for the following reasons: - * - security, this is currently how imported user bufs are checked. -- * - userspace ddk guaranteed to assume region was mapped as GPU_WR */ -+ * - userspace ddk guaranteed to assume region was mapped as GPU_WR -+ */ - user_result = kbase_vmap_prot(kctx, jc, sizeof(data), KBASE_REG_GPU_WR, &map); - if (!user_result) - return 0; -@@ -292,7 +298,7 @@ static void kbase_fence_debug_check_atom(struct kbase_jd_atom *katom) - - if (!kbase_sync_fence_in_info_get(dep, &info)) { - dev_warn(dev, -- "\tVictim trigger atom %d fence [%p] %s: %s\n", -+ "\tVictim trigger atom %d fence [%pK] %s: %s\n", - kbase_jd_atom_id(kctx, dep), - info.fence, - info.name, -@@ -321,11 +327,11 @@ static void kbase_fence_debug_wait_timeout(struct kbase_jd_atom *katom) - return; - } - -- dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%p] after %dms\n", -+ dev_warn(dev, "ctx %d_%d: Atom %d still waiting for fence [%pK] after %dms\n", - kctx->tgid, kctx->id, - kbase_jd_atom_id(kctx, katom), - info.fence, timeout_ms); -- dev_warn(dev, "\tGuilty fence [%p] %s: %s\n", -+ dev_warn(dev, "\tGuilty fence [%pK] %s: %s\n", - info.fence, info.name, - kbase_sync_status_string(info.status)); - -@@ -713,14 +719,16 @@ out_unlock: - - out_cleanup: - /* Frees allocated memory for kbase_debug_copy_job struct, including -- * members, and sets jc to 0 */ -+ * members, and sets jc to 0 -+ */ - kbase_debug_copy_finish(katom); - kfree(user_buffers); - - return ret; - } -+#endif /* !MALI_USE_CSF */ - --#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) -+#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE - static void *dma_buf_kmap_page(struct kbase_mem_phy_alloc *gpu_alloc, - unsigned long page_num, struct page **page) - { -@@ -801,16 +809,16 @@ int kbase_mem_copy_from_extres(struct kbase_context *kctx, - dma_to_copy = min(dma_buf->size, - (size_t)(buf_data->nr_extres_pages * PAGE_SIZE)); - ret = dma_buf_begin_cpu_access(dma_buf, --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) -- 0, dma_to_copy, -+#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS) -+ 0, dma_to_copy, - #endif -- DMA_FROM_DEVICE); -+ DMA_FROM_DEVICE); - if (ret) - goto out_unlock; - - for (i = 0; i < dma_to_copy/PAGE_SIZE && - target_page_nr < buf_data->nr_pages; i++) { --#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) -+#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE - struct page *pg; - void *extres_page = dma_buf_kmap_page(gpu_alloc, i, &pg); - #else -@@ -822,20 +830,20 @@ int kbase_mem_copy_from_extres(struct kbase_context *kctx, - buf_data->nr_pages, - &target_page_nr, offset); - --#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0) -+#if KERNEL_VERSION(5, 6, 0) <= LINUX_VERSION_CODE - kunmap(pg); - #else - dma_buf_kunmap(dma_buf, i, extres_page); - #endif - if (ret) -- goto out_unlock; -+ break; - } - } - dma_buf_end_cpu_access(dma_buf, --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 6, 0) && !defined(CONFIG_CHROMEOS) -- 0, dma_to_copy, -+#if KERNEL_VERSION(4, 6, 0) > LINUX_VERSION_CODE && !defined(CONFIG_CHROMEOS) -+ 0, dma_to_copy, - #endif -- DMA_FROM_DEVICE); -+ DMA_FROM_DEVICE); - break; - } - default: -@@ -846,6 +854,7 @@ out_unlock: - return ret; - } - -+#if !MALI_USE_CSF - static int kbase_debug_copy(struct kbase_jd_atom *katom) - { - struct kbase_debug_copy_buffer *buffers = katom->softjob_data; -@@ -863,6 +872,7 @@ static int kbase_debug_copy(struct kbase_jd_atom *katom) - - return 0; - } -+#endif /* !MALI_USE_CSF */ - - #define KBASEP_JIT_ALLOC_GPU_ADDR_ALIGNMENT ((u32)0x7) - -@@ -899,7 +909,7 @@ int kbasep_jit_alloc_validate(struct kbase_context *kctx, - if (info->flags & ~(BASE_JIT_ALLOC_VALID_FLAGS)) - return -EINVAL; - --#if !MALI_JIT_PRESSURE_LIMIT -+#if !MALI_JIT_PRESSURE_LIMIT_BASE - /* If just-in-time memory allocation pressure limit feature is disabled, - * heap_info_gpu_addr must be zeroed-out - */ -@@ -907,21 +917,19 @@ int kbasep_jit_alloc_validate(struct kbase_context *kctx, - return -EINVAL; - #endif - -+#if !MALI_USE_CSF - /* If BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE is set, heap_info_gpu_addr - * cannot be 0 - */ - if ((info->flags & BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) && - !info->heap_info_gpu_addr) - return -EINVAL; -+#endif /* !MALI_USE_CSF */ - - return 0; - } - -- --#if (KERNEL_VERSION(3, 18, 63) > LINUX_VERSION_CODE) --#define offsetofend(TYPE, MEMBER) \ -- (offsetof(TYPE, MEMBER) + sizeof(((TYPE *)0)->MEMBER)) --#endif -+#if !MALI_USE_CSF - - /* - * Sizes of user data to copy for each just-in-time memory interface version -@@ -998,10 +1006,10 @@ static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) - ret = kbasep_jit_alloc_validate(kctx, info); - if (ret) - goto free_info; -- KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO(kbdev, katom, -- info->va_pages, info->commit_pages, info->extent, -- info->id, info->bin_id, info->max_allocations, -- info->flags, info->usage_id); -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_JITALLOCINFO( -+ kbdev, katom, info->va_pages, info->commit_pages, -+ info->extension, info->id, info->bin_id, -+ info->max_allocations, info->flags, info->usage_id); - } - - katom->jit_blocked = false; -@@ -1016,7 +1024,7 @@ static int kbase_jit_allocate_prepare(struct kbase_jd_atom *katom) - * though the region is valid it doesn't represent the - * same thing it used to. - * -- * Complete validation of va_pages, commit_pages and extent -+ * Complete validation of va_pages, commit_pages and extension - * isn't done here as it will be done during the call to - * kbase_mem_alloc. - */ -@@ -1091,14 +1099,19 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) - } - } - --#if MALI_JIT_PRESSURE_LIMIT -- /** -- * If this is the only JIT_ALLOC atom in-flight then allow it to exceed -- * the defined pressure limit. -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ /* -+ * If this is the only JIT_ALLOC atom in-flight or if JIT pressure limit -+ * is disabled at the context scope, then bypass JIT pressure limit -+ * logic in kbase_jit_allocate(). - */ -- if (kctx->jit_current_allocations == 0) -+ if (!kbase_ctx_flag(kctx, KCTX_JPL_ENABLED) -+ || (kctx->jit_current_allocations == 0)) { - ignore_pressure_limit = true; --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+ } -+#else -+ ignore_pressure_limit = true; -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - for (i = 0, info = katom->softjob_data; i < count; i++, info++) { - if (kctx->jit_alloc[info->id]) { -@@ -1215,10 +1228,10 @@ static int kbase_jit_allocate_process(struct kbase_jd_atom *katom) - MIDGARD_MMU_BOTTOMLEVEL, kctx->jit_group_id); - #endif - -- KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT(kbdev, katom, -- info->gpu_alloc_addr, new_addr, info->flags, -- entry_mmu_flags, info->id, info->commit_pages, -- info->extent, info->va_pages); -+ KBASE_TLSTREAM_TL_ATTRIB_ATOM_JIT( -+ kbdev, katom, info->gpu_alloc_addr, new_addr, -+ info->flags, entry_mmu_flags, info->id, -+ info->commit_pages, info->extension, info->va_pages); - kbase_vunmap(kctx, &mapping); - - kbase_trace_jit_report_gpu_mem(kctx, reg, -@@ -1358,12 +1371,16 @@ void kbase_jit_retry_pending_alloc(struct kbase_context *kctx) - list_for_each_safe(i, tmp, &jit_pending_alloc_list) { - struct kbase_jd_atom *pending_atom = list_entry(i, - struct kbase_jd_atom, queue); -+ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kctx->kbdev, pending_atom); -+ kbase_kinstr_jm_atom_sw_start(pending_atom); - if (kbase_jit_allocate_process(pending_atom) == 0) { - /* Atom has completed */ - INIT_WORK(&pending_atom->work, - kbasep_jit_finish_worker); - queue_work(kctx->jctx.job_done_wq, &pending_atom->work); - } -+ KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kctx->kbdev, pending_atom); -+ kbase_kinstr_jm_atom_sw_stop(pending_atom); - } - } - -@@ -1412,41 +1429,27 @@ static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) - struct base_external_resource_list *ext_res; - u64 count = 0; - size_t copy_size; -- int ret; - - user_ext_res = (__user struct base_external_resource_list *) - (uintptr_t) katom->jc; - - /* Fail the job if there is no info structure */ -- if (!user_ext_res) { -- ret = -EINVAL; -- goto fail; -- } -+ if (!user_ext_res) -+ return -EINVAL; - -- if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) { -- ret = -EINVAL; -- goto fail; -- } -+ if (copy_from_user(&count, &user_ext_res->count, sizeof(u64)) != 0) -+ return -EINVAL; - - /* Is the number of external resources in range? */ -- if (!count || count > BASE_EXT_RES_COUNT_MAX) { -- ret = -EINVAL; -- goto fail; -- } -+ if (!count || count > BASE_EXT_RES_COUNT_MAX) -+ return -EINVAL; - - /* Copy the information for safe access and future storage */ - copy_size = sizeof(*ext_res); - copy_size += sizeof(struct base_external_resource) * (count - 1); -- ext_res = kzalloc(copy_size, GFP_KERNEL); -- if (!ext_res) { -- ret = -ENOMEM; -- goto fail; -- } -- -- if (copy_from_user(ext_res, user_ext_res, copy_size) != 0) { -- ret = -EINVAL; -- goto free_info; -- } -+ ext_res = memdup_user(user_ext_res, copy_size); -+ if (IS_ERR(ext_res)) -+ return PTR_ERR(ext_res); - - /* - * Overwrite the count with the first value incase it was changed -@@ -1457,11 +1460,6 @@ static int kbase_ext_res_prepare(struct kbase_jd_atom *katom) - katom->softjob_data = ext_res; - - return 0; -- --free_info: -- kfree(ext_res); --fail: -- return ret; - } - - static void kbase_ext_res_process(struct kbase_jd_atom *katom, bool map) -@@ -1538,6 +1536,7 @@ int kbase_process_soft_job(struct kbase_jd_atom *katom) - struct kbase_device *kbdev = kctx->kbdev; - - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_START(kbdev, katom); -+ kbase_kinstr_jm_atom_sw_start(katom); - - trace_sysgraph(SGR_SUBMIT, kctx->id, - kbase_jd_atom_id(kctx, katom)); -@@ -1600,6 +1599,7 @@ int kbase_process_soft_job(struct kbase_jd_atom *katom) - - /* Atom is complete */ - KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTJOB_END(kbdev, katom); -+ kbase_kinstr_jm_atom_sw_stop(katom); - return ret; - } - -@@ -1635,7 +1635,9 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) - struct base_fence fence; - int fd; - -- if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) -+ if (copy_from_user(&fence, -+ (__user void *)(uintptr_t)katom->jc, -+ sizeof(fence)) != 0) - return -EINVAL; - - fd = kbase_sync_fence_out_create(katom, -@@ -1644,7 +1646,8 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) - return -EINVAL; - - fence.basep.fd = fd; -- if (0 != copy_to_user((__user void *)(uintptr_t) katom->jc, &fence, sizeof(fence))) { -+ if (copy_to_user((__user void *)(uintptr_t)katom->jc, -+ &fence, sizeof(fence)) != 0) { - kbase_sync_fence_out_remove(katom); - kbase_sync_fence_close_fd(fd); - fence.basep.fd = -EINVAL; -@@ -1657,7 +1660,9 @@ int kbase_prepare_soft_job(struct kbase_jd_atom *katom) - struct base_fence fence; - int ret; - -- if (0 != copy_from_user(&fence, (__user void *)(uintptr_t) katom->jc, sizeof(fence))) -+ if (copy_from_user(&fence, -+ (__user void *)(uintptr_t)katom->jc, -+ sizeof(fence)) != 0) - return -EINVAL; - - /* Get a reference to the fence object */ -@@ -1776,6 +1781,9 @@ void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) - if (kbase_process_soft_job(katom_iter) == 0) { - kbase_finish_soft_job(katom_iter); - resched |= jd_done_nolock(katom_iter, NULL); -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ atomic_dec(&kbdev->pm.gpu_users_waiting); -+#endif /* CONFIG_MALI_ARBITER_SUPPORT */ - } - mutex_unlock(&kctx->jctx.lock); - } -@@ -1783,3 +1791,4 @@ void kbase_resume_suspended_soft_jobs(struct kbase_device *kbdev) - if (resched) - kbase_js_sched_all(kbdev); - } -+#endif /* !MALI_USE_CSF */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_strings.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_strings.c -index 22caa4a..84784be 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_strings.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_strings.c -@@ -1,11 +1,12 @@ -- /* -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* - * -- * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2016, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,9 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ -+ - #include "mali_kbase_strings.h" - - #define KBASE_DRV_NAME "mali" -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_strings.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_strings.h -index d2f1825..c3f94f9 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_strings.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_strings.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2016, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - extern const char kbase_drv_name[]; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync.h -index 80b54d0..ad05cdf 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,14 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -- * @file mali_kbase_sync.h -- * -- * This file contains our internal "API" for explicit fences. -+ * DOC: This file contains our internal "API" for explicit fences. - * It hides the implementation details of the actual explicit fence mechanism - * used (Android fences or sync file with DMA fences). - */ -@@ -31,11 +28,12 @@ - #ifndef MALI_KBASE_SYNC_H - #define MALI_KBASE_SYNC_H - -+#include <linux/fdtable.h> - #include <linux/syscalls.h> --#ifdef CONFIG_SYNC -+#if IS_ENABLED(CONFIG_SYNC) - #include <sync.h> - #endif --#ifdef CONFIG_SYNC_FILE -+#if IS_ENABLED(CONFIG_SYNC_FILE) - #include "mali_kbase_fence_defs.h" - #include <linux/sync_file.h> - #endif -@@ -72,6 +70,7 @@ struct kbase_sync_fence_info { - */ - int kbase_sync_fence_stream_create(const char *name, int *const out_fd); - -+#if !MALI_USE_CSF - /** - * kbase_sync_fence_out_create Create an explicit output fence to specified atom - * @katom: Atom to assign the new explicit fence to -@@ -92,6 +91,7 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd); - * return: 0 on success, < 0 on error - */ - int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); -+#endif /* !MALI_USE_CSF */ - - /** - * kbase_sync_fence_validate() - Validate a fd to be a valid fence -@@ -104,6 +104,7 @@ int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd); - */ - int kbase_sync_fence_validate(int fd); - -+#if !MALI_USE_CSF - /** - * kbase_sync_fence_out_trigger - Signal explicit output fence attached on katom - * @katom: Atom with an explicit fence to signal -@@ -154,6 +155,7 @@ void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom); - * This will also release the corresponding reference. - */ - void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); -+#endif /* !MALI_USE_CSF */ - - /** - * kbase_sync_fence_close_fd() - Close a file descriptor representing a fence -@@ -161,13 +163,16 @@ void kbase_sync_fence_out_remove(struct kbase_jd_atom *katom); - */ - static inline void kbase_sync_fence_close_fd(int fd) - { --#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 17, 0) -+#if KERNEL_VERSION(5, 11, 0) <= LINUX_VERSION_CODE -+ close_fd(fd); -+#elif KERNEL_VERSION(4, 17, 0) <= LINUX_VERSION_CODE - ksys_close(fd); - #else - sys_close(fd); - #endif - } - -+#if !MALI_USE_CSF - /** - * kbase_sync_fence_in_info_get() - Retrieves information about input fence - * @katom: Atom to get fence information from -@@ -187,6 +192,7 @@ int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, - */ - int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, - struct kbase_sync_fence_info *info); -+#endif /* !MALI_USE_CSF */ - - #if defined(CONFIG_SYNC_FILE) - #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) -@@ -207,6 +213,7 @@ void kbase_sync_fence_info_get(struct dma_fence *fence, - const char *kbase_sync_status_string(int status); - - -+#if !MALI_USE_CSF - /* - * Internal worker used to continue processing of atom. - */ -@@ -219,5 +226,6 @@ void kbase_sync_fence_wait_worker(struct work_struct *data); - */ - void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom); - #endif -+#endif /* !MALI_USE_CSF */ - - #endif /* MALI_KBASE_SYNC_H */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_android.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_android.c -index 75940fb..8af2584 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_android.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_android.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -50,15 +49,6 @@ struct mali_sync_pt { - int result; - }; - --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) --/* For backwards compatibility with kernels before 3.17. After 3.17 -- * sync_pt_parent is included in the kernel. */ --static inline struct sync_timeline *sync_pt_parent(struct sync_pt *pt) --{ -- return pt->parent; --} --#endif -- - static struct mali_sync_timeline *to_mali_sync_timeline( - struct sync_timeline *timeline) - { -@@ -196,6 +186,7 @@ int kbase_sync_fence_stream_create(const char *name, int *const out_fd) - return 0; - } - -+#if !MALI_USE_CSF - /* Allocates a sync point within the timeline. - * - * The timeline must be the one allocated by kbase_sync_timeline_alloc -@@ -225,10 +216,6 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) - struct sync_timeline *tl; - struct sync_pt *pt; - struct sync_fence *fence; --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 7, 0) -- struct files_struct *files; -- struct fdtable *fdt; --#endif - int fd; - struct file *tl_file; - -@@ -259,29 +246,11 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) - /* from here the fence owns the sync_pt */ - - /* create a fd representing the fence */ --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) - fd = get_unused_fd_flags(O_RDWR | O_CLOEXEC); - if (fd < 0) { - sync_fence_put(fence); - goto out; - } --#else -- fd = get_unused_fd(); -- if (fd < 0) { -- sync_fence_put(fence); -- goto out; -- } -- -- files = current->files; -- spin_lock(&files->file_lock); -- fdt = files_fdtable(files); --#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0) -- __set_close_on_exec(fd, fdt); --#else -- FD_SET(fd, fdt->close_on_exec); --#endif -- spin_unlock(&files->file_lock); --#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(3, 7, 0) */ - - /* bind fence to the new fd */ - sync_fence_install(fence, fd); -@@ -289,7 +258,8 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int tl_fd) - katom->fence = sync_fence_fdget(fd); - if (katom->fence == NULL) { - /* The only way the fence can be NULL is if userspace closed it -- * for us, so we don't need to clear it up */ -+ * for us, so we don't need to clear it up -+ */ - fd = -EINVAL; - goto out; - } -@@ -305,6 +275,7 @@ int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) - katom->fence = sync_fence_fdget(fd); - return katom->fence ? 0 : -ENOENT; - } -+#endif /* !MALI_USE_CSF */ - - int kbase_sync_fence_validate(int fd) - { -@@ -318,6 +289,7 @@ int kbase_sync_fence_validate(int fd) - return 0; - } - -+#if !MALI_USE_CSF - /* Returns true if the specified timeline is allocated by Mali */ - static int kbase_sync_timeline_is_ours(struct sync_timeline *timeline) - { -@@ -376,22 +348,14 @@ kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) - if (!katom->fence) - return BASE_JD_EVENT_JOB_CANCELLED; - --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -- if (!list_is_singular(&katom->fence->pt_list_head)) { --#else - if (katom->fence->num_fences != 1) { --#endif - /* Not exactly one item in the list - so it didn't (directly) -- * come from us */ -+ * come from us -+ */ - return BASE_JD_EVENT_JOB_CANCELLED; - } - --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -- pt = list_first_entry(&katom->fence->pt_list_head, -- struct sync_pt, pt_list); --#else - pt = container_of(katom->fence->cbs[0].sync_pt, struct sync_pt, base); --#endif - timeline = sync_pt_parent(pt); - - if (!kbase_sync_timeline_is_ours(timeline)) { -@@ -413,11 +377,7 @@ static inline int kbase_fence_get_status(struct sync_fence *fence) - if (!fence) - return -ENOENT; - --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 17, 0) -- return fence->status; --#else - return atomic_read(&fence->status); --#endif - } - - static void kbase_fence_wait_callback(struct sync_fence *fence, -@@ -461,7 +421,8 @@ int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) - if (ret < 0) { - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - /* We should cause the dependent jobs in the bag to be failed, -- * to do this we schedule the work queue to complete this job */ -+ * to do this we schedule the work queue to complete this job -+ */ - INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); - queue_work(katom->kctx->jctx.job_done_wq, &katom->work); - } -@@ -473,7 +434,8 @@ void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) - { - if (sync_fence_cancel_async(katom->fence, &katom->sync_waiter) != 0) { - /* The wait wasn't cancelled - leave the cleanup for -- * kbase_fence_wait_callback */ -+ * kbase_fence_wait_callback -+ */ - return; - } - -@@ -540,3 +502,4 @@ void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) - sync_fence_wait(katom->fence, 1); - } - #endif -+#endif /* !MALI_USE_CSF */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_common.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_common.c -index 2e1ede5..5ee7fc3 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_common.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_common.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2016, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -- * @file mali_kbase_sync_common.c -+ * @file - * - * Common code for our explicit fence functionality - */ -@@ -30,6 +29,7 @@ - #include "mali_kbase.h" - #include "mali_kbase_sync.h" - -+#if !MALI_USE_CSF - void kbase_sync_fence_wait_worker(struct work_struct *data) - { - struct kbase_jd_atom *katom; -@@ -37,6 +37,7 @@ void kbase_sync_fence_wait_worker(struct work_struct *data) - katom = container_of(data, struct kbase_jd_atom, work); - kbase_soft_event_wait_callback(katom); - } -+#endif /* !MALI_USE_CSF */ - - const char *kbase_sync_status_string(int status) - { -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_file.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_file.c -index 0679c48..25670c4 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_file.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_sync_file.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2012-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -59,9 +58,10 @@ int kbase_sync_fence_stream_create(const char *name, int *const out_fd) - return 0; - } - -+#if !MALI_USE_CSF - int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) - { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; - #else - struct dma_fence *fence; -@@ -106,7 +106,7 @@ int kbase_sync_fence_out_create(struct kbase_jd_atom *katom, int stream_fd) - - int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) - { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence = sync_file_get_fence(fd); - #else - struct dma_fence *fence = sync_file_get_fence(fd); -@@ -119,10 +119,11 @@ int kbase_sync_fence_in_from_fd(struct kbase_jd_atom *katom, int fd) - - return 0; - } -+#endif /* !MALI_USE_CSF */ - - int kbase_sync_fence_validate(int fd) - { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence = sync_file_get_fence(fd); - #else - struct dma_fence *fence = sync_file_get_fence(fd); -@@ -136,6 +137,7 @@ int kbase_sync_fence_validate(int fd) - return 0; /* valid */ - } - -+#if !MALI_USE_CSF - enum base_jd_event_code - kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) - { -@@ -157,7 +159,7 @@ kbase_sync_fence_out_trigger(struct kbase_jd_atom *katom, int result) - return (result != 0) ? BASE_JD_EVENT_JOB_CANCELLED : BASE_JD_EVENT_DONE; - } - --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - static void kbase_fence_wait_callback(struct fence *fence, - struct fence_cb *cb) - #else -@@ -175,7 +177,7 @@ static void kbase_fence_wait_callback(struct dma_fence *fence, - #if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE || \ - (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE && \ - KERNEL_VERSION(4, 9, 68) <= LINUX_VERSION_CODE)) -- if (dma_fence_is_signaled(kcb->fence) && kcb->fence->error) -+ if (dma_fence_is_signaled(kcb->fence) && kcb->fence->error < 0) - #else - if (dma_fence_is_signaled(kcb->fence) && kcb->fence->status < 0) - #endif -@@ -200,7 +202,7 @@ static void kbase_fence_wait_callback(struct dma_fence *fence, - int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) - { - int err; --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; - #else - struct dma_fence *fence; -@@ -233,8 +235,8 @@ int kbase_sync_fence_in_wait(struct kbase_jd_atom *katom) - katom->event_code = BASE_JD_EVENT_JOB_CANCELLED; - - /* We should cause the dependent jobs in the bag to be failed, -- * to do this we schedule the work queue to complete this job */ -- -+ * to do this we schedule the work queue to complete this job -+ */ - INIT_WORK(&katom->work, kbase_sync_fence_wait_worker); - queue_work(katom->kctx->jctx.job_done_wq, &katom->work); - } -@@ -246,7 +248,8 @@ void kbase_sync_fence_in_cancel_wait(struct kbase_jd_atom *katom) - { - if (!kbase_fence_free_callbacks(katom)) { - /* The wait wasn't cancelled - -- * leave the cleanup for kbase_fence_wait_callback */ -+ * leave the cleanup for kbase_fence_wait_callback -+ */ - return; - } - -@@ -273,6 +276,7 @@ void kbase_sync_fence_in_remove(struct kbase_jd_atom *katom) - kbase_fence_free_callbacks(katom); - kbase_fence_in_remove(katom); - } -+#endif /* !MALI_USE_CSF */ - - #if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - void kbase_sync_fence_info_get(struct fence *fence, -@@ -317,10 +321,11 @@ void kbase_sync_fence_info_get(struct dma_fence *fence, - #endif - } - -+#if !MALI_USE_CSF - int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, - struct kbase_sync_fence_info *info) - { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; - #else - struct dma_fence *fence; -@@ -340,7 +345,7 @@ int kbase_sync_fence_in_info_get(struct kbase_jd_atom *katom, - int kbase_sync_fence_out_info_get(struct kbase_jd_atom *katom, - struct kbase_sync_fence_info *info) - { --#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0)) -+#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - struct fence *fence; - #else - struct dma_fence *fence; -@@ -364,3 +369,4 @@ void kbase_sync_fence_in_dump(struct kbase_jd_atom *katom) - /* Not implemented */ - } - #endif -+#endif /* !MALI_USE_CSF*/ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_trace_gpu_mem.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_trace_gpu_mem.c -new file mode 100644 -index 0000000..3088c41 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_trace_gpu_mem.c -@@ -0,0 +1,221 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <mali_kbase_mem_linux.h> -+#include <mali_kbase_defs.h> -+#include <mali_kbase_trace_gpu_mem.h> -+ -+/** -+ * struct kbase_dma_buf - Object instantiated when a dma-buf imported allocation -+ * is mapped to GPU for the first time within a process. -+ * Another instantiation is done for the case when that -+ * allocation is mapped for the first time to GPU. -+ * -+ * @dma_buf: Reference to dma_buf been imported. -+ * @dma_buf_node: Link node to maintain a rb_tree of kbase_dma_buf. -+ * @import_count: The number of times the dma_buf was imported. -+ */ -+struct kbase_dma_buf { -+ struct dma_buf *dma_buf; -+ struct rb_node dma_buf_node; -+ u32 import_count; -+}; -+ -+/** -+ * kbase_delete_dma_buf_mapping - Delete a dma buffer mapping. -+ * -+ * @kctx: Pointer to kbase context. -+ * @dma_buf: Pointer to a dma buffer mapping. -+ * @tree: Pointer to root of rb_tree containing the dma_buf's mapped. -+ * -+ * when we un-map any dma mapping we need to remove them from rb_tree, -+ * rb_tree is maintained at kbase_device level and kbase_process level -+ * by passing the root of kbase_device or kbase_process we can remove -+ * the node from the tree. -+ */ -+static bool kbase_delete_dma_buf_mapping(struct kbase_context *kctx, -+ struct dma_buf *dma_buf, -+ struct rb_root *tree) -+{ -+ struct kbase_dma_buf *buf_node = NULL; -+ struct rb_node *node = tree->rb_node; -+ bool mapping_removed = false; -+ -+ lockdep_assert_held(&kctx->kbdev->dma_buf_lock); -+ -+ while (node) { -+ buf_node = rb_entry(node, struct kbase_dma_buf, dma_buf_node); -+ -+ if (dma_buf == buf_node->dma_buf) { -+ WARN_ON(!buf_node->import_count); -+ -+ buf_node->import_count--; -+ -+ if (!buf_node->import_count) { -+ rb_erase(&buf_node->dma_buf_node, tree); -+ kfree(buf_node); -+ mapping_removed = true; -+ } -+ -+ break; -+ } -+ -+ if (dma_buf < buf_node->dma_buf) -+ node = node->rb_left; -+ else -+ node = node->rb_right; -+ } -+ -+ WARN_ON(!buf_node); -+ return mapping_removed; -+} -+ -+/** -+ * kbase_capture_dma_buf_mapping - capture a dma buffer mapping. -+ * -+ * @kctx: Pointer to kbase context. -+ * @dma_buf: Pointer to a dma buffer mapping. -+ * @root: Pointer to root of rb_tree containing the dma_buf's. -+ * -+ * We maintain a kbase_device level and kbase_process level rb_tree -+ * of all unique dma_buf's mapped to gpu memory. So when attach any -+ * dma_buf add it the rb_tree's. To add the unique mapping we need -+ * check if the mapping is not a duplicate and then add them. -+ */ -+static bool kbase_capture_dma_buf_mapping(struct kbase_context *kctx, -+ struct dma_buf *dma_buf, -+ struct rb_root *root) -+{ -+ struct kbase_dma_buf *buf_node = NULL; -+ struct rb_node *node = root->rb_node; -+ bool unique_buf_imported = true; -+ -+ lockdep_assert_held(&kctx->kbdev->dma_buf_lock); -+ -+ while (node) { -+ buf_node = rb_entry(node, struct kbase_dma_buf, dma_buf_node); -+ -+ if (dma_buf == buf_node->dma_buf) { -+ unique_buf_imported = false; -+ break; -+ } -+ -+ if (dma_buf < buf_node->dma_buf) -+ node = node->rb_left; -+ else -+ node = node->rb_right; -+ } -+ -+ if (unique_buf_imported) { -+ struct kbase_dma_buf *new_buf_node = -+ kzalloc(sizeof(*new_buf_node), GFP_KERNEL); -+ -+ if (new_buf_node == NULL) { -+ dev_err(kctx->kbdev->dev, "Error allocating memory for kbase_dma_buf\n"); -+ /* Dont account for it if we fail to allocate memory */ -+ unique_buf_imported = false; -+ } else { -+ struct rb_node **new = &(root->rb_node), *parent = NULL; -+ -+ new_buf_node->dma_buf = dma_buf; -+ new_buf_node->import_count = 1; -+ while (*new) { -+ struct kbase_dma_buf *new_node; -+ -+ parent = *new; -+ new_node = rb_entry(parent, struct kbase_dma_buf, -+ dma_buf_node); -+ if (dma_buf < new_node->dma_buf) -+ new = &(*new)->rb_left; -+ else -+ new = &(*new)->rb_right; -+ } -+ rb_link_node(&new_buf_node->dma_buf_node, parent, new); -+ rb_insert_color(&new_buf_node->dma_buf_node, root); -+ } -+ } else if (!WARN_ON(!buf_node)) { -+ buf_node->import_count++; -+ } -+ -+ return unique_buf_imported; -+} -+ -+void kbase_remove_dma_buf_usage(struct kbase_context *kctx, -+ struct kbase_mem_phy_alloc *alloc) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ bool dev_mapping_removed, prcs_mapping_removed; -+ -+ mutex_lock(&kbdev->dma_buf_lock); -+ -+ dev_mapping_removed = kbase_delete_dma_buf_mapping( -+ kctx, alloc->imported.umm.dma_buf, &kbdev->dma_buf_root); -+ -+ prcs_mapping_removed = kbase_delete_dma_buf_mapping( -+ kctx, alloc->imported.umm.dma_buf, &kctx->kprcs->dma_buf_root); -+ -+ WARN_ON(dev_mapping_removed && !prcs_mapping_removed); -+ -+ spin_lock(&kbdev->gpu_mem_usage_lock); -+ if (dev_mapping_removed) -+ kbdev->total_gpu_pages -= alloc->nents; -+ -+ if (prcs_mapping_removed) -+ kctx->kprcs->total_gpu_pages -= alloc->nents; -+ -+ if (dev_mapping_removed || prcs_mapping_removed) -+ kbase_trace_gpu_mem_usage(kbdev, kctx); -+ spin_unlock(&kbdev->gpu_mem_usage_lock); -+ -+ mutex_unlock(&kbdev->dma_buf_lock); -+} -+ -+void kbase_add_dma_buf_usage(struct kbase_context *kctx, -+ struct kbase_mem_phy_alloc *alloc) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ bool unique_dev_dmabuf, unique_prcs_dmabuf; -+ -+ mutex_lock(&kbdev->dma_buf_lock); -+ -+ /* add dma_buf to device and process. */ -+ unique_dev_dmabuf = kbase_capture_dma_buf_mapping( -+ kctx, alloc->imported.umm.dma_buf, &kbdev->dma_buf_root); -+ -+ unique_prcs_dmabuf = kbase_capture_dma_buf_mapping( -+ kctx, alloc->imported.umm.dma_buf, &kctx->kprcs->dma_buf_root); -+ -+ WARN_ON(unique_dev_dmabuf && !unique_prcs_dmabuf); -+ -+ spin_lock(&kbdev->gpu_mem_usage_lock); -+ if (unique_dev_dmabuf) -+ kbdev->total_gpu_pages += alloc->nents; -+ -+ if (unique_prcs_dmabuf) -+ kctx->kprcs->total_gpu_pages += alloc->nents; -+ -+ if (unique_prcs_dmabuf || unique_dev_dmabuf) -+ kbase_trace_gpu_mem_usage(kbdev, kctx); -+ spin_unlock(&kbdev->gpu_mem_usage_lock); -+ -+ mutex_unlock(&kbdev->dma_buf_lock); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_trace_gpu_mem.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_trace_gpu_mem.h -new file mode 100644 -index 0000000..fd871fc ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_trace_gpu_mem.h -@@ -0,0 +1,100 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KBASE_TRACE_GPU_MEM_H_ -+#define _KBASE_TRACE_GPU_MEM_H_ -+ -+#if IS_ENABLED(CONFIG_TRACE_GPU_MEM) -+#include <trace/events/gpu_mem.h> -+#endif -+ -+#define DEVICE_TGID ((u32) 0U) -+ -+static void kbase_trace_gpu_mem_usage(struct kbase_device *kbdev, -+ struct kbase_context *kctx) -+{ -+#if IS_ENABLED(CONFIG_TRACE_GPU_MEM) -+ lockdep_assert_held(&kbdev->gpu_mem_usage_lock); -+ -+ trace_gpu_mem_total(kbdev->id, DEVICE_TGID, -+ kbdev->total_gpu_pages << PAGE_SHIFT); -+ -+ if (likely(kctx)) -+ trace_gpu_mem_total(kbdev->id, kctx->kprcs->tgid, -+ kctx->kprcs->total_gpu_pages << PAGE_SHIFT); -+#endif -+} -+ -+static inline void kbase_trace_gpu_mem_usage_dec(struct kbase_device *kbdev, -+ struct kbase_context *kctx, size_t pages) -+{ -+ spin_lock(&kbdev->gpu_mem_usage_lock); -+ -+ if (likely(kctx)) -+ kctx->kprcs->total_gpu_pages -= pages; -+ -+ kbdev->total_gpu_pages -= pages; -+ -+ kbase_trace_gpu_mem_usage(kbdev, kctx); -+ -+ spin_unlock(&kbdev->gpu_mem_usage_lock); -+} -+ -+static inline void kbase_trace_gpu_mem_usage_inc(struct kbase_device *kbdev, -+ struct kbase_context *kctx, size_t pages) -+{ -+ spin_lock(&kbdev->gpu_mem_usage_lock); -+ -+ if (likely(kctx)) -+ kctx->kprcs->total_gpu_pages += pages; -+ -+ kbdev->total_gpu_pages += pages; -+ -+ kbase_trace_gpu_mem_usage(kbdev, kctx); -+ -+ spin_unlock(&kbdev->gpu_mem_usage_lock); -+} -+ -+/** -+ * kbase_remove_dma_buf_usage - Remove a dma-buf entry captured. -+ * -+ * @kctx: Pointer to the kbase context -+ * @alloc: Pointer to the alloc to unmap -+ * -+ * Remove reference to dma buf been unmapped from kbase_device level -+ * rb_tree and Kbase_process level dma buf rb_tree. -+ */ -+void kbase_remove_dma_buf_usage(struct kbase_context *kctx, -+ struct kbase_mem_phy_alloc *alloc); -+ -+/** -+ * kbase_add_dma_buf_usage - Add a dma-buf entry captured. -+ * -+ * @kctx: Pointer to the kbase context -+ * @alloc: Pointer to the alloc to map in -+ * -+ * Add reference to dma buf been mapped to kbase_device level -+ * rb_tree and Kbase_process level dma buf rb_tree. -+ */ -+void kbase_add_dma_buf_usage(struct kbase_context *kctx, -+ struct kbase_mem_phy_alloc *alloc); -+ -+#endif /* _KBASE_TRACE_GPU_MEM_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_utility.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_utility.h -index 8d4f044..2dad49b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_utility.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_utility.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2013, 2015, 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2013, 2015, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #ifndef _KBASE_UTILITY_H - #define _KBASE_UTILITY_H - -@@ -32,7 +29,7 @@ - static inline void kbase_timer_setup(struct timer_list *timer, - void (*callback)(struct timer_list *timer)) - { --#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) -+#if KERNEL_VERSION(4, 14, 0) > LINUX_VERSION_CODE - setup_timer(timer, (void (*)(unsigned long)) callback, - (unsigned long) timer); - #else -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_vinstr.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_vinstr.c -index d96b565..64405af 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_vinstr.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_vinstr.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,16 +17,14 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase_vinstr.h" - #include "mali_kbase_hwcnt_virtualizer.h" - #include "mali_kbase_hwcnt_types.h" --#include "mali_kbase_hwcnt_reader.h" -+#include <uapi/gpu/arm/midgard/mali_kbase_hwcnt_reader.h> - #include "mali_kbase_hwcnt_gpu.h" --#include "mali_kbase_ioctl.h" -+#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h> - #include "mali_malisw.h" - #include "mali_kbase_debug.h" - -@@ -33,6 +32,7 @@ - #include <linux/fcntl.h> - #include <linux/fs.h> - #include <linux/hrtimer.h> -+#include <linux/log2.h> - #include <linux/mm.h> - #include <linux/mutex.h> - #include <linux/poll.h> -@@ -53,6 +53,10 @@ - * counters. - * @hvirt: Hardware counter virtualizer used by vinstr. - * @metadata: Hardware counter metadata provided by virtualizer. -+ * @metadata_user: API compatible hardware counter metadata provided by vinstr. -+ * For compatibility with the user driver interface, this -+ * contains a "truncated" version of the HWCNT metadata limited -+ * to 64 entries per block. NULL when not required. - * @lock: Lock protecting all vinstr state. - * @suspend_count: Suspend reference count. If non-zero, timer and worker are - * prevented from being re-scheduled. -@@ -64,6 +68,7 @@ - struct kbase_vinstr_context { - struct kbase_hwcnt_virtualizer *hvirt; - const struct kbase_hwcnt_metadata *metadata; -+ const struct kbase_hwcnt_metadata *metadata_user; - struct mutex lock; - size_t suspend_count; - size_t client_count; -@@ -83,6 +88,7 @@ struct kbase_vinstr_context { - * @next_dump_time_ns: Time in ns when this client's next periodic dump must - * occur. If 0, not a periodic client. - * @enable_map: Counters enable map. -+ * @tmp_buf: Temporary buffer to use before handing dump to client. - * @dump_bufs: Array of dump buffers allocated by this client. - * @dump_bufs_meta: Metadata of dump buffers. - * @meta_idx: Index of metadata being accessed by userspace. -@@ -97,6 +103,7 @@ struct kbase_vinstr_client { - u64 next_dump_time_ns; - u32 dump_interval_ns; - struct kbase_hwcnt_enable_map enable_map; -+ struct kbase_hwcnt_dump_buffer tmp_buf; - struct kbase_hwcnt_dump_buffer_array dump_bufs; - struct kbase_hwcnt_reader_metadata *dump_bufs_meta; - atomic_t meta_idx; -@@ -182,8 +189,10 @@ static int kbasep_vinstr_client_dump( - u64 ts_end_ns; - unsigned int write_idx; - unsigned int read_idx; -+ struct kbase_hwcnt_dump_buffer *tmp_buf; - struct kbase_hwcnt_dump_buffer *dump_buf; - struct kbase_hwcnt_reader_metadata *meta; -+ u8 clk_cnt; - - WARN_ON(!vcli); - lockdep_assert_held(&vcli->vctx->lock); -@@ -198,23 +207,42 @@ static int kbasep_vinstr_client_dump( - - dump_buf = &vcli->dump_bufs.bufs[write_idx]; - meta = &vcli->dump_bufs_meta[write_idx]; -+ tmp_buf = &vcli->tmp_buf; - - errcode = kbase_hwcnt_virtualizer_client_dump( -- vcli->hvcli, &ts_start_ns, &ts_end_ns, dump_buf); -+ vcli->hvcli, &ts_start_ns, &ts_end_ns, tmp_buf); - if (errcode) - return errcode; - - /* Patch the dump buf headers, to hide the counters that other hwcnt - * clients are using. - */ -- kbase_hwcnt_gpu_patch_dump_headers(dump_buf, &vcli->enable_map); -+ kbase_hwcnt_gpu_patch_dump_headers(tmp_buf, &vcli->enable_map); -+ -+ /* Copy the temp buffer to the userspace visible buffer. The strict -+ * variant will explicitly zero any non-enabled counters to ensure -+ * nothing except exactly what the user asked for is made visible. -+ * -+ * If the metadata in vinstr (vctx->metadata_user) is not NULL, it means -+ * vinstr has the truncated metadata, so do a narrow copy since -+ * virtualizer has a bigger buffer but user only needs part of it. -+ * otherwise we do a full copy. -+ */ -+ if (vcli->vctx->metadata_user) -+ kbase_hwcnt_dump_buffer_copy_strict_narrow(dump_buf, tmp_buf, -+ &vcli->enable_map); -+ else -+ kbase_hwcnt_dump_buffer_copy_strict(dump_buf, tmp_buf, -+ &vcli->enable_map); - -- /* Zero all non-enabled counters (current values are undefined) */ -- kbase_hwcnt_dump_buffer_zero_non_enabled(dump_buf, &vcli->enable_map); -+ clk_cnt = vcli->vctx->metadata->clk_cnt; - - meta->timestamp = ts_end_ns; - meta->event_id = event_id; - meta->buffer_idx = write_idx; -+ meta->cycles.top = (clk_cnt > 0) ? dump_buf->clk_cnt_buf[0] : 0; -+ meta->cycles.shader_cores = -+ (clk_cnt > 1) ? dump_buf->clk_cnt_buf[1] : 0; - - /* Notify client. Make sure all changes to memory are visible. */ - wmb(); -@@ -345,11 +373,7 @@ static enum hrtimer_restart kbasep_vinstr_dump_timer(struct hrtimer *timer) - * cancelled, and the worker itself won't reschedule this timer if - * suspend_count != 0. - */ --#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE -- queue_work(system_wq, &vctx->dump_work); --#else -- queue_work(system_highpri_wq, &vctx->dump_work); --#endif -+ kbase_hwcnt_virtualizer_queue_work(vctx->hvirt, &vctx->dump_work); - return HRTIMER_NORESTART; - } - -@@ -365,6 +389,7 @@ static void kbasep_vinstr_client_destroy(struct kbase_vinstr_client *vcli) - kbase_hwcnt_virtualizer_client_destroy(vcli->hvcli); - kfree(vcli->dump_bufs_meta); - kbase_hwcnt_dump_buffer_array_free(&vcli->dump_bufs); -+ kbase_hwcnt_dump_buffer_free(&vcli->tmp_buf); - kbase_hwcnt_enable_map_free(&vcli->enable_map); - kfree(vcli); - } -@@ -374,7 +399,7 @@ static void kbasep_vinstr_client_destroy(struct kbase_vinstr_client *vcli) - * the vinstr context. - * @vctx: Non-NULL pointer to vinstr context. - * @setup: Non-NULL pointer to hardware counter ioctl setup structure. -- * setup->buffer_count must not be 0. -+ * setup->buffer_count must not be 0 and must be a power of 2. - * @out_vcli: Non-NULL pointer to where created client will be stored on - * success. - * -@@ -392,6 +417,7 @@ static int kbasep_vinstr_client_create( - WARN_ON(!vctx); - WARN_ON(!setup); - WARN_ON(setup->buffer_count == 0); -+ WARN_ON(!is_power_of_2(setup->buffer_count)); - - vcli = kzalloc(sizeof(*vcli), GFP_KERNEL); - if (!vcli) -@@ -404,14 +430,36 @@ static int kbasep_vinstr_client_create( - if (errcode) - goto error; - -- phys_em.jm_bm = setup->jm_bm; -+ phys_em.fe_bm = setup->fe_bm; - phys_em.shader_bm = setup->shader_bm; - phys_em.tiler_bm = setup->tiler_bm; - phys_em.mmu_l2_bm = setup->mmu_l2_bm; - kbase_hwcnt_gpu_enable_map_from_physical(&vcli->enable_map, &phys_em); - -- errcode = kbase_hwcnt_dump_buffer_array_alloc( -- vctx->metadata, setup->buffer_count, &vcli->dump_bufs); -+ /* Use virtualizer's metadata to alloc tmp buffer which interacts with -+ * the HWC virtualizer. -+ */ -+ errcode = kbase_hwcnt_dump_buffer_alloc(vctx->metadata, &vcli->tmp_buf); -+ if (errcode) -+ goto error; -+ -+ /* Enable all the available clk_enable_map. */ -+ vcli->enable_map.clk_enable_map = (1ull << vctx->metadata->clk_cnt) - 1; -+ -+ if (vctx->metadata_user) -+ /* Use vinstr's truncated metadata to alloc dump buffers which -+ * interact with clients. -+ */ -+ errcode = -+ kbase_hwcnt_dump_buffer_array_alloc(vctx->metadata_user, -+ setup->buffer_count, -+ &vcli->dump_bufs); -+ else -+ /* Use metadata from virtualizer to allocate dump buffers if -+ * vinstr doesn't have the truncated metadata. -+ */ -+ errcode = kbase_hwcnt_dump_buffer_array_alloc( -+ vctx->metadata, setup->buffer_count, &vcli->dump_bufs); - if (errcode) - goto error; - -@@ -439,6 +487,7 @@ int kbase_vinstr_init( - struct kbase_hwcnt_virtualizer *hvirt, - struct kbase_vinstr_context **out_vctx) - { -+ int errcode; - struct kbase_vinstr_context *vctx; - const struct kbase_hwcnt_metadata *metadata; - -@@ -455,6 +504,11 @@ int kbase_vinstr_init( - - vctx->hvirt = hvirt; - vctx->metadata = metadata; -+ vctx->metadata_user = NULL; -+ errcode = kbase_hwcnt_gpu_metadata_create_truncate_64( -+ &vctx->metadata_user, metadata); -+ if (errcode) -+ goto err_metadata_create; - - mutex_init(&vctx->lock); - INIT_LIST_HEAD(&vctx->clients); -@@ -464,6 +518,11 @@ int kbase_vinstr_init( - - *out_vctx = vctx; - return 0; -+ -+err_metadata_create: -+ kfree(vctx); -+ -+ return errcode; - } - - void kbase_vinstr_term(struct kbase_vinstr_context *vctx) -@@ -484,6 +543,9 @@ void kbase_vinstr_term(struct kbase_vinstr_context *vctx) - } - } - -+ if (vctx->metadata_user) -+ kbase_hwcnt_metadata_destroy(vctx->metadata_user); -+ - WARN_ON(vctx->client_count != 0); - kfree(vctx); - } -@@ -543,11 +605,8 @@ void kbase_vinstr_resume(struct kbase_vinstr_context *vctx) - } - - if (has_periodic_clients) --#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE -- queue_work(system_wq, &vctx->dump_work); --#else -- queue_work(system_highpri_wq, &vctx->dump_work); --#endif -+ kbase_hwcnt_virtualizer_queue_work( -+ vctx->hvirt, &vctx->dump_work); - } - } - -@@ -564,7 +623,8 @@ int kbase_vinstr_hwcnt_reader_setup( - - if (!vctx || !setup || - (setup->buffer_count == 0) || -- (setup->buffer_count > MAX_BUFFER_COUNT)) -+ (setup->buffer_count > MAX_BUFFER_COUNT) || -+ !is_power_of_2(setup->buffer_count)) - return -EINVAL; - - errcode = kbasep_vinstr_client_create(vctx, setup, &vcli); -@@ -675,26 +735,31 @@ static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( - unsigned int idx = meta_idx % cli->dump_bufs.buf_cnt; - - struct kbase_hwcnt_reader_metadata *meta = &cli->dump_bufs_meta[idx]; -+ const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata); -+ const size_t min_size = min(size, meta_size); - - /* Metadata sanity check. */ - WARN_ON(idx != meta->buffer_idx); - -- if (sizeof(struct kbase_hwcnt_reader_metadata) != size) -- return -EINVAL; -- - /* Check if there is any buffer available. */ -- if (atomic_read(&cli->write_idx) == meta_idx) -+ if (unlikely(atomic_read(&cli->write_idx) == meta_idx)) - return -EAGAIN; - - /* Check if previously taken buffer was put back. */ -- if (atomic_read(&cli->read_idx) != meta_idx) -+ if (unlikely(atomic_read(&cli->read_idx) != meta_idx)) - return -EBUSY; - -+ /* Clear user buffer to zero. */ -+ if (unlikely(meta_size < size && clear_user(buffer, size))) -+ return -EFAULT; -+ - /* Copy next available buffer's metadata to user. */ -- if (copy_to_user(buffer, meta, size)) -+ if (unlikely(copy_to_user(buffer, meta, min_size))) - return -EFAULT; - -- atomic_inc(&cli->meta_idx); -+ /* Compare exchange meta idx to protect against concurrent getters */ -+ if (meta_idx != atomic_cmpxchg(&cli->meta_idx, meta_idx, meta_idx + 1)) -+ return -EBUSY; - - return 0; - } -@@ -715,24 +780,68 @@ static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( - unsigned int read_idx = atomic_read(&cli->read_idx); - unsigned int idx = read_idx % cli->dump_bufs.buf_cnt; - -- struct kbase_hwcnt_reader_metadata meta; -- -- if (sizeof(struct kbase_hwcnt_reader_metadata) != size) -- return -EINVAL; -+ struct kbase_hwcnt_reader_metadata *meta; -+ const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata); -+ const size_t max_size = max(size, meta_size); -+ int ret = 0; -+ u8 stack_kbuf[64]; -+ u8 *kbuf = NULL; -+ size_t i; - - /* Check if any buffer was taken. */ -- if (atomic_read(&cli->meta_idx) == read_idx) -+ if (unlikely(atomic_read(&cli->meta_idx) == read_idx)) - return -EPERM; - -+ if (likely(max_size <= sizeof(stack_kbuf))) { -+ /* Use stack buffer when the size is small enough. */ -+ if (unlikely(meta_size > size)) -+ memset(stack_kbuf, 0, sizeof(stack_kbuf)); -+ kbuf = stack_kbuf; -+ } else { -+ kbuf = kzalloc(max_size, GFP_KERNEL); -+ if (unlikely(!kbuf)) -+ return -ENOMEM; -+ } -+ -+ /* -+ * Copy user buffer to zero cleared kernel buffer which has enough -+ * space for both user buffer and kernel metadata. -+ */ -+ if (unlikely(copy_from_user(kbuf, buffer, size))) { -+ ret = -EFAULT; -+ goto out; -+ } -+ -+ /* -+ * Make sure any "extra" data passed from userspace is zero. -+ * It's meaningful only in case meta_size < size. -+ */ -+ for (i = meta_size; i < size; i++) { -+ /* Check if user data beyond meta size is zero. */ -+ if (unlikely(kbuf[i] != 0)) { -+ ret = -EINVAL; -+ goto out; -+ } -+ } -+ - /* Check if correct buffer is put back. */ -- if (copy_from_user(&meta, buffer, size)) -- return -EFAULT; -- if (idx != meta.buffer_idx) -- return -EINVAL; -+ meta = (struct kbase_hwcnt_reader_metadata *)kbuf; -+ if (unlikely(idx != meta->buffer_idx)) { -+ ret = -EINVAL; -+ goto out; -+ } - -- atomic_inc(&cli->read_idx); -+ /* Compare exchange read idx to protect against concurrent putters */ -+ if (read_idx != -+ atomic_cmpxchg(&cli->read_idx, read_idx, read_idx + 1)) { -+ ret = -EPERM; -+ goto out; -+ } - -- return 0; -+out: -+ if (unlikely(kbuf != stack_kbuf)) -+ kfree(kbuf); -+ return ret; - } - - /** -@@ -760,11 +869,8 @@ static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval( - * worker is already queued. - */ - if ((interval != 0) && (cli->vctx->suspend_count == 0)) --#if KERNEL_VERSION(3, 16, 0) > LINUX_VERSION_CODE -- queue_work(system_wq, &cli->vctx->dump_work); --#else -- queue_work(system_highpri_wq, &cli->vctx->dump_work); --#endif -+ kbase_hwcnt_virtualizer_queue_work(cli->vctx->hvirt, -+ &cli->vctx->dump_work); - - mutex_unlock(&cli->vctx->lock); - -@@ -813,26 +919,58 @@ static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( - struct kbase_vinstr_client *cli, - u32 __user *hwver) - { -- u32 ver = 0; -+ u32 ver = 5; - const enum kbase_hwcnt_gpu_group_type type = - kbase_hwcnt_metadata_group_type(cli->vctx->metadata, 0); - -- switch (type) { -- case KBASE_HWCNT_GPU_GROUP_TYPE_V4: -- ver = 4; -- break; -- case KBASE_HWCNT_GPU_GROUP_TYPE_V5: -- ver = 5; -- break; -- default: -- WARN_ON(true); -- } -- -- if (ver != 0) { -- return put_user(ver, hwver); -- } else { -+ if (WARN_ON(type != KBASE_HWCNT_GPU_GROUP_TYPE_V5)) - return -EINVAL; -+ -+ return put_user(ver, hwver); -+} -+ -+/** -+ * The hwcnt reader's ioctl command - get API version. -+ * @cli: The non-NULL pointer to the client -+ * @arg: Command's argument. -+ * @size: Size of arg. -+ * -+ * @return 0 on success, else error code. -+ */ -+static long kbasep_vinstr_hwcnt_reader_ioctl_get_api_version( -+ struct kbase_vinstr_client *cli, unsigned long arg, size_t size) -+{ -+ long ret = -EINVAL; -+ -+ if (size == sizeof(u32)) { -+ ret = put_user(HWCNT_READER_API, (u32 __user *)arg); -+ } else if (size == sizeof(struct kbase_hwcnt_reader_api_version)) { -+ u8 clk_cnt = cli->vctx->metadata->clk_cnt; -+ unsigned long bytes = 0; -+ struct kbase_hwcnt_reader_api_version api_version = { -+ .version = HWCNT_READER_API, -+ .features = KBASE_HWCNT_READER_API_VERSION_NO_FEATURE, -+ }; -+ -+ if (clk_cnt > 0) -+ api_version.features |= -+ KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_TOP; -+ if (clk_cnt > 1) -+ api_version.features |= -+ KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_SHADER_CORES; -+ -+ bytes = copy_to_user( -+ (void __user *)arg, &api_version, sizeof(api_version)); -+ -+ /* copy_to_user returns zero in case of success. -+ * If it fails, it returns the number of bytes that could NOT be copied -+ */ -+ if (bytes == 0) -+ ret = 0; -+ else -+ ret = -EFAULT; - } -+ return ret; - } - - /** -@@ -858,42 +996,48 @@ static long kbasep_vinstr_hwcnt_reader_ioctl( - if (!cli) - return -EINVAL; - -- switch (cmd) { -- case KBASE_HWCNT_READER_GET_API_VERSION: -- rcode = put_user(HWCNT_READER_API, (u32 __user *)arg); -+ switch (_IOC_NR(cmd)) { -+ case _IOC_NR(KBASE_HWCNT_READER_GET_API_VERSION): -+ rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_api_version( -+ cli, arg, _IOC_SIZE(cmd)); - break; -- case KBASE_HWCNT_READER_GET_HWVER: -+ case _IOC_NR(KBASE_HWCNT_READER_GET_HWVER): - rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver( - cli, (u32 __user *)arg); - break; -- case KBASE_HWCNT_READER_GET_BUFFER_SIZE: -- rcode = put_user( -- (u32)cli->vctx->metadata->dump_buf_bytes, -- (u32 __user *)arg); -+ case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER_SIZE): -+ if (cli->vctx->metadata_user) -+ rcode = put_user( -+ (u32)cli->vctx->metadata_user->dump_buf_bytes, -+ (u32 __user *)arg); -+ else -+ rcode = put_user( -+ (u32)cli->vctx->metadata->dump_buf_bytes, -+ (u32 __user *)arg); - break; -- case KBASE_HWCNT_READER_DUMP: -+ case _IOC_NR(KBASE_HWCNT_READER_DUMP): - rcode = kbasep_vinstr_hwcnt_reader_ioctl_dump(cli); - break; -- case KBASE_HWCNT_READER_CLEAR: -+ case _IOC_NR(KBASE_HWCNT_READER_CLEAR): - rcode = kbasep_vinstr_hwcnt_reader_ioctl_clear(cli); - break; -- case KBASE_HWCNT_READER_GET_BUFFER: -+ case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER): - rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer( - cli, (void __user *)arg, _IOC_SIZE(cmd)); - break; -- case KBASE_HWCNT_READER_PUT_BUFFER: -+ case _IOC_NR(KBASE_HWCNT_READER_PUT_BUFFER): - rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer( - cli, (void __user *)arg, _IOC_SIZE(cmd)); - break; -- case KBASE_HWCNT_READER_SET_INTERVAL: -+ case _IOC_NR(KBASE_HWCNT_READER_SET_INTERVAL): - rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval( - cli, (u32)arg); - break; -- case KBASE_HWCNT_READER_ENABLE_EVENT: -+ case _IOC_NR(KBASE_HWCNT_READER_ENABLE_EVENT): - rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event( - cli, (enum base_hwcnt_reader_event)arg); - break; -- case KBASE_HWCNT_READER_DISABLE_EVENT: -+ case _IOC_NR(KBASE_HWCNT_READER_DISABLE_EVENT): - rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event( - cli, (enum base_hwcnt_reader_event)arg); - break; -@@ -955,7 +1099,16 @@ static int kbasep_vinstr_hwcnt_reader_mmap( - return -EINVAL; - - vm_size = vma->vm_end - vma->vm_start; -- size = cli->dump_bufs.buf_cnt * cli->vctx->metadata->dump_buf_bytes; -+ -+ /* The mapping is allowed to span the entirety of the page allocation, -+ * not just the chunk where the dump buffers are allocated. -+ * This accommodates the corner case where the combined size of the -+ * dump buffers is smaller than a single page. -+ * This does not pose a security risk as the pages are zeroed on -+ * allocation, and anything out of bounds of the dump buffers is never -+ * written to. -+ */ -+ size = (1ull << cli->dump_bufs.page_order) * PAGE_SIZE; - - if (vma->vm_pgoff > (size >> PAGE_SHIFT)) - return -EINVAL; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_vinstr.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_vinstr.h -index 81d315f..6747ec7 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_vinstr.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_vinstr.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2015-2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_linux_trace.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_linux_trace.h -index f618755..2a243dd 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_linux_trace.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_linux_trace.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2016, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #undef TRACE_SYSTEM -@@ -31,7 +30,7 @@ - #if defined(CONFIG_MALI_GATOR_SUPPORT) - #define MALI_JOB_SLOTS_EVENT_CHANGED - --/** -+/* - * mali_job_slots_event - Reports change of job slot status. - * @gpu_id: Kbase device id - * @event_id: ORed together bitfields representing a type of event, -@@ -181,6 +180,23 @@ TRACE_EVENT(mali_total_alloc_pages_change, - __print_symbolic(KBASE_MMU_FAULT_STATUS_ACCESS(status), \ - KBASE_MMU_FAULT_ACCESS_SYMBOLIC_STRINGS) - -+#if MALI_USE_CSF -+#define KBASE_MMU_FAULT_CODE_VALID(code) \ -+ ((code >= 0xC0 && code <= 0xEB) && \ -+ (!(code >= 0xC5 && code <= 0xC7)) && \ -+ (!(code >= 0xCC && code <= 0xD8)) && \ -+ (!(code >= 0xDC && code <= 0xDF)) && \ -+ (!(code >= 0xE1 && code <= 0xE3))) -+#define KBASE_MMU_FAULT_CODE_SYMBOLIC_STRINGS _ENSURE_PARENTHESIS(\ -+ {0xC0, "TRANSLATION_FAULT_" }, \ -+ {0xC4, "TRANSLATION_FAULT_" }, \ -+ {0xC8, "PERMISSION_FAULT_" }, \ -+ {0xD0, "TRANSTAB_BUS_FAULT_" }, \ -+ {0xD8, "ACCESS_FLAG_" }, \ -+ {0xE0, "ADDRESS_SIZE_FAULT_IN" }, \ -+ {0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ -+ {0xE8, "MEMORY_ATTRIBUTES_FAULT_" }) -+#else /* MALI_USE_CSF */ - #define KBASE_MMU_FAULT_CODE_VALID(code) \ - ((code >= 0xC0 && code <= 0xEF) && \ - (!(code >= 0xC5 && code <= 0xC6)) && \ -@@ -197,6 +213,7 @@ TRACE_EVENT(mali_total_alloc_pages_change, - {0xE4, "ADDRESS_SIZE_FAULT_OUT" }, \ - {0xE8, "MEMORY_ATTRIBUTES_FAULT_" }, \ - {0xEC, "MEMORY_ATTRIBUTES_NONCACHEABLE_" }) -+#endif /* MALI_USE_CSF */ - #endif /* __TRACE_MALI_MMU_HELPERS */ - - /* trace_mali_mmu_page_fault_grow -@@ -288,7 +305,8 @@ DEFINE_EVENT_PRINT(mali_jit_softjob_template, mali_jit_free, - TP_printk("start=0x%llx va_pages=0x%zx backed_size=0x%zx", - __entry->start_addr, __entry->nr_pages, __entry->backed_pages)); - --#if MALI_JIT_PRESSURE_LIMIT -+#if !MALI_USE_CSF -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /* trace_mali_jit_report - * - * Tracepoint about the GPU data structure read to form a just-in-time memory -@@ -326,13 +344,11 @@ TRACE_EVENT(mali_jit_report, - ), - __entry->read_val, __entry->used_pages) - ); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -+#endif /* !MALI_USE_CSF */ - --#if (KERNEL_VERSION(4, 1, 0) <= LINUX_VERSION_CODE) - TRACE_DEFINE_ENUM(KBASE_JIT_REPORT_ON_ALLOC_OR_FREE); --#endif -- --#if MALI_JIT_PRESSURE_LIMIT -+#if MALI_JIT_PRESSURE_LIMIT_BASE - /* trace_mali_jit_report_pressure - * - * Tracepoint about change in physical memory pressure, due to the information -@@ -366,14 +382,13 @@ TRACE_EVENT(mali_jit_report_pressure, - { KBASE_JIT_REPORT_ON_ALLOC_OR_FREE, - "HAPPENED_ON_ALLOC_OR_FREE" })) - ); --#endif /* MALI_JIT_PRESSURE_LIMIT */ -+#endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ - - #ifndef __TRACE_SYSGRAPH_ENUM - #define __TRACE_SYSGRAPH_ENUM - /* Enum of sysgraph message IDs */ - enum sysgraph_msg { - SGR_ARRIVE, -- SGR_DEP_RES, - SGR_SUBMIT, - SGR_COMPLETE, - SGR_POST, -@@ -401,7 +416,7 @@ TRACE_EVENT(sysgraph, - __entry->message = message; - __entry->atom_id = atom_id; - ), -- TP_printk("msg=%u proc_id=%u, param1=%d\n", __entry->message, -+ TP_printk("msg=%u proc_id=%u, param1=%d", __entry->message, - __entry->proc_id, __entry->atom_id) - ); - -@@ -427,7 +442,7 @@ TRACE_EVENT(sysgraph_gpu, - __entry->atom_id = atom_id; - __entry->js = js; - ), -- TP_printk("msg=%u proc_id=%u, param1=%d, param2=%d\n", -+ TP_printk("msg=%u proc_id=%u, param1=%d, param2=%d", - __entry->message, __entry->proc_id, - __entry->atom_id, __entry->js) - ); -@@ -516,7 +531,7 @@ TRACE_EVENT(mali_jit_trim, - TP_printk("freed_pages=%zu", __entry->freed_pages) - ); - --#include "mali_kbase_debug_linux_ktrace.h" -+#include "debug/mali_kbase_debug_linux_ktrace.h" - - #endif /* _TRACE_MALI_H */ - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_malisw.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_malisw.h -index 3a4db10..c0649f2 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_malisw.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_malisw.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2015, 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2015, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -@@ -28,26 +27,11 @@ - #define _MALISW_H_ - - #include <linux/version.h> --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 14, 0) --#define U8_MAX ((u8)~0U) --#define S8_MAX ((s8)(U8_MAX>>1)) --#define S8_MIN ((s8)(-S8_MAX - 1)) --#define U16_MAX ((u16)~0U) --#define S16_MAX ((s16)(U16_MAX>>1)) --#define S16_MIN ((s16)(-S16_MAX - 1)) --#define U32_MAX ((u32)~0U) --#define S32_MAX ((s32)(U32_MAX>>1)) --#define S32_MIN ((s32)(-S32_MAX - 1)) --#define U64_MAX ((u64)~0ULL) --#define S64_MAX ((s64)(U64_MAX>>1)) --#define S64_MIN ((s64)(-S64_MAX - 1)) --#endif /* LINUX_VERSION_CODE */ --#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 5, 0) --#define SIZE_MAX (~(size_t)0) --#endif /* LINUX_VERSION_CODE */ - - /** - * MIN - Return the lesser of two values. -+ * @x: value1 -+ * @y: value2 - * - * As a macro it may evaluate its arguments more than once. - * Refer to MAX macro for more details -@@ -55,7 +39,9 @@ - #define MIN(x, y) ((x) < (y) ? (x) : (y)) - - /** -- * MAX - Return the greater of two values. -+ * MAX - Return the greater of two values. -+ * @x: value1 -+ * @y: value2 - * - * As a macro it may evaluate its arguments more than once. - * If called on the same two arguments as MIN it is guaranteed to return -@@ -67,24 +53,27 @@ - #define MAX(x, y) ((x) < (y) ? (y) : (x)) - - /** -- * @hideinitializer -- * Function-like macro for suppressing unused variable warnings. Where possible -- * such variables should be removed; this macro is present for cases where we -- * much support API backwards compatibility. -+ * Function-like macro for suppressing unused variable warnings. -+ * @x: unused variable -+ * -+ * Where possible such variables should be removed; this macro is present for -+ * cases where we much support API backwards compatibility. - */ - #define CSTD_UNUSED(x) ((void)(x)) - - /** -- * @hideinitializer -- * Function-like macro for use where "no behavior" is desired. This is useful -- * when compile time macros turn a function-like macro in to a no-op, but -- * where having no statement is otherwise invalid. -+ * Function-like macro for use where "no behavior" is desired. -+ * @...: no-op -+ * -+ * This is useful when compile time macros turn a function-like macro in to a -+ * no-op, but where having no statement is otherwise invalid. - */ - #define CSTD_NOP(...) ((void)#__VA_ARGS__) - - /** -- * @hideinitializer - * Function-like macro for stringizing a single level macro. -+ * @x: macro's value -+ * - * @code - * #define MY_MACRO 32 - * CSTD_STR1( MY_MACRO ) -@@ -94,10 +83,11 @@ - #define CSTD_STR1(x) #x - - /** -- * @hideinitializer -- * Function-like macro for stringizing a macro's value. This should not be used -- * if the macro is defined in a way which may have no value; use the -- * alternative @c CSTD_STR2N macro should be used instead. -+ * Function-like macro for stringizing a macro's value. -+ * @x: macro's value -+ * -+ * This should not be used if the macro is defined in a way which may have no -+ * value; use the alternative @c CSTD_STR2N macro should be used instead. - * @code - * #define MY_MACRO 32 - * CSTD_STR2( MY_MACRO ) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_power_gpu_frequency_trace.c -similarity index 67% -rename from dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu.h -rename to dvalin/kernel/drivers/gpu/arm/midgard/mali_power_gpu_frequency_trace.c -index 9516e56..1db3abe 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_power_gpu_frequency_trace.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,16 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifndef _KBASE_GPU_H_ --#define _KBASE_GPU_H_ -- --#include "mali_kbase_gpu_regmap.h" --#include "mali_kbase_gpu_fault.h" --#include "mali_kbase_gpu_coherency.h" --#include "mali_kbase_gpu_id.h" -- --#endif /* _KBASE_GPU_H_ */ -+/* Create the trace point if not configured in kernel */ -+#ifndef CONFIG_TRACE_POWER_GPU_FREQUENCY -+#define CREATE_TRACE_POINTS -+#include "mali_power_gpu_frequency_trace.h" -+#endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_power_gpu_frequency_trace.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_power_gpu_frequency_trace.h -new file mode 100644 -index 0000000..ea18fcd ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_power_gpu_frequency_trace.h -@@ -0,0 +1,68 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _TRACE_POWER_GPU_FREQUENCY_MALI -+#define _TRACE_POWER_GPU_FREQUENCY_MALI -+#endif -+ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM power -+#undef TRACE_INCLUDE_FILE -+#define TRACE_INCLUDE_FILE mali_power_gpu_frequency_trace -+#undef TRACE_INCLUDE_PATH -+#define TRACE_INCLUDE_PATH . -+ -+#if !defined(_TRACE_POWER_GPU_FREQUENCY_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_POWER_GPU_FREQUENCY_H -+ -+#include <linux/tracepoint.h> -+ -+DECLARE_EVENT_CLASS(gpu, -+ -+ TP_PROTO(unsigned int state, unsigned int gpu_id), -+ -+ TP_ARGS(state, gpu_id), -+ -+ TP_STRUCT__entry( -+ __field( u32, state ) -+ __field( u32, gpu_id ) -+ ), -+ -+ TP_fast_assign( -+ __entry->state = state; -+ __entry->gpu_id = gpu_id; -+ ), -+ -+ TP_printk("state=%lu gpu_id=%lu", (unsigned long)__entry->state, -+ (unsigned long)__entry->gpu_id) -+); -+ -+DEFINE_EVENT(gpu, gpu_frequency, -+ -+ TP_PROTO(unsigned int frequency, unsigned int gpu_id), -+ -+ TP_ARGS(frequency, gpu_id) -+); -+ -+#endif /* _TRACE_POWER_GPU_FREQUENCY_H */ -+ -+/* This part must be outside protection */ -+#include <trace/define_trace.h> -diff --git a/dvalin/kernel/drivers/base/protected_memory_allocator/Kconfig b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/Kbuild -similarity index 61% -rename from dvalin/kernel/drivers/base/protected_memory_allocator/Kconfig -rename to dvalin/kernel/drivers/gpu/arm/midgard/mmu/Kbuild -index 7b42d6c..7f2bb26 100644 ---- a/dvalin/kernel/drivers/base/protected_memory_allocator/Kconfig -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,14 +16,15 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - -+mali_kbase-y += \ -+ mmu/mali_kbase_mmu.o \ -+ mmu/mali_kbase_mmu_hw_direct.o \ -+ mmu/mali_kbase_mmu_mode_aarch64.o - --config MALI_PROTECTED_MEMORY_ALLOCATOR -- tristate "MALI_PROTECTED_MEMORY_ALLOCATOR" -- help -- This option enables an example implementation of a protected memory allocator -- for allocation and release of pages of secure memory intended to be used -- by the firmware of Mali GPU device drivers. -+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) -+ mali_kbase-y += mmu/backend/mali_kbase_mmu_csf.o -+else -+ mali_kbase-y += mmu/backend/mali_kbase_mmu_jm.o -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/backend/mali_kbase_mmu_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/backend/mali_kbase_mmu_csf.c -new file mode 100644 -index 0000000..05253ae ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/backend/mali_kbase_mmu_csf.c -@@ -0,0 +1,565 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/** -+ * DOC: Base kernel MMU management specific for CSF GPU. -+ */ -+ -+#include <mali_kbase.h> -+#include <gpu/mali_kbase_gpu_fault.h> -+#include <mali_kbase_ctx_sched.h> -+#include <mali_kbase_reset_gpu.h> -+#include <mali_kbase_as_fault_debugfs.h> -+#include <mmu/mali_kbase_mmu_internal.h> -+ -+void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, -+ struct kbase_mmu_setup * const setup) -+{ -+ /* Set up the required caching policies at the correct indices -+ * in the memattr register. -+ */ -+ setup->memattr = -+ (AS_MEMATTR_IMPL_DEF_CACHE_POLICY << -+ (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | -+ (AS_MEMATTR_FORCE_TO_CACHE_ALL << -+ (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | -+ (AS_MEMATTR_WRITE_ALLOC << -+ (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | -+ (AS_MEMATTR_AARCH64_OUTER_IMPL_DEF << -+ (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | -+ (AS_MEMATTR_AARCH64_OUTER_WA << -+ (AS_MEMATTR_INDEX_OUTER_WA * 8)) | -+ (AS_MEMATTR_AARCH64_NON_CACHEABLE << -+ (AS_MEMATTR_INDEX_NON_CACHEABLE * 8)) | -+ (AS_MEMATTR_AARCH64_SHARED << -+ (AS_MEMATTR_INDEX_SHARED * 8)); -+ -+ setup->transtab = (u64)mmut->pgd & AS_TRANSTAB_BASE_MASK; -+ setup->transcfg = AS_TRANSCFG_ADRMODE_AARCH64_4K; -+} -+ -+/** -+ * submit_work_pagefault() - Submit a work for MMU page fault. -+ * -+ * @kbdev: Kbase device pointer -+ * @as_nr: Faulty address space -+ * @fault: Data relating to the fault -+ * -+ * This function submits a work for reporting the details of MMU fault. -+ */ -+static void submit_work_pagefault(struct kbase_device *kbdev, u32 as_nr, -+ struct kbase_fault *fault) -+{ -+ unsigned long flags; -+ struct kbase_as *const as = &kbdev->as[as_nr]; -+ struct kbase_context *kctx; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as_nr); -+ -+ if (kctx) { -+ kbase_ctx_sched_retain_ctx_refcount(kctx); -+ -+ as->pf_data = (struct kbase_fault) { -+ .status = fault->status, -+ .addr = fault->addr, -+ }; -+ -+ /* -+ * A page fault work item could already be pending for the -+ * context's address space, when the page fault occurs for -+ * MCU's address space. -+ */ -+ if (!queue_work(as->pf_wq, &as->work_pagefault)) -+ kbase_ctx_sched_release_ctx(kctx); -+ else { -+ dev_dbg(kbdev->dev, -+ "Page fault is already pending for as %u\n", -+ as_nr); -+ atomic_inc(&kbdev->faults_pending); -+ } -+ } -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, -+ struct kbase_fault *fault) -+{ -+ /* decode the fault status */ -+ u32 exception_type = fault->status & 0xFF; -+ u32 access_type = (fault->status >> 8) & 0x3; -+ u32 source_id = (fault->status >> 16); -+ int as_no; -+ -+ /* terminal fault, print info about the fault */ -+ dev_err(kbdev->dev, -+ "Unexpected Page fault in firmware address space at VA 0x%016llX\n" -+ "raw fault status: 0x%X\n" -+ "exception type 0x%X: %s\n" -+ "access type 0x%X: %s\n" -+ "source id 0x%X\n", -+ fault->addr, -+ fault->status, -+ exception_type, kbase_gpu_exception_name(exception_type), -+ access_type, kbase_gpu_access_type_name(fault->status), -+ source_id); -+ -+ /* Report MMU fault for all address spaces (except MCU_AS_NR) */ -+ for (as_no = 1; as_no < kbdev->nr_hw_address_spaces; as_no++) -+ submit_work_pagefault(kbdev, as_no, fault); -+ -+ /* GPU reset is required to recover */ -+ if (kbase_prepare_to_reset_gpu(kbdev, -+ RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) -+ kbase_reset_gpu(kbdev); -+} -+KBASE_EXPORT_TEST_API(kbase_mmu_report_mcu_as_fault_and_reset); -+ -+void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, -+ struct kbase_as *as, struct kbase_fault *fault) -+{ -+ struct kbase_device *kbdev = kctx->kbdev; -+ u32 const status = fault->status; -+ int exception_type = (status & GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> -+ GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT; -+ int access_type = (status & GPU_FAULTSTATUS_ACCESS_TYPE_MASK) >> -+ GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT; -+ int source_id = (status & GPU_FAULTSTATUS_SOURCE_ID_MASK) >> -+ GPU_FAULTSTATUS_SOURCE_ID_SHIFT; -+ const char *addr_valid = (status & GPU_FAULTSTATUS_ADDR_VALID_FLAG) ? -+ "true" : "false"; -+ int as_no = as->number; -+ unsigned long flags; -+ -+ /* terminal fault, print info about the fault */ -+ dev_err(kbdev->dev, -+ "GPU bus fault in AS%d at VA 0x%016llX\n" -+ "VA_VALID: %s\n" -+ "raw fault status: 0x%X\n" -+ "exception type 0x%X: %s\n" -+ "access type 0x%X: %s\n" -+ "source id 0x%X\n" -+ "pid: %d\n", -+ as_no, fault->addr, -+ addr_valid, -+ status, -+ exception_type, kbase_gpu_exception_name(exception_type), -+ access_type, kbase_gpu_access_type_name(access_type), -+ source_id, -+ kctx->pid); -+ -+ /* AS transaction begin */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_mmu_disable(kctx); -+ kbase_ctx_flag_set(kctx, KCTX_AS_DISABLED_ON_FAULT); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ -+ /* Switching to UNMAPPED mode above would have enabled the firmware to -+ * recover from the fault (if the memory access was made by firmware) -+ * and it can then respond to CSG termination requests to be sent now. -+ * All GPU command queue groups associated with the context would be -+ * affected as they use the same GPU address space. -+ */ -+ kbase_csf_ctx_handle_fault(kctx, fault); -+ -+ /* Now clear the GPU fault */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CLEAR_FAULT); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+/* -+ * The caller must ensure it's retained the ctx to prevent it from being -+ * scheduled out whilst it's being worked on. -+ */ -+void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, -+ struct kbase_as *as, const char *reason_str, -+ struct kbase_fault *fault) -+{ -+ unsigned long flags; -+ unsigned int exception_type; -+ unsigned int access_type; -+ unsigned int source_id; -+ int as_no; -+ struct kbase_device *kbdev; -+ const u32 status = fault->status; -+ -+ as_no = as->number; -+ kbdev = kctx->kbdev; -+ -+ /* Make sure the context was active */ -+ if (WARN_ON(atomic_read(&kctx->refcount) <= 0)) -+ return; -+ -+ /* decode the fault status */ -+ exception_type = AS_FAULTSTATUS_EXCEPTION_TYPE_GET(status); -+ access_type = AS_FAULTSTATUS_ACCESS_TYPE_GET(status); -+ source_id = AS_FAULTSTATUS_SOURCE_ID_GET(status); -+ -+ /* terminal fault, print info about the fault */ -+ dev_err(kbdev->dev, -+ "Unhandled Page fault in AS%d at VA 0x%016llX\n" -+ "Reason: %s\n" -+ "raw fault status: 0x%X\n" -+ "exception type 0x%X: %s\n" -+ "access type 0x%X: %s\n" -+ "source id 0x%X\n" -+ "pid: %d\n", -+ as_no, fault->addr, -+ reason_str, -+ status, -+ exception_type, kbase_gpu_exception_name(exception_type), -+ access_type, kbase_gpu_access_type_name(status), -+ source_id, -+ kctx->pid); -+ -+ /* AS transaction begin */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ -+ /* switch to UNMAPPED mode, -+ * will abort all jobs and stop any hw counter dumping -+ */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_mmu_disable(kctx); -+ kbase_ctx_flag_set(kctx, KCTX_AS_DISABLED_ON_FAULT); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ /* AS transaction end */ -+ -+ /* Switching to UNMAPPED mode above would have enabled the firmware to -+ * recover from the fault (if the memory access was made by firmware) -+ * and it can then respond to CSG termination requests to be sent now. -+ * All GPU command queue groups associated with the context would be -+ * affected as they use the same GPU address space. -+ */ -+ kbase_csf_ctx_handle_fault(kctx, fault); -+ -+ /* Clear down the fault */ -+ kbase_mmu_hw_clear_fault(kbdev, as, -+ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); -+ kbase_mmu_hw_enable_fault(kbdev, as, -+ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); -+} -+ -+/** -+ * kbase_mmu_interrupt_process() - Process a bus or page fault. -+ * @kbdev: The kbase_device the fault happened on -+ * @kctx: The kbase_context for the faulting address space if one was -+ * found. -+ * @as: The address space that has the fault -+ * @fault: Data relating to the fault -+ * -+ * This function will process a fault on a specific address space -+ */ -+static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, -+ struct kbase_context *kctx, struct kbase_as *as, -+ struct kbase_fault *fault) -+{ -+ lockdep_assert_held(&kbdev->hwaccess_lock); -+ -+ if (!kctx) { -+ dev_warn(kbdev->dev, "%s in AS%d at 0x%016llx with no context present! Spurious IRQ or SW Design Error?\n", -+ kbase_as_has_bus_fault(as, fault) ? -+ "Bus error" : "Page fault", -+ as->number, fault->addr); -+ -+ /* Since no ctx was found, the MMU must be disabled. */ -+ WARN_ON(as->current_setup.transtab); -+ -+ if (kbase_as_has_bus_fault(as, fault)) -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CLEAR_FAULT); -+ else if (kbase_as_has_page_fault(as, fault)) { -+ kbase_mmu_hw_clear_fault(kbdev, as, -+ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); -+ kbase_mmu_hw_enable_fault(kbdev, as, -+ KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); -+ } -+ -+ return; -+ } -+ -+ if (kbase_as_has_bus_fault(as, fault)) { -+ /* -+ * We need to switch to UNMAPPED mode - but we do this in a -+ * worker so that we can sleep -+ */ -+ WARN_ON(!queue_work(as->pf_wq, &as->work_busfault)); -+ atomic_inc(&kbdev->faults_pending); -+ } else { -+ WARN_ON(!queue_work(as->pf_wq, &as->work_pagefault)); -+ atomic_inc(&kbdev->faults_pending); -+ } -+} -+ -+int kbase_mmu_bus_fault_interrupt(struct kbase_device *kbdev, -+ u32 status, u32 as_nr) -+{ -+ struct kbase_context *kctx; -+ unsigned long flags; -+ struct kbase_as *as; -+ struct kbase_fault *fault; -+ -+ if (WARN_ON(as_nr == MCU_AS_NR)) -+ return -EINVAL; -+ -+ if (WARN_ON(as_nr >= BASE_MAX_NR_AS)) -+ return -EINVAL; -+ -+ as = &kbdev->as[as_nr]; -+ fault = &as->bf_data; -+ fault->status = status; -+ fault->addr = (u64) kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32; -+ fault->addr |= kbase_reg_read(kbdev, -+ GPU_CONTROL_REG(GPU_FAULTADDRESS_LO)); -+ fault->protected_mode = false; -+ -+ /* report the fault to debugfs */ -+ kbase_as_fault_debugfs_new(kbdev, as_nr); -+ -+ kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_nr); -+ -+ /* Process the bus fault interrupt for this address space */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_mmu_interrupt_process(kbdev, kctx, as, fault); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ return 0; -+} -+ -+void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) -+{ -+ const int num_as = 16; -+ const int pf_shift = 0; -+ const unsigned long as_bit_mask = (1UL << num_as) - 1; -+ unsigned long flags; -+ u32 new_mask; -+ u32 tmp; -+ u32 pf_bits = ((irq_stat >> pf_shift) & as_bit_mask); -+ -+ /* remember current mask */ -+ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); -+ new_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); -+ /* mask interrupts for now */ -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0); -+ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); -+ -+ while (pf_bits) { -+ struct kbase_context *kctx; -+ int as_no = ffs(pf_bits) - 1; -+ struct kbase_as *as = &kbdev->as[as_no]; -+ struct kbase_fault *fault = &as->pf_data; -+ -+ /* find faulting address */ -+ fault->addr = kbase_reg_read(kbdev, MMU_AS_REG(as_no, -+ AS_FAULTADDRESS_HI)); -+ fault->addr <<= 32; -+ fault->addr |= kbase_reg_read(kbdev, MMU_AS_REG(as_no, -+ AS_FAULTADDRESS_LO)); -+ -+ /* Mark the fault protected or not */ -+ fault->protected_mode = false; -+ -+ /* report the fault to debugfs */ -+ kbase_as_fault_debugfs_new(kbdev, as_no); -+ -+ /* record the fault status */ -+ fault->status = kbase_reg_read(kbdev, MMU_AS_REG(as_no, -+ AS_FAULTSTATUS)); -+ -+ fault->extra_addr = kbase_reg_read(kbdev, -+ MMU_AS_REG(as_no, AS_FAULTEXTRA_HI)); -+ fault->extra_addr <<= 32; -+ fault->extra_addr |= kbase_reg_read(kbdev, -+ MMU_AS_REG(as_no, AS_FAULTEXTRA_LO)); -+ -+ /* Mark page fault as handled */ -+ pf_bits &= ~(1UL << as_no); -+ -+ /* remove the queued PF from the mask */ -+ new_mask &= ~MMU_PAGE_FAULT(as_no); -+ -+ if (as_no == MCU_AS_NR) { -+ kbase_mmu_report_mcu_as_fault_and_reset(kbdev, fault); -+ /* Pointless to handle remaining faults */ -+ break; -+ } -+ -+ /* -+ * Refcount the kctx - it shouldn't disappear anyway, since -+ * Page faults _should_ only occur whilst GPU commands are -+ * executing, and a command causing the Page fault shouldn't -+ * complete until the MMU is updated. -+ * Reference is released at the end of bottom half of page -+ * fault handling. -+ */ -+ kctx = kbase_ctx_sched_as_to_ctx_refcount(kbdev, as_no); -+ -+ /* Process the interrupt for this address space */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_mmu_interrupt_process(kbdev, kctx, as, fault); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ } -+ -+ /* reenable interrupts */ -+ spin_lock_irqsave(&kbdev->mmu_mask_change, flags); -+ tmp = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)); -+ new_mask |= tmp; -+ kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), new_mask); -+ spin_unlock_irqrestore(&kbdev->mmu_mask_change, flags); -+} -+ -+int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, -+ struct kbase_va_region *const reg) -+{ -+ /* Can't soft-stop the provoking job */ -+ return -EPERM; -+} -+ -+/** -+ * kbase_mmu_gpu_fault_worker() - Process a GPU fault for the device. -+ * -+ * @data: work_struct passed by queue_work() -+ * -+ * Report a GPU fatal error for all GPU command queue groups that are -+ * using the address space and terminate them. -+ */ -+static void kbase_mmu_gpu_fault_worker(struct work_struct *data) -+{ -+ struct kbase_as *const faulting_as = container_of(data, struct kbase_as, -+ work_gpufault); -+ const u32 as_nr = faulting_as->number; -+ struct kbase_device *const kbdev = container_of(faulting_as, struct -+ kbase_device, as[as_nr]); -+ struct kbase_fault *fault; -+ struct kbase_context *kctx; -+ u32 status; -+ u64 address; -+ u32 as_valid; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ fault = &faulting_as->gf_data; -+ status = fault->status; -+ as_valid = status & GPU_FAULTSTATUS_JASID_VALID_FLAG; -+ address = fault->addr; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ dev_warn(kbdev->dev, -+ "GPU Fault 0x%08x (%s) in AS%u at 0x%016llx\n" -+ "ASID_VALID: %s, ADDRESS_VALID: %s\n", -+ status, -+ kbase_gpu_exception_name( -+ GPU_FAULTSTATUS_EXCEPTION_TYPE_GET(status)), -+ as_nr, address, -+ as_valid ? "true" : "false", -+ status & GPU_FAULTSTATUS_ADDR_VALID_FLAG ? "true" : "false"); -+ -+ kctx = kbase_ctx_sched_as_to_ctx(kbdev, as_nr); -+ kbase_csf_ctx_handle_fault(kctx, fault); -+ kbase_ctx_sched_release_ctx_lock(kctx); -+ -+ atomic_dec(&kbdev->faults_pending); -+ -+ /* A work for GPU fault is complete. -+ * Till reaching here, no further GPU fault will be reported. -+ * Now clear the GPU fault to allow next GPU fault interrupt report. -+ */ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), -+ GPU_COMMAND_CLEAR_FAULT); -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+/** -+ * submit_work_gpufault() - Submit a work for GPU fault. -+ * -+ * @kbdev: Kbase device pointer -+ * @status: GPU fault status -+ * @as_nr: Faulty address space -+ * @address: GPU fault address -+ * -+ * This function submits a work for reporting the details of GPU fault. -+ */ -+static void submit_work_gpufault(struct kbase_device *kbdev, u32 status, -+ u32 as_nr, u64 address) -+{ -+ unsigned long flags; -+ struct kbase_as *const as = &kbdev->as[as_nr]; -+ struct kbase_context *kctx; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ kctx = kbase_ctx_sched_as_to_ctx_nolock(kbdev, as_nr); -+ -+ if (kctx) { -+ kbase_ctx_sched_retain_ctx_refcount(kctx); -+ -+ as->gf_data = (struct kbase_fault) { -+ .status = status, -+ .addr = address, -+ }; -+ -+ if (WARN_ON(!queue_work(as->pf_wq, &as->work_gpufault))) -+ kbase_ctx_sched_release_ctx(kctx); -+ else -+ atomic_inc(&kbdev->faults_pending); -+ } -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+} -+ -+void kbase_mmu_gpu_fault_interrupt(struct kbase_device *kbdev, u32 status, -+ u32 as_nr, u64 address, bool as_valid) -+{ -+ if (!as_valid || (as_nr == MCU_AS_NR)) { -+ int as; -+ -+ /* Report GPU fault for all contexts (except MCU_AS_NR) in case either -+ * the address space is invalid or it's MCU address space. -+ */ -+ for (as = 1; as < kbdev->nr_hw_address_spaces; as++) -+ submit_work_gpufault(kbdev, status, as, address); -+ } else -+ submit_work_gpufault(kbdev, status, as_nr, address); -+} -+KBASE_EXPORT_TEST_API(kbase_mmu_gpu_fault_interrupt); -+ -+int kbase_mmu_as_init(struct kbase_device *kbdev, int i) -+{ -+ kbdev->as[i].number = i; -+ kbdev->as[i].bf_data.addr = 0ULL; -+ kbdev->as[i].pf_data.addr = 0ULL; -+ kbdev->as[i].gf_data.addr = 0ULL; -+ -+ kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%d", 0, 1, i); -+ if (!kbdev->as[i].pf_wq) -+ return -ENOMEM; -+ -+ INIT_WORK(&kbdev->as[i].work_pagefault, kbase_mmu_page_fault_worker); -+ INIT_WORK(&kbdev->as[i].work_busfault, kbase_mmu_bus_fault_worker); -+ INIT_WORK(&kbdev->as[i].work_gpufault, kbase_mmu_gpu_fault_worker); -+ -+ return 0; -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/backend/mali_kbase_mmu_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/backend/mali_kbase_mmu_jm.c -index 2d8fb51..01ca419 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/backend/mali_kbase_mmu_jm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/backend/mali_kbase_mmu_jm.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,21 +17,18 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -- * Base kernel MMU management specific for Job Manager GPU. -+ * DOC: Base kernel MMU management specific for Job Manager GPU. - */ - - #include <mali_kbase.h> - #include <gpu/mali_kbase_gpu_fault.h> - #include <mali_kbase_hwaccess_jm.h> --#include <backend/gpu/mali_kbase_device_internal.h> -+#include <device/mali_kbase_device.h> - #include <mali_kbase_as_fault_debugfs.h> --#include "../mali_kbase_mmu_internal.h" --#include "mali_kbase_device_internal.h" -+#include <mmu/mali_kbase_mmu_internal.h> - - void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, - struct kbase_mmu_setup * const setup) -@@ -98,7 +96,7 @@ void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, - KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED); - } - --/** -+/* - * The caller must ensure it's retained the ctx to prevent it from being - * scheduled out whilst it's being worked on. - */ -@@ -145,6 +143,7 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, - kctx->pid); - - /* hardware counters dump fault handling */ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - if ((kbdev->hwcnt.kctx) && (kbdev->hwcnt.kctx->as_nr == as_no) && - (kbdev->hwcnt.backend.state == - KBASE_INSTR_STATE_DUMPING)) { -@@ -153,6 +152,7 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, - kbdev->hwcnt.addr_bytes))) - kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_FAULT; - } -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - /* Stop the kctx from submitting more jobs and cause it to be scheduled - * out/rescheduled - this will occur on releasing the context's refcount -@@ -187,14 +187,26 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, - KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED); - } - --void kbase_mmu_interrupt_process(struct kbase_device *kbdev, -+/** -+ * kbase_mmu_interrupt_process() - Process a bus or page fault. -+ * @kbdev: The kbase_device the fault happened on -+ * @kctx: The kbase_context for the faulting address space if one was -+ * found. -+ * @as: The address space that has the fault -+ * @fault: Data relating to the fault -+ * -+ * This function will process a fault on a specific address space -+ */ -+static void kbase_mmu_interrupt_process(struct kbase_device *kbdev, - struct kbase_context *kctx, struct kbase_as *as, - struct kbase_fault *fault) - { -+ unsigned long flags; -+ - lockdep_assert_held(&kbdev->hwaccess_lock); - - dev_dbg(kbdev->dev, -- "Entering %s kctx %p, as %p\n", -+ "Entering %s kctx %pK, as %pK\n", - __func__, (void *)kctx, (void *)as); - - if (!kctx) { -@@ -228,11 +240,13 @@ void kbase_mmu_interrupt_process(struct kbase_device *kbdev, - * hw counters dumping in progress, signal the - * other thread that it failed - */ -+ spin_lock_irqsave(&kbdev->hwcnt.lock, flags); - if ((kbdev->hwcnt.kctx == kctx) && - (kbdev->hwcnt.backend.state == - KBASE_INSTR_STATE_DUMPING)) - kbdev->hwcnt.backend.state = - KBASE_INSTR_STATE_FAULT; -+ spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags); - - /* - * Stop the kctx from submitting more jobs and cause it -@@ -241,14 +255,10 @@ void kbase_mmu_interrupt_process(struct kbase_device *kbdev, - */ - kbasep_js_clear_submit_allowed(js_devdata, kctx); - -- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -- dev_warn(kbdev->dev, -- "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", -- as->number, fault->addr, -- fault->extra_addr); -- else -- dev_warn(kbdev->dev, "Bus error in AS%d at 0x%016llx\n", -- as->number, fault->addr); -+ dev_warn(kbdev->dev, -+ "Bus error in AS%d at VA=0x%016llx, IPA=0x%016llx\n", -+ as->number, fault->addr, -+ fault->extra_addr); - - /* - * We need to switch to UNMAPPED mode - but we do this in a -@@ -262,7 +272,7 @@ void kbase_mmu_interrupt_process(struct kbase_device *kbdev, - } - - dev_dbg(kbdev->dev, -- "Leaving %s kctx %p, as %p\n", -+ "Leaving %s kctx %pK, as %pK\n", - __func__, (void *)kctx, (void *)as); - } - -@@ -296,7 +306,6 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) - unsigned long flags; - u32 new_mask; - u32 tmp, bf_bits, pf_bits; -- bool gpu_lost = false; - - dev_dbg(kbdev->dev, "Entering %s irq_stat %u\n", - __func__, irq_stat); -@@ -362,22 +371,11 @@ void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat) - /* record the fault status */ - fault->status = kbase_reg_read(kbdev, MMU_AS_REG(as_no, - AS_FAULTSTATUS)); -- -- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { -- fault->extra_addr = kbase_reg_read(kbdev, -- MMU_AS_REG(as_no, AS_FAULTEXTRA_HI)); -- fault->extra_addr <<= 32; -- fault->extra_addr |= kbase_reg_read(kbdev, -- MMU_AS_REG(as_no, AS_FAULTEXTRA_LO)); -- } -- -- /* check if we still have GPU */ -- gpu_lost = kbase_is_gpu_lost(kbdev); -- if (gpu_lost) { -- if (kctx) -- kbasep_js_runpool_release_ctx(kbdev, kctx); -- return; -- } -+ fault->extra_addr = kbase_reg_read(kbdev, -+ MMU_AS_REG(as_no, AS_FAULTEXTRA_HI)); -+ fault->extra_addr <<= 32; -+ fault->extra_addr |= kbase_reg_read(kbdev, -+ MMU_AS_REG(as_no, AS_FAULTEXTRA_LO)); - - if (kbase_as_has_bus_fault(as, fault)) { - /* Mark bus fault as handled. -@@ -418,7 +416,23 @@ int kbase_mmu_switch_to_ir(struct kbase_context *const kctx, - struct kbase_va_region *const reg) - { - dev_dbg(kctx->kbdev->dev, -- "Switching to incremental rendering for region %p\n", -+ "Switching to incremental rendering for region %pK\n", - (void *)reg); - return kbase_job_slot_softstop_start_rp(kctx, reg); - } -+ -+int kbase_mmu_as_init(struct kbase_device *kbdev, int i) -+{ -+ kbdev->as[i].number = i; -+ kbdev->as[i].bf_data.addr = 0ULL; -+ kbdev->as[i].pf_data.addr = 0ULL; -+ -+ kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%d", 0, 1, i); -+ if (!kbdev->as[i].pf_wq) -+ return -ENOMEM; -+ -+ INIT_WORK(&kbdev->as[i].work_pagefault, kbase_mmu_page_fault_worker); -+ INIT_WORK(&kbdev->as[i].work_busfault, kbase_mmu_bus_fault_worker); -+ -+ return 0; -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c -index c2c7c4b..41ee07f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,13 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -- * @file mali_kbase_mmu.c -- * Base kernel MMU management. -+ * DOC: Base kernel MMU management. - */ - - #include <linux/kernel.h> -@@ -31,20 +29,20 @@ - #include <gpu/mali_kbase_gpu_fault.h> - #include <gpu/mali_kbase_gpu_regmap.h> - #include <tl/mali_kbase_tracepoints.h> --#include <mali_kbase_instr_defs.h> -+#include <backend/gpu/mali_kbase_instr_defs.h> - #include <mali_kbase_ctx_sched.h> - #include <mali_kbase_debug.h> - #include <mali_kbase_defs.h> - #include <mali_kbase_hw.h> - #include <mmu/mali_kbase_mmu_hw.h> --#include <mali_kbase_hwaccess_jm.h> --#include <mali_kbase_hwaccess_time.h> - #include <mali_kbase_mem.h> - #include <mali_kbase_reset_gpu.h> - #include <mmu/mali_kbase_mmu.h> - #include <mmu/mali_kbase_mmu_internal.h> - #include <mali_kbase_cs_experimental.h> -+#include <device/mali_kbase_device.h> - -+#include <mali_kbase_trace_gpu_mem.h> - #define KBASE_MMU_PAGE_ENTRIES 512 - - /** -@@ -82,21 +80,20 @@ static void kbase_mmu_flush_invalidate_no_ctx(struct kbase_device *kbdev, - u64 vpfn, size_t nr, bool sync, int as_nr); - - /** -- * kbase_mmu_sync_pgd - sync page directory to memory -+ * kbase_mmu_sync_pgd() - sync page directory to memory when needed. - * @kbdev: Device pointer. - * @handle: Address of DMA region. - * @size: Size of the region to sync. - * - * This should be called after each page directory update. - */ -- - static void kbase_mmu_sync_pgd(struct kbase_device *kbdev, - dma_addr_t handle, size_t size) - { -- /* If page table is not coherent then ensure the gpu can read -+ /* In non-coherent system, ensure the GPU can read - * the pages from memory - */ -- if (kbdev->system_coherency != COHERENCY_ACE) -+ if (kbdev->system_coherency == COHERENCY_NONE) - dma_sync_single_for_device(kbdev->dev, handle, size, - DMA_TO_DEVICE); - } -@@ -117,7 +114,7 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_context *kctx, u64 vpfn, - /** - * reg_grow_calc_extra_pages() - Calculate the number of backed pages to add to - * a region on a GPU page fault -- * -+ * @kbdev: KBase device - * @reg: The region that will be backed with more pages - * @fault_rel_pfn: PFN of the fault relative to the start of the region - * -@@ -133,23 +130,32 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_context *kctx, u64 vpfn, - static size_t reg_grow_calc_extra_pages(struct kbase_device *kbdev, - struct kbase_va_region *reg, size_t fault_rel_pfn) - { -- size_t multiple = reg->extent; -+ size_t multiple = reg->extension; - size_t reg_current_size = kbase_reg_current_backed_size(reg); - size_t minimum_extra = fault_rel_pfn - reg_current_size + 1; - size_t remainder; - - if (!multiple) { -- dev_warn(kbdev->dev, -- "VA Region 0x%llx extent was 0, allocator needs to set this properly for KBASE_REG_PF_GROW\n", -+ dev_warn( -+ kbdev->dev, -+ "VA Region 0x%llx extension was 0, allocator needs to set this properly for KBASE_REG_PF_GROW\n", - ((unsigned long long)reg->start_pfn) << PAGE_SHIFT); - return minimum_extra; - } - - /* Calculate the remainder to subtract from minimum_extra to make it -- * the desired (rounded down) multiple of the extent. -+ * the desired (rounded down) multiple of the extension. - * Depending on reg's flags, the base used for calculating multiples is - * different - */ -+ -+ /* multiple is based from the current backed size, even if the -+ * current backed size/pfn for end of committed memory are not -+ * themselves aligned to multiple -+ */ -+ remainder = minimum_extra % multiple; -+ -+#if !MALI_USE_CSF - if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { - /* multiple is based from the top of the initial commit, which - * has been allocated in such a way that (start_pfn + -@@ -175,13 +181,8 @@ static size_t reg_grow_calc_extra_pages(struct kbase_device *kbdev, - - remainder = pages_after_initial % multiple; - } -- } else { -- /* multiple is based from the current backed size, even if the -- * current backed size/pfn for end of committed memory are not -- * themselves aligned to multiple -- */ -- remainder = minimum_extra % multiple; - } -+#endif /* !MALI_USE_CSF */ - - if (remainder == 0) - return minimum_extra; -@@ -522,10 +523,15 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, - static void release_ctx(struct kbase_device *kbdev, - struct kbase_context *kctx) - { -+#if MALI_USE_CSF -+ CSTD_UNUSED(kbdev); -+ kbase_ctx_sched_release_ctx_lock(kctx); -+#else /* MALI_USE_CSF */ - kbasep_js_runpool_release_ctx(kbdev, kctx); -+#endif /* MALI_USE_CSF */ - } - --void page_fault_worker(struct work_struct *data) -+void kbase_mmu_page_fault_worker(struct work_struct *data) - { - u64 fault_pfn; - u32 fault_status; -@@ -544,7 +550,9 @@ void page_fault_worker(struct work_struct *data) - struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL }; - int i; - size_t current_backed_size; -- -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ size_t pages_trimmed = 0; -+#endif - - faulting_as = container_of(data, struct kbase_as, work_pagefault); - fault = &faulting_as->pf_data; -@@ -553,7 +561,7 @@ void page_fault_worker(struct work_struct *data) - - kbdev = container_of(faulting_as, struct kbase_device, as[as_no]); - dev_dbg(kbdev->dev, -- "Entering %s %p, fault_pfn %lld, as_no %d\n", -+ "Entering %s %pK, fault_pfn %lld, as_no %d\n", - __func__, (void *)data, fault_pfn, as_no); - - /* Grab the context that was already refcounted in kbase_mmu_interrupt() -@@ -568,6 +576,21 @@ void page_fault_worker(struct work_struct *data) - - KBASE_DEBUG_ASSERT(kctx->kbdev == kbdev); - -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+#if !MALI_USE_CSF -+ mutex_lock(&kctx->jctx.lock); -+#endif -+#endif -+ -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ /* check if we still have GPU */ -+ if (unlikely(kbase_is_gpu_removed(kbdev))) { -+ dev_dbg(kbdev->dev, -+ "%s: GPU has been removed\n", __func__); -+ goto fault_done; -+ } -+#endif -+ - if (unlikely(fault->protected_mode)) { - kbase_mmu_report_fault_and_kill(kctx, faulting_as, - "Protected mode fault", fault); -@@ -611,21 +634,13 @@ void page_fault_worker(struct work_struct *data) - goto fault_done; - - case AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT: -- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -- kbase_mmu_report_fault_and_kill(kctx, faulting_as, -- "Address size fault", fault); -- else -- kbase_mmu_report_fault_and_kill(kctx, faulting_as, -- "Unknown fault code", fault); -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Address size fault", fault); - goto fault_done; - - case AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT: -- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) -- kbase_mmu_report_fault_and_kill(kctx, faulting_as, -- "Memory attributes fault", fault); -- else -- kbase_mmu_report_fault_and_kill(kctx, faulting_as, -- "Unknown fault code", fault); -+ kbase_mmu_report_fault_and_kill(kctx, faulting_as, -+ "Memory attributes fault", fault); - goto fault_done; - - default: -@@ -691,6 +706,10 @@ page_fault_retry: - goto fault_done; - } - -+ if (AS_FAULTSTATUS_ACCESS_TYPE_GET(fault_status) == -+ AS_FAULTSTATUS_ACCESS_TYPE_READ) -+ dev_warn(kbdev->dev, "Grow on pagefault while reading"); -+ - /* find the size we need to grow it by - * we know the result fit in a size_t due to - * kbase_region_tracker_find_region_enclosing_address -@@ -758,6 +777,13 @@ page_fault_retry: - - pages_to_grow = 0; - -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if ((region->flags & KBASE_REG_ACTIVE_JIT_ALLOC) && !pages_trimmed) { -+ kbase_jit_request_phys_increase(kctx, new_pages); -+ pages_trimmed = new_pages; -+ } -+#endif -+ - spin_lock(&kctx->mem_partials_lock); - grown = page_fault_try_alloc(kctx, region, new_pages, &pages_to_grow, - &grow_2mb_pool, prealloc_sas); -@@ -818,7 +844,7 @@ page_fault_retry: - - if (kbase_mmu_switch_to_ir(kctx, region) >= 0) { - dev_dbg(kctx->kbdev->dev, -- "Get region %p for IR\n", -+ "Get region %pK for IR\n", - (void *)region); - kbase_va_region_alloc_get(kctx, region); - } -@@ -872,6 +898,13 @@ page_fault_retry: - } - } - #endif -+ -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (pages_trimmed) { -+ kbase_jit_done_phys_increase(kctx, pages_trimmed); -+ pages_trimmed = 0; -+ } -+#endif - kbase_gpu_vm_unlock(kctx); - } else { - int ret = -ENOMEM; -@@ -918,6 +951,17 @@ page_fault_retry: - } - - fault_done: -+#if MALI_JIT_PRESSURE_LIMIT_BASE -+ if (pages_trimmed) { -+ kbase_gpu_vm_lock(kctx); -+ kbase_jit_done_phys_increase(kctx, pages_trimmed); -+ kbase_gpu_vm_unlock(kctx); -+ } -+#if !MALI_USE_CSF -+ mutex_unlock(&kctx->jctx.lock); -+#endif -+#endif -+ - for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) - kfree(prealloc_sas[i]); - -@@ -928,7 +972,7 @@ fault_done: - release_ctx(kbdev, kctx); - - atomic_dec(&kbdev->faults_pending); -- dev_dbg(kbdev->dev, "Leaving page_fault_worker %p\n", (void *)data); -+ dev_dbg(kbdev->dev, "Leaving page_fault_worker %pK\n", (void *)data); - } - - static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, -@@ -938,7 +982,11 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, - int i; - struct page *p; - -+#ifdef CONFIG_MALI_2MB_ALLOC -+ p = kbase_mem_pool_alloc(&kbdev->mem_pools.large[mmut->group_id]); -+#else /* CONFIG_MALI_2MB_ALLOC */ - p = kbase_mem_pool_alloc(&kbdev->mem_pools.small[mmut->group_id]); -+#endif /* CONFIG_MALI_2MB_ALLOC */ - if (!p) - return 0; - -@@ -964,6 +1012,8 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, - - atomic_add(1, &kbdev->memdev.used_pages); - -+ kbase_trace_gpu_mem_usage_inc(kbdev, mmut->kctx, 1); -+ - for (i = 0; i < KBASE_MMU_PAGE_ENTRIES; i++) - kbdev->mmu_mode->entry_invalidate(&page[i]); - -@@ -973,8 +1023,12 @@ static phys_addr_t kbase_mmu_alloc_pgd(struct kbase_device *kbdev, - return page_to_phys(p); - - alloc_free: -- kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], p, -- false); -+ -+#ifdef CONFIG_MALI_2MB_ALLOC -+ kbase_mem_pool_free(&kbdev->mem_pools.large[mmut->group_id], p, false); -+#else /* CONFIG_MALI_2MB_ALLOC */ -+ kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], p, false); -+#endif /* CONFIG_MALI_2MB_ALLOC */ - - return 0; - } -@@ -1200,7 +1254,11 @@ int kbase_mmu_insert_single_page(struct kbase_context *kctx, u64 vpfn, - */ - mutex_unlock(&kctx->mmu.mmu_lock); - err = kbase_mem_pool_grow( -+#ifdef CONFIG_MALI_2MB_ALLOC -+ &kbdev->mem_pools.large[ -+#else - &kbdev->mem_pools.small[ -+#endif - kctx->mmu.group_id], - MIDGARD_MMU_BOTTOMLEVEL); - mutex_lock(&kctx->mmu.mmu_lock); -@@ -1279,7 +1337,11 @@ static inline void cleanup_empty_pte(struct kbase_device *kbdev, - - tmp_pgd = kbdev->mmu_mode->pte_to_phy_addr(*pte); - tmp_p = phys_to_page(tmp_pgd); -+#ifdef CONFIG_MALI_2MB_ALLOC -+ kbase_mem_pool_free(&kbdev->mem_pools.large[mmut->group_id], -+#else - kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], -+#endif - tmp_p, false); - - /* If the MMU tables belong to a context then we accounted the memory -@@ -1290,6 +1352,8 @@ static inline void cleanup_empty_pte(struct kbase_device *kbdev, - atomic_sub(1, &mmut->kctx->used_pages); - } - atomic_sub(1, &kbdev->memdev.used_pages); -+ -+ kbase_trace_gpu_mem_usage_dec(kbdev, mmut->kctx, 1); - } - - u64 kbase_mmu_create_ate(struct kbase_device *const kbdev, -@@ -1361,7 +1425,11 @@ int kbase_mmu_insert_pages_no_flush(struct kbase_device *kbdev, - */ - mutex_unlock(&mmut->mmu_lock); - err = kbase_mem_pool_grow( -+#ifdef CONFIG_MALI_2MB_ALLOC -+ &kbdev->mem_pools.large[mmut->group_id], -+#else - &kbdev->mem_pools.small[mmut->group_id], -+#endif - cur_level); - mutex_lock(&mmut->mmu_lock); - } while (!err); -@@ -1509,7 +1577,7 @@ static void kbase_mmu_flush_invalidate_noretain(struct kbase_context *kctx, - */ - dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); - -- if (kbase_prepare_to_reset_gpu_locked(kbdev)) -+ if (kbase_prepare_to_reset_gpu_locked(kbdev, RESET_FLAGS_NONE)) - kbase_reset_gpu_locked(kbdev); - } - } -@@ -1522,10 +1590,29 @@ static void kbase_mmu_flush_invalidate_as(struct kbase_device *kbdev, - { - int err; - u32 op; -+ bool gpu_powered; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags); -+ gpu_powered = kbdev->pm.backend.gpu_powered; -+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); -+ -+ /* GPU is off so there's no need to perform flush/invalidate. -+ * But even if GPU is not actually powered down, after gpu_powered flag -+ * was set to false, it is still safe to skip the flush/invalidate. -+ * The TLB invalidation will anyways be performed due to AS_COMMAND_UPDATE -+ * which is sent when address spaces are restored after gpu_powered flag -+ * is set to true. Flushing of L2 cache is certainly not required as L2 -+ * cache is definitely off if gpu_powered is false. -+ */ -+ if (!gpu_powered) -+ return; - - if (kbase_pm_context_active_handle_suspend(kbdev, - KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) { -- /* GPU is off so there's no need to perform flush/invalidate */ -+ /* GPU has just been powered off due to system suspend. -+ * So again, no need to perform flush/invalidate. -+ */ - return; - } - -@@ -1544,9 +1631,10 @@ static void kbase_mmu_flush_invalidate_as(struct kbase_device *kbdev, - /* Flush failed to complete, assume the GPU has hung and - * perform a reset to recover - */ -- dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issueing GPU soft-reset to recover\n"); -+ dev_err(kbdev->dev, "Flush for GPU page table update did not complete. Issuing GPU soft-reset to recover\n"); - -- if (kbase_prepare_to_reset_gpu(kbdev)) -+ if (kbase_prepare_to_reset_gpu( -+ kbdev, RESET_FLAGS_HWC_UNRECOVERABLE_ERROR)) - kbase_reset_gpu(kbdev); - } - -@@ -1577,9 +1665,13 @@ static void kbase_mmu_flush_invalidate(struct kbase_context *kctx, - return; - - kbdev = kctx->kbdev; -+#if !MALI_USE_CSF - mutex_lock(&kbdev->js_data.queue_mutex); - ctx_is_in_runpool = kbase_ctx_sched_inc_refcount(kctx); - mutex_unlock(&kbdev->js_data.queue_mutex); -+#else -+ ctx_is_in_runpool = kbase_ctx_sched_inc_refcount_if_as_valid(kctx); -+#endif /* !MALI_USE_CSF */ - - if (ctx_is_in_runpool) { - KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); -@@ -1621,6 +1713,7 @@ void kbase_mmu_disable(struct kbase_context *kctx) - KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID); - - lockdep_assert_held(&kctx->kbdev->hwaccess_lock); -+ lockdep_assert_held(&kctx->kbdev->mmu_hw_mutex); - - /* - * The address space is being disabled, drain all knowledge of it out -@@ -1832,7 +1925,11 @@ static int kbase_mmu_update_pages_no_flush(struct kbase_context *kctx, u64 vpfn, - */ - mutex_unlock(&kctx->mmu.mmu_lock); - err = kbase_mem_pool_grow( -+#ifdef CONFIG_MALI_2MB_ALLOC -+ &kbdev->mem_pools.large[ -+#else - &kbdev->mem_pools.small[ -+#endif - kctx->mmu.group_id], - MIDGARD_MMU_BOTTOMLEVEL); - mutex_lock(&kctx->mmu.mmu_lock); -@@ -1927,8 +2024,11 @@ static void mmu_teardown_level(struct kbase_device *kbdev, - } - - p = pfn_to_page(PFN_DOWN(pgd)); -- -+#ifdef CONFIG_MALI_2MB_ALLOC -+ kbase_mem_pool_free(&kbdev->mem_pools.large[mmut->group_id], -+#else - kbase_mem_pool_free(&kbdev->mem_pools.small[mmut->group_id], -+#endif - p, true); - - atomic_sub(1, &kbdev->memdev.used_pages); -@@ -1940,6 +2040,8 @@ static void mmu_teardown_level(struct kbase_device *kbdev, - kbase_process_page_usage_dec(mmut->kctx, 1); - atomic_sub(1, &mmut->kctx->used_pages); - } -+ -+ kbase_trace_gpu_mem_usage_dec(kbdev, mmut->kctx, 1); - } - - int kbase_mmu_init(struct kbase_device *const kbdev, -@@ -1969,7 +2071,11 @@ int kbase_mmu_init(struct kbase_device *const kbdev, - int err; - - err = kbase_mem_pool_grow( -+#ifdef CONFIG_MALI_2MB_ALLOC -+ &kbdev->mem_pools.large[mmut->group_id], -+#else - &kbdev->mem_pools.small[mmut->group_id], -+#endif - MIDGARD_MMU_BOTTOMLEVEL); - if (err) { - kbase_mmu_term(kbdev, mmut); -@@ -2000,6 +2106,11 @@ void kbase_mmu_term(struct kbase_device *kbdev, struct kbase_mmu_table *mmut) - mutex_destroy(&mmut->mmu_lock); - } - -+void kbase_mmu_as_term(struct kbase_device *kbdev, int i) -+{ -+ destroy_workqueue(kbdev->as[i].pf_wq); -+} -+ - static size_t kbasep_mmu_dump_level(struct kbase_context *kctx, phys_addr_t pgd, - int level, char ** const buffer, size_t *size_left) - { -@@ -2140,7 +2251,7 @@ fail_free: - } - KBASE_EXPORT_TEST_API(kbase_mmu_dump); - --void bus_fault_worker(struct work_struct *data) -+void kbase_mmu_bus_fault_worker(struct work_struct *data) - { - struct kbase_as *faulting_as; - int as_no; -@@ -2168,6 +2279,17 @@ void bus_fault_worker(struct work_struct *data) - return; - } - -+#ifdef CONFIG_MALI_ARBITER_SUPPORT -+ /* check if we still have GPU */ -+ if (unlikely(kbase_is_gpu_removed(kbdev))) { -+ dev_dbg(kbdev->dev, -+ "%s: GPU has been removed\n", __func__); -+ release_ctx(kbdev, kctx); -+ atomic_dec(&kbdev->faults_pending); -+ return; -+ } -+#endif -+ - if (unlikely(fault->protected_mode)) { - kbase_mmu_report_fault_and_kill(kctx, faulting_as, - "Permission failure", fault); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.h b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.h -index c9e27b1..a2d1a8e 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,13 +17,35 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_MMU_H_ - #define _KBASE_MMU_H_ - -+/** -+ * kbase_mmu_as_init() - Initialising GPU address space object. -+ * -+ * This is called from device probe to initialise an address space object -+ * of the device. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer). -+ * @i: Array index of address space object. -+ * -+ * Return: 0 on success and non-zero value on failure. -+ */ -+int kbase_mmu_as_init(struct kbase_device *kbdev, int i); -+ -+/** -+ * kbase_mmu_as_term() - Terminate address space object. -+ * -+ * This is called upon device termination to destroy -+ * the address space object of the device. -+ * -+ * @kbdev: The kbase device structure for the device (must be a valid pointer). -+ * @i: Array index of address space object. -+ */ -+void kbase_mmu_as_term(struct kbase_device *kbdev, int i); -+ - /** - * kbase_mmu_init - Initialise an object representing GPU page tables - * -@@ -115,4 +138,18 @@ int kbase_mmu_update_pages(struct kbase_context *kctx, u64 vpfn, - int kbase_mmu_bus_fault_interrupt(struct kbase_device *kbdev, u32 status, - u32 as_nr); - -+/** -+ * kbase_mmu_gpu_fault_interrupt() - Report a GPU fault. -+ * @kbdev: Kbase device pointer -+ * @status: GPU fault status -+ * @as_nr: Faulty address space -+ * @address: GPU fault address -+ * @as_valid: true if address space is valid -+ * -+ * This function builds GPU fault information to submit a work -+ * for reporting the details of the fault. -+ */ -+void kbase_mmu_gpu_fault_interrupt(struct kbase_device *kbdev, u32 status, -+ u32 as_nr, u64 address, bool as_valid); -+ - #endif /* _KBASE_MMU_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_hw.h b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_hw.h -index e6eef86..d1f1ff2 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_hw.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_hw.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2015, 2018-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2015, 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -@@ -39,6 +38,11 @@ struct kbase_context; - - /** - * enum kbase_mmu_fault_type - MMU fault type descriptor. -+ * @KBASE_MMU_FAULT_TYPE_UNKNOWN: unknown fault -+ * @KBASE_MMU_FAULT_TYPE_PAGE: page fault -+ * @KBASE_MMU_FAULT_TYPE_BUS: nus fault -+ * @KBASE_MMU_FAULT_TYPE_PAGE_UNEXPECTED: page_unexpected fault -+ * @KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED: bus_unexpected fault - */ - enum kbase_mmu_fault_type { - KBASE_MMU_FAULT_TYPE_UNKNOWN = 0, -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_hw_direct.c b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_hw_direct.c -index f22e73e..a99b988 100755 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_hw_direct.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_hw_direct.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/bitops.h> -@@ -25,8 +24,7 @@ - #include <mali_kbase_mem.h> - #include <mmu/mali_kbase_mmu_hw.h> - #include <tl/mali_kbase_tracepoints.h> --#include <backend/gpu/mali_kbase_device_internal.h> --#include <mali_kbase_as_fault_debugfs.h> -+#include <device/mali_kbase_device.h> - - /** - * lock_region() - Generate lockaddr to lock memory region in MMU -@@ -126,38 +124,33 @@ void kbase_mmu_hw_configure(struct kbase_device *kbdev, struct kbase_as *as) - struct kbase_mmu_setup *current_setup = &as->current_setup; - u64 transcfg = 0; - -- if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU)) { -- transcfg = current_setup->transcfg; -+ transcfg = current_setup->transcfg; - -- /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK -- * Clear PTW_MEMATTR bits -- */ -- transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; -- /* Enable correct PTW_MEMATTR bits */ -- transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; -- /* Ensure page-tables reads use read-allocate cache-policy in -- * the L2 -- */ -- transcfg |= AS_TRANSCFG_R_ALLOCATE; -- -- if (kbdev->system_coherency == COHERENCY_ACE) { -- /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) -- * Clear PTW_SH bits -- */ -- transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); -- /* Enable correct PTW_SH bits */ -- transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); -- } -+ /* Set flag AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK -+ * Clear PTW_MEMATTR bits -+ */ -+ transcfg &= ~AS_TRANSCFG_PTW_MEMATTR_MASK; -+ /* Enable correct PTW_MEMATTR bits */ -+ transcfg |= AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK; -+ /* Ensure page-tables reads use read-allocate cache-policy in -+ * the L2 -+ */ -+ transcfg |= AS_TRANSCFG_R_ALLOCATE; - -- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), -- transcfg); -- kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), -- (transcfg >> 32) & 0xFFFFFFFFUL); -- } else { -- if (kbdev->system_coherency == COHERENCY_ACE) -- current_setup->transtab |= AS_TRANSTAB_LPAE_SHARE_OUTER; -+ if (kbdev->system_coherency != COHERENCY_NONE) { -+ /* Set flag AS_TRANSCFG_PTW_SH_OS (outer shareable) -+ * Clear PTW_SH bits -+ */ -+ transcfg = (transcfg & ~AS_TRANSCFG_PTW_SH_MASK); -+ /* Enable correct PTW_SH bits */ -+ transcfg = (transcfg | AS_TRANSCFG_PTW_SH_OS); - } - -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_LO), -+ transcfg); -+ kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSCFG_HI), -+ (transcfg >> 32) & 0xFFFFFFFFUL); -+ - kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_LO), - current_setup->transtab & 0xFFFFFFFFUL); - kbase_reg_write(kbdev, MMU_AS_REG(as->number, AS_TRANSTAB_HI), -@@ -230,10 +223,11 @@ void kbase_mmu_hw_clear_fault(struct kbase_device *kbdev, struct kbase_as *as, - - /* Clear the page (and bus fault IRQ as well in case one occurred) */ - pf_bf_mask = MMU_PAGE_FAULT(as->number); -+#if !MALI_USE_CSF - if (type == KBASE_MMU_FAULT_TYPE_BUS || - type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) - pf_bf_mask |= MMU_BUS_ERROR(as->number); -- -+#endif - kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), pf_bf_mask); - - unlock: -@@ -261,10 +255,11 @@ void kbase_mmu_hw_enable_fault(struct kbase_device *kbdev, struct kbase_as *as, - irq_mask = kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK)) | - MMU_PAGE_FAULT(as->number); - -+#if !MALI_USE_CSF - if (type == KBASE_MMU_FAULT_TYPE_BUS || - type == KBASE_MMU_FAULT_TYPE_BUS_UNEXPECTED) - irq_mask |= MMU_BUS_ERROR(as->number); -- -+#endif - kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), irq_mask); - - unlock: -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_internal.h b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_internal.h -index 28bd341..b8cd55f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_internal.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_internal.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KBASE_MMU_INTERNAL_H_ -@@ -26,6 +25,15 @@ - void kbase_mmu_get_as_setup(struct kbase_mmu_table *mmut, - struct kbase_mmu_setup * const setup); - -+/** -+ * kbase_mmu_report_mcu_as_fault_and_reset - Report page fault for all -+ * address spaces and reset the GPU. -+ * @kbdev: The kbase_device the fault happened on -+ * @fault: Data relating to the fault -+ */ -+void kbase_mmu_report_mcu_as_fault_and_reset(struct kbase_device *kbdev, -+ struct kbase_fault *fault); -+ - void kbase_gpu_report_bus_fault_and_kill(struct kbase_context *kctx, - struct kbase_as *as, struct kbase_fault *fault); - -@@ -33,24 +41,11 @@ void kbase_mmu_report_fault_and_kill(struct kbase_context *kctx, - struct kbase_as *as, const char *reason_str, - struct kbase_fault *fault); - --/** -- * kbase_mmu_interrupt_process - Process a bus or page fault. -- * @kbdev The kbase_device the fault happened on -- * @kctx The kbase_context for the faulting address space if one was found. -- * @as The address space that has the fault -- * @fault Data relating to the fault -- * -- * This function will process a fault on a specific address space -- */ --void kbase_mmu_interrupt_process(struct kbase_device *kbdev, -- struct kbase_context *kctx, struct kbase_as *as, -- struct kbase_fault *fault); -- - /** - * kbase_mmu_switch_to_ir() - Switch to incremental rendering if possible -- * @kctx The kbase_context for the faulting address space. -- * @reg Reference of a growable GPU memory region in the same context. -- * Takes ownership of the reference if successful. -+ * @kctx: kbase_context for the faulting address space. -+ * @reg: of a growable GPU memory region in the same context. -+ * Takes ownership of the reference if successful. - * - * Used to switch to incremental rendering if we have nearly run out of - * virtual address space in a growable memory region. -@@ -60,4 +55,18 @@ void kbase_mmu_interrupt_process(struct kbase_device *kbdev, - int kbase_mmu_switch_to_ir(struct kbase_context *kctx, - struct kbase_va_region *reg); - -+/** -+ * kbase_mmu_page_fault_worker() - Process a page fault. -+ * -+ * @data: work_struct passed by queue_work() -+ */ -+void kbase_mmu_page_fault_worker(struct work_struct *data); -+ -+/** -+ * kbase_mmu_bus_fault_worker() - Process a bus fault. -+ * -+ * @data: work_struct passed by queue_work() -+ */ -+void kbase_mmu_bus_fault_worker(struct work_struct *data); -+ - #endif /* _KBASE_MMU_INTERNAL_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_mode_aarch64.c b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_mode_aarch64.c -index 02493e9..16b928d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_mode_aarch64.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_mode_aarch64.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2014, 2016-2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2014, 2016-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase.h" -@@ -48,25 +47,7 @@ - */ - static inline void page_table_entry_set(u64 *pte, u64 phy) - { --#if KERNEL_VERSION(3, 18, 13) <= LINUX_VERSION_CODE - WRITE_ONCE(*pte, phy); --#else --#ifdef CONFIG_64BIT -- barrier(); -- *pte = phy; -- barrier(); --#elif defined(CONFIG_ARM) -- barrier(); -- asm volatile("ldrd r0, [%1]\n\t" -- "strd r0, %0\n\t" -- : "=m" (*pte) -- : "r" (&phy) -- : "r0", "r1"); -- barrier(); --#else --#error "64-bit atomic write must be implemented for your architecture" --#endif --#endif - } - - static void mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_mode_lpae.c b/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_mode_lpae.c -deleted file mode 100644 -index 91a2d7a..0000000 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu_mode_lpae.c -+++ /dev/null -@@ -1,215 +0,0 @@ --/* -- * -- * (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- */ -- -- --#include "mali_kbase.h" --#include <gpu/mali_kbase_gpu_regmap.h> --#include "mali_kbase_defs.h" -- --#define ENTRY_TYPE_MASK 3ULL --#define ENTRY_IS_ATE 1ULL --#define ENTRY_IS_INVAL 2ULL --#define ENTRY_IS_PTE 3ULL -- --#define ENTRY_ATTR_BITS (7ULL << 2) /* bits 4:2 */ --#define ENTRY_RD_BIT (1ULL << 6) --#define ENTRY_WR_BIT (1ULL << 7) --#define ENTRY_SHARE_BITS (3ULL << 8) /* bits 9:8 */ --#define ENTRY_ACCESS_BIT (1ULL << 10) --#define ENTRY_NX_BIT (1ULL << 54) -- --#define ENTRY_FLAGS_MASK (ENTRY_ATTR_BITS | ENTRY_RD_BIT | ENTRY_WR_BIT | \ -- ENTRY_SHARE_BITS | ENTRY_ACCESS_BIT | ENTRY_NX_BIT) -- --/* Helper Function to perform assignment of page table entries, to -- * ensure the use of strd, which is required on LPAE systems. -- */ --static inline void page_table_entry_set(u64 *pte, u64 phy) --{ --#if KERNEL_VERSION(3, 18, 13) <= LINUX_VERSION_CODE -- WRITE_ONCE(*pte, phy); --#else --#ifdef CONFIG_64BIT -- barrier(); -- *pte = phy; -- barrier(); --#elif defined(CONFIG_ARM) -- barrier(); -- asm volatile("ldrd r0, [%1]\n\t" -- "strd r0, %0\n\t" -- : "=m" (*pte) -- : "r" (&phy) -- : "r0", "r1"); -- barrier(); --#else --#error "64-bit atomic write must be implemented for your architecture" --#endif --#endif --} -- --static void mmu_get_as_setup(struct kbase_mmu_table *mmut, -- struct kbase_mmu_setup * const setup) --{ -- /* Set up the required caching policies at the correct indices -- * in the memattr register. -- */ -- setup->memattr = -- (AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY << -- (AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY * 8)) | -- (AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL << -- (AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL * 8)) | -- (AS_MEMATTR_LPAE_WRITE_ALLOC << -- (AS_MEMATTR_INDEX_WRITE_ALLOC * 8)) | -- (AS_MEMATTR_LPAE_OUTER_IMPL_DEF << -- (AS_MEMATTR_INDEX_OUTER_IMPL_DEF * 8)) | -- (AS_MEMATTR_LPAE_OUTER_WA << -- (AS_MEMATTR_INDEX_OUTER_WA * 8)) | -- 0; /* The other indices are unused for now */ -- -- setup->transtab = ((u64)mmut->pgd & -- ((0xFFFFFFFFULL << 32) | AS_TRANSTAB_LPAE_ADDR_SPACE_MASK)) | -- AS_TRANSTAB_LPAE_ADRMODE_TABLE | -- AS_TRANSTAB_LPAE_READ_INNER; -- -- setup->transcfg = 0; --} -- --static void mmu_update(struct kbase_device *kbdev, -- struct kbase_mmu_table *mmut, -- int as_nr) --{ -- struct kbase_as *as; -- struct kbase_mmu_setup *current_setup; -- -- if (WARN_ON(as_nr == KBASEP_AS_NR_INVALID)) -- return; -- -- as = &kbdev->as[as_nr]; -- current_setup = &as->current_setup; -- -- mmu_get_as_setup(mmut, current_setup); -- -- /* Apply the address space setting */ -- kbase_mmu_hw_configure(kbdev, as); --} -- --static void mmu_disable_as(struct kbase_device *kbdev, int as_nr) --{ -- struct kbase_as * const as = &kbdev->as[as_nr]; -- struct kbase_mmu_setup * const current_setup = &as->current_setup; -- -- current_setup->transtab = AS_TRANSTAB_LPAE_ADRMODE_UNMAPPED; -- -- /* Apply the address space setting */ -- kbase_mmu_hw_configure(kbdev, as); --} -- --static phys_addr_t pte_to_phy_addr(u64 entry) --{ -- if (!(entry & 1)) -- return 0; -- -- return entry & ~0xFFF; --} -- --static int ate_is_valid(u64 ate, int const level) --{ -- return ((ate & ENTRY_TYPE_MASK) == ENTRY_IS_ATE); --} -- --static int pte_is_valid(u64 pte, int const level) --{ -- return ((pte & ENTRY_TYPE_MASK) == ENTRY_IS_PTE); --} -- --/* -- * Map KBASE_REG flags to MMU flags -- */ --static u64 get_mmu_flags(unsigned long flags) --{ -- u64 mmu_flags; -- unsigned long memattr_idx; -- -- memattr_idx = KBASE_REG_MEMATTR_VALUE(flags); -- if (WARN(memattr_idx == AS_MEMATTR_INDEX_NON_CACHEABLE, -- "Legacy Mode MMU cannot honor GPU non-cachable memory, will use default instead\n")) -- memattr_idx = AS_MEMATTR_INDEX_DEFAULT; -- /* store mem_attr index as 4:2, noting that: -- * - macro called above ensures 3 bits already -- * - all AS_MEMATTR_INDEX_<...> macros only use 3 bits -- */ -- mmu_flags = memattr_idx << 2; -- -- /* write perm if requested */ -- mmu_flags |= (flags & KBASE_REG_GPU_WR) ? ENTRY_WR_BIT : 0; -- /* read perm if requested */ -- mmu_flags |= (flags & KBASE_REG_GPU_RD) ? ENTRY_RD_BIT : 0; -- /* nx if requested */ -- mmu_flags |= (flags & KBASE_REG_GPU_NX) ? ENTRY_NX_BIT : 0; -- -- if (flags & KBASE_REG_SHARE_BOTH) { -- /* inner and outer shareable */ -- mmu_flags |= SHARE_BOTH_BITS; -- } else if (flags & KBASE_REG_SHARE_IN) { -- /* inner shareable coherency */ -- mmu_flags |= SHARE_INNER_BITS; -- } -- -- return mmu_flags; --} -- --static void entry_set_ate(u64 *entry, -- struct tagged_addr phy, -- unsigned long flags, -- int const level) --{ -- page_table_entry_set(entry, as_phys_addr_t(phy) | get_mmu_flags(flags) | -- ENTRY_IS_ATE); --} -- --static void entry_set_pte(u64 *entry, phys_addr_t phy) --{ -- page_table_entry_set(entry, (phy & ~0xFFF) | ENTRY_IS_PTE); --} -- --static void entry_invalidate(u64 *entry) --{ -- page_table_entry_set(entry, ENTRY_IS_INVAL); --} -- --static struct kbase_mmu_mode const lpae_mode = { -- .update = mmu_update, -- .get_as_setup = mmu_get_as_setup, -- .disable_as = mmu_disable_as, -- .pte_to_phy_addr = pte_to_phy_addr, -- .ate_is_valid = ate_is_valid, -- .pte_is_valid = pte_is_valid, -- .entry_set_ate = entry_set_ate, -- .entry_set_pte = entry_set_pte, -- .entry_invalidate = entry_invalidate, -- .flags = 0 --}; -- --struct kbase_mmu_mode const *kbase_mmu_mode_get_lpae(void) --{ -- return &lpae_mode; --} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/Kconfig b/dvalin/kernel/drivers/gpu/arm/midgard/platform/Kconfig -index ef9fb96..5f0118d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/Kconfig -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/Kconfig -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012-2013, 2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2013, 2017, 2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,11 +16,7 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # --# -- -- - - # Add your platform specific Kconfig file here - # -@@ -27,4 +24,3 @@ - # - # Where xxx is the platform name is the name set in MALI_PLATFORM_NAME - # -- -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/Kbuild -old mode 100644 -new mode 100755 -index 0a82eaf..4f3fc0d ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012-2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2017, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,8 +16,6 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/Kbuild.rej b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/Kbuild.rej -new file mode 100644 -index 0000000..dbc2029 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/Kbuild.rej -@@ -0,0 +1,17 @@ -+--- drivers/gpu/arm/midgard/platform/devicetree/Kbuild -++++ drivers/gpu/arm/midgard/platform/devicetree/Kbuild -+@@ -16,10 +17,10 @@ -+ # along with this program; if not, you can access it online at -+ # http://www.gnu.org/licenses/gpl-2.0.html. -+ # -+-# SPDX-License-Identifier: GPL-2.0 -+-# -+ # -+ -+ mali_kbase-y += \ -+- $(MALI_PLATFORM_DIR)/mali_kbase_config_devicetree.o \ -+- $(MALI_PLATFORM_DIR)/mali_kbase_runtime_pm.o -++ platform/$(MALI_PLATFORM_DIR)/mali_kbase_config_devicetree.o \ -++ platform/$(MALI_PLATFORM_DIR)/mali_kbase_config_platform.o \ -++ platform/$(MALI_PLATFORM_DIR)/mali_kbase_runtime_pm.o \ -++ platform/$(MALI_PLATFORM_DIR)/mali_kbase_clk_rate_trace.o -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_clk_rate_trace.c b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_clk_rate_trace.c -new file mode 100644 -index 0000000..4bcd585 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_clk_rate_trace.c -@@ -0,0 +1,105 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2015, 2017-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <mali_kbase_defs.h> -+#include <linux/clk.h> -+#include "mali_kbase_config_platform.h" -+ -+#if MALI_USE_CSF -+#include <asm/arch_timer.h> -+#endif -+ -+static void *enumerate_gpu_clk(struct kbase_device *kbdev, -+ unsigned int index) -+{ -+ if (index >= kbdev->nr_clocks) -+ return NULL; -+ -+#if MALI_USE_CSF -+ if (of_machine_is_compatible("arm,juno")) -+ WARN_ON(kbdev->nr_clocks != 1); -+#endif -+ -+ return kbdev->clocks[index]; -+} -+ -+static unsigned long get_gpu_clk_rate(struct kbase_device *kbdev, -+ void *gpu_clk_handle) -+{ -+#if MALI_USE_CSF -+ /* On Juno fpga platforms, the GPU clock rate is reported as 600 MHZ at -+ * the boot time. Then after the first call to kbase_devfreq_target() -+ * the clock rate is reported as 450 MHZ and the frequency does not -+ * change after that. But the actual frequency at which GPU operates -+ * is always 50 MHz, which is equal to the frequency of system counter -+ * and HW counters also increment at the same rate. -+ * DVFS, which is a client of kbase_ipa_control, needs normalization of -+ * GPU_ACTIVE counter to calculate the time for which GPU has been busy. -+ * So for the correct normalization need to return the system counter -+ * frequency value. -+ * This is a reasonable workaround as the frequency value remains same -+ * throughout. It can be removed after GPUCORE-25693. -+ */ -+ if (of_machine_is_compatible("arm,juno")) -+ return arch_timer_get_cntfrq(); -+#endif -+ -+ return clk_get_rate((struct clk *)gpu_clk_handle); -+} -+ -+static int gpu_clk_notifier_register(struct kbase_device *kbdev, -+ void *gpu_clk_handle, struct notifier_block *nb) -+{ -+ compiletime_assert(offsetof(struct clk_notifier_data, clk) == -+ offsetof(struct kbase_gpu_clk_notifier_data, gpu_clk_handle), -+ "mismatch in the offset of clk member"); -+ -+ compiletime_assert(sizeof(((struct clk_notifier_data *)0)->clk) == -+ sizeof(((struct kbase_gpu_clk_notifier_data *)0)->gpu_clk_handle), -+ "mismatch in the size of clk member"); -+ -+#if MALI_USE_CSF -+ /* Frequency is fixed on Juno platforms */ -+ if (of_machine_is_compatible("arm,juno")) -+ return 0; -+#endif -+ -+ return clk_notifier_register((struct clk *)gpu_clk_handle, nb); -+} -+ -+static void gpu_clk_notifier_unregister(struct kbase_device *kbdev, -+ void *gpu_clk_handle, struct notifier_block *nb) -+{ -+#if MALI_USE_CSF -+ if (of_machine_is_compatible("arm,juno")) -+ return; -+#endif -+ -+ clk_notifier_unregister((struct clk *)gpu_clk_handle, nb); -+} -+ -+struct kbase_clk_rate_trace_op_conf clk_rate_trace_ops = { -+ .get_gpu_clk_rate = get_gpu_clk_rate, -+ .enumerate_gpu_clk = enumerate_gpu_clk, -+ .gpu_clk_notifier_register = gpu_clk_notifier_register, -+ .gpu_clk_notifier_unregister = gpu_clk_notifier_unregister, -+}; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c -old mode 100644 -new mode 100755 -index 55a50c4..d729ffb ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_devicetree.c -@@ -1,20 +1,28 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* -- * mali_kbase_config_devicetree.c - * -- * Copyright (C) 2017 Amlogic, Inc. All rights reserved. -+ * (C) COPYRIGHT 2015, 2017, 2020-2021 ARM Limited. All rights reserved. - * -- * This program is free software; you can redistribute it and/or modify -- * it under the terms of the GNU General Public License as published by -- * the Free Software Foundation; either version 2 of the License, or -- * (at your option) any later version. -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. - * -- * This program is distributed in the hope that it will be useful, but WITHOUT -- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -- * more details. -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -+#include <mali_kbase.h> -+#include <mali_kbase_config.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> -+ - #ifdef CONFIG_DEVFREQ_THERMAL - #include <linux/version.h> - #include <linux/devfreq_cooling.h> -@@ -82,3 +90,14 @@ void kbase_platform_unregister(void) - { - } - #endif -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+#if MALI_USE_CSF -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -+#else -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, u32 util_cl_share[2]) -+#endif -+{ -+ return 1; -+} -+#endif /* CONFIG_MALI_MIDGARD_DVFS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.c b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.c -new file mode 100644 -index 0000000..63aa33f ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.c -@@ -0,0 +1,43 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <mali_kbase.h> -+#include <mali_kbase_defs.h> -+#include <mali_kbase_config.h> -+#include "mali_kbase_config_platform.h" -+#include <device/mali_kbase_device.h> -+#include <mali_kbase_hwaccess_time.h> -+#include <gpu/mali_kbase_gpu_regmap.h> -+ -+#include <linux/kthread.h> -+#include <linux/timer.h> -+#include <linux/jiffies.h> -+#include <linux/wait.h> -+#include <linux/delay.h> -+#include <linux/gcd.h> -+#include <asm/arch_timer.h> -+ -+struct kbase_platform_funcs_conf platform_funcs = { -+ .platform_init_func = NULL, -+ .platform_term_func = NULL, -+ .platform_late_init_func = NULL, -+ .platform_late_term_func = NULL, -+}; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h -old mode 100644 -new mode 100755 -index 233a18e..6cb3b53 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h -@@ -1,45 +1,22 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* -- * mali_kbase_config_platform.h - * -- * Copyright (C) 2017 Amlogic, Inc. All rights reserved. -+ * (C) COPYRIGHT 2014-2017, 2020-2021 ARM Limited. All rights reserved. - * -- * This program is free software; you can redistribute it and/or modify -- * it under the terms of the GNU General Public License as published by -- * the Free Software Foundation; either version 2 of the License, or -- * (at your option) any later version. -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. - * -- * This program is distributed in the hope that it will be useful, but WITHOUT -- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -- * more details. -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. - * -- */ -- --/** -- * Maximum frequency GPU will be clocked at. Given in kHz. -- * This must be specified as there is no default value. -- * -- * Attached value: number in kHz -- * Default value: NA -- */ --#define GPU_FREQ_KHZ_MAX (750000) --/** -- * Minimum frequency GPU will be clocked at. Given in kHz. -- * This must be specified as there is no default value. -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * Attached value: number in kHz -- * Default value: NA -- */ --#define GPU_FREQ_KHZ_MIN (100000) -- --/** -- * CPU_SPEED_FUNC - A pointer to a function that calculates the CPU clock -- * -- * CPU clock speed of the platform is in MHz - see kbase_cpu_clk_speed_func -- * for the function prototype. -- * -- * Attached value: A kbase_cpu_clk_speed_func. -- * Default Value: NA - */ - #define CPU_SPEED_FUNC (NULL) - -@@ -81,7 +58,12 @@ extern struct devfreq_cooling_ops t83x_model_ops; - #else - #define POWER_MODEL_CALLBACKS (NULL) - #endif -+ -+#define CLK_RATE_TRACE_OPS (&clk_rate_trace_ops) -+ - extern struct kbase_pm_callback_conf pm_callbacks; -+extern struct kbase_clk_rate_trace_op_conf clk_rate_trace_ops; -+extern struct kbase_platform_funcs_conf platform_funcs; - - void mali_dev_freeze(void); - void mali_dev_restore(void); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h.rej b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h.rej -new file mode 100644 -index 0000000..277acaa ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h.rej -@@ -0,0 +1,42 @@ -+--- drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h -++++ drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_config_platform.h -+@@ -1,11 +1,12 @@ -++/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+ /* -+ * -+- * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -++ * (C) COPYRIGHT 2014-2017, 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+- * of such GNU licence. -++ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+@@ -16,8 +17,6 @@ -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+- * SPDX-License-Identifier: GPL-2.0 -+- * -+ */ -+ -+ /** -+@@ -34,10 +33,13 @@ -+ * Attached value: pointer to @ref kbase_platform_funcs_conf -+ * Default value: See @ref kbase_platform_funcs_conf -+ */ -+-#define PLATFORM_FUNCS (NULL) -++#define PLATFORM_FUNCS (&platform_funcs) -+ -+-extern struct kbase_pm_callback_conf pm_callbacks; -++#define CLK_RATE_TRACE_OPS (&clk_rate_trace_ops) -+ -++extern struct kbase_pm_callback_conf pm_callbacks; -++extern struct kbase_clk_rate_trace_op_conf clk_rate_trace_ops; -++extern struct kbase_platform_funcs_conf platform_funcs; -+ /** -+ * Autosuspend delay -+ * -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c -old mode 100644 -new mode 100755 -index ee18718..4a0b72c ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/devicetree/mali_kbase_runtime_pm.c -@@ -1,21 +1,24 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* -- * mali_kbase_runtime_pm.c - * -- * Copyright (C) 2017 Amlogic, Inc. All rights reserved. -+ * (C) COPYRIGHT 2015-2021 ARM Limited. All rights reserved. - * -- * This program is free software; you can redistribute it and/or modify -- * it under the terms of the GNU General Public License as published by -- * the Free Software Foundation; either version 2 of the License, or -- * (at your option) any later version. -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. - * -- * This program is distributed in the hope that it will be useful, but WITHOUT -- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -- * more details. -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - --//#define DEBUG - #include <mali_kbase.h> - #include <mali_kbase_defs.h> - #include <linux/pm_runtime.h> -@@ -25,6 +28,7 @@ - #include <linux/delay.h> - #include <linux/io.h> - #include <backend/gpu/mali_kbase_device_internal.h> -+ - #include "mali_kbase_config_platform.h" - #include "mali_scaling.h" - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/Kbuild -index 6780e4c..b547366 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2013, 2016-2017, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,10 +16,8 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - mali_kbase-y += \ -- $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ -- mali_kbase_platform_fake.o -+ platform/$(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ -+ mali_kbase_platform_fake.o -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h -index fac3cd5..d8682db 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_platform.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c -index d165ce2..ff1ee65 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress/mali_kbase_config_vexpress.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,18 +17,16 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #include <linux/ioport.h> - #include <mali_kbase.h> - #include <mali_kbase_defs.h> - #include <mali_kbase_config.h> - #include "mali_kbase_config_platform.h" - -+#include <backend/gpu/mali_kbase_pm_internal.h> -+ - #ifndef CONFIG_OF - static struct kbase_io_resources io_resources = { - .job_irq_number = 68, -@@ -67,3 +66,14 @@ struct kbase_platform_config *kbase_get_platform_config(void) - { - return &versatile_platform_config; - } -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+#if MALI_USE_CSF -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -+#else -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, u32 util_cl_share[2]) -+#endif -+{ -+ return 1; -+} -+#endif /* CONFIG_MALI_MIDGARD_DVFS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild -index 51b408e..b547366 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2013-2014, 2016-2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2013, 2016-2017, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,10 +16,8 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - mali_kbase-y += \ -- $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ -- mali_kbase_platform_fake.o -+ platform/$(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ -+ mali_kbase_platform_fake.o -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h -index fac3cd5..d8682db 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_platform.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c -index efca0a5..fee6a36 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_1xv7_a57/mali_kbase_config_vexpress.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2014, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2014, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/ioport.h> -@@ -25,6 +24,8 @@ - #include <mali_kbase_defs.h> - #include <mali_kbase_config.h> - -+#include <backend/gpu/mali_kbase_pm_internal.h> -+ - #ifndef CONFIG_OF - static struct kbase_io_resources io_resources = { - .job_irq_number = 68, -@@ -63,3 +64,14 @@ struct kbase_platform_config *kbase_get_platform_config(void) - { - return &versatile_platform_config; - } -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+#if MALI_USE_CSF -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -+#else -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, u32 util_cl_share[2]) -+#endif -+{ -+ return 1; -+} -+#endif /* CONFIG_MALI_MIDGARD_DVFS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild -index e07709c..bb92c47 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2012-2013, 2016-2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2012-2013, 2016-2017, 2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,11 +16,9 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - - mali_kbase-y += \ -- $(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ -- $(MALI_PLATFORM_DIR)/mali_kbase_cpu_vexpress.o \ -- mali_kbase_platform_fake.o -+ platform/$(MALI_PLATFORM_DIR)/mali_kbase_config_vexpress.o \ -+ platform/$(MALI_PLATFORM_DIR)/mali_kbase_cpu_vexpress.o \ -+ mali_kbase_platform_fake.o -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h -index fac3cd5..d8682db 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_platform.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014-2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014-2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /** -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c -index b6714b9..f6fb9aa 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/platform/vexpress_6xvirtex7_10mhz/mali_kbase_config_vexpress.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2011-2014, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2011-2014, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,17 +17,15 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - #include <linux/ioport.h> - #include <mali_kbase.h> - #include <mali_kbase_defs.h> - #include <mali_kbase_config.h> - -+#include <backend/gpu/mali_kbase_pm_internal.h> -+ - #ifndef CONFIG_OF - static struct kbase_io_resources io_resources = { - .job_irq_number = 75, -@@ -65,3 +64,14 @@ struct kbase_platform_config *kbase_get_platform_config(void) - { - return &versatile_platform_config; - } -+ -+#ifdef CONFIG_MALI_MIDGARD_DVFS -+#if MALI_USE_CSF -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation) -+#else -+int kbase_platform_dvfs_event(struct kbase_device *kbdev, u32 utilisation, u32 util_gl_share, u32 util_cl_share[2]) -+#endif -+{ -+ return 1; -+} -+#endif /* CONFIG_MALI_MIDGARD_DVFS */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/protected_mode_switcher.h b/dvalin/kernel/drivers/gpu/arm/midgard/protected_mode_switcher.h -index 8778d81..9dd9253 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/protected_mode_switcher.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/protected_mode_switcher.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _PROTECTED_MODE_SWITCH_H_ -@@ -28,35 +27,23 @@ struct protected_mode_device; - /** - * struct protected_mode_ops - Callbacks for protected mode switch operations - * -- * @protected_mode_enable: Callback to enable protected mode for device -+ * @protected_mode_enable: Callback to enable protected mode for device, and -+ * reset device -+ * Returns 0 on success, non-zero on error - * @protected_mode_disable: Callback to disable protected mode for device -+ * Returns 0 on success, non-zero on error - */ - struct protected_mode_ops { -- /** -- * protected_mode_enable() - Enable protected mode on device -- * @dev: The struct device -- * -- * Return: 0 on success, non-zero on error -- */ - int (*protected_mode_enable)( - struct protected_mode_device *protected_dev); -- -- /** -- * protected_mode_disable() - Disable protected mode on device, and -- * reset device -- * @dev: The struct device -- * -- * Return: 0 on success, non-zero on error -- */ - int (*protected_mode_disable)( - struct protected_mode_device *protected_dev); - }; - - /** - * struct protected_mode_device - Device structure for protected mode devices -- * -- * @ops - Callbacks associated with this device -- * @data - Pointer to device private data -+ * @ops: Callbacks associated with this device -+ * @data: Pointer to device private data - * - * This structure should be registered with the platform device using - * platform_set_drvdata(). -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/tests/Kbuild -index df16a77..ee3de7b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,9 +16,15 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - -+ccflags-y += -I$(src)/include \ -+ -I$(src) -+ -+subdir-ccflags-y += -I$(src)/include \ -+ -I$(src) -+ - obj-$(CONFIG_MALI_KUTF) += kutf/ --obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test/ -+obj-$(CONFIG_MALI_KUTF_IRQ_TEST) += mali_kutf_irq_test/ -+obj-$(CONFIG_MALI_KUTF_CLK_RATE_TRACE) += mali_kutf_clk_rate_trace/kernel/ -+ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/Kconfig b/dvalin/kernel/drivers/gpu/arm/midgard/tests/Kconfig -index fa91aea..a86e1ce 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/Kconfig -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/Kconfig -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,9 +16,42 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - --source "drivers/gpu/arm/midgard/tests/kutf/Kconfig" --source "drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig" -+menuconfig MALI_KUTF -+ bool "Build Mali Kernel Unit Test Framework modules" -+ depends on MALI_MIDGARD && MALI_DEBUG -+ default y if MALI_DEBUG -+ help -+ This option will build the Mali testing framework modules. -+ -+ Modules: -+ - kutf.ko -+ - kutf_test.ko -+ -+config MALI_KUTF_IRQ_TEST -+ bool "Build Mali KUTF IRQ test module" -+ depends on MALI_KUTF -+ default y -+ help -+ This option will build the IRQ latency measurement test module. -+ It can determine the latency of the Mali GPU IRQ on your system. -+ -+ Modules: -+ - mali_kutf_irq_test.ko -+ -+config MALI_KUTF_CLK_RATE_TRACE -+ bool "Build Mali KUTF Clock rate trace test module" -+ depends on MALI_KUTF -+ default y -+ help -+ This option will build the clock rate trace portal test module. -+ It can test the clocks integration into the platform and exercise some -+ basic trace test in the system. -+ -+ Modules: -+ - mali_kutf_clk_rate_trace_test_portal.ko -+ -+ -+comment "Enable MALI_DEBUG for KUTF modules support" -+ depends on MALI_MIDGARD && !MALI_DEBUG && MALI_KUTF -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/Mconfig b/dvalin/kernel/drivers/gpu/arm/midgard/tests/Mconfig -index be3fedb..167facd 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/Mconfig -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/Mconfig -@@ -1,38 +1,61 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. - # --# A copy of the licence is included with the program, and can also be obtained --# from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, --# Boston, MA 02110-1301, USA. - # - --config UNIT_TEST_KERNEL_MODULES -- bool -- default y if UNIT_TEST_CODE && BUILD_KERNEL_MODULES -- default n -- --config BUILD_IPA_TESTS -- bool -- default y if UNIT_TEST_KERNEL_MODULES && MALI_DEVFREQ -- default n -- --config BUILD_IPA_UNIT_TESTS -- bool -- default y if NO_MALI && BUILD_IPA_TESTS -- default n -- --config BUILD_CSF_TESTS -- bool -- default y if UNIT_TEST_KERNEL_MODULES && GPU_HAS_CSF -- default n -- --config BUILD_ARBIF_TESTS -- bool -- default y if UNIT_TEST_KERNEL_MODULES && MALI_ARBITER_SUPPORT -- default n -+menuconfig MALI_KUTF -+ bool "Build Mali Kernel Unit Test Framework modules" -+ depends on MALI_MIDGARD && MALI_DEBUG -+ default y if BACKEND_KERNEL && MALI_DEBUG -+ help -+ This option will build the Mali testing framework modules. -+ -+ Modules: -+ - kutf.ko -+ - kutf_test.ko -+ -+config MALI_KUTF_IRQ_TEST -+ bool "Build Mali KUTF IRQ test module" -+ depends on MALI_KUTF -+ default y -+ help -+ This option will build the IRQ latency measurement test module. -+ It can determine the latency of the Mali GPU IRQ on your system. -+ -+ Modules: -+ - mali_kutf_irq_test.ko - -+config MALI_KUTF_CLK_RATE_TRACE -+ bool "Build Mali KUTF Clock rate trace test module" -+ depends on MALI_KUTF -+ default y -+ help -+ This option will build the clock rate trace portal test module. -+ It can test the clocks integration into the platform and exercise some -+ basic trace test in the system. -+ -+ Modules: -+ - mali_kutf_clk_rate_trace_test_portal.ko -+ -+ -+# Enable MALI_DEBUG for KUTF modules support -+ -+config UNIT_TEST_KERNEL_MODULES -+ bool -+ default y if UNIT_TEST_CODE && BACKEND_KERNEL -+ default n -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/build.bp b/dvalin/kernel/drivers/gpu/arm/midgard/tests/build.bp -new file mode 100644 -index 0000000..9d6137d ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/build.bp -@@ -0,0 +1,40 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+bob_defaults { -+ name: "kernel_test_includes", -+ local_include_dirs: [ -+ "include", -+ "./../../", -+ "./../", -+ "./" -+ ], -+} -+ -+bob_defaults { -+ name: "kernel_test_configs", -+ mali_kutf: { -+ kbuild_options: ["CONFIG_MALI_KUTF=y"], -+ }, -+ unit_test_kernel_modules: { -+ kbuild_options: ["CONFIG_UNIT_TEST_KERNEL_MODULES=y"], -+ }, -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_helpers.h b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_helpers.h -index 15e168c..c4c713c 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_helpers.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_helpers.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KERNEL_UTF_HELPERS_H_ -@@ -33,6 +32,14 @@ - - #include <kutf/kutf_suite.h> - -+/** -+ * kutf_helper_pending_input() - Check any pending lines sent by user space -+ * @context: KUTF context -+ * -+ * Return: true if there are pending lines, otherwise false -+ */ -+bool kutf_helper_pending_input(struct kutf_context *context); -+ - /** - * kutf_helper_input_dequeue() - Dequeue a line sent by user space - * @context: KUTF context -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_helpers_user.h b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_helpers_user.h -index 3b1300e..e147cbb 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_helpers_user.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_helpers_user.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KERNEL_UTF_HELPERS_USER_H_ -@@ -63,7 +62,8 @@ struct kutf_helper_named_val { - * unrecoverable) - * - * Positive values indicate correct access but invalid parsing (can be -- * recovered from assuming data in the future is correct) */ -+ * recovered from assuming data in the future is correct) -+ */ - enum kutf_helper_err { - /* No error - must be zero */ - KUTF_HELPER_ERR_NONE = 0, -@@ -71,14 +71,16 @@ enum kutf_helper_err { - KUTF_HELPER_ERR_INVALID_NAME, - /* Named value parsing of string or u64 type encountered extra - * characters after the value (after the last digit for a u64 type or -- * after the string end delimiter for string type) */ -+ * after the string end delimiter for string type) -+ */ - KUTF_HELPER_ERR_CHARS_AFTER_VAL, - /* Named value parsing of string type couldn't find the string end - * delimiter. - * - * This cannot be encountered when the NAME="value" message exceeds the - * textbuf's maximum line length, because such messages are not checked -- * for an end string delimiter */ -+ * for an end string delimiter -+ */ - KUTF_HELPER_ERR_NO_END_DELIMITER, - /* Named value didn't parse as any of the known types */ - KUTF_HELPER_ERR_INVALID_VALUE, -@@ -122,7 +124,8 @@ int kutf_helper_max_str_len_for_kern(const char *val_name, int kern_buf_sz); - * - * Any failure will be logged on the suite's current test fixture - * -- * Returns 0 on success, non-zero on failure */ -+ * Returns 0 on success, non-zero on failure -+ */ - int kutf_helper_send_named_str(struct kutf_context *context, - const char *val_name, const char *val_str); - -@@ -138,7 +141,8 @@ int kutf_helper_send_named_str(struct kutf_context *context, - * - * Returns 0 on success. Negative value on failure to receive from the 'run' - * file, positive value indicates an enum kutf_helper_err value for correct -- * reception of data but invalid parsing */ -+ * reception of data but invalid parsing -+ */ - int kutf_helper_receive_named_val( - struct kutf_context *context, - struct kutf_helper_named_val *named_val); -@@ -165,7 +169,8 @@ int kutf_helper_receive_named_val( - * - return value will be 0 to indicate success - * - * The rationale behind this is that we'd prefer to continue the rest of the -- * test with failures propagated, rather than hitting a timeout */ -+ * test with failures propagated, rather than hitting a timeout -+ */ - int kutf_helper_receive_check_val( - struct kutf_helper_named_val *named_val, - struct kutf_context *context, -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h -index 988559d..5d4d96e 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_mem.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KERNEL_UTF_MEM_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h -index 49ebeb4..2fb1a47 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_resultset.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KERNEL_UTF_RESULTSET_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h -index 8d75f50..b9c333b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_suite.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KERNEL_UTF_SUITE_H_ -@@ -264,9 +263,10 @@ struct kutf_suite { - struct list_head test_list; - }; - --/* ============================================================================ -- Application functions --============================================================================ */ -+/** =========================================================================== -+ * Application functions -+ * ============================================================================ -+ */ - - /** - * kutf_create_application() - Create an in kernel test application. -@@ -284,9 +284,10 @@ struct kutf_application *kutf_create_application(const char *name); - */ - void kutf_destroy_application(struct kutf_application *app); - --/* ============================================================================ -- Suite functions --============================================================================ */ -+/**============================================================================ -+ * Suite functions -+ * ============================================================================ -+ */ - - /** - * kutf_create_suite() - Create a kernel test suite. -@@ -416,10 +417,10 @@ void kutf_add_test_with_filters_and_data( - unsigned int filters, - union kutf_callback_data test_data); - -- --/* ============================================================================ -- Test functions --============================================================================ */ -+/** =========================================================================== -+ * Test functions -+ * ============================================================================ -+ */ - /** - * kutf_test_log_result_external() - Log a result which has been created - * externally into a in a standard form -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h -index 25b8285..18dcc3d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/include/kutf/kutf_utils.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _KERNEL_UTF_UTILS_H_ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Kbuild -index 2531d41..c4790bc 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,12 +16,16 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # --# -- --ccflags-y += -I$(src)/../include - --obj-$(CONFIG_MALI_KUTF) += kutf.o -+ifeq ($(CONFIG_MALI_KUTF),y) -+obj-m += kutf.o - --kutf-y := kutf_mem.o kutf_resultset.o kutf_suite.o kutf_utils.o kutf_helpers.o kutf_helpers_user.o -+kutf-y := \ -+ kutf_mem.o \ -+ kutf_resultset.o \ -+ kutf_suite.o \ -+ kutf_utils.o \ -+ kutf_helpers.o \ -+ kutf_helpers_user.o -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/build.bp b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/build.bp -index 32eab14..89edae9 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/build.bp -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/build.bp -@@ -1,23 +1,30 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * -- * A copy of the licence is included with the program, and can also be obtained -- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- * Boston, MA 02110-1301, USA. -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - - bob_kernel_module { - name: "kutf", - defaults: [ -- "kernel_defaults", -- "kutf_includes", -+ "mali_kbase_shared_config_defaults", -+ "kernel_test_configs", -+ "kernel_test_includes", - ], - srcs: [ - "Kbuild", -@@ -28,9 +35,8 @@ bob_kernel_module { - "kutf_suite.c", - "kutf_utils.c", - ], -- kbuild_options: ["CONFIG_MALI_KUTF=m"], - enabled: false, -- base_build_kutf: { -+ mali_kutf: { - enabled: true, - }, - } -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_helpers.c b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_helpers.c -index cab5add..c075428 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_helpers.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_helpers.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* Kernel UTF test helpers */ -@@ -29,10 +28,11 @@ - #include <linux/preempt.h> - #include <linux/wait.h> - #include <linux/uaccess.h> -+#include <linux/export.h> - - static DEFINE_SPINLOCK(kutf_input_lock); - --static bool pending_input(struct kutf_context *context) -+bool kutf_helper_pending_input(struct kutf_context *context) - { - bool input_pending; - -@@ -44,6 +44,7 @@ static bool pending_input(struct kutf_context *context) - - return input_pending; - } -+EXPORT_SYMBOL(kutf_helper_pending_input); - - char *kutf_helper_input_dequeue(struct kutf_context *context, size_t *str_size) - { -@@ -59,7 +60,7 @@ char *kutf_helper_input_dequeue(struct kutf_context *context, size_t *str_size) - spin_unlock(&kutf_input_lock); - - err = wait_event_interruptible(context->userdata.input_waitq, -- pending_input(context)); -+ kutf_helper_pending_input(context)); - - if (err) - return ERR_PTR(-EINTR); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_helpers_user.c b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_helpers_user.c -index 108fa82..a8b59f7 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_helpers_user.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_helpers_user.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* Kernel UTF test helpers that mirror those for kutf-userside */ -@@ -42,7 +41,8 @@ static const char *get_val_type_name(enum kutf_helper_valtype valtype) - * a) "<0 comparison on unsigned type" warning - if we did both upper - * and lower bound check - * b) incorrect range checking if it was a signed type - if we did -- * upper bound check only */ -+ * upper bound check only -+ */ - unsigned int type_idx = (unsigned int)valtype; - - if (type_idx >= (unsigned int)KUTF_HELPER_VALTYPE_COUNT) -@@ -54,7 +54,8 @@ static const char *get_val_type_name(enum kutf_helper_valtype valtype) - /* Check up to str_len chars of val_str to see if it's a valid value name: - * - * - Has between 1 and KUTF_HELPER_MAX_VAL_NAME_LEN characters before the \0 terminator -- * - And, each char is in the character set [A-Z0-9_] */ -+ * - And, each char is in the character set [A-Z0-9_] -+ */ - static int validate_val_name(const char *val_str, int str_len) - { - int i = 0; -@@ -87,7 +88,8 @@ static int validate_val_name(const char *val_str, int str_len) - * e.g. "str" - * - * That is, before any '\\', '\n' or '"' characters. This is so we don't have -- * to escape the string */ -+ * to escape the string -+ */ - static int find_quoted_string_valid_len(const char *str) - { - char *ptr; -@@ -207,7 +209,8 @@ int kutf_helper_send_named_str(struct kutf_context *context, - str_buf_sz = val_name_len + start_delim_len + val_str_len + end_delim_len + 1; - - /* Using kmalloc() here instead of mempool since we know we need to free -- * before we return */ -+ * before we return -+ */ - str_buf = kmalloc(str_buf_sz, GFP_KERNEL); - if (!str_buf) { - errmsg = kutf_dsprintf(&context->fixture_pool, -@@ -218,7 +221,8 @@ int kutf_helper_send_named_str(struct kutf_context *context, - copy_ptr = str_buf; - - /* Manually copy each string component instead of snprintf because -- * val_str may need to end early, and less error path handling */ -+ * val_str may need to end early, and less error path handling -+ */ - - /* name */ - memcpy(copy_ptr, val_name, val_name_len); -@@ -331,7 +335,8 @@ int kutf_helper_receive_named_val( - /* possibly a number value - strtoull will parse it */ - err = kstrtoull(recv_str, 0, &u64val); - /* unlike userspace can't get an end ptr, but if kstrtoull() -- * reads characters after the number it'll report -EINVAL */ -+ * reads characters after the number it'll report -EINVAL -+ */ - if (!err) { - int len_remain = strnlen(recv_str, recv_sz); - -@@ -399,7 +404,8 @@ int kutf_helper_receive_check_val( - goto out_fail_and_fixup; - } - -- if (strcmp(named_val->val_name, expect_val_name) != 0) { -+ if (named_val->val_name != NULL && -+ strcmp(named_val->val_name, expect_val_name) != 0) { - const char *msg = kutf_dsprintf(&context->fixture_pool, - "Expecting to receive value named '%s' but got '%s'", - expect_val_name, named_val->val_name); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c -index fd98bea..716970a 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_mem.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* Kernel UTF memory management functions */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c -index 94ecfa4..c7572bd 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_resultset.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* Kernel UTF result management functions */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c -index 3f15669..6745299 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_suite.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014, 2017-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2017-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,11 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* Kernel UTF suite, test and fixture management including user to kernel -- * interaction */ -+ * interaction -+ */ - - #include <linux/list.h> - #include <linux/slab.h> -@@ -598,7 +598,7 @@ static int create_fixture_variant(struct kutf_test_function *test_func, - goto fail_file; - } - --#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) -+#if KERNEL_VERSION(4, 7, 0) <= LINUX_VERSION_CODE - tmp = debugfs_create_file_unsafe( - #else - tmp = debugfs_create_file( -@@ -634,7 +634,7 @@ static void kutf_remove_test_variant(struct kutf_test_fixture *test_fix) - kfree(test_fix); - } - --#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) -+#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE - /* Adapting to the upstream debugfs_create_x32() change */ - static int ktufp_u32_get(void *data, u64 *val) - { -@@ -679,7 +679,7 @@ void kutf_add_test_with_filters_and_data( - } - - test_func->filters = filters; --#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 5, 0) -+#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE - tmp = debugfs_create_file_unsafe("filters", S_IROTH, test_func->dir, - &test_func->filters, &kutfp_fops_x32_ro); - #else -@@ -692,12 +692,17 @@ void kutf_add_test_with_filters_and_data( - } - - test_func->test_id = id; -+#if KERNEL_VERSION(5, 5, 0) <= LINUX_VERSION_CODE -+ debugfs_create_u32("test_id", S_IROTH, test_func->dir, -+ &test_func->test_id); -+#else - tmp = debugfs_create_u32("test_id", S_IROTH, test_func->dir, - &test_func->test_id); - if (!tmp) { - pr_err("Failed to create debugfs file \"test_id\" when adding test %s\n", name); - goto fail_file; - } -+#endif - - for (i = 0; i < suite->fixture_variants; i++) { - if (create_fixture_variant(test_func, i)) { -@@ -1153,7 +1158,7 @@ void kutf_test_abort(struct kutf_context *context) - } - EXPORT_SYMBOL(kutf_test_abort); - --#ifdef CONFIG_DEBUG_FS -+#if IS_ENABLED(CONFIG_DEBUG_FS) - - /** - * init_kutf_core() - Module entry point. -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c -index 7f5ac51..c0fb3ba 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/kutf/kutf_utils.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2014, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2014, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* Kernel UTF utility functions */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/Kbuild -new file mode 100644 -index 0000000..027bc27 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/Kbuild -@@ -0,0 +1,25 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+ifeq ($(CONFIG_MALI_KUTF_CLK_RATE_TRACE),y) -+obj-m += mali_kutf_clk_rate_trace_test_portal.o -+ -+mali_kutf_clk_rate_trace_test_portal-y := mali_kutf_clk_rate_trace_test.o -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/build.bp b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/build.bp -new file mode 100644 -index 0000000..225ad69 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/build.bp -@@ -0,0 +1,43 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+bob_kernel_module { -+ name: "mali_kutf_clk_rate_trace_test_portal", -+ defaults: [ -+ "mali_kbase_shared_config_defaults", -+ "kernel_test_configs", -+ "kernel_test_includes", -+ ], -+ srcs: [ -+ "Kbuild", -+ "mali_kutf_clk_rate_trace_test.c", -+ "../mali_kutf_clk_rate_trace_test.h", -+ ], -+ extra_symbols: [ -+ "mali_kbase", -+ "kutf", -+ ], -+ enabled: false, -+ mali_kutf_clk_rate_trace: { -+ kbuild_options: ["CONFIG_MALI_KUTF_CLK_RATE_TRACE=y"], -+ enabled: true, -+ }, -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c -new file mode 100644 -index 0000000..f9410a5 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/kernel/mali_kutf_clk_rate_trace_test.c -@@ -0,0 +1,957 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <linux/fdtable.h> -+#include <linux/module.h> -+ -+#include <linux/delay.h> -+#include <linux/mutex.h> -+#include <linux/ktime.h> -+#include <linux/version.h> -+#if (KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE) -+#include <linux/sched/task.h> -+#else -+#include <linux/sched.h> -+#endif -+#include "mali_kbase.h" -+#include "backend/gpu/mali_kbase_irq_internal.h" -+#include "backend/gpu/mali_kbase_pm_internal.h" -+#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h" -+ -+#include <kutf/kutf_suite.h> -+#include <kutf/kutf_utils.h> -+#include <kutf/kutf_helpers.h> -+#include <kutf/kutf_helpers_user.h> -+ -+#include "../mali_kutf_clk_rate_trace_test.h" -+ -+#define MINOR_FOR_FIRST_KBASE_DEV (-1) -+ -+/* KUTF test application pointer for this test */ -+struct kutf_application *kutf_app; -+ -+enum portal_server_state { -+ PORTAL_STATE_NO_CLK, -+ PORTAL_STATE_LIVE, -+ PORTAL_STATE_CLOSING, -+}; -+ -+/** -+ * struct clk_trace_snapshot - Trace info data on a clock. -+ * @previous_rate: Snapshot start point clock rate. -+ * @current_rate: End point clock rate. It becomes the start rate of the -+ * next trace snapshot. -+ * @rate_up_cnt: Count in the snapshot duration when the clock trace -+ * write is a rate of higher value than the last. -+ * @rate_down_cnt: Count in the snapshot duration when the clock trace write -+ * is a rate of lower value than the last. -+ */ -+struct clk_trace_snapshot { -+ unsigned long previous_rate; -+ unsigned long current_rate; -+ u32 rate_up_cnt; -+ u32 rate_down_cnt; -+}; -+ -+/** -+ * struct kutf_clk_rate_trace_fixture_data - Fixture data for the test. -+ * @kbdev: kbase device for the GPU. -+ * @listener: Clock rate change listener structure. -+ * @invoke_notify: When true, invoke notify command is being executed. -+ * @snapshot: Clock trace update snapshot data array. A snapshot -+ * for each clock contains info accumulated beteen two -+ * GET_TRACE_SNAPSHOT requests. -+ * @nclks: Number of clocks visible to the trace portal. -+ * @pm_ctx_cnt: Net count of PM (Power Management) context INC/DEC -+ * PM_CTX_CNT requests made to the portal. On change from -+ * 0 to 1 (INC), or, 1 to 0 (DEC), a PM context action is -+ * triggered. -+ * @total_update_cnt: Total number of received trace write callbacks. -+ * @server_state: Portal server operational state. -+ * @result_msg: Message for the test result. -+ * @test_status: Portal test reslt status. -+ */ -+struct kutf_clk_rate_trace_fixture_data { -+ struct kbase_device *kbdev; -+ struct kbase_clk_rate_listener listener; -+ bool invoke_notify; -+ struct clk_trace_snapshot snapshot[BASE_MAX_NR_CLOCKS_REGULATORS]; -+ unsigned int nclks; -+ unsigned int pm_ctx_cnt; -+ unsigned int total_update_cnt; -+ enum portal_server_state server_state; -+ char const *result_msg; -+ enum kutf_result_status test_status; -+}; -+ -+struct clk_trace_portal_input { -+ struct kutf_helper_named_val cmd_input; -+ enum kbasep_clk_rate_trace_req portal_cmd; -+ int named_val_err; -+}; -+ -+struct kbasep_cmd_name_pair { -+ enum kbasep_clk_rate_trace_req cmd; -+ const char *name; -+}; -+ -+struct kbasep_cmd_name_pair kbasep_portal_cmd_name_map[] = { -+ { PORTAL_CMD_GET_PLATFORM, GET_PLATFORM }, -+ { PORTAL_CMD_GET_CLK_RATE_MGR, GET_CLK_RATE_MGR }, -+ { PORTAL_CMD_GET_CLK_RATE_TRACE, GET_CLK_RATE_TRACE }, -+ { PORTAL_CMD_GET_TRACE_SNAPSHOT, GET_TRACE_SNAPSHOT }, -+ { PORTAL_CMD_INC_PM_CTX_CNT, INC_PM_CTX_CNT }, -+ { PORTAL_CMD_DEC_PM_CTX_CNT, DEC_PM_CTX_CNT }, -+ { PORTAL_CMD_CLOSE_PORTAL, CLOSE_PORTAL }, -+ { PORTAL_CMD_INVOKE_NOTIFY_42KHZ, INVOKE_NOTIFY_42KHZ }, -+}; -+ -+/* Global pointer for the kutf_portal_trace_write() to use. When -+ * this pointer is engaged, new requests for create fixture will fail -+ * hence limiting the use of the portal at any time to a singleton. -+ */ -+struct kutf_clk_rate_trace_fixture_data *g_ptr_portal_data; -+ -+#define PORTAL_MSG_LEN (KUTF_MAX_LINE_LENGTH - MAX_REPLY_NAME_LEN) -+static char portal_msg_buf[PORTAL_MSG_LEN]; -+ -+static void kutf_portal_trace_write( -+ struct kbase_clk_rate_listener *listener, -+ u32 index, u32 new_rate) -+{ -+ struct clk_trace_snapshot *snapshot; -+ struct kutf_clk_rate_trace_fixture_data *data; -+ -+ if (listener == NULL) { -+ pr_err("%s - index: %u, new_rate: %u, listener is NULL\n", -+ __func__, index, new_rate); -+ return; -+ } -+ -+ data = container_of(listener, struct kutf_clk_rate_trace_fixture_data, -+ listener); -+ -+ lockdep_assert_held(&data->kbdev->pm.clk_rtm.lock); -+ -+ if (WARN_ON(g_ptr_portal_data == NULL)) -+ return; -+ if (WARN_ON(index >= g_ptr_portal_data->nclks)) -+ return; -+ -+ /* This callback is triggered by invoke notify command, skipping */ -+ if (data->invoke_notify) -+ return; -+ -+ snapshot = &g_ptr_portal_data->snapshot[index]; -+ if (new_rate > snapshot->current_rate) -+ snapshot->rate_up_cnt++; -+ else -+ snapshot->rate_down_cnt++; -+ snapshot->current_rate = new_rate; -+ g_ptr_portal_data->total_update_cnt++; -+} -+ -+static void kutf_set_pm_ctx_active(struct kutf_context *context) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ -+ if (WARN_ON(data->pm_ctx_cnt != 1)) -+ return; -+ -+ kbase_pm_context_active(data->kbdev); -+ kbase_pm_wait_for_desired_state(data->kbdev); -+#if !MALI_USE_CSF -+ kbase_pm_request_gpu_cycle_counter(data->kbdev); -+#endif -+} -+ -+static void kutf_set_pm_ctx_idle(struct kutf_context *context) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ -+ if (WARN_ON(data->pm_ctx_cnt > 0)) -+ return; -+#if !MALI_USE_CSF -+ kbase_pm_release_gpu_cycle_counter(data->kbdev); -+#endif -+ kbase_pm_context_idle(data->kbdev); -+} -+ -+static char const *kutf_clk_trace_do_change_pm_ctx(struct kutf_context *context, -+ struct clk_trace_portal_input *cmd) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ int seq = cmd->cmd_input.u.val_u64 & 0xFF; -+ const unsigned int cnt = data->pm_ctx_cnt; -+ const enum kbasep_clk_rate_trace_req req = cmd->portal_cmd; -+ char const *errmsg = NULL; -+ -+ WARN_ON(req != PORTAL_CMD_INC_PM_CTX_CNT && -+ req != PORTAL_CMD_DEC_PM_CTX_CNT); -+ -+ if (req == PORTAL_CMD_INC_PM_CTX_CNT && cnt < UINT_MAX) { -+ data->pm_ctx_cnt++; -+ if (data->pm_ctx_cnt == 1) -+ kutf_set_pm_ctx_active(context); -+ } -+ -+ if (req == PORTAL_CMD_DEC_PM_CTX_CNT && cnt > 0) { -+ data->pm_ctx_cnt--; -+ if (data->pm_ctx_cnt == 0) -+ kutf_set_pm_ctx_idle(context); -+ } -+ -+ /* Skip the length check, no chance of overflow for two ints */ -+ snprintf(portal_msg_buf, PORTAL_MSG_LEN, -+ "{SEQ:%d, PM_CTX_CNT:%u}", seq, data->pm_ctx_cnt); -+ -+ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { -+ pr_warn("Error in sending ack for adjusting pm_ctx_cnt\n"); -+ errmsg = kutf_dsprintf(&context->fixture_pool, -+ "Error in sending ack for adjusting pm_ctx_cnt"); -+ } -+ -+ return errmsg; -+} -+ -+static char const *kutf_clk_trace_do_get_rate(struct kutf_context *context, -+ struct clk_trace_portal_input *cmd) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ struct kbase_device *kbdev = data->kbdev; -+ int seq = cmd->cmd_input.u.val_u64 & 0xFF; -+ unsigned long rate; -+ bool idle; -+ int ret; -+ int i; -+ char const *errmsg = NULL; -+ -+ WARN_ON((cmd->portal_cmd != PORTAL_CMD_GET_CLK_RATE_MGR) && -+ (cmd->portal_cmd != PORTAL_CMD_GET_CLK_RATE_TRACE)); -+ -+ ret = snprintf(portal_msg_buf, PORTAL_MSG_LEN, -+ "{SEQ:%d, RATE:[", seq); -+ -+ for (i = 0; i < data->nclks; i++) { -+ spin_lock(&kbdev->pm.clk_rtm.lock); -+ if (cmd->portal_cmd == PORTAL_CMD_GET_CLK_RATE_MGR) -+ rate = kbdev->pm.clk_rtm.clks[i]->clock_val; -+ else -+ rate = data->snapshot[i].current_rate; -+ idle = kbdev->pm.clk_rtm.gpu_idle; -+ spin_unlock(&kbdev->pm.clk_rtm.lock); -+ -+ if ((i + 1) == data->nclks) -+ ret += snprintf(portal_msg_buf + ret, -+ PORTAL_MSG_LEN - ret, "0x%lx], GPU_IDLE:%d}", -+ rate, idle); -+ else -+ ret += snprintf(portal_msg_buf + ret, -+ PORTAL_MSG_LEN - ret, "0x%lx, ", rate); -+ -+ if (ret >= PORTAL_MSG_LEN) { -+ pr_warn("Message buf overflow with rate array data\n"); -+ return kutf_dsprintf(&context->fixture_pool, -+ "Message buf overflow with rate array data"); -+ } -+ } -+ -+ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { -+ pr_warn("Error in sending back rate array\n"); -+ errmsg = kutf_dsprintf(&context->fixture_pool, -+ "Error in sending rate array"); -+ } -+ -+ return errmsg; -+} -+ -+/** -+ * kutf_clk_trace_do_get_snapshot() - Send back the current snapshot -+ * @context: KUTF context -+ * @cmd: The decoded portal input request -+ * -+ * The accumulated clock rate trace information is kept inside as an snapshot -+ * record. A user request of getting the snapshot marks the closure of the -+ * current snapshot record, and the start of the next one. The response -+ * message contains the current snapshot record, with each clock's -+ * data sequentially placed inside (array marker) [ ]. -+ */ -+static char const *kutf_clk_trace_do_get_snapshot(struct kutf_context *context, -+ struct clk_trace_portal_input *cmd) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ struct clk_trace_snapshot snapshot; -+ int seq = cmd->cmd_input.u.val_u64 & 0xFF; -+ int ret; -+ int i; -+ char const *fmt; -+ char const *errmsg = NULL; -+ -+ WARN_ON(cmd->portal_cmd != PORTAL_CMD_GET_TRACE_SNAPSHOT); -+ -+ ret = snprintf(portal_msg_buf, PORTAL_MSG_LEN, -+ "{SEQ:%d, SNAPSHOT_ARRAY:[", seq); -+ -+ for (i = 0; i < data->nclks; i++) { -+ spin_lock(&data->kbdev->pm.clk_rtm.lock); -+ /* copy out the snapshot of the clock */ -+ snapshot = data->snapshot[i]; -+ /* Set the next snapshot start condition */ -+ data->snapshot[i].previous_rate = snapshot.current_rate; -+ data->snapshot[i].rate_up_cnt = 0; -+ data->snapshot[i].rate_down_cnt = 0; -+ spin_unlock(&data->kbdev->pm.clk_rtm.lock); -+ -+ /* Check i corresponding to the last clock */ -+ if ((i + 1) == data->nclks) -+ fmt = "(0x%lx, 0x%lx, %u, %u)]}"; -+ else -+ fmt = "(0x%lx, 0x%lx, %u, %u), "; -+ ret += snprintf(portal_msg_buf + ret, PORTAL_MSG_LEN - ret, -+ fmt, snapshot.previous_rate, snapshot.current_rate, -+ snapshot.rate_up_cnt, snapshot.rate_down_cnt); -+ if (ret >= PORTAL_MSG_LEN) { -+ pr_warn("Message buf overflow with snapshot data\n"); -+ return kutf_dsprintf(&context->fixture_pool, -+ "Message buf overflow with snapshot data"); -+ } -+ } -+ -+ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { -+ pr_warn("Error in sending back snapshot array\n"); -+ errmsg = kutf_dsprintf(&context->fixture_pool, -+ "Error in sending snapshot array"); -+ } -+ -+ return errmsg; -+} -+ -+/** -+ * kutf_clk_trace_do_invoke_notify_42k() - Invokes the stored notification callback -+ * @context: KUTF context -+ * @cmd: The decoded portal input request -+ * -+ * Invokes frequency change notification callbacks with a fake -+ * GPU frequency 42 kHz for the top clock domain. -+ */ -+static char const *kutf_clk_trace_do_invoke_notify_42k( -+ struct kutf_context *context, -+ struct clk_trace_portal_input *cmd) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ int seq = cmd->cmd_input.u.val_u64 & 0xFF; -+ const unsigned long new_rate_hz = 42000; -+ int ret; -+ char const *errmsg = NULL; -+ struct kbase_clk_rate_trace_manager *clk_rtm = &data->kbdev->pm.clk_rtm; -+ -+ WARN_ON(cmd->portal_cmd != PORTAL_CMD_INVOKE_NOTIFY_42KHZ); -+ -+ spin_lock(&clk_rtm->lock); -+ -+ data->invoke_notify = true; -+ kbase_clk_rate_trace_manager_notify_all( -+ clk_rtm, 0, new_rate_hz); -+ data->invoke_notify = false; -+ -+ spin_unlock(&clk_rtm->lock); -+ -+ ret = snprintf(portal_msg_buf, PORTAL_MSG_LEN, -+ "{SEQ:%d, HZ:%lu}", seq, new_rate_hz); -+ -+ if (ret >= PORTAL_MSG_LEN) { -+ pr_warn("Message buf overflow with invoked data\n"); -+ return kutf_dsprintf(&context->fixture_pool, -+ "Message buf overflow with invoked data"); -+ } -+ -+ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { -+ pr_warn("Error in sending ack for " INVOKE_NOTIFY_42KHZ "request\n"); -+ errmsg = kutf_dsprintf(&context->fixture_pool, -+ "Error in sending ack for " INVOKE_NOTIFY_42KHZ "request"); -+ } -+ -+ return errmsg; -+} -+ -+static char const *kutf_clk_trace_do_close_portal(struct kutf_context *context, -+ struct clk_trace_portal_input *cmd) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ int seq = cmd->cmd_input.u.val_u64 & 0xFF; -+ char const *errmsg = NULL; -+ -+ WARN_ON(cmd->portal_cmd != PORTAL_CMD_CLOSE_PORTAL); -+ -+ data->server_state = PORTAL_STATE_CLOSING; -+ -+ /* Skip the length check, no chance of overflow for two ints */ -+ snprintf(portal_msg_buf, PORTAL_MSG_LEN, -+ "{SEQ:%d, PM_CTX_CNT:%u}", seq, data->pm_ctx_cnt); -+ -+ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { -+ pr_warn("Error in sending ack for " CLOSE_PORTAL "reuquest\n"); -+ errmsg = kutf_dsprintf(&context->fixture_pool, -+ "Error in sending ack for " CLOSE_PORTAL "reuquest"); -+ } -+ -+ return errmsg; -+} -+ -+/** -+ * kutf_clk_trace_do_get_platform() - Gets platform information -+ * @context: KUTF context -+ * @cmd: The decoded portal input request -+ * -+ * Checks the gpu node in the device tree to see if arbitration is enabled -+ * If so determines device tree whether platform is PV or PTM -+ * -+ * Return: A string to indicate the platform (PV/PTM/GPU/UNKNOWN) -+ */ -+static char const *kutf_clk_trace_do_get_platform( -+ struct kutf_context *context, -+ struct clk_trace_portal_input *cmd) -+{ -+ int seq = cmd->cmd_input.u.val_u64 & 0xFF; -+ char const *errmsg = NULL; -+ const void *arbiter_if_node = NULL; -+ const void *power_node = NULL; -+ const char *platform = "GPU"; -+#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF) -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ -+ arbiter_if_node = -+ of_get_property(data->kbdev->dev->of_node, "arbiter_if", NULL); -+#endif -+ if (arbiter_if_node) { -+ power_node = of_find_compatible_node(NULL, NULL, -+ "arm,mali-gpu-power"); -+ if (power_node) { -+ platform = "PV"; -+ } else { -+ power_node = of_find_compatible_node(NULL, NULL, -+ "arm,mali-ptm"); -+ if (power_node) -+ platform = "PTM"; -+ else -+ platform = "UNKNOWN"; -+ } -+ } else { -+ platform = "GPU"; -+ } -+ -+ pr_debug("%s - platform is %s\n", __func__, platform); -+ snprintf(portal_msg_buf, PORTAL_MSG_LEN, -+ "{SEQ:%d, PLATFORM:%s}", seq, platform); -+ -+ WARN_ON(cmd->portal_cmd != PORTAL_CMD_GET_PLATFORM); -+ -+ if (kutf_helper_send_named_str(context, "ACK", portal_msg_buf)) { -+ pr_warn("Error in sending ack for " CLOSE_PORTAL "reuquest\n"); -+ errmsg = kutf_dsprintf(&context->fixture_pool, -+ "Error in sending ack for " GET_PLATFORM "request"); -+ } -+ -+ return errmsg; -+} -+ -+static bool kutf_clk_trace_dequeue_portal_cmd(struct kutf_context *context, -+ struct clk_trace_portal_input *cmd) -+{ -+ int i; -+ int err = kutf_helper_receive_named_val(context, &cmd->cmd_input); -+ -+ cmd->named_val_err = err; -+ if (err == KUTF_HELPER_ERR_NONE && -+ cmd->cmd_input.type == KUTF_HELPER_VALTYPE_U64) { -+ /* All portal request commands are of format (named u64): -+ * CMD_NAME=1234 -+ * where, 1234 is a (variable) sequence number tag. -+ */ -+ for (i = 0; i < PORTAL_TOTAL_CMDS; i++) { -+ if (strcmp(cmd->cmd_input.val_name, -+ kbasep_portal_cmd_name_map[i].name)) -+ continue; -+ -+ cmd->portal_cmd = kbasep_portal_cmd_name_map[i].cmd; -+ return true; -+ } -+ } -+ -+ cmd->portal_cmd = PORTAL_CMD_INVALID; -+ return false; -+} -+ -+static void kutf_clk_trace_flag_result(struct kutf_context *context, -+ enum kutf_result_status result, char const *msg) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ -+ if (result > data->test_status) { -+ data->test_status = result; -+ if (msg) -+ data->result_msg = msg; -+ if (data->server_state == PORTAL_STATE_LIVE && -+ result > KUTF_RESULT_WARN) { -+ data->server_state = PORTAL_STATE_CLOSING; -+ } -+ } -+} -+ -+static bool kutf_clk_trace_process_portal_cmd(struct kutf_context *context, -+ struct clk_trace_portal_input *cmd) -+{ -+ char const *errmsg = NULL; -+ -+ BUILD_BUG_ON(ARRAY_SIZE(kbasep_portal_cmd_name_map) != -+ PORTAL_TOTAL_CMDS); -+ WARN_ON(cmd->portal_cmd == PORTAL_CMD_INVALID); -+ -+ switch (cmd->portal_cmd) { -+ case PORTAL_CMD_GET_PLATFORM: -+ errmsg = kutf_clk_trace_do_get_platform(context, cmd); -+ break; -+ case PORTAL_CMD_GET_CLK_RATE_MGR: -+ /* Fall through */ -+ case PORTAL_CMD_GET_CLK_RATE_TRACE: -+ errmsg = kutf_clk_trace_do_get_rate(context, cmd); -+ break; -+ case PORTAL_CMD_GET_TRACE_SNAPSHOT: -+ errmsg = kutf_clk_trace_do_get_snapshot(context, cmd); -+ break; -+ case PORTAL_CMD_INC_PM_CTX_CNT: -+ /* Fall through */ -+ case PORTAL_CMD_DEC_PM_CTX_CNT: -+ errmsg = kutf_clk_trace_do_change_pm_ctx(context, cmd); -+ break; -+ case PORTAL_CMD_CLOSE_PORTAL: -+ errmsg = kutf_clk_trace_do_close_portal(context, cmd); -+ break; -+ case PORTAL_CMD_INVOKE_NOTIFY_42KHZ: -+ errmsg = kutf_clk_trace_do_invoke_notify_42k(context, cmd); -+ break; -+ default: -+ pr_warn("Don't know how to handle portal_cmd: %d, abort session.\n", -+ cmd->portal_cmd); -+ errmsg = kutf_dsprintf(&context->fixture_pool, -+ "Don't know how to handle portal_cmd: %d", -+ cmd->portal_cmd); -+ break; -+ } -+ -+ if (errmsg) -+ kutf_clk_trace_flag_result(context, KUTF_RESULT_FAIL, errmsg); -+ -+ return (errmsg == NULL); -+} -+ -+/** -+ * kutf_clk_trace_do_nack_response() - respond a NACK to erroneous input -+ * @context: KUTF context -+ * @cmd: The erroneous input request -+ * -+ * This function deal with an erroneous input request, and respond with -+ * a proper 'NACK' message. -+ */ -+static int kutf_clk_trace_do_nack_response(struct kutf_context *context, -+ struct clk_trace_portal_input *cmd) -+{ -+ int seq; -+ int err; -+ char const *errmsg = NULL; -+ -+ WARN_ON(cmd->portal_cmd != PORTAL_CMD_INVALID); -+ -+ if (cmd->named_val_err == KUTF_HELPER_ERR_NONE && -+ cmd->cmd_input.type == KUTF_HELPER_VALTYPE_U64) { -+ /* Keep seq number as % 256 */ -+ seq = cmd->cmd_input.u.val_u64 & 255; -+ snprintf(portal_msg_buf, PORTAL_MSG_LEN, -+ "{SEQ:%d, MSG: Unknown command '%s'.}", seq, -+ cmd->cmd_input.val_name); -+ err = kutf_helper_send_named_str(context, "NACK", -+ portal_msg_buf); -+ } else -+ err = kutf_helper_send_named_str(context, "NACK", -+ "Wrong portal cmd format (Ref example: CMD_NAME=0X16)"); -+ -+ if (err) { -+ errmsg = kutf_dsprintf(&context->fixture_pool, -+ "Failed to send portal NACK response"); -+ kutf_clk_trace_flag_result(context, KUTF_RESULT_FAIL, errmsg); -+ } -+ -+ return err; -+} -+ -+/** -+ * kutf_clk_trace_barebone_check() - Sanity test on the clock tracing -+ * @context: KUTF context -+ * -+ * This function carries out some basic test on the tracing operation: -+ * 1). GPU idle on test start, trace rate should be 0 (low power state) -+ * 2). Make sure GPU is powered up, the trace rate should match -+ * that from the clcok manager's internal recorded rate -+ * 3). If the GPU active transition occurs following 2), there -+ * must be rate change event from tracing. -+ */ -+void kutf_clk_trace_barebone_check(struct kutf_context *context) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ struct kbase_device *kbdev = data->kbdev; -+ bool fail = false; -+ bool idle[2] = { false }; -+ char const *msg = NULL; -+ int i; -+ -+ /* Check consistency if gpu happens to be idle */ -+ spin_lock(&kbdev->pm.clk_rtm.lock); -+ idle[0] = kbdev->pm.clk_rtm.gpu_idle; -+ if (kbdev->pm.clk_rtm.gpu_idle) { -+ for (i = 0; i < data->nclks; i++) { -+ if (data->snapshot[i].current_rate) { -+ /* Idle should have a rate 0 */ -+ fail = true; -+ break; -+ } -+ } -+ } -+ spin_unlock(&kbdev->pm.clk_rtm.lock); -+ if (fail) { -+ msg = kutf_dsprintf(&context->fixture_pool, -+ "GPU Idle not yielding 0-rate"); -+ pr_err("Trace did not see idle rate\n"); -+ } else { -+ /* Make local PM active if not done so yet */ -+ if (data->pm_ctx_cnt == 0) { -+ /* Ensure the GPU is powered */ -+ data->pm_ctx_cnt++; -+ kutf_set_pm_ctx_active(context); -+ } -+ /* Checking the rate is consistent */ -+ spin_lock(&kbdev->pm.clk_rtm.lock); -+ idle[1] = kbdev->pm.clk_rtm.gpu_idle; -+ for (i = 0; i < data->nclks; i++) { -+ /* Rate match between the manager and the trace */ -+ if (kbdev->pm.clk_rtm.clks[i]->clock_val != -+ data->snapshot[i].current_rate) { -+ fail = true; -+ break; -+ } -+ } -+ spin_unlock(&kbdev->pm.clk_rtm.lock); -+ -+ if (idle[1]) { -+ msg = kutf_dsprintf(&context->fixture_pool, -+ "GPU still idle after set_pm_ctx_active"); -+ pr_err("GPU still idle after set_pm_ctx_active\n"); -+ } -+ -+ if (!msg && fail) { -+ msg = kutf_dsprintf(&context->fixture_pool, -+ "Trace rate not matching Clk manager's read"); -+ pr_err("Trace rate not matching Clk manager's read\n"); -+ } -+ } -+ -+ if (!msg && idle[0] && !idle[1] && !data->total_update_cnt) { -+ msg = kutf_dsprintf(&context->fixture_pool, -+ "Trace update did not occur"); -+ pr_err("Trace update did not occur\n"); -+ } -+ if (msg) -+ kutf_clk_trace_flag_result(context, KUTF_RESULT_FAIL, msg); -+ else if (!data->total_update_cnt) { -+ msg = kutf_dsprintf(&context->fixture_pool, -+ "No trace update seen during the test!"); -+ kutf_clk_trace_flag_result(context, KUTF_RESULT_WARN, msg); -+ } -+} -+ -+static bool kutf_clk_trace_end_of_stream(struct clk_trace_portal_input *cmd) -+{ -+ return (cmd->named_val_err == -EBUSY); -+} -+ -+void kutf_clk_trace_no_clks_dummy(struct kutf_context *context) -+{ -+ struct clk_trace_portal_input cmd; -+ unsigned long timeout = jiffies + HZ * 2; -+ bool has_cmd; -+ -+ while (time_before(jiffies, timeout)) { -+ if (kutf_helper_pending_input(context)) { -+ has_cmd = kutf_clk_trace_dequeue_portal_cmd(context, -+ &cmd); -+ if (!has_cmd && kutf_clk_trace_end_of_stream(&cmd)) -+ break; -+ -+ kutf_helper_send_named_str(context, "NACK", -+ "Fatal! No clocks visible, aborting"); -+ } -+ msleep(20); -+ } -+ -+ kutf_clk_trace_flag_result(context, KUTF_RESULT_FATAL, -+ "No clocks visble to the portal"); -+} -+ -+/** -+ * mali_kutf_clk_rate_trace_test_portal() - Service portal input -+ * @context: KUTF context -+ * -+ * The test portal operates on input requests. If the input request is one -+ * of the recognized portal commands, it handles it accordingly. Otherwise -+ * a negative response 'NACK' is returned. The portal service terminates -+ * when a 'CLOSE_PORTAL' request is received, or due to an internal error. -+ * Both case would result in the server_state transitioned to CLOSING. -+ * -+ * If the portal is closed on request, a sanity test on the clock rate -+ * trace operation is undertaken via function: -+ * kutf_clk_trace_barebone_check(); -+ */ -+static void mali_kutf_clk_rate_trace_test_portal(struct kutf_context *context) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ struct clk_trace_portal_input new_cmd; -+ -+ pr_debug("Test portal service start\n"); -+ -+ while (data->server_state == PORTAL_STATE_LIVE) { -+ if (kutf_clk_trace_dequeue_portal_cmd(context, &new_cmd)) -+ kutf_clk_trace_process_portal_cmd(context, &new_cmd); -+ else if (kutf_clk_trace_end_of_stream(&new_cmd)) -+ /* Dequeue on portal input, end of stream */ -+ data->server_state = PORTAL_STATE_CLOSING; -+ else -+ kutf_clk_trace_do_nack_response(context, &new_cmd); -+ } -+ -+ /* Closing, exhausting all the pending inputs with NACKs. */ -+ if (data->server_state == PORTAL_STATE_CLOSING) { -+ while (kutf_helper_pending_input(context) && -+ (kutf_clk_trace_dequeue_portal_cmd(context, &new_cmd) || -+ !kutf_clk_trace_end_of_stream(&new_cmd))) { -+ kutf_helper_send_named_str(context, "NACK", -+ "Portal closing down"); -+ } -+ } -+ -+ /* If no portal error, do a barebone test here irrespective -+ * whatever the portal live session has been testing, which -+ * is entirely driven by the user-side via portal requests. -+ */ -+ if (data->test_status <= KUTF_RESULT_WARN) { -+ if (data->server_state != PORTAL_STATE_NO_CLK) -+ kutf_clk_trace_barebone_check(context); -+ else { -+ /* No clocks case, NACK 2-sec for the fatal situation */ -+ kutf_clk_trace_no_clks_dummy(context); -+ } -+ } -+ -+ /* If we have changed pm_ctx count, drop it back */ -+ if (data->pm_ctx_cnt) { -+ /* Although we count on portal requests, it only has material -+ * impact when from 0 -> 1. So the reverse is a simple one off. -+ */ -+ data->pm_ctx_cnt = 0; -+ kutf_set_pm_ctx_idle(context); -+ } -+ -+ /* Finally log the test result line */ -+ if (data->test_status < KUTF_RESULT_WARN) -+ kutf_test_pass(context, data->result_msg); -+ else if (data->test_status == KUTF_RESULT_WARN) -+ kutf_test_warn(context, data->result_msg); -+ else if (data->test_status == KUTF_RESULT_FATAL) -+ kutf_test_fatal(context, data->result_msg); -+ else -+ kutf_test_fail(context, data->result_msg); -+ -+ pr_debug("Test end\n"); -+} -+ -+/** -+ * mali_kutf_clk_rate_trace_create_fixture() - Creates the fixture data -+ * required for mali_kutf_clk_rate_trace_test_portal. -+ * @context: KUTF context. -+ * -+ * Return: Fixture data created on success or NULL on failure -+ */ -+static void *mali_kutf_clk_rate_trace_create_fixture( -+ struct kutf_context *context) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data; -+ struct kbase_device *kbdev; -+ unsigned long rate; -+ int i; -+ -+ /* Acquire the kbase device */ -+ pr_debug("Finding device\n"); -+ kbdev = kbase_find_device(MINOR_FOR_FIRST_KBASE_DEV); -+ if (kbdev == NULL) { -+ kutf_test_fail(context, "Failed to find kbase device"); -+ return NULL; -+ } -+ -+ pr_debug("Creating fixture\n"); -+ data = kutf_mempool_alloc(&context->fixture_pool, -+ sizeof(struct kutf_clk_rate_trace_fixture_data)); -+ if (!data) -+ return NULL; -+ -+ *data = (const struct kutf_clk_rate_trace_fixture_data) { 0 }; -+ pr_debug("Hooking up the test portal to kbdev clk rate trace\n"); -+ spin_lock(&kbdev->pm.clk_rtm.lock); -+ -+ if (g_ptr_portal_data != NULL) { -+ pr_warn("Test portal is already in use, run aborted\n"); -+ kutf_test_fail(context, "Portal allows single session only"); -+ spin_unlock(&kbdev->pm.clk_rtm.lock); -+ return NULL; -+ } -+ -+ for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) { -+ if (kbdev->pm.clk_rtm.clks[i]) { -+ data->nclks++; -+ if (kbdev->pm.clk_rtm.gpu_idle) -+ rate = 0; -+ else -+ rate = kbdev->pm.clk_rtm.clks[i]->clock_val; -+ data->snapshot[i].previous_rate = rate; -+ data->snapshot[i].current_rate = rate; -+ } -+ } -+ -+ spin_unlock(&kbdev->pm.clk_rtm.lock); -+ -+ if (data->nclks) { -+ /* Subscribe this test server portal */ -+ data->listener.notify = kutf_portal_trace_write; -+ data->invoke_notify = false; -+ -+ kbase_clk_rate_trace_manager_subscribe( -+ &kbdev->pm.clk_rtm, &data->listener); -+ /* Update the kutf_server_portal fixture_data pointer */ -+ g_ptr_portal_data = data; -+ } -+ -+ data->kbdev = kbdev; -+ data->result_msg = NULL; -+ data->test_status = KUTF_RESULT_PASS; -+ -+ if (data->nclks == 0) { -+ data->server_state = PORTAL_STATE_NO_CLK; -+ pr_debug("Kbdev has no clocks for rate trace"); -+ } else -+ data->server_state = PORTAL_STATE_LIVE; -+ -+ pr_debug("Created fixture\n"); -+ -+ return data; -+} -+ -+/** -+ * Destroy fixture data previously created by -+ * mali_kutf_clk_rate_trace_create_fixture. -+ * -+ * @context: KUTF context. -+ */ -+static void mali_kutf_clk_rate_trace_remove_fixture( -+ struct kutf_context *context) -+{ -+ struct kutf_clk_rate_trace_fixture_data *data = context->fixture; -+ struct kbase_device *kbdev = data->kbdev; -+ -+ if (data->nclks) { -+ /* Clean up the portal trace write arrangement */ -+ g_ptr_portal_data = NULL; -+ -+ kbase_clk_rate_trace_manager_unsubscribe( -+ &kbdev->pm.clk_rtm, &data->listener); -+ } -+ pr_debug("Destroying fixture\n"); -+ kbase_release_device(kbdev); -+ pr_debug("Destroyed fixture\n"); -+} -+ -+/** -+ * mali_kutf_clk_rate_trace_test_module_init() - Entry point for test mdoule. -+ */ -+int mali_kutf_clk_rate_trace_test_module_init(void) -+{ -+ struct kutf_suite *suite; -+ unsigned int filters; -+ union kutf_callback_data suite_data = { 0 }; -+ -+ pr_debug("Creating app\n"); -+ -+ g_ptr_portal_data = NULL; -+ kutf_app = kutf_create_application(CLK_RATE_TRACE_APP_NAME); -+ -+ if (!kutf_app) { -+ pr_warn("Creation of app " CLK_RATE_TRACE_APP_NAME -+ " failed!\n"); -+ return -ENOMEM; -+ } -+ -+ pr_debug("Create suite %s\n", CLK_RATE_TRACE_SUITE_NAME); -+ suite = kutf_create_suite_with_filters_and_data( -+ kutf_app, CLK_RATE_TRACE_SUITE_NAME, 1, -+ mali_kutf_clk_rate_trace_create_fixture, -+ mali_kutf_clk_rate_trace_remove_fixture, -+ KUTF_F_TEST_GENERIC, -+ suite_data); -+ -+ if (!suite) { -+ pr_warn("Creation of suite %s failed!\n", -+ CLK_RATE_TRACE_SUITE_NAME); -+ kutf_destroy_application(kutf_app); -+ return -ENOMEM; -+ } -+ -+ filters = suite->suite_default_flags; -+ kutf_add_test_with_filters( -+ suite, 0x0, CLK_RATE_TRACE_PORTAL, -+ mali_kutf_clk_rate_trace_test_portal, -+ filters); -+ -+ pr_debug("Init complete\n"); -+ return 0; -+} -+ -+/** -+ * mali_kutf_clk_rate_trace_test_module_exit() - Module exit point for this -+ * test. -+ */ -+void mali_kutf_clk_rate_trace_test_module_exit(void) -+{ -+ pr_debug("Exit start\n"); -+ kutf_destroy_application(kutf_app); -+ pr_debug("Exit complete\n"); -+} -+ -+ -+module_init(mali_kutf_clk_rate_trace_test_module_init); -+module_exit(mali_kutf_clk_rate_trace_test_module_exit); -+ -+MODULE_LICENSE("GPL"); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h -new file mode 100644 -index 0000000..f37efa8 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_clk_rate_trace/mali_kutf_clk_rate_trace_test.h -@@ -0,0 +1,151 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _KUTF_CLK_RATE_TRACE_TEST_H_ -+#define _KUTF_CLK_RATE_TRACE_TEST_H_ -+ -+#define CLK_RATE_TRACE_APP_NAME "clk_rate_trace" -+#define CLK_RATE_TRACE_SUITE_NAME "rate_trace" -+#define CLK_RATE_TRACE_PORTAL "portal" -+ -+/** -+ * enum kbasep_clk_rate_trace_req - request command to the clock rate trace -+ * service portal. -+ * -+ * @PORTAL_CMD_GET_PLATFORM: Request the platform that the tests are -+ * to be run on. -+ * @PORTAL_CMD_GET_CLK_RATE_MGR: Request the clock trace manager internal -+ * data record. On a positive acknowledgement -+ * the prevailing clock rates and the GPU idle -+ * condition flag are returned. -+ * @PORTAL_CMD_GET_CLK_RATE_TRACE: Request the clock trace portal to return its -+ * data record. On a positive acknowledgement -+ * the last trace recorded clock rates and the -+ * GPU idle condition flag are returned. -+ * @PORTAL_CMD_GET_TRACE_SNAPSHOT: Request the clock trace portal to return its -+ * current snapshot data record. On a positive -+ * acknowledgement the snapshot array matching -+ * the number of clocks are returned. It also -+ * starts a fresh snapshot inside the clock -+ * trace portal. -+ * @PORTAL_CMD_INC_PM_CTX_CNT: Request the clock trace portal to increase -+ * its internal PM_CTX_COUNT. If this increase -+ * yielded a count of 0 -> 1 change, the portal -+ * will initiate a PM_CTX_ACTIVE call to the -+ * Kbase power management. Futher increase -+ * requests will limit to only affect the -+ * portal internal count value. -+ * @PORTAL_CMD_DEC_PM_CTX_CNT: Request the clock trace portal to decrease -+ * its internal PM_CTX_COUNT. If this decrease -+ * yielded a count of 1 -> 0 change, the portal -+ * will initiate a PM_CTX_IDLE call to the -+ * Kbase power management. -+ * @PORTAL_CMD_CLOSE_PORTAL: Inform the clock trace portal service the -+ * client has completed its session. The portal -+ * will start the close down action. If no -+ * error has occurred during the dynamic -+ * interactive session, an inherent basic test -+ * carrying out some sanity check on the clock -+ * trace is undertaken. -+ * @PORTAL_CMD_INVOKE_NOTIFY_42KHZ: Invokes all clock rate trace manager callbacks -+ * for the top clock domain with a new GPU frequency -+ * set to 42 kHZ. -+ * @PORTAL_CMD_INVALID: Valid commands termination marker. Must be -+ * the highest enumeration value, as it -+ * represents valid command array size. -+ * @PORTAL_TOTAL_CMDS: Alias of PORTAL_CMD_INVALID. -+ */ -+/* PORTAL_CMD_INVALID must be the last one, serving the size */ -+enum kbasep_clk_rate_trace_req { -+ PORTAL_CMD_GET_PLATFORM, -+ PORTAL_CMD_GET_CLK_RATE_MGR, -+ PORTAL_CMD_GET_CLK_RATE_TRACE, -+ PORTAL_CMD_GET_TRACE_SNAPSHOT, -+ PORTAL_CMD_INC_PM_CTX_CNT, -+ PORTAL_CMD_DEC_PM_CTX_CNT, -+ PORTAL_CMD_CLOSE_PORTAL, -+ PORTAL_CMD_INVOKE_NOTIFY_42KHZ, -+ PORTAL_CMD_INVALID, -+ PORTAL_TOTAL_CMDS = PORTAL_CMD_INVALID, -+}; -+ -+/** -+ * Portal service request command names. The portal request consists of a kutf -+ * named u64-value. For those above enumerated PORTAL_CMD, the names defined -+ * here are used to mark the name and then followed with a sequence number -+ * value. Example (manual script here for illustration): -+ * exec 5<>run # open the portal kutf run as fd-5 -+ * echo GET_CLK_RATE_MGR=1 >&5 # send the cmd and sequence number 1 -+ * head -n 1 <&5 # read back the 1-line server reseponse -+ * ACK="{SEQ:1, RATE:[0x1ad27480], GPU_IDLE:1}" # response string -+ * echo GET_TRACE_SNAPSHOT=1 >&5 # send the cmd and sequence number 1 -+ * head -n 1 <&5 # read back the 1-line server reseponse -+ * ACK="{SEQ:1, SNAPSHOT_ARRAY:[(0x0, 0x1ad27480, 1, 0)]}" -+ * echo CLOSE_PORTAL=1 >&5 # close the portal -+ * cat <&5 # read back all the response lines -+ * ACK="{SEQ:1, PM_CTX_CNT:0}" # response to close command -+ * KUTF_RESULT_PASS:(explicit pass) # internal sanity test passed. -+ * exec 5>&- # close the service portal fd. -+ * -+ * Expected request command return format: -+ * GET_CLK_RATE_MGR: ACK="{SEQ:12, RATE:[1080, 1280], GPU_IDLE:1}" -+ * Note, the above contains 2-clock with rates in [], GPU idle -+ * GET_CLK_RATE_TRACE: ACK="{SEQ:6, RATE:[0x1ad27480], GPU_IDLE:0}" -+ * Note, 1-clock with rate in [], GPU not idle -+ * GET_TRACE_SNAPSHOT: ACK="{SEQ:8, SNAPSHOT_ARRAY:[(0x0, 0x1ad27480, 1, 0)]}" -+ * Note, 1-clock, (start_rate : 0, last_rate : 0x1ad27480, -+ * trace_rate_up_count: 1, trace_rate_down_count : 0) -+ * For the specific sample case here, there is a single rate_trace event -+ * that yielded a rate increase change. No rate drop event recorded in the -+ * reporting snapshot duration. -+ * INC_PM_CTX_CNT: ACK="{SEQ:1, PM_CTX_CNT:1}" -+ * Note, after the increment, M_CTX_CNT is 1. (i.e. 0 -> 1) -+ * DEC_PM_CTX_CNT: ACK="{SEQ:3, PM_CTX_CNT:0}" -+ * Note, after the decrement, PM_CTX_CNT is 0. (i.e. 1 -> 0) -+ * CLOSE_PORTAL: ACK="{SEQ:1, PM_CTX_CNT:1}" -+ * Note, at the close, PM_CTX_CNT is 1. The PM_CTX_CNT will internally be -+ * dropped down to 0 as part of the portal close clean up. -+ */ -+#define GET_PLATFORM "GET_PLATFORM" -+#define GET_CLK_RATE_MGR "GET_CLK_RATE_MGR" -+#define GET_CLK_RATE_TRACE "GET_CLK_RATE_TRACE" -+#define GET_TRACE_SNAPSHOT "GET_TRACE_SNAPSHOT" -+#define INC_PM_CTX_CNT "INC_PM_CTX_CNT" -+#define DEC_PM_CTX_CNT "DEC_PM_CTX_CNT" -+#define CLOSE_PORTAL "CLOSE_PORTAL" -+#define INVOKE_NOTIFY_42KHZ "INVOKE_NOTIFY_42KHZ" -+ -+/** -+ * Portal service response tag names. The response consists of a kutf -+ * named string-value. In case of a 'NACK' (negative acknowledgement), it -+ * can be one of the two formats: -+ * 1. NACK="{SEQ:2, MSG:xyzed}" # NACK on command with sequence tag-2. -+ * Note, the portal has received a valid name and valid sequence number -+ * but can't carry-out the request, reason in the MSG field. -+ * 2. NACK="Failing-message" -+ * Note, unable to parse a valid name or valid sequence number, -+ * or some internal error condition. Reason in the quoted string. -+ */ -+#define ACK "ACK" -+#define NACK "NACK" -+#define MAX_REPLY_NAME_LEN 32 -+ -+#endif /* _KUTF_CLK_RATE_TRACE_TEST_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild -index ca8c512..213d6d5 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - # --# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,12 +16,10 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 - # --# -- --ccflags-y += -I$(src)/../include -I$(src)/../../../ -I$(src)/../../ -I$(src)/../../backend/gpu -I$(srctree)/drivers/staging/android - --obj-$(CONFIG_MALI_IRQ_LATENCY) += mali_kutf_irq_test.o -+ifeq ($(CONFIG_MALI_KUTF_IRQ_TEST),y) -+obj-m += mali_kutf_irq_test.o - - mali_kutf_irq_test-y := mali_kutf_irq_test_main.o -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig -deleted file mode 100644 -index 4a3863a..0000000 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Kconfig -+++ /dev/null -@@ -1,29 +0,0 @@ --# --# (C) COPYRIGHT 2017 ARM Limited. All rights reserved. --# --# This program is free software and is provided to you under the terms of the --# GNU General Public License version 2 as published by the Free Software --# Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. --# --# This program is distributed in the hope that it will be useful, --# but WITHOUT ANY WARRANTY; without even the implied warranty of --# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --# GNU General Public License for more details. --# --# You should have received a copy of the GNU General Public License --# along with this program; if not, you can access it online at --# http://www.gnu.org/licenses/gpl-2.0.html. --# --# SPDX-License-Identifier: GPL-2.0 --# --# -- --config MALI_IRQ_LATENCY -- tristate "Mali GPU IRQ latency measurement" -- depends on MALI_MIDGARD && MALI_DEBUG && MALI_KUTF -- default m -- help -- This option will build a test module mali_kutf_irq_test that -- can determine the latency of the Mali GPU IRQ on your system. -- Choosing M here will generate a single module called mali_kutf_irq_test. -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile -deleted file mode 100644 -index bc4d654..0000000 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/Makefile -+++ /dev/null -@@ -1,51 +0,0 @@ --# --# (C) COPYRIGHT 2015, 2017-2018, 2020 ARM Limited. All rights reserved. --# --# This program is free software and is provided to you under the terms of the --# GNU General Public License version 2 as published by the Free Software --# Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. --# --# This program is distributed in the hope that it will be useful, --# but WITHOUT ANY WARRANTY; without even the implied warranty of --# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --# GNU General Public License for more details. --# --# You should have received a copy of the GNU General Public License --# along with this program; if not, you can access it online at --# http://www.gnu.org/licenses/gpl-2.0.html. --# --# SPDX-License-Identifier: GPL-2.0 --# --# -- --# linux build system bootstrap for out-of-tree module -- --# default to building for the host --ARCH ?= $(shell uname -m) -- --ifeq ($(KDIR),) --$(error Must specify KDIR to point to the kernel to target)) --endif -- --TEST_CCFLAGS := \ -- -DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \ -- -DMALI_CUSTOMER_RELEASE=$(MALI_CUSTOMER_RELEASE) \ -- -DMALI_USE_CSF=$(MALI_USE_CSF) \ -- $(SCONS_CFLAGS) \ -- -I$(CURDIR)/../include \ -- -I$(CURDIR)/../../../../../../include \ -- -I$(CURDIR)/../../../ \ -- -I$(CURDIR)/../../ \ -- -I$(CURDIR)/../../backend/gpu \ -- -I$(CURDIR)/../../debug \ -- -I$(CURDIR)/../../debug/backend \ -- -I$(CURDIR)/ \ -- -I$(srctree)/drivers/staging/android \ -- -I$(srctree)/include/linux -- --all: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) $(SCONS_CONFIGS) EXTRA_CFLAGS="$(TEST_CCFLAGS)" KBUILD_EXTRA_SYMBOLS="$(CURDIR)/../kutf/Module.symvers $(CURDIR)/../../Module.symvers" modules -- --clean: -- $(MAKE) ARCH=$(ARCH) -C $(KDIR) M=$(CURDIR) clean -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/build.bp b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/build.bp -index 90efdcf..155875b 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/build.bp -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/build.bp -@@ -1,15 +1,21 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2018-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * -- * A copy of the licence is included with the program, and can also be obtained -- * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, -- * Boston, MA 02110-1301, USA. -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -@@ -17,6 +23,7 @@ bob_kernel_module { - name: "mali_kutf_irq_test", - defaults: [ - "mali_kbase_shared_config_defaults", -+ "kernel_test_configs", - "kernel_test_includes", - ], - srcs: [ -@@ -28,8 +35,8 @@ bob_kernel_module { - "kutf", - ], - enabled: false, -- base_build_kutf: { -+ mali_kutf_irq_test: { -+ kbuild_options: ["CONFIG_MALI_KUTF_IRQ_TEST=y"], - enabled: true, -- kbuild_options: ["CONFIG_MALI_IRQ_LATENCY=m"], - }, - } -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c -index 26b442a..fdc5437 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tests/mali_kutf_irq_test/mali_kutf_irq_test_main.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2016-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2016-2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include <linux/module.h> -@@ -25,8 +24,8 @@ - #include <linux/interrupt.h> - - #include "mali_kbase.h" --#include <midgard/backend/gpu/mali_kbase_device_internal.h> --#include <midgard/backend/gpu/mali_kbase_pm_internal.h> -+#include <device/mali_kbase_device.h> -+#include <backend/gpu/mali_kbase_pm_internal.h> - - #include <kutf/kutf_suite.h> - #include <kutf/kutf_utils.h> -@@ -242,7 +241,7 @@ int mali_kutf_irq_test_main_init(void) - - irq_app = kutf_create_application("irq"); - -- if (NULL == irq_app) { -+ if (irq_app == NULL) { - pr_warn("Creation of test application failed!\n"); - return -ENOMEM; - } -@@ -251,7 +250,7 @@ int mali_kutf_irq_test_main_init(void) - 1, mali_kutf_irq_default_create_fixture, - mali_kutf_irq_default_remove_fixture); - -- if (NULL == suite) { -+ if (suite == NULL) { - pr_warn("Creation of test suite failed!\n"); - kutf_destroy_application(irq_app); - return -ENOMEM; -diff --git a/dvalin/kernel/drivers/base/memory_group_manager/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/thirdparty/Kbuild -similarity index 82% -rename from dvalin/kernel/drivers/base/memory_group_manager/Kbuild -rename to dvalin/kernel/drivers/gpu/arm/midgard/thirdparty/Kbuild -index a049bed..c723f3a 100644 ---- a/dvalin/kernel/drivers/base/memory_group_manager/Kbuild -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/thirdparty/Kbuild -@@ -1,10 +1,11 @@ -+# SPDX-License-Identifier: GPL-2.0 - # --# (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. - # - # This program is free software and is provided to you under the terms of the - # GNU General Public License version 2 as published by the Free Software - # Foundation, and any use by you of this program is subject to the terms --# of such GNU licence. -+# of such GNU license. - # - # This program is distributed in the hope that it will be useful, - # but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -15,8 +16,6 @@ - # along with this program; if not, you can access it online at - # http://www.gnu.org/licenses/gpl-2.0.html. - # --# SPDX-License-Identifier: GPL-2.0 --# - # - --obj-$(CONFIG_MALI_MEMORY_GROUP_MANAGER) := memory_group_manager.o -\ No newline at end of file -+mali_kbase-y += thirdparty/mali_kbase_mmap.o -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/thirdparty/mali_kbase_mmap.c b/dvalin/kernel/drivers/gpu/arm/midgard/thirdparty/mali_kbase_mmap.c -index f266d8e..de1199a 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/thirdparty/mali_kbase_mmap.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/thirdparty/mali_kbase_mmap.c -@@ -1,24 +1,4 @@ - /* -- * -- * (C) COPYRIGHT ARM Limited. All rights reserved. -- * -- * This program is free software and is provided to you under the terms of the -- * GNU General Public License version 2 as published by the Free Software -- * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -- * -- * This program is distributed in the hope that it will be useful, -- * but WITHOUT ANY WARRANTY; without even the implied warranty of -- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -- * GNU General Public License for more details. -- * -- * You should have received a copy of the GNU General Public License -- * along with this program; if not, you can access it online at -- * http://www.gnu.org/licenses/gpl-2.0.html. -- * -- * SPDX-License-Identifier: GPL-2.0 -- * -- *//* - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -@@ -30,7 +10,7 @@ - */ - - #include "linux/mman.h" --#include "../mali_kbase.h" -+#include <mali_kbase.h> - - /* mali_kbase_mmap.c - * -@@ -209,7 +189,8 @@ check_current: - return -ENOMEM; - if (gap_start <= high_limit && gap_end - gap_start >= length) { - /* We found a suitable gap. Clip it with the original -- * high_limit. */ -+ * high_limit. -+ */ - if (gap_end > info->high_limit) - gap_end = info->high_limit; - -@@ -270,19 +251,38 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, - bool is_same_4gb_page = false; - unsigned long ret; - -+ /* the 'nolock' form is used here: -+ * - the base_pfn of the SAME_VA zone does not change -+ * - in normal use, va_size_pages is constant once the first allocation -+ * begins -+ * -+ * However, in abnormal use this function could be processing whilst -+ * another new zone is being setup in a different thread (e.g. to -+ * borrow part of the SAME_VA zone). In the worst case, this path may -+ * witness a higher SAME_VA end_pfn than the code setting up the new -+ * zone. -+ * -+ * This is safe because once we reach the main allocation functions, -+ * we'll see the updated SAME_VA end_pfn and will determine that there -+ * is no free region at the address found originally by too large a -+ * same_va_end_addr here, and will fail the allocation gracefully. -+ */ -+ struct kbase_reg_zone *zone = -+ kbase_ctx_reg_zone_get_nolock(kctx, KBASE_REG_ZONE_SAME_VA); -+ u64 same_va_end_addr = kbase_reg_zone_end_pfn(zone) << PAGE_SHIFT; -+ - /* err on fixed address */ - if ((flags & MAP_FIXED) || addr) - return -EINVAL; - --#ifdef CONFIG_64BIT -+#if IS_ENABLED(CONFIG_64BIT) - /* too big? */ - if (len > TASK_SIZE - SZ_2M) - return -ENOMEM; - - if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) { -- -- high_limit = min_t(unsigned long, mm->mmap_base, -- (kctx->same_va_end << PAGE_SHIFT)); -+ high_limit = -+ min_t(unsigned long, mm->mmap_base, same_va_end_addr); - - /* If there's enough (> 33 bits) of GPU VA space, align - * to 2MB boundaries. -@@ -317,19 +317,25 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, - align_mask = align_offset - 1; - is_shader_code = true; - } -+#if !MALI_USE_CSF - } else if (reg->flags & KBASE_REG_TILER_ALIGN_TOP) { -- unsigned long extent_bytes = -- (unsigned long)(reg->extent << PAGE_SHIFT); -+ unsigned long extension_bytes = -+ (unsigned long)(reg->extension -+ << PAGE_SHIFT); - /* kbase_check_alloc_sizes() already satisfies - * these checks, but they're here to avoid - * maintenance hazards due to the assumptions -- * involved */ -- WARN_ON(reg->extent > (ULONG_MAX >> PAGE_SHIFT)); -+ * involved -+ */ -+ WARN_ON(reg->extension > -+ (ULONG_MAX >> PAGE_SHIFT)); - WARN_ON(reg->initial_commit > (ULONG_MAX >> PAGE_SHIFT)); -- WARN_ON(!is_power_of_2(extent_bytes)); -- align_mask = extent_bytes - 1; -+ WARN_ON(!is_power_of_2(extension_bytes)); -+ align_mask = extension_bytes - 1; - align_offset = -- extent_bytes - (reg->initial_commit << PAGE_SHIFT); -+ extension_bytes - -+ (reg->initial_commit << PAGE_SHIFT); -+#endif /* !MALI_USE_CSF */ - } else if (reg->flags & KBASE_REG_GPU_VA_SAME_4GB_PAGE) { - is_same_4gb_page = true; - } -@@ -352,11 +358,10 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *const kctx, - is_same_4gb_page); - - if (IS_ERR_VALUE(ret) && high_limit == mm->mmap_base && -- high_limit < (kctx->same_va_end << PAGE_SHIFT)) { -+ high_limit < same_va_end_addr) { - /* Retry above mmap_base */ - info.low_limit = mm->mmap_base; -- info.high_limit = min_t(u64, TASK_SIZE, -- (kctx->same_va_end << PAGE_SHIFT)); -+ info.high_limit = min_t(u64, TASK_SIZE, same_va_end_addr); - - ret = kbase_unmapped_area_topdown(&info, is_shader_code, - is_same_4gb_page); -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/Kbuild b/dvalin/kernel/drivers/gpu/arm/midgard/tl/Kbuild -new file mode 100644 -index 0000000..4344850 ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/Kbuild -@@ -0,0 +1,32 @@ -+# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+# -+# (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+# -+# This program is free software and is provided to you under the terms of the -+# GNU General Public License version 2 as published by the Free Software -+# Foundation, and any use by you of this program is subject to the terms -+# of such GNU license. -+# -+# This program is distributed in the hope that it will be useful, -+# but WITHOUT ANY WARRANTY; without even the implied warranty of -+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+# GNU General Public License for more details. -+# -+# You should have received a copy of the GNU General Public License -+# along with this program; if not, you can access it online at -+# http://www.gnu.org/licenses/gpl-2.0.html. -+# -+# -+ -+mali_kbase-y += \ -+ tl/mali_kbase_timeline.o \ -+ tl/mali_kbase_timeline_io.o \ -+ tl/mali_kbase_tlstream.o \ -+ tl/mali_kbase_tracepoints.o -+ -+ -+ifeq ($(CONFIG_MALI_CSF_SUPPORT),y) -+ mali_kbase-y += tl/backend/mali_kbase_timeline_csf.o -+else -+ mali_kbase-y += tl/backend/mali_kbase_timeline_jm.o -+endif -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/backend/mali_kbase_timeline_csf.c b/dvalin/kernel/drivers/gpu/arm/midgard/tl/backend/mali_kbase_timeline_csf.c -new file mode 100644 -index 0000000..a2868da ---- /dev/null -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/backend/mali_kbase_timeline_csf.c -@@ -0,0 +1,171 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#include <tl/mali_kbase_tracepoints.h> -+#include <tl/mali_kbase_timeline.h> -+#include <tl/mali_kbase_timeline_priv.h> -+ -+#include <mali_kbase.h> -+ -+void kbase_create_timeline_objects(struct kbase_device *kbdev) -+{ -+ unsigned int as_nr; -+ unsigned int slot_i; -+ struct kbase_context *kctx; -+ struct kbase_timeline *timeline = kbdev->timeline; -+ struct kbase_tlstream *summary = -+ &kbdev->timeline->streams[TL_STREAM_TYPE_OBJ_SUMMARY]; -+ -+ /* Summarize the Address Space objects. */ -+ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) -+ __kbase_tlstream_tl_new_as(summary, &kbdev->as[as_nr], as_nr); -+ -+ /* Create Legacy GPU object to track in AOM for dumping */ -+ __kbase_tlstream_tl_new_gpu(summary, -+ kbdev, -+ kbdev->gpu_props.props.raw_props.gpu_id, -+ kbdev->gpu_props.num_cores); -+ -+ -+ for (as_nr = 0; as_nr < kbdev->nr_hw_address_spaces; as_nr++) -+ __kbase_tlstream_tl_lifelink_as_gpu(summary, -+ &kbdev->as[as_nr], -+ kbdev); -+ -+ /* Trace the creation of a new kbase device and set its properties. */ -+ __kbase_tlstream_tl_kbase_new_device(summary, -+ kbdev->gpu_props.props.raw_props.gpu_id, -+ kbdev->gpu_props.num_cores, kbdev->csf.global_iface.group_num, -+ kbdev->nr_hw_address_spaces); -+ -+ /* Lock the context list, to ensure no changes to the list are made -+ * while we're summarizing the contexts and their contents. -+ */ -+ mutex_lock(&timeline->tl_kctx_list_lock); -+ -+ /* Hold the scheduler lock while we emit the current state -+ * We also need to continue holding the lock until after the first body -+ * stream tracepoints are emitted to ensure we don't change the -+ * scheduler until after then -+ */ -+ mutex_lock(&kbdev->csf.scheduler.lock); -+ -+ for (slot_i = 0; slot_i < kbdev->csf.global_iface.group_num; slot_i++) { -+ -+ struct kbase_queue_group *group = -+ kbdev->csf.scheduler.csg_slots[slot_i].resident_group; -+ -+ if (group) -+ __kbase_tlstream_tl_kbase_device_program_csg(summary, -+ kbdev->gpu_props.props.raw_props.gpu_id, -+ group->handle, slot_i); -+ } -+ -+ /* Reset body stream buffers while holding the kctx lock. -+ * As we are holding the lock, we can guarantee that no kctx creation or -+ * deletion tracepoints can be fired from outside of this function by -+ * some other thread. -+ */ -+ kbase_timeline_streams_body_reset(timeline); -+ -+ mutex_unlock(&kbdev->csf.scheduler.lock); -+ -+ /* For each context in the device... */ -+ list_for_each_entry(kctx, &timeline->tl_kctx_list, tl_kctx_list_node) { -+ size_t i; -+ struct kbase_tlstream *body = -+ &timeline->streams[TL_STREAM_TYPE_OBJ]; -+ -+ /* Lock the context's KCPU queues, to ensure no KCPU-queue -+ * related actions can occur in this context from now on. -+ */ -+ mutex_lock(&kctx->csf.kcpu_queues.lock); -+ -+ /* Acquire the MMU lock, to ensure we don't get a concurrent -+ * address space assignment while summarizing this context's -+ * address space. -+ */ -+ mutex_lock(&kbdev->mmu_hw_mutex); -+ -+ /* Trace the context itself into the body stream, not the -+ * summary stream. -+ * We place this in the body to ensure it is ordered after any -+ * other tracepoints related to the contents of the context that -+ * might have been fired before acquiring all of the per-context -+ * locks. -+ * This ensures that those tracepoints will not actually affect -+ * the object model state, as they reference a context that -+ * hasn't been traced yet. They may, however, cause benign -+ * errors to be emitted. -+ */ -+ __kbase_tlstream_tl_kbase_new_ctx(body, kctx->id, -+ kbdev->gpu_props.props.raw_props.gpu_id); -+ -+ /* Also trace with the legacy AOM tracepoint for dumping */ -+ __kbase_tlstream_tl_new_ctx(body, -+ kctx, -+ kctx->id, -+ (u32)(kctx->tgid)); -+ -+ /* Trace the currently assigned address space */ -+ if (kctx->as_nr != KBASEP_AS_NR_INVALID) -+ __kbase_tlstream_tl_kbase_ctx_assign_as(body, kctx->id, -+ kctx->as_nr); -+ -+ -+ /* Trace all KCPU queues in the context into the body stream. -+ * As we acquired the KCPU lock after resetting the body stream, -+ * it's possible that some KCPU-related events for this context -+ * occurred between that reset and now. -+ * These will cause errors to be emitted when parsing the -+ * timeline, but they will not affect the correctness of the -+ * object model. -+ */ -+ for (i = 0; i < KBASEP_MAX_KCPU_QUEUES; i++) { -+ const struct kbase_kcpu_command_queue *kcpu_queue = -+ kctx->csf.kcpu_queues.array[i]; -+ -+ if (kcpu_queue) -+ __kbase_tlstream_tl_kbase_new_kcpuqueue( -+ body, kcpu_queue, kcpu_queue->kctx->id, -+ kcpu_queue->num_pending_cmds); -+ } -+ -+ mutex_unlock(&kbdev->mmu_hw_mutex); -+ mutex_unlock(&kctx->csf.kcpu_queues.lock); -+ -+ /* Now that all per-context locks for this context have been -+ * released, any per-context tracepoints that are fired from -+ * any other threads will go into the body stream after -+ * everything that was just summarised into the body stream in -+ * this iteration of the loop, so will start to correctly update -+ * the object model state. -+ */ -+ } -+ -+ mutex_unlock(&timeline->tl_kctx_list_lock); -+ -+ /* Static object are placed into summary packet that needs to be -+ * transmitted first. Flush all streams to make it available to -+ * user space. -+ */ -+ kbase_timeline_streams_flush(timeline); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/backend/mali_kbase_timeline_jm.c b/dvalin/kernel/drivers/gpu/arm/midgard/tl/backend/mali_kbase_timeline_jm.c -index c368ac7..9ba89f5 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/backend/mali_kbase_timeline_jm.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/backend/mali_kbase_timeline_jm.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,13 +17,11 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#include "../mali_kbase_tracepoints.h" --#include "../mali_kbase_timeline.h" --#include "../mali_kbase_timeline_priv.h" -+#include <tl/mali_kbase_tracepoints.h> -+#include <tl/mali_kbase_timeline.h> -+#include <tl/mali_kbase_timeline_priv.h> - - #include <mali_kbase.h> - -@@ -66,16 +65,16 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) - /* Lock the context list, to ensure no changes to the list are made - * while we're summarizing the contexts and their contents. - */ -- mutex_lock(&kbdev->kctx_list_lock); -+ mutex_lock(&timeline->tl_kctx_list_lock); - - /* For each context in the device... */ -- list_for_each_entry(kctx, &kbdev->kctx_list, kctx_list_link) { -+ list_for_each_entry(kctx, &timeline->tl_kctx_list, tl_kctx_list_node) { - /* Summarize the context itself */ - __kbase_tlstream_tl_new_ctx(summary, - kctx, - kctx->id, - (u32)(kctx->tgid)); -- }; -+ } - - /* Reset body stream buffers while holding the kctx lock. - * This ensures we can't fire both summary and normal tracepoints for -@@ -87,11 +86,11 @@ void kbase_create_timeline_objects(struct kbase_device *kbdev) - */ - kbase_timeline_streams_body_reset(timeline); - -- mutex_unlock(&kbdev->kctx_list_lock); -+ mutex_unlock(&timeline->tl_kctx_list_lock); - - /* Static object are placed into summary packet that needs to be - * transmitted first. Flush all streams to make it available to - * user space. - */ - kbase_timeline_streams_flush(timeline); --} -\ No newline at end of file -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline.c b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline.c -index 88fba83..09818a5 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase_timeline.h" -@@ -109,11 +108,14 @@ int kbase_timeline_init(struct kbase_timeline **timeline, - { - enum tl_stream_type i; - struct kbase_timeline *result; -+#if MALI_USE_CSF -+ struct kbase_tlstream *csffw_stream; -+#endif - - if (!timeline || !timeline_flags) - return -EINVAL; - -- result = kzalloc(sizeof(*result), GFP_KERNEL); -+ result = vzalloc(sizeof(*result)); - if (!result) - return -ENOMEM; - -@@ -125,12 +127,20 @@ int kbase_timeline_init(struct kbase_timeline **timeline, - kbase_tlstream_init(&result->streams[i], i, - &result->event_queue); - -+ /* Initialize the kctx list */ -+ mutex_init(&result->tl_kctx_list_lock); -+ INIT_LIST_HEAD(&result->tl_kctx_list); -+ - /* Initialize autoflush timer. */ - atomic_set(&result->autoflush_timer_active, 0); - kbase_timer_setup(&result->autoflush_timer, - kbasep_timeline_autoflush_timer_callback); - result->timeline_flags = timeline_flags; - -+#if MALI_USE_CSF -+ csffw_stream = &result->streams[TL_STREAM_TYPE_CSFFW]; -+ kbase_csf_tl_reader_init(&result->csf_tl_reader, csffw_stream); -+#endif - - *timeline = result; - return 0; -@@ -143,11 +153,16 @@ void kbase_timeline_term(struct kbase_timeline *timeline) - if (!timeline) - return; - -+#if MALI_USE_CSF -+ kbase_csf_tl_reader_term(&timeline->csf_tl_reader); -+#endif -+ -+ WARN_ON(!list_empty(&timeline->tl_kctx_list)); - - for (i = (enum tl_stream_type)0; i < TL_STREAM_TYPE_COUNT; i++) - kbase_tlstream_term(&timeline->streams[i]); - -- kfree(timeline); -+ vfree(timeline); - } - - #ifdef CONFIG_MALI_DEVFREQ -@@ -162,11 +177,7 @@ static void kbase_tlstream_current_devfreq_target(struct kbase_device *kbdev) - unsigned long cur_freq = 0; - - mutex_lock(&devfreq->lock); --#if KERNEL_VERSION(4, 3, 0) > LINUX_VERSION_CODE -- cur_freq = kbdev->current_nominal_freq; --#else - cur_freq = devfreq->last_status.current_frequency; --#endif - KBASE_TLSTREAM_AUX_DEVFREQ_TARGET(kbdev, (u64)cur_freq); - mutex_unlock(&devfreq->lock); - } -@@ -175,13 +186,24 @@ static void kbase_tlstream_current_devfreq_target(struct kbase_device *kbdev) - - int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags) - { -- int ret; -+ int ret = 0; - u32 timeline_flags = TLSTREAM_ENABLED | flags; - struct kbase_timeline *timeline = kbdev->timeline; - - if (!atomic_cmpxchg(timeline->timeline_flags, 0, timeline_flags)) { - int rcode; - -+#if MALI_USE_CSF -+ if (flags & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) { -+ ret = kbase_csf_tl_reader_start( -+ &timeline->csf_tl_reader, kbdev); -+ if (ret) -+ { -+ atomic_set(timeline->timeline_flags, 0); -+ return ret; -+ } -+ } -+#endif - ret = anon_inode_getfd( - "[mali_tlstream]", - &kbasep_tlstream_fops, -@@ -189,6 +211,9 @@ int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags) - O_RDONLY | O_CLOEXEC); - if (ret < 0) { - atomic_set(timeline->timeline_flags, 0); -+#if MALI_USE_CSF -+ kbase_csf_tl_reader_stop(&timeline->csf_tl_reader); -+#endif - return ret; - } - -@@ -206,6 +231,7 @@ int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags) - jiffies + msecs_to_jiffies(AUTOFLUSH_INTERVAL)); - CSTD_UNUSED(rcode); - -+#if !MALI_USE_CSF - /* If job dumping is enabled, readjust the software event's - * timeout as the default value of 3 seconds is often - * insufficient. -@@ -216,6 +242,7 @@ int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags) - atomic_set(&kbdev->js_data.soft_job_timeout_ms, - 1800000); - } -+#endif /* !MALI_USE_CSF */ - - /* Summary stream was cleared during acquire. - * Create static timeline objects that will be -@@ -235,15 +262,30 @@ int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags) - ret = -EBUSY; - } - -+ if (ret >= 0) -+ timeline->last_acquire_time = ktime_get(); -+ - return ret; - } - --void kbase_timeline_streams_flush(struct kbase_timeline *timeline) -+int kbase_timeline_streams_flush(struct kbase_timeline *timeline) - { - enum tl_stream_type stype; -+ bool has_bytes = false; -+ size_t nbytes = 0; -+#if MALI_USE_CSF -+ int ret = kbase_csf_tl_reader_flush_buffer(&timeline->csf_tl_reader); -+ -+ if (ret > 0) -+ has_bytes = true; -+#endif - -- for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) -- kbase_tlstream_flush_stream(&timeline->streams[stype]); -+ for (stype = 0; stype < TL_STREAM_TYPE_COUNT; stype++) { -+ nbytes = kbase_tlstream_flush_stream(&timeline->streams[stype]); -+ if (nbytes > 0) -+ has_bytes = true; -+ } -+ return has_bytes ? 0 : -EIO; - } - - void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline) -@@ -252,6 +294,78 @@ void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline) - &timeline->streams[TL_STREAM_TYPE_OBJ]); - kbase_tlstream_reset( - &timeline->streams[TL_STREAM_TYPE_AUX]); -+#if MALI_USE_CSF -+ kbase_tlstream_reset( -+ &timeline->streams[TL_STREAM_TYPE_CSFFW]); -+#endif -+} -+ -+void kbase_timeline_pre_kbase_context_destroy(struct kbase_context *kctx) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ struct kbase_timeline *timeline = kbdev->timeline; -+ -+ /* Remove the context from the list to ensure we don't try and -+ * summarize a context that is being destroyed. -+ * -+ * It's unsafe to try and summarize a context being destroyed as the -+ * locks we might normally attempt to acquire, and the data structures -+ * we would normally attempt to traverse could already be destroyed. -+ * -+ * In the case where the tlstream is acquired between this pre destroy -+ * call and the post destroy call, we will get a context destroy -+ * tracepoint without the corresponding context create tracepoint, -+ * but this will not affect the correctness of the object model. -+ */ -+ mutex_lock(&timeline->tl_kctx_list_lock); -+ list_del_init(&kctx->tl_kctx_list_node); -+ mutex_unlock(&timeline->tl_kctx_list_lock); -+} -+ -+void kbase_timeline_post_kbase_context_create(struct kbase_context *kctx) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ struct kbase_timeline *timeline = kbdev->timeline; -+ -+ /* On context create, add the context to the list to ensure it is -+ * summarized when timeline is acquired -+ */ -+ mutex_lock(&timeline->tl_kctx_list_lock); -+ -+ list_add(&kctx->tl_kctx_list_node, &timeline->tl_kctx_list); -+ -+ /* Fire the tracepoints with the lock held to ensure the tracepoints -+ * are either fired before or after the summarization, -+ * never in parallel with it. If fired in parallel, we could get -+ * duplicate creation tracepoints. -+ */ -+#if MALI_USE_CSF -+ KBASE_TLSTREAM_TL_KBASE_NEW_CTX( -+ kbdev, kctx->id, kbdev->gpu_props.props.raw_props.gpu_id); -+#endif -+ /* Trace with the AOM tracepoint even in CSF for dumping */ -+ KBASE_TLSTREAM_TL_NEW_CTX(kbdev, kctx, kctx->id, 0); -+ -+ mutex_unlock(&timeline->tl_kctx_list_lock); -+} -+ -+void kbase_timeline_post_kbase_context_destroy(struct kbase_context *kctx) -+{ -+ struct kbase_device *const kbdev = kctx->kbdev; -+ -+ /* Trace with the AOM tracepoint even in CSF for dumping */ -+ KBASE_TLSTREAM_TL_DEL_CTX(kbdev, kctx); -+#if MALI_USE_CSF -+ KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kbdev, kctx->id); -+#endif -+ -+ /* Flush the timeline stream, so the user can see the termination -+ * tracepoints being fired. -+ * The "if" statement below is for optimization. It is safe to call -+ * kbase_timeline_streams_flush when timeline is disabled. -+ */ -+ if (atomic_read(&kbdev->timeline_flags) != 0) -+ kbase_timeline_streams_flush(kbdev->timeline); - } - - #if MALI_UNIT_TEST -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline.h b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline.h -index cd48411..63926eb 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #if !defined(_KBASE_TIMELINE_H) -@@ -70,8 +69,10 @@ int kbase_timeline_io_acquire(struct kbase_device *kbdev, u32 flags); - * @timeline: Timeline instance - * - * Function will flush pending data in all timeline streams. -+ * -+ * Return: Zero on success, errno on failure. - */ --void kbase_timeline_streams_flush(struct kbase_timeline *timeline); -+int kbase_timeline_streams_flush(struct kbase_timeline *timeline); - - /** - * kbase_timeline_streams_body_reset - reset timeline body streams. -@@ -81,33 +82,31 @@ void kbase_timeline_streams_flush(struct kbase_timeline *timeline); - */ - void kbase_timeline_streams_body_reset(struct kbase_timeline *timeline); - --#if MALI_UNIT_TEST - /** -- * kbase_timeline_test - start timeline stream data generator -- * @kbdev: Kernel common context -- * @tpw_count: Number of trace point writers in each context -- * @msg_delay: Time delay in milliseconds between trace points written by one -- * writer -- * @msg_count: Number of trace points written by one writer -- * @aux_msg: If non-zero aux messages will be included -+ * kbase_timeline_post_kbase_context_create - Inform timeline that a new KBase -+ * Context has been created. -+ * @kctx: KBase Context -+ */ -+void kbase_timeline_post_kbase_context_create(struct kbase_context *kctx); -+ -+/** -+ * kbase_timeline_pre_kbase_context_destroy - Inform timeline that a KBase -+ * Context is about to be destroyed. -+ * @kctx: KBase Context -+ */ -+void kbase_timeline_pre_kbase_context_destroy(struct kbase_context *kctx); -+ -+/** -+ * kbase_timeline_post_kbase_context_destroy - Inform timeline that a KBase -+ * Context has been destroyed. -+ * @kctx: KBase Context - * -- * This test starts a requested number of asynchronous writers in both IRQ and -- * thread context. Each writer will generate required number of test -- * tracepoints (tracepoints with embedded information about writer that -- * should be verified by user space reader). Tracepoints will be emitted in -- * all timeline body streams. If aux_msg is non-zero writer will also -- * generate not testable tracepoints (tracepoints without information about -- * writer). These tracepoints are used to check correctness of remaining -- * timeline message generating functions. Writer will wait requested time -- * between generating another set of messages. This call blocks until all -- * writers finish. -+ * Should be called immediately before the memory is freed, and the context ID -+ * and kbdev pointer should still be valid. - */ --void kbase_timeline_test( -- struct kbase_device *kbdev, -- unsigned int tpw_count, -- unsigned int msg_delay, -- unsigned int msg_count, -- int aux_msg); -+void kbase_timeline_post_kbase_context_destroy(struct kbase_context *kctx); -+ -+#if MALI_UNIT_TEST - - /** - * kbase_timeline_stats - read timeline stream statistics -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline_io.c b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline_io.c -index cdde928..eff30a0 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline_io.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline_io.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,35 +17,38 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase_timeline_priv.h" - #include "mali_kbase_tlstream.h" - #include "mali_kbase_tracepoints.h" -+#include "mali_kbase_timeline.h" - -+#include <linux/delay.h> - #include <linux/poll.h> - - /* The timeline stream file operations functions. */ --static ssize_t kbasep_timeline_io_read( -- struct file *filp, -- char __user *buffer, -- size_t size, -- loff_t *f_pos); --static unsigned int kbasep_timeline_io_poll(struct file *filp, poll_table *wait); -+static ssize_t kbasep_timeline_io_read(struct file *filp, char __user *buffer, -+ size_t size, loff_t *f_pos); -+static unsigned int kbasep_timeline_io_poll(struct file *filp, -+ poll_table *wait); - static int kbasep_timeline_io_release(struct inode *inode, struct file *filp); -+static int kbasep_timeline_io_fsync(struct file *filp, loff_t start, loff_t end, -+ int datasync); - - /* The timeline stream file operations structure. */ - const struct file_operations kbasep_tlstream_fops = { - .owner = THIS_MODULE, - .release = kbasep_timeline_io_release, -- .read = kbasep_timeline_io_read, -- .poll = kbasep_timeline_io_poll, -+ .read = kbasep_timeline_io_read, -+ .poll = kbasep_timeline_io_poll, -+ .fsync = kbasep_timeline_io_fsync, - }; - - /** -- * kbasep_timeline_io_packet_pending - check timeline streams for pending packets -+ * kbasep_timeline_io_packet_pending - check timeline streams for pending -+ * packets -+ * - * @timeline: Timeline instance - * @ready_stream: Pointer to variable where stream will be placed - * @rb_idx_raw: Pointer to variable where read buffer index will be placed -@@ -56,10 +60,10 @@ const struct file_operations kbasep_tlstream_fops = { - * - * Return: non-zero if any of timeline streams has at last one packet ready - */ --static int kbasep_timeline_io_packet_pending( -- struct kbase_timeline *timeline, -- struct kbase_tlstream **ready_stream, -- unsigned int *rb_idx_raw) -+static int -+kbasep_timeline_io_packet_pending(struct kbase_timeline *timeline, -+ struct kbase_tlstream **ready_stream, -+ unsigned int *rb_idx_raw) - { - enum tl_stream_type i; - -@@ -78,12 +82,28 @@ static int kbasep_timeline_io_packet_pending( - *ready_stream = stream; - return 1; - } -- - } - - return 0; - } - -+/** -+ * kbasep_timeline_has_header_data() - check timeline headers for pending -+ * packets -+ * -+ * @timeline: Timeline instance -+ * -+ * Return: non-zero if any of timeline headers has at last one packet ready. -+ */ -+static int kbasep_timeline_has_header_data(struct kbase_timeline *timeline) -+{ -+ return timeline->obj_header_btc || timeline->aux_header_btc -+#if MALI_USE_CSF -+ || timeline->csf_tl_reader.tl_header.btc -+#endif -+ ; -+} -+ - /** - * copy_stream_header() - copy timeline stream header. - * -@@ -97,11 +117,9 @@ static int kbasep_timeline_io_packet_pending( - * - * Returns: 0 if success, -1 otherwise. - */ --static inline int copy_stream_header( -- char __user *buffer, size_t size, ssize_t *copy_len, -- const char *hdr, -- size_t hdr_size, -- size_t *hdr_btc) -+static inline int copy_stream_header(char __user *buffer, size_t size, -+ ssize_t *copy_len, const char *hdr, -+ size_t hdr_size, size_t *hdr_btc) - { - const size_t offset = hdr_size - *hdr_btc; - const size_t copy_size = MIN(size - *copy_len, *hdr_btc); -@@ -123,6 +141,7 @@ static inline int copy_stream_header( - - /** - * kbasep_timeline_copy_header - copy timeline headers to the user -+ * - * @timeline: Timeline instance - * @buffer: Pointer to the buffer provided by user - * @size: Maximum amount of data that can be stored in the buffer -@@ -135,29 +154,30 @@ static inline int copy_stream_header( - * - * Returns: 0 if success, -1 if copy_to_user has failed. - */ --static inline int kbasep_timeline_copy_headers( -- struct kbase_timeline *timeline, -- char __user *buffer, -- size_t size, -- ssize_t *copy_len) -+static inline int kbasep_timeline_copy_headers(struct kbase_timeline *timeline, -+ char __user *buffer, size_t size, -+ ssize_t *copy_len) - { -- if (copy_stream_header(buffer, size, copy_len, -- obj_desc_header, -- obj_desc_header_size, -- &timeline->obj_header_btc)) -+ if (copy_stream_header(buffer, size, copy_len, obj_desc_header, -+ obj_desc_header_size, &timeline->obj_header_btc)) - return -1; - -+ if (copy_stream_header(buffer, size, copy_len, aux_desc_header, -+ aux_desc_header_size, &timeline->aux_header_btc)) -+ return -1; -+#if MALI_USE_CSF - if (copy_stream_header(buffer, size, copy_len, -- aux_desc_header, -- aux_desc_header_size, -- &timeline->aux_header_btc)) -+ timeline->csf_tl_reader.tl_header.data, -+ timeline->csf_tl_reader.tl_header.size, -+ &timeline->csf_tl_reader.tl_header.btc)) - return -1; -+#endif - return 0; - } - -- - /** - * kbasep_timeline_io_read - copy data from streams to buffer provided by user -+ * - * @filp: Pointer to file structure - * @buffer: Pointer to the buffer provided by user - * @size: Maximum amount of data that can be stored in the buffer -@@ -165,11 +185,8 @@ static inline int kbasep_timeline_copy_headers( - * - * Return: number of bytes stored in the buffer - */ --static ssize_t kbasep_timeline_io_read( -- struct file *filp, -- char __user *buffer, -- size_t size, -- loff_t *f_pos) -+static ssize_t kbasep_timeline_io_read(struct file *filp, char __user *buffer, -+ size_t size, loff_t *f_pos) - { - ssize_t copy_len = 0; - struct kbase_timeline *timeline; -@@ -180,25 +197,25 @@ static ssize_t kbasep_timeline_io_read( - if (WARN_ON(!filp->private_data)) - return -EFAULT; - -- timeline = (struct kbase_timeline *) filp->private_data; -+ timeline = (struct kbase_timeline *)filp->private_data; - - if (!buffer) - return -EINVAL; - -- if ((*f_pos < 0) || (size < PACKET_SIZE)) -+ if (*f_pos < 0) - return -EINVAL; - - mutex_lock(&timeline->reader_lock); - - while (copy_len < size) { - struct kbase_tlstream *stream = NULL; -- unsigned int rb_idx_raw = 0; -- unsigned int wb_idx_raw; -- unsigned int rb_idx; -- size_t rb_size; -+ unsigned int rb_idx_raw = 0; -+ unsigned int wb_idx_raw; -+ unsigned int rb_idx; -+ size_t rb_size; - -- if (kbasep_timeline_copy_headers( -- timeline, buffer, size, ©_len)) { -+ if (kbasep_timeline_copy_headers(timeline, buffer, size, -+ ©_len)) { - copy_len = -EFAULT; - break; - } -@@ -210,17 +227,13 @@ static ssize_t kbasep_timeline_io_read( - */ - if (copy_len > 0) { - if (!kbasep_timeline_io_packet_pending( -- timeline, -- &stream, -- &rb_idx_raw)) -+ timeline, &stream, &rb_idx_raw)) - break; - } else { - if (wait_event_interruptible( -- timeline->event_queue, -- kbasep_timeline_io_packet_pending( -- timeline, -- &stream, -- &rb_idx_raw))) { -+ timeline->event_queue, -+ kbasep_timeline_io_packet_pending( -+ timeline, &stream, &rb_idx_raw))) { - copy_len = -ERESTARTSYS; - break; - } -@@ -238,10 +251,8 @@ static ssize_t kbasep_timeline_io_read( - rb_size = atomic_read(&stream->buffer[rb_idx].size); - if (rb_size > size - copy_len) - break; -- if (copy_to_user( -- &buffer[copy_len], -- stream->buffer[rb_idx].data, -- rb_size)) { -+ if (copy_to_user(&buffer[copy_len], stream->buffer[rb_idx].data, -+ rb_size)) { - copy_len = -EFAULT; - break; - } -@@ -283,7 +294,7 @@ static ssize_t kbasep_timeline_io_read( - static unsigned int kbasep_timeline_io_poll(struct file *filp, poll_table *wait) - { - struct kbase_tlstream *stream; -- unsigned int rb_idx; -+ unsigned int rb_idx; - struct kbase_timeline *timeline; - - KBASE_DEBUG_ASSERT(filp); -@@ -292,7 +303,11 @@ static unsigned int kbasep_timeline_io_poll(struct file *filp, poll_table *wait) - if (WARN_ON(!filp->private_data)) - return -EFAULT; - -- timeline = (struct kbase_timeline *) filp->private_data; -+ timeline = (struct kbase_timeline *)filp->private_data; -+ -+ /* If there are header bytes to copy, read will not block */ -+ if (kbasep_timeline_has_header_data(timeline)) -+ return POLLIN; - - poll_wait(filp, &timeline->event_queue, wait); - if (kbasep_timeline_io_packet_pending(timeline, &stream, &rb_idx)) -@@ -310,6 +325,8 @@ static unsigned int kbasep_timeline_io_poll(struct file *filp, poll_table *wait) - static int kbasep_timeline_io_release(struct inode *inode, struct file *filp) - { - struct kbase_timeline *timeline; -+ ktime_t elapsed_time; -+ s64 elapsed_time_ms, time_to_sleep; - - KBASE_DEBUG_ASSERT(inode); - KBASE_DEBUG_ASSERT(filp); -@@ -317,8 +334,23 @@ static int kbasep_timeline_io_release(struct inode *inode, struct file *filp) - - CSTD_UNUSED(inode); - -- timeline = (struct kbase_timeline *) filp->private_data; -+ timeline = (struct kbase_timeline *)filp->private_data; -+ -+ /* Get the amount of time passed since the timeline was acquired and ensure -+ * we sleep for long enough such that it has been at least -+ * TIMELINE_HYSTERESIS_TIMEOUT_MS amount of time between acquire and release. -+ * This prevents userspace from spamming acquire and release too quickly. -+ */ -+ elapsed_time = ktime_sub(ktime_get(), timeline->last_acquire_time); -+ elapsed_time_ms = ktime_to_ms(elapsed_time); -+ time_to_sleep = MIN(TIMELINE_HYSTERESIS_TIMEOUT_MS, -+ TIMELINE_HYSTERESIS_TIMEOUT_MS - elapsed_time_ms); -+ if (time_to_sleep > 0) -+ msleep(time_to_sleep); - -+#if MALI_USE_CSF -+ kbase_csf_tl_reader_stop(&timeline->csf_tl_reader); -+#endif - - /* Stop autoflush timer before releasing access to streams. */ - atomic_set(&timeline->autoflush_timer_active, 0); -@@ -327,3 +359,20 @@ static int kbasep_timeline_io_release(struct inode *inode, struct file *filp) - atomic_set(timeline->timeline_flags, 0); - return 0; - } -+ -+static int kbasep_timeline_io_fsync(struct file *filp, loff_t start, loff_t end, -+ int datasync) -+{ -+ struct kbase_timeline *timeline; -+ -+ CSTD_UNUSED(start); -+ CSTD_UNUSED(end); -+ CSTD_UNUSED(datasync); -+ -+ if (WARN_ON(!filp->private_data)) -+ return -EFAULT; -+ -+ timeline = (struct kbase_timeline *)filp->private_data; -+ -+ return kbase_timeline_streams_flush(timeline); -+} -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline_priv.h b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline_priv.h -index 3596584..bf2c385 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline_priv.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_timeline_priv.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #if !defined(_KBASE_TIMELINE_PRIV_H) -@@ -26,14 +25,25 @@ - #include <mali_kbase.h> - #include "mali_kbase_tlstream.h" - -+#if MALI_USE_CSF -+#include "csf/mali_kbase_csf_tl_reader.h" -+#include "csf/mali_kbase_csf_trace_buffer.h" -+#endif - - #include <linux/timer.h> - #include <linux/atomic.h> - #include <linux/mutex.h> - -+/* The minimum amount of time timeline must be acquired for before release is -+ * allowed, to prevent DoS attacks. -+ */ -+#define TIMELINE_HYSTERESIS_TIMEOUT_MS ((s64)500) -+ - /** - * struct kbase_timeline - timeline state structure - * @streams: The timeline streams generated by kernel -+ * @tl_kctx_list: List of contexts for timeline. -+ * @tl_kctx_list_lock: Lock to protect @tl_kctx_list. - * @autoflush_timer: Autoflush timer - * @autoflush_timer_active: If non-zero autoflush timer is active - * @reader_lock: Reader lock. Only one reader is allowed to -@@ -44,9 +54,13 @@ - * otherwise. See kbase_timeline_io_acquire(). - * @obj_header_btc: Remaining bytes to copy for the object stream header - * @aux_header_btc: Remaining bytes to copy for the aux stream header -+ * @last_acquire_time: The time at which timeline was last acquired. -+ * @csf_tl_reader: CSFFW timeline reader - */ - struct kbase_timeline { - struct kbase_tlstream streams[TL_STREAM_TYPE_COUNT]; -+ struct list_head tl_kctx_list; -+ struct mutex tl_kctx_list_lock; - struct timer_list autoflush_timer; - atomic_t autoflush_timer_active; - struct mutex reader_lock; -@@ -57,6 +71,10 @@ struct kbase_timeline { - atomic_t *timeline_flags; - size_t obj_header_btc; - size_t aux_header_btc; -+ ktime_t last_acquire_time; -+#if MALI_USE_CSF -+ struct kbase_csf_tl_reader csf_tl_reader; -+#endif - }; - - extern const struct file_operations kbasep_tlstream_fops; -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tl_serialize.h b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tl_serialize.h -index 3e37827..30d120d 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tl_serialize.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tl_serialize.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #if !defined(_KBASE_TL_SERIALIZE_H) -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tlstream.c b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tlstream.c -index bec4be7..0275bad 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tlstream.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tlstream.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #include "mali_kbase_tlstream.h" -@@ -57,20 +56,19 @@ static void kbasep_packet_header_setup( - * @numbered: non-zero if the stream is numbered - * - * Function updates mutable part of packet header in the given buffer. -- * Note that value of data_size must not including size of the header. -+ * Note that value of data_size must not include size of the header. - */ - static void kbasep_packet_header_update( - char *buffer, - size_t data_size, - int numbered) - { -- u32 word0; - u32 word1 = MIPE_PACKET_HEADER_W1((u32)data_size, !!numbered); - - KBASE_DEBUG_ASSERT(buffer); -- CSTD_UNUSED(word0); - -- memcpy(&buffer[sizeof(word0)], &word1, sizeof(word1)); -+ /* we copy the contents of word1 to its respective position in the buffer */ -+ memcpy(&buffer[sizeof(u32)], &word1, sizeof(word1)); - } - - /** -@@ -131,6 +129,14 @@ static const struct { - TL_PACKET_TYPE_BODY, - TL_STREAM_ID_KERNEL, - }, -+#if MALI_USE_CSF -+ { -+ TL_PACKET_FAMILY_TL, -+ TL_PACKET_CLASS_OBJ, -+ TL_PACKET_TYPE_BODY, -+ TL_STREAM_ID_CSFFW, -+ }, -+#endif - }; - - void kbase_tlstream_init( -@@ -141,12 +147,12 @@ void kbase_tlstream_init( - unsigned int i; - - KBASE_DEBUG_ASSERT(stream); -- KBASE_DEBUG_ASSERT(TL_STREAM_TYPE_COUNT > stream_type); -+ KBASE_DEBUG_ASSERT(stream_type < TL_STREAM_TYPE_COUNT); - - spin_lock_init(&stream->lock); - - /* All packets carrying tracepoints shall be numbered. */ -- if (TL_PACKET_TYPE_BODY == tl_stream_cfg[stream_type].pkt_type) -+ if (tl_stream_cfg[stream_type].pkt_type == TL_PACKET_TYPE_BODY) - stream->numbered = 1; - else - stream->numbered = 0; -@@ -209,7 +215,8 @@ static size_t kbasep_tlstream_msgbuf_submit( - - /* Increasing write buffer index will expose this packet to the reader. - * As stream->lock is not taken on reader side we must make sure memory -- * is updated correctly before this will happen. */ -+ * is updated correctly before this will happen. -+ */ - smp_wmb(); - atomic_inc(&stream->wbi); - -@@ -243,7 +250,7 @@ char *kbase_tlstream_msgbuf_acquire( - wb_size = atomic_read(&stream->buffer[wb_idx].size); - - /* Select next buffer if data will not fit into current one. */ -- if (PACKET_SIZE < wb_size + msg_size) { -+ if (wb_size + msg_size > PACKET_SIZE) { - wb_size = kbasep_tlstream_msgbuf_submit( - stream, wb_idx_raw, wb_size); - wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; -@@ -269,7 +276,7 @@ void kbase_tlstream_msgbuf_release( - spin_unlock_irqrestore(&stream->lock, flags); - } - --void kbase_tlstream_flush_stream( -+size_t kbase_tlstream_flush_stream( - struct kbase_tlstream *stream) - { - unsigned long flags; -@@ -278,6 +285,7 @@ void kbase_tlstream_flush_stream( - size_t wb_size; - size_t min_size = PACKET_HEADER_SIZE; - -+ - if (stream->numbered) - min_size += PACKET_NUMBER_SIZE; - -@@ -292,7 +300,14 @@ void kbase_tlstream_flush_stream( - stream, wb_idx_raw, wb_size); - wb_idx = (wb_idx_raw + 1) % PACKET_COUNT; - atomic_set(&stream->buffer[wb_idx].size, wb_size); -+ } else { -+ /* we return that there is no bytes to be read.*/ -+ /* Timeline io fsync will use this info the decide whether -+ * fsync should return an error -+ */ -+ wb_size = 0; - } -+ - spin_unlock_irqrestore(&stream->lock, flags); -+ return wb_size; - } -- -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tlstream.h b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tlstream.h -index 427bb09..6660cf5 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tlstream.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tlstream.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #if !defined(_KBASE_TLSTREAM_H) -@@ -44,6 +43,8 @@ - * struct kbase_tlstream - timeline stream structure - * @lock: Message order lock - * @buffer: Array of buffers -+ * @buffer.size: Number of bytes in buffer -+ * @buffer.data: Buffer's data - * @wbi: Write buffer index - * @rbi: Read buffer index - * @numbered: If non-zero stream's packets are sequentially numbered -@@ -76,8 +77,8 @@ struct kbase_tlstream { - spinlock_t lock; - - struct { -- atomic_t size; /* number of bytes in buffer */ -- char data[PACKET_SIZE]; /* buffer's data */ -+ atomic_t size; -+ char data[PACKET_SIZE]; - } buffer[PACKET_COUNT]; - - atomic_t wbi; -@@ -97,6 +98,9 @@ enum tl_stream_type { - TL_STREAM_TYPE_OBJ_SUMMARY = TL_STREAM_TYPE_FIRST, - TL_STREAM_TYPE_OBJ, - TL_STREAM_TYPE_AUX, -+#if MALI_USE_CSF -+ TL_STREAM_TYPE_CSFFW, -+#endif - TL_STREAM_TYPE_COUNT - }; - -@@ -159,8 +163,10 @@ void kbase_tlstream_msgbuf_release(struct kbase_tlstream *stream, - * @stream: Pointer to the stream structure - * - * Flush pending data in the timeline stream. -+ * -+ * Return: Number of bytes available flushed and available to be read -+ * - */ --void kbase_tlstream_flush_stream(struct kbase_tlstream *stream); -+size_t kbase_tlstream_flush_stream(struct kbase_tlstream *stream); - - #endif /* _KBASE_TLSTREAM_H */ -- -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tracepoints.c b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tracepoints.c -index e445a3a..2c0de01 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tracepoints.c -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tracepoints.c -@@ -1,11 +1,12 @@ -+// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -66,10 +65,11 @@ enum tl_msg_id_obj { - KBASE_TL_EVENT_ATOM_SOFTSTOP_ISSUE, - KBASE_TL_EVENT_ATOM_SOFTJOB_START, - KBASE_TL_EVENT_ATOM_SOFTJOB_END, -- KBASE_TL_EVENT_ARB_GRANTED, -- KBASE_TL_EVENT_ARB_STARTED, -- KBASE_TL_EVENT_ARB_STOP_REQUESTED, -- KBASE_TL_EVENT_ARB_STOPPED, -+ KBASE_TL_ARBITER_GRANTED, -+ KBASE_TL_ARBITER_STARTED, -+ KBASE_TL_ARBITER_STOP_REQUESTED, -+ KBASE_TL_ARBITER_STOPPED, -+ KBASE_TL_ARBITER_REQUESTED, - KBASE_JD_GPU_SOFT_RESET, - KBASE_TL_KBASE_NEW_DEVICE, - KBASE_TL_KBASE_DEVICE_PROGRAM_CSG, -@@ -82,18 +82,13 @@ enum tl_msg_id_obj { - KBASE_TL_KBASE_DEL_KCPUQUEUE, - KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL, - KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT, -- KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT, -- KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_WAIT, -- KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_WAIT, -- KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_SET, -- KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_SET, -- KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_SET, -- KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_DEBUGCOPY, -- KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_DEBUGCOPY, -- KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY, -+ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT, -+ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET, - KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT, - KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT, - KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE, -+ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER, -+ KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND, - KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC, - KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC, - KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC, -@@ -107,8 +102,6 @@ enum tl_msg_id_obj { - KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START, - KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END, - KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET, -- KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_START, -- KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_END, - KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START, - KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END, - KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START, -@@ -123,8 +116,11 @@ enum tl_msg_id_obj { - KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END, - KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END, - KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END, -- KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER, -+ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER, -+ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START, -+ KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END, - KBASE_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW, -+ KBASE_TL_KBASE_CSFFW_RESET, - KBASE_OBJ_MSG_COUNT, - }; - -@@ -139,6 +135,7 @@ enum tl_msg_id_aux { - KBASE_AUX_PROTECTED_LEAVE_START, - KBASE_AUX_PROTECTED_LEAVE_END, - KBASE_AUX_JIT_STATS, -+ KBASE_AUX_TILER_HEAP_STATS, - KBASE_AUX_EVENT_JOB_SLOT, - KBASE_AUX_MSG_COUNT, - }; -@@ -276,22 +273,26 @@ enum tl_msg_id_aux { - "atom soft job has completed", \ - "@p", \ - "atom") \ -- TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_GRANTED, \ -+ TRACEPOINT_DESC(KBASE_TL_ARBITER_GRANTED, \ - "Arbiter has granted gpu access", \ - "@p", \ - "gpu") \ -- TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_STARTED, \ -+ TRACEPOINT_DESC(KBASE_TL_ARBITER_STARTED, \ - "Driver is running again and able to process jobs", \ - "@p", \ - "gpu") \ -- TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_STOP_REQUESTED, \ -+ TRACEPOINT_DESC(KBASE_TL_ARBITER_STOP_REQUESTED, \ - "Arbiter has requested driver to stop using gpu", \ - "@p", \ - "gpu") \ -- TRACEPOINT_DESC(KBASE_TL_EVENT_ARB_STOPPED, \ -+ TRACEPOINT_DESC(KBASE_TL_ARBITER_STOPPED, \ - "Driver has stopped using gpu", \ - "@p", \ - "gpu") \ -+ TRACEPOINT_DESC(KBASE_TL_ARBITER_REQUESTED, \ -+ "Driver has requested the arbiter for gpu access", \ -+ "@p", \ -+ "gpu") \ - TRACEPOINT_DESC(KBASE_JD_GPU_SOFT_RESET, \ - "gpu soft reset", \ - "@p", \ -@@ -340,42 +341,14 @@ enum tl_msg_id_aux { - "KCPU Queue enqueues Wait on Fence", \ - "@pp", \ - "kcpu_queue,fence") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT, \ -- "Begin array of KCPU Queue enqueues Wait on Cross Queue Sync Object", \ -- "@p", \ -- "kcpu_queue") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_WAIT, \ -- "Array item of KCPU Queue enqueues Wait on Cross Queue Sync Object", \ -- "@pLI", \ -- "kcpu_queue,cqs_obj_gpu_addr,cqs_obj_compare_value") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_WAIT, \ -- "End array of KCPU Queue enqueues Wait on Cross Queue Sync Object", \ -- "@p", \ -- "kcpu_queue") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_SET, \ -- "Begin array of KCPU Queue enqueues Set on Cross Queue Sync Object", \ -- "@p", \ -- "kcpu_queue") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_SET, \ -- "Array item of KCPU Queue enqueues Set on Cross Queue Sync Object", \ -+ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT, \ -+ "KCPU Queue enqueues Wait on Cross Queue Sync Object", \ -+ "@pLII", \ -+ "kcpu_queue,cqs_obj_gpu_addr,cqs_obj_compare_value,cqs_obj_inherit_error") \ -+ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET, \ -+ "KCPU Queue enqueues Set on Cross Queue Sync Object", \ - "@pL", \ - "kcpu_queue,cqs_obj_gpu_addr") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_SET, \ -- "End array of KCPU Queue enqueues Set on Cross Queue Sync Object", \ -- "@p", \ -- "kcpu_queue") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_DEBUGCOPY, \ -- "Begin array of KCPU Queue enqueues Debug Copy", \ -- "@p", \ -- "kcpu_queue") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_DEBUGCOPY, \ -- "Array item of KCPU Queue enqueues Debug Copy", \ -- "@pL", \ -- "kcpu_queue,debugcopy_dst_size") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY, \ -- "End array of KCPU Queue enqueues Debug Copy", \ -- "@p", \ -- "kcpu_queue") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT, \ - "KCPU Queue enqueues Map Import", \ - "@pL", \ -@@ -388,6 +361,14 @@ enum tl_msg_id_aux { - "KCPU Queue enqueues Unmap Import ignoring reference count", \ - "@pL", \ - "kcpu_queue,map_import_buf_gpu_addr") \ -+ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER, \ -+ "KCPU Queue enqueues Error Barrier", \ -+ "@p", \ -+ "kcpu_queue") \ -+ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND, \ -+ "KCPU Queue enqueues Group Suspend", \ -+ "@ppI", \ -+ "kcpu_queue,group_suspend_buf,gpu_cmdq_grp_handle") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC, \ - "Begin array of KCPU Queue enqueues JIT Alloc", \ - "@p", \ -@@ -418,60 +399,52 @@ enum tl_msg_id_aux { - "kcpu_queue") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END, \ - "KCPU Queue ends a Signal on Fence", \ -- "@p", \ -- "kcpu_queue") \ -+ "@pI", \ -+ "kcpu_queue,execute_error") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START, \ - "KCPU Queue starts a Wait on Fence", \ - "@p", \ - "kcpu_queue") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END, \ - "KCPU Queue ends a Wait on Fence", \ -- "@p", \ -- "kcpu_queue") \ -+ "@pI", \ -+ "kcpu_queue,execute_error") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START, \ - "KCPU Queue starts a Wait on an array of Cross Queue Sync Objects", \ - "@p", \ - "kcpu_queue") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END, \ - "KCPU Queue ends a Wait on an array of Cross Queue Sync Objects", \ -- "@p", \ -- "kcpu_queue") \ -+ "@pI", \ -+ "kcpu_queue,execute_error") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET, \ - "KCPU Queue executes a Set on an array of Cross Queue Sync Objects", \ -- "@p", \ -- "kcpu_queue") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_START, \ -- "KCPU Queue starts an array of Debug Copys", \ -- "@p", \ -- "kcpu_queue") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_END, \ -- "KCPU Queue ends an array of Debug Copys", \ -- "@p", \ -- "kcpu_queue") \ -+ "@pI", \ -+ "kcpu_queue,execute_error") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START, \ - "KCPU Queue starts a Map Import", \ - "@p", \ - "kcpu_queue") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END, \ - "KCPU Queue ends a Map Import", \ -- "@p", \ -- "kcpu_queue") \ -+ "@pI", \ -+ "kcpu_queue,execute_error") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START, \ - "KCPU Queue starts an Unmap Import", \ - "@p", \ - "kcpu_queue") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END, \ - "KCPU Queue ends an Unmap Import", \ -- "@p", \ -- "kcpu_queue") \ -+ "@pI", \ -+ "kcpu_queue,execute_error") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START, \ - "KCPU Queue starts an Unmap Import ignoring reference count", \ - "@p", \ - "kcpu_queue") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END, \ - "KCPU Queue ends an Unmap Import ignoring reference count", \ -- "@p", \ -- "kcpu_queue") \ -+ "@pI", \ -+ "kcpu_queue,execute_error") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START, \ - "KCPU Queue starts an array of JIT Allocs", \ - "@p", \ -@@ -482,8 +455,8 @@ enum tl_msg_id_aux { - "kcpu_queue") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, \ - "Array item of KCPU Queue ends an array of JIT Allocs", \ -- "@pLL", \ -- "kcpu_queue,jit_alloc_gpu_alloc_addr,jit_alloc_mmu_flags") \ -+ "@pILL", \ -+ "kcpu_queue,execute_error,jit_alloc_gpu_alloc_addr,jit_alloc_mmu_flags") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END, \ - "End array of KCPU Queue ends an array of JIT Allocs", \ - "@p", \ -@@ -498,20 +471,32 @@ enum tl_msg_id_aux { - "kcpu_queue") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END, \ - "Array item of KCPU Queue ends an array of JIT Frees", \ -- "@pL", \ -- "kcpu_queue,jit_free_pages_used") \ -+ "@pIL", \ -+ "kcpu_queue,execute_error,jit_free_pages_used") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END, \ - "End array of KCPU Queue ends an array of JIT Frees", \ - "@p", \ - "kcpu_queue") \ -- TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER, \ -+ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER, \ - "KCPU Queue executes an Error Barrier", \ - "@p", \ - "kcpu_queue") \ -+ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START, \ -+ "KCPU Queue starts a group suspend", \ -+ "@p", \ -+ "kcpu_queue") \ -+ TRACEPOINT_DESC(KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END, \ -+ "KCPU Queue ends a group suspend", \ -+ "@pI", \ -+ "kcpu_queue,execute_error") \ - TRACEPOINT_DESC(KBASE_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW, \ - "An overflow has happened with the CSFFW Timeline stream", \ - "@LL", \ - "csffw_timestamp,csffw_cycle") \ -+ TRACEPOINT_DESC(KBASE_TL_KBASE_CSFFW_RESET, \ -+ "A reset has happened with the CSFFW", \ -+ "@L", \ -+ "csffw_cycle") \ - - #define MIPE_HEADER_BLOB_VAR_NAME __obj_desc_header - #define MIPE_HEADER_STREAM_ID TL_STREAM_ID_KERNEL -@@ -561,6 +546,10 @@ const size_t obj_desc_header_size = sizeof(__obj_desc_header); - "per-bin JIT statistics", \ - "@IIIIII", \ - "ctx_nr,bid,max_allocs,allocs,va_pages,ph_pages") \ -+ TRACEPOINT_DESC(KBASE_AUX_TILER_HEAP_STATS, \ -+ "Tiler Heap statistics", \ -+ "@ILIIIIIII", \ -+ "ctx_nr,heap_id,va_pages,ph_pages,max_chunks,chunk_size,chunk_count,target_in_flight,nr_in_flight") \ - TRACEPOINT_DESC(KBASE_AUX_EVENT_JOB_SLOT, \ - "event on a given job slot", \ - "@pIII", \ -@@ -1493,11 +1482,33 @@ void __kbase_tlstream_tl_event_atom_softjob_end( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_event_arb_granted( -+void __kbase_tlstream_tl_arbiter_granted( -+ struct kbase_tlstream *stream, -+ const void *gpu) -+{ -+ const u32 msg_id = KBASE_TL_ARBITER_GRANTED; -+ const size_t msg_size = sizeof(msg_id) + sizeof(u64) -+ + sizeof(gpu) -+ ; -+ char *buffer; -+ unsigned long acq_flags; -+ size_t pos = 0; -+ -+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -+ -+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_serialize_timestamp(buffer, pos); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &gpu, sizeof(gpu)); -+ -+ kbase_tlstream_msgbuf_release(stream, acq_flags); -+} -+ -+void __kbase_tlstream_tl_arbiter_started( - struct kbase_tlstream *stream, - const void *gpu) - { -- const u32 msg_id = KBASE_TL_EVENT_ARB_GRANTED; -+ const u32 msg_id = KBASE_TL_ARBITER_STARTED; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(gpu) - ; -@@ -1515,11 +1526,11 @@ void __kbase_tlstream_tl_event_arb_granted( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_event_arb_started( -+void __kbase_tlstream_tl_arbiter_stop_requested( - struct kbase_tlstream *stream, - const void *gpu) - { -- const u32 msg_id = KBASE_TL_EVENT_ARB_STARTED; -+ const u32 msg_id = KBASE_TL_ARBITER_STOP_REQUESTED; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(gpu) - ; -@@ -1537,11 +1548,11 @@ void __kbase_tlstream_tl_event_arb_started( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_event_arb_stop_requested( -+void __kbase_tlstream_tl_arbiter_stopped( - struct kbase_tlstream *stream, - const void *gpu) - { -- const u32 msg_id = KBASE_TL_EVENT_ARB_STOP_REQUESTED; -+ const u32 msg_id = KBASE_TL_ARBITER_STOPPED; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(gpu) - ; -@@ -1559,11 +1570,11 @@ void __kbase_tlstream_tl_event_arb_stop_requested( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_event_arb_stopped( -+void __kbase_tlstream_tl_arbiter_requested( - struct kbase_tlstream *stream, - const void *gpu) - { -- const u32 msg_id = KBASE_TL_EVENT_ARB_STOPPED; -+ const u32 msg_id = KBASE_TL_ARBITER_REQUESTED; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(gpu) - ; -@@ -1837,6 +1848,60 @@ void __kbase_tlstream_aux_jit_stats( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - -+void __kbase_tlstream_aux_tiler_heap_stats( -+ struct kbase_tlstream *stream, -+ u32 ctx_nr, -+ u64 heap_id, -+ u32 va_pages, -+ u32 ph_pages, -+ u32 max_chunks, -+ u32 chunk_size, -+ u32 chunk_count, -+ u32 target_in_flight, -+ u32 nr_in_flight) -+{ -+ const u32 msg_id = KBASE_AUX_TILER_HEAP_STATS; -+ const size_t msg_size = sizeof(msg_id) + sizeof(u64) -+ + sizeof(ctx_nr) -+ + sizeof(heap_id) -+ + sizeof(va_pages) -+ + sizeof(ph_pages) -+ + sizeof(max_chunks) -+ + sizeof(chunk_size) -+ + sizeof(chunk_count) -+ + sizeof(target_in_flight) -+ + sizeof(nr_in_flight) -+ ; -+ char *buffer; -+ unsigned long acq_flags; -+ size_t pos = 0; -+ -+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -+ -+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_serialize_timestamp(buffer, pos); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &ctx_nr, sizeof(ctx_nr)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &heap_id, sizeof(heap_id)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &va_pages, sizeof(va_pages)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &ph_pages, sizeof(ph_pages)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &max_chunks, sizeof(max_chunks)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &chunk_size, sizeof(chunk_size)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &chunk_count, sizeof(chunk_count)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &target_in_flight, sizeof(target_in_flight)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &nr_in_flight, sizeof(nr_in_flight)); -+ -+ kbase_tlstream_msgbuf_release(stream, acq_flags); -+} -+ - void __kbase_tlstream_aux_event_job_slot( - struct kbase_tlstream *stream, - const void *ctx, -@@ -2161,39 +2226,19 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_cqs_wait( -- struct kbase_tlstream *stream, -- const void *kcpu_queue) --{ -- const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT; -- const size_t msg_size = sizeof(msg_id) + sizeof(u64) -- + sizeof(kcpu_queue) -- ; -- char *buffer; -- unsigned long acq_flags; -- size_t pos = 0; -- -- buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -- -- pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -- pos = kbasep_serialize_timestamp(buffer, pos); -- pos = kbasep_serialize_bytes(buffer, -- pos, &kcpu_queue, sizeof(kcpu_queue)); -- -- kbase_tlstream_msgbuf_release(stream, acq_flags); --} -- --void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_wait( -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( - struct kbase_tlstream *stream, - const void *kcpu_queue, - u64 cqs_obj_gpu_addr, -- u32 cqs_obj_compare_value) -+ u32 cqs_obj_compare_value, -+ u32 cqs_obj_inherit_error) - { -- const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_WAIT; -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) - + sizeof(cqs_obj_gpu_addr) - + sizeof(cqs_obj_compare_value) -+ + sizeof(cqs_obj_inherit_error) - ; - char *buffer; - unsigned long acq_flags; -@@ -2209,60 +2254,18 @@ void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_wait( - pos, &cqs_obj_gpu_addr, sizeof(cqs_obj_gpu_addr)); - pos = kbasep_serialize_bytes(buffer, - pos, &cqs_obj_compare_value, sizeof(cqs_obj_compare_value)); -- -- kbase_tlstream_msgbuf_release(stream, acq_flags); --} -- --void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_cqs_wait( -- struct kbase_tlstream *stream, -- const void *kcpu_queue) --{ -- const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_WAIT; -- const size_t msg_size = sizeof(msg_id) + sizeof(u64) -- + sizeof(kcpu_queue) -- ; -- char *buffer; -- unsigned long acq_flags; -- size_t pos = 0; -- -- buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -- -- pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -- pos = kbasep_serialize_timestamp(buffer, pos); -- pos = kbasep_serialize_bytes(buffer, -- pos, &kcpu_queue, sizeof(kcpu_queue)); -- -- kbase_tlstream_msgbuf_release(stream, acq_flags); --} -- --void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_cqs_set( -- struct kbase_tlstream *stream, -- const void *kcpu_queue) --{ -- const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_SET; -- const size_t msg_size = sizeof(msg_id) + sizeof(u64) -- + sizeof(kcpu_queue) -- ; -- char *buffer; -- unsigned long acq_flags; -- size_t pos = 0; -- -- buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -- -- pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -- pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, -- pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos, &cqs_obj_inherit_error, sizeof(cqs_obj_inherit_error)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_set( -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( - struct kbase_tlstream *stream, - const void *kcpu_queue, - u64 cqs_obj_gpu_addr) - { -- const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_SET; -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) - + sizeof(cqs_obj_gpu_addr) -@@ -2283,13 +2286,15 @@ void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_set( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_cqs_set( -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( - struct kbase_tlstream *stream, -- const void *kcpu_queue) -+ const void *kcpu_queue, -+ u64 map_import_buf_gpu_addr) - { -- const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_SET; -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(map_import_buf_gpu_addr) - ; - char *buffer; - unsigned long acq_flags; -@@ -2301,41 +2306,21 @@ void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_cqs_set( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -- -- kbase_tlstream_msgbuf_release(stream, acq_flags); --} -- --void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_debugcopy( -- struct kbase_tlstream *stream, -- const void *kcpu_queue) --{ -- const u32 msg_id = KBASE_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_DEBUGCOPY; -- const size_t msg_size = sizeof(msg_id) + sizeof(u64) -- + sizeof(kcpu_queue) -- ; -- char *buffer; -- unsigned long acq_flags; -- size_t pos = 0; -- -- buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -- -- pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -- pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, -- pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_debugcopy( -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( - struct kbase_tlstream *stream, - const void *kcpu_queue, -- u64 debugcopy_dst_size) -+ u64 map_import_buf_gpu_addr) - { -- const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_DEBUGCOPY; -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -- + sizeof(debugcopy_dst_size) -+ + sizeof(map_import_buf_gpu_addr) - ; - char *buffer; - unsigned long acq_flags; -@@ -2348,39 +2333,17 @@ void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_debugcopy( - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); - pos = kbasep_serialize_bytes(buffer, -- pos, &debugcopy_dst_size, sizeof(debugcopy_dst_size)); -- -- kbase_tlstream_msgbuf_release(stream, acq_flags); --} -- --void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_debugcopy( -- struct kbase_tlstream *stream, -- const void *kcpu_queue) --{ -- const u32 msg_id = KBASE_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY; -- const size_t msg_size = sizeof(msg_id) + sizeof(u64) -- + sizeof(kcpu_queue) -- ; -- char *buffer; -- unsigned long acq_flags; -- size_t pos = 0; -- -- buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -- -- pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -- pos = kbasep_serialize_timestamp(buffer, pos); -- pos = kbasep_serialize_bytes(buffer, -- pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( - struct kbase_tlstream *stream, - const void *kcpu_queue, - u64 map_import_buf_gpu_addr) - { -- const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT; -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) - + sizeof(map_import_buf_gpu_addr) -@@ -2401,15 +2364,13 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_error_barrier( - struct kbase_tlstream *stream, -- const void *kcpu_queue, -- u64 map_import_buf_gpu_addr) -+ const void *kcpu_queue) - { -- const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT; -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -- + sizeof(map_import_buf_gpu_addr) - ; - char *buffer; - unsigned long acq_flags; -@@ -2421,21 +2382,21 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -- pos = kbasep_serialize_bytes(buffer, -- pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_group_suspend( - struct kbase_tlstream *stream, - const void *kcpu_queue, -- u64 map_import_buf_gpu_addr) -+ const void *group_suspend_buf, -+ u32 gpu_cmdq_grp_handle) - { -- const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE; -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -- + sizeof(map_import_buf_gpu_addr) -+ + sizeof(group_suspend_buf) -+ + sizeof(gpu_cmdq_grp_handle) - ; - char *buffer; - unsigned long acq_flags; -@@ -2448,7 +2409,9 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); - pos = kbasep_serialize_bytes(buffer, -- pos, &map_import_buf_gpu_addr, sizeof(map_import_buf_gpu_addr)); -+ pos, &group_suspend_buf, sizeof(group_suspend_buf)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &gpu_cmdq_grp_handle, sizeof(gpu_cmdq_grp_handle)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } -@@ -2649,11 +2612,13 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( - - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue) -+ const void *kcpu_queue, -+ u32 execute_error) - { - const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - ; - char *buffer; - unsigned long acq_flags; -@@ -2665,6 +2630,8 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &execute_error, sizeof(execute_error)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } -@@ -2693,11 +2660,13 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( - - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue) -+ const void *kcpu_queue, -+ u32 execute_error) - { - const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - ; - char *buffer; - unsigned long acq_flags; -@@ -2709,6 +2678,8 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &execute_error, sizeof(execute_error)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } -@@ -2737,11 +2708,13 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( - - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue) -+ const void *kcpu_queue, -+ u32 execute_error) - { - const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - ; - char *buffer; - unsigned long acq_flags; -@@ -2753,17 +2726,21 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &execute_error, sizeof(execute_error)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( - struct kbase_tlstream *stream, -- const void *kcpu_queue) -+ const void *kcpu_queue, -+ u32 execute_error) - { - const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - ; - char *buffer; - unsigned long acq_flags; -@@ -2775,50 +2752,8 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -- -- kbase_tlstream_msgbuf_release(stream, acq_flags); --} -- --void __kbase_tlstream_tl_kbase_kcpuqueue_execute_debugcopy_start( -- struct kbase_tlstream *stream, -- const void *kcpu_queue) --{ -- const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_START; -- const size_t msg_size = sizeof(msg_id) + sizeof(u64) -- + sizeof(kcpu_queue) -- ; -- char *buffer; -- unsigned long acq_flags; -- size_t pos = 0; -- -- buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -- -- pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -- pos = kbasep_serialize_timestamp(buffer, pos); -- pos = kbasep_serialize_bytes(buffer, -- pos, &kcpu_queue, sizeof(kcpu_queue)); -- -- kbase_tlstream_msgbuf_release(stream, acq_flags); --} -- --void __kbase_tlstream_tl_kbase_kcpuqueue_execute_debugcopy_end( -- struct kbase_tlstream *stream, -- const void *kcpu_queue) --{ -- const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_END; -- const size_t msg_size = sizeof(msg_id) + sizeof(u64) -- + sizeof(kcpu_queue) -- ; -- char *buffer; -- unsigned long acq_flags; -- size_t pos = 0; -- -- buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -- -- pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -- pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, -- pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos, &execute_error, sizeof(execute_error)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } -@@ -2847,11 +2782,13 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( - - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue) -+ const void *kcpu_queue, -+ u32 execute_error) - { - const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - ; - char *buffer; - unsigned long acq_flags; -@@ -2863,6 +2800,8 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &execute_error, sizeof(execute_error)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } -@@ -2891,11 +2830,13 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( - - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue) -+ const void *kcpu_queue, -+ u32 execute_error) - { - const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - ; - char *buffer; - unsigned long acq_flags; -@@ -2907,6 +2848,8 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &execute_error, sizeof(execute_error)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } -@@ -2935,11 +2878,13 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( - - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue) -+ const void *kcpu_queue, -+ u32 execute_error) - { - const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - ; - char *buffer; - unsigned long acq_flags; -@@ -2951,6 +2896,8 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &execute_error, sizeof(execute_error)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } -@@ -3002,12 +2949,14 @@ void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( - void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( - struct kbase_tlstream *stream, - const void *kcpu_queue, -+ u32 execute_error, - u64 jit_alloc_gpu_alloc_addr, - u64 jit_alloc_mmu_flags) - { - const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - + sizeof(jit_alloc_gpu_alloc_addr) - + sizeof(jit_alloc_mmu_flags) - ; -@@ -3021,6 +2970,8 @@ void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &execute_error, sizeof(execute_error)); - pos = kbasep_serialize_bytes(buffer, - pos, &jit_alloc_gpu_alloc_addr, sizeof(jit_alloc_gpu_alloc_addr)); - pos = kbasep_serialize_bytes(buffer, -@@ -3098,11 +3049,13 @@ void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( - void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( - struct kbase_tlstream *stream, - const void *kcpu_queue, -+ u32 execute_error, - u64 jit_free_pages_used) - { - const u32 msg_id = KBASE_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - + sizeof(jit_free_pages_used) - ; - char *buffer; -@@ -3115,6 +3068,8 @@ void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &execute_error, sizeof(execute_error)); - pos = kbasep_serialize_bytes(buffer, - pos, &jit_free_pages_used, sizeof(jit_free_pages_used)); - -@@ -3143,13 +3098,59 @@ void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - --void __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( -+void __kbase_tlstream_tl_kbase_kcpuqueue_execute_error_barrier( -+ struct kbase_tlstream *stream, -+ const void *kcpu_queue) -+{ -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER; -+ const size_t msg_size = sizeof(msg_id) + sizeof(u64) -+ + sizeof(kcpu_queue) -+ ; -+ char *buffer; -+ unsigned long acq_flags; -+ size_t pos = 0; -+ -+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -+ -+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_serialize_timestamp(buffer, pos); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &kcpu_queue, sizeof(kcpu_queue)); -+ -+ kbase_tlstream_msgbuf_release(stream, acq_flags); -+} -+ -+void __kbase_tlstream_tl_kbase_kcpuqueue_execute_group_suspend_start( - struct kbase_tlstream *stream, - const void *kcpu_queue) - { -- const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER; -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START; -+ const size_t msg_size = sizeof(msg_id) + sizeof(u64) -+ + sizeof(kcpu_queue) -+ ; -+ char *buffer; -+ unsigned long acq_flags; -+ size_t pos = 0; -+ -+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -+ -+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_serialize_timestamp(buffer, pos); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &kcpu_queue, sizeof(kcpu_queue)); -+ -+ kbase_tlstream_msgbuf_release(stream, acq_flags); -+} -+ -+void __kbase_tlstream_tl_kbase_kcpuqueue_execute_group_suspend_end( -+ struct kbase_tlstream *stream, -+ const void *kcpu_queue, -+ u32 execute_error) -+{ -+ const u32 msg_id = KBASE_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END; - const size_t msg_size = sizeof(msg_id) + sizeof(u64) - + sizeof(kcpu_queue) -+ + sizeof(execute_error) - ; - char *buffer; - unsigned long acq_flags; -@@ -3161,6 +3162,8 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( - pos = kbasep_serialize_timestamp(buffer, pos); - pos = kbasep_serialize_bytes(buffer, - pos, &kcpu_queue, sizeof(kcpu_queue)); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &execute_error, sizeof(execute_error)); - - kbase_tlstream_msgbuf_release(stream, acq_flags); - } -@@ -3191,4 +3194,26 @@ void __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( - kbase_tlstream_msgbuf_release(stream, acq_flags); - } - -+void __kbase_tlstream_tl_kbase_csffw_reset( -+ struct kbase_tlstream *stream, -+ u64 csffw_cycle) -+{ -+ const u32 msg_id = KBASE_TL_KBASE_CSFFW_RESET; -+ const size_t msg_size = sizeof(msg_id) + sizeof(u64) -+ + sizeof(csffw_cycle) -+ ; -+ char *buffer; -+ unsigned long acq_flags; -+ size_t pos = 0; -+ -+ buffer = kbase_tlstream_msgbuf_acquire(stream, msg_size, &acq_flags); -+ -+ pos = kbasep_serialize_bytes(buffer, pos, &msg_id, sizeof(msg_id)); -+ pos = kbasep_serialize_timestamp(buffer, pos); -+ pos = kbasep_serialize_bytes(buffer, -+ pos, &csffw_cycle, sizeof(csffw_cycle)); -+ -+ kbase_tlstream_msgbuf_release(stream, acq_flags); -+} -+ - /* clang-format on */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tracepoints.h b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tracepoints.h -index 7ea8ba8..887a1aa 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tracepoints.h -+++ b/dvalin/kernel/drivers/gpu/arm/midgard/tl/mali_kbase_tracepoints.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - /* -@@ -226,16 +225,19 @@ void __kbase_tlstream_tl_event_atom_softjob_start( - void __kbase_tlstream_tl_event_atom_softjob_end( - struct kbase_tlstream *stream, - const void *atom); --void __kbase_tlstream_tl_event_arb_granted( -+void __kbase_tlstream_tl_arbiter_granted( -+ struct kbase_tlstream *stream, -+ const void *gpu); -+void __kbase_tlstream_tl_arbiter_started( - struct kbase_tlstream *stream, - const void *gpu); --void __kbase_tlstream_tl_event_arb_started( -+void __kbase_tlstream_tl_arbiter_stop_requested( - struct kbase_tlstream *stream, - const void *gpu); --void __kbase_tlstream_tl_event_arb_stop_requested( -+void __kbase_tlstream_tl_arbiter_stopped( - struct kbase_tlstream *stream, - const void *gpu); --void __kbase_tlstream_tl_event_arb_stopped( -+void __kbase_tlstream_tl_arbiter_requested( - struct kbase_tlstream *stream, - const void *gpu); - void __kbase_tlstream_jd_gpu_soft_reset( -@@ -277,6 +279,17 @@ void __kbase_tlstream_aux_jit_stats( - u32 allocs, - u32 va_pages, - u32 ph_pages); -+void __kbase_tlstream_aux_tiler_heap_stats( -+ struct kbase_tlstream *stream, -+ u32 ctx_nr, -+ u64 heap_id, -+ u32 va_pages, -+ u32 ph_pages, -+ u32 max_chunks, -+ u32 chunk_size, -+ u32 chunk_count, -+ u32 target_in_flight, -+ u32 nr_in_flight); - void __kbase_tlstream_aux_event_job_slot( - struct kbase_tlstream *stream, - const void *ctx, -@@ -328,37 +341,16 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( - struct kbase_tlstream *stream, - const void *kcpu_queue, - const void *fence); --void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_cqs_wait( -- struct kbase_tlstream *stream, -- const void *kcpu_queue); --void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_wait( -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( - struct kbase_tlstream *stream, - const void *kcpu_queue, - u64 cqs_obj_gpu_addr, -- u32 cqs_obj_compare_value); --void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_cqs_wait( -- struct kbase_tlstream *stream, -- const void *kcpu_queue); --void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_cqs_set( -- struct kbase_tlstream *stream, -- const void *kcpu_queue); --void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_cqs_set( -+ u32 cqs_obj_compare_value, -+ u32 cqs_obj_inherit_error); -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( - struct kbase_tlstream *stream, - const void *kcpu_queue, - u64 cqs_obj_gpu_addr); --void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_cqs_set( -- struct kbase_tlstream *stream, -- const void *kcpu_queue); --void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_debugcopy( -- struct kbase_tlstream *stream, -- const void *kcpu_queue); --void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_debugcopy( -- struct kbase_tlstream *stream, -- const void *kcpu_queue, -- u64 debugcopy_dst_size); --void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_debugcopy( -- struct kbase_tlstream *stream, -- const void *kcpu_queue); - void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( - struct kbase_tlstream *stream, - const void *kcpu_queue, -@@ -371,6 +363,14 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( - struct kbase_tlstream *stream, - const void *kcpu_queue, - u64 map_import_buf_gpu_addr); -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_error_barrier( -+ struct kbase_tlstream *stream, -+ const void *kcpu_queue); -+void __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_group_suspend( -+ struct kbase_tlstream *stream, -+ const void *kcpu_queue, -+ const void *group_suspend_buf, -+ u32 gpu_cmdq_grp_handle); - void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( - struct kbase_tlstream *stream, - const void *kcpu_queue); -@@ -404,46 +404,47 @@ void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( - const void *kcpu_queue); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue); -+ const void *kcpu_queue, -+ u32 execute_error); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( - struct kbase_tlstream *stream, - const void *kcpu_queue); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue); -+ const void *kcpu_queue, -+ u32 execute_error); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( - struct kbase_tlstream *stream, - const void *kcpu_queue); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue); -+ const void *kcpu_queue, -+ u32 execute_error); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( - struct kbase_tlstream *stream, -- const void *kcpu_queue); --void __kbase_tlstream_tl_kbase_kcpuqueue_execute_debugcopy_start( -- struct kbase_tlstream *stream, -- const void *kcpu_queue); --void __kbase_tlstream_tl_kbase_kcpuqueue_execute_debugcopy_end( -- struct kbase_tlstream *stream, -- const void *kcpu_queue); -+ const void *kcpu_queue, -+ u32 execute_error); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( - struct kbase_tlstream *stream, - const void *kcpu_queue); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue); -+ const void *kcpu_queue, -+ u32 execute_error); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( - struct kbase_tlstream *stream, - const void *kcpu_queue); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue); -+ const void *kcpu_queue, -+ u32 execute_error); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( - struct kbase_tlstream *stream, - const void *kcpu_queue); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( - struct kbase_tlstream *stream, -- const void *kcpu_queue); -+ const void *kcpu_queue, -+ u32 execute_error); - void __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( - struct kbase_tlstream *stream, - const void *kcpu_queue); -@@ -453,6 +454,7 @@ void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( - void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( - struct kbase_tlstream *stream, - const void *kcpu_queue, -+ u32 execute_error, - u64 jit_alloc_gpu_alloc_addr, - u64 jit_alloc_mmu_flags); - void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( -@@ -467,17 +469,28 @@ void __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( - void __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( - struct kbase_tlstream *stream, - const void *kcpu_queue, -+ u32 execute_error, - u64 jit_free_pages_used); - void __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( - struct kbase_tlstream *stream, - const void *kcpu_queue); --void __kbase_tlstream_tl_kbase_kcpuqueue_execute_errorbarrier( -+void __kbase_tlstream_tl_kbase_kcpuqueue_execute_error_barrier( -+ struct kbase_tlstream *stream, -+ const void *kcpu_queue); -+void __kbase_tlstream_tl_kbase_kcpuqueue_execute_group_suspend_start( - struct kbase_tlstream *stream, - const void *kcpu_queue); -+void __kbase_tlstream_tl_kbase_kcpuqueue_execute_group_suspend_end( -+ struct kbase_tlstream *stream, -+ const void *kcpu_queue, -+ u32 execute_error); - void __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( - struct kbase_tlstream *stream, - u64 csffw_timestamp, - u64 csffw_cycle); -+void __kbase_tlstream_tl_kbase_csffw_reset( -+ struct kbase_tlstream *stream, -+ u64 csffw_cycle); - - struct kbase_tlstream; - -@@ -1215,77 +1228,96 @@ struct kbase_tlstream; - } while (0) - - /** -- * KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED - -+ * KBASE_TLSTREAM_TL_ARBITER_GRANTED - - * Arbiter has granted gpu access - * - * @kbdev: Kbase device - * @gpu: Name of the GPU object - */ --#define KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED( \ -+#define KBASE_TLSTREAM_TL_ARBITER_GRANTED( \ - kbdev, \ - gpu \ - ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ - if (enabled & TLSTREAM_ENABLED) \ -- __kbase_tlstream_tl_event_arb_granted( \ -+ __kbase_tlstream_tl_arbiter_granted( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - gpu); \ - } while (0) - - /** -- * KBASE_TLSTREAM_TL_EVENT_ARB_STARTED - -+ * KBASE_TLSTREAM_TL_ARBITER_STARTED - - * Driver is running again and able to process jobs - * - * @kbdev: Kbase device - * @gpu: Name of the GPU object - */ --#define KBASE_TLSTREAM_TL_EVENT_ARB_STARTED( \ -+#define KBASE_TLSTREAM_TL_ARBITER_STARTED( \ - kbdev, \ - gpu \ - ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ - if (enabled & TLSTREAM_ENABLED) \ -- __kbase_tlstream_tl_event_arb_started( \ -+ __kbase_tlstream_tl_arbiter_started( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - gpu); \ - } while (0) - - /** -- * KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED - -+ * KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED - - * Arbiter has requested driver to stop using gpu - * - * @kbdev: Kbase device - * @gpu: Name of the GPU object - */ --#define KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED( \ -+#define KBASE_TLSTREAM_TL_ARBITER_STOP_REQUESTED( \ - kbdev, \ - gpu \ - ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ - if (enabled & TLSTREAM_ENABLED) \ -- __kbase_tlstream_tl_event_arb_stop_requested( \ -+ __kbase_tlstream_tl_arbiter_stop_requested( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - gpu); \ - } while (0) - - /** -- * KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED - -+ * KBASE_TLSTREAM_TL_ARBITER_STOPPED - - * Driver has stopped using gpu - * - * @kbdev: Kbase device - * @gpu: Name of the GPU object - */ --#define KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED( \ -+#define KBASE_TLSTREAM_TL_ARBITER_STOPPED( \ - kbdev, \ - gpu \ - ) \ - do { \ - int enabled = atomic_read(&kbdev->timeline_flags); \ - if (enabled & TLSTREAM_ENABLED) \ -- __kbase_tlstream_tl_event_arb_stopped( \ -+ __kbase_tlstream_tl_arbiter_stopped( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ gpu); \ -+ } while (0) -+ -+/** -+ * KBASE_TLSTREAM_TL_ARBITER_REQUESTED - -+ * Driver has requested the arbiter for gpu access -+ * -+ * @kbdev: Kbase device -+ * @gpu: Name of the GPU object -+ */ -+#define KBASE_TLSTREAM_TL_ARBITER_REQUESTED( \ -+ kbdev, \ -+ gpu \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & TLSTREAM_ENABLED) \ -+ __kbase_tlstream_tl_arbiter_requested( \ - __TL_DISPATCH_STREAM(kbdev, obj), \ - gpu); \ - } while (0) -@@ -1499,6 +1531,42 @@ struct kbase_tlstream; - ctx_nr, bid, max_allocs, allocs, va_pages, ph_pages); \ - } while (0) - -+/** -+ * KBASE_TLSTREAM_AUX_TILER_HEAP_STATS - -+ * Tiler Heap statistics -+ * -+ * @kbdev: Kbase device -+ * @ctx_nr: Kernel context number -+ * @heap_id: Unique id used to represent a heap under a context -+ * @va_pages: Number of virtual pages allocated in this bin -+ * @ph_pages: Number of physical pages allocated in this bin -+ * @max_chunks: The maximum number of chunks that the heap should be allowed to use -+ * @chunk_size: Size of each chunk in tiler heap, in bytes -+ * @chunk_count: The number of chunks currently allocated in the tiler heap -+ * @target_in_flight: Number of render-passes that the driver should attempt -+ * to keep in flight for which allocation of new chunks is allowed -+ * @nr_in_flight: Number of render-passes that are in flight -+ */ -+#define KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( \ -+ kbdev, \ -+ ctx_nr, \ -+ heap_id, \ -+ va_pages, \ -+ ph_pages, \ -+ max_chunks, \ -+ chunk_size, \ -+ chunk_count, \ -+ target_in_flight, \ -+ nr_in_flight \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & TLSTREAM_ENABLED) \ -+ __kbase_tlstream_aux_tiler_heap_stats( \ -+ __TL_DISPATCH_STREAM(kbdev, aux), \ -+ ctx_nr, heap_id, va_pages, ph_pages, max_chunks, chunk_size, chunk_count, target_in_flight, nr_in_flight); \ -+ } while (0) -+ - /** - * KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT - - * event on a given job slot -@@ -1534,6 +1602,22 @@ struct kbase_tlstream; - * @kbase_device_max_num_csgs: The max number of CSGs the physical hardware supports - * @kbase_device_as_count: The number of address spaces the physical hardware has available - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ -+ kbdev, \ -+ kbase_device_id, \ -+ kbase_device_gpu_core_count, \ -+ kbase_device_max_num_csgs, \ -+ kbase_device_as_count \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_new_device( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kbase_device_id, kbase_device_gpu_core_count, kbase_device_max_num_csgs, kbase_device_as_count); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_NEW_DEVICE( \ - kbdev, \ - kbase_device_id, \ -@@ -1542,6 +1626,7 @@ struct kbase_tlstream; - kbase_device_as_count \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG - -@@ -1552,6 +1637,21 @@ struct kbase_tlstream; - * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace - * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ -+ kbdev, \ -+ kbase_device_id, \ -+ gpu_cmdq_grp_handle, \ -+ kbase_device_csg_slot_index \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_device_program_csg( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kbase_device_id, gpu_cmdq_grp_handle, kbase_device_csg_slot_index); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_DEVICE_PROGRAM_CSG( \ - kbdev, \ - kbase_device_id, \ -@@ -1559,6 +1659,7 @@ struct kbase_tlstream; - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG - -@@ -1568,12 +1669,27 @@ struct kbase_tlstream; - * @kbase_device_id: The id of the physical hardware - * @kbase_device_csg_slot_index: The index of the slot in the scheduler being programmed - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ -+ kbdev, \ -+ kbase_device_id, \ -+ kbase_device_csg_slot_index \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_device_deprogram_csg( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kbase_device_id, kbase_device_csg_slot_index); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_DEVICE_DEPROGRAM_CSG( \ - kbdev, \ - kbase_device_id, \ - kbase_device_csg_slot_index \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_NEW_CTX - -@@ -1583,12 +1699,27 @@ struct kbase_tlstream; - * @kernel_ctx_id: Unique ID for the KBase Context - * @kbase_device_id: The id of the physical hardware - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ -+ kbdev, \ -+ kernel_ctx_id, \ -+ kbase_device_id \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_new_ctx( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kernel_ctx_id, kbase_device_id); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_NEW_CTX( \ - kbdev, \ - kernel_ctx_id, \ - kbase_device_id \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_DEL_CTX - -@@ -1597,11 +1728,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kernel_ctx_id: Unique ID for the KBase Context - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ -+ kbdev, \ -+ kernel_ctx_id \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_del_ctx( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kernel_ctx_id); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_DEL_CTX( \ - kbdev, \ - kernel_ctx_id \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS - -@@ -1611,12 +1756,27 @@ struct kbase_tlstream; - * @kernel_ctx_id: Unique ID for the KBase Context - * @kbase_device_as_index: The index of the device address space being assigned - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( \ -+ kbdev, \ -+ kernel_ctx_id, \ -+ kbase_device_as_index \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_ctx_assign_as( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kernel_ctx_id, kbase_device_as_index); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS( \ - kbdev, \ - kernel_ctx_id, \ - kbase_device_as_index \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS - -@@ -1625,11 +1785,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kernel_ctx_id: Unique ID for the KBase Context - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( \ -+ kbdev, \ -+ kernel_ctx_id \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_ctx_unassign_as( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kernel_ctx_id); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS( \ - kbdev, \ - kernel_ctx_id \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE - -@@ -1641,6 +1815,21 @@ struct kbase_tlstream; - * @kcpuq_num_pending_cmds: Number of commands already enqueued - * in the KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ -+ kbdev, \ -+ kcpu_queue, \ -+ kernel_ctx_id, \ -+ kcpuq_num_pending_cmds \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_new_kcpuqueue( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, kernel_ctx_id, kcpuq_num_pending_cmds); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_NEW_KCPUQUEUE( \ - kbdev, \ - kcpu_queue, \ -@@ -1648,6 +1837,7 @@ struct kbase_tlstream; - kcpuq_num_pending_cmds \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE - -@@ -1656,11 +1846,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_del_kcpuqueue( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_DEL_KCPUQUEUE( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL - -@@ -1670,12 +1874,27 @@ struct kbase_tlstream; - * @kcpu_queue: KCPU queue - * @fence: Fence object handle - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ -+ kbdev, \ -+ kcpu_queue, \ -+ fence \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_signal( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, fence); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_SIGNAL( \ - kbdev, \ - kcpu_queue, \ - fence \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT - -@@ -1685,183 +1904,244 @@ struct kbase_tlstream; - * @kcpu_queue: KCPU queue - * @fence: Fence object handle - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ - kbdev, \ - kcpu_queue, \ - fence \ - ) \ -- do { } while (0) -- --/** -- * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT - -- * Begin array of KCPU Queue enqueues Wait on Cross Queue Sync Object -- * -- * @kbdev: Kbase device -- * @kcpu_queue: KCPU queue -- */ --#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_fence_wait( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, fence); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_FENCE_WAIT( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ fence \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** -- * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_WAIT - -- * Array item of KCPU Queue enqueues Wait on Cross Queue Sync Object -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT - -+ * KCPU Queue enqueues Wait on Cross Queue Sync Object - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -- * @cqs_obj_gpu_addr: CQS Object GPU ptr -+ * @cqs_obj_gpu_addr: CQS Object GPU pointer - * @cqs_obj_compare_value: Semaphore value that should be exceeded - * for the WAIT to pass -+ * @cqs_obj_inherit_error: Indicates the error state should be inherited into the queue or not - */ --#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ -+ kbdev, \ -+ kcpu_queue, \ -+ cqs_obj_gpu_addr, \ -+ cqs_obj_compare_value, \ -+ cqs_obj_inherit_error \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_wait( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, cqs_obj_gpu_addr, cqs_obj_compare_value, cqs_obj_inherit_error); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ - kbdev, \ - kcpu_queue, \ - cqs_obj_gpu_addr, \ -- cqs_obj_compare_value \ -+ cqs_obj_compare_value, \ -+ cqs_obj_inherit_error \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** -- * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_WAIT - -- * End array of KCPU Queue enqueues Wait on Cross Queue Sync Object -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET - -+ * KCPU Queue enqueues Set on Cross Queue Sync Object - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @cqs_obj_gpu_addr: CQS Object GPU pointer - */ --#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_WAIT( \ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ cqs_obj_gpu_addr \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_cqs_set( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, cqs_obj_gpu_addr); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_CQS_SET( \ -+ kbdev, \ -+ kcpu_queue, \ -+ cqs_obj_gpu_addr \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** -- * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_SET - -- * Begin array of KCPU Queue enqueues Set on Cross Queue Sync Object -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT - -+ * KCPU Queue enqueues Map Import - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @map_import_buf_gpu_addr: Map import buffer GPU pointer - */ --#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_CQS_SET( \ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ map_import_buf_gpu_addr \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_map_import( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, map_import_buf_gpu_addr); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ -+ kbdev, \ -+ kcpu_queue, \ -+ map_import_buf_gpu_addr \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** -- * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_SET - -- * Array item of KCPU Queue enqueues Set on Cross Queue Sync Object -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT - -+ * KCPU Queue enqueues Unmap Import - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -- * @cqs_obj_gpu_addr: CQS Object GPU ptr -+ * @map_import_buf_gpu_addr: Map import buffer GPU pointer - */ --#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_CQS_SET( \ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ - kbdev, \ - kcpu_queue, \ -- cqs_obj_gpu_addr \ -+ map_import_buf_gpu_addr \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, map_import_buf_gpu_addr); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ -+ kbdev, \ -+ kcpu_queue, \ -+ map_import_buf_gpu_addr \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** -- * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_SET - -- * End array of KCPU Queue enqueues Set on Cross Queue Sync Object -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE - -+ * KCPU Queue enqueues Unmap Import ignoring reference count - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @map_import_buf_gpu_addr: Map import buffer GPU pointer - */ --#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_CQS_SET( \ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ map_import_buf_gpu_addr \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_unmap_import_force( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, map_import_buf_gpu_addr); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ -+ kbdev, \ -+ kcpu_queue, \ -+ map_import_buf_gpu_addr \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** -- * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_DEBUGCOPY - -- * Begin array of KCPU Queue enqueues Debug Copy -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER - -+ * KCPU Queue enqueues Error Barrier - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ --#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_DEBUGCOPY( \ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_error_barrier( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_ERROR_BARRIER( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** -- * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_DEBUGCOPY - -- * Array item of KCPU Queue enqueues Debug Copy -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND - -+ * KCPU Queue enqueues Group Suspend - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -- * @debugcopy_dst_size: Debug Copy destination size -- */ --#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_DEBUGCOPY( \ -- kbdev, \ -- kcpu_queue, \ -- debugcopy_dst_size \ -- ) \ -- do { } while (0) -- --/** -- * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY - -- * End array of KCPU Queue enqueues Debug Copy -- * -- * @kbdev: Kbase device -- * @kcpu_queue: KCPU queue -- */ --#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_DEBUGCOPY( \ -- kbdev, \ -- kcpu_queue \ -- ) \ -- do { } while (0) -- --/** -- * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT - -- * KCPU Queue enqueues Map Import -- * -- * @kbdev: Kbase device -- * @kcpu_queue: KCPU queue -- * @map_import_buf_gpu_addr: Map import buffer GPU ptr -- */ --#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_MAP_IMPORT( \ -- kbdev, \ -- kcpu_queue, \ -- map_import_buf_gpu_addr \ -- ) \ -- do { } while (0) -- --/** -- * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT - -- * KCPU Queue enqueues Unmap Import -- * -- * @kbdev: Kbase device -- * @kcpu_queue: KCPU queue -- * @map_import_buf_gpu_addr: Map import buffer GPU ptr -+ * @group_suspend_buf: Pointer to the suspend buffer structure -+ * @gpu_cmdq_grp_handle: GPU Command Queue Group handle which will match userspace - */ --#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT( \ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND( \ - kbdev, \ - kcpu_queue, \ -- map_import_buf_gpu_addr \ -+ group_suspend_buf, \ -+ gpu_cmdq_grp_handle \ - ) \ -- do { } while (0) -- --/** -- * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE - -- * KCPU Queue enqueues Unmap Import ignoring reference count -- * -- * @kbdev: Kbase device -- * @kcpu_queue: KCPU queue -- * @map_import_buf_gpu_addr: Map import buffer GPU ptr -- */ --#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_UNMAP_IMPORT_FORCE( \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_enqueue_group_suspend( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, group_suspend_buf, gpu_cmdq_grp_handle); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_ENQUEUE_GROUP_SUSPEND( \ - kbdev, \ - kcpu_queue, \ -- map_import_buf_gpu_addr \ -+ group_suspend_buf, \ -+ gpu_cmdq_grp_handle \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC - -@@ -1870,11 +2150,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_alloc( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC - -@@ -1902,6 +2196,28 @@ struct kbase_tlstream; - * reused. The kernel should attempt to use a previous allocation with the same - * usage_id - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ -+ kbdev, \ -+ kcpu_queue, \ -+ jit_alloc_gpu_alloc_addr_dest, \ -+ jit_alloc_va_pages, \ -+ jit_alloc_commit_pages, \ -+ jit_alloc_extent, \ -+ jit_alloc_jit_id, \ -+ jit_alloc_bin_id, \ -+ jit_alloc_max_allocations, \ -+ jit_alloc_flags, \ -+ jit_alloc_usage_id \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_alloc( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, jit_alloc_gpu_alloc_addr_dest, jit_alloc_va_pages, jit_alloc_commit_pages, jit_alloc_extent, jit_alloc_jit_id, jit_alloc_bin_id, jit_alloc_max_allocations, jit_alloc_flags, jit_alloc_usage_id); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ - kbdev, \ - kcpu_queue, \ -@@ -1916,6 +2232,7 @@ struct kbase_tlstream; - jit_alloc_usage_id \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC - -@@ -1924,11 +2241,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_alloc( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_ALLOC( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE - -@@ -1937,11 +2268,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_enqueue_jit_free( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_ENQUEUE_JIT_FREE( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE - -@@ -1952,12 +2297,27 @@ struct kbase_tlstream; - * @jit_alloc_jit_id: Unique ID provided by the caller, this is used - * to pair allocation and free requests. Zero is not a valid value - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ -+ kbdev, \ -+ kcpu_queue, \ -+ jit_alloc_jit_id \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_enqueue_jit_free( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, jit_alloc_jit_id); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_ENQUEUE_JIT_FREE( \ - kbdev, \ - kcpu_queue, \ - jit_alloc_jit_id \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE - -@@ -1966,11 +2326,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_enqueue_jit_free( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_ENQUEUE_JIT_FREE( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START - -@@ -1979,11 +2353,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_start( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END - -@@ -1991,12 +2379,29 @@ struct kbase_tlstream; - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ execute_error \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_signal_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_SIGNAL_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START - -@@ -2005,11 +2410,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_start( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END - -@@ -2017,12 +2436,29 @@ struct kbase_tlstream; - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ execute_error \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_fence_wait_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_FENCE_WAIT_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START - -@@ -2031,11 +2467,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_start( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END - -@@ -2043,12 +2493,29 @@ struct kbase_tlstream; - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ execute_error \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_wait_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_WAIT_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET - -@@ -2056,51 +2523,56 @@ struct kbase_tlstream; - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ execute_error \ - ) \ -- do { } while (0) -- --/** -- * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_START - -- * KCPU Queue starts an array of Debug Copys -- * -- * @kbdev: Kbase device -- * @kcpu_queue: KCPU queue -- */ --#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_START( \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_cqs_set( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_CQS_SET( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ execute_error \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** -- * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_END - -- * KCPU Queue ends an array of Debug Copys -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START - -+ * KCPU Queue starts a Map Import - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ --#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_DEBUGCOPY_END( \ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ - kbdev, \ - kcpu_queue \ - ) \ -- do { } while (0) -- --/** -- * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START - -- * KCPU Queue starts a Map Import -- * -- * @kbdev: Kbase device -- * @kcpu_queue: KCPU queue -- */ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_start( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END - -@@ -2108,12 +2580,29 @@ struct kbase_tlstream; - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ execute_error \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_map_import_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_MAP_IMPORT_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START - -@@ -2122,11 +2611,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_start( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END - -@@ -2134,12 +2637,29 @@ struct kbase_tlstream; - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ execute_error \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START - -@@ -2148,11 +2668,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_start( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END - -@@ -2160,12 +2694,29 @@ struct kbase_tlstream; - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ - kbdev, \ -- kcpu_queue \ -+ kcpu_queue, \ -+ execute_error \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_unmap_import_force_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_UNMAP_IMPORT_FORCE_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START - -@@ -2174,11 +2725,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_alloc_start( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_ALLOC_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - -@@ -2187,11 +2752,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_alloc_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - -@@ -2199,16 +2778,35 @@ struct kbase_tlstream; - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero - * @jit_alloc_gpu_alloc_addr: The JIT allocated GPU virtual address - * @jit_alloc_mmu_flags: The MMU flags for the JIT allocation - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ - kbdev, \ - kcpu_queue, \ -+ execute_error, \ -+ jit_alloc_gpu_alloc_addr, \ -+ jit_alloc_mmu_flags \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_alloc_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error, jit_alloc_gpu_alloc_addr, jit_alloc_mmu_flags); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error, \ - jit_alloc_gpu_alloc_addr, \ - jit_alloc_mmu_flags \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END - -@@ -2217,11 +2815,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_alloc_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_ALLOC_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START - -@@ -2230,11 +2842,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_jit_free_start( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_JIT_FREE_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END - -@@ -2243,11 +2869,25 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_begin_kcpuqueue_execute_jit_free_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_BEGIN_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END - -@@ -2255,15 +2895,33 @@ struct kbase_tlstream; - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero - * @jit_free_pages_used: The actual number of pages used by the JIT - * allocation - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error, \ -+ jit_free_pages_used \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_item_kcpuqueue_execute_jit_free_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error, jit_free_pages_used); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_ITEM_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ - kbdev, \ - kcpu_queue, \ -+ execute_error, \ - jit_free_pages_used \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END - -@@ -2272,24 +2930,109 @@ struct kbase_tlstream; - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_array_end_kcpuqueue_execute_jit_free_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else - #define KBASE_TLSTREAM_TL_KBASE_ARRAY_END_KCPUQUEUE_EXECUTE_JIT_FREE_END( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** -- * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER - -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER - - * KCPU Queue executes an Error Barrier - * - * @kbdev: Kbase device - * @kcpu_queue: KCPU queue - */ --#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERRORBARRIER( \ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_error_barrier( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_ERROR_BARRIER( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { } while (0) -+#endif /* MALI_USE_CSF */ -+ -+/** -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START - -+ * KCPU Queue starts a group suspend -+ * -+ * @kbdev: Kbase device -+ * @kcpu_queue: KCPU queue -+ */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START( \ -+ kbdev, \ -+ kcpu_queue \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_group_suspend_start( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_START( \ - kbdev, \ - kcpu_queue \ - ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ -+ -+/** -+ * KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END - -+ * KCPU Queue ends a group suspend -+ * -+ * @kbdev: Kbase device -+ * @kcpu_queue: KCPU queue -+ * @execute_error: Non-zero error code if KCPU Queue item completed with error, else zero -+ */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_kcpuqueue_execute_group_suspend_end( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ kcpu_queue, execute_error); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_KCPUQUEUE_EXECUTE_GROUP_SUSPEND_END( \ -+ kbdev, \ -+ kcpu_queue, \ -+ execute_error \ -+ ) \ -+ do { } while (0) -+#endif /* MALI_USE_CSF */ - - /** - * KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW - -@@ -2299,12 +3042,54 @@ struct kbase_tlstream; - * @csffw_timestamp: Timestamp of a CSFFW event - * @csffw_cycle: Cycle number of a CSFFW event - */ -+#if MALI_USE_CSF - #define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ - kbdev, \ - csffw_timestamp, \ - csffw_cycle \ - ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_csffw_tlstream_overflow( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ csffw_timestamp, csffw_cycle); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_CSFFW_TLSTREAM_OVERFLOW( \ -+ kbdev, \ -+ csffw_timestamp, \ -+ csffw_cycle \ -+ ) \ -+ do { } while (0) -+#endif /* MALI_USE_CSF */ -+ -+/** -+ * KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET - -+ * A reset has happened with the CSFFW -+ * -+ * @kbdev: Kbase device -+ * @csffw_cycle: Cycle number of a CSFFW event -+ */ -+#if MALI_USE_CSF -+#define KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET( \ -+ kbdev, \ -+ csffw_cycle \ -+ ) \ -+ do { \ -+ int enabled = atomic_read(&kbdev->timeline_flags); \ -+ if (enabled & BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) \ -+ __kbase_tlstream_tl_kbase_csffw_reset( \ -+ __TL_DISPATCH_STREAM(kbdev, obj), \ -+ csffw_cycle); \ -+ } while (0) -+#else -+#define KBASE_TLSTREAM_TL_KBASE_CSFFW_RESET( \ -+ kbdev, \ -+ csffw_cycle \ -+ ) \ - do { } while (0) -+#endif /* MALI_USE_CSF */ - - - /* Gator tracepoints are hooked into TLSTREAM interface. -diff --git a/dvalin/kernel/include/linux/dma-buf-test-exporter.h b/dvalin/kernel/include/linux/dma-buf-test-exporter.h -index 95bc6f8..5a310f6 100644 ---- a/dvalin/kernel/include/linux/dma-buf-test-exporter.h -+++ b/dvalin/kernel/include/linux/dma-buf-test-exporter.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2012-2013, 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2012-2013, 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,8 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- - #ifndef _LINUX_DMA_BUF_TEST_EXPORTER_H_ - #define _LINUX_DMA_BUF_TEST_EXPORTER_H_ - -diff --git a/dvalin/kernel/include/linux/memory_group_manager.h b/dvalin/kernel/include/linux/memory_group_manager.h -index b1ac253..c0e808b 100644 ---- a/dvalin/kernel/include/linux/memory_group_manager.h -+++ b/dvalin/kernel/include/linux/memory_group_manager.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _MEMORY_GROUP_MANAGER_H_ -diff --git a/dvalin/kernel/include/linux/priority_control_manager.h b/dvalin/kernel/include/linux/priority_control_manager.h -new file mode 100644 -index 0000000..002e78b ---- /dev/null -+++ b/dvalin/kernel/include/linux/priority_control_manager.h -@@ -0,0 +1,77 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _PRIORITY_CONTROL_MANAGER_H_ -+#define _PRIORITY_CONTROL_MANAGER_H_ -+ -+#include <linux/mm.h> -+#include <linux/of.h> -+#include <linux/version.h> -+ -+struct priority_control_manager_device; -+ -+/** -+ * struct priority_control_manager_ops - Callbacks for priority control manager operations -+ * -+ * @pcm_scheduler_priority_check: Callback to check if scheduling priority level can be requested -+ */ -+struct priority_control_manager_ops { -+ /** -+ * pcm_scheduler_priority_check: This function can be used to check what priority its work -+ * would be treated as based on the requested_priority value. -+ * -+ * @pcm_dev: The priority control manager through which the request is -+ * being made. -+ * @task: The task struct of the process requesting the priority check. -+ * @requested_priority: The priority level being requested. -+ * -+ * The returned value will be: -+ * The same as requested_priority if the process has permission to use requested_priority -+ * A lower priority value if the process does not have permission to use requested_priority -+ * -+ * requested_priority has the following value range: -+ * 0-3 : Priority level, 0 being highest and 3 being lowest -+ * -+ * Return: The priority that would actually be given, could be lower than requested_priority -+ */ -+ int (*pcm_scheduler_priority_check)( -+ struct priority_control_manager_device *pcm_dev, -+ struct task_struct *task, int requested_priority); -+}; -+ -+/** -+ * struct priority_control_manager_device - Device structure for priority -+ * control manager -+ * -+ * @ops: Callbacks associated with this device -+ * @data: Pointer to device private data -+ * @owner: Pointer to the module owner -+ * -+ * This structure should be registered with the platform device using -+ * platform_set_drvdata(). -+ */ -+struct priority_control_manager_device { -+ struct priority_control_manager_ops ops; -+ void *data; -+ struct module *owner; -+}; -+ -+#endif /* _PRIORITY_CONTROL_MANAGER_H_ */ -diff --git a/dvalin/kernel/include/linux/protected_memory_allocator.h b/dvalin/kernel/include/linux/protected_memory_allocator.h -index 3b9205b..1e05bb8 100644 ---- a/dvalin/kernel/include/linux/protected_memory_allocator.h -+++ b/dvalin/kernel/include/linux/protected_memory_allocator.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _PROTECTED_MEMORY_ALLOCATOR_H_ -diff --git a/dvalin/kernel/include/linux/protected_mode_switcher.h b/dvalin/kernel/include/linux/protected_mode_switcher.h -index 8778d81..1a3c5f7 100644 ---- a/dvalin/kernel/include/linux/protected_mode_switcher.h -+++ b/dvalin/kernel/include/linux/protected_mode_switcher.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,8 +17,6 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - - #ifndef _PROTECTED_MODE_SWITCH_H_ -diff --git a/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_base_csf_kernel.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_base_csf_kernel.h -new file mode 100644 -index 0000000..72572e5 ---- /dev/null -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_base_csf_kernel.h -@@ -0,0 +1,765 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _UAPI_BASE_CSF_KERNEL_H_ -+#define _UAPI_BASE_CSF_KERNEL_H_ -+ -+#include <linux/types.h> -+ -+/* Memory allocation, access/hint flags. -+ * -+ * See base_mem_alloc_flags. -+ */ -+ -+/* IN */ -+/* Read access CPU side -+ */ -+#define BASE_MEM_PROT_CPU_RD ((base_mem_alloc_flags)1 << 0) -+ -+/* Write access CPU side -+ */ -+#define BASE_MEM_PROT_CPU_WR ((base_mem_alloc_flags)1 << 1) -+ -+/* Read access GPU side -+ */ -+#define BASE_MEM_PROT_GPU_RD ((base_mem_alloc_flags)1 << 2) -+ -+/* Write access GPU side -+ */ -+#define BASE_MEM_PROT_GPU_WR ((base_mem_alloc_flags)1 << 3) -+ -+/* Execute allowed on the GPU side -+ */ -+#define BASE_MEM_PROT_GPU_EX ((base_mem_alloc_flags)1 << 4) -+ -+/* Will be permanently mapped in kernel space. -+ * Flag is only allowed on allocations originating from kbase. -+ */ -+#define BASEP_MEM_PERMANENT_KERNEL_MAPPING ((base_mem_alloc_flags)1 << 5) -+ -+/* The allocation will completely reside within the same 4GB chunk in the GPU -+ * virtual space. -+ * Since this flag is primarily required only for the TLS memory which will -+ * not be used to contain executable code and also not used for Tiler heap, -+ * it can't be used along with BASE_MEM_PROT_GPU_EX and TILER_ALIGN_TOP flags. -+ */ -+#define BASE_MEM_GPU_VA_SAME_4GB_PAGE ((base_mem_alloc_flags)1 << 6) -+ -+/* Userspace is not allowed to free this memory. -+ * Flag is only allowed on allocations originating from kbase. -+ */ -+#define BASEP_MEM_NO_USER_FREE ((base_mem_alloc_flags)1 << 7) -+ -+#define BASE_MEM_RESERVED_BIT_8 ((base_mem_alloc_flags)1 << 8) -+ -+/* Grow backing store on GPU Page Fault -+ */ -+#define BASE_MEM_GROW_ON_GPF ((base_mem_alloc_flags)1 << 9) -+ -+/* Page coherence Outer shareable, if available -+ */ -+#define BASE_MEM_COHERENT_SYSTEM ((base_mem_alloc_flags)1 << 10) -+ -+/* Page coherence Inner shareable -+ */ -+#define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) -+ -+/* IN/OUT */ -+/* Should be cached on the CPU, returned if actually cached -+ */ -+#define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) -+ -+/* IN/OUT */ -+/* Must have same VA on both the GPU and the CPU -+ */ -+#define BASE_MEM_SAME_VA ((base_mem_alloc_flags)1 << 13) -+ -+/* OUT */ -+/* Must call mmap to acquire a GPU address for the alloc -+ */ -+#define BASE_MEM_NEED_MMAP ((base_mem_alloc_flags)1 << 14) -+ -+/* IN */ -+/* Page coherence Outer shareable, required. -+ */ -+#define BASE_MEM_COHERENT_SYSTEM_REQUIRED ((base_mem_alloc_flags)1 << 15) -+ -+/* Protected memory -+ */ -+#define BASE_MEM_PROTECTED ((base_mem_alloc_flags)1 << 16) -+ -+/* Not needed physical memory -+ */ -+#define BASE_MEM_DONT_NEED ((base_mem_alloc_flags)1 << 17) -+ -+/* Must use shared CPU/GPU zone (SAME_VA zone) but doesn't require the -+ * addresses to be the same -+ */ -+#define BASE_MEM_IMPORT_SHARED ((base_mem_alloc_flags)1 << 18) -+ -+/* CSF event memory -+ * -+ * If Outer shareable coherence is not specified or not available, then on -+ * allocation kbase will automatically use the uncached GPU mapping. -+ * There is no need for the client to specify BASE_MEM_UNCACHED_GPU -+ * themselves when allocating memory with the BASE_MEM_CSF_EVENT flag. -+ * -+ * This memory requires a permanent mapping -+ * -+ * See also kbase_reg_needs_kernel_mapping() -+ */ -+#define BASE_MEM_CSF_EVENT ((base_mem_alloc_flags)1 << 19) -+ -+#define BASE_MEM_RESERVED_BIT_20 ((base_mem_alloc_flags)1 << 20) -+ -+/* Should be uncached on the GPU, will work only for GPUs using AARCH64 mmu -+ * mode. Some components within the GPU might only be able to access memory -+ * that is GPU cacheable. Refer to the specific GPU implementation for more -+ * details. The 3 shareability flags will be ignored for GPU uncached memory. -+ * If used while importing USER_BUFFER type memory, then the import will fail -+ * if the memory is not aligned to GPU and CPU cache line width. -+ */ -+#define BASE_MEM_UNCACHED_GPU ((base_mem_alloc_flags)1 << 21) -+ -+/* -+ * Bits [22:25] for group_id (0~15). -+ * -+ * base_mem_group_id_set() should be used to pack a memory group ID into a -+ * base_mem_alloc_flags value instead of accessing the bits directly. -+ * base_mem_group_id_get() should be used to extract the memory group ID from -+ * a base_mem_alloc_flags value. -+ */ -+#define BASEP_MEM_GROUP_ID_SHIFT 22 -+#define BASE_MEM_GROUP_ID_MASK \ -+ ((base_mem_alloc_flags)0xF << BASEP_MEM_GROUP_ID_SHIFT) -+ -+/* Must do CPU cache maintenance when imported memory is mapped/unmapped -+ * on GPU. Currently applicable to dma-buf type only. -+ */ -+#define BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP ((base_mem_alloc_flags)1 << 26) -+ -+/* OUT */ -+/* Kernel side cache sync ops required */ -+#define BASE_MEM_KERNEL_SYNC ((base_mem_alloc_flags)1 << 28) -+ -+/* Number of bits used as flags for base memory management -+ * -+ * Must be kept in sync with the base_mem_alloc_flags flags -+ */ -+#define BASE_MEM_FLAGS_NR_BITS 29 -+ -+/* A mask of all the flags which are only valid for allocations within kbase, -+ * and may not be passed from user space. -+ */ -+#define BASEP_MEM_FLAGS_KERNEL_ONLY \ -+ (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE) -+ -+/* A mask for all output bits, excluding IN/OUT bits. -+ */ -+#define BASE_MEM_FLAGS_OUTPUT_MASK BASE_MEM_NEED_MMAP -+ -+/* A mask for all input bits, including IN/OUT bits. -+ */ -+#define BASE_MEM_FLAGS_INPUT_MASK \ -+ (((1 << BASE_MEM_FLAGS_NR_BITS) - 1) & ~BASE_MEM_FLAGS_OUTPUT_MASK) -+ -+/* A mask of all currently reserved flags -+ */ -+#define BASE_MEM_FLAGS_RESERVED \ -+ BASE_MEM_RESERVED_BIT_8 | BASE_MEM_RESERVED_BIT_20 -+ -+#define BASEP_MEM_INVALID_HANDLE (0ull << 12) -+#define BASE_MEM_MMU_DUMP_HANDLE (1ull << 12) -+#define BASE_MEM_TRACE_BUFFER_HANDLE (2ull << 12) -+#define BASE_MEM_MAP_TRACKING_HANDLE (3ull << 12) -+#define BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE (4ull << 12) -+/* reserved handles ..-47<<PAGE_SHIFT> for future special handles */ -+#define BASEP_MEM_CSF_USER_REG_PAGE_HANDLE (47ul << 12) -+#define BASEP_MEM_CSF_USER_IO_PAGES_HANDLE (48ul << 12) -+#define BASE_MEM_COOKIE_BASE (64ul << 12) -+#define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ -+ BASE_MEM_COOKIE_BASE) -+ -+#define KBASE_CSF_NUM_USER_IO_PAGES_HANDLE \ -+ ((BASE_MEM_COOKIE_BASE - BASEP_MEM_CSF_USER_IO_PAGES_HANDLE) >> \ -+ LOCAL_PAGE_SHIFT) -+ -+/** -+ * Valid set of just-in-time memory allocation flags -+ */ -+#define BASE_JIT_ALLOC_VALID_FLAGS ((__u8)0) -+ -+/* Flags to pass to ::base_context_init. -+ * Flags can be ORed together to enable multiple things. -+ * -+ * These share the same space as BASEP_CONTEXT_FLAG_*, and so must -+ * not collide with them. -+ */ -+typedef __u32 base_context_create_flags; -+ -+/* No flags set */ -+#define BASE_CONTEXT_CREATE_FLAG_NONE ((base_context_create_flags)0) -+ -+/* Base context is embedded in a cctx object (flag used for CINSTR -+ * software counter macros) -+ */ -+#define BASE_CONTEXT_CCTX_EMBEDDED ((base_context_create_flags)1 << 0) -+ -+/* Base context is a 'System Monitor' context for Hardware counters. -+ * -+ * One important side effect of this is that job submission is disabled. -+ */ -+#define BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED \ -+ ((base_context_create_flags)1 << 1) -+ -+/* Base context creates a CSF event notification thread. -+ * -+ * The creation of a CSF event notification thread is conditional but -+ * mandatory for the handling of CSF events. -+ */ -+#define BASE_CONTEXT_CSF_EVENT_THREAD ((base_context_create_flags)1 << 2) -+ -+/* Bit-shift used to encode a memory group ID in base_context_create_flags -+ */ -+#define BASEP_CONTEXT_MMU_GROUP_ID_SHIFT (3) -+ -+/* Bitmask used to encode a memory group ID in base_context_create_flags -+ */ -+#define BASEP_CONTEXT_MMU_GROUP_ID_MASK \ -+ ((base_context_create_flags)0xF << BASEP_CONTEXT_MMU_GROUP_ID_SHIFT) -+ -+/* Bitpattern describing the base_context_create_flags that can be -+ * passed to the kernel -+ */ -+#define BASEP_CONTEXT_CREATE_KERNEL_FLAGS \ -+ (BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED | \ -+ BASEP_CONTEXT_MMU_GROUP_ID_MASK) -+ -+/* Bitpattern describing the ::base_context_create_flags that can be -+ * passed to base_context_init() -+ */ -+#define BASEP_CONTEXT_CREATE_ALLOWED_FLAGS \ -+ (BASE_CONTEXT_CCTX_EMBEDDED | \ -+ BASE_CONTEXT_CSF_EVENT_THREAD | \ -+ BASEP_CONTEXT_CREATE_KERNEL_FLAGS) -+ -+/* Enable additional tracepoints for latency measurements (TL_ATOM_READY, -+ * TL_ATOM_DONE, TL_ATOM_PRIO_CHANGE, TL_ATOM_EVENT_POST) -+ */ -+#define BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS (1 << 0) -+ -+/* Indicate that job dumping is enabled. This could affect certain timers -+ * to account for the performance impact. -+ */ -+#define BASE_TLSTREAM_JOB_DUMPING_ENABLED (1 << 1) -+ -+/* Enable KBase tracepoints for CSF builds */ -+#define BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS (1 << 2) -+ -+/* Enable additional CSF Firmware side tracepoints */ -+#define BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS (1 << 3) -+ -+#define BASE_TLSTREAM_FLAGS_MASK (BASE_TLSTREAM_ENABLE_LATENCY_TRACEPOINTS | \ -+ BASE_TLSTREAM_JOB_DUMPING_ENABLED | \ -+ BASE_TLSTREAM_ENABLE_CSF_TRACEPOINTS | \ -+ BASE_TLSTREAM_ENABLE_CSFFW_TRACEPOINTS) -+ -+/* Number of pages mapped into the process address space for a bound GPU -+ * command queue. A pair of input/output pages and a Hw doorbell page -+ * are mapped to enable direct submission of commands to Hw. -+ */ -+#define BASEP_QUEUE_NR_MMAP_USER_PAGES ((size_t)3) -+ -+#define BASE_QUEUE_MAX_PRIORITY (15U) -+ -+/* CQS Sync object is an array of __u32 event_mem[2], error field index is 1 */ -+#define BASEP_EVENT_VAL_INDEX (0U) -+#define BASEP_EVENT_ERR_INDEX (1U) -+ -+/* The upper limit for number of objects that could be waited/set per command. -+ * This limit is now enforced as internally the error inherit inputs are -+ * converted to 32-bit flags in a __u32 variable occupying a previously padding -+ * field. -+ */ -+#define BASEP_KCPU_CQS_MAX_NUM_OBJS ((size_t)32) -+ -+#if MALI_UNIT_TEST -+/** -+ * enum base_kcpu_command_type - Kernel CPU queue command type. -+ * @BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: fence_signal, -+ * @BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: fence_wait, -+ * @BASE_KCPU_COMMAND_TYPE_CQS_WAIT: cqs_wait, -+ * @BASE_KCPU_COMMAND_TYPE_CQS_SET: cqs_set, -+ * @BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION: cqs_wait_operation, -+ * @BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION: cqs_set_operation, -+ * @BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: map_import, -+ * @BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: unmap_import, -+ * @BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: unmap_import_force, -+ * @BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: jit_alloc, -+ * @BASE_KCPU_COMMAND_TYPE_JIT_FREE: jit_free, -+ * @BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: group_suspend, -+ * @BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: error_barrier, -+ * @BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME: sample_time, -+ */ -+enum base_kcpu_command_type { -+ BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL, -+ BASE_KCPU_COMMAND_TYPE_FENCE_WAIT, -+ BASE_KCPU_COMMAND_TYPE_CQS_WAIT, -+ BASE_KCPU_COMMAND_TYPE_CQS_SET, -+ BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION, -+ BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION, -+ BASE_KCPU_COMMAND_TYPE_MAP_IMPORT, -+ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT, -+ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE, -+ BASE_KCPU_COMMAND_TYPE_JIT_ALLOC, -+ BASE_KCPU_COMMAND_TYPE_JIT_FREE, -+ BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND, -+ BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER, -+ BASE_KCPU_COMMAND_TYPE_SAMPLE_TIME, -+}; -+#else -+/** -+ * enum base_kcpu_command_type - Kernel CPU queue command type. -+ * @BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL: fence_signal, -+ * @BASE_KCPU_COMMAND_TYPE_FENCE_WAIT: fence_wait, -+ * @BASE_KCPU_COMMAND_TYPE_CQS_WAIT: cqs_wait, -+ * @BASE_KCPU_COMMAND_TYPE_CQS_SET: cqs_set, -+ * @BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION: cqs_wait_operation, -+ * @BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION: cqs_set_operation, -+ * @BASE_KCPU_COMMAND_TYPE_MAP_IMPORT: map_import, -+ * @BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT: unmap_import, -+ * @BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE: unmap_import_force, -+ * @BASE_KCPU_COMMAND_TYPE_JIT_ALLOC: jit_alloc, -+ * @BASE_KCPU_COMMAND_TYPE_JIT_FREE: jit_free, -+ * @BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND: group_suspend, -+ * @BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER: error_barrier, -+ */ -+enum base_kcpu_command_type { -+ BASE_KCPU_COMMAND_TYPE_FENCE_SIGNAL, -+ BASE_KCPU_COMMAND_TYPE_FENCE_WAIT, -+ BASE_KCPU_COMMAND_TYPE_CQS_WAIT, -+ BASE_KCPU_COMMAND_TYPE_CQS_SET, -+ BASE_KCPU_COMMAND_TYPE_CQS_WAIT_OPERATION, -+ BASE_KCPU_COMMAND_TYPE_CQS_SET_OPERATION, -+ BASE_KCPU_COMMAND_TYPE_MAP_IMPORT, -+ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT, -+ BASE_KCPU_COMMAND_TYPE_UNMAP_IMPORT_FORCE, -+ BASE_KCPU_COMMAND_TYPE_JIT_ALLOC, -+ BASE_KCPU_COMMAND_TYPE_JIT_FREE, -+ BASE_KCPU_COMMAND_TYPE_GROUP_SUSPEND, -+ BASE_KCPU_COMMAND_TYPE_ERROR_BARRIER, -+}; -+#endif /* MALI_UNIT_TEST */ -+ -+/** -+ * enum base_queue_group_priority - Priority of a GPU Command Queue Group. -+ * @BASE_QUEUE_GROUP_PRIORITY_HIGH: GPU Command Queue Group is of high -+ * priority. -+ * @BASE_QUEUE_GROUP_PRIORITY_MEDIUM: GPU Command Queue Group is of medium -+ * priority. -+ * @BASE_QUEUE_GROUP_PRIORITY_LOW: GPU Command Queue Group is of low -+ * priority. -+ * @BASE_QUEUE_GROUP_PRIORITY_REALTIME: GPU Command Queue Group is of real-time -+ * priority. -+ * @BASE_QUEUE_GROUP_PRIORITY_COUNT: Number of GPU Command Queue Group -+ * priority levels. -+ * -+ * Currently this is in order of highest to lowest, but if new levels are added -+ * then those new levels may be out of order to preserve the ABI compatibility -+ * with previous releases. At that point, ensure assignment to -+ * the 'priority' member in &kbase_queue_group is updated to ensure it remains -+ * a linear ordering. -+ * -+ * There should be no gaps in the enum, otherwise use of -+ * BASE_QUEUE_GROUP_PRIORITY_COUNT in kbase must be updated. -+ */ -+enum base_queue_group_priority { -+ BASE_QUEUE_GROUP_PRIORITY_HIGH = 0, -+ BASE_QUEUE_GROUP_PRIORITY_MEDIUM, -+ BASE_QUEUE_GROUP_PRIORITY_LOW, -+ BASE_QUEUE_GROUP_PRIORITY_REALTIME, -+ BASE_QUEUE_GROUP_PRIORITY_COUNT -+}; -+ -+struct base_kcpu_command_fence_info { -+ __u64 fence; -+}; -+ -+struct base_cqs_wait_info { -+ __u64 addr; -+ __u32 val; -+ __u32 padding; -+}; -+ -+struct base_kcpu_command_cqs_wait_info { -+ __u64 objs; -+ __u32 nr_objs; -+ __u32 inherit_err_flags; -+}; -+ -+struct base_cqs_set { -+ __u64 addr; -+}; -+ -+struct base_kcpu_command_cqs_set_info { -+ __u64 objs; -+ __u32 nr_objs; -+ __u32 padding; -+}; -+ -+/** -+ * basep_cqs_data_type - Enumeration of CQS Data Types -+ * -+ * @BASEP_CQS_DATA_TYPE_U32: The Data Type of a CQS Object's value -+ * is an unsigned 32-bit integer -+ * @BASEP_CQS_DATA_TYPE_U64: The Data Type of a CQS Object's value -+ * is an unsigned 64-bit integer -+ */ -+typedef enum PACKED { -+ BASEP_CQS_DATA_TYPE_U32 = 0, -+ BASEP_CQS_DATA_TYPE_U64 = 1, -+} basep_cqs_data_type; -+ -+/** -+ * basep_cqs_wait_operation_op - Enumeration of CQS Object Wait -+ * Operation conditions -+ * -+ * @BASEP_CQS_WAIT_OPERATION_LE: CQS Wait Operation indicating that a -+ * wait will be satisfied when a CQS Object's -+ * value is Less than or Equal to -+ * the Wait Operation value -+ * @BASEP_CQS_WAIT_OPERATION_GT: CQS Wait Operation indicating that a -+ * wait will be satisfied when a CQS Object's -+ * value is Greater than the Wait Operation value -+ */ -+typedef enum { -+ BASEP_CQS_WAIT_OPERATION_LE = 0, -+ BASEP_CQS_WAIT_OPERATION_GT = 1, -+} basep_cqs_wait_operation_op; -+ -+struct base_cqs_wait_operation_info { -+ __u64 addr; -+ __u64 val; -+ __u8 operation; -+ __u8 data_type; -+ __u8 padding[6]; -+}; -+ -+/** -+ * struct base_kcpu_command_cqs_wait_operation_info - structure which contains information -+ * about the Timeline CQS wait objects -+ * -+ * @objs: An array of Timeline CQS waits. -+ * @nr_objs: Number of Timeline CQS waits in the array. -+ * @inherit_err_flags: Bit-pattern for the CQSs in the array who's error field -+ * to be served as the source for importing into the -+ * queue's error-state. -+ */ -+struct base_kcpu_command_cqs_wait_operation_info { -+ __u64 objs; -+ __u32 nr_objs; -+ __u32 inherit_err_flags; -+}; -+ -+/** -+ * basep_cqs_set_operation_op - Enumeration of CQS Set Operations -+ * -+ * @BASEP_CQS_SET_OPERATION_ADD: CQS Set operation for adding a value -+ * to a synchronization object -+ * @BASEP_CQS_SET_OPERATION_SET: CQS Set operation for setting the value -+ * of a synchronization object -+ */ -+typedef enum { -+ BASEP_CQS_SET_OPERATION_ADD = 0, -+ BASEP_CQS_SET_OPERATION_SET = 1, -+} basep_cqs_set_operation_op; -+ -+struct base_cqs_set_operation_info { -+ __u64 addr; -+ __u64 val; -+ __u8 operation; -+ __u8 data_type; -+ __u8 padding[6]; -+}; -+ -+/** -+ * struct base_kcpu_command_cqs_set_operation_info - structure which contains information -+ * about the Timeline CQS set objects -+ * -+ * @objs: An array of Timeline CQS sets. -+ * @nr_objs: Number of Timeline CQS sets in the array. -+ * @padding: Structure padding, unused bytes. -+ */ -+struct base_kcpu_command_cqs_set_operation_info { -+ __u64 objs; -+ __u32 nr_objs; -+ __u32 padding; -+}; -+ -+/** -+ * struct base_kcpu_command_import_info - structure which contains information -+ * about the imported buffer. -+ * -+ * @handle: Address of imported user buffer. -+ */ -+struct base_kcpu_command_import_info { -+ __u64 handle; -+}; -+ -+/** -+ * struct base_kcpu_command_jit_alloc_info - structure which contains -+ * information about jit memory allocation. -+ * -+ * @info: An array of elements of the -+ * struct base_jit_alloc_info type. -+ * @count: The number of elements in the info array. -+ * @padding: Padding to a multiple of 64 bits. -+ */ -+struct base_kcpu_command_jit_alloc_info { -+ __u64 info; -+ __u8 count; -+ __u8 padding[7]; -+}; -+ -+/** -+ * struct base_kcpu_command_jit_free_info - structure which contains -+ * information about jit memory which is to be freed. -+ * -+ * @ids: An array containing the JIT IDs to free. -+ * @count: The number of elements in the ids array. -+ * @padding: Padding to a multiple of 64 bits. -+ */ -+struct base_kcpu_command_jit_free_info { -+ __u64 ids; -+ __u8 count; -+ __u8 padding[7]; -+}; -+ -+/** -+ * struct base_kcpu_command_group_suspend_info - structure which contains -+ * suspend buffer data captured for a suspended queue group. -+ * -+ * @buffer: Pointer to an array of elements of the type char. -+ * @size: Number of elements in the @buffer array. -+ * @group_handle: Handle to the mapping of CSG. -+ * @padding: padding to a multiple of 64 bits. -+ */ -+struct base_kcpu_command_group_suspend_info { -+ __u64 buffer; -+ __u32 size; -+ __u8 group_handle; -+ __u8 padding[3]; -+}; -+ -+#if MALI_UNIT_TEST -+struct base_kcpu_command_sample_time_info { -+ __u64 time; -+}; -+#endif /* MALI_UNIT_TEST */ -+ -+/** -+ * struct base_kcpu_command - kcpu command. -+ * @type: type of the kcpu command, one enum base_kcpu_command_type -+ * @padding: padding to a multiple of 64 bits -+ * @info: structure which contains information about the kcpu command; -+ * actual type is determined by @p type -+ * @info.fence: Fence -+ * @info.cqs_wait: CQS wait -+ * @info.cqs_set: CQS set -+ * @info.import: import -+ * @info.jit_alloc: jit allocation -+ * @info.jit_free: jit deallocation -+ * @info.suspend_buf_copy: suspend buffer copy -+ * @info.sample_time: sample time -+ * @info.padding: padding -+ */ -+struct base_kcpu_command { -+ __u8 type; -+ __u8 padding[sizeof(__u64) - sizeof(__u8)]; -+ union { -+ struct base_kcpu_command_fence_info fence; -+ struct base_kcpu_command_cqs_wait_info cqs_wait; -+ struct base_kcpu_command_cqs_set_info cqs_set; -+ struct base_kcpu_command_cqs_wait_operation_info cqs_wait_operation; -+ struct base_kcpu_command_cqs_set_operation_info cqs_set_operation; -+ struct base_kcpu_command_import_info import; -+ struct base_kcpu_command_jit_alloc_info jit_alloc; -+ struct base_kcpu_command_jit_free_info jit_free; -+ struct base_kcpu_command_group_suspend_info suspend_buf_copy; -+#if MALI_UNIT_TEST -+ struct base_kcpu_command_sample_time_info sample_time; -+#endif /* MALI_UNIT_TEST */ -+ __u64 padding[2]; /* No sub-struct should be larger */ -+ } info; -+}; -+ -+/** -+ * struct basep_cs_stream_control - CSI capabilities. -+ * -+ * @features: Features of this stream -+ * @padding: Padding to a multiple of 64 bits. -+ */ -+struct basep_cs_stream_control { -+ __u32 features; -+ __u32 padding; -+}; -+ -+/** -+ * struct basep_cs_group_control - CSG interface capabilities. -+ * -+ * @features: Features of this group -+ * @stream_num: Number of streams in this group -+ * @suspend_size: Size in bytes of the suspend buffer for this group -+ * @padding: Padding to a multiple of 64 bits. -+ */ -+struct basep_cs_group_control { -+ __u32 features; -+ __u32 stream_num; -+ __u32 suspend_size; -+ __u32 padding; -+}; -+ -+/** -+ * struct base_gpu_queue_group_error_fatal_payload - Unrecoverable fault -+ * error information associated with GPU command queue group. -+ * -+ * @sideband: Additional information of the unrecoverable fault. -+ * @status: Unrecoverable fault information. -+ * This consists of exception type (least significant byte) and -+ * data (remaining bytes). One example of exception type is -+ * CS_INVALID_INSTRUCTION (0x49). -+ * @padding: Padding to make multiple of 64bits -+ */ -+struct base_gpu_queue_group_error_fatal_payload { -+ __u64 sideband; -+ __u32 status; -+ __u32 padding; -+}; -+ -+/** -+ * struct base_gpu_queue_error_fatal_payload - Unrecoverable fault -+ * error information related to GPU command queue. -+ * -+ * @sideband: Additional information about this unrecoverable fault. -+ * @status: Unrecoverable fault information. -+ * This consists of exception type (least significant byte) and -+ * data (remaining bytes). One example of exception type is -+ * CS_INVALID_INSTRUCTION (0x49). -+ * @csi_index: Index of the CSF interface the queue is bound to. -+ * @padding: Padding to make multiple of 64bits -+ */ -+struct base_gpu_queue_error_fatal_payload { -+ __u64 sideband; -+ __u32 status; -+ __u8 csi_index; -+ __u8 padding[3]; -+}; -+ -+/** -+ * enum base_gpu_queue_group_error_type - GPU Fatal error type. -+ * -+ * @BASE_GPU_QUEUE_GROUP_ERROR_FATAL: Fatal error associated with GPU -+ * command queue group. -+ * @BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL: Fatal error associated with GPU -+ * command queue. -+ * @BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT: Fatal error associated with -+ * progress timeout. -+ * @BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM: Fatal error due to running out -+ * of tiler heap memory. -+ * @BASE_GPU_QUEUE_GROUP_ERROR_FATAL_COUNT: The number of fatal error types -+ * -+ * This type is used for &struct_base_gpu_queue_group_error.error_type. -+ */ -+enum base_gpu_queue_group_error_type { -+ BASE_GPU_QUEUE_GROUP_ERROR_FATAL = 0, -+ BASE_GPU_QUEUE_GROUP_QUEUE_ERROR_FATAL, -+ BASE_GPU_QUEUE_GROUP_ERROR_TIMEOUT, -+ BASE_GPU_QUEUE_GROUP_ERROR_TILER_HEAP_OOM, -+ BASE_GPU_QUEUE_GROUP_ERROR_FATAL_COUNT -+}; -+ -+/** -+ * struct base_gpu_queue_group_error - Unrecoverable fault information -+ * @error_type: Error type of @base_gpu_queue_group_error_type -+ * indicating which field in union payload is filled -+ * @padding: Unused bytes for 64bit boundary -+ * @payload: Input Payload -+ * @payload.fatal_group: Unrecoverable fault error associated with -+ * GPU command queue group -+ * @payload.fatal_queue: Unrecoverable fault error associated with command queue -+ */ -+struct base_gpu_queue_group_error { -+ __u8 error_type; -+ __u8 padding[7]; -+ union { -+ struct base_gpu_queue_group_error_fatal_payload fatal_group; -+ struct base_gpu_queue_error_fatal_payload fatal_queue; -+ } payload; -+}; -+ -+/** -+ * enum base_csf_notification_type - Notification type -+ * -+ * @BASE_CSF_NOTIFICATION_EVENT: Notification with kernel event -+ * @BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR: Notification with GPU fatal -+ * error -+ * @BASE_CSF_NOTIFICATION_CPU_QUEUE_DUMP: Notification with dumping cpu -+ * queue -+ * @BASE_CSF_NOTIFICATION_COUNT: The number of notification type -+ * -+ * This type is used for &struct_base_csf_notification.type. -+ */ -+enum base_csf_notification_type { -+ BASE_CSF_NOTIFICATION_EVENT = 0, -+ BASE_CSF_NOTIFICATION_GPU_QUEUE_GROUP_ERROR, -+ BASE_CSF_NOTIFICATION_CPU_QUEUE_DUMP, -+ BASE_CSF_NOTIFICATION_COUNT -+}; -+ -+/** -+ * struct base_csf_notification - Event or error notification -+ * -+ * @type: Notification type of @base_csf_notification_type -+ * @padding: Padding for 64bit boundary -+ * @payload: Input Payload -+ * @payload.align: To fit the struct into a 64-byte cache line -+ * @payload.csg_error: CSG error -+ * @payload.csg_error.handle: Handle of GPU command queue group associated with -+ * fatal error -+ * @payload.csg_error.padding: Padding -+ * @payload.csg_error.error: Unrecoverable fault error -+ * -+ */ -+struct base_csf_notification { -+ __u8 type; -+ __u8 padding[7]; -+ union { -+ struct { -+ __u8 handle; -+ __u8 padding[7]; -+ struct base_gpu_queue_group_error error; -+ } csg_error; -+ -+ __u8 align[56]; -+ } payload; -+}; -+ -+#endif /* _UAPI_BASE_CSF_KERNEL_H_ */ -diff --git a/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_gpu_csf_control_registers.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_gpu_csf_control_registers.h -new file mode 100644 -index 0000000..b62a8b0 ---- /dev/null -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_gpu_csf_control_registers.h -@@ -0,0 +1,32 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * This header was autogenerated, it should not be edited. -+ */ -+ -+#ifndef _UAPI_GPU_CSF_CONTROL_REGISTERS_H_ -+#define _UAPI_GPU_CSF_CONTROL_REGISTERS_H_ -+ -+/* GPU_REGISTERS register offsets */ -+#define GPU_CONTROL_MCU 0x3000 /* () MCU control registers */ -+ -+#endif /* _UAPI_GPU_CSF_CONTROL_REGISTERS_H_ */ -diff --git a/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h -new file mode 100644 -index 0000000..06cc4c2 ---- /dev/null -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_gpu_csf_registers.h -@@ -0,0 +1,1488 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2018-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * This header was autogenerated, it should not be edited. -+ */ -+ -+#ifndef _UAPI_GPU_CSF_REGISTERS_H_ -+#define _UAPI_GPU_CSF_REGISTERS_H_ -+ -+/* -+ * Begin register sets -+ */ -+ -+/* DOORBELLS base address */ -+#define DOORBELLS_BASE 0x0080000 -+#define DOORBELLS_REG(r) (DOORBELLS_BASE + (r)) -+ -+/* CS_KERNEL_INPUT_BLOCK base address */ -+#define CS_KERNEL_INPUT_BLOCK_BASE 0x0000 -+#define CS_KERNEL_INPUT_BLOCK_REG(r) (CS_KERNEL_INPUT_BLOCK_BASE + (r)) -+ -+/* CS_KERNEL_OUTPUT_BLOCK base address */ -+#define CS_KERNEL_OUTPUT_BLOCK_BASE 0x0000 -+#define CS_KERNEL_OUTPUT_BLOCK_REG(r) (CS_KERNEL_OUTPUT_BLOCK_BASE + (r)) -+ -+/* CS_USER_INPUT_BLOCK base address */ -+#define CS_USER_INPUT_BLOCK_BASE 0x0000 -+#define CS_USER_INPUT_BLOCK_REG(r) (CS_USER_INPUT_BLOCK_BASE + (r)) -+ -+/* CS_USER_OUTPUT_BLOCK base address */ -+#define CS_USER_OUTPUT_BLOCK_BASE 0x0000 -+#define CS_USER_OUTPUT_BLOCK_REG(r) (CS_USER_OUTPUT_BLOCK_BASE + (r)) -+ -+/* CSG_INPUT_BLOCK base address */ -+#define CSG_INPUT_BLOCK_BASE 0x0000 -+#define CSG_INPUT_BLOCK_REG(r) (CSG_INPUT_BLOCK_BASE + (r)) -+ -+/* CSG_OUTPUT_BLOCK base address */ -+#define CSG_OUTPUT_BLOCK_BASE 0x0000 -+#define CSG_OUTPUT_BLOCK_REG(r) (CSG_OUTPUT_BLOCK_BASE + (r)) -+ -+/* GLB_CONTROL_BLOCK base address */ -+#define GLB_CONTROL_BLOCK_BASE 0x04000000 -+#define GLB_CONTROL_BLOCK_REG(r) (GLB_CONTROL_BLOCK_BASE + (r)) -+ -+/* GLB_INPUT_BLOCK base address */ -+#define GLB_INPUT_BLOCK_BASE 0x0000 -+#define GLB_INPUT_BLOCK_REG(r) (GLB_INPUT_BLOCK_BASE + (r)) -+ -+/* GLB_OUTPUT_BLOCK base address */ -+#define GLB_OUTPUT_BLOCK_BASE 0x0000 -+#define GLB_OUTPUT_BLOCK_REG(r) (GLB_OUTPUT_BLOCK_BASE + (r)) -+ -+/* USER base address */ -+#define USER_BASE 0x0010000 -+#define USER_REG(r) (USER_BASE + (r)) -+ -+/* End register sets */ -+ -+/* -+ * Begin register offsets -+ */ -+ -+/* DOORBELLS register offsets */ -+#define DOORBELL_0 0x0000 /* () Doorbell 0 register */ -+#define DOORBELL(n) (DOORBELL_0 + (n)*65536) -+#define DOORBELL_REG(n, r) (DOORBELL(n) + DOORBELL_BLOCK_REG(r)) -+#define DOORBELL_COUNT 1024 -+ -+/* DOORBELL_BLOCK register offsets */ -+#define DB_BLK_DOORBELL 0x0000 /* (WO) Doorbell request */ -+ -+/* CS_KERNEL_INPUT_BLOCK register offsets */ -+#define CS_REQ 0x0000 /* () CS request flags */ -+#define CS_CONFIG 0x0004 /* () CS configuration */ -+#define CS_ACK_IRQ_MASK 0x000C /* () Command steam interrupt mask */ -+#define CS_BASE_LO 0x0010 /* () Base pointer for the ring buffer, low word */ -+#define CS_BASE_HI 0x0014 /* () Base pointer for the ring buffer, high word */ -+#define CS_SIZE 0x0018 /* () Size of the ring buffer */ -+#define CS_TILER_HEAP_START_LO 0x0020 /* () Pointer to heap start, low word */ -+#define CS_TILER_HEAP_START_HI 0x0024 /* () Pointer to heap start, high word */ -+#define CS_TILER_HEAP_END_LO 0x0028 /* () Tiler heap descriptor address, low word */ -+#define CS_TILER_HEAP_END_HI 0x002C /* () Tiler heap descriptor address, high word */ -+#define CS_USER_INPUT_LO 0x0030 /* () CS user mode input page address, low word */ -+#define CS_USER_INPUT_HI 0x0034 /* () CS user mode input page address, high word */ -+#define CS_USER_OUTPUT_LO 0x0038 /* () CS user mode input page address, low word */ -+#define CS_USER_OUTPUT_HI 0x003C /* () CS user mode input page address, high word */ -+#define CS_INSTR_CONFIG 0x0040 /* () Instrumentation buffer configuration */ -+#define CS_INSTR_BUFFER_SIZE 0x0044 /* () Instrumentation buffer size */ -+#define CS_INSTR_BUFFER_BASE_LO 0x0048 /* () Instrumentation buffer base pointer, low word */ -+#define CS_INSTR_BUFFER_BASE_HI 0x004C /* () Instrumentation buffer base pointer, high word */ -+#define CS_INSTR_BUFFER_OFFSET_POINTER_LO 0x0050 /* () Instrumentation buffer pointer to insert offset, low word */ -+#define CS_INSTR_BUFFER_OFFSET_POINTER_HI 0x0054 /* () Instrumentation buffer pointer to insert offset, high word */ -+ -+/* CS_KERNEL_OUTPUT_BLOCK register offsets */ -+#define CS_ACK 0x0000 /* () CS acknowledge flags */ -+#define CS_STATUS_CMD_PTR_LO 0x0040 /* () Program pointer current value, low word */ -+#define CS_STATUS_CMD_PTR_HI 0x0044 /* () Program pointer current value, high word */ -+#define CS_STATUS_WAIT 0x0048 /* () Wait condition status register */ -+#define CS_STATUS_REQ_RESOURCE 0x004C /* () Indicates the resources requested by the CS */ -+#define CS_STATUS_WAIT_SYNC_POINTER_LO 0x0050 /* () Sync object pointer, low word */ -+#define CS_STATUS_WAIT_SYNC_POINTER_HI 0x0054 /* () Sync object pointer, high word */ -+#define CS_STATUS_WAIT_SYNC_VALUE 0x0058 /* () Sync object test value */ -+#define CS_STATUS_SCOREBOARDS 0x005C /* () Scoreboard status */ -+#define CS_STATUS_BLOCKED_REASON 0x0060 /* () Blocked reason */ -+#define CS_FAULT 0x0080 /* () Recoverable fault information */ -+#define CS_FATAL 0x0084 /* () Unrecoverable fault information */ -+#define CS_FAULT_INFO_LO 0x0088 /* () Additional information about a recoverable fault, low word */ -+#define CS_FAULT_INFO_HI 0x008C /* () Additional information about a recoverable fault, high word */ -+#define CS_FATAL_INFO_LO 0x0090 /* () Additional information about a non-recoverable fault, low word */ -+#define CS_FATAL_INFO_HI 0x0094 /* () Additional information about a non-recoverable fault, high word */ -+#define CS_HEAP_VT_START 0x00C0 /* () Number of vertex/tiling operations started */ -+#define CS_HEAP_VT_END 0x00C4 /* () Number of vertex/tiling operations completed */ -+#define CS_HEAP_FRAG_END 0x00CC /* () Number of fragment completed */ -+#define CS_HEAP_ADDRESS_LO 0x00D0 /* () Heap address, low word */ -+#define CS_HEAP_ADDRESS_HI 0x00D4 /* () Heap address, high word */ -+ -+/* CS_USER_INPUT_BLOCK register offsets */ -+#define CS_INSERT_LO 0x0000 /* () Current insert offset for ring buffer, low word */ -+#define CS_INSERT_HI 0x0004 /* () Current insert offset for ring buffer, high word */ -+#define CS_EXTRACT_INIT_LO 0x0008 /* () Initial extract offset for ring buffer, low word */ -+#define CS_EXTRACT_INIT_HI 0x000C /* () Initial extract offset for ring buffer, high word */ -+ -+/* CS_USER_OUTPUT_BLOCK register offsets */ -+#define CS_EXTRACT_LO 0x0000 /* () Current extract offset for ring buffer, low word */ -+#define CS_EXTRACT_HI 0x0004 /* () Current extract offset for ring buffer, high word */ -+#define CS_ACTIVE 0x0008 /* () Initial extract offset when the CS is started */ -+ -+/* CSG_INPUT_BLOCK register offsets */ -+#define CSG_REQ 0x0000 /* () CSG request */ -+#define CSG_ACK_IRQ_MASK 0x0004 /* () Global acknowledge interrupt mask */ -+#define CSG_DB_REQ 0x0008 /* () Global doorbell request */ -+#define CSG_IRQ_ACK 0x000C /* () CS IRQ acknowledge */ -+#define CSG_ALLOW_COMPUTE_LO 0x0020 /* () Allowed compute endpoints, low word */ -+#define CSG_ALLOW_COMPUTE_HI 0x0024 /* () Allowed compute endpoints, high word */ -+#define CSG_ALLOW_FRAGMENT_LO 0x0028 /* () Allowed fragment endpoints, low word */ -+#define CSG_ALLOW_FRAGMENT_HI 0x002C /* () Allowed fragment endpoints, high word */ -+#define CSG_ALLOW_OTHER 0x0030 /* () Allowed other endpoints */ -+#define CSG_EP_REQ 0x0034 /* () Maximum number of endpoints allowed */ -+#define CSG_SUSPEND_BUF_LO 0x0040 /* () Normal mode suspend buffer, low word */ -+#define CSG_SUSPEND_BUF_HI 0x0044 /* () Normal mode suspend buffer, high word */ -+#define CSG_PROTM_SUSPEND_BUF_LO 0x0048 /* () Protected mode suspend buffer, low word */ -+#define CSG_PROTM_SUSPEND_BUF_HI 0x004C /* () Protected mode suspend buffer, high word */ -+#define CSG_CONFIG 0x0050 /* () CSG configuration options */ -+#define CSG_ITER_TRACE_CONFIG 0x0054 /* () CSG trace configuration */ -+ -+/* CSG_OUTPUT_BLOCK register offsets */ -+#define CSG_ACK 0x0000 /* () CSG acknowledge flags */ -+#define CSG_DB_ACK 0x0008 /* () CS kernel doorbell acknowledge flags */ -+#define CSG_IRQ_REQ 0x000C /* () CS interrupt request flags */ -+#define CSG_STATUS_EP_CURRENT 0x0010 /* () Endpoint allocation status register */ -+#define CSG_STATUS_EP_REQ 0x0014 /* () Endpoint request status register */ -+#define CSG_RESOURCE_DEP 0x001C /* () Current resource dependencies */ -+ -+/* GLB_CONTROL_BLOCK register offsets */ -+#define GLB_VERSION 0x0000 /* () Global interface version */ -+#define GLB_FEATURES 0x0004 /* () Global interface features */ -+#define GLB_INPUT_VA 0x0008 /* () Address of GLB_INPUT_BLOCK */ -+#define GLB_OUTPUT_VA 0x000C /* () Address of GLB_OUTPUT_BLOCK */ -+#define GLB_GROUP_NUM 0x0010 /* () Number of CSG interfaces */ -+#define GLB_GROUP_STRIDE 0x0014 /* () Stride between CSG interfaces */ -+#define GLB_PRFCNT_SIZE 0x0018 /* () Size of CSF performance counters */ -+#define GLB_INSTR_FEATURES \ -+ 0x001C /* () TRACE_POINT instrumentation. (csf >= 1.1.0) */ -+#define GROUP_CONTROL_0 0x1000 /* () CSG control and capabilities */ -+#define GROUP_CONTROL(n) (GROUP_CONTROL_0 + (n)*256) -+#define GROUP_CONTROL_REG(n, r) (GROUP_CONTROL(n) + GROUP_CONTROL_BLOCK_REG(r)) -+#define GROUP_CONTROL_COUNT 16 -+ -+/* STREAM_CONTROL_BLOCK register offsets */ -+#define STREAM_FEATURES 0x0000 /* () CSI features */ -+#define STREAM_INPUT_VA 0x0004 /* () Address of CS_KERNEL_INPUT_BLOCK */ -+#define STREAM_OUTPUT_VA 0x0008 /* () Address of CS_KERNEL_OUTPUT_BLOCK */ -+ -+/* GROUP_CONTROL_BLOCK register offsets */ -+#define GROUP_FEATURES 0x0000 /* () CSG interface features */ -+#define GROUP_INPUT_VA 0x0004 /* () Address of CSG_INPUT_BLOCK */ -+#define GROUP_OUTPUT_VA 0x0008 /* () Address of CSG_OUTPUT_BLOCK */ -+#define GROUP_SUSPEND_SIZE 0x000C /* () Size of CSG suspend buffer */ -+#define GROUP_PROTM_SUSPEND_SIZE 0x0010 /* () Size of CSG protected-mode suspend buffer */ -+#define GROUP_STREAM_NUM 0x0014 /* () Number of CS interfaces */ -+#define GROUP_STREAM_STRIDE 0x0018 /* () Stride between CS interfaces */ -+#define STREAM_CONTROL_0 0x0040 /* () CS control and capabilities */ -+#define STREAM_CONTROL(n) (STREAM_CONTROL_0 + (n)*12) -+#define STREAM_CONTROL_REG(n, r) (STREAM_CONTROL(n) + STREAM_CONTROL_BLOCK_REG(r)) -+#define STREAM_CONTROL_COUNT 16 -+ -+/* GLB_INPUT_BLOCK register offsets */ -+#define GLB_REQ 0x0000 /* () Global request */ -+#define GLB_ACK_IRQ_MASK 0x0004 /* () Global acknowledge interrupt mask */ -+#define GLB_DB_REQ 0x0008 /* () Global doorbell request */ -+#define GLB_PROGRESS_TIMER 0x0010 /* () Global progress timeout */ -+#define GLB_PWROFF_TIMER 0x0014 /* () Global shader core power off timer */ -+#define GLB_ALLOC_EN_LO 0x0018 /* () Global shader core allocation enable mask, low word */ -+#define GLB_ALLOC_EN_HI 0x001C /* () Global shader core allocation enable mask, high word */ -+#define GLB_PROTM_COHERENCY 0x0020 /* () Configure COHERENCY_ENABLE register value to use in protected mode execution */ -+ -+#define GLB_PRFCNT_JASID 0x0024 /* () Performance counter address space */ -+#define GLB_PRFCNT_BASE_LO 0x0028 /* () Performance counter buffer address, low word */ -+#define GLB_PRFCNT_BASE_HI 0x002C /* () Performance counter buffer address, high word */ -+#define GLB_PRFCNT_EXTRACT 0x0030 /* () Performance counter buffer extract index */ -+#define GLB_PRFCNT_CONFIG 0x0040 /* () Performance counter configuration */ -+#define GLB_PRFCNT_CSG_SELECT 0x0044 /* () CSG performance counting enable */ -+#define GLB_PRFCNT_FW_EN 0x0048 /* () Performance counter enable for firmware */ -+#define GLB_PRFCNT_CSG_EN 0x004C /* () Performance counter enable for CSG */ -+#define GLB_PRFCNT_CSF_EN 0x0050 /* () Performance counter enable for CSF */ -+#define GLB_PRFCNT_SHADER_EN 0x0054 /* () Performance counter enable for shader cores */ -+#define GLB_PRFCNT_TILER_EN 0x0058 /* () Performance counter enable for tiler */ -+#define GLB_PRFCNT_MMU_L2_EN 0x005C /* () Performance counter enable for MMU/L2 cache */ -+ -+#define GLB_DEBUG_FWUTF_DESTROY 0x0FE0 /* () Test fixture destroy function address */ -+#define GLB_DEBUG_FWUTF_TEST 0x0FE4 /* () Test index */ -+#define GLB_DEBUG_FWUTF_FIXTURE 0x0FE8 /* () Test fixture index */ -+#define GLB_DEBUG_FWUTF_CREATE 0x0FEC /* () Test fixture create function address */ -+#define GLB_DEBUG_ACK_IRQ_MASK 0x0FF8 /* () Global debug acknowledge interrupt mask */ -+#define GLB_DEBUG_REQ 0x0FFC /* () Global debug request */ -+ -+/* GLB_OUTPUT_BLOCK register offsets */ -+#define GLB_ACK 0x0000 /* () Global acknowledge */ -+#define GLB_DB_ACK 0x0008 /* () Global doorbell acknowledge */ -+#define GLB_HALT_STATUS 0x0010 /* () Global halt status */ -+#define GLB_PRFCNT_STATUS 0x0014 /* () Performance counter status */ -+#define GLB_PRFCNT_INSERT 0x0018 /* () Performance counter buffer insert index */ -+#define GLB_DEBUG_FWUTF_RESULT 0x0FE0 /* () Firmware debug test result */ -+#define GLB_DEBUG_ACK 0x0FFC /* () Global debug acknowledge */ -+ -+/* USER register offsets */ -+#define LATEST_FLUSH 0x0000 /* () Flush ID of latest clean-and-invalidate operation */ -+ -+/* End register offsets */ -+ -+/* CS_KERNEL_INPUT_BLOCK register set definitions */ -+/* GLB_VERSION register */ -+#define GLB_VERSION_PATCH_SHIFT (0) -+#define GLB_VERSION_MINOR_SHIFT (16) -+#define GLB_VERSION_MAJOR_SHIFT (24) -+ -+/* CS_REQ register */ -+#define CS_REQ_STATE_SHIFT 0 -+#define CS_REQ_STATE_MASK (0x7 << CS_REQ_STATE_SHIFT) -+#define CS_REQ_STATE_GET(reg_val) (((reg_val)&CS_REQ_STATE_MASK) >> CS_REQ_STATE_SHIFT) -+#define CS_REQ_STATE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_STATE_MASK) | (((value) << CS_REQ_STATE_SHIFT) & CS_REQ_STATE_MASK)) -+/* CS_REQ_STATE values */ -+#define CS_REQ_STATE_STOP 0x0 -+#define CS_REQ_STATE_START 0x1 -+/* End of CS_REQ_STATE values */ -+#define CS_REQ_EXTRACT_EVENT_SHIFT 4 -+#define CS_REQ_EXTRACT_EVENT_MASK (0x1 << CS_REQ_EXTRACT_EVENT_SHIFT) -+#define CS_REQ_EXTRACT_EVENT_GET(reg_val) (((reg_val)&CS_REQ_EXTRACT_EVENT_MASK) >> CS_REQ_EXTRACT_EVENT_SHIFT) -+#define CS_REQ_EXTRACT_EVENT_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_EXTRACT_EVENT_MASK) | (((value) << CS_REQ_EXTRACT_EVENT_SHIFT) & CS_REQ_EXTRACT_EVENT_MASK)) -+ -+#define CS_REQ_IDLE_SYNC_WAIT_SHIFT 8 -+#define CS_REQ_IDLE_SYNC_WAIT_MASK (0x1 << CS_REQ_IDLE_SYNC_WAIT_SHIFT) -+#define CS_REQ_IDLE_SYNC_WAIT_GET(reg_val) (((reg_val)&CS_REQ_IDLE_SYNC_WAIT_MASK) >> CS_REQ_IDLE_SYNC_WAIT_SHIFT) -+#define CS_REQ_IDLE_SYNC_WAIT_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_IDLE_SYNC_WAIT_MASK) | \ -+ (((value) << CS_REQ_IDLE_SYNC_WAIT_SHIFT) & CS_REQ_IDLE_SYNC_WAIT_MASK)) -+#define CS_REQ_IDLE_PROTM_PEND_SHIFT 9 -+#define CS_REQ_IDLE_PROTM_PEND_MASK (0x1 << CS_REQ_IDLE_PROTM_PEND_SHIFT) -+#define CS_REQ_IDLE_PROTM_PEND_GET(reg_val) (((reg_val)&CS_REQ_IDLE_PROTM_PEND_MASK) >> CS_REQ_IDLE_PROTM_PEND_SHIFT) -+#define CS_REQ_IDLE_PROTM_PEND_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_IDLE_PROTM_PEND_MASK) | \ -+ (((value) << CS_REQ_IDLE_PROTM_PEND_SHIFT) & CS_REQ_IDLE_PROTM_PEND_MASK)) -+#define CS_REQ_IDLE_EMPTY_SHIFT 10 -+#define CS_REQ_IDLE_EMPTY_MASK (0x1 << CS_REQ_IDLE_EMPTY_SHIFT) -+#define CS_REQ_IDLE_EMPTY_GET(reg_val) (((reg_val)&CS_REQ_IDLE_EMPTY_MASK) >> CS_REQ_IDLE_EMPTY_SHIFT) -+#define CS_REQ_IDLE_EMPTY_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_IDLE_EMPTY_MASK) | (((value) << CS_REQ_IDLE_EMPTY_SHIFT) & CS_REQ_IDLE_EMPTY_MASK)) -+#define CS_REQ_IDLE_RESOURCE_REQ_SHIFT 11 -+#define CS_REQ_IDLE_RESOURCE_REQ_MASK (0x1 << CS_REQ_IDLE_RESOURCE_REQ_SHIFT) -+#define CS_REQ_IDLE_RESOURCE_REQ_GET(reg_val) \ -+ (((reg_val)&CS_REQ_IDLE_RESOURCE_REQ_MASK) >> CS_REQ_IDLE_RESOURCE_REQ_SHIFT) -+#define CS_REQ_IDLE_RESOURCE_REQ_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_IDLE_RESOURCE_REQ_MASK) | \ -+ (((value) << CS_REQ_IDLE_RESOURCE_REQ_SHIFT) & CS_REQ_IDLE_RESOURCE_REQ_MASK)) -+#define CS_REQ_TILER_OOM_SHIFT 26 -+#define CS_REQ_TILER_OOM_MASK (0x1 << CS_REQ_TILER_OOM_SHIFT) -+#define CS_REQ_TILER_OOM_GET(reg_val) (((reg_val)&CS_REQ_TILER_OOM_MASK) >> CS_REQ_TILER_OOM_SHIFT) -+#define CS_REQ_TILER_OOM_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_TILER_OOM_MASK) | (((value) << CS_REQ_TILER_OOM_SHIFT) & CS_REQ_TILER_OOM_MASK)) -+#define CS_REQ_PROTM_PEND_SHIFT 27 -+#define CS_REQ_PROTM_PEND_MASK (0x1 << CS_REQ_PROTM_PEND_SHIFT) -+#define CS_REQ_PROTM_PEND_GET(reg_val) (((reg_val)&CS_REQ_PROTM_PEND_MASK) >> CS_REQ_PROTM_PEND_SHIFT) -+#define CS_REQ_PROTM_PEND_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_PROTM_PEND_MASK) | (((value) << CS_REQ_PROTM_PEND_SHIFT) & CS_REQ_PROTM_PEND_MASK)) -+#define CS_REQ_FATAL_SHIFT 30 -+#define CS_REQ_FATAL_MASK (0x1 << CS_REQ_FATAL_SHIFT) -+#define CS_REQ_FATAL_GET(reg_val) (((reg_val)&CS_REQ_FATAL_MASK) >> CS_REQ_FATAL_SHIFT) -+#define CS_REQ_FATAL_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_FATAL_MASK) | (((value) << CS_REQ_FATAL_SHIFT) & CS_REQ_FATAL_MASK)) -+#define CS_REQ_FAULT_SHIFT 31 -+#define CS_REQ_FAULT_MASK (0x1 << CS_REQ_FAULT_SHIFT) -+#define CS_REQ_FAULT_GET(reg_val) (((reg_val)&CS_REQ_FAULT_MASK) >> CS_REQ_FAULT_SHIFT) -+#define CS_REQ_FAULT_SET(reg_val, value) \ -+ (((reg_val) & ~CS_REQ_FAULT_MASK) | (((value) << CS_REQ_FAULT_SHIFT) & CS_REQ_FAULT_MASK)) -+ -+/* CS_CONFIG register */ -+#define CS_CONFIG_PRIORITY_SHIFT 0 -+#define CS_CONFIG_PRIORITY_MASK (0xF << CS_CONFIG_PRIORITY_SHIFT) -+#define CS_CONFIG_PRIORITY_GET(reg_val) (((reg_val)&CS_CONFIG_PRIORITY_MASK) >> CS_CONFIG_PRIORITY_SHIFT) -+#define CS_CONFIG_PRIORITY_SET(reg_val, value) \ -+ (((reg_val) & ~CS_CONFIG_PRIORITY_MASK) | (((value) << CS_CONFIG_PRIORITY_SHIFT) & CS_CONFIG_PRIORITY_MASK)) -+#define CS_CONFIG_USER_DOORBELL_SHIFT 8 -+#define CS_CONFIG_USER_DOORBELL_MASK (0xFF << CS_CONFIG_USER_DOORBELL_SHIFT) -+#define CS_CONFIG_USER_DOORBELL_GET(reg_val) (((reg_val)&CS_CONFIG_USER_DOORBELL_MASK) >> CS_CONFIG_USER_DOORBELL_SHIFT) -+#define CS_CONFIG_USER_DOORBELL_SET(reg_val, value) \ -+ (((reg_val) & ~CS_CONFIG_USER_DOORBELL_MASK) | \ -+ (((value) << CS_CONFIG_USER_DOORBELL_SHIFT) & CS_CONFIG_USER_DOORBELL_MASK)) -+ -+/* CS_ACK_IRQ_MASK register */ -+#define CS_ACK_IRQ_MASK_STATE_SHIFT 0 -+#define CS_ACK_IRQ_MASK_STATE_MASK (0x7 << CS_ACK_IRQ_MASK_STATE_SHIFT) -+#define CS_ACK_IRQ_MASK_STATE_GET(reg_val) (((reg_val)&CS_ACK_IRQ_MASK_STATE_MASK) >> CS_ACK_IRQ_MASK_STATE_SHIFT) -+#define CS_ACK_IRQ_MASK_STATE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_IRQ_MASK_STATE_MASK) | \ -+ (((value) << CS_ACK_IRQ_MASK_STATE_SHIFT) & CS_ACK_IRQ_MASK_STATE_MASK)) -+/* CS_ACK_IRQ_MASK_STATE values */ -+#define CS_ACK_IRQ_MASK_STATE_DISABLED 0x0 -+#define CS_ACK_IRQ_MASK_STATE_ENABLED 0x7 -+/* End of CS_ACK_IRQ_MASK_STATE values */ -+#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT 4 -+#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK (0x1 << CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT) -+#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_GET(reg_val) \ -+ (((reg_val)&CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK) >> CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT) -+#define CS_ACK_IRQ_MASK_EXTRACT_EVENT_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK) | \ -+ (((value) << CS_ACK_IRQ_MASK_EXTRACT_EVENT_SHIFT) & CS_ACK_IRQ_MASK_EXTRACT_EVENT_MASK)) -+#define CS_ACK_IRQ_MASK_TILER_OOM_SHIFT 26 -+#define CS_ACK_IRQ_MASK_TILER_OOM_MASK (0x1 << CS_ACK_IRQ_MASK_TILER_OOM_SHIFT) -+#define CS_ACK_IRQ_MASK_TILER_OOM_GET(reg_val) \ -+ (((reg_val)&CS_ACK_IRQ_MASK_TILER_OOM_MASK) >> CS_ACK_IRQ_MASK_TILER_OOM_SHIFT) -+#define CS_ACK_IRQ_MASK_TILER_OOM_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_IRQ_MASK_TILER_OOM_MASK) | \ -+ (((value) << CS_ACK_IRQ_MASK_TILER_OOM_SHIFT) & CS_ACK_IRQ_MASK_TILER_OOM_MASK)) -+#define CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT 27 -+#define CS_ACK_IRQ_MASK_PROTM_PEND_MASK (0x1 << CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT) -+#define CS_ACK_IRQ_MASK_PROTM_PEND_GET(reg_val) \ -+ (((reg_val)&CS_ACK_IRQ_MASK_PROTM_PEND_MASK) >> CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT) -+#define CS_ACK_IRQ_MASK_PROTM_PEND_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_IRQ_MASK_PROTM_PEND_MASK) | \ -+ (((value) << CS_ACK_IRQ_MASK_PROTM_PEND_SHIFT) & CS_ACK_IRQ_MASK_PROTM_PEND_MASK)) -+#define CS_ACK_IRQ_MASK_FATAL_SHIFT 30 -+#define CS_ACK_IRQ_MASK_FATAL_MASK (0x1 << CS_ACK_IRQ_MASK_FATAL_SHIFT) -+#define CS_ACK_IRQ_MASK_FATAL_GET(reg_val) (((reg_val)&CS_ACK_IRQ_MASK_FATAL_MASK) >> CS_ACK_IRQ_MASK_FATAL_SHIFT) -+#define CS_ACK_IRQ_MASK_FATAL_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_IRQ_MASK_FATAL_MASK) | \ -+ (((value) << CS_ACK_IRQ_MASK_FATAL_SHIFT) & CS_ACK_IRQ_MASK_FATAL_MASK)) -+#define CS_ACK_IRQ_MASK_FAULT_SHIFT 31 -+#define CS_ACK_IRQ_MASK_FAULT_MASK (0x1 << CS_ACK_IRQ_MASK_FAULT_SHIFT) -+#define CS_ACK_IRQ_MASK_FAULT_GET(reg_val) (((reg_val)&CS_ACK_IRQ_MASK_FAULT_MASK) >> CS_ACK_IRQ_MASK_FAULT_SHIFT) -+#define CS_ACK_IRQ_MASK_FAULT_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_IRQ_MASK_FAULT_MASK) | \ -+ (((value) << CS_ACK_IRQ_MASK_FAULT_SHIFT) & CS_ACK_IRQ_MASK_FAULT_MASK)) -+ -+/* CS_BASE register */ -+#define CS_BASE_POINTER_SHIFT 0 -+#define CS_BASE_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_BASE_POINTER_SHIFT) -+#define CS_BASE_POINTER_GET(reg_val) (((reg_val)&CS_BASE_POINTER_MASK) >> CS_BASE_POINTER_SHIFT) -+#define CS_BASE_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_BASE_POINTER_MASK) | (((value) << CS_BASE_POINTER_SHIFT) & CS_BASE_POINTER_MASK)) -+ -+/* CS_SIZE register */ -+#define CS_SIZE_SIZE_SHIFT 0 -+#define CS_SIZE_SIZE_MASK (0xFFFFFFFF << CS_SIZE_SIZE_SHIFT) -+#define CS_SIZE_SIZE_GET(reg_val) (((reg_val)&CS_SIZE_SIZE_MASK) >> CS_SIZE_SIZE_SHIFT) -+#define CS_SIZE_SIZE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_SIZE_SIZE_MASK) | (((value) << CS_SIZE_SIZE_SHIFT) & CS_SIZE_SIZE_MASK)) -+ -+/* CS_TILER_HEAP_START register */ -+#define CS_TILER_HEAP_START_POINTER_SHIFT 0 -+#define CS_TILER_HEAP_START_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_TILER_HEAP_START_POINTER_SHIFT) -+#define CS_TILER_HEAP_START_POINTER_GET(reg_val) \ -+ (((reg_val)&CS_TILER_HEAP_START_POINTER_MASK) >> CS_TILER_HEAP_START_POINTER_SHIFT) -+#define CS_TILER_HEAP_START_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_TILER_HEAP_START_POINTER_MASK) | \ -+ (((value) << CS_TILER_HEAP_START_POINTER_SHIFT) & CS_TILER_HEAP_START_POINTER_MASK)) -+/* HeapChunkPointer nested in CS_TILER_HEAP_START_POINTER */ -+/* End of HeapChunkPointer nested in CS_TILER_HEAP_START_POINTER */ -+ -+/* CS_TILER_HEAP_END register */ -+#define CS_TILER_HEAP_END_POINTER_SHIFT 0 -+#define CS_TILER_HEAP_END_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_TILER_HEAP_END_POINTER_SHIFT) -+#define CS_TILER_HEAP_END_POINTER_GET(reg_val) \ -+ (((reg_val)&CS_TILER_HEAP_END_POINTER_MASK) >> CS_TILER_HEAP_END_POINTER_SHIFT) -+#define CS_TILER_HEAP_END_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_TILER_HEAP_END_POINTER_MASK) | \ -+ (((value) << CS_TILER_HEAP_END_POINTER_SHIFT) & CS_TILER_HEAP_END_POINTER_MASK)) -+/* HeapChunkPointer nested in CS_TILER_HEAP_END_POINTER */ -+/* End of HeapChunkPointer nested in CS_TILER_HEAP_END_POINTER */ -+ -+/* CS_USER_INPUT register */ -+#define CS_USER_INPUT_POINTER_SHIFT 0 -+#define CS_USER_INPUT_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_USER_INPUT_POINTER_SHIFT) -+#define CS_USER_INPUT_POINTER_GET(reg_val) (((reg_val)&CS_USER_INPUT_POINTER_MASK) >> CS_USER_INPUT_POINTER_SHIFT) -+#define CS_USER_INPUT_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_USER_INPUT_POINTER_MASK) | \ -+ (((value) << CS_USER_INPUT_POINTER_SHIFT) & CS_USER_INPUT_POINTER_MASK)) -+ -+/* CS_USER_OUTPUT register */ -+#define CS_USER_OUTPUT_POINTER_SHIFT 0 -+#define CS_USER_OUTPUT_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_USER_OUTPUT_POINTER_SHIFT) -+#define CS_USER_OUTPUT_POINTER_GET(reg_val) (((reg_val)&CS_USER_OUTPUT_POINTER_MASK) >> CS_USER_OUTPUT_POINTER_SHIFT) -+#define CS_USER_OUTPUT_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_USER_OUTPUT_POINTER_MASK) | \ -+ (((value) << CS_USER_OUTPUT_POINTER_SHIFT) & CS_USER_OUTPUT_POINTER_MASK)) -+ -+/* CS_INSTR_CONFIG register */ -+#define CS_INSTR_CONFIG_JASID_SHIFT (0) -+#define CS_INSTR_CONFIG_JASID_MASK ((u32)0xF << CS_INSTR_CONFIG_JASID_SHIFT) -+#define CS_INSTR_CONFIG_JASID_GET(reg_val) (((reg_val)&CS_INSTR_CONFIG_JASID_MASK) >> CS_INSTR_CONFIG_JASID_SHIFT) -+#define CS_INSTR_CONFIG_JASID_SET(reg_val, value) \ -+ (((reg_val) & ~CS_INSTR_CONFIG_JASID_MASK) | \ -+ (((value) << CS_INSTR_CONFIG_JASID_SHIFT) & CS_INSTR_CONFIG_JASID_MASK)) -+#define CS_INSTR_CONFIG_EVENT_SIZE_SHIFT (4) -+#define CS_INSTR_CONFIG_EVENT_SIZE_MASK ((u32)0xF << CS_INSTR_CONFIG_EVENT_SIZE_SHIFT) -+#define CS_INSTR_CONFIG_EVENT_SIZE_GET(reg_val) \ -+ (((reg_val)&CS_INSTR_CONFIG_EVENT_SIZE_MASK) >> CS_INSTR_CONFIG_EVENT_SIZE_SHIFT) -+#define CS_INSTR_CONFIG_EVENT_SIZE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_INSTR_CONFIG_EVENT_SIZE_MASK) | \ -+ (((value) << CS_INSTR_CONFIG_EVENT_SIZE_SHIFT) & CS_INSTR_CONFIG_EVENT_SIZE_MASK)) -+#define CS_INSTR_CONFIG_EVENT_STATE_SHIFT (16) -+#define CS_INSTR_CONFIG_EVENT_STATE_MASK ((u32)0xFF << CS_INSTR_CONFIG_EVENT_STATE_SHIFT) -+#define CS_INSTR_CONFIG_EVENT_STATE_GET(reg_val) \ -+ (((reg_val)&CS_INSTR_CONFIG_EVENT_STATE_MASK) >> CS_INSTR_CONFIG_EVENT_STATE_SHIFT) -+#define CS_INSTR_CONFIG_EVENT_STATE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_INSTR_CONFIG_EVENT_STATE_MASK) | \ -+ (((value) << CS_INSTR_CONFIG_EVENT_STATE_SHIFT) & CS_INSTR_CONFIG_EVENT_STATE_MASK)) -+ -+/* CS_INSTR_BUFFER_SIZE register */ -+#define CS_INSTR_BUFFER_SIZE_SIZE_SHIFT (0) -+#define CS_INSTR_BUFFER_SIZE_SIZE_MASK ((u32)0xFFFFFFFF << CS_INSTR_BUFFER_SIZE_SIZE_SHIFT) -+#define CS_INSTR_BUFFER_SIZE_SIZE_GET(reg_val) \ -+ (((reg_val)&CS_INSTR_BUFFER_SIZE_SIZE_MASK) >> CS_INSTR_BUFFER_SIZE_SIZE_SHIFT) -+#define CS_INSTR_BUFFER_SIZE_SIZE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_INSTR_BUFFER_SIZE_SIZE_MASK) | \ -+ (((value) << CS_INSTR_BUFFER_SIZE_SIZE_SHIFT) & CS_INSTR_BUFFER_SIZE_SIZE_MASK)) -+ -+/* CS_INSTR_BUFFER_BASE register */ -+#define CS_INSTR_BUFFER_BASE_POINTER_SHIFT (0) -+#define CS_INSTR_BUFFER_BASE_POINTER_MASK ((u64)0xFFFFFFFFFFFFFFFF << CS_INSTR_BUFFER_BASE_POINTER_SHIFT) -+#define CS_INSTR_BUFFER_BASE_POINTER_GET(reg_val) \ -+ (((reg_val)&CS_INSTR_BUFFER_BASE_POINTER_MASK) >> CS_INSTR_BUFFER_BASE_POINTER_SHIFT) -+#define CS_INSTR_BUFFER_BASE_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_INSTR_BUFFER_BASE_POINTER_MASK) | \ -+ (((value) << CS_INSTR_BUFFER_BASE_POINTER_SHIFT) & CS_INSTR_BUFFER_BASE_POINTER_MASK)) -+ -+/* CS_INSTR_BUFFER_OFFSET_POINTER register */ -+#define CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_SHIFT (0) -+#define CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_MASK \ -+ ((u64)0xFFFFFFFFFFFFFFFF) << CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_SHIFT) -+#define CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_GET(reg_val) \ -+ (((reg_val)&CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_MASK) >> CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_SHIFT) -+#define CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_MASK) | \ -+ (((value) << CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_SHIFT) & CS_INSTR_BUFFER_OFFSET_POINTER_POINTER_MASK)) -+ -+/* End of CS_KERNEL_INPUT_BLOCK register set definitions */ -+ -+/* CS_KERNEL_OUTPUT_BLOCK register set definitions */ -+ -+/* CS_ACK register */ -+#define CS_ACK_STATE_SHIFT 0 -+#define CS_ACK_STATE_MASK (0x7 << CS_ACK_STATE_SHIFT) -+#define CS_ACK_STATE_GET(reg_val) (((reg_val)&CS_ACK_STATE_MASK) >> CS_ACK_STATE_SHIFT) -+#define CS_ACK_STATE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_STATE_MASK) | (((value) << CS_ACK_STATE_SHIFT) & CS_ACK_STATE_MASK)) -+/* CS_ACK_STATE values */ -+#define CS_ACK_STATE_STOP 0x0 -+#define CS_ACK_STATE_START 0x1 -+/* End of CS_ACK_STATE values */ -+#define CS_ACK_EXTRACT_EVENT_SHIFT 4 -+#define CS_ACK_EXTRACT_EVENT_MASK (0x1 << CS_ACK_EXTRACT_EVENT_SHIFT) -+#define CS_ACK_EXTRACT_EVENT_GET(reg_val) (((reg_val)&CS_ACK_EXTRACT_EVENT_MASK) >> CS_ACK_EXTRACT_EVENT_SHIFT) -+#define CS_ACK_EXTRACT_EVENT_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_EXTRACT_EVENT_MASK) | (((value) << CS_ACK_EXTRACT_EVENT_SHIFT) & CS_ACK_EXTRACT_EVENT_MASK)) -+#define CS_ACK_TILER_OOM_SHIFT 26 -+#define CS_ACK_TILER_OOM_MASK (0x1 << CS_ACK_TILER_OOM_SHIFT) -+#define CS_ACK_TILER_OOM_GET(reg_val) (((reg_val)&CS_ACK_TILER_OOM_MASK) >> CS_ACK_TILER_OOM_SHIFT) -+#define CS_ACK_TILER_OOM_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_TILER_OOM_MASK) | (((value) << CS_ACK_TILER_OOM_SHIFT) & CS_ACK_TILER_OOM_MASK)) -+#define CS_ACK_PROTM_PEND_SHIFT 27 -+#define CS_ACK_PROTM_PEND_MASK (0x1 << CS_ACK_PROTM_PEND_SHIFT) -+#define CS_ACK_PROTM_PEND_GET(reg_val) (((reg_val)&CS_ACK_PROTM_PEND_MASK) >> CS_ACK_PROTM_PEND_SHIFT) -+#define CS_ACK_PROTM_PEND_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_PROTM_PEND_MASK) | (((value) << CS_ACK_PROTM_PEND_SHIFT) & CS_ACK_PROTM_PEND_MASK)) -+#define CS_ACK_FATAL_SHIFT 30 -+#define CS_ACK_FATAL_MASK (0x1 << CS_ACK_FATAL_SHIFT) -+#define CS_ACK_FATAL_GET(reg_val) (((reg_val)&CS_ACK_FATAL_MASK) >> CS_ACK_FATAL_SHIFT) -+#define CS_ACK_FATAL_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_FATAL_MASK) | (((value) << CS_ACK_FATAL_SHIFT) & CS_ACK_FATAL_MASK)) -+#define CS_ACK_FAULT_SHIFT 31 -+#define CS_ACK_FAULT_MASK (0x1 << CS_ACK_FAULT_SHIFT) -+#define CS_ACK_FAULT_GET(reg_val) (((reg_val)&CS_ACK_FAULT_MASK) >> CS_ACK_FAULT_SHIFT) -+#define CS_ACK_FAULT_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACK_FAULT_MASK) | (((value) << CS_ACK_FAULT_SHIFT) & CS_ACK_FAULT_MASK)) -+ -+/* CS_STATUS_CMD_PTR register */ -+#define CS_STATUS_CMD_PTR_POINTER_SHIFT 0 -+#define CS_STATUS_CMD_PTR_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_STATUS_CMD_PTR_POINTER_SHIFT) -+#define CS_STATUS_CMD_PTR_POINTER_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_CMD_PTR_POINTER_MASK) >> CS_STATUS_CMD_PTR_POINTER_SHIFT) -+#define CS_STATUS_CMD_PTR_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_CMD_PTR_POINTER_MASK) | \ -+ (((value) << CS_STATUS_CMD_PTR_POINTER_SHIFT) & CS_STATUS_CMD_PTR_POINTER_MASK)) -+ -+/* CS_STATUS_WAIT register */ -+#define CS_STATUS_WAIT_SB_MASK_SHIFT 0 -+#define CS_STATUS_WAIT_SB_MASK_MASK (0xFFFF << CS_STATUS_WAIT_SB_MASK_SHIFT) -+#define CS_STATUS_WAIT_SB_MASK_GET(reg_val) (((reg_val)&CS_STATUS_WAIT_SB_MASK_MASK) >> CS_STATUS_WAIT_SB_MASK_SHIFT) -+#define CS_STATUS_WAIT_SB_MASK_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_WAIT_SB_MASK_MASK) | \ -+ (((value) << CS_STATUS_WAIT_SB_MASK_SHIFT) & CS_STATUS_WAIT_SB_MASK_MASK)) -+#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT 24 -+#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK (0xF << CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) -+#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK) >> CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) -+#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK) | \ -+ (((value) << CS_STATUS_WAIT_SYNC_WAIT_CONDITION_SHIFT) & CS_STATUS_WAIT_SYNC_WAIT_CONDITION_MASK)) -+/* CS_STATUS_WAIT_SYNC_WAIT_CONDITION values */ -+#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_LE 0x0 -+#define CS_STATUS_WAIT_SYNC_WAIT_CONDITION_GT 0x1 -+/* End of CS_STATUS_WAIT_SYNC_WAIT_CONDITION values */ -+#define CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT 28 -+#define CS_STATUS_WAIT_PROGRESS_WAIT_MASK (0x1 << CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) -+#define CS_STATUS_WAIT_PROGRESS_WAIT_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_WAIT_PROGRESS_WAIT_MASK) >> CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) -+#define CS_STATUS_WAIT_PROGRESS_WAIT_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_WAIT_PROGRESS_WAIT_MASK) | \ -+ (((value) << CS_STATUS_WAIT_PROGRESS_WAIT_SHIFT) & CS_STATUS_WAIT_PROGRESS_WAIT_MASK)) -+#define CS_STATUS_WAIT_PROTM_PEND_SHIFT 29 -+#define CS_STATUS_WAIT_PROTM_PEND_MASK (0x1 << CS_STATUS_WAIT_PROTM_PEND_SHIFT) -+#define CS_STATUS_WAIT_PROTM_PEND_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_WAIT_PROTM_PEND_MASK) >> CS_STATUS_WAIT_PROTM_PEND_SHIFT) -+#define CS_STATUS_WAIT_PROTM_PEND_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_WAIT_PROTM_PEND_MASK) | \ -+ (((value) << CS_STATUS_WAIT_PROTM_PEND_SHIFT) & CS_STATUS_WAIT_PROTM_PEND_MASK)) -+#define CS_STATUS_WAIT_SYNC_WAIT_SHIFT 31 -+#define CS_STATUS_WAIT_SYNC_WAIT_MASK (0x1 << CS_STATUS_WAIT_SYNC_WAIT_SHIFT) -+#define CS_STATUS_WAIT_SYNC_WAIT_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_WAIT_SYNC_WAIT_MASK) >> CS_STATUS_WAIT_SYNC_WAIT_SHIFT) -+#define CS_STATUS_WAIT_SYNC_WAIT_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_WAIT_SYNC_WAIT_MASK) | \ -+ (((value) << CS_STATUS_WAIT_SYNC_WAIT_SHIFT) & CS_STATUS_WAIT_SYNC_WAIT_MASK)) -+ -+/* CS_STATUS_REQ_RESOURCE register */ -+#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT 0 -+#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT) -+#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT) -+#define CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK) | \ -+ (((value) << CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_COMPUTE_RESOURCES_MASK)) -+#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT 1 -+#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT) -+#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT) -+#define CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK) | \ -+ (((value) << CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_FRAGMENT_RESOURCES_MASK)) -+#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT 2 -+#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT) -+#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT) -+#define CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK) | \ -+ (((value) << CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_TILER_RESOURCES_MASK)) -+#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT 3 -+#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK (0x1 << CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT) -+#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK) >> CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT) -+#define CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK) | \ -+ (((value) << CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_SHIFT) & CS_STATUS_REQ_RESOURCE_IDVS_RESOURCES_MASK)) -+ -+/* CS_STATUS_WAIT_SYNC_POINTER register */ -+#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT 0 -+#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT) -+#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK) >> CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT) -+#define CS_STATUS_WAIT_SYNC_POINTER_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK) | \ -+ (((value) << CS_STATUS_WAIT_SYNC_POINTER_POINTER_SHIFT) & CS_STATUS_WAIT_SYNC_POINTER_POINTER_MASK)) -+ -+/* CS_STATUS_WAIT_SYNC_VALUE register */ -+#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT 0 -+#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK (0xFFFFFFFF << CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) -+#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK) >> CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) -+#define CS_STATUS_WAIT_SYNC_VALUE_VALUE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK) | \ -+ (((value) << CS_STATUS_WAIT_SYNC_VALUE_VALUE_SHIFT) & CS_STATUS_WAIT_SYNC_VALUE_VALUE_MASK)) -+ -+/* CS_STATUS_SCOREBOARDS register */ -+#define CS_STATUS_SCOREBOARDS_NONZERO_SHIFT (0) -+#define CS_STATUS_SCOREBOARDS_NONZERO_MASK \ -+ ((0xFFFF) << CS_STATUS_SCOREBOARDS_NONZERO_SHIFT) -+#define CS_STATUS_SCOREBOARDS_NONZERO_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_SCOREBOARDS_NONZERO_MASK) >> \ -+ CS_STATUS_SCOREBOARDS_NONZERO_SHIFT) -+#define CS_STATUS_SCOREBOARDS_NONZERO_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_SCOREBOARDS_NONZERO_MASK) | \ -+ (((value) << CS_STATUS_SCOREBOARDS_NONZERO_SHIFT) & \ -+ CS_STATUS_SCOREBOARDS_NONZERO_MASK)) -+ -+/* CS_STATUS_BLOCKED_REASON register */ -+#define CS_STATUS_BLOCKED_REASON_REASON_SHIFT (0) -+#define CS_STATUS_BLOCKED_REASON_REASON_MASK \ -+ ((0xF) << CS_STATUS_BLOCKED_REASON_REASON_SHIFT) -+#define CS_STATUS_BLOCKED_REASON_REASON_GET(reg_val) \ -+ (((reg_val)&CS_STATUS_BLOCKED_REASON_REASON_MASK) >> \ -+ CS_STATUS_BLOCKED_REASON_REASON_SHIFT) -+#define CS_STATUS_BLOCKED_REASON_REASON_SET(reg_val, value) \ -+ (((reg_val) & ~CS_STATUS_BLOCKED_REASON_REASON_MASK) | \ -+ (((value) << CS_STATUS_BLOCKED_REASON_REASON_SHIFT) & \ -+ CS_STATUS_BLOCKED_REASON_REASON_MASK)) -+/* CS_STATUS_BLOCKED_REASON_reason values */ -+#define CS_STATUS_BLOCKED_REASON_REASON_UNBLOCKED 0x0 -+#define CS_STATUS_BLOCKED_REASON_REASON_WAIT 0x1 -+#define CS_STATUS_BLOCKED_REASON_REASON_PROGRESS_WAIT 0x2 -+#define CS_STATUS_BLOCKED_REASON_REASON_SYNC_WAIT 0x3 -+#define CS_STATUS_BLOCKED_REASON_REASON_DEFERRED 0x4 -+#define CS_STATUS_BLOCKED_REASON_REASON_RESOURCE 0x5 -+#define CS_STATUS_BLOCKED_REASON_REASON_FLUSH 0x6 -+/* End of CS_STATUS_BLOCKED_REASON_reason values */ -+ -+/* CS_FAULT register */ -+#define CS_FAULT_EXCEPTION_TYPE_SHIFT 0 -+#define CS_FAULT_EXCEPTION_TYPE_MASK (0xFF << CS_FAULT_EXCEPTION_TYPE_SHIFT) -+#define CS_FAULT_EXCEPTION_TYPE_GET(reg_val) (((reg_val)&CS_FAULT_EXCEPTION_TYPE_MASK) >> CS_FAULT_EXCEPTION_TYPE_SHIFT) -+#define CS_FAULT_EXCEPTION_TYPE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_FAULT_EXCEPTION_TYPE_MASK) | \ -+ (((value) << CS_FAULT_EXCEPTION_TYPE_SHIFT) & CS_FAULT_EXCEPTION_TYPE_MASK)) -+/* CS_FAULT_EXCEPTION_TYPE values */ -+#define CS_FAULT_EXCEPTION_TYPE_CS_RESOURCE_TERMINATED 0x0F -+#define CS_FAULT_EXCEPTION_TYPE_CS_INHERIT_FAULT 0x4B -+#define CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_PC 0x50 -+#define CS_FAULT_EXCEPTION_TYPE_INSTR_INVALID_ENC 0x51 -+#define CS_FAULT_EXCEPTION_TYPE_INSTR_BARRIER_FAULT 0x55 -+#define CS_FAULT_EXCEPTION_TYPE_DATA_INVALID_FAULT 0x58 -+#define CS_FAULT_EXCEPTION_TYPE_TILE_RANGE_FAULT 0x59 -+#define CS_FAULT_EXCEPTION_TYPE_ADDR_RANGE_FAULT 0x5A -+#define CS_FAULT_EXCEPTION_TYPE_IMPRECISE_FAULT 0x5B -+#define CS_FAULT_EXCEPTION_TYPE_RESOURCE_EVICTION_TIMEOUT 0x69 -+/* End of CS_FAULT_EXCEPTION_TYPE values */ -+#define CS_FAULT_EXCEPTION_DATA_SHIFT 8 -+#define CS_FAULT_EXCEPTION_DATA_MASK (0xFFFFFF << CS_FAULT_EXCEPTION_DATA_SHIFT) -+#define CS_FAULT_EXCEPTION_DATA_GET(reg_val) (((reg_val)&CS_FAULT_EXCEPTION_DATA_MASK) >> CS_FAULT_EXCEPTION_DATA_SHIFT) -+#define CS_FAULT_EXCEPTION_DATA_SET(reg_val, value) \ -+ (((reg_val) & ~CS_FAULT_EXCEPTION_DATA_MASK) | \ -+ (((value) << CS_FAULT_EXCEPTION_DATA_SHIFT) & CS_FAULT_EXCEPTION_DATA_MASK)) -+ -+/* CS_FATAL register */ -+#define CS_FATAL_EXCEPTION_TYPE_SHIFT 0 -+#define CS_FATAL_EXCEPTION_TYPE_MASK (0xFF << CS_FATAL_EXCEPTION_TYPE_SHIFT) -+#define CS_FATAL_EXCEPTION_TYPE_GET(reg_val) (((reg_val)&CS_FATAL_EXCEPTION_TYPE_MASK) >> CS_FATAL_EXCEPTION_TYPE_SHIFT) -+#define CS_FATAL_EXCEPTION_TYPE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_FATAL_EXCEPTION_TYPE_MASK) | \ -+ (((value) << CS_FATAL_EXCEPTION_TYPE_SHIFT) & CS_FATAL_EXCEPTION_TYPE_MASK)) -+/* CS_FATAL_EXCEPTION_TYPE values */ -+#define CS_FATAL_EXCEPTION_TYPE_CS_CONFIG_FAULT 0x40 -+#define CS_FATAL_EXCEPTION_TYPE_CS_ENDPOINT_FAULT 0x44 -+#define CS_FATAL_EXCEPTION_TYPE_CS_BUS_FAULT 0x48 -+#define CS_FATAL_EXCEPTION_TYPE_CS_INVALID_INSTRUCTION 0x49 -+#define CS_FATAL_EXCEPTION_TYPE_CS_CALL_STACK_OVERFLOW 0x4A -+#define CS_FATAL_EXCEPTION_TYPE_FIRMWARE_INTERNAL_ERROR 0x68 -+/* End of CS_FATAL_EXCEPTION_TYPE values */ -+#define CS_FATAL_EXCEPTION_DATA_SHIFT 8 -+#define CS_FATAL_EXCEPTION_DATA_MASK (0xFFFFFF << CS_FATAL_EXCEPTION_DATA_SHIFT) -+#define CS_FATAL_EXCEPTION_DATA_GET(reg_val) (((reg_val)&CS_FATAL_EXCEPTION_DATA_MASK) >> CS_FATAL_EXCEPTION_DATA_SHIFT) -+#define CS_FATAL_EXCEPTION_DATA_SET(reg_val, value) \ -+ (((reg_val) & ~CS_FATAL_EXCEPTION_DATA_MASK) | \ -+ (((value) << CS_FATAL_EXCEPTION_DATA_SHIFT) & CS_FATAL_EXCEPTION_DATA_MASK)) -+ -+/* CS_FAULT_INFO register */ -+#define CS_FAULT_INFO_EXCEPTION_DATA_SHIFT 0 -+#define CS_FAULT_INFO_EXCEPTION_DATA_MASK (0xFFFFFFFFFFFFFFFF << CS_FAULT_INFO_EXCEPTION_DATA_SHIFT) -+#define CS_FAULT_INFO_EXCEPTION_DATA_GET(reg_val) \ -+ (((reg_val)&CS_FAULT_INFO_EXCEPTION_DATA_MASK) >> CS_FAULT_INFO_EXCEPTION_DATA_SHIFT) -+#define CS_FAULT_INFO_EXCEPTION_DATA_SET(reg_val, value) \ -+ (((reg_val) & ~CS_FAULT_INFO_EXCEPTION_DATA_MASK) | \ -+ (((value) << CS_FAULT_INFO_EXCEPTION_DATA_SHIFT) & CS_FAULT_INFO_EXCEPTION_DATA_MASK)) -+ -+/* CS_FATAL_INFO register */ -+#define CS_FATAL_INFO_EXCEPTION_DATA_SHIFT 0 -+#define CS_FATAL_INFO_EXCEPTION_DATA_MASK (0xFFFFFFFFFFFFFFFF << CS_FATAL_INFO_EXCEPTION_DATA_SHIFT) -+#define CS_FATAL_INFO_EXCEPTION_DATA_GET(reg_val) \ -+ (((reg_val)&CS_FATAL_INFO_EXCEPTION_DATA_MASK) >> CS_FATAL_INFO_EXCEPTION_DATA_SHIFT) -+#define CS_FATAL_INFO_EXCEPTION_DATA_SET(reg_val, value) \ -+ (((reg_val) & ~CS_FATAL_INFO_EXCEPTION_DATA_MASK) | \ -+ (((value) << CS_FATAL_INFO_EXCEPTION_DATA_SHIFT) & CS_FATAL_INFO_EXCEPTION_DATA_MASK)) -+ -+/* CS_HEAP_VT_START register */ -+#define CS_HEAP_VT_START_VALUE_SHIFT 0 -+#define CS_HEAP_VT_START_VALUE_MASK (0xFFFFFFFF << CS_HEAP_VT_START_VALUE_SHIFT) -+#define CS_HEAP_VT_START_VALUE_GET(reg_val) (((reg_val)&CS_HEAP_VT_START_VALUE_MASK) >> CS_HEAP_VT_START_VALUE_SHIFT) -+#define CS_HEAP_VT_START_VALUE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_HEAP_VT_START_VALUE_MASK) | \ -+ (((value) << CS_HEAP_VT_START_VALUE_SHIFT) & CS_HEAP_VT_START_VALUE_MASK)) -+ -+/* CS_HEAP_VT_END register */ -+#define CS_HEAP_VT_END_VALUE_SHIFT 0 -+#define CS_HEAP_VT_END_VALUE_MASK (0xFFFFFFFF << CS_HEAP_VT_END_VALUE_SHIFT) -+#define CS_HEAP_VT_END_VALUE_GET(reg_val) (((reg_val)&CS_HEAP_VT_END_VALUE_MASK) >> CS_HEAP_VT_END_VALUE_SHIFT) -+#define CS_HEAP_VT_END_VALUE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_HEAP_VT_END_VALUE_MASK) | (((value) << CS_HEAP_VT_END_VALUE_SHIFT) & CS_HEAP_VT_END_VALUE_MASK)) -+ -+/* CS_HEAP_FRAG_END register */ -+#define CS_HEAP_FRAG_END_VALUE_SHIFT 0 -+#define CS_HEAP_FRAG_END_VALUE_MASK (0xFFFFFFFF << CS_HEAP_FRAG_END_VALUE_SHIFT) -+#define CS_HEAP_FRAG_END_VALUE_GET(reg_val) (((reg_val)&CS_HEAP_FRAG_END_VALUE_MASK) >> CS_HEAP_FRAG_END_VALUE_SHIFT) -+#define CS_HEAP_FRAG_END_VALUE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_HEAP_FRAG_END_VALUE_MASK) | \ -+ (((value) << CS_HEAP_FRAG_END_VALUE_SHIFT) & CS_HEAP_FRAG_END_VALUE_MASK)) -+ -+/* CS_HEAP_ADDRESS register */ -+#define CS_HEAP_ADDRESS_POINTER_SHIFT 0 -+#define CS_HEAP_ADDRESS_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CS_HEAP_ADDRESS_POINTER_SHIFT) -+#define CS_HEAP_ADDRESS_POINTER_GET(reg_val) (((reg_val)&CS_HEAP_ADDRESS_POINTER_MASK) >> CS_HEAP_ADDRESS_POINTER_SHIFT) -+#define CS_HEAP_ADDRESS_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CS_HEAP_ADDRESS_POINTER_MASK) | \ -+ (((value) << CS_HEAP_ADDRESS_POINTER_SHIFT) & CS_HEAP_ADDRESS_POINTER_MASK)) -+/* End of CS_KERNEL_OUTPUT_BLOCK register set definitions */ -+ -+/* CS_USER_INPUT_BLOCK register set definitions */ -+ -+/* CS_INSERT register */ -+#define CS_INSERT_VALUE_SHIFT 0 -+#define CS_INSERT_VALUE_MASK (0xFFFFFFFFFFFFFFFF << CS_INSERT_VALUE_SHIFT) -+#define CS_INSERT_VALUE_GET(reg_val) (((reg_val)&CS_INSERT_VALUE_MASK) >> CS_INSERT_VALUE_SHIFT) -+#define CS_INSERT_VALUE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_INSERT_VALUE_MASK) | (((value) << CS_INSERT_VALUE_SHIFT) & CS_INSERT_VALUE_MASK)) -+ -+/* CS_EXTRACT_INIT register */ -+#define CS_EXTRACT_INIT_VALUE_SHIFT 0 -+#define CS_EXTRACT_INIT_VALUE_MASK (0xFFFFFFFFFFFFFFFF << CS_EXTRACT_INIT_VALUE_SHIFT) -+#define CS_EXTRACT_INIT_VALUE_GET(reg_val) (((reg_val)&CS_EXTRACT_INIT_VALUE_MASK) >> CS_EXTRACT_INIT_VALUE_SHIFT) -+#define CS_EXTRACT_INIT_VALUE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_EXTRACT_INIT_VALUE_MASK) | \ -+ (((value) << CS_EXTRACT_INIT_VALUE_SHIFT) & CS_EXTRACT_INIT_VALUE_MASK)) -+/* End of CS_USER_INPUT_BLOCK register set definitions */ -+ -+/* CS_USER_OUTPUT_BLOCK register set definitions */ -+ -+/* CS_EXTRACT register */ -+#define CS_EXTRACT_VALUE_SHIFT 0 -+#define CS_EXTRACT_VALUE_MASK (0xFFFFFFFFFFFFFFFF << CS_EXTRACT_VALUE_SHIFT) -+#define CS_EXTRACT_VALUE_GET(reg_val) (((reg_val)&CS_EXTRACT_VALUE_MASK) >> CS_EXTRACT_VALUE_SHIFT) -+#define CS_EXTRACT_VALUE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_EXTRACT_VALUE_MASK) | (((value) << CS_EXTRACT_VALUE_SHIFT) & CS_EXTRACT_VALUE_MASK)) -+ -+/* CS_ACTIVE register */ -+#define CS_ACTIVE_HW_ACTIVE_SHIFT 0 -+#define CS_ACTIVE_HW_ACTIVE_MASK (0x1 << CS_ACTIVE_HW_ACTIVE_SHIFT) -+#define CS_ACTIVE_HW_ACTIVE_GET(reg_val) (((reg_val)&CS_ACTIVE_HW_ACTIVE_MASK) >> CS_ACTIVE_HW_ACTIVE_SHIFT) -+#define CS_ACTIVE_HW_ACTIVE_SET(reg_val, value) \ -+ (((reg_val) & ~CS_ACTIVE_HW_ACTIVE_MASK) | (((value) << CS_ACTIVE_HW_ACTIVE_SHIFT) & CS_ACTIVE_HW_ACTIVE_MASK)) -+/* End of CS_USER_OUTPUT_BLOCK register set definitions */ -+ -+/* CSG_INPUT_BLOCK register set definitions */ -+ -+/* CSG_REQ register */ -+#define CSG_REQ_STATE_SHIFT 0 -+#define CSG_REQ_STATE_MASK (0x7 << CSG_REQ_STATE_SHIFT) -+#define CSG_REQ_STATE_GET(reg_val) (((reg_val)&CSG_REQ_STATE_MASK) >> CSG_REQ_STATE_SHIFT) -+#define CSG_REQ_STATE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_REQ_STATE_MASK) | (((value) << CSG_REQ_STATE_SHIFT) & CSG_REQ_STATE_MASK)) -+/* CSG_REQ_STATE values */ -+#define CSG_REQ_STATE_TERMINATE 0x0 -+#define CSG_REQ_STATE_START 0x1 -+#define CSG_REQ_STATE_SUSPEND 0x2 -+#define CSG_REQ_STATE_RESUME 0x3 -+/* End of CSG_REQ_STATE values */ -+#define CSG_REQ_EP_CFG_SHIFT 4 -+#define CSG_REQ_EP_CFG_MASK (0x1 << CSG_REQ_EP_CFG_SHIFT) -+#define CSG_REQ_EP_CFG_GET(reg_val) (((reg_val)&CSG_REQ_EP_CFG_MASK) >> CSG_REQ_EP_CFG_SHIFT) -+#define CSG_REQ_EP_CFG_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_REQ_EP_CFG_MASK) | (((value) << CSG_REQ_EP_CFG_SHIFT) & CSG_REQ_EP_CFG_MASK)) -+#define CSG_REQ_STATUS_UPDATE_SHIFT 5 -+#define CSG_REQ_STATUS_UPDATE_MASK (0x1 << CSG_REQ_STATUS_UPDATE_SHIFT) -+#define CSG_REQ_STATUS_UPDATE_GET(reg_val) (((reg_val)&CSG_REQ_STATUS_UPDATE_MASK) >> CSG_REQ_STATUS_UPDATE_SHIFT) -+#define CSG_REQ_STATUS_UPDATE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_REQ_STATUS_UPDATE_MASK) | \ -+ (((value) << CSG_REQ_STATUS_UPDATE_SHIFT) & CSG_REQ_STATUS_UPDATE_MASK)) -+#define CSG_REQ_SYNC_UPDATE_SHIFT 28 -+#define CSG_REQ_SYNC_UPDATE_MASK (0x1 << CSG_REQ_SYNC_UPDATE_SHIFT) -+#define CSG_REQ_SYNC_UPDATE_GET(reg_val) (((reg_val)&CSG_REQ_SYNC_UPDATE_MASK) >> CSG_REQ_SYNC_UPDATE_SHIFT) -+#define CSG_REQ_SYNC_UPDATE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_REQ_SYNC_UPDATE_MASK) | (((value) << CSG_REQ_SYNC_UPDATE_SHIFT) & CSG_REQ_SYNC_UPDATE_MASK)) -+#define CSG_REQ_IDLE_SHIFT 29 -+#define CSG_REQ_IDLE_MASK (0x1 << CSG_REQ_IDLE_SHIFT) -+#define CSG_REQ_IDLE_GET(reg_val) (((reg_val)&CSG_REQ_IDLE_MASK) >> CSG_REQ_IDLE_SHIFT) -+#define CSG_REQ_IDLE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_REQ_IDLE_MASK) | (((value) << CSG_REQ_IDLE_SHIFT) & CSG_REQ_IDLE_MASK)) -+#define CSG_REQ_DOORBELL_SHIFT 30 -+#define CSG_REQ_DOORBELL_MASK (0x1 << CSG_REQ_DOORBELL_SHIFT) -+#define CSG_REQ_DOORBELL_GET(reg_val) (((reg_val)&CSG_REQ_DOORBELL_MASK) >> CSG_REQ_DOORBELL_SHIFT) -+#define CSG_REQ_DOORBELL_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_REQ_DOORBELL_MASK) | (((value) << CSG_REQ_DOORBELL_SHIFT) & CSG_REQ_DOORBELL_MASK)) -+#define CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT 31 -+#define CSG_REQ_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) -+#define CSG_REQ_PROGRESS_TIMER_EVENT_GET(reg_val) \ -+ (((reg_val)&CSG_REQ_PROGRESS_TIMER_EVENT_MASK) >> CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) -+#define CSG_REQ_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_REQ_PROGRESS_TIMER_EVENT_MASK) | \ -+ (((value) << CSG_REQ_PROGRESS_TIMER_EVENT_SHIFT) & CSG_REQ_PROGRESS_TIMER_EVENT_MASK)) -+ -+/* CSG_ACK_IRQ_MASK register */ -+#define CSG_ACK_IRQ_MASK_STATE_SHIFT 0 -+#define CSG_ACK_IRQ_MASK_STATE_MASK (0x7 << CSG_ACK_IRQ_MASK_STATE_SHIFT) -+#define CSG_ACK_IRQ_MASK_STATE_GET(reg_val) (((reg_val)&CSG_ACK_IRQ_MASK_STATE_MASK) >> CSG_ACK_IRQ_MASK_STATE_SHIFT) -+#define CSG_ACK_IRQ_MASK_STATE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_IRQ_MASK_STATE_MASK) | \ -+ (((value) << CSG_ACK_IRQ_MASK_STATE_SHIFT) & CSG_ACK_IRQ_MASK_STATE_MASK)) -+/* CSG_ACK_IRQ_MASK_STATE values */ -+#define CSG_ACK_IRQ_MASK_STATE_DISABLED 0x0 -+#define CSG_ACK_IRQ_MASK_STATE_ENABLED 0x7 -+/* End of CSG_ACK_IRQ_MASK_STATE values */ -+#define CSG_ACK_IRQ_MASK_EP_CFG_SHIFT 4 -+#define CSG_ACK_IRQ_MASK_EP_CFG_MASK (0x1 << CSG_ACK_IRQ_MASK_EP_CFG_SHIFT) -+#define CSG_ACK_IRQ_MASK_EP_CFG_GET(reg_val) (((reg_val)&CSG_ACK_IRQ_MASK_EP_CFG_MASK) >> CSG_ACK_IRQ_MASK_EP_CFG_SHIFT) -+#define CSG_ACK_IRQ_MASK_EP_CFG_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_IRQ_MASK_EP_CFG_MASK) | \ -+ (((value) << CSG_ACK_IRQ_MASK_EP_CFG_SHIFT) & CSG_ACK_IRQ_MASK_EP_CFG_MASK)) -+#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT 5 -+#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK (0x1 << CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT) -+#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_GET(reg_val) \ -+ (((reg_val)&CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK) >> CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT) -+#define CSG_ACK_IRQ_MASK_STATUS_UPDATE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK) | \ -+ (((value) << CSG_ACK_IRQ_MASK_STATUS_UPDATE_SHIFT) & CSG_ACK_IRQ_MASK_STATUS_UPDATE_MASK)) -+#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT 28 -+#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK (0x1 << CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT) -+#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_GET(reg_val) \ -+ (((reg_val)&CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK) >> CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT) -+#define CSG_ACK_IRQ_MASK_SYNC_UPDATE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK) | \ -+ (((value) << CSG_ACK_IRQ_MASK_SYNC_UPDATE_SHIFT) & CSG_ACK_IRQ_MASK_SYNC_UPDATE_MASK)) -+#define CSG_ACK_IRQ_MASK_IDLE_SHIFT 29 -+#define CSG_ACK_IRQ_MASK_IDLE_MASK (0x1 << CSG_ACK_IRQ_MASK_IDLE_SHIFT) -+#define CSG_ACK_IRQ_MASK_IDLE_GET(reg_val) (((reg_val)&CSG_ACK_IRQ_MASK_IDLE_MASK) >> CSG_ACK_IRQ_MASK_IDLE_SHIFT) -+#define CSG_ACK_IRQ_MASK_IDLE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_IRQ_MASK_IDLE_MASK) | \ -+ (((value) << CSG_ACK_IRQ_MASK_IDLE_SHIFT) & CSG_ACK_IRQ_MASK_IDLE_MASK)) -+#define CSG_ACK_IRQ_MASK_DOORBELL_SHIFT 30 -+#define CSG_ACK_IRQ_MASK_DOORBELL_MASK (0x1 << CSG_ACK_IRQ_MASK_DOORBELL_SHIFT) -+#define CSG_ACK_IRQ_MASK_DOORBELL_GET(reg_val) \ -+ (((reg_val)&CSG_ACK_IRQ_MASK_DOORBELL_MASK) >> CSG_ACK_IRQ_MASK_DOORBELL_SHIFT) -+#define CSG_ACK_IRQ_MASK_DOORBELL_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_IRQ_MASK_DOORBELL_MASK) | \ -+ (((value) << CSG_ACK_IRQ_MASK_DOORBELL_SHIFT) & CSG_ACK_IRQ_MASK_DOORBELL_MASK)) -+#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT 31 -+#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT) -+#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_GET(reg_val) \ -+ (((reg_val)&CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK) >> CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT) -+#define CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK) | \ -+ (((value) << CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_SHIFT) & CSG_ACK_IRQ_MASK_PROGRESS_TIMER_EVENT_MASK)) -+ -+/* CSG_EP_REQ register */ -+#define CSG_EP_REQ_COMPUTE_EP_SHIFT 0 -+#define CSG_EP_REQ_COMPUTE_EP_MASK (0xFF << CSG_EP_REQ_COMPUTE_EP_SHIFT) -+#define CSG_EP_REQ_COMPUTE_EP_GET(reg_val) (((reg_val)&CSG_EP_REQ_COMPUTE_EP_MASK) >> CSG_EP_REQ_COMPUTE_EP_SHIFT) -+#define CSG_EP_REQ_COMPUTE_EP_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_EP_REQ_COMPUTE_EP_MASK) | \ -+ (((value) << CSG_EP_REQ_COMPUTE_EP_SHIFT) & CSG_EP_REQ_COMPUTE_EP_MASK)) -+#define CSG_EP_REQ_FRAGMENT_EP_SHIFT 8 -+#define CSG_EP_REQ_FRAGMENT_EP_MASK (0xFF << CSG_EP_REQ_FRAGMENT_EP_SHIFT) -+#define CSG_EP_REQ_FRAGMENT_EP_GET(reg_val) (((reg_val)&CSG_EP_REQ_FRAGMENT_EP_MASK) >> CSG_EP_REQ_FRAGMENT_EP_SHIFT) -+#define CSG_EP_REQ_FRAGMENT_EP_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_EP_REQ_FRAGMENT_EP_MASK) | \ -+ (((value) << CSG_EP_REQ_FRAGMENT_EP_SHIFT) & CSG_EP_REQ_FRAGMENT_EP_MASK)) -+#define CSG_EP_REQ_TILER_EP_SHIFT 16 -+#define CSG_EP_REQ_TILER_EP_MASK (0xF << CSG_EP_REQ_TILER_EP_SHIFT) -+#define CSG_EP_REQ_TILER_EP_GET(reg_val) (((reg_val)&CSG_EP_REQ_TILER_EP_MASK) >> CSG_EP_REQ_TILER_EP_SHIFT) -+#define CSG_EP_REQ_TILER_EP_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_EP_REQ_TILER_EP_MASK) | (((value) << CSG_EP_REQ_TILER_EP_SHIFT) & CSG_EP_REQ_TILER_EP_MASK)) -+#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT 20 -+#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK (0x1 << CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) -+#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_GET(reg_val) \ -+ (((reg_val)&CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK) >> CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) -+#define CSG_EP_REQ_EXCLUSIVE_COMPUTE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK) | \ -+ (((value) << CSG_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) & CSG_EP_REQ_EXCLUSIVE_COMPUTE_MASK)) -+#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT 21 -+#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK (0x1 << CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) -+#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_GET(reg_val) \ -+ (((reg_val)&CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) >> CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) -+#define CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) | \ -+ (((value) << CSG_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) & CSG_EP_REQ_EXCLUSIVE_FRAGMENT_MASK)) -+#define CSG_EP_REQ_PRIORITY_SHIFT 28 -+#define CSG_EP_REQ_PRIORITY_MASK (0xF << CSG_EP_REQ_PRIORITY_SHIFT) -+#define CSG_EP_REQ_PRIORITY_GET(reg_val) (((reg_val)&CSG_EP_REQ_PRIORITY_MASK) >> CSG_EP_REQ_PRIORITY_SHIFT) -+#define CSG_EP_REQ_PRIORITY_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_EP_REQ_PRIORITY_MASK) | (((value) << CSG_EP_REQ_PRIORITY_SHIFT) & CSG_EP_REQ_PRIORITY_MASK)) -+ -+/* CSG_SUSPEND_BUF register */ -+#define CSG_SUSPEND_BUF_POINTER_SHIFT 0 -+#define CSG_SUSPEND_BUF_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CSG_SUSPEND_BUF_POINTER_SHIFT) -+#define CSG_SUSPEND_BUF_POINTER_GET(reg_val) (((reg_val)&CSG_SUSPEND_BUF_POINTER_MASK) >> CSG_SUSPEND_BUF_POINTER_SHIFT) -+#define CSG_SUSPEND_BUF_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_SUSPEND_BUF_POINTER_MASK) | \ -+ (((value) << CSG_SUSPEND_BUF_POINTER_SHIFT) & CSG_SUSPEND_BUF_POINTER_MASK)) -+ -+/* CSG_PROTM_SUSPEND_BUF register */ -+#define CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT 0 -+#define CSG_PROTM_SUSPEND_BUF_POINTER_MASK (0xFFFFFFFFFFFFFFFF << CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT) -+#define CSG_PROTM_SUSPEND_BUF_POINTER_GET(reg_val) \ -+ (((reg_val)&CSG_PROTM_SUSPEND_BUF_POINTER_MASK) >> CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT) -+#define CSG_PROTM_SUSPEND_BUF_POINTER_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_PROTM_SUSPEND_BUF_POINTER_MASK) | \ -+ (((value) << CSG_PROTM_SUSPEND_BUF_POINTER_SHIFT) & CSG_PROTM_SUSPEND_BUF_POINTER_MASK)) -+ -+/* End of CSG_INPUT_BLOCK register set definitions */ -+ -+/* CSG_OUTPUT_BLOCK register set definitions */ -+ -+/* CSG_ACK register */ -+#define CSG_ACK_STATE_SHIFT 0 -+#define CSG_ACK_STATE_MASK (0x7 << CSG_ACK_STATE_SHIFT) -+#define CSG_ACK_STATE_GET(reg_val) (((reg_val)&CSG_ACK_STATE_MASK) >> CSG_ACK_STATE_SHIFT) -+#define CSG_ACK_STATE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_STATE_MASK) | (((value) << CSG_ACK_STATE_SHIFT) & CSG_ACK_STATE_MASK)) -+/* CSG_ACK_STATE values */ -+#define CSG_ACK_STATE_TERMINATE 0x0 -+#define CSG_ACK_STATE_START 0x1 -+#define CSG_ACK_STATE_SUSPEND 0x2 -+#define CSG_ACK_STATE_RESUME 0x3 -+/* End of CSG_ACK_STATE values */ -+#define CSG_ACK_EP_CFG_SHIFT 4 -+#define CSG_ACK_EP_CFG_MASK (0x1 << CSG_ACK_EP_CFG_SHIFT) -+#define CSG_ACK_EP_CFG_GET(reg_val) (((reg_val)&CSG_ACK_EP_CFG_MASK) >> CSG_ACK_EP_CFG_SHIFT) -+#define CSG_ACK_EP_CFG_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_EP_CFG_MASK) | (((value) << CSG_ACK_EP_CFG_SHIFT) & CSG_ACK_EP_CFG_MASK)) -+#define CSG_ACK_STATUS_UPDATE_SHIFT 5 -+#define CSG_ACK_STATUS_UPDATE_MASK (0x1 << CSG_ACK_STATUS_UPDATE_SHIFT) -+#define CSG_ACK_STATUS_UPDATE_GET(reg_val) (((reg_val)&CSG_ACK_STATUS_UPDATE_MASK) >> CSG_ACK_STATUS_UPDATE_SHIFT) -+#define CSG_ACK_STATUS_UPDATE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_STATUS_UPDATE_MASK) | \ -+ (((value) << CSG_ACK_STATUS_UPDATE_SHIFT) & CSG_ACK_STATUS_UPDATE_MASK)) -+#define CSG_ACK_SYNC_UPDATE_SHIFT 28 -+#define CSG_ACK_SYNC_UPDATE_MASK (0x1 << CSG_ACK_SYNC_UPDATE_SHIFT) -+#define CSG_ACK_SYNC_UPDATE_GET(reg_val) (((reg_val)&CSG_ACK_SYNC_UPDATE_MASK) >> CSG_ACK_SYNC_UPDATE_SHIFT) -+#define CSG_ACK_SYNC_UPDATE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_SYNC_UPDATE_MASK) | (((value) << CSG_ACK_SYNC_UPDATE_SHIFT) & CSG_ACK_SYNC_UPDATE_MASK)) -+#define CSG_ACK_IDLE_SHIFT 29 -+#define CSG_ACK_IDLE_MASK (0x1 << CSG_ACK_IDLE_SHIFT) -+#define CSG_ACK_IDLE_GET(reg_val) (((reg_val)&CSG_ACK_IDLE_MASK) >> CSG_ACK_IDLE_SHIFT) -+#define CSG_ACK_IDLE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_IDLE_MASK) | (((value) << CSG_ACK_IDLE_SHIFT) & CSG_ACK_IDLE_MASK)) -+#define CSG_ACK_DOORBELL_SHIFT 30 -+#define CSG_ACK_DOORBELL_MASK (0x1 << CSG_ACK_DOORBELL_SHIFT) -+#define CSG_ACK_DOORBELL_GET(reg_val) (((reg_val)&CSG_ACK_DOORBELL_MASK) >> CSG_ACK_DOORBELL_SHIFT) -+#define CSG_ACK_DOORBELL_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_DOORBELL_MASK) | (((value) << CSG_ACK_DOORBELL_SHIFT) & CSG_ACK_DOORBELL_MASK)) -+#define CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT 31 -+#define CSG_ACK_PROGRESS_TIMER_EVENT_MASK (0x1 << CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) -+#define CSG_ACK_PROGRESS_TIMER_EVENT_GET(reg_val) \ -+ (((reg_val)&CSG_ACK_PROGRESS_TIMER_EVENT_MASK) >> CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) -+#define CSG_ACK_PROGRESS_TIMER_EVENT_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_ACK_PROGRESS_TIMER_EVENT_MASK) | \ -+ (((value) << CSG_ACK_PROGRESS_TIMER_EVENT_SHIFT) & CSG_ACK_PROGRESS_TIMER_EVENT_MASK)) -+ -+/* CSG_STATUS_EP_CURRENT register */ -+#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT 0 -+#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK (0xFF << CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT) -+#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_GET(reg_val) \ -+ (((reg_val)&CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK) >> CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT) -+#define CSG_STATUS_EP_CURRENT_COMPUTE_EP_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK) | \ -+ (((value) << CSG_STATUS_EP_CURRENT_COMPUTE_EP_SHIFT) & CSG_STATUS_EP_CURRENT_COMPUTE_EP_MASK)) -+#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT 8 -+#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK (0xFF << CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT) -+#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_GET(reg_val) \ -+ (((reg_val)&CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK) >> CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT) -+#define CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK) | \ -+ (((value) << CSG_STATUS_EP_CURRENT_FRAGMENT_EP_SHIFT) & CSG_STATUS_EP_CURRENT_FRAGMENT_EP_MASK)) -+#define CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT 16 -+#define CSG_STATUS_EP_CURRENT_TILER_EP_MASK (0xF << CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT) -+#define CSG_STATUS_EP_CURRENT_TILER_EP_GET(reg_val) \ -+ (((reg_val)&CSG_STATUS_EP_CURRENT_TILER_EP_MASK) >> CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT) -+#define CSG_STATUS_EP_CURRENT_TILER_EP_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_STATUS_EP_CURRENT_TILER_EP_MASK) | \ -+ (((value) << CSG_STATUS_EP_CURRENT_TILER_EP_SHIFT) & CSG_STATUS_EP_CURRENT_TILER_EP_MASK)) -+ -+/* CSG_STATUS_EP_REQ register */ -+#define CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT 0 -+#define CSG_STATUS_EP_REQ_COMPUTE_EP_MASK (0xFF << CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT) -+#define CSG_STATUS_EP_REQ_COMPUTE_EP_GET(reg_val) \ -+ (((reg_val)&CSG_STATUS_EP_REQ_COMPUTE_EP_MASK) >> CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT) -+#define CSG_STATUS_EP_REQ_COMPUTE_EP_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_STATUS_EP_REQ_COMPUTE_EP_MASK) | \ -+ (((value) << CSG_STATUS_EP_REQ_COMPUTE_EP_SHIFT) & CSG_STATUS_EP_REQ_COMPUTE_EP_MASK)) -+#define CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT 8 -+#define CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK (0xFF << CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT) -+#define CSG_STATUS_EP_REQ_FRAGMENT_EP_GET(reg_val) \ -+ (((reg_val)&CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK) >> CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT) -+#define CSG_STATUS_EP_REQ_FRAGMENT_EP_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK) | \ -+ (((value) << CSG_STATUS_EP_REQ_FRAGMENT_EP_SHIFT) & CSG_STATUS_EP_REQ_FRAGMENT_EP_MASK)) -+#define CSG_STATUS_EP_REQ_TILER_EP_SHIFT 16 -+#define CSG_STATUS_EP_REQ_TILER_EP_MASK (0xF << CSG_STATUS_EP_REQ_TILER_EP_SHIFT) -+#define CSG_STATUS_EP_REQ_TILER_EP_GET(reg_val) \ -+ (((reg_val)&CSG_STATUS_EP_REQ_TILER_EP_MASK) >> CSG_STATUS_EP_REQ_TILER_EP_SHIFT) -+#define CSG_STATUS_EP_REQ_TILER_EP_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_STATUS_EP_REQ_TILER_EP_MASK) | \ -+ (((value) << CSG_STATUS_EP_REQ_TILER_EP_SHIFT) & CSG_STATUS_EP_REQ_TILER_EP_MASK)) -+#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT 20 -+#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK (0x1 << CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) -+#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_GET(reg_val) \ -+ (((reg_val)&CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK) >> CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) -+#define CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK) | \ -+ (((value) << CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_SHIFT) & CSG_STATUS_EP_REQ_EXCLUSIVE_COMPUTE_MASK)) -+#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT 21 -+#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK (0x1 << CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) -+#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_GET(reg_val) \ -+ (((reg_val)&CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) >> CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) -+#define CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK) | \ -+ (((value) << CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_SHIFT) & CSG_STATUS_EP_REQ_EXCLUSIVE_FRAGMENT_MASK)) -+ -+/* End of CSG_OUTPUT_BLOCK register set definitions */ -+ -+/* STREAM_CONTROL_BLOCK register set definitions */ -+ -+/* STREAM_FEATURES register */ -+#define STREAM_FEATURES_WORK_REGISTERS_SHIFT 0 -+#define STREAM_FEATURES_WORK_REGISTERS_MASK (0xFF << STREAM_FEATURES_WORK_REGISTERS_SHIFT) -+#define STREAM_FEATURES_WORK_REGISTERS_GET(reg_val) \ -+ (((reg_val)&STREAM_FEATURES_WORK_REGISTERS_MASK) >> STREAM_FEATURES_WORK_REGISTERS_SHIFT) -+#define STREAM_FEATURES_WORK_REGISTERS_SET(reg_val, value) \ -+ (((reg_val) & ~STREAM_FEATURES_WORK_REGISTERS_MASK) | \ -+ (((value) << STREAM_FEATURES_WORK_REGISTERS_SHIFT) & STREAM_FEATURES_WORK_REGISTERS_MASK)) -+#define STREAM_FEATURES_SCOREBOARDS_SHIFT 8 -+#define STREAM_FEATURES_SCOREBOARDS_MASK (0xFF << STREAM_FEATURES_SCOREBOARDS_SHIFT) -+#define STREAM_FEATURES_SCOREBOARDS_GET(reg_val) \ -+ (((reg_val)&STREAM_FEATURES_SCOREBOARDS_MASK) >> STREAM_FEATURES_SCOREBOARDS_SHIFT) -+#define STREAM_FEATURES_SCOREBOARDS_SET(reg_val, value) \ -+ (((reg_val) & ~STREAM_FEATURES_SCOREBOARDS_MASK) | \ -+ (((value) << STREAM_FEATURES_SCOREBOARDS_SHIFT) & STREAM_FEATURES_SCOREBOARDS_MASK)) -+#define STREAM_FEATURES_COMPUTE_SHIFT 16 -+#define STREAM_FEATURES_COMPUTE_MASK (0x1 << STREAM_FEATURES_COMPUTE_SHIFT) -+#define STREAM_FEATURES_COMPUTE_GET(reg_val) (((reg_val)&STREAM_FEATURES_COMPUTE_MASK) >> STREAM_FEATURES_COMPUTE_SHIFT) -+#define STREAM_FEATURES_COMPUTE_SET(reg_val, value) \ -+ (((reg_val) & ~STREAM_FEATURES_COMPUTE_MASK) | \ -+ (((value) << STREAM_FEATURES_COMPUTE_SHIFT) & STREAM_FEATURES_COMPUTE_MASK)) -+#define STREAM_FEATURES_FRAGMENT_SHIFT 17 -+#define STREAM_FEATURES_FRAGMENT_MASK (0x1 << STREAM_FEATURES_FRAGMENT_SHIFT) -+#define STREAM_FEATURES_FRAGMENT_GET(reg_val) \ -+ (((reg_val)&STREAM_FEATURES_FRAGMENT_MASK) >> STREAM_FEATURES_FRAGMENT_SHIFT) -+#define STREAM_FEATURES_FRAGMENT_SET(reg_val, value) \ -+ (((reg_val) & ~STREAM_FEATURES_FRAGMENT_MASK) | \ -+ (((value) << STREAM_FEATURES_FRAGMENT_SHIFT) & STREAM_FEATURES_FRAGMENT_MASK)) -+#define STREAM_FEATURES_TILER_SHIFT 18 -+#define STREAM_FEATURES_TILER_MASK (0x1 << STREAM_FEATURES_TILER_SHIFT) -+#define STREAM_FEATURES_TILER_GET(reg_val) (((reg_val)&STREAM_FEATURES_TILER_MASK) >> STREAM_FEATURES_TILER_SHIFT) -+#define STREAM_FEATURES_TILER_SET(reg_val, value) \ -+ (((reg_val) & ~STREAM_FEATURES_TILER_MASK) | \ -+ (((value) << STREAM_FEATURES_TILER_SHIFT) & STREAM_FEATURES_TILER_MASK)) -+ -+/* STREAM_INPUT_VA register */ -+#define STREAM_INPUT_VA_VALUE_SHIFT 0 -+#define STREAM_INPUT_VA_VALUE_MASK (0xFFFFFFFF << STREAM_INPUT_VA_VALUE_SHIFT) -+#define STREAM_INPUT_VA_VALUE_GET(reg_val) (((reg_val)&STREAM_INPUT_VA_VALUE_MASK) >> STREAM_INPUT_VA_VALUE_SHIFT) -+#define STREAM_INPUT_VA_VALUE_SET(reg_val, value) \ -+ (((reg_val) & ~STREAM_INPUT_VA_VALUE_MASK) | \ -+ (((value) << STREAM_INPUT_VA_VALUE_SHIFT) & STREAM_INPUT_VA_VALUE_MASK)) -+ -+/* STREAM_OUTPUT_VA register */ -+#define STREAM_OUTPUT_VA_VALUE_SHIFT 0 -+#define STREAM_OUTPUT_VA_VALUE_MASK (0xFFFFFFFF << STREAM_OUTPUT_VA_VALUE_SHIFT) -+#define STREAM_OUTPUT_VA_VALUE_GET(reg_val) (((reg_val)&STREAM_OUTPUT_VA_VALUE_MASK) >> STREAM_OUTPUT_VA_VALUE_SHIFT) -+#define STREAM_OUTPUT_VA_VALUE_SET(reg_val, value) \ -+ (((reg_val) & ~STREAM_OUTPUT_VA_VALUE_MASK) | \ -+ (((value) << STREAM_OUTPUT_VA_VALUE_SHIFT) & STREAM_OUTPUT_VA_VALUE_MASK)) -+/* End of STREAM_CONTROL_BLOCK register set definitions */ -+ -+/* GLB_INPUT_BLOCK register set definitions */ -+ -+/* GLB_REQ register */ -+#define GLB_REQ_HALT_SHIFT 0 -+#define GLB_REQ_HALT_MASK (0x1 << GLB_REQ_HALT_SHIFT) -+#define GLB_REQ_HALT_GET(reg_val) (((reg_val)&GLB_REQ_HALT_MASK) >> GLB_REQ_HALT_SHIFT) -+#define GLB_REQ_HALT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_HALT_MASK) | (((value) << GLB_REQ_HALT_SHIFT) & GLB_REQ_HALT_MASK)) -+#define GLB_REQ_CFG_PROGRESS_TIMER_SHIFT 1 -+#define GLB_REQ_CFG_PROGRESS_TIMER_MASK (0x1 << GLB_REQ_CFG_PROGRESS_TIMER_SHIFT) -+#define GLB_REQ_CFG_PROGRESS_TIMER_GET(reg_val) \ -+ (((reg_val)&GLB_REQ_CFG_PROGRESS_TIMER_MASK) >> GLB_REQ_CFG_PROGRESS_TIMER_SHIFT) -+#define GLB_REQ_CFG_PROGRESS_TIMER_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_CFG_PROGRESS_TIMER_MASK) | \ -+ (((value) << GLB_REQ_CFG_PROGRESS_TIMER_SHIFT) & GLB_REQ_CFG_PROGRESS_TIMER_MASK)) -+#define GLB_REQ_CFG_ALLOC_EN_SHIFT 2 -+#define GLB_REQ_CFG_ALLOC_EN_MASK (0x1 << GLB_REQ_CFG_ALLOC_EN_SHIFT) -+#define GLB_REQ_CFG_ALLOC_EN_GET(reg_val) (((reg_val)&GLB_REQ_CFG_ALLOC_EN_MASK) >> GLB_REQ_CFG_ALLOC_EN_SHIFT) -+#define GLB_REQ_CFG_ALLOC_EN_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_CFG_ALLOC_EN_MASK) | (((value) << GLB_REQ_CFG_ALLOC_EN_SHIFT) & GLB_REQ_CFG_ALLOC_EN_MASK)) -+#define GLB_REQ_CFG_PWROFF_TIMER_SHIFT 3 -+#define GLB_REQ_CFG_PWROFF_TIMER_MASK (0x1 << GLB_REQ_CFG_PWROFF_TIMER_SHIFT) -+#define GLB_REQ_CFG_PWROFF_TIMER_GET(reg_val) \ -+ (((reg_val)&GLB_REQ_CFG_PWROFF_TIMER_MASK) >> GLB_REQ_CFG_PWROFF_TIMER_SHIFT) -+#define GLB_REQ_CFG_PWROFF_TIMER_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_CFG_PWROFF_TIMER_MASK) | \ -+ (((value) << GLB_REQ_CFG_PWROFF_TIMER_SHIFT) & GLB_REQ_CFG_PWROFF_TIMER_MASK)) -+#define GLB_REQ_PROTM_ENTER_SHIFT 4 -+#define GLB_REQ_PROTM_ENTER_MASK (0x1 << GLB_REQ_PROTM_ENTER_SHIFT) -+#define GLB_REQ_PROTM_ENTER_GET(reg_val) (((reg_val)&GLB_REQ_PROTM_ENTER_MASK) >> GLB_REQ_PROTM_ENTER_SHIFT) -+#define GLB_REQ_PROTM_ENTER_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_PROTM_ENTER_MASK) | (((value) << GLB_REQ_PROTM_ENTER_SHIFT) & GLB_REQ_PROTM_ENTER_MASK)) -+#define GLB_REQ_PRFCNT_ENABLE_SHIFT 5 -+#define GLB_REQ_PRFCNT_ENABLE_MASK (0x1 << GLB_REQ_PRFCNT_ENABLE_SHIFT) -+#define GLB_REQ_PRFCNT_ENABLE_GET(reg_val) (((reg_val)&GLB_REQ_PRFCNT_ENABLE_MASK) >> GLB_REQ_PRFCNT_ENABLE_SHIFT) -+#define GLB_REQ_PRFCNT_ENABLE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_PRFCNT_ENABLE_MASK) | \ -+ (((value) << GLB_REQ_PRFCNT_ENABLE_SHIFT) & GLB_REQ_PRFCNT_ENABLE_MASK)) -+#define GLB_REQ_PRFCNT_SAMPLE_SHIFT 6 -+#define GLB_REQ_PRFCNT_SAMPLE_MASK (0x1 << GLB_REQ_PRFCNT_SAMPLE_SHIFT) -+#define GLB_REQ_PRFCNT_SAMPLE_GET(reg_val) (((reg_val)&GLB_REQ_PRFCNT_SAMPLE_MASK) >> GLB_REQ_PRFCNT_SAMPLE_SHIFT) -+#define GLB_REQ_PRFCNT_SAMPLE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_PRFCNT_SAMPLE_MASK) | \ -+ (((value) << GLB_REQ_PRFCNT_SAMPLE_SHIFT) & GLB_REQ_PRFCNT_SAMPLE_MASK)) -+#define GLB_REQ_COUNTER_ENABLE_SHIFT 7 -+#define GLB_REQ_COUNTER_ENABLE_MASK (0x1 << GLB_REQ_COUNTER_ENABLE_SHIFT) -+#define GLB_REQ_COUNTER_ENABLE_GET(reg_val) (((reg_val)&GLB_REQ_COUNTER_ENABLE_MASK) >> GLB_REQ_COUNTER_ENABLE_SHIFT) -+#define GLB_REQ_COUNTER_ENABLE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_COUNTER_ENABLE_MASK) | \ -+ (((value) << GLB_REQ_COUNTER_ENABLE_SHIFT) & GLB_REQ_COUNTER_ENABLE_MASK)) -+#define GLB_REQ_PING_SHIFT 8 -+#define GLB_REQ_PING_MASK (0x1 << GLB_REQ_PING_SHIFT) -+#define GLB_REQ_PING_GET(reg_val) (((reg_val)&GLB_REQ_PING_MASK) >> GLB_REQ_PING_SHIFT) -+#define GLB_REQ_PING_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_PING_MASK) | (((value) << GLB_REQ_PING_SHIFT) & GLB_REQ_PING_MASK)) -+#define GLB_REQ_FIRMWARE_CONFIG_UPDATE_SHIFT 9 -+#define GLB_REQ_FIRMWARE_CONFIG_UPDATE_MASK \ -+ (0x1 << GLB_REQ_FIRMWARE_CONFIG_UPDATE_SHIFT) -+#define GLB_REQ_FIRMWARE_CONFIG_UPDATE_GET(reg_val) \ -+ (((reg_val)&GLB_REQ_FIRMWARE_CONFIG_UPDATE_MASK) >> \ -+ GLB_REQ_FIRMWARE_CONFIG_UPDATE_SHIFT) -+#define GLB_REQ_FIRMWARE_CONFIG_UPDATE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_FIRMWARE_CONFIG_UPDATE_MASK) | \ -+ (((value) << GLB_REQ_FIRMWARE_CONFIG_UPDATE_SHIFT) & \ -+ GLB_REQ_FIRMWARE_CONFIG_UPDATE_MASK)) -+#define GLB_REQ_INACTIVE_COMPUTE_SHIFT 20 -+#define GLB_REQ_INACTIVE_COMPUTE_MASK (0x1 << GLB_REQ_INACTIVE_COMPUTE_SHIFT) -+#define GLB_REQ_INACTIVE_COMPUTE_GET(reg_val) \ -+ (((reg_val)&GLB_REQ_INACTIVE_COMPUTE_MASK) >> GLB_REQ_INACTIVE_COMPUTE_SHIFT) -+#define GLB_REQ_INACTIVE_COMPUTE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_INACTIVE_COMPUTE_MASK) | \ -+ (((value) << GLB_REQ_INACTIVE_COMPUTE_SHIFT) & GLB_REQ_INACTIVE_COMPUTE_MASK)) -+#define GLB_REQ_INACTIVE_FRAGMENT_SHIFT 21 -+#define GLB_REQ_INACTIVE_FRAGMENT_MASK (0x1 << GLB_REQ_INACTIVE_FRAGMENT_SHIFT) -+#define GLB_REQ_INACTIVE_FRAGMENT_GET(reg_val) \ -+ (((reg_val)&GLB_REQ_INACTIVE_FRAGMENT_MASK) >> GLB_REQ_INACTIVE_FRAGMENT_SHIFT) -+#define GLB_REQ_INACTIVE_FRAGMENT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_INACTIVE_FRAGMENT_MASK) | \ -+ (((value) << GLB_REQ_INACTIVE_FRAGMENT_SHIFT) & GLB_REQ_INACTIVE_FRAGMENT_MASK)) -+#define GLB_REQ_INACTIVE_TILER_SHIFT 22 -+#define GLB_REQ_INACTIVE_TILER_MASK (0x1 << GLB_REQ_INACTIVE_TILER_SHIFT) -+#define GLB_REQ_INACTIVE_TILER_GET(reg_val) (((reg_val)&GLB_REQ_INACTIVE_TILER_MASK) >> GLB_REQ_INACTIVE_TILER_SHIFT) -+#define GLB_REQ_INACTIVE_TILER_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_INACTIVE_TILER_MASK) | \ -+ (((value) << GLB_REQ_INACTIVE_TILER_SHIFT) & GLB_REQ_INACTIVE_TILER_MASK)) -+#define GLB_REQ_PROTM_EXIT_SHIFT 23 -+#define GLB_REQ_PROTM_EXIT_MASK (0x1 << GLB_REQ_PROTM_EXIT_SHIFT) -+#define GLB_REQ_PROTM_EXIT_GET(reg_val) (((reg_val)&GLB_REQ_PROTM_EXIT_MASK) >> GLB_REQ_PROTM_EXIT_SHIFT) -+#define GLB_REQ_PROTM_EXIT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_PROTM_EXIT_MASK) | (((value) << GLB_REQ_PROTM_EXIT_SHIFT) & GLB_REQ_PROTM_EXIT_MASK)) -+#define GLB_REQ_PRFCNT_THRESHOLD_SHIFT 24 -+#define GLB_REQ_PRFCNT_THRESHOLD_MASK (0x1 << GLB_REQ_PRFCNT_THRESHOLD_SHIFT) -+#define GLB_REQ_PRFCNT_THRESHOLD_GET(reg_val) \ -+ (((reg_val)&GLB_REQ_PRFCNT_THRESHOLD_MASK) >> \ -+ GLB_REQ_PRFCNT_THRESHOLD_SHIFT) -+#define GLB_REQ_PRFCNT_THRESHOLD_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_PRFCNT_THRESHOLD_MASK) | \ -+ (((value) << GLB_REQ_PRFCNT_THRESHOLD_SHIFT) & \ -+ GLB_REQ_PRFCNT_THRESHOLD_MASK)) -+#define GLB_REQ_PRFCNT_OVERFLOW_SHIFT 25 -+#define GLB_REQ_PRFCNT_OVERFLOW_MASK (0x1 << GLB_REQ_PRFCNT_OVERFLOW_SHIFT) -+#define GLB_REQ_PRFCNT_OVERFLOW_GET(reg_val) \ -+ (((reg_val)&GLB_REQ_PRFCNT_OVERFLOW_MASK) >> \ -+ GLB_REQ_PRFCNT_OVERFLOW_SHIFT) -+#define GLB_REQ_PRFCNT_OVERFLOW_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_PRFCNT_OVERFLOW_MASK) | \ -+ (((value) << GLB_REQ_PRFCNT_OVERFLOW_SHIFT) & \ -+ GLB_REQ_PRFCNT_OVERFLOW_MASK)) -+#define GLB_REQ_DEBUG_CSF_REQ_SHIFT 30 -+#define GLB_REQ_DEBUG_CSF_REQ_MASK (0x1 << GLB_REQ_DEBUG_CSF_REQ_SHIFT) -+#define GLB_REQ_DEBUG_CSF_REQ_GET(reg_val) (((reg_val)&GLB_REQ_DEBUG_CSF_REQ_MASK) >> GLB_REQ_DEBUG_CSF_REQ_SHIFT) -+#define GLB_REQ_DEBUG_CSF_REQ_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_DEBUG_CSF_REQ_MASK) | \ -+ (((value) << GLB_REQ_DEBUG_CSF_REQ_SHIFT) & GLB_REQ_DEBUG_CSF_REQ_MASK)) -+#define GLB_REQ_DEBUG_HOST_REQ_SHIFT 31 -+#define GLB_REQ_DEBUG_HOST_REQ_MASK (0x1 << GLB_REQ_DEBUG_HOST_REQ_SHIFT) -+#define GLB_REQ_DEBUG_HOST_REQ_GET(reg_val) (((reg_val)&GLB_REQ_DEBUG_HOST_REQ_MASK) >> GLB_REQ_DEBUG_HOST_REQ_SHIFT) -+#define GLB_REQ_DEBUG_HOST_REQ_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_REQ_DEBUG_HOST_REQ_MASK) | \ -+ (((value) << GLB_REQ_DEBUG_HOST_REQ_SHIFT) & GLB_REQ_DEBUG_HOST_REQ_MASK)) -+ -+/* GLB_ACK_IRQ_MASK register */ -+#define GLB_ACK_IRQ_MASK_HALT_SHIFT 0 -+#define GLB_ACK_IRQ_MASK_HALT_MASK (0x1 << GLB_ACK_IRQ_MASK_HALT_SHIFT) -+#define GLB_ACK_IRQ_MASK_HALT_GET(reg_val) (((reg_val)&GLB_ACK_IRQ_MASK_HALT_MASK) >> GLB_ACK_IRQ_MASK_HALT_SHIFT) -+#define GLB_ACK_IRQ_MASK_HALT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_HALT_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_HALT_SHIFT) & GLB_ACK_IRQ_MASK_HALT_MASK)) -+#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT 1 -+#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT) -+#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK) >> GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT) -+#define GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_SHIFT) & GLB_ACK_IRQ_MASK_CFG_PROGRESS_TIMER_MASK)) -+#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT 2 -+#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT) -+#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK) >> GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT) -+#define GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_SHIFT) & GLB_ACK_IRQ_MASK_CFG_ALLOC_EN_MASK)) -+#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT 3 -+#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK (0x1 << GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT) -+#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK) >> GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT) -+#define GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_SHIFT) & GLB_ACK_IRQ_MASK_CFG_PWROFF_TIMER_MASK)) -+#define GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT 4 -+#define GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK (0x1 << GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT) -+#define GLB_ACK_IRQ_MASK_PROTM_ENTER_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK) >> GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT) -+#define GLB_ACK_IRQ_MASK_PROTM_ENTER_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_PROTM_ENTER_SHIFT) & GLB_ACK_IRQ_MASK_PROTM_ENTER_MASK)) -+#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT 5 -+#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT) -+#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK) >> GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT) -+#define GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_SHIFT) & GLB_ACK_IRQ_MASK_PRFCNT_ENABLE_MASK)) -+#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT 6 -+#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT) -+#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK) >> GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT) -+#define GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_SHIFT) & GLB_ACK_IRQ_MASK_PRFCNT_SAMPLE_MASK)) -+#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT 7 -+#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK (0x1 << GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT) -+#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK) >> GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT) -+#define GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_COUNTER_ENABLE_SHIFT) & GLB_ACK_IRQ_MASK_COUNTER_ENABLE_MASK)) -+#define GLB_ACK_IRQ_MASK_PING_SHIFT 8 -+#define GLB_ACK_IRQ_MASK_PING_MASK (0x1 << GLB_ACK_IRQ_MASK_PING_SHIFT) -+#define GLB_ACK_IRQ_MASK_PING_GET(reg_val) (((reg_val)&GLB_ACK_IRQ_MASK_PING_MASK) >> GLB_ACK_IRQ_MASK_PING_SHIFT) -+#define GLB_ACK_IRQ_MASK_PING_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_PING_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_PING_SHIFT) & GLB_ACK_IRQ_MASK_PING_MASK)) -+#define GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_SHIFT 9 -+#define GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK \ -+ (0x1 << GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_SHIFT) -+#define GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK) >> \ -+ GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_SHIFT) -+#define GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_SHIFT) & \ -+ GLB_ACK_IRQ_MASK_FIRMWARE_CONFIG_UPDATE_MASK)) -+#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT 20 -+#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) -+#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK) >> GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) -+#define GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_SHIFT) & GLB_ACK_IRQ_MASK_INACTIVE_COMPUTE_MASK)) -+#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT 21 -+#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT) -+#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK) >> GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT) -+#define GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_SHIFT) & GLB_ACK_IRQ_MASK_INACTIVE_FRAGMENT_MASK)) -+#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT 22 -+#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK (0x1 << GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT) -+#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK) >> GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT) -+#define GLB_ACK_IRQ_MASK_INACTIVE_TILER_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_INACTIVE_TILER_SHIFT) & GLB_ACK_IRQ_MASK_INACTIVE_TILER_MASK)) -+#define GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT 23 -+#define GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK (0x1 << GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) -+#define GLB_ACK_IRQ_MASK_PROTM_EXIT_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK) >> GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) -+#define GLB_ACK_IRQ_MASK_PROTM_EXIT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_PROTM_EXIT_SHIFT) & GLB_ACK_IRQ_MASK_PROTM_EXIT_MASK)) -+#define GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SHIFT 24 -+#define GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK \ -+ (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SHIFT) -+#define GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK) >> \ -+ GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SHIFT) -+#define GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_SHIFT) & \ -+ GLB_ACK_IRQ_MASK_PRFCNT_THRESHOLD_MASK)) -+#define GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SHIFT 25 -+#define GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK \ -+ (0x1 << GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SHIFT) -+#define GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK) >> \ -+ GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SHIFT) -+#define GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_SHIFT) & \ -+ GLB_ACK_IRQ_MASK_PRFCNT_OVERFLOW_MASK)) -+#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT 30 -+#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK (0x1 << GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) -+#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK) >> GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) -+#define GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_SHIFT) & GLB_ACK_IRQ_MASK_DEBUG_CSF_REQ_MASK)) -+#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT 31 -+#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK (0x1 << GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT) -+#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK) >> GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT) -+#define GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK) | \ -+ (((value) << GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_SHIFT) & GLB_ACK_IRQ_MASK_DEBUG_HOST_REQ_MASK)) -+ -+/* GLB_PROGRESS_TIMER register */ -+#define GLB_PROGRESS_TIMER_TIMEOUT_SHIFT 0 -+#define GLB_PROGRESS_TIMER_TIMEOUT_MASK (0xFFFFFFFF << GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) -+#define GLB_PROGRESS_TIMER_TIMEOUT_GET(reg_val) \ -+ (((reg_val)&GLB_PROGRESS_TIMER_TIMEOUT_MASK) >> GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) -+#define GLB_PROGRESS_TIMER_TIMEOUT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_PROGRESS_TIMER_TIMEOUT_MASK) | \ -+ (((value) << GLB_PROGRESS_TIMER_TIMEOUT_SHIFT) & GLB_PROGRESS_TIMER_TIMEOUT_MASK)) -+ -+/* GLB_PWROFF_TIMER register */ -+#define GLB_PWROFF_TIMER_TIMEOUT_SHIFT 0 -+#define GLB_PWROFF_TIMER_TIMEOUT_MASK (0x7FFFFFFF << GLB_PWROFF_TIMER_TIMEOUT_SHIFT) -+#define GLB_PWROFF_TIMER_TIMEOUT_GET(reg_val) \ -+ (((reg_val)&GLB_PWROFF_TIMER_TIMEOUT_MASK) >> GLB_PWROFF_TIMER_TIMEOUT_SHIFT) -+#define GLB_PWROFF_TIMER_TIMEOUT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_PWROFF_TIMER_TIMEOUT_MASK) | \ -+ (((value) << GLB_PWROFF_TIMER_TIMEOUT_SHIFT) & GLB_PWROFF_TIMER_TIMEOUT_MASK)) -+#define GLB_PWROFF_TIMER_TIMER_SOURCE_SHIFT 31 -+#define GLB_PWROFF_TIMER_TIMER_SOURCE_MASK (0x1 << GLB_PWROFF_TIMER_TIMER_SOURCE_SHIFT) -+#define GLB_PWROFF_TIMER_TIMER_SOURCE_GET(reg_val) \ -+ (((reg_val)&GLB_PWROFF_TIMER_TIMER_SOURCE_MASK) >> GLB_PWROFF_TIMER_TIMER_SOURCE_SHIFT) -+#define GLB_PWROFF_TIMER_TIMER_SOURCE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_PWROFF_TIMER_TIMER_SOURCE_MASK) | \ -+ (((value) << GLB_PWROFF_TIMER_TIMER_SOURCE_SHIFT) & GLB_PWROFF_TIMER_TIMER_SOURCE_MASK)) -+/* GLB_PWROFF_TIMER_TIMER_SOURCE values */ -+#define GLB_PWROFF_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP 0x0 -+#define GLB_PWROFF_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1 -+/* End of GLB_PWROFF_TIMER_TIMER_SOURCE values */ -+ -+/* GLB_ALLOC_EN register */ -+#define GLB_ALLOC_EN_MASK_SHIFT 0 -+#define GLB_ALLOC_EN_MASK_MASK (0xFFFFFFFFFFFFFFFF << GLB_ALLOC_EN_MASK_SHIFT) -+#define GLB_ALLOC_EN_MASK_GET(reg_val) (((reg_val)&GLB_ALLOC_EN_MASK_MASK) >> GLB_ALLOC_EN_MASK_SHIFT) -+#define GLB_ALLOC_EN_MASK_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ALLOC_EN_MASK_MASK) | (((value) << GLB_ALLOC_EN_MASK_SHIFT) & GLB_ALLOC_EN_MASK_MASK)) -+ -+/* GLB_PROTM_COHERENCY register */ -+#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT 0 -+#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK \ -+ (0xFFFFFFFF << GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT) -+#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_GET(reg_val) \ -+ (((reg_val)&GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK) >> \ -+ GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT) -+#define GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK) | \ -+ (((value) << GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_SHIFT) & \ -+ GLB_PROTM_COHERENCY_L2_CACHE_PROTOCOL_SELECT_MASK)) -+/* End of GLB_INPUT_BLOCK register set definitions */ -+ -+/* GLB_OUTPUT_BLOCK register set definitions */ -+ -+/* GLB_ACK register */ -+#define GLB_ACK_CFG_PROGRESS_TIMER_SHIFT 1 -+#define GLB_ACK_CFG_PROGRESS_TIMER_MASK (0x1 << GLB_ACK_CFG_PROGRESS_TIMER_SHIFT) -+#define GLB_ACK_CFG_PROGRESS_TIMER_GET(reg_val) \ -+ (((reg_val)&GLB_ACK_CFG_PROGRESS_TIMER_MASK) >> GLB_ACK_CFG_PROGRESS_TIMER_SHIFT) -+#define GLB_ACK_CFG_PROGRESS_TIMER_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_CFG_PROGRESS_TIMER_MASK) | \ -+ (((value) << GLB_ACK_CFG_PROGRESS_TIMER_SHIFT) & GLB_ACK_CFG_PROGRESS_TIMER_MASK)) -+#define GLB_ACK_CFG_ALLOC_EN_SHIFT 2 -+#define GLB_ACK_CFG_ALLOC_EN_MASK (0x1 << GLB_ACK_CFG_ALLOC_EN_SHIFT) -+#define GLB_ACK_CFG_ALLOC_EN_GET(reg_val) (((reg_val)&GLB_ACK_CFG_ALLOC_EN_MASK) >> GLB_ACK_CFG_ALLOC_EN_SHIFT) -+#define GLB_ACK_CFG_ALLOC_EN_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_ACK_CFG_ALLOC_EN_MASK) | (((value) << GLB_ACK_CFG_ALLOC_EN_SHIFT) & GLB_ACK_CFG_ALLOC_EN_MASK)) -+/* End of GLB_OUTPUT_BLOCK register set definitions */ -+ -+/* The following register and fields are for headers before 10.x.7/11.x.4 */ -+#define GLB_REQ_IDLE_ENABLE_SHIFT (10) -+#define GLB_REQ_REQ_IDLE_ENABLE (1 << GLB_REQ_IDLE_ENABLE_SHIFT) -+#define GLB_REQ_REQ_IDLE_DISABLE (0 << GLB_REQ_IDLE_ENABLE_SHIFT) -+#define GLB_REQ_IDLE_ENABLE_MASK (0x1 << GLB_REQ_IDLE_ENABLE_SHIFT) -+#define GLB_REQ_IDLE_DISABLE_MASK (0x1 << GLB_REQ_IDLE_ENABLE_SHIFT) -+#define GLB_REQ_IDLE_EVENT_SHIFT (26) -+#define GLB_REQ_IDLE_EVENT_MASK (0x1 << GLB_REQ_IDLE_EVENT_SHIFT) -+#define GLB_ACK_IDLE_ENABLE_SHIFT (10) -+#define GLB_ACK_ACK_IDLE_ENABLE (1 << GLB_ACK_IDLE_ENABLE_SHIFT) -+#define GLB_ACK_ACK_IDLE_DISABLE (0 << GLB_ACK_IDLE_ENABLE_SHIFT) -+#define GLB_ACK_IDLE_ENABLE_MASK (0x1 << GLB_ACK_IDLE_ENABLE_SHIFT) -+#define GLB_ACK_IDLE_EVENT_SHIFT (26) -+#define GLB_ACK_IDLE_EVENT_MASK (0x1 << GLB_REQ_IDLE_EVENT_SHIFT) -+ -+#define GLB_ACK_IRQ_MASK_IDLE_EVENT_SHIFT (26) -+#define GLB_ACK_IRQ_MASK_IDLE_EVENT_MASK (0x1 << GLB_ACK_IRQ_MASK_IDLE_EVENT_SHIFT) -+ -+#define GLB_IDLE_TIMER (0x0080) -+/* GLB_IDLE_TIMER register */ -+#define GLB_IDLE_TIMER_TIMEOUT_SHIFT (0) -+#define GLB_IDLE_TIMER_TIMEOUT_MASK ((0x7FFFFFFF) << GLB_IDLE_TIMER_TIMEOUT_SHIFT) -+#define GLB_IDLE_TIMER_TIMEOUT_GET(reg_val) (((reg_val)&GLB_IDLE_TIMER_TIMEOUT_MASK) >> GLB_IDLE_TIMER_TIMEOUT_SHIFT) -+#define GLB_IDLE_TIMER_TIMEOUT_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_IDLE_TIMER_TIMEOUT_MASK) | \ -+ (((value) << GLB_IDLE_TIMER_TIMEOUT_SHIFT) & GLB_IDLE_TIMER_TIMEOUT_MASK)) -+#define GLB_IDLE_TIMER_TIMER_SOURCE_SHIFT (31) -+#define GLB_IDLE_TIMER_TIMER_SOURCE_MASK ((0x1) << GLB_IDLE_TIMER_TIMER_SOURCE_SHIFT) -+#define GLB_IDLE_TIMER_TIMER_SOURCE_GET(reg_val) \ -+ (((reg_val)&GLB_IDLE_TIMER_TIMER_SOURCE_MASK) >> GLB_IDLE_TIMER_TIMER_SOURCE_SHIFT) -+#define GLB_IDLE_TIMER_TIMER_SOURCE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_IDLE_TIMER_TIMER_SOURCE_MASK) | \ -+ (((value) << GLB_IDLE_TIMER_TIMER_SOURCE_SHIFT) & GLB_IDLE_TIMER_TIMER_SOURCE_MASK)) -+/* GLB_IDLE_TIMER_TIMER_SOURCE values */ -+#define GLB_IDLE_TIMER_TIMER_SOURCE_SYSTEM_TIMESTAMP 0x0 -+#define GLB_IDLE_TIMER_TIMER_SOURCE_GPU_COUNTER 0x1 -+/* End of GLB_IDLE_TIMER_TIMER_SOURCE values */ -+ -+/* GLB_INSTR_FEATURES register */ -+#define GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_SHIFT (0) -+#define GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_MASK ((u32)0xF << GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_SHIFT) -+#define GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_GET(reg_val) \ -+ (((reg_val)&GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_MASK) >> GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_SHIFT) -+#define GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_MASK) | \ -+ (((value) << GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_SHIFT) & GLB_INSTR_FEATURES_OFFSET_UPDATE_RATE_MASK)) -+#define GLB_INSTR_FEATURES_EVENT_SIZE_MAX_SHIFT (4) -+#define GLB_INSTR_FEATURES_EVENT_SIZE_MAX_MASK ((u32)0xF << GLB_INSTR_FEATURES_EVENT_SIZE_MAX_SHIFT) -+#define GLB_INSTR_FEATURES_EVENT_SIZE_MAX_GET(reg_val) \ -+ (((reg_val)&GLB_INSTR_FEATURES_EVENT_SIZE_MAX_MASK) >> GLB_INSTR_FEATURES_EVENT_SIZE_MAX_SHIFT) -+#define GLB_INSTR_FEATURES_EVENT_SIZE_MAX_SET(reg_val, value) \ -+ (((reg_val) & ~GLB_INSTR_FEATURES_EVENT_SIZE_MAX_MASK) | \ -+ (((value) << GLB_INSTR_FEATURES_EVENT_SIZE_MAX_SHIFT) & GLB_INSTR_FEATURES_EVENT_SIZE_MAX_MASK)) -+ -+#define CSG_STATUS_STATE (0x0018) /* CSG state status register */ -+/* CSG_STATUS_STATE register */ -+#define CSG_STATUS_STATE_IDLE_SHIFT (0) -+#define CSG_STATUS_STATE_IDLE_MASK ((0x1) << CSG_STATUS_STATE_IDLE_SHIFT) -+#define CSG_STATUS_STATE_IDLE_GET(reg_val) \ -+ (((reg_val)&CSG_STATUS_STATE_IDLE_MASK) >> CSG_STATUS_STATE_IDLE_SHIFT) -+#define CSG_STATUS_STATE_IDLE_SET(reg_val, value) \ -+ (((reg_val) & ~CSG_STATUS_STATE_IDLE_MASK) | \ -+ (((value) << CSG_STATUS_STATE_IDLE_SHIFT) & CSG_STATUS_STATE_IDLE_MASK)) -+ -+#endif /* _UAPI_GPU_CSF_REGISTERS_H_ */ -diff --git a/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_kbase_csf_ioctl.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_kbase_csf_ioctl.h -new file mode 100644 -index 0000000..d2d7ce2 ---- /dev/null -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/csf/mali_kbase_csf_ioctl.h -@@ -0,0 +1,433 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _UAPI_KBASE_CSF_IOCTL_H_ -+#define _UAPI_KBASE_CSF_IOCTL_H_ -+ -+#include <asm-generic/ioctl.h> -+#include <linux/types.h> -+ -+/* -+ * 1.0: -+ * - CSF IOCTL header separated from JM -+ * 1.1: -+ * - Add a new priority level BASE_QUEUE_GROUP_PRIORITY_REALTIME -+ * - Add ioctl 54: This controls the priority setting. -+ * 1.2: -+ * - Add new CSF GPU_FEATURES register into the property structure -+ * returned by KBASE_IOCTL_GET_GPUPROPS -+ * 1.3: -+ * - Add __u32 group_uid member to -+ * &struct_kbase_ioctl_cs_queue_group_create.out -+ * 1.4: -+ * - Replace padding in kbase_ioctl_cs_get_glb_iface with -+ * instr_features member of same size -+ * 1.5: -+ * - Add ioctl 40: kbase_ioctl_cs_queue_register_ex, this is a new -+ * queue registration call with extended format for supporting CS -+ * trace configurations with CSF trace_command. -+ */ -+ -+#define BASE_UK_VERSION_MAJOR 1 -+#define BASE_UK_VERSION_MINOR 5 -+ -+/** -+ * struct kbase_ioctl_version_check - Check version compatibility between -+ * kernel and userspace -+ * -+ * @major: Major version number -+ * @minor: Minor version number -+ */ -+struct kbase_ioctl_version_check { -+ __u16 major; -+ __u16 minor; -+}; -+ -+#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ -+ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) -+ -+ -+/** -+ * struct kbase_ioctl_cs_queue_register - Register a GPU command queue with the -+ * base back-end -+ * -+ * @buffer_gpu_addr: GPU address of the buffer backing the queue -+ * @buffer_size: Size of the buffer in bytes -+ * @priority: Priority of the queue within a group when run within a process -+ * @padding: Currently unused, must be zero -+ * -+ * @Note: There is an identical sub-section in kbase_ioctl_cs_queue_register_ex. -+ * Any change of this struct should also be mirrored to the latter. -+ */ -+struct kbase_ioctl_cs_queue_register { -+ __u64 buffer_gpu_addr; -+ __u32 buffer_size; -+ __u8 priority; -+ __u8 padding[3]; -+}; -+ -+#define KBASE_IOCTL_CS_QUEUE_REGISTER \ -+ _IOW(KBASE_IOCTL_TYPE, 36, struct kbase_ioctl_cs_queue_register) -+ -+/** -+ * struct kbase_ioctl_cs_queue_kick - Kick the GPU command queue group scheduler -+ * to notify that a queue has been updated -+ * -+ * @buffer_gpu_addr: GPU address of the buffer backing the queue -+ */ -+struct kbase_ioctl_cs_queue_kick { -+ __u64 buffer_gpu_addr; -+}; -+ -+#define KBASE_IOCTL_CS_QUEUE_KICK \ -+ _IOW(KBASE_IOCTL_TYPE, 37, struct kbase_ioctl_cs_queue_kick) -+ -+/** -+ * union kbase_ioctl_cs_queue_bind - Bind a GPU command queue to a group -+ * -+ * @in: Input parameters -+ * @in.buffer_gpu_addr: GPU address of the buffer backing the queue -+ * @in.group_handle: Handle of the group to which the queue should be bound -+ * @in.csi_index: Index of the CSF interface the queue should be bound to -+ * @in.padding: Currently unused, must be zero -+ * @out: Output parameters -+ * @out.mmap_handle: Handle to be used for creating the mapping of CS -+ * input/output pages -+ */ -+union kbase_ioctl_cs_queue_bind { -+ struct { -+ __u64 buffer_gpu_addr; -+ __u8 group_handle; -+ __u8 csi_index; -+ __u8 padding[6]; -+ } in; -+ struct { -+ __u64 mmap_handle; -+ } out; -+}; -+ -+#define KBASE_IOCTL_CS_QUEUE_BIND \ -+ _IOWR(KBASE_IOCTL_TYPE, 39, union kbase_ioctl_cs_queue_bind) -+ -+/** -+ * struct kbase_ioctl_cs_queue_register_ex - Register a GPU command queue with the -+ * base back-end in extended format, -+ * involving trace buffer configuration -+ * -+ * @buffer_gpu_addr: GPU address of the buffer backing the queue -+ * @buffer_size: Size of the buffer in bytes -+ * @priority: Priority of the queue within a group when run within a process -+ * @padding: Currently unused, must be zero -+ * @ex_offset_var_addr: GPU address of the trace buffer write offset variable -+ * @ex_buffer_base: Trace buffer GPU base address for the queue -+ * @ex_buffer_size: Size of the trace buffer in bytes -+ * @ex_event_size: Trace event write size, in log2 designation -+ * @ex_event_state: Trace event states configuration -+ * @ex_padding: Currently unused, must be zero -+ * -+ * @Note: There is an identical sub-section at the start of this struct to that -+ * of @ref kbase_ioctl_cs_queue_register. Any change of this sub-section -+ * must also be mirrored to the latter. Following the said sub-section, -+ * the remaining fields forms the extension, marked with ex_*. -+ */ -+struct kbase_ioctl_cs_queue_register_ex { -+ __u64 buffer_gpu_addr; -+ __u32 buffer_size; -+ __u8 priority; -+ __u8 padding[3]; -+ __u64 ex_offset_var_addr; -+ __u64 ex_buffer_base; -+ __u32 ex_buffer_size; -+ __u8 ex_event_size; -+ __u8 ex_event_state; -+ __u8 ex_padding[2]; -+}; -+ -+#define KBASE_IOCTL_CS_QUEUE_REGISTER_EX \ -+ _IOW(KBASE_IOCTL_TYPE, 40, struct kbase_ioctl_cs_queue_register_ex) -+ -+/** -+ * struct kbase_ioctl_cs_queue_terminate - Terminate a GPU command queue -+ * -+ * @buffer_gpu_addr: GPU address of the buffer backing the queue -+ */ -+struct kbase_ioctl_cs_queue_terminate { -+ __u64 buffer_gpu_addr; -+}; -+ -+#define KBASE_IOCTL_CS_QUEUE_TERMINATE \ -+ _IOW(KBASE_IOCTL_TYPE, 41, struct kbase_ioctl_cs_queue_terminate) -+ -+/** -+ * union kbase_ioctl_cs_queue_group_create - Create a GPU command queue group -+ * @in: Input parameters -+ * @in.tiler_mask: Mask of tiler endpoints the group is allowed to use. -+ * @in.fragment_mask: Mask of fragment endpoints the group is allowed to use. -+ * @in.compute_mask: Mask of compute endpoints the group is allowed to use. -+ * @in.cs_min: Minimum number of CSs required. -+ * @in.priority: Queue group's priority within a process. -+ * @in.tiler_max: Maximum number of tiler endpoints the group is allowed -+ * to use. -+ * @in.fragment_max: Maximum number of fragment endpoints the group is -+ * allowed to use. -+ * @in.compute_max: Maximum number of compute endpoints the group is allowed -+ * to use. -+ * @in.padding: Currently unused, must be zero -+ * @out: Output parameters -+ * @out.group_handle: Handle of a newly created queue group. -+ * @out.padding: Currently unused, must be zero -+ * @out.group_uid: UID of the queue group available to base. -+ */ -+union kbase_ioctl_cs_queue_group_create { -+ struct { -+ __u64 tiler_mask; -+ __u64 fragment_mask; -+ __u64 compute_mask; -+ __u8 cs_min; -+ __u8 priority; -+ __u8 tiler_max; -+ __u8 fragment_max; -+ __u8 compute_max; -+ __u8 padding[3]; -+ -+ } in; -+ struct { -+ __u8 group_handle; -+ __u8 padding[3]; -+ __u32 group_uid; -+ } out; -+}; -+ -+#define KBASE_IOCTL_CS_QUEUE_GROUP_CREATE \ -+ _IOWR(KBASE_IOCTL_TYPE, 42, union kbase_ioctl_cs_queue_group_create) -+ -+/** -+ * struct kbase_ioctl_cs_queue_group_term - Terminate a GPU command queue group -+ * -+ * @group_handle: Handle of the queue group to be terminated -+ * @padding: Padding to round up to a multiple of 8 bytes, must be zero -+ */ -+struct kbase_ioctl_cs_queue_group_term { -+ __u8 group_handle; -+ __u8 padding[7]; -+}; -+ -+#define KBASE_IOCTL_CS_QUEUE_GROUP_TERMINATE \ -+ _IOW(KBASE_IOCTL_TYPE, 43, struct kbase_ioctl_cs_queue_group_term) -+ -+#define KBASE_IOCTL_CS_EVENT_SIGNAL \ -+ _IO(KBASE_IOCTL_TYPE, 44) -+ -+typedef __u8 base_kcpu_queue_id; /* We support up to 256 active KCPU queues */ -+ -+/** -+ * struct kbase_ioctl_kcpu_queue_new - Create a KCPU command queue -+ * -+ * @id: ID of the new command queue returned by the kernel -+ * @padding: Padding to round up to a multiple of 8 bytes, must be zero -+ */ -+struct kbase_ioctl_kcpu_queue_new { -+ base_kcpu_queue_id id; -+ __u8 padding[7]; -+}; -+ -+#define KBASE_IOCTL_KCPU_QUEUE_CREATE \ -+ _IOR(KBASE_IOCTL_TYPE, 45, struct kbase_ioctl_kcpu_queue_new) -+ -+/** -+ * struct kbase_ioctl_kcpu_queue_delete - Destroy a KCPU command queue -+ * -+ * @id: ID of the command queue to be destroyed -+ * @padding: Padding to round up to a multiple of 8 bytes, must be zero -+ */ -+struct kbase_ioctl_kcpu_queue_delete { -+ base_kcpu_queue_id id; -+ __u8 padding[7]; -+}; -+ -+#define KBASE_IOCTL_KCPU_QUEUE_DELETE \ -+ _IOW(KBASE_IOCTL_TYPE, 46, struct kbase_ioctl_kcpu_queue_delete) -+ -+/** -+ * struct kbase_ioctl_kcpu_queue_enqueue - Enqueue commands into the KCPU queue -+ * -+ * @addr: Memory address of an array of struct base_kcpu_queue_command -+ * @nr_commands: Number of commands in the array -+ * @id: kcpu queue identifier, returned by KBASE_IOCTL_KCPU_QUEUE_CREATE ioctl -+ * @padding: Padding to round up to a multiple of 8 bytes, must be zero -+ */ -+struct kbase_ioctl_kcpu_queue_enqueue { -+ __u64 addr; -+ __u32 nr_commands; -+ base_kcpu_queue_id id; -+ __u8 padding[3]; -+}; -+ -+#define KBASE_IOCTL_KCPU_QUEUE_ENQUEUE \ -+ _IOW(KBASE_IOCTL_TYPE, 47, struct kbase_ioctl_kcpu_queue_enqueue) -+ -+/** -+ * union kbase_ioctl_cs_tiler_heap_init - Initialize chunked tiler memory heap -+ * @in: Input parameters -+ * @in.chunk_size: Size of each chunk. -+ * @in.initial_chunks: Initial number of chunks that heap will be created with. -+ * @in.max_chunks: Maximum number of chunks that the heap is allowed to use. -+ * @in.target_in_flight: Number of render-passes that the driver should attempt to -+ * keep in flight for which allocation of new chunks is -+ * allowed. -+ * @in.group_id: Group ID to be used for physical allocations. -+ * @in.padding: Padding -+ * @out: Output parameters -+ * @out.gpu_heap_va: GPU VA (virtual address) of Heap context that was set up -+ * for the heap. -+ * @out.first_chunk_va: GPU VA of the first chunk allocated for the heap, -+ * actually points to the header of heap chunk and not to -+ * the low address of free memory in the chunk. -+ */ -+union kbase_ioctl_cs_tiler_heap_init { -+ struct { -+ __u32 chunk_size; -+ __u32 initial_chunks; -+ __u32 max_chunks; -+ __u16 target_in_flight; -+ __u8 group_id; -+ __u8 padding; -+ } in; -+ struct { -+ __u64 gpu_heap_va; -+ __u64 first_chunk_va; -+ } out; -+}; -+ -+#define KBASE_IOCTL_CS_TILER_HEAP_INIT \ -+ _IOWR(KBASE_IOCTL_TYPE, 48, union kbase_ioctl_cs_tiler_heap_init) -+ -+/** -+ * struct kbase_ioctl_cs_tiler_heap_term - Terminate a chunked tiler heap -+ * instance -+ * -+ * @gpu_heap_va: GPU VA of Heap context that was set up for the heap. -+ */ -+struct kbase_ioctl_cs_tiler_heap_term { -+ __u64 gpu_heap_va; -+}; -+ -+#define KBASE_IOCTL_CS_TILER_HEAP_TERM \ -+ _IOW(KBASE_IOCTL_TYPE, 49, struct kbase_ioctl_cs_tiler_heap_term) -+ -+/** -+ * union kbase_ioctl_cs_get_glb_iface - Request the global control block -+ * of CSF interface capabilities -+ * -+ * @in: Input parameters -+ * @in.max_group_num: The maximum number of groups to be read. Can be 0, in -+ * which case groups_ptr is unused. -+ * @in.max_total_stream _num: The maximum number of CSs to be read. Can be 0, in -+ * which case streams_ptr is unused. -+ * @in.groups_ptr: Pointer where to store all the group data (sequentially). -+ * @in.streams_ptr: Pointer where to store all the CS data (sequentially). -+ * @out: Output parameters -+ * @out.glb_version: Global interface version. -+ * @out.features: Bit mask of features (e.g. whether certain types of job -+ * can be suspended). -+ * @out.group_num: Number of CSGs supported. -+ * @out.prfcnt_size: Size of CSF performance counters, in bytes. Bits 31:16 -+ * hold the size of firmware performance counter data -+ * and 15:0 hold the size of hardware performance counter -+ * data. -+ * @out.total_stream_num: Total number of CSs, summed across all groups. -+ * @out.instr_features: Instrumentation features. Bits 7:4 hold the maximum -+ * size of events. Bits 3:0 hold the offset update rate. -+ * (csf >= 1.1.0) -+ * -+ */ -+union kbase_ioctl_cs_get_glb_iface { -+ struct { -+ __u32 max_group_num; -+ __u32 max_total_stream_num; -+ __u64 groups_ptr; -+ __u64 streams_ptr; -+ } in; -+ struct { -+ __u32 glb_version; -+ __u32 features; -+ __u32 group_num; -+ __u32 prfcnt_size; -+ __u32 total_stream_num; -+ __u32 instr_features; -+ } out; -+}; -+ -+#define KBASE_IOCTL_CS_GET_GLB_IFACE \ -+ _IOWR(KBASE_IOCTL_TYPE, 51, union kbase_ioctl_cs_get_glb_iface) -+ -+struct kbase_ioctl_cs_cpu_queue_info { -+ __u64 buffer; -+ __u64 size; -+}; -+ -+#define KBASE_IOCTL_VERSION_CHECK \ -+ _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) -+ -+#define KBASE_IOCTL_CS_CPU_QUEUE_DUMP \ -+ _IOW(KBASE_IOCTL_TYPE, 53, struct kbase_ioctl_cs_cpu_queue_info) -+ -+/*************** -+ * test ioctls * -+ ***************/ -+#if MALI_UNIT_TEST -+/* These ioctls are purely for test purposes and are not used in the production -+ * driver, they therefore may change without notice -+ */ -+ -+/** -+ * struct kbase_ioctl_cs_event_memory_write - Write an event memory address -+ * @cpu_addr: Memory address to write -+ * @value: Value to write -+ * @padding: Currently unused, must be zero -+ */ -+struct kbase_ioctl_cs_event_memory_write { -+ __u64 cpu_addr; -+ __u8 value; -+ __u8 padding[7]; -+}; -+ -+/** -+ * union kbase_ioctl_cs_event_memory_read - Read an event memory address -+ * @in: Input parameters -+ * @in.cpu_addr: Memory address to read -+ * @out: Output parameters -+ * @out.value: Value read -+ * @out.padding: Currently unused, must be zero -+ */ -+union kbase_ioctl_cs_event_memory_read { -+ struct { -+ __u64 cpu_addr; -+ } in; -+ struct { -+ __u8 value; -+ __u8 padding[7]; -+ } out; -+}; -+ -+#endif /* MALI_UNIT_TEST */ -+ -+#endif /* _UAPI_KBASE_CSF_IOCTL_H_ */ -diff --git a/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_csf.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_csf.h -new file mode 100644 -index 0000000..2041739 ---- /dev/null -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_csf.h -@@ -0,0 +1,335 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _UAPI_KBASE_GPU_REGMAP_CSF_H_ -+#define _UAPI_KBASE_GPU_REGMAP_CSF_H_ -+ -+#include <linux/types.h> -+ -+#if !MALI_USE_CSF && defined(__KERNEL__) -+#error "Cannot be compiled with JM" -+#endif -+ -+/* IPA control registers */ -+ -+#define IPA_CONTROL_BASE 0x40000 -+#define IPA_CONTROL_REG(r) (IPA_CONTROL_BASE+(r)) -+#define COMMAND 0x000 /* (WO) Command register */ -+#define STATUS 0x004 /* (RO) Status register */ -+#define TIMER 0x008 /* (RW) Timer control register */ -+ -+#define SELECT_CSHW_LO 0x010 /* (RW) Counter select for CS hardware, low word */ -+#define SELECT_CSHW_HI 0x014 /* (RW) Counter select for CS hardware, high word */ -+#define SELECT_MEMSYS_LO 0x018 /* (RW) Counter select for Memory system, low word */ -+#define SELECT_MEMSYS_HI 0x01C /* (RW) Counter select for Memory system, high word */ -+#define SELECT_TILER_LO 0x020 /* (RW) Counter select for Tiler cores, low word */ -+#define SELECT_TILER_HI 0x024 /* (RW) Counter select for Tiler cores, high word */ -+#define SELECT_SHADER_LO 0x028 /* (RW) Counter select for Shader cores, low word */ -+#define SELECT_SHADER_HI 0x02C /* (RW) Counter select for Shader cores, high word */ -+ -+/* Accumulated counter values for CS hardware */ -+#define VALUE_CSHW_BASE 0x100 -+#define VALUE_CSHW_REG_LO(n) (VALUE_CSHW_BASE + ((n) << 3)) /* (RO) Counter value #n, low word */ -+#define VALUE_CSHW_REG_HI(n) (VALUE_CSHW_BASE + ((n) << 3) + 4) /* (RO) Counter value #n, high word */ -+ -+/* Accumulated counter values for memory system */ -+#define VALUE_MEMSYS_BASE 0x140 -+#define VALUE_MEMSYS_REG_LO(n) (VALUE_MEMSYS_BASE + ((n) << 3)) /* (RO) Counter value #n, low word */ -+#define VALUE_MEMSYS_REG_HI(n) (VALUE_MEMSYS_BASE + ((n) << 3) + 4) /* (RO) Counter value #n, high word */ -+ -+#define VALUE_TILER_BASE 0x180 -+#define VALUE_TILER_REG_LO(n) (VALUE_TILER_BASE + ((n) << 3)) /* (RO) Counter value #n, low word */ -+#define VALUE_TILER_REG_HI(n) (VALUE_TILER_BASE + ((n) << 3) + 4) /* (RO) Counter value #n, high word */ -+ -+#define VALUE_SHADER_BASE 0x1C0 -+#define VALUE_SHADER_REG_LO(n) (VALUE_SHADER_BASE + ((n) << 3)) /* (RO) Counter value #n, low word */ -+#define VALUE_SHADER_REG_HI(n) (VALUE_SHADER_BASE + ((n) << 3) + 4) /* (RO) Counter value #n, high word */ -+ -+#include "../../csf/mali_gpu_csf_control_registers.h" -+ -+/* Set to implementation defined, outer caching */ -+#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull -+/* Set to write back memory, outer caching */ -+#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull -+/* Set to inner non-cacheable, outer-non-cacheable -+ * Setting defined by the alloc bits is ignored, but set to a valid encoding: -+ * - no-alloc on read -+ * - no alloc on write -+ */ -+#define AS_MEMATTR_AARCH64_NON_CACHEABLE 0x4Cull -+/* Set to shared memory, that is inner cacheable on ACE and inner or outer -+ * shared, otherwise inner non-cacheable. -+ * Outer cacheable if inner or outer shared, otherwise outer non-cacheable. -+ */ -+#define AS_MEMATTR_AARCH64_SHARED 0x8ull -+ -+/* Symbols for default MEMATTR to use -+ * Default is - HW implementation defined caching -+ */ -+#define AS_MEMATTR_INDEX_DEFAULT 0 -+#define AS_MEMATTR_INDEX_DEFAULT_ACE 3 -+ -+/* HW implementation defined caching */ -+#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0 -+/* Force cache on */ -+#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1 -+/* Write-alloc */ -+#define AS_MEMATTR_INDEX_WRITE_ALLOC 2 -+/* Outer coherent, inner implementation defined policy */ -+#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3 -+/* Outer coherent, write alloc inner */ -+#define AS_MEMATTR_INDEX_OUTER_WA 4 -+/* Normal memory, inner non-cacheable, outer non-cacheable (ARMv8 mode only) */ -+#define AS_MEMATTR_INDEX_NON_CACHEABLE 5 -+/* Normal memory, shared between MCU and Host */ -+#define AS_MEMATTR_INDEX_SHARED 6 -+ -+/* Configuration bits for the CSF. */ -+#define CSF_CONFIG 0xF00 -+ -+/* CSF_CONFIG register */ -+#define CSF_CONFIG_FORCE_COHERENCY_FEATURES_SHIFT 2 -+ -+/* GPU control registers */ -+#define CORE_FEATURES 0x008 /* () Shader Core Features */ -+#define MCU_CONTROL 0x700 -+#define MCU_STATUS 0x704 -+ -+#define MCU_CNTRL_ENABLE (1 << 0) -+#define MCU_CNTRL_AUTO (1 << 1) -+#define MCU_CNTRL_DISABLE (0) -+ -+#define MCU_STATUS_HALTED (1 << 1) -+ -+#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory -+ * region base address, low word -+ */ -+#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory -+ * region base address, high word -+ */ -+#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter -+ * configuration -+ */ -+ -+#define PRFCNT_CSHW_EN 0x06C /* (RW) Performance counter -+ * enable for CS Hardware -+ */ -+ -+#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable -+ * flags for shader cores -+ */ -+#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable -+ * flags for tiler -+ */ -+#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable -+ * flags for MMU/L2 cache -+ */ -+ -+/* JOB IRQ flags */ -+#define JOB_IRQ_GLOBAL_IF (1 << 31) /* Global interface interrupt received */ -+ -+/* GPU_COMMAND codes */ -+#define GPU_COMMAND_CODE_NOP 0x00 /* No operation, nothing happens */ -+#define GPU_COMMAND_CODE_RESET 0x01 /* Reset the GPU */ -+#define GPU_COMMAND_CODE_PRFCNT 0x02 /* Clear or sample performance counters */ -+#define GPU_COMMAND_CODE_TIME 0x03 /* Configure time sources */ -+#define GPU_COMMAND_CODE_FLUSH_CACHES 0x04 /* Flush caches */ -+#define GPU_COMMAND_CODE_SET_PROTECTED_MODE 0x05 /* Places the GPU in protected mode */ -+#define GPU_COMMAND_CODE_FINISH_HALT 0x06 /* Halt CSF */ -+#define GPU_COMMAND_CODE_CLEAR_FAULT 0x07 /* Clear GPU_FAULTSTATUS and GPU_FAULTADDRESS, TODX */ -+ -+/* GPU_COMMAND_RESET payloads */ -+ -+/* This will leave the state of active jobs UNDEFINED, but will leave the external bus in a defined and idle state. -+ * Power domains will remain powered on. -+ */ -+#define GPU_COMMAND_RESET_PAYLOAD_FAST_RESET 0x00 -+ -+/* This will leave the state of active CSs UNDEFINED, but will leave the external bus in a defined and -+ * idle state. -+ */ -+#define GPU_COMMAND_RESET_PAYLOAD_SOFT_RESET 0x01 -+ -+/* This reset will leave the state of currently active streams UNDEFINED, will likely lose data, and may leave -+ * the system bus in an inconsistent state. Use only as a last resort when nothing else works. -+ */ -+#define GPU_COMMAND_RESET_PAYLOAD_HARD_RESET 0x02 -+ -+/* GPU_COMMAND_PRFCNT payloads */ -+#define GPU_COMMAND_PRFCNT_PAYLOAD_SAMPLE 0x01 /* Sample performance counters */ -+#define GPU_COMMAND_PRFCNT_PAYLOAD_CLEAR 0x02 /* Clear performance counters */ -+ -+/* GPU_COMMAND_TIME payloads */ -+#define GPU_COMMAND_TIME_DISABLE 0x00 /* Disable cycle counter */ -+#define GPU_COMMAND_TIME_ENABLE 0x01 /* Enable cycle counter */ -+ -+/* GPU_COMMAND_FLUSH_CACHES payloads */ -+#define GPU_COMMAND_FLUSH_PAYLOAD_NONE 0x00 /* No flush */ -+#define GPU_COMMAND_FLUSH_PAYLOAD_CLEAN 0x01 /* Clean the caches */ -+#define GPU_COMMAND_FLUSH_PAYLOAD_INVALIDATE 0x02 /* Invalidate the caches */ -+#define GPU_COMMAND_FLUSH_PAYLOAD_CLEAN_INVALIDATE 0x03 /* Clean and invalidate the caches */ -+ -+/* GPU_COMMAND command + payload */ -+#define GPU_COMMAND_CODE_PAYLOAD(opcode, payload) \ -+ ((__u32)opcode | ((__u32)payload << 8)) -+ -+/* Final GPU_COMMAND form */ -+/* No operation, nothing happens */ -+#define GPU_COMMAND_NOP \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_NOP, 0) -+ -+/* Stop all external bus interfaces, and then reset the entire GPU. */ -+#define GPU_COMMAND_SOFT_RESET \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_RESET, GPU_COMMAND_RESET_PAYLOAD_SOFT_RESET) -+ -+/* Immediately reset the entire GPU. */ -+#define GPU_COMMAND_HARD_RESET \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_RESET, GPU_COMMAND_RESET_PAYLOAD_HARD_RESET) -+ -+/* Clear all performance counters, setting them all to zero. */ -+#define GPU_COMMAND_PRFCNT_CLEAR \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_PRFCNT, GPU_COMMAND_PRFCNT_PAYLOAD_CLEAR) -+ -+/* Sample all performance counters, writing them out to memory */ -+#define GPU_COMMAND_PRFCNT_SAMPLE \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_PRFCNT, GPU_COMMAND_PRFCNT_PAYLOAD_SAMPLE) -+ -+/* Starts the cycle counter, and system timestamp propagation */ -+#define GPU_COMMAND_CYCLE_COUNT_START \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_TIME, GPU_COMMAND_TIME_ENABLE) -+ -+/* Stops the cycle counter, and system timestamp propagation */ -+#define GPU_COMMAND_CYCLE_COUNT_STOP \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_TIME, GPU_COMMAND_TIME_DISABLE) -+ -+/* Clean all caches */ -+#define GPU_COMMAND_CLEAN_CACHES \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_FLUSH_CACHES, GPU_COMMAND_FLUSH_PAYLOAD_CLEAN) -+ -+/* Clean and invalidate all caches */ -+#define GPU_COMMAND_CLEAN_INV_CACHES \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_FLUSH_CACHES, GPU_COMMAND_FLUSH_PAYLOAD_CLEAN_INVALIDATE) -+ -+/* Places the GPU in protected mode */ -+#define GPU_COMMAND_SET_PROTECTED_MODE \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_SET_PROTECTED_MODE, 0) -+ -+/* Halt CSF */ -+#define GPU_COMMAND_FINISH_HALT \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_FINISH_HALT, 0) -+ -+/* Clear GPU faults */ -+#define GPU_COMMAND_CLEAR_FAULT \ -+ GPU_COMMAND_CODE_PAYLOAD(GPU_COMMAND_CODE_CLEAR_FAULT, 0) -+ -+/* End Command Values */ -+ -+/* GPU_FAULTSTATUS register */ -+#define GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT 0 -+#define GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK (0xFFul) -+#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GET(reg_val) \ -+ (((reg_val)&GPU_FAULTSTATUS_EXCEPTION_TYPE_MASK) \ -+ >> GPU_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) -+#define GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT 8 -+#define GPU_FAULTSTATUS_ACCESS_TYPE_MASK \ -+ (0x3ul << GPU_FAULTSTATUS_ACCESS_TYPE_SHIFT) -+ -+#define GPU_FAULTSTATUS_ADDR_VALID_SHIFT 10 -+#define GPU_FAULTSTATUS_ADDR_VALID_FLAG \ -+ (1ul << GPU_FAULTSTATUS_ADDR_VALID_SHIFT) -+ -+#define GPU_FAULTSTATUS_JASID_VALID_SHIFT 11 -+#define GPU_FAULTSTATUS_JASID_VALID_FLAG \ -+ (1ul << GPU_FAULTSTATUS_JASID_VALID_SHIFT) -+ -+#define GPU_FAULTSTATUS_JASID_SHIFT 12 -+#define GPU_FAULTSTATUS_JASID_MASK (0xF << GPU_FAULTSTATUS_JASID_SHIFT) -+#define GPU_FAULTSTATUS_JASID_GET(reg_val) \ -+ (((reg_val)&GPU_FAULTSTATUS_JASID_MASK) >> GPU_FAULTSTATUS_JASID_SHIFT) -+#define GPU_FAULTSTATUS_JASID_SET(reg_val, value) \ -+ (((reg_val) & ~GPU_FAULTSTATUS_JASID_MASK) | \ -+ (((value) << GPU_FAULTSTATUS_JASID_SHIFT) & GPU_FAULTSTATUS_JASID_MASK)) -+ -+#define GPU_FAULTSTATUS_SOURCE_ID_SHIFT 16 -+#define GPU_FAULTSTATUS_SOURCE_ID_MASK \ -+ (0xFFFFul << GPU_FAULTSTATUS_SOURCE_ID_SHIFT) -+/* End GPU_FAULTSTATUS register */ -+ -+/* GPU_FAULTSTATUS_ACCESS_TYPE values */ -+#define GPU_FAULTSTATUS_ACCESS_TYPE_ATOMIC 0x0 -+#define GPU_FAULTSTATUS_ACCESS_TYPE_EXECUTE 0x1 -+#define GPU_FAULTSTATUS_ACCESS_TYPE_READ 0x2 -+#define GPU_FAULTSTATUS_ACCESS_TYPE_WRITE 0x3 -+/* End of GPU_FAULTSTATUS_ACCESS_TYPE values */ -+ -+/* Implementation-dependent exception codes used to indicate CSG -+ * and CS errors that are not specified in the specs. -+ */ -+#define GPU_EXCEPTION_TYPE_SW_FAULT_0 ((__u8)0x70) -+#define GPU_EXCEPTION_TYPE_SW_FAULT_1 ((__u8)0x71) -+#define GPU_EXCEPTION_TYPE_SW_FAULT_2 ((__u8)0x72) -+ -+/* GPU_FAULTSTATUS_EXCEPTION_TYPE values */ -+#define GPU_FAULTSTATUS_EXCEPTION_TYPE_OK 0x00 -+#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_BUS_FAULT 0x80 -+#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_SHAREABILITY_FAULT 0x88 -+#define GPU_FAULTSTATUS_EXCEPTION_TYPE_SYSTEM_SHAREABILITY_FAULT 0x89 -+#define GPU_FAULTSTATUS_EXCEPTION_TYPE_GPU_CACHEABILITY_FAULT 0x8A -+/* End of GPU_FAULTSTATUS_EXCEPTION_TYPE values */ -+ -+#define GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT GPU_U(10) -+#define GPU_FAULTSTATUS_ADDRESS_VALID_MASK (GPU_U(0x1) << GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) -+#define GPU_FAULTSTATUS_ADDRESS_VALID_GET(reg_val) \ -+ (((reg_val)&GPU_FAULTSTATUS_ADDRESS_VALID_MASK) >> GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) -+#define GPU_FAULTSTATUS_ADDRESS_VALID_SET(reg_val, value) \ -+ (((reg_val) & ~GPU_FAULTSTATUS_ADDRESS_VALID_MASK) | \ -+ (((value) << GPU_FAULTSTATUS_ADDRESS_VALID_SHIFT) & GPU_FAULTSTATUS_ADDRESS_VALID_MASK)) -+ -+/* IRQ flags */ -+#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ -+#define GPU_PROTECTED_FAULT (1 << 1) /* A GPU fault has occurred in protected mode */ -+#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. */ -+#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ -+#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down. */ -+#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ -+#define DOORBELL_MIRROR (1 << 18) /* Mirrors the doorbell interrupt line to the CPU */ -+#define MCU_STATUS_GPU_IRQ (1 << 19) /* MCU requires attention */ -+ -+/* -+ * In Debug build, -+ * GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE is used to clear and unmask interupts sources of GPU_IRQ -+ * by writing it onto GPU_IRQ_CLEAR/MASK registers. -+ * -+ * In Release build, -+ * GPU_IRQ_REG_COMMON is used. -+ * -+ * Note: -+ * CLEAN_CACHES_COMPLETED - Used separately for cache operation. -+ * DOORBELL_MIRROR - Do not have it included for GPU_IRQ_REG_COMMON -+ * as it can't be cleared by GPU_IRQ_CLEAR, thus interrupt storm might happen -+ */ -+#define GPU_IRQ_REG_COMMON (GPU_FAULT | GPU_PROTECTED_FAULT | RESET_COMPLETED \ -+ | POWER_CHANGED_ALL | MCU_STATUS_GPU_IRQ) -+ -+/* GPU_CONTROL_MCU.GPU_IRQ_RAWSTAT */ -+#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when performance count sample has completed */ -+ -+#endif /* _UAPI_KBASE_GPU_REGMAP_CSF_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_jm.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_jm.h -similarity index 89% -rename from dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_jm.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_jm.h -index 258ff33..1be3541 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_jm.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/backend/mali_kbase_gpu_regmap_jm.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2019 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,13 +17,14 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifndef _KBASE_GPU_REGMAP_JM_H_ --#define _KBASE_GPU_REGMAP_JM_H_ -+#ifndef _UAPI_KBASE_GPU_REGMAP_JM_H_ -+#define _UAPI_KBASE_GPU_REGMAP_JM_H_ - -+#if MALI_USE_CSF && defined(__KERNEL__) -+#error "Cannot be compiled with CSF" -+#endif - - /* Set to implementation defined, outer caching */ - #define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull -@@ -136,8 +138,8 @@ - #define JS_AFFINITY_LO 0x10 /* (RO) Core affinity mask for job slot n, low word */ - #define JS_AFFINITY_HI 0x14 /* (RO) Core affinity mask for job slot n, high word */ - #define JS_CONFIG 0x18 /* (RO) Configuration settings for job slot n */ --#define JS_XAFFINITY 0x1C /* (RO) Extended affinity mask for job -- slot n */ -+/* (RO) Extended affinity mask for job slot n*/ -+#define JS_XAFFINITY 0x1C - - #define JS_COMMAND 0x20 /* (WO) Command register for job slot n */ - #define JS_STATUS 0x24 /* (RO) Status register for job slot n */ -@@ -148,8 +150,8 @@ - #define JS_AFFINITY_NEXT_LO 0x50 /* (RW) Next core affinity mask for job slot n, low word */ - #define JS_AFFINITY_NEXT_HI 0x54 /* (RW) Next core affinity mask for job slot n, high word */ - #define JS_CONFIG_NEXT 0x58 /* (RW) Next configuration settings for job slot n */ --#define JS_XAFFINITY_NEXT 0x5C /* (RW) Next extended affinity mask for -- job slot n */ -+/* (RW) Next extended affinity mask for job slot n */ -+#define JS_XAFFINITY_NEXT 0x5C - - #define JS_COMMAND_NEXT 0x60 /* (RW) Next command register for job slot n */ - -@@ -259,4 +261,27 @@ - #define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */ - #define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */ - --#endif /* _KBASE_GPU_REGMAP_JM_H_ */ -+/* IRQ flags */ -+#define GPU_FAULT (1 << 0) /* A GPU Fault has occurred */ -+#define MULTIPLE_GPU_FAULTS (1 << 7) /* More than one GPU Fault occurred. */ -+#define RESET_COMPLETED (1 << 8) /* Set when a reset has completed. */ -+#define POWER_CHANGED_SINGLE (1 << 9) /* Set when a single core has finished powering up or down. */ -+#define POWER_CHANGED_ALL (1 << 10) /* Set when all cores have finished powering up or down. */ -+#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */ -+#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */ -+ -+/* -+ * In Debug build, -+ * GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE is used to clear and enable interupts sources of GPU_IRQ -+ * by writing it onto GPU_IRQ_CLEAR/MASK registers. -+ * -+ * In Release build, -+ * GPU_IRQ_REG_COMMON is used. -+ * -+ * Note: -+ * CLEAN_CACHES_COMPLETED - Used separately for cache operation. -+ */ -+#define GPU_IRQ_REG_COMMON (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \ -+ | POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED) -+ -+#endif /* _UAPI_KBASE_GPU_REGMAP_JM_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_coherency.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_coherency.h -similarity index 75% -rename from dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_coherency.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_coherency.h -index bb2b161..83d8413 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_coherency.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_coherency.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,16 +17,14 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifndef _KBASE_GPU_COHERENCY_H_ --#define _KBASE_GPU_COHERENCY_H_ -+#ifndef _UAPI_KBASE_GPU_COHERENCY_H_ -+#define _UAPI_KBASE_GPU_COHERENCY_H_ - - #define COHERENCY_ACE_LITE 0 - #define COHERENCY_ACE 1 - #define COHERENCY_NONE 31 - #define COHERENCY_FEATURE_BIT(x) (1 << (x)) - --#endif /* _KBASE_GPU_COHERENCY_H_ */ -+#endif /* _UAPI_KBASE_GPU_COHERENCY_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_id.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_id.h -similarity index 74% -rename from dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_id.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_id.h -index 9f3d6b1..d093ce4 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/gpu/mali_kbase_gpu_id.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_id.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,12 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifndef _KBASE_GPU_ID_H_ --#define _KBASE_GPU_ID_H_ -+#ifndef _UAPI_KBASE_GPU_ID_H_ -+#define _UAPI_KBASE_GPU_ID_H_ -+ -+#include <linux/types.h> - - /* GPU_ID register */ - #define GPU_ID_VERSION_STATUS_SHIFT 0 -@@ -53,19 +54,21 @@ - GPU_ID2_VERSION_STATUS) - - /* Helper macro to create a partial GPU_ID (new format) that defines -- a product ignoring its version. */ -+ * a product ignoring its version. -+ */ - #define GPU_ID2_PRODUCT_MAKE(arch_major, arch_minor, arch_rev, product_major) \ -- ((((u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ -- (((u32)arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ -- (((u32)arch_rev) << GPU_ID2_ARCH_REV_SHIFT) | \ -- (((u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) -+ ((((__u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ -+ (((__u32)arch_minor) << GPU_ID2_ARCH_MINOR_SHIFT) | \ -+ (((__u32)arch_rev) << GPU_ID2_ARCH_REV_SHIFT) | \ -+ (((__u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) - - /* Helper macro to create a partial GPU_ID (new format) that specifies the -- revision (major, minor, status) of a product */ -+ * revision (major, minor, status) of a product -+ */ - #define GPU_ID2_VERSION_MAKE(version_major, version_minor, version_status) \ -- ((((u32)version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ -- (((u32)version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ -- (((u32)version_status) << GPU_ID2_VERSION_STATUS_SHIFT)) -+ ((((__u32)version_major) << GPU_ID2_VERSION_MAJOR_SHIFT) | \ -+ (((__u32)version_minor) << GPU_ID2_VERSION_MINOR_SHIFT) | \ -+ (((__u32)version_status) << GPU_ID2_VERSION_STATUS_SHIFT)) - - /* Helper macro to create a complete GPU_ID (new format) */ - #define GPU_ID2_MAKE(arch_major, arch_minor, arch_rev, product_major, \ -@@ -76,16 +79,18 @@ - version_status)) - - /* Helper macro to create a partial GPU_ID (new format) that identifies -- a particular GPU model by its arch_major and product_major. */ -+ * a particular GPU model by its arch_major and product_major. -+ */ - #define GPU_ID2_MODEL_MAKE(arch_major, product_major) \ -- ((((u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ -- (((u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) -+ ((((__u32)arch_major) << GPU_ID2_ARCH_MAJOR_SHIFT) | \ -+ (((__u32)product_major) << GPU_ID2_PRODUCT_MAJOR_SHIFT)) - - /* Strip off the non-relevant bits from a product_id value and make it suitable -- for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU -- model. */ -+ * for comparison against the GPU_ID2_PRODUCT_xxx values which identify a GPU -+ * model. -+ */ - #define GPU_ID2_MODEL_MATCH_VALUE(product_id) \ -- ((((u32)product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ -+ ((((__u32)product_id) << GPU_ID2_PRODUCT_MAJOR_SHIFT) & \ - GPU_ID2_PRODUCT_MODEL) - - #define GPU_ID2_PRODUCT_TMIX GPU_ID2_MODEL_MAKE(6, 0) -@@ -98,21 +103,20 @@ - #define GPU_ID2_PRODUCT_TNAX GPU_ID2_MODEL_MAKE(9, 1) - #define GPU_ID2_PRODUCT_TBEX GPU_ID2_MODEL_MAKE(9, 2) - #define GPU_ID2_PRODUCT_LBEX GPU_ID2_MODEL_MAKE(9, 4) -+#define GPU_ID2_PRODUCT_TBAX GPU_ID2_MODEL_MAKE(9, 5) - #define GPU_ID2_PRODUCT_TDUX GPU_ID2_MODEL_MAKE(10, 1) - #define GPU_ID2_PRODUCT_TODX GPU_ID2_MODEL_MAKE(10, 2) - #define GPU_ID2_PRODUCT_TGRX GPU_ID2_MODEL_MAKE(10, 3) - #define GPU_ID2_PRODUCT_TVAX GPU_ID2_MODEL_MAKE(10, 4) - #define GPU_ID2_PRODUCT_LODX GPU_ID2_MODEL_MAKE(10, 7) --#define GPU_ID2_PRODUCT_TTUX GPU_ID2_MODEL_MAKE(11, 2) --#define GPU_ID2_PRODUCT_LTUX GPU_ID2_MODEL_MAKE(11, 3) --#define GPU_ID2_PRODUCT_TE2X GPU_ID2_MODEL_MAKE(11, 1) - - /* Helper macro to create a GPU_ID assuming valid values for id, major, -- minor, status */ -+ * minor, status -+ */ - #define GPU_ID_MAKE(id, major, minor, status) \ -- ((((u32)id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ -- (((u32)major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ -- (((u32)minor) << GPU_ID_VERSION_MINOR_SHIFT) | \ -- (((u32)status) << GPU_ID_VERSION_STATUS_SHIFT)) -+ ((((__u32)id) << GPU_ID_VERSION_PRODUCT_ID_SHIFT) | \ -+ (((__u32)major) << GPU_ID_VERSION_MAJOR_SHIFT) | \ -+ (((__u32)minor) << GPU_ID_VERSION_MINOR_SHIFT) | \ -+ (((__u32)status) << GPU_ID_VERSION_STATUS_SHIFT)) - --#endif /* _KBASE_GPU_ID_H_ */ -+#endif /* _UAPI_KBASE_GPU_ID_H_ */ -diff --git a/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_regmap.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_regmap.h -new file mode 100644 -index 0000000..84fad8d ---- /dev/null -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/gpu/mali_kbase_gpu_regmap.h -@@ -0,0 +1,434 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+#ifndef _UAPI_KBASE_GPU_REGMAP_H_ -+#define _UAPI_KBASE_GPU_REGMAP_H_ -+ -+#include "mali_kbase_gpu_coherency.h" -+#include "mali_kbase_gpu_id.h" -+#if MALI_USE_CSF -+#include "backend/mali_kbase_gpu_regmap_csf.h" -+#else -+#include "backend/mali_kbase_gpu_regmap_jm.h" -+#endif -+ -+/* Begin Register Offsets */ -+/* GPU control registers */ -+ -+#define GPU_CONTROL_BASE 0x0000 -+#define GPU_CONTROL_REG(r) (GPU_CONTROL_BASE + (r)) -+#define GPU_ID 0x000 /* (RO) GPU and revision identifier */ -+#define L2_FEATURES 0x004 /* (RO) Level 2 cache features */ -+#define TILER_FEATURES 0x00C /* (RO) Tiler Features */ -+#define MEM_FEATURES 0x010 /* (RO) Memory system features */ -+#define MMU_FEATURES 0x014 /* (RO) MMU features */ -+#define AS_PRESENT 0x018 /* (RO) Address space slots present */ -+#define GPU_IRQ_RAWSTAT 0x020 /* (RW) */ -+#define GPU_IRQ_CLEAR 0x024 /* (WO) */ -+#define GPU_IRQ_MASK 0x028 /* (RW) */ -+#define GPU_IRQ_STATUS 0x02C /* (RO) */ -+ -+#define GPU_COMMAND 0x030 /* (WO) */ -+#define GPU_STATUS 0x034 /* (RO) */ -+ -+#define GPU_DBGEN (1 << 8) /* DBGEN wire status */ -+ -+#define GPU_FAULTSTATUS 0x03C /* (RO) GPU exception type and fault status */ -+#define GPU_FAULTADDRESS_LO 0x040 /* (RO) GPU exception fault address, low word */ -+#define GPU_FAULTADDRESS_HI 0x044 /* (RO) GPU exception fault address, high word */ -+ -+#define L2_CONFIG 0x048 /* (RW) Level 2 cache configuration */ -+ -+#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */ -+#define SUPER_L2_COHERENT (1 << 1) /* Shader cores within a core -+ * supergroup are l2 coherent -+ */ -+ -+#define PWR_KEY 0x050 /* (WO) Power manager key register */ -+#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */ -+#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */ -+#define GPU_FEATURES_LO 0x060 /* (RO) GPU features, low word */ -+#define GPU_FEATURES_HI 0x064 /* (RO) GPU features, high word */ -+#define PRFCNT_FEATURES 0x068 /* (RO) Performance counter features */ -+#define TIMESTAMP_OFFSET_LO 0x088 /* (RW) Global time stamp offset, low word */ -+#define TIMESTAMP_OFFSET_HI 0x08C /* (RW) Global time stamp offset, high word */ -+#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */ -+#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */ -+#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */ -+#define TIMESTAMP_HI 0x09C /* (RO) Global time stamp counter, high word */ -+ -+#define THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ -+#define THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ -+#define THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ -+#define THREAD_FEATURES 0x0AC /* (RO) Thread features */ -+#define THREAD_TLS_ALLOC 0x310 /* (RO) Number of threads per core that TLS must be allocated for */ -+ -+#define TEXTURE_FEATURES_0 0x0B0 /* (RO) Support flags for indexed texture formats 0..31 */ -+#define TEXTURE_FEATURES_1 0x0B4 /* (RO) Support flags for indexed texture formats 32..63 */ -+#define TEXTURE_FEATURES_2 0x0B8 /* (RO) Support flags for indexed texture formats 64..95 */ -+#define TEXTURE_FEATURES_3 0x0BC /* (RO) Support flags for texture order */ -+ -+#define TEXTURE_FEATURES_REG(n) GPU_CONTROL_REG(TEXTURE_FEATURES_0 + ((n) << 2)) -+ -+#define SHADER_PRESENT_LO 0x100 /* (RO) Shader core present bitmap, low word */ -+#define SHADER_PRESENT_HI 0x104 /* (RO) Shader core present bitmap, high word */ -+ -+#define TILER_PRESENT_LO 0x110 /* (RO) Tiler core present bitmap, low word */ -+#define TILER_PRESENT_HI 0x114 /* (RO) Tiler core present bitmap, high word */ -+ -+#define L2_PRESENT_LO 0x120 /* (RO) Level 2 cache present bitmap, low word */ -+#define L2_PRESENT_HI 0x124 /* (RO) Level 2 cache present bitmap, high word */ -+ -+#define STACK_PRESENT_LO 0xE00 /* (RO) Core stack present bitmap, low word */ -+#define STACK_PRESENT_HI 0xE04 /* (RO) Core stack present bitmap, high word */ -+ -+#define SHADER_READY_LO 0x140 /* (RO) Shader core ready bitmap, low word */ -+#define SHADER_READY_HI 0x144 /* (RO) Shader core ready bitmap, high word */ -+ -+#define TILER_READY_LO 0x150 /* (RO) Tiler core ready bitmap, low word */ -+#define TILER_READY_HI 0x154 /* (RO) Tiler core ready bitmap, high word */ -+ -+#define L2_READY_LO 0x160 /* (RO) Level 2 cache ready bitmap, low word */ -+#define L2_READY_HI 0x164 /* (RO) Level 2 cache ready bitmap, high word */ -+ -+#define STACK_READY_LO 0xE10 /* (RO) Core stack ready bitmap, low word */ -+#define STACK_READY_HI 0xE14 /* (RO) Core stack ready bitmap, high word */ -+ -+#define SHADER_PWRON_LO 0x180 /* (WO) Shader core power on bitmap, low word */ -+#define SHADER_PWRON_HI 0x184 /* (WO) Shader core power on bitmap, high word */ -+ -+#define TILER_PWRON_LO 0x190 /* (WO) Tiler core power on bitmap, low word */ -+#define TILER_PWRON_HI 0x194 /* (WO) Tiler core power on bitmap, high word */ -+ -+#define L2_PWRON_LO 0x1A0 /* (WO) Level 2 cache power on bitmap, low word */ -+#define L2_PWRON_HI 0x1A4 /* (WO) Level 2 cache power on bitmap, high word */ -+ -+#define STACK_PWRON_LO 0xE20 /* (RO) Core stack power on bitmap, low word */ -+#define STACK_PWRON_HI 0xE24 /* (RO) Core stack power on bitmap, high word */ -+ -+#define SHADER_PWROFF_LO 0x1C0 /* (WO) Shader core power off bitmap, low word */ -+#define SHADER_PWROFF_HI 0x1C4 /* (WO) Shader core power off bitmap, high word */ -+ -+#define TILER_PWROFF_LO 0x1D0 /* (WO) Tiler core power off bitmap, low word */ -+#define TILER_PWROFF_HI 0x1D4 /* (WO) Tiler core power off bitmap, high word */ -+ -+#define L2_PWROFF_LO 0x1E0 /* (WO) Level 2 cache power off bitmap, low word */ -+#define L2_PWROFF_HI 0x1E4 /* (WO) Level 2 cache power off bitmap, high word */ -+ -+#define STACK_PWROFF_LO 0xE30 /* (RO) Core stack power off bitmap, low word */ -+#define STACK_PWROFF_HI 0xE34 /* (RO) Core stack power off bitmap, high word */ -+ -+#define SHADER_PWRTRANS_LO 0x200 /* (RO) Shader core power transition bitmap, low word */ -+#define SHADER_PWRTRANS_HI 0x204 /* (RO) Shader core power transition bitmap, high word */ -+ -+#define TILER_PWRTRANS_LO 0x210 /* (RO) Tiler core power transition bitmap, low word */ -+#define TILER_PWRTRANS_HI 0x214 /* (RO) Tiler core power transition bitmap, high word */ -+ -+#define L2_PWRTRANS_LO 0x220 /* (RO) Level 2 cache power transition bitmap, low word */ -+#define L2_PWRTRANS_HI 0x224 /* (RO) Level 2 cache power transition bitmap, high word */ -+ -+#define ASN_HASH_0 0x02C0 /* (RW) ASN hash function argument 0 */ -+#define ASN_HASH(n) (ASN_HASH_0 + (n)*4) -+#define ASN_HASH_COUNT 3 -+ -+#define STACK_PWRTRANS_LO 0xE40 /* (RO) Core stack power transition bitmap, low word */ -+#define STACK_PWRTRANS_HI 0xE44 /* (RO) Core stack power transition bitmap, high word */ -+ -+#define SHADER_PWRACTIVE_LO 0x240 /* (RO) Shader core active bitmap, low word */ -+#define SHADER_PWRACTIVE_HI 0x244 /* (RO) Shader core active bitmap, high word */ -+ -+#define TILER_PWRACTIVE_LO 0x250 /* (RO) Tiler core active bitmap, low word */ -+#define TILER_PWRACTIVE_HI 0x254 /* (RO) Tiler core active bitmap, high word */ -+ -+#define L2_PWRACTIVE_LO 0x260 /* (RO) Level 2 cache active bitmap, low word */ -+#define L2_PWRACTIVE_HI 0x264 /* (RO) Level 2 cache active bitmap, high word */ -+ -+#define COHERENCY_FEATURES 0x300 /* (RO) Coherency features present */ -+#define COHERENCY_ENABLE 0x304 /* (RW) Coherency enable */ -+ -+#define SHADER_CONFIG 0xF04 /* (RW) Shader core configuration (implementation-specific) */ -+#define TILER_CONFIG 0xF08 /* (RW) Tiler core configuration (implementation-specific) */ -+#define L2_MMU_CONFIG 0xF0C /* (RW) L2 cache and MMU configuration (implementation-specific) */ -+ -+/* Job control registers */ -+ -+#define JOB_CONTROL_BASE 0x1000 -+ -+#define JOB_CONTROL_REG(r) (JOB_CONTROL_BASE + (r)) -+ -+#define JOB_IRQ_RAWSTAT 0x000 /* Raw interrupt status register */ -+#define JOB_IRQ_CLEAR 0x004 /* Interrupt clear register */ -+#define JOB_IRQ_MASK 0x008 /* Interrupt mask register */ -+#define JOB_IRQ_STATUS 0x00C /* Interrupt status register */ -+ -+/* MMU control registers */ -+ -+#define MEMORY_MANAGEMENT_BASE 0x2000 -+#define MMU_REG(r) (MEMORY_MANAGEMENT_BASE + (r)) -+ -+#define MMU_IRQ_RAWSTAT 0x000 /* (RW) Raw interrupt status register */ -+#define MMU_IRQ_CLEAR 0x004 /* (WO) Interrupt clear register */ -+#define MMU_IRQ_MASK 0x008 /* (RW) Interrupt mask register */ -+#define MMU_IRQ_STATUS 0x00C /* (RO) Interrupt status register */ -+ -+#define MMU_AS0 0x400 /* Configuration registers for address space 0 */ -+#define MMU_AS1 0x440 /* Configuration registers for address space 1 */ -+#define MMU_AS2 0x480 /* Configuration registers for address space 2 */ -+#define MMU_AS3 0x4C0 /* Configuration registers for address space 3 */ -+#define MMU_AS4 0x500 /* Configuration registers for address space 4 */ -+#define MMU_AS5 0x540 /* Configuration registers for address space 5 */ -+#define MMU_AS6 0x580 /* Configuration registers for address space 6 */ -+#define MMU_AS7 0x5C0 /* Configuration registers for address space 7 */ -+#define MMU_AS8 0x600 /* Configuration registers for address space 8 */ -+#define MMU_AS9 0x640 /* Configuration registers for address space 9 */ -+#define MMU_AS10 0x680 /* Configuration registers for address space 10 */ -+#define MMU_AS11 0x6C0 /* Configuration registers for address space 11 */ -+#define MMU_AS12 0x700 /* Configuration registers for address space 12 */ -+#define MMU_AS13 0x740 /* Configuration registers for address space 13 */ -+#define MMU_AS14 0x780 /* Configuration registers for address space 14 */ -+#define MMU_AS15 0x7C0 /* Configuration registers for address space 15 */ -+ -+/* MMU address space control registers */ -+ -+#define MMU_AS_REG(n, r) (MMU_REG(MMU_AS0 + ((n) << 6)) + (r)) -+ -+#define AS_TRANSTAB_LO 0x00 /* (RW) Translation Table Base Address for address space n, low word */ -+#define AS_TRANSTAB_HI 0x04 /* (RW) Translation Table Base Address for address space n, high word */ -+#define AS_MEMATTR_LO 0x08 /* (RW) Memory attributes for address space n, low word. */ -+#define AS_MEMATTR_HI 0x0C /* (RW) Memory attributes for address space n, high word. */ -+#define AS_LOCKADDR_LO 0x10 /* (RW) Lock region address for address space n, low word */ -+#define AS_LOCKADDR_HI 0x14 /* (RW) Lock region address for address space n, high word */ -+#define AS_COMMAND 0x18 /* (WO) MMU command register for address space n */ -+#define AS_FAULTSTATUS 0x1C /* (RO) MMU fault status register for address space n */ -+#define AS_FAULTADDRESS_LO 0x20 /* (RO) Fault Address for address space n, low word */ -+#define AS_FAULTADDRESS_HI 0x24 /* (RO) Fault Address for address space n, high word */ -+#define AS_STATUS 0x28 /* (RO) Status flags for address space n */ -+ -+/* (RW) Translation table configuration for address space n, low word */ -+#define AS_TRANSCFG_LO 0x30 -+/* (RW) Translation table configuration for address space n, high word */ -+#define AS_TRANSCFG_HI 0x34 -+/* (RO) Secondary fault address for address space n, low word */ -+#define AS_FAULTEXTRA_LO 0x38 -+/* (RO) Secondary fault address for address space n, high word */ -+#define AS_FAULTEXTRA_HI 0x3C -+ -+/* End Register Offsets */ -+ -+#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON) -+ -+/* -+ * MMU_IRQ_RAWSTAT register values. Values are valid also for -+ * MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers. -+ */ -+ -+#define MMU_PAGE_FAULT_FLAGS 16 -+ -+/* Macros returning a bitmask to retrieve page fault or bus error flags from -+ * MMU registers -+ */ -+#define MMU_PAGE_FAULT(n) (1UL << (n)) -+#define MMU_BUS_ERROR(n) (1UL << ((n) + MMU_PAGE_FAULT_FLAGS)) -+ -+/* -+ * Begin AARCH64 MMU TRANSTAB register values -+ */ -+#define MMU_HW_OUTA_BITS 40 -+#define AS_TRANSTAB_BASE_MASK ((1ULL << MMU_HW_OUTA_BITS) - (1ULL << 4)) -+ -+/* -+ * Begin MMU STATUS register values -+ */ -+#define AS_STATUS_AS_ACTIVE 0x01 -+ -+#define AS_FAULTSTATUS_EXCEPTION_CODE_MASK (0x7<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSLATION_FAULT (0x0<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_PERMISSION_FAULT (0x1<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_TRANSTAB_BUS_FAULT (0x2<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_ACCESS_FLAG (0x3<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3) -+#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3) -+ -+#define AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT 0 -+#define AS_FAULTSTATUS_EXCEPTION_TYPE_MASK (0xFF << AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) -+#define AS_FAULTSTATUS_EXCEPTION_TYPE_GET(reg_val) \ -+ (((reg_val)&AS_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT) -+#define AS_FAULTSTATUS_EXCEPTION_TYPE_TRANSLATION_FAULT_0 0xC0 -+ -+#define AS_FAULTSTATUS_ACCESS_TYPE_SHIFT 8 -+#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3 << AS_FAULTSTATUS_ACCESS_TYPE_SHIFT) -+#define AS_FAULTSTATUS_ACCESS_TYPE_GET(reg_val) \ -+ (((reg_val)&AS_FAULTSTATUS_ACCESS_TYPE_MASK) >> AS_FAULTSTATUS_ACCESS_TYPE_SHIFT) -+ -+#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0) -+#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1) -+#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2) -+#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3) -+ -+#define AS_FAULTSTATUS_SOURCE_ID_SHIFT 16 -+#define AS_FAULTSTATUS_SOURCE_ID_MASK (0xFFFF << AS_FAULTSTATUS_SOURCE_ID_SHIFT) -+#define AS_FAULTSTATUS_SOURCE_ID_GET(reg_val) \ -+ (((reg_val)&AS_FAULTSTATUS_SOURCE_ID_MASK) >> AS_FAULTSTATUS_SOURCE_ID_SHIFT) -+ -+#define PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_SHIFT (0) -+#define PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_MASK \ -+ ((0xFF) << PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_SHIFT) -+#define PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_GET(reg_val) \ -+ (((reg_val)&PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_MASK) >> \ -+ PRFCNT_FEATURES_COUNTER_BLOCK_SIZE_SHIFT) -+ -+/* -+ * Begin MMU TRANSCFG register values -+ */ -+#define AS_TRANSCFG_ADRMODE_LEGACY 0 -+#define AS_TRANSCFG_ADRMODE_UNMAPPED 1 -+#define AS_TRANSCFG_ADRMODE_IDENTITY 2 -+#define AS_TRANSCFG_ADRMODE_AARCH64_4K 6 -+#define AS_TRANSCFG_ADRMODE_AARCH64_64K 8 -+ -+#define AS_TRANSCFG_ADRMODE_MASK 0xF -+ -+/* -+ * Begin TRANSCFG register values -+ */ -+#define AS_TRANSCFG_PTW_MEMATTR_MASK (3ull << 24) -+#define AS_TRANSCFG_PTW_MEMATTR_NON_CACHEABLE (1ull << 24) -+#define AS_TRANSCFG_PTW_MEMATTR_WRITE_BACK (2ull << 24) -+ -+#define AS_TRANSCFG_PTW_SH_MASK ((3ull << 28)) -+#define AS_TRANSCFG_PTW_SH_OS (2ull << 28) -+#define AS_TRANSCFG_PTW_SH_IS (3ull << 28) -+#define AS_TRANSCFG_R_ALLOCATE (1ull << 30) -+ -+/* -+ * Begin Command Values -+ */ -+ -+/* AS_COMMAND register commands */ -+#define AS_COMMAND_NOP 0x00 /* NOP Operation */ -+#define AS_COMMAND_UPDATE 0x01 /* Broadcasts the values in AS_TRANSTAB and ASn_MEMATTR to all MMUs */ -+#define AS_COMMAND_LOCK 0x02 /* Issue a lock region command to all MMUs */ -+#define AS_COMMAND_UNLOCK 0x03 /* Issue a flush region command to all MMUs */ -+/* Flush all L2 caches then issue a flush region command to all MMUs -+ * (deprecated - only for use with T60x) -+ */ -+#define AS_COMMAND_FLUSH 0x04 -+/* Flush all L2 caches then issue a flush region command to all MMUs */ -+#define AS_COMMAND_FLUSH_PT 0x04 -+/* Wait for memory accesses to complete, flush all the L1s cache then flush all -+ * L2 caches then issue a flush region command to all MMUs -+ */ -+#define AS_COMMAND_FLUSH_MEM 0x05 -+ -+/* GPU_STATUS values */ -+#define GPU_STATUS_PRFCNT_ACTIVE (1 << 2) /* Set if the performance counters are active. */ -+#define GPU_STATUS_CYCLE_COUNT_ACTIVE (1 << 6) /* Set if the cycle counter is active. */ -+#define GPU_STATUS_PROTECTED_MODE_ACTIVE (1 << 7) /* Set if protected mode is active */ -+ -+/* PRFCNT_CONFIG register values */ -+#define PRFCNT_CONFIG_MODE_SHIFT 0 /* Counter mode position. */ -+#define PRFCNT_CONFIG_AS_SHIFT 4 /* Address space bitmap position. */ -+#define PRFCNT_CONFIG_SETSELECT_SHIFT 8 /* Set select position. */ -+ -+/* The performance counters are disabled. */ -+#define PRFCNT_CONFIG_MODE_OFF 0 -+/* The performance counters are enabled, but are only written out when a -+ * PRFCNT_SAMPLE command is issued using the GPU_COMMAND register. -+ */ -+#define PRFCNT_CONFIG_MODE_MANUAL 1 -+/* The performance counters are enabled, and are written out each time a tile -+ * finishes rendering. -+ */ -+#define PRFCNT_CONFIG_MODE_TILE 2 -+ -+/* AS<n>_MEMATTR values from MMU_MEMATTR_STAGE1: */ -+/* Use GPU implementation-defined caching policy. */ -+#define AS_MEMATTR_IMPL_DEF_CACHE_POLICY 0x88ull -+/* The attribute set to force all resources to be cached. */ -+#define AS_MEMATTR_FORCE_TO_CACHE_ALL 0x8Full -+/* Inner write-alloc cache setup, no outer caching */ -+#define AS_MEMATTR_WRITE_ALLOC 0x8Dull -+ -+/* Use GPU implementation-defined caching policy. */ -+#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull -+/* The attribute set to force all resources to be cached. */ -+#define AS_MEMATTR_LPAE_FORCE_TO_CACHE_ALL 0x4Full -+/* Inner write-alloc cache setup, no outer caching */ -+#define AS_MEMATTR_LPAE_WRITE_ALLOC 0x4Dull -+/* Set to implementation defined, outer caching */ -+#define AS_MEMATTR_LPAE_OUTER_IMPL_DEF 0x88ull -+/* Set to write back memory, outer caching */ -+#define AS_MEMATTR_LPAE_OUTER_WA 0x8Dull -+/* There is no LPAE support for non-cacheable, since the memory type is always -+ * write-back. -+ * Marking this setting as reserved for LPAE -+ */ -+#define AS_MEMATTR_LPAE_NON_CACHEABLE_RESERVED -+ -+/* L2_MMU_CONFIG register */ -+#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23) -+#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT) -+ -+/* End L2_MMU_CONFIG register */ -+ -+/* THREAD_* registers */ -+ -+/* THREAD_FEATURES IMPLEMENTATION_TECHNOLOGY values */ -+#define IMPLEMENTATION_UNSPECIFIED 0 -+#define IMPLEMENTATION_SILICON 1 -+#define IMPLEMENTATION_FPGA 2 -+#define IMPLEMENTATION_MODEL 3 -+ -+/* Default values when registers are not supported by the implemented hardware */ -+#define THREAD_MT_DEFAULT 256 -+#define THREAD_MWS_DEFAULT 256 -+#define THREAD_MBS_DEFAULT 256 -+#define THREAD_MR_DEFAULT 1024 -+#define THREAD_MTQ_DEFAULT 4 -+#define THREAD_MTGS_DEFAULT 10 -+ -+/* End THREAD_* registers */ -+ -+/* SHADER_CONFIG register */ -+#define SC_LS_ALLOW_ATTR_TYPES (1ul << 16) -+#define SC_TLS_HASH_ENABLE (1ul << 17) -+#define SC_LS_ATTR_CHECK_DISABLE (1ul << 18) -+#define SC_VAR_ALGORITHM (1ul << 29) -+/* End SHADER_CONFIG register */ -+ -+/* TILER_CONFIG register */ -+#define TC_CLOCK_GATE_OVERRIDE (1ul << 0) -+/* End TILER_CONFIG register */ -+ -+/* L2_CONFIG register */ -+#define L2_CONFIG_SIZE_SHIFT 16 -+#define L2_CONFIG_SIZE_MASK (0xFFul << L2_CONFIG_SIZE_SHIFT) -+#define L2_CONFIG_HASH_SHIFT 24 -+#define L2_CONFIG_HASH_MASK (0xFFul << L2_CONFIG_HASH_SHIFT) -+#define L2_CONFIG_ASN_HASH_ENABLE_SHIFT 24 -+#define L2_CONFIG_ASN_HASH_ENABLE_MASK (1ul << L2_CONFIG_ASN_HASH_ENABLE_SHIFT) -+/* End L2_CONFIG register */ -+ -+/* IDVS_GROUP register */ -+#define IDVS_GROUP_SIZE_SHIFT (16) -+#define IDVS_GROUP_MAX_SIZE (0x3F) -+ -+#endif /* _UAPI_KBASE_GPU_REGMAP_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_base_jm_kernel.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/jm/mali_base_jm_kernel.h -similarity index 75% -rename from dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_base_jm_kernel.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/jm/mali_base_jm_kernel.h -index 879a436..749e1fa 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_base_jm_kernel.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/jm/mali_base_jm_kernel.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,11 +17,12 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ --#ifndef _BASE_JM_KERNEL_H_ --#define _BASE_JM_KERNEL_H_ -+ -+#ifndef _UAPI_BASE_JM_KERNEL_H_ -+#define _UAPI_BASE_JM_KERNEL_H_ -+ -+#include <linux/types.h> - - /* Memory allocation, access/hint flags. - * -@@ -80,7 +82,8 @@ - */ - #define BASE_MEM_COHERENT_LOCAL ((base_mem_alloc_flags)1 << 11) - --/* Should be cached on the CPU -+/* IN/OUT */ -+/* Should be cached on the CPU, returned if actually cached - */ - #define BASE_MEM_CACHED_CPU ((base_mem_alloc_flags)1 << 12) - -@@ -120,9 +123,9 @@ - #define BASE_MEM_RESERVED_BIT_19 ((base_mem_alloc_flags)1 << 19) - - /** -- * Memory starting from the end of the initial commit is aligned to 'extent' -- * pages, where 'extent' must be a power of 2 and no more than -- * BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES -+ * Memory starting from the end of the initial commit is aligned to 'extension' -+ * pages, where 'extension' must be a power of 2 and no more than -+ * BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES - */ - #define BASE_MEM_TILER_ALIGN_TOP ((base_mem_alloc_flags)1 << 20) - -@@ -155,18 +158,25 @@ - /* Use the GPU VA chosen by the kernel client */ - #define BASE_MEM_FLAG_MAP_FIXED ((base_mem_alloc_flags)1 << 27) - -+/* OUT */ -+/* Kernel side cache sync ops required */ -+#define BASE_MEM_KERNEL_SYNC ((base_mem_alloc_flags)1 << 28) -+ -+/* Force trimming of JIT allocations when creating a new allocation */ -+#define BASEP_MEM_PERFORM_JIT_TRIM ((base_mem_alloc_flags)1 << 29) -+ - /* Number of bits used as flags for base memory management - * - * Must be kept in sync with the base_mem_alloc_flags flags - */ --#define BASE_MEM_FLAGS_NR_BITS 28 -+#define BASE_MEM_FLAGS_NR_BITS 30 - - /* A mask of all the flags which are only valid for allocations within kbase, - * and may not be passed from user space. - */ - #define BASEP_MEM_FLAGS_KERNEL_ONLY \ - (BASEP_MEM_PERMANENT_KERNEL_MAPPING | BASEP_MEM_NO_USER_FREE | \ -- BASE_MEM_FLAG_MAP_FIXED) -+ BASE_MEM_FLAG_MAP_FIXED | BASEP_MEM_PERFORM_JIT_TRIM) - - /* A mask for all output bits, excluding IN/OUT bits. - */ -@@ -192,6 +202,28 @@ - #define BASE_MEM_FIRST_FREE_ADDRESS ((BITS_PER_LONG << 12) + \ - BASE_MEM_COOKIE_BASE) - -+/* Similar to BASE_MEM_TILER_ALIGN_TOP, memory starting from the end of the -+ * initial commit is aligned to 'extension' pages, where 'extension' must be a power -+ * of 2 and no more than BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES -+ */ -+#define BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP (1 << 0) -+ -+/** -+ * If set, the heap info address points to a __u32 holding the used size in bytes; -+ * otherwise it points to a __u64 holding the lowest address of unused memory. -+ */ -+#define BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE (1 << 1) -+ -+/** -+ * Valid set of just-in-time memory allocation flags -+ * -+ * Note: BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE cannot be set if heap_info_gpu_addr -+ * in %base_jit_alloc_info is 0 (atom with BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE set -+ * and heap_info_gpu_addr being 0 will be rejected). -+ */ -+#define BASE_JIT_ALLOC_VALID_FLAGS \ -+ (BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP | BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) -+ - /** - * typedef base_context_create_flags - Flags to pass to ::base_context_init. - * -@@ -200,7 +232,7 @@ - * These share the same space as BASEP_CONTEXT_FLAG_*, and so must - * not collide with them. - */ --typedef u32 base_context_create_flags; -+typedef __u32 base_context_create_flags; - - /* No flags set */ - #define BASE_CONTEXT_CREATE_FLAG_NONE ((base_context_create_flags)0) -@@ -290,7 +322,7 @@ typedef u32 base_context_create_flags; - * @blob: per-job data array - */ - struct base_jd_udata { -- u64 blob[2]; -+ __u64 blob[2]; - }; - - /** -@@ -303,7 +335,7 @@ struct base_jd_udata { - * When the flag is set for a particular dependency to signal that it is an - * ordering only dependency then errors will not be propagated. - */ --typedef u8 base_jd_dep_type; -+typedef __u8 base_jd_dep_type; - - #define BASE_JD_DEP_TYPE_INVALID (0) /**< Invalid dependency */ - #define BASE_JD_DEP_TYPE_DATA (1U << 0) /**< Data dependency */ -@@ -319,7 +351,7 @@ typedef u8 base_jd_dep_type; - * Special case is ::BASE_JD_REQ_DEP, which is used to express complex - * dependencies, and that doesn't execute anything on the hardware. - */ --typedef u32 base_jd_core_req; -+typedef __u32 base_jd_core_req; - - /* Requirements that come from the HW */ - -@@ -551,6 +583,13 @@ typedef u32 base_jd_core_req; - */ - #define BASE_JD_REQ_END_RENDERPASS ((base_jd_core_req)1 << 19) - -+/* SW-only requirement: The atom needs to run on a limited core mask affinity. -+ * -+ * If this bit is set then the kbase_context.limited_core_mask will be applied -+ * to the affinity. -+ */ -+#define BASE_JD_REQ_LIMITED_CORE_MASK ((base_jd_core_req)1 << 20) -+ - /* These requirement bits are currently unused in base_jd_core_req - */ - #define BASEP_JD_REQ_RESERVED \ -@@ -561,7 +600,7 @@ typedef u32 base_jd_core_req; - BASE_JD_REQ_FS_AFBC | BASE_JD_REQ_PERMON | \ - BASE_JD_REQ_SKIP_CACHE_START | BASE_JD_REQ_SKIP_CACHE_END | \ - BASE_JD_REQ_JOB_SLOT | BASE_JD_REQ_START_RENDERPASS | \ -- BASE_JD_REQ_END_RENDERPASS)) -+ BASE_JD_REQ_END_RENDERPASS | BASE_JD_REQ_LIMITED_CORE_MASK)) - - /* Mask of all bits in base_jd_core_req that control the type of the atom. - * -@@ -606,7 +645,7 @@ enum kbase_jd_atom_state { - /** - * typedef base_atom_id - Type big enough to store an atom number in. - */ --typedef u8 base_atom_id; -+typedef __u8 base_atom_id; - - /** - * struct base_dependency - -@@ -669,10 +708,10 @@ struct base_dependency { - * BASE_JD_REQ_END_RENDERPASS is set in the base_jd_core_req. - */ - struct base_jd_fragment { -- u64 norm_read_norm_write; -- u64 norm_read_forced_write; -- u64 forced_read_forced_write; -- u64 forced_read_norm_write; -+ __u64 norm_read_norm_write; -+ __u64 norm_read_forced_write; -+ __u64 forced_read_forced_write; -+ __u64 forced_read_norm_write; - }; - - /** -@@ -712,7 +751,7 @@ struct base_jd_fragment { - * the same context. See KBASE_JS_SYSTEM_PRIORITY_MODE and - * KBASE_JS_PROCESS_LOCAL_PRIORITY_MODE for more details. - */ --typedef u8 base_jd_prio; -+typedef __u8 base_jd_prio; - - /* Medium atom priority. This is a priority higher than BASE_JD_PRIO_LOW */ - #define BASE_JD_PRIO_MEDIUM ((base_jd_prio)0) -@@ -722,11 +761,15 @@ typedef u8 base_jd_prio; - #define BASE_JD_PRIO_HIGH ((base_jd_prio)1) - /* Low atom priority. */ - #define BASE_JD_PRIO_LOW ((base_jd_prio)2) -+/* Real-Time atom priority. This is a priority higher than BASE_JD_PRIO_HIGH, -+ * BASE_JD_PRIO_MEDIUM, and BASE_JD_PRIO_LOW -+ */ -+#define BASE_JD_PRIO_REALTIME ((base_jd_prio)3) - - /* Count of the number of priority levels. This itself is not a valid - * base_jd_prio setting - */ --#define BASE_JD_NR_PRIO_LEVELS 3 -+#define BASE_JD_NR_PRIO_LEVELS 4 - - /** - * struct base_jd_atom_v2 - Node of a dependency graph used to submit a -@@ -759,34 +802,82 @@ typedef u8 base_jd_prio; - * @padding: Unused. Must be zero. - * - * This structure has changed since UK 10.2 for which base_jd_core_req was a -- * u16 value. -+ * __u16 value. - * -- * In UK 10.3 a core_req field of a u32 type was added to the end of the -- * structure, and the place in the structure previously occupied by u16 -+ * In UK 10.3 a core_req field of a __u32 type was added to the end of the -+ * structure, and the place in the structure previously occupied by __u16 - * core_req was kept but renamed to compat_core_req. - * -- * From UK 11.20 - compat_core_req is now occupied by u8 jit_id[2]. -+ * From UK 11.20 - compat_core_req is now occupied by __u8 jit_id[2]. - * Compatibility with UK 10.x from UK 11.y is not handled because - * the major version increase prevents this. - * - * For UK 11.20 jit_id[2] must be initialized to zero. - */ - struct base_jd_atom_v2 { -- u64 jc; -+ __u64 jc; - struct base_jd_udata udata; -- u64 extres_list; -- u16 nr_extres; -- u8 jit_id[2]; -+ __u64 extres_list; -+ __u16 nr_extres; -+ __u8 jit_id[2]; - struct base_dependency pre_dep[2]; - base_atom_id atom_number; - base_jd_prio prio; -- u8 device_nr; -- u8 jobslot; -+ __u8 device_nr; -+ __u8 jobslot; - base_jd_core_req core_req; -- u8 renderpass_id; -- u8 padding[7]; -+ __u8 renderpass_id; -+ __u8 padding[7]; - }; - -+/** -+ * struct base_jd_atom - Same as base_jd_atom_v2, but has an extra seq_nr -+ * at the beginning. -+ * -+ * @seq_nr: Sequence number of logical grouping of atoms. -+ * @jc: GPU address of a job chain or (if BASE_JD_REQ_END_RENDERPASS -+ * is set in the base_jd_core_req) the CPU address of a -+ * base_jd_fragment object. -+ * @udata: User data. -+ * @extres_list: List of external resources. -+ * @nr_extres: Number of external resources or JIT allocations. -+ * @jit_id: Zero-terminated array of IDs of just-in-time memory -+ * allocations written to by the atom. When the atom -+ * completes, the value stored at the -+ * &struct_base_jit_alloc_info.heap_info_gpu_addr of -+ * each allocation is read in order to enforce an -+ * overall physical memory usage limit. -+ * @pre_dep: Pre-dependencies. One need to use SETTER function to assign -+ * this field; this is done in order to reduce possibility of -+ * improper assignment of a dependency field. -+ * @atom_number: Unique number to identify the atom. -+ * @prio: Atom priority. Refer to base_jd_prio for more details. -+ * @device_nr: Core group when BASE_JD_REQ_SPECIFIC_COHERENT_GROUP -+ * specified. -+ * @jobslot: Job slot to use when BASE_JD_REQ_JOB_SLOT is specified. -+ * @core_req: Core requirements. -+ * @renderpass_id: Renderpass identifier used to associate an atom that has -+ * BASE_JD_REQ_START_RENDERPASS set in its core requirements -+ * with an atom that has BASE_JD_REQ_END_RENDERPASS set. -+ * @padding: Unused. Must be zero. -+ */ -+typedef struct base_jd_atom { -+ __u64 seq_nr; -+ __u64 jc; -+ struct base_jd_udata udata; -+ __u64 extres_list; -+ __u16 nr_extres; -+ __u8 jit_id[2]; -+ struct base_dependency pre_dep[2]; -+ base_atom_id atom_number; -+ base_jd_prio prio; -+ __u8 device_nr; -+ __u8 jobslot; -+ base_jd_core_req core_req; -+ __u8 renderpass_id; -+ __u8 padding[7]; -+} base_jd_atom; -+ - /* Job chain event code bits - * Defines the bits used to create ::base_jd_event_code - */ -@@ -834,6 +925,109 @@ enum { - * Such codes are never returned to - * user-space. - * @BASE_JD_EVENT_RANGE_KERNEL_ONLY_END: End of kernel-only status codes. -+ * @BASE_JD_EVENT_DONE: atom has completed successfull -+ * @BASE_JD_EVENT_JOB_CONFIG_FAULT: Atom dependencies configuration error which -+ * shall result in a failed atom -+ * @BASE_JD_EVENT_JOB_POWER_FAULT: The job could not be executed because the -+ * part of the memory system required to access -+ * job descriptors was not powered on -+ * @BASE_JD_EVENT_JOB_READ_FAULT: Reading a job descriptor into the Job -+ * manager failed -+ * @BASE_JD_EVENT_JOB_WRITE_FAULT: Writing a job descriptor from the Job -+ * manager failed -+ * @BASE_JD_EVENT_JOB_AFFINITY_FAULT: The job could not be executed because the -+ * specified affinity mask does not intersect -+ * any available cores -+ * @BASE_JD_EVENT_JOB_BUS_FAULT: A bus access failed while executing a job -+ * @BASE_JD_EVENT_INSTR_INVALID_PC: A shader instruction with an illegal program -+ * counter was executed. -+ * @BASE_JD_EVENT_INSTR_INVALID_ENC: A shader instruction with an illegal -+ * encoding was executed. -+ * @BASE_JD_EVENT_INSTR_TYPE_MISMATCH: A shader instruction was executed where -+ * the instruction encoding did not match the -+ * instruction type encoded in the program -+ * counter. -+ * @BASE_JD_EVENT_INSTR_OPERAND_FAULT: A shader instruction was executed that -+ * contained invalid combinations of operands. -+ * @BASE_JD_EVENT_INSTR_TLS_FAULT: A shader instruction was executed that tried -+ * to access the thread local storage section -+ * of another thread. -+ * @BASE_JD_EVENT_INSTR_ALIGN_FAULT: A shader instruction was executed that -+ * tried to do an unsupported unaligned memory -+ * access. -+ * @BASE_JD_EVENT_INSTR_BARRIER_FAULT: A shader instruction was executed that -+ * failed to complete an instruction barrier. -+ * @BASE_JD_EVENT_DATA_INVALID_FAULT: Any data structure read as part of the job -+ * contains invalid combinations of data. -+ * @BASE_JD_EVENT_TILE_RANGE_FAULT: Tile or fragment shading was asked to -+ * process a tile that is entirely outside the -+ * bounding box of the frame. -+ * @BASE_JD_EVENT_STATE_FAULT: Matches ADDR_RANGE_FAULT. A virtual address -+ * has been found that exceeds the virtual -+ * address range. -+ * @BASE_JD_EVENT_OUT_OF_MEMORY: The tiler ran out of memory when executing a job. -+ * @BASE_JD_EVENT_UNKNOWN: If multiple jobs in a job chain fail, only -+ * the first one the reports an error will set -+ * and return full error information. -+ * Subsequent failing jobs will not update the -+ * error status registers, and may write an -+ * error status of UNKNOWN. -+ * @BASE_JD_EVENT_DELAYED_BUS_FAULT: The GPU received a bus fault for access to -+ * physical memory where the original virtual -+ * address is no longer available. -+ * @BASE_JD_EVENT_SHAREABILITY_FAULT: Matches GPU_SHAREABILITY_FAULT. A cache -+ * has detected that the same line has been -+ * accessed as both shareable and non-shareable -+ * memory from inside the GPU. -+ * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL1: A memory access hit an invalid table -+ * entry at level 1 of the translation table. -+ * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL2: A memory access hit an invalid table -+ * entry at level 2 of the translation table. -+ * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL3: A memory access hit an invalid table -+ * entry at level 3 of the translation table. -+ * @BASE_JD_EVENT_TRANSLATION_FAULT_LEVEL4: A memory access hit an invalid table -+ * entry at level 4 of the translation table. -+ * @BASE_JD_EVENT_PERMISSION_FAULT: A memory access could not be allowed due to -+ * the permission flags set in translation -+ * table -+ * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL1: A bus fault occurred while reading -+ * level 0 of the translation tables. -+ * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL2: A bus fault occurred while reading -+ * level 1 of the translation tables. -+ * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL3: A bus fault occurred while reading -+ * level 2 of the translation tables. -+ * @BASE_JD_EVENT_TRANSTAB_BUS_FAULT_LEVEL4: A bus fault occurred while reading -+ * level 3 of the translation tables. -+ * @BASE_JD_EVENT_ACCESS_FLAG: Matches ACCESS_FLAG_0. A memory access hit a -+ * translation table entry with the ACCESS_FLAG -+ * bit set to zero in level 0 of the -+ * page table, and the DISABLE_AF_FAULT flag -+ * was not set. -+ * @BASE_JD_EVENT_MEM_GROWTH_FAILED: raised for JIT_ALLOC atoms that failed to -+ * grow memory on demand -+ * @BASE_JD_EVENT_JOB_CANCELLED: raised when this atom was hard-stopped or its -+ * dependencies failed -+ * @BASE_JD_EVENT_JOB_INVALID: raised for many reasons, including invalid data -+ * in the atom which overlaps with -+ * BASE_JD_EVENT_JOB_CONFIG_FAULT, or if the -+ * platform doesn't support the feature specified in -+ * the atom. -+ * @BASE_JD_EVENT_PM_EVENT: TODO: remove as it's not used -+ * @BASE_JD_EVENT_TIMED_OUT: TODO: remove as it's not used -+ * @BASE_JD_EVENT_BAG_INVALID: TODO: remove as it's not used -+ * @BASE_JD_EVENT_PROGRESS_REPORT: TODO: remove as it's not used -+ * @BASE_JD_EVENT_BAG_DONE: TODO: remove as it's not used -+ * @BASE_JD_EVENT_DRV_TERMINATED: this is a special event generated to indicate -+ * to userspace that the KBase context has been -+ * destroyed and Base should stop listening for -+ * further events -+ * @BASE_JD_EVENT_REMOVED_FROM_NEXT: raised when an atom that was configured in -+ * the GPU has to be retried (but it has not -+ * started) due to e.g., GPU reset -+ * @BASE_JD_EVENT_END_RP_DONE: this is used for incremental rendering to signal -+ * the completion of a renderpass. This value -+ * shouldn't be returned to userspace but I haven't -+ * seen where it is reset back to JD_EVENT_DONE. - * - * HW and low-level SW events are represented by event codes. - * The status of jobs which succeeded are also represented by -@@ -980,9 +1174,14 @@ struct base_jd_event_v2 { - * struct base_dump_cpu_gpu_counters - Structure for - * BASE_JD_REQ_SOFT_DUMP_CPU_GPU_COUNTERS - * jobs. -+ * @system_time: gpu timestamp -+ * @cycle_counter: gpu cycle count -+ * @sec: cpu time(sec) -+ * @usec: cpu time(usec) -+ * @padding: padding - * - * This structure is stored into the memory pointed to by the @jc field -- * of &struct base_jd_atom_v2. -+ * of &struct base_jd_atom. - * - * It must not occupy the same CPU cache line(s) as any neighboring data. - * This is to avoid cases where access to pages containing the structure -@@ -991,11 +1190,11 @@ struct base_jd_event_v2 { - */ - - struct base_dump_cpu_gpu_counters { -- u64 system_time; -- u64 cycle_counter; -- u64 sec; -- u32 usec; -- u8 padding[36]; -+ __u64 system_time; -+ __u64 cycle_counter; -+ __u64 sec; -+ __u32 usec; -+ __u8 padding[36]; - }; - --#endif /* _BASE_JM_KERNEL_H_ */ -+#endif /* _UAPI_BASE_JM_KERNEL_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h -similarity index 57% -rename from dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h -index 408e98e..72d75cb 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/jm/mali_kbase_jm_ioctl.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifndef _KBASE_JM_IOCTL_H_ --#define _KBASE_JM_IOCTL_H_ -+#ifndef _UAPI_KBASE_JM_IOCTL_H_ -+#define _UAPI_KBASE_JM_IOCTL_H_ - - #include <asm-generic/ioctl.h> - #include <linux/types.h> -@@ -94,16 +93,58 @@ - * - The above changes are checked for safe values in usual builds - * 11.21: - * - v2.0 of mali_trace debugfs file, which now versions the file separately -+ * 11.22: -+ * - Added base_jd_atom (v3), which is seq_nr + base_jd_atom_v2. -+ * KBASE_IOCTL_JOB_SUBMIT supports both in parallel. -+ * 11.23: -+ * - Modified KBASE_IOCTL_MEM_COMMIT behavior to reject requests to modify -+ * the physical memory backing of JIT allocations. This was not supposed -+ * to be a valid use case, but it was allowed by the previous implementation. -+ * 11.24: -+ * - Added a sysfs file 'serialize_jobs' inside a new sub-directory -+ * 'scheduling'. -+ * 11.25: -+ * - Enabled JIT pressure limit in base/kbase by default -+ * 11.26 -+ * - Added kinstr_jm API -+ * 11.27 -+ * - Backwards compatible extension to HWC ioctl. -+ * 11.28: -+ * - Added kernel side cache ops needed hint -+ * 11.29: -+ * - Reserve ioctl 52 -+ * 11.30: -+ * - Add a new priority level BASE_JD_PRIO_REALTIME -+ * - Add ioctl 54: This controls the priority setting. -+ * 11.31: -+ * - Added BASE_JD_REQ_LIMITED_CORE_MASK. -+ * - Added ioctl 55: set_limited_core_count. - */ - #define BASE_UK_VERSION_MAJOR 11 --#define BASE_UK_VERSION_MINOR 21 -+#define BASE_UK_VERSION_MINOR 31 -+ -+/** -+ * struct kbase_ioctl_version_check - Check version compatibility between -+ * kernel and userspace -+ * -+ * @major: Major version number -+ * @minor: Minor version number -+ */ -+struct kbase_ioctl_version_check { -+ __u16 major; -+ __u16 minor; -+}; -+ -+#define KBASE_IOCTL_VERSION_CHECK \ -+ _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) -+ - - /** - * struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel - * -- * @addr: Memory address of an array of struct base_jd_atom_v2 -+ * @addr: Memory address of an array of struct base_jd_atom_v2 or v3 - * @nr_atoms: Number of entries in the array -- * @stride: sizeof(struct base_jd_atom_v2) -+ * @stride: sizeof(struct base_jd_atom_v2) or sizeof(struct base_jd_atom) - */ - struct kbase_ioctl_job_submit { - __u64 addr; -@@ -132,5 +173,51 @@ struct kbase_ioctl_soft_event_update { - #define KBASE_IOCTL_SOFT_EVENT_UPDATE \ - _IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update) - -+/** -+ * struct kbase_kinstr_jm_fd_out - Explains the compatibility information for -+ * the `struct kbase_kinstr_jm_atom_state_change` structure returned from the -+ * kernel -+ * -+ * @size: The size of the `struct kbase_kinstr_jm_atom_state_change` -+ * @version: Represents a breaking change in the -+ * `struct kbase_kinstr_jm_atom_state_change` -+ * @padding: Explicit padding to get the structure up to 64bits. See -+ * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst -+ * -+ * The `struct kbase_kinstr_jm_atom_state_change` may have extra members at the -+ * end of the structure that older user space might not understand. If the -+ * `version` is the same, the structure is still compatible with newer kernels. -+ * The `size` can be used to cast the opaque memory returned from the kernel. -+ */ -+struct kbase_kinstr_jm_fd_out { -+ __u16 size; -+ __u8 version; -+ __u8 padding[5]; -+}; -+ -+/** -+ * struct kbase_kinstr_jm_fd_in - Options when creating the file descriptor -+ * -+ * @count: Number of atom states that can be stored in the kernel circular -+ * buffer. Must be a power of two -+ * @padding: Explicit padding to get the structure up to 64bits. See -+ * https://www.kernel.org/doc/Documentation/ioctl/botching-up-ioctls.rst -+ */ -+struct kbase_kinstr_jm_fd_in { -+ __u16 count; -+ __u8 padding[6]; -+}; -+ -+union kbase_kinstr_jm_fd { -+ struct kbase_kinstr_jm_fd_in in; -+ struct kbase_kinstr_jm_fd_out out; -+}; -+ -+#define KBASE_IOCTL_KINSTR_JM_FD \ -+ _IOWR(KBASE_IOCTL_TYPE, 51, union kbase_kinstr_jm_fd) -+ -+ -+#define KBASE_IOCTL_VERSION_CHECK_RESERVED \ -+ _IOWR(KBASE_IOCTL_TYPE, 52, struct kbase_ioctl_version_check) - --#endif /* _KBASE_JM_IOCTL_H_ */ -+#endif /* _UAPI_KBASE_JM_IOCTL_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_kernel.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_base_kernel.h -similarity index 72% -rename from dvalin/kernel/drivers/gpu/arm/midgard/mali_base_kernel.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/mali_base_kernel.h -index 1e2744d..a46c41f 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_kernel.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_base_kernel.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,48 +17,52 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /* - * Base structures shared with the kernel. - */ - --#ifndef _BASE_KERNEL_H_ --#define _BASE_KERNEL_H_ -+#ifndef _UAPI_BASE_KERNEL_H_ -+#define _UAPI_BASE_KERNEL_H_ -+ -+#include <linux/types.h> - - struct base_mem_handle { - struct { -- u64 handle; -+ __u64 handle; - } basep; - }; - - #include "mali_base_mem_priv.h" --#include "gpu/mali_kbase_gpu_coherency.h" - #include "gpu/mali_kbase_gpu_id.h" -+#include "gpu/mali_kbase_gpu_coherency.h" - - #define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 4 - - #define BASE_MAX_COHERENT_GROUPS 16 - --#if defined CDBG_ASSERT -+#if defined(CDBG_ASSERT) - #define LOCAL_ASSERT CDBG_ASSERT --#elif defined KBASE_DEBUG_ASSERT -+#elif defined(KBASE_DEBUG_ASSERT) - #define LOCAL_ASSERT KBASE_DEBUG_ASSERT - #else -+#if defined(__KERNEL__) - #error assert macro not defined! -+#else -+#define LOCAL_ASSERT(...) ((void)#__VA_ARGS__) -+#endif - #endif - - #if defined(PAGE_MASK) && defined(PAGE_SHIFT) - #define LOCAL_PAGE_SHIFT PAGE_SHIFT - #define LOCAL_PAGE_LSB ~PAGE_MASK - #else --#include <osu/mali_osu.h> -+#ifndef OSU_CONFIG_CPU_PAGE_SIZE_LOG2 -+#define OSU_CONFIG_CPU_PAGE_SIZE_LOG2 12 -+#endif - --#if defined OSU_CONFIG_CPU_PAGE_SIZE_LOG2 -+#if defined(OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - #define LOCAL_PAGE_SHIFT OSU_CONFIG_CPU_PAGE_SIZE_LOG2 - #define LOCAL_PAGE_LSB ((1ul << OSU_CONFIG_CPU_PAGE_SIZE_LOG2) - 1) - #else -@@ -85,7 +90,7 @@ struct base_mem_handle { - * More flags can be added to this list, as long as they don't clash - * (see BASE_MEM_FLAGS_NR_BITS for the number of the first free bit). - */ --typedef u32 base_mem_alloc_flags; -+typedef __u32 base_mem_alloc_flags; - - /* A mask for all the flags which are modifiable via the base_mem_set_flags - * interface. -@@ -121,7 +126,7 @@ typedef u32 base_mem_alloc_flags; - */ - enum base_mem_import_type { - BASE_MEM_IMPORT_TYPE_INVALID = 0, -- /** -+ /* - * Import type with value 1 is deprecated. - */ - BASE_MEM_IMPORT_TYPE_UMM = 2, -@@ -138,8 +143,8 @@ enum base_mem_import_type { - */ - - struct base_mem_import_user_buffer { -- u64 ptr; -- u64 length; -+ __u64 ptr; -+ __u64 length; - }; - - /* Mask to detect 4GB boundary alignment */ -@@ -147,15 +152,15 @@ struct base_mem_import_user_buffer { - /* Mask to detect 4GB boundary (in page units) alignment */ - #define BASE_MEM_PFN_MASK_4GB (BASE_MEM_MASK_4GB >> LOCAL_PAGE_SHIFT) - --/* Limit on the 'extent' parameter for an allocation with the -+/* Limit on the 'extension' parameter for an allocation with the - * BASE_MEM_TILER_ALIGN_TOP flag set - * - * This is the same as the maximum limit for a Buffer Descriptor's chunk size - */ --#define BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES_LOG2 \ -- (21u - (LOCAL_PAGE_SHIFT)) --#define BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES \ -- (1ull << (BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES_LOG2)) -+#define BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES_LOG2 \ -+ (21u - (LOCAL_PAGE_SHIFT)) -+#define BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES \ -+ (1ull << (BASE_MEM_TILER_ALIGN_TOP_EXTENSION_MAX_PAGES_LOG2)) - - /* Bit mask of cookies used for for memory allocation setup */ - #define KBASE_COOKIE_MASK ~1UL /* bit 0 is reserved */ -@@ -163,7 +168,7 @@ struct base_mem_import_user_buffer { - /* Maximum size allowed in a single KBASE_IOCTL_MEM_ALLOC call */ - #define KBASE_MEM_ALLOC_MAX_SIZE ((8ull << 30) >> PAGE_SHIFT) /* 8 GB */ - --/** -+/* - * struct base_fence - Cross-device synchronisation fence. - * - * A fence is used to signal when the GPU has finished accessing a resource that -@@ -200,8 +205,8 @@ struct base_fence { - */ - struct base_mem_aliasing_info { - struct base_mem_handle handle; -- u64 offset; -- u64 length; -+ __u64 offset; -+ __u64 length; - }; - - /* Maximum percentage of just-in-time memory allocation trimming to perform -@@ -213,28 +218,6 @@ struct base_mem_aliasing_info { - */ - #define BASE_JIT_ALLOC_COUNT (255) - --/* Similar to BASE_MEM_TILER_ALIGN_TOP, memory starting from the end of the -- * initial commit is aligned to 'extent' pages, where 'extent' must be a power -- * of 2 and no more than BASE_MEM_TILER_ALIGN_TOP_EXTENT_MAX_PAGES -- */ --#define BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP (1 << 0) -- --/** -- * If set, the heap info address points to a u32 holding the used size in bytes; -- * otherwise it points to a u64 holding the lowest address of unused memory. -- */ --#define BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE (1 << 1) -- --/** -- * Valid set of just-in-time memory allocation flags -- * -- * Note: BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE cannot be set if heap_info_gpu_addr -- * in %base_jit_alloc_info is 0 (atom with BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE set -- * and heap_info_gpu_addr being 0 will be rejected). -- */ --#define BASE_JIT_ALLOC_VALID_FLAGS \ -- (BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP | BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE) -- - /* base_jit_alloc_info in use for kernel driver versions 10.2 to early 11.5 - * - * jit_version is 1 -@@ -245,11 +228,11 @@ struct base_mem_aliasing_info { - * An array of structures was not supported - */ - struct base_jit_alloc_info_10_2 { -- u64 gpu_alloc_addr; -- u64 va_pages; -- u64 commit_pages; -- u64 extent; -- u8 id; -+ __u64 gpu_alloc_addr; -+ __u64 va_pages; -+ __u64 commit_pages; -+ __u64 extension; -+ __u8 id; - }; - - /* base_jit_alloc_info introduced by kernel driver version 11.5, and in use up -@@ -272,16 +255,16 @@ struct base_jit_alloc_info_10_2 { - * 11.10: Arrays of this structure are supported - */ - struct base_jit_alloc_info_11_5 { -- u64 gpu_alloc_addr; -- u64 va_pages; -- u64 commit_pages; -- u64 extent; -- u8 id; -- u8 bin_id; -- u8 max_allocations; -- u8 flags; -- u8 padding[2]; -- u16 usage_id; -+ __u64 gpu_alloc_addr; -+ __u64 va_pages; -+ __u64 commit_pages; -+ __u64 extension; -+ __u8 id; -+ __u8 bin_id; -+ __u8 max_allocations; -+ __u8 flags; -+ __u8 padding[2]; -+ __u16 usage_id; - }; - - /** -@@ -292,7 +275,7 @@ struct base_jit_alloc_info_11_5 { - * @va_pages: The minimum number of virtual pages required. - * @commit_pages: The minimum number of physical pages which - * should back the allocation. -- * @extent: Granularity of physical pages to grow the -+ * @extension: Granularity of physical pages to grow the - * allocation by during a fault. - * @id: Unique ID provided by the caller, this is used - * to pair allocation and free requests. -@@ -327,17 +310,17 @@ struct base_jit_alloc_info_11_5 { - * 11.20: added @heap_info_gpu_addr - */ - struct base_jit_alloc_info { -- u64 gpu_alloc_addr; -- u64 va_pages; -- u64 commit_pages; -- u64 extent; -- u8 id; -- u8 bin_id; -- u8 max_allocations; -- u8 flags; -- u8 padding[2]; -- u16 usage_id; -- u64 heap_info_gpu_addr; -+ __u64 gpu_alloc_addr; -+ __u64 va_pages; -+ __u64 commit_pages; -+ __u64 extension; -+ __u8 id; -+ __u8 bin_id; -+ __u8 max_allocations; -+ __u8 flags; -+ __u8 padding[2]; -+ __u16 usage_id; -+ __u64 heap_info_gpu_addr; - }; - - enum base_external_resource_access { -@@ -346,7 +329,7 @@ enum base_external_resource_access { - }; - - struct base_external_resource { -- u64 ext_resource; -+ __u64 ext_resource; - }; - - -@@ -364,13 +347,13 @@ struct base_external_resource { - * sized at allocation time. - */ - struct base_external_resource_list { -- u64 count; -+ __u64 count; - struct base_external_resource ext_res[1]; - }; - - struct base_jd_debug_copy_buffer { -- u64 address; -- u64 size; -+ __u64 address; -+ __u64 size; - struct base_external_resource extres; - }; - -@@ -482,7 +465,7 @@ struct base_jd_debug_copy_buffer { - * population count, since faulty cores may be disabled during production, - * producing a non-contiguous mask. - * -- * The memory requirements for this algorithm can be determined either by a u64 -+ * The memory requirements for this algorithm can be determined either by a __u64 - * population count on the L2_PRESENT mask (a LUT helper already is - * required for the above), or simple assumption that there can be no more than - * 16 coherent groups, since core groups are typically 4 cores. -@@ -491,165 +474,170 @@ struct base_jd_debug_copy_buffer { - #define BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS 4 - - #define BASE_MAX_COHERENT_GROUPS 16 -- -+/** -+ * struct mali_base_gpu_core_props - GPU core props info -+ * @product_id: Pro specific value. -+ * @version_status: Status of the GPU release. No defined values, but starts at -+ * 0 and increases by one for each release status (alpha, beta, EAC, etc.). -+ * 4 bit values (0-15). -+ * @minor_revision: Minor release number of the GPU. "P" part of an "RnPn" -+ * release number. -+ * 8 bit values (0-255). -+ * @major_revision: Major release number of the GPU. "R" part of an "RnPn" -+ * release number. -+ * 4 bit values (0-15). -+ * @padding: padding to allign to 8-byte -+ * @gpu_freq_khz_max: The maximum GPU frequency. Reported to applications by -+ * clGetDeviceInfo() -+ * @log2_program_counter_size: Size of the shader program counter, in bits. -+ * @texture_features: TEXTURE_FEATURES_x registers, as exposed by the GPU. This -+ * is a bitpattern where a set bit indicates that the format is supported. -+ * Before using a texture format, it is recommended that the corresponding -+ * bit be checked. -+ * @gpu_available_memory_size: Theoretical maximum memory available to the GPU. -+ * It is unlikely that a client will be able to allocate all of this memory -+ * for their own purposes, but this at least provides an upper bound on the -+ * memory available to the GPU. -+ * This is required for OpenCL's clGetDeviceInfo() call when -+ * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The -+ * client will not be expecting to allocate anywhere near this value. -+ * @num_exec_engines: The number of execution engines. -+ */ - struct mali_base_gpu_core_props { -- /** -- * Product specific value. -- */ -- u32 product_id; -- -- /** -- * Status of the GPU release. -- * No defined values, but starts at 0 and increases by one for each -- * release status (alpha, beta, EAC, etc.). -- * 4 bit values (0-15). -- */ -- u16 version_status; -- -- /** -- * Minor release number of the GPU. "P" part of an "RnPn" release number. -- * 8 bit values (0-255). -- */ -- u16 minor_revision; -- -- /** -- * Major release number of the GPU. "R" part of an "RnPn" release number. -- * 4 bit values (0-15). -- */ -- u16 major_revision; -- -- u16 padding; -- -- /* The maximum GPU frequency. Reported to applications by -- * clGetDeviceInfo() -- */ -- u32 gpu_freq_khz_max; -- -- /** -- * Size of the shader program counter, in bits. -- */ -- u32 log2_program_counter_size; -- -- /** -- * TEXTURE_FEATURES_x registers, as exposed by the GPU. This is a -- * bitpattern where a set bit indicates that the format is supported. -- * -- * Before using a texture format, it is recommended that the corresponding -- * bit be checked. -- */ -- u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; -- -- /** -- * Theoretical maximum memory available to the GPU. It is unlikely that a -- * client will be able to allocate all of this memory for their own -- * purposes, but this at least provides an upper bound on the memory -- * available to the GPU. -- * -- * This is required for OpenCL's clGetDeviceInfo() call when -- * CL_DEVICE_GLOBAL_MEM_SIZE is requested, for OpenCL GPU devices. The -- * client will not be expecting to allocate anywhere near this value. -- */ -- u64 gpu_available_memory_size; -- -- /** -- * The number of execution engines. -- */ -- u8 num_exec_engines; -+ __u32 product_id; -+ __u16 version_status; -+ __u16 minor_revision; -+ __u16 major_revision; -+ __u16 padding; -+ __u32 gpu_freq_khz_max; -+ __u32 log2_program_counter_size; -+ __u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; -+ __u64 gpu_available_memory_size; -+ __u8 num_exec_engines; - }; - --/** -- * -+/* - * More information is possible - but associativity and bus width are not - * required by upper-level apis. - */ - struct mali_base_gpu_l2_cache_props { -- u8 log2_line_size; -- u8 log2_cache_size; -- u8 num_l2_slices; /* Number of L2C slices. 1 or higher */ -- u8 padding[5]; -+ __u8 log2_line_size; -+ __u8 log2_cache_size; -+ __u8 num_l2_slices; /* Number of L2C slices. 1 or higher */ -+ __u8 padding[5]; - }; - - struct mali_base_gpu_tiler_props { -- u32 bin_size_bytes; /* Max is 4*2^15 */ -- u32 max_active_levels; /* Max is 2^15 */ -+ __u32 bin_size_bytes; /* Max is 4*2^15 */ -+ __u32 max_active_levels; /* Max is 2^15 */ - }; - - /** -- * GPU threading system details. -+ * struct mali_base_gpu_thread_props - GPU threading system details. -+ * @max_threads: Max. number of threads per core -+ * @max_workgroup_size: Max. number of threads per workgroup -+ * @max_barrier_size: Max. number of threads that can synchronize on a -+ * simple barrier -+ * @max_registers: Total size [1..65535] of the register file available -+ * per core. -+ * @max_task_queue: Max. tasks [1..255] which may be sent to a core -+ * before it becomes blocked. -+ * @max_thread_group_split: Max. allowed value [1..15] of the Thread Group Split -+ * field. -+ * @impl_tech: 0 = Not specified, 1 = Silicon, 2 = FPGA, -+ * 3 = SW Model/Emulation -+ * @padding: padding to allign to 8-byte -+ * @tls_alloc: Number of threads per core that TLS must be -+ * allocated for - */ - struct mali_base_gpu_thread_props { -- u32 max_threads; /* Max. number of threads per core */ -- u32 max_workgroup_size; /* Max. number of threads per workgroup */ -- u32 max_barrier_size; /* Max. number of threads that can synchronize on a simple barrier */ -- u16 max_registers; /* Total size [1..65535] of the register file available per core. */ -- u8 max_task_queue; /* Max. tasks [1..255] which may be sent to a core before it becomes blocked. */ -- u8 max_thread_group_split; /* Max. allowed value [1..15] of the Thread Group Split field. */ -- u8 impl_tech; /* 0 = Not specified, 1 = Silicon, 2 = FPGA, 3 = SW Model/Emulation */ -- u8 padding[3]; -- u32 tls_alloc; /* Number of threads per core that TLS must -- * be allocated for -- */ -+ __u32 max_threads; -+ __u32 max_workgroup_size; -+ __u32 max_barrier_size; -+ __u16 max_registers; -+ __u8 max_task_queue; -+ __u8 max_thread_group_split; -+ __u8 impl_tech; -+ __u8 padding[3]; -+ __u32 tls_alloc; - }; - - /** - * struct mali_base_gpu_coherent_group - descriptor for a coherent group -+ * @core_mask: Core restriction mask required for the group -+ * @num_cores: Number of cores in the group -+ * @padding: padding to allign to 8-byte - * - * \c core_mask exposes all cores in that coherent group, and \c num_cores -- * provides a cached population-count for that mask. -+ * provides a cached population-count for that mask. - * - * @note Whilst all cores are exposed in the mask, not all may be available to -- * the application, depending on the Kernel Power policy. -+ * the application, depending on the Kernel Power policy. - * -- * @note if u64s must be 8-byte aligned, then this structure has 32-bits of wastage. -+ * @note if u64s must be 8-byte aligned, then this structure has 32-bits of -+ * wastage. - */ - struct mali_base_gpu_coherent_group { -- u64 core_mask; /**< Core restriction mask required for the group */ -- u16 num_cores; /**< Number of cores in the group */ -- u16 padding[3]; -+ __u64 core_mask; -+ __u16 num_cores; -+ __u16 padding[3]; - }; - - /** - * struct mali_base_gpu_coherent_group_info - Coherency group information -+ * @num_groups: Number of coherent groups in the GPU. -+ * @num_core_groups: Number of core groups (coherent or not) in the GPU. -+ * Equivalent to the number of L2 Caches. -+ * The GPU Counter dumping writes 2048 bytes per core group, regardless -+ * of whether the core groups are coherent or not. Hence this member is -+ * needed to calculate how much memory is required for dumping. -+ * @note Do not use it to work out how many valid elements are in the -+ * group[] member. Use num_groups instead. -+ * @coherency: Coherency features of the memory, accessed by gpu_mem_features -+ * methods -+ * @padding: padding to allign to 8-byte -+ * @group: Descriptors of coherent groups - * - * Note that the sizes of the members could be reduced. However, the \c group -- * member might be 8-byte aligned to ensure the u64 core_mask is 8-byte -+ * member might be 8-byte aligned to ensure the __u64 core_mask is 8-byte - * aligned, thus leading to wastage if the other members sizes were reduced. - * - * The groups are sorted by core mask. The core masks are non-repeating and do - * not intersect. - */ - struct mali_base_gpu_coherent_group_info { -- u32 num_groups; -- -- /** -- * Number of core groups (coherent or not) in the GPU. Equivalent to the number of L2 Caches. -- * -- * The GPU Counter dumping writes 2048 bytes per core group, regardless of -- * whether the core groups are coherent or not. Hence this member is needed -- * to calculate how much memory is required for dumping. -- * -- * @note Do not use it to work out how many valid elements are in the -- * group[] member. Use num_groups instead. -- */ -- u32 num_core_groups; -- -- /** -- * Coherency features of the memory, accessed by gpu_mem_features -- * methods -- */ -- u32 coherency; -- -- u32 padding; -- -- /** -- * Descriptors of coherent groups -- */ -+ __u32 num_groups; -+ __u32 num_core_groups; -+ __u32 coherency; -+ __u32 padding; - struct mali_base_gpu_coherent_group group[BASE_MAX_COHERENT_GROUPS]; - }; - - /** - * struct gpu_raw_gpu_props - A complete description of the GPU's Hardware - * Configuration Discovery registers. -+ * @shader_present: Shader core present bitmap -+ * @tiler_present: Tiler core present bitmap -+ * @l2_present: Level 2 cache present bitmap -+ * @stack_present: Core stack present bitmap -+ * @l2_features: L2 features -+ * @core_features: Core features -+ * @mem_features: Mem features -+ * @mmu_features: Mmu features -+ * @as_present: Bitmap of address spaces present -+ * @js_present: Job slots present -+ * @js_features: Array of job slot features. -+ * @tiler_features: Tiler features -+ * @texture_features: TEXTURE_FEATURES_x registers, as exposed by the GPU -+ * @gpu_id: GPU and revision identifier -+ * @thread_max_threads: Maximum number of threads per core -+ * @thread_max_workgroup_size: Maximum number of threads per workgroup -+ * @thread_max_barrier_size: Maximum number of threads per barrier -+ * @thread_features: Thread features -+ * @coherency_mode: Note: This is the _selected_ coherency mode rather than the -+ * available modes as exposed in the coherency_features register -+ * @thread_tls_alloc: Number of threads per core that TLS must be allocated for -+ * @gpu_features: GPU features - * - * The information is presented inefficiently for access. For frequent access, - * the values should be better expressed in an unpacked form in the -@@ -664,62 +652,69 @@ struct mali_base_gpu_coherent_group_info { - * - */ - struct gpu_raw_gpu_props { -- u64 shader_present; -- u64 tiler_present; -- u64 l2_present; -- u64 stack_present; -+ __u64 shader_present; -+ __u64 tiler_present; -+ __u64 l2_present; -+ __u64 stack_present; -+ __u32 l2_features; -+ __u32 core_features; -+ __u32 mem_features; -+ __u32 mmu_features; - -- u32 l2_features; -- u32 core_features; -- u32 mem_features; -- u32 mmu_features; -+ __u32 as_present; - -- u32 as_present; -+ __u32 js_present; -+ __u32 js_features[GPU_MAX_JOB_SLOTS]; -+ __u32 tiler_features; -+ __u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; - -- u32 js_present; -- u32 js_features[GPU_MAX_JOB_SLOTS]; -- u32 tiler_features; -- u32 texture_features[BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS]; -+ __u32 gpu_id; - -- u32 gpu_id; -- -- u32 thread_max_threads; -- u32 thread_max_workgroup_size; -- u32 thread_max_barrier_size; -- u32 thread_features; -+ __u32 thread_max_threads; -+ __u32 thread_max_workgroup_size; -+ __u32 thread_max_barrier_size; -+ __u32 thread_features; - - /* - * Note: This is the _selected_ coherency mode rather than the - * available modes as exposed in the coherency_features register. - */ -- u32 coherency_mode; -+ __u32 coherency_mode; - -- u32 thread_tls_alloc; -+ __u32 thread_tls_alloc; -+ __u64 gpu_features; - }; - - /** - * struct base_gpu_props - Return structure for base_get_gpu_props(). -+ * @core_props: Core props. -+ * @l2_props: L2 props. -+ * @unused_1: Keep for backwards compatibility. -+ * @tiler_props: Tiler props. -+ * @thread_props: Thread props. -+ * @raw_props: This member is large, likely to be 128 bytes. -+ * @coherency_info: This must be last member of the structure. - * - * NOTE: the raw_props member in this data structure contains the register - * values from which the value of the other members are derived. The derived - * members exist to allow for efficient access and/or shielding the details - * of the layout of the registers. -- * -- * @unused_1: Keep for backwards compatibility. -- * @raw_props: This member is large, likely to be 128 bytes. -- * @coherency_info: This must be last member of the structure. -- */ -+ * */ - struct base_gpu_props { - struct mali_base_gpu_core_props core_props; - struct mali_base_gpu_l2_cache_props l2_props; -- u64 unused_1; -+ __u64 unused_1; - struct mali_base_gpu_tiler_props tiler_props; - struct mali_base_gpu_thread_props thread_props; - struct gpu_raw_gpu_props raw_props; - struct mali_base_gpu_coherent_group_info coherency_info; - }; - -+#if MALI_USE_CSF -+#include "csf/mali_base_csf_kernel.h" -+#else - #include "jm/mali_base_jm_kernel.h" -+#endif - - /** - * base_mem_group_id_get() - Get group ID from flags -@@ -730,7 +725,7 @@ struct base_gpu_props { - * - * Return: group ID(0~15) extracted from the parameter - */ --static inline int base_mem_group_id_get(base_mem_alloc_flags flags) -+static __inline__ int base_mem_group_id_get(base_mem_alloc_flags flags) - { - LOCAL_ASSERT((flags & ~BASE_MEM_FLAGS_INPUT_MASK) == 0); - return (int)((flags & BASE_MEM_GROUP_ID_MASK) >> -@@ -749,10 +744,12 @@ static inline int base_mem_group_id_get(base_mem_alloc_flags flags) - * The return value can be combined with other flags against base_mem_alloc - * to identify a specific memory group. - */ --static inline base_mem_alloc_flags base_mem_group_id_set(int id) -+static __inline__ base_mem_alloc_flags base_mem_group_id_set(int id) - { -- LOCAL_ASSERT(id >= 0); -- LOCAL_ASSERT(id < BASE_MEM_GROUP_COUNT); -+ if ((id < 0) || (id >= BASE_MEM_GROUP_COUNT)) { -+ /* Set to default value when id is out of range. */ -+ id = BASE_MEM_GROUP_DEFAULT; -+ } - - return ((base_mem_alloc_flags)id << BASEP_MEM_GROUP_ID_SHIFT) & - BASE_MEM_GROUP_ID_MASK; -@@ -768,7 +765,7 @@ static inline base_mem_alloc_flags base_mem_group_id_set(int id) - * - * Return: Bitmask of flags to pass to base_context_init. - */ --static inline base_context_create_flags base_context_mmu_group_id_set( -+static __inline__ base_context_create_flags base_context_mmu_group_id_set( - int const group_id) - { - LOCAL_ASSERT(group_id >= 0); -@@ -788,7 +785,7 @@ static inline base_context_create_flags base_context_mmu_group_id_set( - * - * Return: Physical memory group ID. Valid range is 0..(BASE_MEM_GROUP_COUNT-1). - */ --static inline int base_context_mmu_group_id_get( -+static __inline__ int base_context_mmu_group_id_get( - base_context_create_flags const flags) - { - LOCAL_ASSERT(flags == (flags & BASEP_CONTEXT_CREATE_ALLOWED_FLAGS)); -@@ -820,4 +817,10 @@ static inline int base_context_mmu_group_id_get( - BASE_TIMEINFO_KERNEL_SOURCE_FLAG | \ - BASE_TIMEINFO_USER_SOURCE_FLAG) - --#endif /* _BASE_KERNEL_H_ */ -+/* Maximum number of source allocations allowed to create an alias allocation. -+ * This needs to be 4096 * 6 to allow cube map arrays with up to 4096 array -+ * layers, since each cube map in the array will have 6 faces. -+ */ -+#define BASE_MEM_ALIAS_MAX_ENTS ((size_t)24576) -+ -+#endif /* _UAPI_BASE_KERNEL_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_mem_priv.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_base_mem_priv.h -similarity index 80% -rename from dvalin/kernel/drivers/gpu/arm/midgard/mali_base_mem_priv.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/mali_base_mem_priv.h -index 844a025..304a334 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_base_mem_priv.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_base_mem_priv.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010-2015, 2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010-2015, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,14 +17,14 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -+#ifndef _UAPI_BASE_MEM_PRIV_H_ -+#define _UAPI_BASE_MEM_PRIV_H_ - -+#include <linux/types.h> - --#ifndef _BASE_MEM_PRIV_H_ --#define _BASE_MEM_PRIV_H_ -+#include "mali_base_kernel.h" - - #define BASE_SYNCSET_OP_MSYNC (1U << 0) - #define BASE_SYNCSET_OP_CSYNC (1U << 1) -@@ -48,10 +49,10 @@ - */ - struct basep_syncset { - struct base_mem_handle mem_handle; -- u64 user_addr; -- u64 size; -- u8 type; -- u8 padding[7]; -+ __u64 user_addr; -+ __u64 size; -+ __u8 type; -+ __u8 padding[7]; - }; - --#endif -+#endif /* _UAPI_BASE_MEM_PRIV_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_hwcnt_reader.h -similarity index 52% -rename from dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_hwcnt_reader.h -index 10706b8..9baaec1 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_hwcnt_reader.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_hwcnt_reader.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2015 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2015, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,38 +17,59 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifndef _KBASE_HWCNT_READER_H_ --#define _KBASE_HWCNT_READER_H_ -+#ifndef _UAPI_KBASE_HWCNT_READER_H_ -+#define _UAPI_KBASE_HWCNT_READER_H_ -+ -+#include <stddef.h> -+#include <linux/types.h> - - /* The ids of ioctl commands. */ - #define KBASE_HWCNT_READER 0xBE --#define KBASE_HWCNT_READER_GET_HWVER _IOR(KBASE_HWCNT_READER, 0x00, u32) --#define KBASE_HWCNT_READER_GET_BUFFER_SIZE _IOR(KBASE_HWCNT_READER, 0x01, u32) --#define KBASE_HWCNT_READER_DUMP _IOW(KBASE_HWCNT_READER, 0x10, u32) --#define KBASE_HWCNT_READER_CLEAR _IOW(KBASE_HWCNT_READER, 0x11, u32) --#define KBASE_HWCNT_READER_GET_BUFFER _IOR(KBASE_HWCNT_READER, 0x20,\ -+#define KBASE_HWCNT_READER_GET_HWVER _IOR(KBASE_HWCNT_READER, 0x00, __u32) -+#define KBASE_HWCNT_READER_GET_BUFFER_SIZE _IOR(KBASE_HWCNT_READER, 0x01, __u32) -+#define KBASE_HWCNT_READER_DUMP _IOW(KBASE_HWCNT_READER, 0x10, __u32) -+#define KBASE_HWCNT_READER_CLEAR _IOW(KBASE_HWCNT_READER, 0x11, __u32) -+#define KBASE_HWCNT_READER_GET_BUFFER _IOC(_IOC_READ, KBASE_HWCNT_READER, 0x20,\ -+ offsetof(struct kbase_hwcnt_reader_metadata, cycles)) -+#define KBASE_HWCNT_READER_GET_BUFFER_WITH_CYCLES _IOR(KBASE_HWCNT_READER, 0x20,\ - struct kbase_hwcnt_reader_metadata) --#define KBASE_HWCNT_READER_PUT_BUFFER _IOW(KBASE_HWCNT_READER, 0x21,\ -+#define KBASE_HWCNT_READER_PUT_BUFFER _IOC(_IOC_WRITE, KBASE_HWCNT_READER, 0x21,\ -+ offsetof(struct kbase_hwcnt_reader_metadata, cycles)) -+#define KBASE_HWCNT_READER_PUT_BUFFER_WITH_CYCLES _IOW(KBASE_HWCNT_READER, 0x21,\ - struct kbase_hwcnt_reader_metadata) --#define KBASE_HWCNT_READER_SET_INTERVAL _IOW(KBASE_HWCNT_READER, 0x30, u32) --#define KBASE_HWCNT_READER_ENABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x40, u32) --#define KBASE_HWCNT_READER_DISABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x41, u32) --#define KBASE_HWCNT_READER_GET_API_VERSION _IOW(KBASE_HWCNT_READER, 0xFF, u32) -+#define KBASE_HWCNT_READER_SET_INTERVAL _IOW(KBASE_HWCNT_READER, 0x30, __u32) -+#define KBASE_HWCNT_READER_ENABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x40, __u32) -+#define KBASE_HWCNT_READER_DISABLE_EVENT _IOW(KBASE_HWCNT_READER, 0x41, __u32) -+#define KBASE_HWCNT_READER_GET_API_VERSION _IOW(KBASE_HWCNT_READER, 0xFF, __u32) -+#define KBASE_HWCNT_READER_GET_API_VERSION_WITH_FEATURES \ -+ _IOW(KBASE_HWCNT_READER, 0xFF, \ -+ struct kbase_hwcnt_reader_api_version) -+ -+/** -+ * struct kbase_hwcnt_reader_metadata_cycles - GPU clock cycles -+ * @top: the number of cycles associated with the main clock for the -+ * GPU -+ * @shader_cores: the cycles that have elapsed on the GPU shader cores -+ */ -+struct kbase_hwcnt_reader_metadata_cycles { -+ __u64 top; -+ __u64 shader_cores; -+}; - - /** - * struct kbase_hwcnt_reader_metadata - hwcnt reader sample buffer metadata - * @timestamp: time when sample was collected - * @event_id: id of an event that triggered sample collection - * @buffer_idx: position in sampling area where sample buffer was stored -+ * @cycles: the GPU cycles that occurred since the last sample - */ - struct kbase_hwcnt_reader_metadata { -- u64 timestamp; -- u32 event_id; -- u32 buffer_idx; -+ __u64 timestamp; -+ __u32 event_id; -+ __u32 buffer_idx; -+ struct kbase_hwcnt_reader_metadata_cycles cycles; - }; - - /** -@@ -63,9 +85,21 @@ enum base_hwcnt_reader_event { - BASE_HWCNT_READER_EVENT_PERIODIC, - BASE_HWCNT_READER_EVENT_PREJOB, - BASE_HWCNT_READER_EVENT_POSTJOB, -- - BASE_HWCNT_READER_EVENT_COUNT - }; - --#endif /* _KBASE_HWCNT_READER_H_ */ -+#define KBASE_HWCNT_READER_API_VERSION_NO_FEATURE (0) -+#define KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_TOP (1 << 0) -+#define KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_SHADER_CORES (1 << 1) -+/** -+ * struct kbase_hwcnt_reader_api_version - hwcnt reader API version -+ * @version: API version -+ * @features: available features in this API version -+ */ -+struct kbase_hwcnt_reader_api_version { -+ __u32 version; -+ __u32 features; -+}; -+ -+#endif /* _UAPI_KBASE_HWCNT_READER_H_ */ - -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ioctl.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_ioctl.h -similarity index 83% -rename from dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ioctl.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_ioctl.h -index 977b194..29ff32a 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_ioctl.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_ioctl.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2017-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,12 +17,10 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - --#ifndef _KBASE_IOCTL_H_ --#define _KBASE_IOCTL_H_ -+#ifndef _UAPI_KBASE_IOCTL_H_ -+#define _UAPI_KBASE_IOCTL_H_ - - #ifdef __cpluscplus - extern "C" { -@@ -30,24 +29,14 @@ extern "C" { - #include <asm-generic/ioctl.h> - #include <linux/types.h> - -+#if MALI_USE_CSF -+#include "csf/mali_kbase_csf_ioctl.h" -+#else - #include "jm/mali_kbase_jm_ioctl.h" -+#endif /* MALI_USE_CSF */ - - #define KBASE_IOCTL_TYPE 0x80 - --/** -- * struct kbase_ioctl_version_check - Check version compatibility with kernel -- * -- * @major: Major version number -- * @minor: Minor version number -- */ --struct kbase_ioctl_version_check { -- __u16 major; -- __u16 minor; --}; -- --#define KBASE_IOCTL_VERSION_CHECK \ -- _IOWR(KBASE_IOCTL_TYPE, 0, struct kbase_ioctl_version_check) -- - /** - * struct kbase_ioctl_set_flags - Set kernel context creation flags - * -@@ -75,16 +64,16 @@ struct kbase_ioctl_set_flags { - * @flags may be used in the future to request a different format for the - * buffer. With @flags == 0 the following format is used. - * -- * The buffer will be filled with pairs of values, a u32 key identifying the -+ * The buffer will be filled with pairs of values, a __u32 key identifying the - * property followed by the value. The size of the value is identified using - * the bottom bits of the key. The value then immediately followed the key and - * is tightly packed (there is no padding). All keys and values are - * little-endian. - * -- * 00 = u8 -- * 01 = u16 -- * 10 = u32 -- * 11 = u64 -+ * 00 = __u8 -+ * 01 = __u16 -+ * 10 = __u32 -+ * 11 = __u64 - */ - struct kbase_ioctl_get_gpuprops { - __u64 buffer; -@@ -97,22 +86,20 @@ struct kbase_ioctl_get_gpuprops { - - /** - * union kbase_ioctl_mem_alloc - Allocate memory on the GPU -- * -- * @va_pages: The number of pages of virtual address space to reserve -- * @commit_pages: The number of physical pages to allocate -- * @extent: The number of extra pages to allocate on each GPU fault which grows -- * the region -- * @flags: Flags -- * @gpu_va: The GPU virtual address which is allocated -- * - * @in: Input parameters -+ * @in.va_pages: The number of pages of virtual address space to reserve -+ * @in.commit_pages: The number of physical pages to allocate -+ * @in.extension: The number of extra pages to allocate on each GPU fault which grows the region -+ * @in.flags: Flags - * @out: Output parameters -+ * @out.flags: Flags -+ * @out.gpu_va: The GPU virtual address which is allocated - */ - union kbase_ioctl_mem_alloc { - struct { - __u64 va_pages; - __u64 commit_pages; -- __u64 extent; -+ __u64 extension; - __u64 flags; - } in; - struct { -@@ -126,14 +113,13 @@ union kbase_ioctl_mem_alloc { - - /** - * struct kbase_ioctl_mem_query - Query properties of a GPU memory region -- * @gpu_addr: A GPU address contained within the region -- * @query: The type of query -- * @value: The result of the query -- * -- * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. -- * - * @in: Input parameters -+ * @in.gpu_addr: A GPU address contained within the region -+ * @in.query: The type of query - * @out: Output parameters -+ * @out.value: The result of the query -+ * -+ * Use a %KBASE_MEM_QUERY_xxx flag as input for @query. - */ - union kbase_ioctl_mem_query { - struct { -@@ -148,9 +134,9 @@ union kbase_ioctl_mem_query { - #define KBASE_IOCTL_MEM_QUERY \ - _IOWR(KBASE_IOCTL_TYPE, 6, union kbase_ioctl_mem_query) - --#define KBASE_MEM_QUERY_COMMIT_SIZE ((u64)1) --#define KBASE_MEM_QUERY_VA_SIZE ((u64)2) --#define KBASE_MEM_QUERY_FLAGS ((u64)3) -+#define KBASE_MEM_QUERY_COMMIT_SIZE ((__u64)1) -+#define KBASE_MEM_QUERY_VA_SIZE ((__u64)2) -+#define KBASE_MEM_QUERY_FLAGS ((__u64)3) - - /** - * struct kbase_ioctl_mem_free - Free a memory region -@@ -166,7 +152,7 @@ struct kbase_ioctl_mem_free { - /** - * struct kbase_ioctl_hwcnt_reader_setup - Setup HWC dumper/reader - * @buffer_count: requested number of dumping buffers -- * @jm_bm: counters selection bitmask (JM) -+ * @fe_bm: counters selection bitmask (Front end) - * @shader_bm: counters selection bitmask (Shader) - * @tiler_bm: counters selection bitmask (Tiler) - * @mmu_l2_bm: counters selection bitmask (MMU_L2) -@@ -175,7 +161,7 @@ struct kbase_ioctl_mem_free { - */ - struct kbase_ioctl_hwcnt_reader_setup { - __u32 buffer_count; -- __u32 jm_bm; -+ __u32 fe_bm; - __u32 shader_bm; - __u32 tiler_bm; - __u32 mmu_l2_bm; -@@ -187,14 +173,14 @@ struct kbase_ioctl_hwcnt_reader_setup { - /** - * struct kbase_ioctl_hwcnt_enable - Enable hardware counter collection - * @dump_buffer: GPU address to write counters to -- * @jm_bm: counters selection bitmask (JM) -+ * @fe_bm: counters selection bitmask (Front end) - * @shader_bm: counters selection bitmask (Shader) - * @tiler_bm: counters selection bitmask (Tiler) - * @mmu_l2_bm: counters selection bitmask (MMU_L2) - */ - struct kbase_ioctl_hwcnt_enable { - __u64 dump_buffer; -- __u32 jm_bm; -+ __u32 fe_bm; - __u32 shader_bm; - __u32 tiler_bm; - __u32 mmu_l2_bm; -@@ -353,13 +339,12 @@ struct kbase_ioctl_mem_sync { - /** - * union kbase_ioctl_mem_find_cpu_offset - Find the offset of a CPU pointer - * -- * @gpu_addr: The GPU address of the memory region -- * @cpu_addr: The CPU address to locate -- * @size: A size in bytes to validate is contained within the region -- * @offset: The offset from the start of the memory region to @cpu_addr -- * - * @in: Input parameters -+ * @in.gpu_addr: The GPU address of the memory region -+ * @in.cpu_addr: The CPU address to locate -+ * @in.size: A size in bytes to validate is contained within the region - * @out: Output parameters -+ * @out.offset: The offset from the start of the memory region to @cpu_addr - */ - union kbase_ioctl_mem_find_cpu_offset { - struct { -@@ -424,15 +409,15 @@ struct kbase_ioctl_mem_commit { - - /** - * union kbase_ioctl_mem_alias - Create an alias of memory regions -- * @flags: Flags, see BASE_MEM_xxx -- * @stride: Bytes between start of each memory region -- * @nents: The number of regions to pack together into the alias -- * @aliasing_info: Pointer to an array of struct base_mem_aliasing_info -- * @gpu_va: Address of the new alias -- * @va_pages: Size of the new alias -- * - * @in: Input parameters -+ * @in.flags: Flags, see BASE_MEM_xxx -+ * @in.stride: Bytes between start of each memory region -+ * @in.nents: The number of regions to pack together into the alias -+ * @in.aliasing_info: Pointer to an array of struct base_mem_aliasing_info - * @out: Output parameters -+ * @out.flags: Flags, see BASE_MEM_xxx -+ * @out.gpu_va: Address of the new alias -+ * @out.va_pages: Size of the new alias - */ - union kbase_ioctl_mem_alias { - struct { -@@ -453,15 +438,15 @@ union kbase_ioctl_mem_alias { - - /** - * union kbase_ioctl_mem_import - Import memory for use by the GPU -- * @flags: Flags, see BASE_MEM_xxx -- * @phandle: Handle to the external memory -- * @type: Type of external memory, see base_mem_import_type -- * @padding: Amount of extra VA pages to append to the imported buffer -- * @gpu_va: Address of the new alias -- * @va_pages: Size of the new alias -- * - * @in: Input parameters -+ * @in.flags: Flags, see BASE_MEM_xxx -+ * @in.phandle: Handle to the external memory -+ * @in.type: Type of external memory, see base_mem_import_type -+ * @in.padding: Amount of extra VA pages to append to the imported buffer - * @out: Output parameters -+ * @out.flags: Flags, see BASE_MEM_xxx -+ * @out.gpu_va: Address of the new alias -+ * @out.va_pages: Size of the new alias - */ - union kbase_ioctl_mem_import { - struct { -@@ -544,7 +529,7 @@ struct kbase_ioctl_mem_profile_add { - /** - * struct kbase_ioctl_sticky_resource_map - Permanently map an external resource - * @count: Number of resources -- * @address: Array of u64 GPU addresses of the external resources to map -+ * @address: Array of __u64 GPU addresses of the external resources to map - */ - struct kbase_ioctl_sticky_resource_map { - __u64 count; -@@ -558,7 +543,7 @@ struct kbase_ioctl_sticky_resource_map { - * struct kbase_ioctl_sticky_resource_map - Unmap a resource mapped which was - * previously permanently mapped - * @count: Number of resources -- * @address: Array of u64 GPU addresses of the external resources to unmap -+ * @address: Array of __u64 GPU addresses of the external resources to unmap - */ - struct kbase_ioctl_sticky_resource_unmap { - __u64 count; -@@ -574,15 +559,13 @@ struct kbase_ioctl_sticky_resource_unmap { - * the given gpu address and - * the offset of that address - * into the region -- * -- * @gpu_addr: GPU virtual address -- * @size: Size in bytes within the region -- * @start: Address of the beginning of the memory region enclosing @gpu_addr -- * for the length of @offset bytes -- * @offset: The offset from the start of the memory region to @gpu_addr -- * - * @in: Input parameters -+ * @in.gpu_addr: GPU virtual address -+ * @in.size: Size in bytes within the region - * @out: Output parameters -+ * @out.start: Address of the beginning of the memory region enclosing @gpu_addr -+ * for the length of @offset bytes -+ * @out.offset: The offset from the start of the memory region to @gpu_addr - */ - union kbase_ioctl_mem_find_gpu_start_and_offset { - struct { -@@ -598,7 +581,6 @@ union kbase_ioctl_mem_find_gpu_start_and_offset { - #define KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET \ - _IOWR(KBASE_IOCTL_TYPE, 31, union kbase_ioctl_mem_find_gpu_start_and_offset) - -- - #define KBASE_IOCTL_CINSTR_GWT_START \ - _IO(KBASE_IOCTL_TYPE, 33) - -@@ -607,14 +589,15 @@ union kbase_ioctl_mem_find_gpu_start_and_offset { - - /** - * union kbase_ioctl_gwt_dump - Used to collect all GPU write fault addresses. -- * @addr_buffer: Address of buffer to hold addresses of gpu modified areas. -- * @size_buffer: Address of buffer to hold size of modified areas (in pages) -- * @len: Number of addresses the buffers can hold. -- * @more_data_available: Status indicating if more addresses are available. -- * @no_of_addr_collected: Number of addresses collected into addr_buffer. -- * - * @in: Input parameters -+ * @in.addr_buffer: Address of buffer to hold addresses of gpu modified areas. -+ * @in.size_buffer: Address of buffer to hold size of modified areas (in pages) -+ * @in.len: Number of addresses the buffers can hold. -+ * @in.padding: padding - * @out: Output parameters -+ * @out.no_of_addr_collected: Number of addresses collected into addr_buffer. -+ * @out.more_data_available: Status indicating if more addresses are available. -+ * @out.padding: padding - * - * This structure is used when performing a call to dump GPU write fault - * addresses. -@@ -652,18 +635,15 @@ struct kbase_ioctl_mem_exec_init { - /** - * union kbase_ioctl_get_cpu_gpu_timeinfo - Request zero or more types of - * cpu/gpu time (counter values) -- * -- * @request_flags: Bit-flags indicating the requested types. -- * @paddings: Unused, size alignment matching the out. -- * @sec: Integer field of the monotonic time, unit in seconds. -- * @nsec: Fractional sec of the monotonic time, in nano-seconds. -- * @padding: Unused, for u64 alignment -- * @timestamp: System wide timestamp (counter) value. -- * @cycle_counter: GPU cycle counter value. -- * - * @in: Input parameters -+ * @in.request_flags: Bit-flags indicating the requested types. -+ * @in.paddings: Unused, size alignment matching the out. - * @out: Output parameters -- * -+ * @out.sec: Integer field of the monotonic time, unit in seconds. -+ * @out.nsec: Fractional sec of the monotonic time, in nano-seconds. -+ * @out.padding: Unused, for __u64 alignment -+ * @out.timestamp: System wide timestamp (counter) value. -+ * @out.cycle_counter: GPU cycle counter value. - */ - union kbase_ioctl_get_cpu_gpu_timeinfo { - struct { -@@ -682,6 +662,31 @@ union kbase_ioctl_get_cpu_gpu_timeinfo { - #define KBASE_IOCTL_GET_CPU_GPU_TIMEINFO \ - _IOWR(KBASE_IOCTL_TYPE, 50, union kbase_ioctl_get_cpu_gpu_timeinfo) - -+/** -+ * struct kbase_ioctl_context_priority_check - Check the max possible priority -+ * @priority: Input priority & output priority -+ */ -+ -+struct kbase_ioctl_context_priority_check { -+ __u8 priority; -+}; -+ -+#define KBASE_IOCTL_CONTEXT_PRIORITY_CHECK \ -+ _IOWR(KBASE_IOCTL_TYPE, 54, struct kbase_ioctl_context_priority_check) -+ -+/** -+ * struct kbase_ioctl_set_limited_core_count - Set the limited core count. -+ * -+ * @max_core_count: Maximum core count -+ */ -+struct kbase_ioctl_set_limited_core_count { -+ __u8 max_core_count; -+}; -+ -+#define KBASE_IOCTL_SET_LIMITED_CORE_COUNT \ -+ _IOW(KBASE_IOCTL_TYPE, 55, struct kbase_ioctl_set_limited_core_count) -+ -+ - /*************** - * test ioctls * - ***************/ -@@ -692,23 +697,6 @@ union kbase_ioctl_get_cpu_gpu_timeinfo { - - #define KBASE_IOCTL_TEST_TYPE (KBASE_IOCTL_TYPE + 1) - --/** -- * struct kbase_ioctl_tlstream_test - Start a timeline stream test -- * -- * @tpw_count: number of trace point writers in each context -- * @msg_delay: time delay between tracepoints from one writer in milliseconds -- * @msg_count: number of trace points written by one writer -- * @aux_msg: if non-zero aux messages will be included -- */ --struct kbase_ioctl_tlstream_test { -- __u32 tpw_count; -- __u32 msg_delay; -- __u32 msg_count; -- __u32 aux_msg; --}; -- --#define KBASE_IOCTL_TLSTREAM_TEST \ -- _IOW(KBASE_IOCTL_TEST_TYPE, 1, struct kbase_ioctl_tlstream_test) - - /** - * struct kbase_ioctl_tlstream_stats - Read tlstream stats for test purposes -@@ -836,13 +824,13 @@ struct kbase_ioctl_tlstream_stats { - #define KBASE_GPUPROP_TEXTURE_FEATURES_3 80 - #define KBASE_GPUPROP_RAW_TEXTURE_FEATURES_3 81 - --#define KBASE_GPUPROP_NUM_EXEC_ENGINES 82 -+#define KBASE_GPUPROP_NUM_EXEC_ENGINES 82 - - #define KBASE_GPUPROP_RAW_THREAD_TLS_ALLOC 83 - #define KBASE_GPUPROP_TLS_ALLOC 84 -- -+#define KBASE_GPUPROP_RAW_GPU_FEATURES 85 - #ifdef __cpluscplus - } - #endif - --#endif -+#endif /* _UAPI_KBASE_IOCTL_H_ */ -diff --git a/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_kinstr_jm_reader.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_kinstr_jm_reader.h -new file mode 100644 -index 0000000..72e1b9d ---- /dev/null -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_kbase_kinstr_jm_reader.h -@@ -0,0 +1,69 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ -+/* -+ * -+ * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved. -+ * -+ * This program is free software and is provided to you under the terms of the -+ * GNU General Public License version 2 as published by the Free Software -+ * Foundation, and any use by you of this program is subject to the terms -+ * of such GNU license. -+ * -+ * This program is distributed in the hope that it will be useful, -+ * but WITHOUT ANY WARRANTY; without even the implied warranty of -+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -+ * GNU General Public License for more details. -+ * -+ * You should have received a copy of the GNU General Public License -+ * along with this program; if not, you can access it online at -+ * http://www.gnu.org/licenses/gpl-2.0.html. -+ * -+ */ -+ -+/* -+ * mali_kbase_kinstr_jm_reader.h -+ * Provides an ioctl API to read kernel atom state changes. The flow of the -+ * API is: -+ * 1. Obtain the file descriptor with ``KBASE_IOCTL_KINSTR_JM_FD`` -+ * 2. Determine the buffer structure layout via the above ioctl's returned -+ * size and version fields in ``struct kbase_kinstr_jm_fd_out`` -+ * 4. Poll the file descriptor for ``POLLIN`` -+ * 5. Get data with read() on the fd -+ * 6. Use the structure version to understand how to read the data from the -+ * buffer -+ * 7. Repeat 4-6 -+ * 8. Close the file descriptor -+ */ -+ -+#ifndef _UAPI_KBASE_KINSTR_JM_READER_H_ -+#define _UAPI_KBASE_KINSTR_JM_READER_H_ -+ -+/** -+ * enum kbase_kinstr_jm_reader_atom_state - Determines the work state of an atom -+ * @KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE: Signifies that an atom has -+ * entered a hardware queue -+ * @KBASE_KINSTR_JM_READER_ATOM_STATE_START: Signifies that work has started -+ * on an atom -+ * @KBASE_KINSTR_JM_READER_ATOM_STATE_STOP: Signifies that work has stopped -+ * on an atom -+ * @KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE: Signifies that work has -+ * completed on an atom -+ * @KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT: The number of state enumerations -+ * -+ * We can add new states to the end of this if they do not break the existing -+ * state machine. Old user mode code can gracefully ignore states they do not -+ * understand. -+ * -+ * If we need to make a breaking change to the state machine, we can do that by -+ * changing the version reported by KBASE_IOCTL_KINSTR_JM_FD. This will -+ * mean that old user mode code will fail to understand the new state field in -+ * the structure and gracefully not use the state change API. -+ */ -+enum kbase_kinstr_jm_reader_atom_state { -+ KBASE_KINSTR_JM_READER_ATOM_STATE_QUEUE, -+ KBASE_KINSTR_JM_READER_ATOM_STATE_START, -+ KBASE_KINSTR_JM_READER_ATOM_STATE_STOP, -+ KBASE_KINSTR_JM_READER_ATOM_STATE_COMPLETE, -+ KBASE_KINSTR_JM_READER_ATOM_STATE_COUNT -+}; -+ -+#endif /* _UAPI_KBASE_KINSTR_JM_READER_H_ */ -diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_uk.h b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_uk.h -similarity index 69% -rename from dvalin/kernel/drivers/gpu/arm/midgard/mali_uk.h -rename to dvalin/kernel/include/uapi/gpu/arm/midgard/mali_uk.h -index 701f390..fcb6cb8 100644 ---- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_uk.h -+++ b/dvalin/kernel/include/uapi/gpu/arm/midgard/mali_uk.h -@@ -1,11 +1,12 @@ -+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ - /* - * -- * (C) COPYRIGHT 2010, 2012-2015, 2018 ARM Limited. All rights reserved. -+ * (C) COPYRIGHT 2010, 2012-2015, 2018, 2020-2021 ARM Limited. All rights reserved. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms -- * of such GNU licence. -+ * of such GNU license. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of -@@ -16,32 +17,22 @@ - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * -- * SPDX-License-Identifier: GPL-2.0 -- * - */ - -- -- - /** -- * @file mali_uk.h - * Types and definitions that are common across OSs for both the user - * and kernel side of the User-Kernel interface. - */ - --#ifndef _UK_H_ --#define _UK_H_ -+#ifndef _UAPI_UK_H_ -+#define _UAPI_UK_H_ - - #ifdef __cplusplus - extern "C" { --#endif /* __cplusplus */ -- --/** -- * @addtogroup base_api -- * @{ -- */ -+#endif /* __cplusplus */ - - /** -- * @defgroup uk_api User-Kernel Interface API -+ * DOC: uk_api User-Kernel Interface API - * - * The User-Kernel Interface abstracts the communication mechanism between the user and kernel-side code of device - * drivers developed as part of the Midgard DDK. Currently that includes the Base driver. -@@ -51,12 +42,16 @@ extern "C" { - * - * This API is internal to the Midgard DDK and is not exposed to any applications. - * -- * @{ - */ - - /** -- * These are identifiers for kernel-side drivers implementing a UK interface, aka UKK clients. The -- * UK module maps this to an OS specific device name, e.g. "gpu_base" -> "GPU0:". Specify this -+ * enum uk_client_id - These are identifiers for kernel-side drivers -+ * implementing a UK interface, aka UKK clients. -+ * @UK_CLIENT_MALI_T600_BASE: Value used to identify the Base driver UK client. -+ * @UK_CLIENT_COUNT: The number of uk clients supported. This must be -+ * the last member of the enum -+ * -+ * The UK module maps this to an OS specific device name, e.g. "gpu_base" -> "GPU0:". Specify this - * identifier to select a UKK client to the uku_open() function. - * - * When a new UKK client driver is created a new identifier needs to be added to the uk_client_id -@@ -65,20 +60,11 @@ extern "C" { - * - */ - enum uk_client_id { -- /** -- * Value used to identify the Base driver UK client. -- */ - UK_CLIENT_MALI_T600_BASE, -- -- /** The number of uk clients supported. This must be the last member of the enum */ - UK_CLIENT_COUNT - }; - --/** @} end group uk_api */ -- --/** @} *//* end group base_api */ -- - #ifdef __cplusplus - } --#endif /* __cplusplus */ --#endif /* _UK_H_ */ -+#endif /* __cplusplus */ -+#endif /* _UAPI_UK_H_ */ --- -2.29.0 - diff --git a/bifrost/Makefile b/bifrost/Makefile index 8a7fdf4..0b9a600 100644 --- a/bifrost/Makefile +++ b/bifrost/Makefile @@ -26,7 +26,9 @@ EXTRA_INCLUDE := -I$(KERNEL_SRC)/$(M)/../bifrost/$(GPU_DRV_VERSION)/kernel/drive -I$(KERNEL_SRC)/$(M)/../dvalin/kernel/include KBUILD_CFLAGS_MODULE += $(GKI_EXT_MODULE_PREDEFINE) - +$(warning "CC:"$(CC) ) +$(warning "HOSTCC:"$(HOSTCC) ) +$(warning "CROSS_COMPILE:"$(CROSS_COMPILE) ) modules: $(MAKE) -C $(KERNEL_SRC) M=$(M)/$(GPU_DRV_VERSION)/kernel/drivers/gpu/arm/midgard \ EXTRA_CFLAGS="-DCONFIG_MALI_PLATFORM_DEVICETREE -DCONFIG_MALI_MIDGARD_DVFS -DCONFIG_MALI_BACKEND=gpu " \ diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h index 146695c..63abc8c 100644 --- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h +++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_defs.h @@ -1076,9 +1076,7 @@ struct kbase_device { atomic_t ctx_num; -#if IS_ENABLED(CONFIG_DEBUG_FS) struct kbase_io_history io_history; -#endif /* CONFIG_DEBUG_FS */ struct kbase_hwaccess_data hwaccess; diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c index 1e807d7..3a7595e 100644 --- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c +++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.c @@ -22,10 +22,6 @@ #include "mali_kbase.h" #include "mali_kbase_regs_history_debugfs.h" -#if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_NO_MALI) - -#include <linux/debugfs.h> - /** * kbase_io_history_resize - resize the register access history buffer. * @@ -142,6 +138,10 @@ void kbase_io_history_dump(struct kbase_device *kbdev) spin_unlock_irqrestore(&h->lock, flags); } +#if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_NO_MALI) + +#include <linux/debugfs.h> + static int regs_history_size_get(void *data, u64 *val) { struct kbase_io_history *const h = data; diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h index 3b181d3..0283f94 100644 --- a/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h +++ b/dvalin/kernel/drivers/gpu/arm/midgard/mali_kbase_regs_history_debugfs.h @@ -36,8 +36,6 @@ struct kbase_device; -#if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_NO_MALI) - /** * kbase_io_history_init - initialize data struct for register access history * @@ -62,6 +60,7 @@ void kbase_io_history_term(struct kbase_io_history *h); */ void kbase_io_history_dump(struct kbase_device *kbdev); +#if defined(CONFIG_DEBUG_FS) && !IS_ENABLED(CONFIG_MALI_NO_MALI) /** * kbasep_regs_history_debugfs_init - add debugfs entries for register history * |