1 files changed, 128 insertions, 76 deletions

diff --git a/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.c b/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.c
index a56b689..bbf2e4e 100644
--- a/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.c
+++ b/mali_kbase/csf/ipa_control/mali_kbase_csf_ipa_control.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2020-2023 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -20,6 +20,7 @@
  */
 
 #include <mali_kbase.h>
+#include <mali_kbase_config_defaults.h>
 #include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"
 #include "mali_kbase_csf_ipa_control.h"
 
@@ -27,8 +28,6 @@
  * Status flags from the STATUS register of the IPA Control interface.
  */
 #define STATUS_COMMAND_ACTIVE ((u32)1 << 0)
-#define STATUS_TIMER_ACTIVE ((u32)1 << 1)
-#define STATUS_AUTO_ACTIVE ((u32)1 << 2)
 #define STATUS_PROTECTED_MODE ((u32)1 << 8)
 #define STATUS_RESET ((u32)1 << 9)
 #define STATUS_TIMER_ENABLED ((u32)1 << 31)
@@ -36,27 +35,15 @@
 /*
  * Commands for the COMMAND register of the IPA Control interface.
  */
-#define COMMAND_NOP ((u32)0)
 #define COMMAND_APPLY ((u32)1)
-#define COMMAND_CLEAR ((u32)2)
 #define COMMAND_SAMPLE ((u32)3)
 #define COMMAND_PROTECTED_ACK ((u32)4)
 #define COMMAND_RESET_ACK ((u32)5)
 
 /*
- * Default value for the TIMER register of the IPA Control interface,
- * expressed in milliseconds.
- *
- * The chosen value is a trade off between two requirements: the IPA Control
- * interface should sample counters with a resolution in the order of
- * milliseconds, while keeping GPU overhead as limited as possible.
- */
-#define TIMER_DEFAULT_VALUE_MS ((u32)10) /* 10 milliseconds */
-
-/*
  * Number of timer events per second.
  */
-#define TIMER_EVENTS_PER_SECOND ((u32)1000 / TIMER_DEFAULT_VALUE_MS)
+#define TIMER_EVENTS_PER_SECOND ((u32)1000 / IPA_CONTROL_TIMER_DEFAULT_VALUE_MS)
 
 /*
  * Maximum number of loops polling the GPU before we assume the GPU has hung.
@@ -77,12 +64,19 @@
  * struct kbase_ipa_control_listener_data - Data for the GPU clock frequency
  *                                          listener
  *
- * @listener: GPU clock frequency listener.
- * @kbdev:    Pointer to kbase device.
+ * @listener:     GPU clock frequency listener.
+ * @kbdev:        Pointer to kbase device.
+ * @clk_chg_wq:   Dedicated workqueue to process the work item corresponding to
+ *                a clock rate notification.
+ * @clk_chg_work: Work item to process the clock rate change
+ * @rate:         The latest notified rate change, in unit of Hz
  */
 struct kbase_ipa_control_listener_data {
 	struct kbase_clk_rate_listener listener;
 	struct kbase_device *kbdev;
+	struct workqueue_struct *clk_chg_wq;
+	struct work_struct clk_chg_work;
+	atomic_t rate;
 };
 
 static u32 timer_value(u32 gpu_rate)
@@ -284,58 +278,61 @@ kbase_ipa_control_rate_change_notify(struct kbase_clk_rate_listener *listener,
 				     u32 clk_index, u32 clk_rate_hz)
 {
 	if ((clk_index == KBASE_CLOCK_DOMAIN_TOP) && (clk_rate_hz != 0)) {
-		size_t i;
-		unsigned long flags;
 		struct kbase_ipa_control_listener_data *listener_data =
-			container_of(listener,
-				     struct kbase_ipa_control_listener_data,
-				     listener);
-		struct kbase_device *kbdev = listener_data->kbdev;
-		struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
+			container_of(listener, struct kbase_ipa_control_listener_data, listener);
 
-		spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
+		/* Save the rate and delegate the job to a work item */
+		atomic_set(&listener_data->rate, clk_rate_hz);
+		queue_work(listener_data->clk_chg_wq, &listener_data->clk_chg_work);
+	}
+}
 
-		if (!kbdev->pm.backend.gpu_ready) {
-			dev_err(kbdev->dev,
-				"%s: GPU frequency cannot change while GPU is off",
-				__func__);
-			spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
-			return;
-		}
+static void kbase_ipa_ctrl_rate_change_worker(struct work_struct *data)
+{
+	struct kbase_ipa_control_listener_data *listener_data =
+		container_of(data, struct kbase_ipa_control_listener_data, clk_chg_work);
+	struct kbase_device *kbdev = listener_data->kbdev;
+	struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
+	unsigned long flags;
+	u32 rate;
+	size_t i;
 
-		/* Interrupts are already disabled and interrupt state is also saved */
-		spin_lock(&ipa_ctrl->lock);
+	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 
-		for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) {
-			struct kbase_ipa_control_session *session = &ipa_ctrl->sessions[i];
+	if (!kbdev->pm.backend.gpu_ready) {
+		dev_err(kbdev->dev, "%s: GPU frequency cannot change while GPU is off", __func__);
+		spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
+		return;
+	}
 
-			if (session->active) {
-				size_t j;
+	spin_lock(&ipa_ctrl->lock);
+	/* Picking up the latest notified rate */
+	rate = (u32)atomic_read(&listener_data->rate);
 
-				for (j = 0; j < session->num_prfcnts; j++) {
-					struct kbase_ipa_control_prfcnt *prfcnt =
-						&session->prfcnts[j];
+	for (i = 0; i < KBASE_IPA_CONTROL_MAX_SESSIONS; i++) {
+		struct kbase_ipa_control_session *session = &ipa_ctrl->sessions[i];
 
-					if (prfcnt->gpu_norm)
-						calc_prfcnt_delta(kbdev, prfcnt, true);
-				}
-			}
-		}
+		if (session->active) {
+			size_t j;
 
-		ipa_ctrl->cur_gpu_rate = clk_rate_hz;
+			for (j = 0; j < session->num_prfcnts; j++) {
+				struct kbase_ipa_control_prfcnt *prfcnt = &session->prfcnts[j];
 
-		/* Update the timer for automatic sampling if active sessions
-		 * are present. Counters have already been manually sampled.
-		 */
-		if (ipa_ctrl->num_active_sessions > 0) {
-			kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER),
-					timer_value(ipa_ctrl->cur_gpu_rate));
+				if (prfcnt->gpu_norm)
+					calc_prfcnt_delta(kbdev, prfcnt, true);
+			}
 		}
+	}
 
-		spin_unlock(&ipa_ctrl->lock);
+	ipa_ctrl->cur_gpu_rate = rate;
+	/* Update the timer for automatic sampling if active sessions
+	 * are present. Counters have already been manually sampled.
+	 */
+	if (ipa_ctrl->num_active_sessions > 0)
+		kbase_reg_write(kbdev, IPA_CONTROL_REG(TIMER), timer_value(rate));
 
-		spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
-	}
+	spin_unlock(&ipa_ctrl->lock);
+	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 }
 
 void kbase_ipa_control_init(struct kbase_device *kbdev)
@@ -344,6 +341,7 @@ void kbase_ipa_control_init(struct kbase_device *kbdev)
 	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
 	struct kbase_ipa_control_listener_data *listener_data;
 	size_t i, j;
+	unsigned long flags;
 
 	for (i = 0; i < KBASE_IPA_CORE_TYPE_NUM; i++) {
 		for (j = 0; j < KBASE_IPA_CONTROL_NUM_BLOCK_COUNTERS; j++) {
@@ -362,20 +360,35 @@ void kbase_ipa_control_init(struct kbase_device *kbdev)
 	listener_data = kmalloc(sizeof(struct kbase_ipa_control_listener_data),
 				GFP_KERNEL);
 	if (listener_data) {
-		listener_data->listener.notify =
-			kbase_ipa_control_rate_change_notify;
-		listener_data->kbdev = kbdev;
-		ipa_ctrl->rtm_listener_data = listener_data;
-	}
+		listener_data->clk_chg_wq =
+			alloc_workqueue("ipa_ctrl_wq", WQ_HIGHPRI | WQ_UNBOUND, 1);
+		if (listener_data->clk_chg_wq) {
+			INIT_WORK(&listener_data->clk_chg_work, kbase_ipa_ctrl_rate_change_worker);
+			listener_data->listener.notify = kbase_ipa_control_rate_change_notify;
+			listener_data->kbdev = kbdev;
+			ipa_ctrl->rtm_listener_data = listener_data;
+			/* Initialise to 0, which is out of normal notified rates */
+			atomic_set(&listener_data->rate, 0);
+		} else {
+			dev_warn(kbdev->dev,
+				 "%s: failed to allocate workqueue, clock rate update disabled",
+				 __func__);
+			kfree(listener_data);
+			listener_data = NULL;
+		}
+	} else
+		dev_warn(kbdev->dev,
+			 "%s: failed to allocate memory, IPA control clock rate update disabled",
+			 __func__);
 
-	spin_lock(&clk_rtm->lock);
+	spin_lock_irqsave(&clk_rtm->lock, flags);
 	if (clk_rtm->clks[KBASE_CLOCK_DOMAIN_TOP])
 		ipa_ctrl->cur_gpu_rate =
 			clk_rtm->clks[KBASE_CLOCK_DOMAIN_TOP]->clock_val;
 	if (listener_data)
 		kbase_clk_rate_trace_manager_subscribe_no_lock(
 			clk_rtm, &listener_data->listener);
-	spin_unlock(&clk_rtm->lock);
+	spin_unlock_irqrestore(&clk_rtm->lock, flags);
 }
 KBASE_EXPORT_TEST_API(kbase_ipa_control_init);
 
@@ -389,8 +402,10 @@ void kbase_ipa_control_term(struct kbase_device *kbdev)
 
 	WARN_ON(ipa_ctrl->num_active_sessions);
 
-	if (listener_data)
+	if (listener_data) {
 		kbase_clk_rate_trace_manager_unsubscribe(clk_rtm, &listener_data->listener);
+		destroy_workqueue(listener_data->clk_chg_wq);
+	}
 	kfree(ipa_ctrl->rtm_listener_data);
 
 	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
@@ -602,9 +617,10 @@ int kbase_ipa_control_register(
 	 */
 	for (session_idx = 0; session_idx < KBASE_IPA_CONTROL_MAX_SESSIONS;
 	     session_idx++) {
-		session = &ipa_ctrl->sessions[session_idx];
-		if (!session->active)
+		if (!ipa_ctrl->sessions[session_idx].active) {
+			session = &ipa_ctrl->sessions[session_idx];
 			break;
+		}
 	}
 
 	if (!session) {
@@ -659,7 +675,7 @@ int kbase_ipa_control_register(
 		/* Reports to this client for GPU time spent in protected mode
 		 * should begin from the point of registration.
 		 */
-		session->last_query_time = ktime_get_ns();
+		session->last_query_time = ktime_get_raw_ns();
 
 		/* Initially, no time has been spent in protected mode */
 		session->protm_time = 0;
@@ -829,7 +845,7 @@ int kbase_ipa_control_query(struct kbase_device *kbdev, const void *client,
 	}
 
 	if (protected_time) {
-		u64 time_now = ktime_get_ns();
+		u64 time_now = ktime_get_raw_ns();
 
 		/* This is the amount of protected-mode time spent prior to
 		 * the current protm period.
@@ -973,16 +989,53 @@ void kbase_ipa_control_handle_gpu_reset_post(struct kbase_device *kbdev)
 }
 KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_reset_post);
 
+#ifdef KBASE_PM_RUNTIME
+void kbase_ipa_control_handle_gpu_sleep_enter(struct kbase_device *kbdev)
+{
+	lockdep_assert_held(&kbdev->hwaccess_lock);
+
+	if (kbdev->pm.backend.mcu_state == KBASE_MCU_IN_SLEEP) {
+		/* GPU Sleep is treated as a power down */
+		kbase_ipa_control_handle_gpu_power_off(kbdev);
+
+		/* SELECT_CSHW register needs to be cleared to prevent any
+		 * IPA control message to be sent to the top level GPU HWCNT.
+		 */
+		kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_LO), 0);
+		kbase_reg_write(kbdev, IPA_CONTROL_REG(SELECT_CSHW_HI), 0);
+
+		/* No need to issue the APPLY command here */
+	}
+}
+KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_sleep_enter);
+
+void kbase_ipa_control_handle_gpu_sleep_exit(struct kbase_device *kbdev)
+{
+	lockdep_assert_held(&kbdev->hwaccess_lock);
+
+	if (kbdev->pm.backend.mcu_state == KBASE_MCU_IN_SLEEP) {
+		/* To keep things simple, currently exit from
+		 * GPU Sleep is treated as a power on event where
+		 * all 4 SELECT registers are reconfigured.
+		 * On exit from sleep, reconfiguration is needed
+		 * only for the SELECT_CSHW register.
+		 */
+		kbase_ipa_control_handle_gpu_power_on(kbdev);
+	}
+}
+KBASE_EXPORT_TEST_API(kbase_ipa_control_handle_gpu_sleep_exit);
+#endif
+
 #if MALI_UNIT_TEST
 void kbase_ipa_control_rate_change_notify_test(struct kbase_device *kbdev,
 					       u32 clk_index, u32 clk_rate_hz)
 {
 	struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
-	struct kbase_ipa_control_listener_data *listener_data =
-		ipa_ctrl->rtm_listener_data;
+	struct kbase_ipa_control_listener_data *listener_data = ipa_ctrl->rtm_listener_data;
 
-	kbase_ipa_control_rate_change_notify(&listener_data->listener,
-					     clk_index, clk_rate_hz);
+	kbase_ipa_control_rate_change_notify(&listener_data->listener, clk_index, clk_rate_hz);
+	/* Ensure the callback has taken effect before returning back to the test caller */
+	flush_work(&listener_data->clk_chg_work);
 }
 KBASE_EXPORT_TEST_API(kbase_ipa_control_rate_change_notify_test);
 #endif
@@ -992,14 +1045,14 @@ void kbase_ipa_control_protm_entered(struct kbase_device *kbdev)
 	struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
 
 	lockdep_assert_held(&kbdev->hwaccess_lock);
-	ipa_ctrl->protm_start = ktime_get_ns();
+	ipa_ctrl->protm_start = ktime_get_raw_ns();
 }
 
 void kbase_ipa_control_protm_exited(struct kbase_device *kbdev)
 {
 	struct kbase_ipa_control *ipa_ctrl = &kbdev->csf.ipa_control;
 	size_t i;
-	u64 time_now = ktime_get_ns();
+	u64 time_now = ktime_get_raw_ns();
 	u32 status;
 
 	lockdep_assert_held(&kbdev->hwaccess_lock);
@@ -1035,4 +1088,3 @@ void kbase_ipa_control_protm_exited(struct kbase_device *kbdev)
 		}
 	}
 }
-