path: root/mali_kbase/backend/gpu/mali_kbase_clk_rate_trace_mgr.c

// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
 *
 * (C) COPYRIGHT 2020-2021 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU license.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
 *
 */

/*
 * Implementation of the GPU clock rate trace manager.
 */

#include <mali_kbase.h>
#include <mali_kbase_config_defaults.h>
#include <linux/clk.h>
#include <linux/pm_opp.h>
#include <asm/div64.h>
#include "backend/gpu/mali_kbase_clk_rate_trace_mgr.h"

#ifdef CONFIG_TRACE_POWER_GPU_FREQUENCY
#include <trace/events/power_gpu_frequency.h>
#else
#include "mali_power_gpu_frequency_trace.h"
#endif

#ifndef CLK_RATE_TRACE_OPS
#define CLK_RATE_TRACE_OPS (NULL)
#endif

/**
 * get_clk_rate_trace_callbacks() - Returns pointer to clk trace ops.
 * @kbdev: Pointer to kbase device, used to check if arbitration is enabled
 *         when compiled with arbiter support.
 * Return: Pointer to clk trace ops if supported or NULL.
 */
static struct kbase_clk_rate_trace_op_conf *
get_clk_rate_trace_callbacks(__maybe_unused struct kbase_device *kbdev)
{
	/* base case */
	struct kbase_clk_rate_trace_op_conf *callbacks =
		(struct kbase_clk_rate_trace_op_conf *)CLK_RATE_TRACE_OPS;
#if defined(CONFIG_MALI_ARBITER_SUPPORT) && defined(CONFIG_OF)
	const void *arbiter_if_node;

	if (WARN_ON(!kbdev) || WARN_ON(!kbdev->dev))
		return callbacks;

	arbiter_if_node =
		of_get_property(kbdev->dev->of_node, "arbiter_if", NULL);
	/* Arbitration enabled, override the callback pointer.*/
	if (arbiter_if_node)
		callbacks = &arb_clk_rate_trace_ops;
	else
		dev_dbg(kbdev->dev,
			"Arbitration supported but disabled by platform. Leaving clk rate callbacks as default.\n");

#endif

	return callbacks;
}

int kbase_lowest_gpu_freq_init(struct kbase_device *kbdev)
{
	/* Uses default reference frequency defined in below macro */
	u64 lowest_freq_khz = DEFAULT_REF_TIMEOUT_FREQ_KHZ;

	/* Only check lowest frequency in cases when OPPs are used and
	 * present in the device tree.
	 */
#ifdef CONFIG_PM_OPP
	struct dev_pm_opp *opp_ptr;
	unsigned long found_freq = 0;

	/* find lowest frequency OPP */
	opp_ptr = dev_pm_opp_find_freq_ceil(kbdev->dev, &found_freq);
	if (IS_ERR(opp_ptr)) {
		dev_err(kbdev->dev,
			"No OPPs found in device tree! Scaling timeouts using %llu kHz",
			(unsigned long long)lowest_freq_khz);
	} else {
#if KERNEL_VERSION(4, 11, 0) <= LINUX_VERSION_CODE
		dev_pm_opp_put(opp_ptr); /* decrease OPP refcount */
#endif
		/* convert found frequency to KHz */
		found_freq /= 1000;

		/* If lowest frequency in OPP table is still higher
		 * than the reference, then keep the reference frequency
		 * as the one to use for scaling .
		 */
		if (found_freq < lowest_freq_khz)
			lowest_freq_khz = found_freq;
	}
#else
	dev_err(kbdev->dev,
		"No operating-points-v2 node or operating-points property in DT");
#endif

	kbdev->lowest_gpu_freq_khz = lowest_freq_khz;
	dev_dbg(kbdev->dev, "Lowest frequency identified is %llu kHz",
		kbdev->lowest_gpu_freq_khz);
	return 0;
}

static int gpu_clk_rate_change_notifier(struct notifier_block *nb,
			unsigned long event, void *data)
{
	struct kbase_gpu_clk_notifier_data *ndata = data;
	struct kbase_clk_data *clk_data =
		container_of(nb, struct kbase_clk_data, clk_rate_change_nb);
	struct kbase_clk_rate_trace_manager *clk_rtm = clk_data->clk_rtm;
	unsigned long flags;

	if (WARN_ON_ONCE(clk_data->gpu_clk_handle != ndata->gpu_clk_handle))
		return NOTIFY_BAD;

	spin_lock_irqsave(&clk_rtm->lock, flags);
	if (event == POST_RATE_CHANGE) {
		if (!clk_rtm->gpu_idle &&
		    (clk_data->clock_val != ndata->new_rate)) {
			kbase_clk_rate_trace_manager_notify_all(
				clk_rtm, clk_data->index, ndata->new_rate);
		}

		clk_data->clock_val = ndata->new_rate;
	}
	spin_unlock_irqrestore(&clk_rtm->lock, flags);

	return NOTIFY_DONE;
}

static int gpu_clk_data_init(struct kbase_device *kbdev,
		void *gpu_clk_handle, unsigned int index)
{
	struct kbase_clk_rate_trace_op_conf *callbacks;
	struct kbase_clk_data *clk_data;
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	int ret = 0;

	callbacks = get_clk_rate_trace_callbacks(kbdev);

	if (WARN_ON(!callbacks) ||
	    WARN_ON(!gpu_clk_handle) ||
	    WARN_ON(index >= BASE_MAX_NR_CLOCKS_REGULATORS))
		return -EINVAL;

	clk_data = kzalloc(sizeof(*clk_data), GFP_KERNEL);
	if (!clk_data) {
		dev_err(kbdev->dev, "Failed to allocate data for clock enumerated at index %u", index);
		return -ENOMEM;
	}

	clk_data->index = (u8)index;
	clk_data->gpu_clk_handle = gpu_clk_handle;
	/* Store the initial value of clock */
	clk_data->clock_val =
		callbacks->get_gpu_clk_rate(kbdev, gpu_clk_handle);

	{
		/* At the initialization time, GPU is powered off. */
		unsigned long flags;

		spin_lock_irqsave(&clk_rtm->lock, flags);
		kbase_clk_rate_trace_manager_notify_all(
			clk_rtm, clk_data->index, 0);
		spin_unlock_irqrestore(&clk_rtm->lock, flags);
	}

	clk_data->clk_rtm = clk_rtm;
	clk_rtm->clks[index] = clk_data;

	clk_data->clk_rate_change_nb.notifier_call =
			gpu_clk_rate_change_notifier;

	if (callbacks->gpu_clk_notifier_register)
		ret = callbacks->gpu_clk_notifier_register(kbdev,
				gpu_clk_handle, &clk_data->clk_rate_change_nb);
	if (ret) {
		dev_err(kbdev->dev, "Failed to register notifier for clock enumerated at index %u", index);
		kfree(clk_data);
	}

	return ret;
}

int kbase_clk_rate_trace_manager_init(struct kbase_device *kbdev)
{
	struct kbase_clk_rate_trace_op_conf *callbacks;
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	unsigned int i;
	int ret = 0;

	callbacks = get_clk_rate_trace_callbacks(kbdev);

	spin_lock_init(&clk_rtm->lock);
	INIT_LIST_HEAD(&clk_rtm->listeners);

	/* Return early if no callbacks provided for clock rate tracing */
	if (!callbacks) {
		WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
		return 0;
	}

	clk_rtm->gpu_idle = true;

	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
		void *gpu_clk_handle =
			callbacks->enumerate_gpu_clk(kbdev, i);

		if (!gpu_clk_handle)
			break;

		ret = gpu_clk_data_init(kbdev, gpu_clk_handle, i);
		if (ret)
			goto error;
	}

	/* Activate clock rate trace manager if at least one GPU clock was
	 * enumerated.
	 */
	if (i) {
		WRITE_ONCE(clk_rtm->clk_rate_trace_ops, callbacks);
	} else {
		dev_info(kbdev->dev, "No clock(s) available for rate tracing");
		WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
	}

	return 0;

error:
	while (i--) {
		clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister(
				kbdev, clk_rtm->clks[i]->gpu_clk_handle,
				&clk_rtm->clks[i]->clk_rate_change_nb);
		kfree(clk_rtm->clks[i]);
	}

	return ret;
}

void kbase_clk_rate_trace_manager_term(struct kbase_device *kbdev)
{
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	unsigned int i;

	WARN_ON(!list_empty(&clk_rtm->listeners));

	if (!clk_rtm->clk_rate_trace_ops)
		return;

	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
		if (!clk_rtm->clks[i])
			break;

		if (clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister)
			clk_rtm->clk_rate_trace_ops->gpu_clk_notifier_unregister
			(kbdev, clk_rtm->clks[i]->gpu_clk_handle,
			&clk_rtm->clks[i]->clk_rate_change_nb);
		kfree(clk_rtm->clks[i]);
	}

	WRITE_ONCE(clk_rtm->clk_rate_trace_ops, NULL);
}

void kbase_clk_rate_trace_manager_gpu_active(struct kbase_device *kbdev)
{
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	unsigned int i;
	unsigned long flags;

	if (!clk_rtm->clk_rate_trace_ops)
		return;

	spin_lock_irqsave(&clk_rtm->lock, flags);

	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
		struct kbase_clk_data *clk_data = clk_rtm->clks[i];

		if (!clk_data)
			break;

		if (unlikely(!clk_data->clock_val))
			continue;

		kbase_clk_rate_trace_manager_notify_all(
			clk_rtm, clk_data->index, clk_data->clock_val);
	}

	clk_rtm->gpu_idle = false;
	spin_unlock_irqrestore(&clk_rtm->lock, flags);
}

void kbase_clk_rate_trace_manager_gpu_idle(struct kbase_device *kbdev)
{
	struct kbase_clk_rate_trace_manager *clk_rtm = &kbdev->pm.clk_rtm;
	unsigned int i;
	unsigned long flags;

	if (!clk_rtm->clk_rate_trace_ops)
		return;

	spin_lock_irqsave(&clk_rtm->lock, flags);

	for (i = 0; i < BASE_MAX_NR_CLOCKS_REGULATORS; i++) {
		struct kbase_clk_data *clk_data = clk_rtm->clks[i];

		if (!clk_data)
			break;

		if (unlikely(!clk_data->clock_val))
			continue;

		kbase_clk_rate_trace_manager_notify_all(
			clk_rtm, clk_data->index, 0);
	}

	clk_rtm->gpu_idle = true;
	spin_unlock_irqrestore(&clk_rtm->lock, flags);
}

void kbase_clk_rate_trace_manager_notify_all(
	struct kbase_clk_rate_trace_manager *clk_rtm,
	u32 clk_index,
	unsigned long new_rate)
{
	struct kbase_clk_rate_listener *pos;
	struct kbase_device *kbdev;

	lockdep_assert_held(&clk_rtm->lock);

	kbdev = container_of(clk_rtm, struct kbase_device, pm.clk_rtm);

	dev_dbg(kbdev->dev, "%s - GPU clock %u rate changed to %lu, pid: %d",
		__func__, clk_index, new_rate, current->pid);

	/* Raise standard `power/gpu_frequency` ftrace event */
	{
		unsigned long new_rate_khz = new_rate;

#if BITS_PER_LONG == 64
		do_div(new_rate_khz, 1000);
#elif BITS_PER_LONG == 32
		new_rate_khz /= 1000;
#else
#error "unsigned long division is not supported for this architecture"
#endif

		trace_gpu_frequency(new_rate_khz, clk_index);
	}

	/* Notify the listeners. */
	list_for_each_entry(pos, &clk_rtm->listeners, node) {
		pos->notify(pos, clk_index, new_rate);
	}
}
KBASE_EXPORT_TEST_API(kbase_clk_rate_trace_manager_notify_all);