Merge 8fa442b60b2c40b57e44d43663f315dc76371ade on remote branch

Change-Id: I1a7803126bbf0104c6f4d1f5b6c8a3acb124642f

24 files changed, 340 insertions, 158 deletions

diff --git a/adreno-gpulist.h b/adreno-gpulist.h
index 96b8e70..8897f11 100644
--- a/adreno-gpulist.h
+++ b/adreno-gpulist.h
@@ -2139,7 +2139,7 @@ static const struct adreno_gen7_core adreno_gpu_core_gen7_2_0 = {
 				ADRENO_LPAC | ADRENO_BCL | ADRENO_L3_VOTE |
 				ADRENO_PREEMPTION | ADRENO_DMS,
 		.gpudev = &adreno_gen7_hwsched_gpudev.base,
-		.perfcounters = &adreno_gen7_2_0_perfcounters,
+		.perfcounters = &adreno_gen7_hwsched_perfcounters,
 		.uche_gmem_alignment = SZ_16M,
 		.gmem_size = 3 * SZ_1M,
 		.bus_width = 32,
@@ -2175,7 +2175,7 @@ static const struct adreno_gen7_core adreno_gpu_core_gen7_2_1 = {
 				ADRENO_BCL | ADRENO_L3_VOTE | ADRENO_ACD |
 				ADRENO_PREEMPTION | ADRENO_DMS,
 		.gpudev = &adreno_gen7_hwsched_gpudev.base,
-		.perfcounters = &adreno_gen7_2_0_perfcounters,
+		.perfcounters = &adreno_gen7_hwsched_perfcounters,
 		.uche_gmem_alignment = SZ_16M,
 		.gmem_size = 3 * SZ_1M,
 		.bus_width = 32,
@@ -2300,7 +2300,7 @@ static const struct adreno_gen7_core adreno_gpu_core_gen7_9_0 = {
 			ADRENO_CONTENT_PROTECTION | ADRENO_LPAC | ADRENO_IFPC |
 			ADRENO_L3_VOTE | ADRENO_BCL | ADRENO_ACD | ADRENO_DMS | ADRENO_HW_FENCE,
 		.gpudev = &adreno_gen7_9_0_hwsched_gpudev.base,
-		.perfcounters = &adreno_gen7_2_0_perfcounters,
+		.perfcounters = &adreno_gen7_hwsched_perfcounters,
 		.uche_gmem_alignment = SZ_16M,
 		.gmem_size = 3 * SZ_1M,
 		.bus_width = 32,
@@ -2320,6 +2320,7 @@ static const struct adreno_gen7_core adreno_gpu_core_gen7_9_0 = {
 	.gmu_hub_clk_freq = 200000000,
 	.gen7_snapshot_block_list = &gen7_9_0_snapshot_block_list,
 	.bcl_data = 1,
+	.acv_perfmode_vote = BIT(2),
 };
 
 static const struct kgsl_regmap_list a663_hwcg_regs[] = {
diff --git a/adreno.c b/adreno.c
index 86717a9..d132cbe 100644
--- a/adreno.c
+++ b/adreno.c
@@ -1195,10 +1195,12 @@ static void adreno_setup_device(struct adreno_device *adreno_dev)
 				adreno_dev->gpucore->uche_gmem_alignment);
 }
 
-static const struct of_device_id adreno_gmu_match[] = {
+static const struct of_device_id adreno_component_match[] = {
 	{ .compatible = "qcom,gen7-gmu" },
 	{ .compatible = "qcom,gpu-gmu" },
 	{ .compatible = "qcom,gpu-rgmu" },
+	{ .compatible = "qcom,kgsl-smmu-v2" },
+	{ .compatible = "qcom,smmu-kgsl-cb" },
 	{},
 };
 
@@ -1242,18 +1244,6 @@ int adreno_device_probe(struct platform_device *pdev,
 		goto err;
 
 	/*
-	 * Bind the GMU components (if applicable) before doing the KGSL
-	 * platform probe
-	 */
-	if (of_find_matching_node(dev->of_node, adreno_gmu_match)) {
-		status = component_bind_all(dev, NULL);
-		if (status) {
-			kgsl_bus_close(device);
-			return status;
-		}
-	}
-
-	/*
 	 * The SMMU APIs use unsigned long for virtual addresses which means
 	 * that we cannot use 64 bit virtual addresses on a 32 bit kernel even
 	 * though the hardware and the rest of the KGSL driver supports it.
@@ -1298,6 +1288,11 @@ int adreno_device_probe(struct platform_device *pdev,
 	if (!IS_ERR_OR_NULL(adreno_dev->gpuhtw_llc_slice))
 		kgsl_mmu_set_feature(device, KGSL_MMU_LLCC_ENABLE);
 
+	 /* Bind the components before doing the KGSL platform probe. */
+	status = component_bind_all(dev, NULL);
+	if (status)
+		goto err;
+
 	status = kgsl_request_irq(pdev, "kgsl_3d0_irq", adreno_irq_handler, device);
 	if (status < 0)
 		goto err;
@@ -1382,8 +1377,7 @@ int adreno_device_probe(struct platform_device *pdev,
 err:
 	device->pdev = NULL;
 
-	if (of_find_matching_node(dev->of_node, adreno_gmu_match))
-		component_unbind_all(dev, NULL);
+	component_unbind_all(dev, NULL);
 
 	kgsl_bus_close(device);
 
@@ -1457,8 +1451,7 @@ static void adreno_unbind(struct device *dev)
 
 	kgsl_device_platform_remove(device);
 
-	if (of_find_matching_node(dev->of_node, adreno_gmu_match))
-		component_unbind_all(dev, NULL);
+	component_unbind_all(dev, NULL);
 
 	kgsl_bus_close(device);
 
@@ -3394,43 +3387,40 @@ static void _release_of(struct device *dev, void *data)
 	of_node_put(data);
 }
 
-static void adreno_add_gmu_components(struct device *dev,
+static void adreno_add_components(struct device *dev,
 		struct component_match **match)
 {
 	struct device_node *node;
 
-	node = of_find_matching_node(NULL, adreno_gmu_match);
-	if (!node)
-		return;
+	/*
+	 * Add kgsl-smmu, context banks and gmu as components, if supported.
+	 * Master bind (adreno_bind) will be called only once all added
+	 * components are available.
+	 */
+	for_each_matching_node(node, adreno_component_match) {
+		if (!of_device_is_available(node))
+			continue;
 
-	if (!of_device_is_available(node)) {
-		of_node_put(node);
-		return;
+		component_match_add_release(dev, match, _release_of, _compare_of, node);
 	}
-
-	component_match_add_release(dev, match, _release_of,
-		_compare_of, node);
 }
 
 static int adreno_probe(struct platform_device *pdev)
 {
 	struct component_match *match = NULL;
 
-	adreno_add_gmu_components(&pdev->dev, &match);
+	adreno_add_components(&pdev->dev, &match);
 
-	if (match)
-		return component_master_add_with_match(&pdev->dev,
-				&adreno_ops, match);
-	else
-		return adreno_bind(&pdev->dev);
+	if (!match)
+		return -ENODEV;
+
+	return component_master_add_with_match(&pdev->dev,
+			&adreno_ops, match);
 }
 
 static int adreno_remove(struct platform_device *pdev)
 {
-	if (of_find_matching_node(NULL, adreno_gmu_match))
-		component_master_del(&pdev->dev, &adreno_ops);
-	else
-		adreno_unbind(&pdev->dev);
+	component_master_del(&pdev->dev, &adreno_ops);
 
 	return 0;
 }
@@ -3652,10 +3642,19 @@ static int __init kgsl_3d_init(void)
 	if (ret)
 		return ret;
 
+	ret = kgsl_mmu_init();
+	if (ret) {
+		kgsl_core_exit();
+		return ret;
+	}
+
 	gmu_core_register();
 	ret = platform_driver_register(&adreno_platform_driver);
-	if (ret)
+	if (ret) {
+		gmu_core_unregister();
+		kgsl_mmu_exit();
 		kgsl_core_exit();
+	}
 
 	return ret;
 }
@@ -3664,6 +3663,7 @@ static void __exit kgsl_3d_exit(void)
 {
 	platform_driver_unregister(&adreno_platform_driver);
 	gmu_core_unregister();
+	kgsl_mmu_exit();
 	kgsl_core_exit();
 }
 
diff --git a/adreno_a6xx.h b/adreno_a6xx.h
index 52a384d..a00f389 100644
--- a/adreno_a6xx.h
+++ b/adreno_a6xx.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2017-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef _ADRENO_A6XX_H_
@@ -352,19 +352,6 @@ void a6xx_spin_idle_debug(struct adreno_device *adreno_dev,
 	const char *str);
 
 /**
- * a6xx_counter_enable - Configure a performance counter for a countable
- * @adreno_dev -  Adreno device to configure
- * @group - Desired performance counter group
- * @counter - Desired performance counter in the group
- * @countable - Desired countable
- *
- * Physically set up a counter within a group with the desired countable
- * Return 0 on success else error code
- */
-int a6xx_counter_enable(struct adreno_device *adreno_dev,
-		const struct adreno_perfcount_group *group,
-		unsigned int counter, unsigned int countable);
-/**
  * a6xx_perfcounter_update - Update the IFPC perfcounter list
  * @adreno_dev: An Adreno GPU handle
  * @reg: Perfcounter reg struct to add/remove to the list
diff --git a/adreno_a6xx_hwsched_hfi.c b/adreno_a6xx_hwsched_hfi.c
index 829f869..85d375b 100644
--- a/adreno_a6xx_hwsched_hfi.c
+++ b/adreno_a6xx_hwsched_hfi.c
@@ -1387,17 +1387,12 @@ int a6xx_hwsched_cp_init(struct adreno_device *adreno_dev)
 
 int a6xx_hwsched_counter_inline_enable(struct adreno_device *adreno_dev,
 		const struct adreno_perfcount_group *group,
-		unsigned int counter, unsigned int countable)
+		u32 counter, u32 countable)
 {
 	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
 	struct adreno_perfcount_register *reg = &group->regs[counter];
-	u32 cmds[A6XX_PERF_COUNTER_ENABLE_DWORDS + 1];
+	u32 val, cmds[A6XX_PERF_COUNTER_ENABLE_DWORDS + 1];
 	int ret;
-	char str[64];
-
-	if (!(device->state == KGSL_STATE_ACTIVE))
-		return a6xx_counter_enable(adreno_dev, group, counter,
-			countable);
 
 	if (group->flags & ADRENO_PERFCOUNTER_GROUP_RESTORE)
 		a6xx_perfcounter_update(adreno_dev, reg, false);
@@ -1409,13 +1404,21 @@ int a6xx_hwsched_counter_inline_enable(struct adreno_device *adreno_dev,
 	cmds[2] = cp_type4_packet(reg->select, 1);
 	cmds[3] = countable;
 
-	snprintf(str, sizeof(str), "Perfcounter %s/%u/%u start via commands failed\n",
-			group->name, counter, countable);
+	ret = a6xx_hfi_send_cmd_async(adreno_dev, cmds);
+	if (ret)
+		goto err;
 
-	ret = submit_raw_cmds(adreno_dev, cmds, str);
-	if (!ret)
+	/* Wait till the register is programmed with the countable */
+	ret = kgsl_regmap_read_poll_timeout(&device->regmap, reg->select, val,
+				val == countable, 100, ADRENO_IDLE_TIMEOUT);
+	if (!ret) {
 		reg->value = 0;
+		return ret;
+	}
 
+err:
+	dev_err(device->dev, "Perfcounter %s/%u/%u start via commands failed\n",
+			group->name, counter, countable);
 	return ret;
 }
 
diff --git a/adreno_a6xx_hwsched_hfi.h b/adreno_a6xx_hwsched_hfi.h
index 6c1063e..995bb63 100644
--- a/adreno_a6xx_hwsched_hfi.h
+++ b/adreno_a6xx_hwsched_hfi.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef _ADRENO_A6XX_HWSCHED_HFI_H_
@@ -91,13 +91,12 @@ int a6xx_hwsched_cp_init(struct adreno_device *adreno_dev);
  * @counter - Desired performance counter in the group
  * @countable - Desired countable
  *
- * Function is used for adreno cores
  * Physically set up a counter within a group with the desired countable
- * Return 0 on success else error code
+ * Return 0 on success or negative error on failure.
  */
 int a6xx_hwsched_counter_inline_enable(struct adreno_device *adreno_dev,
 		const struct adreno_perfcount_group *group,
-		unsigned int counter, unsigned int countable);
+		u32 counter, u32 countable);
 
 /**
  * a6xx_hfi_send_cmd_async - Send an hfi packet
diff --git a/adreno_a6xx_perfcounter.c b/adreno_a6xx_perfcounter.c
index 7da9291..7d07cfc 100644
--- a/adreno_a6xx_perfcounter.c
+++ b/adreno_a6xx_perfcounter.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include "adreno.h"
@@ -72,9 +72,9 @@ static u64 a6xx_counter_read_norestore(struct adreno_device *adreno_dev,
 	return ((((u64) hi) << 32) | lo) + reg->value;
 }
 
-int a6xx_counter_enable(struct adreno_device *adreno_dev,
+static int a6xx_counter_enable(struct adreno_device *adreno_dev,
 		const struct adreno_perfcount_group *group,
-		unsigned int counter, unsigned int countable)
+		u32 counter, u32 countable)
 {
 	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
 	struct adreno_perfcount_register *reg = &group->regs[counter];
@@ -91,6 +91,16 @@ int a6xx_counter_enable(struct adreno_device *adreno_dev,
 	return ret;
 }
 
+static int a6xx_hwsched_counter_enable(struct adreno_device *adreno_dev,
+		const struct adreno_perfcount_group *group,
+		u32 counter, u32 countable)
+{
+	if (!(KGSL_DEVICE(adreno_dev)->state == KGSL_STATE_ACTIVE))
+		return a6xx_counter_enable(adreno_dev, group, counter, countable);
+
+	return a6xx_hwsched_counter_inline_enable(adreno_dev, group, counter, countable);
+}
+
 static int a6xx_counter_inline_enable(struct adreno_device *adreno_dev,
 		const struct adreno_perfcount_group *group,
 		unsigned int counter, unsigned int countable)
@@ -941,7 +951,7 @@ static const struct adreno_perfcount_group a6xx_hwsched_perfcounter_groups
 		a6xx_counter_enable, a6xx_counter_read, a6xx_counter_load),
 	A6XX_REGULAR_PERFCOUNTER_GROUP(PC, pc),
 	A6XX_REGULAR_PERFCOUNTER_GROUP(VFD, vfd),
-	A6XX_PERFCOUNTER_GROUP(HLSQ, hlsq, a6xx_hwsched_counter_inline_enable,
+	A6XX_PERFCOUNTER_GROUP(HLSQ, hlsq, a6xx_hwsched_counter_enable,
 			a6xx_counter_read, a6xx_counter_load),
 	A6XX_REGULAR_PERFCOUNTER_GROUP(VPC, vpc),
 	A6XX_REGULAR_PERFCOUNTER_GROUP(CCU, ccu),
@@ -950,9 +960,9 @@ static const struct adreno_perfcount_group a6xx_hwsched_perfcounter_groups
 	A6XX_REGULAR_PERFCOUNTER_GROUP(RAS, ras),
 	A6XX_REGULAR_PERFCOUNTER_GROUP(LRZ, lrz),
 	A6XX_REGULAR_PERFCOUNTER_GROUP(UCHE, uche),
-	A6XX_PERFCOUNTER_GROUP(TP, tp, a6xx_hwsched_counter_inline_enable,
+	A6XX_PERFCOUNTER_GROUP(TP, tp, a6xx_hwsched_counter_enable,
 			a6xx_counter_read, a6xx_counter_load),
-	A6XX_PERFCOUNTER_GROUP(SP, sp, a6xx_hwsched_counter_inline_enable,
+	A6XX_PERFCOUNTER_GROUP(SP, sp, a6xx_hwsched_counter_enable,
 			a6xx_counter_read, a6xx_counter_load),
 	A6XX_REGULAR_PERFCOUNTER_GROUP(RB, rb),
 	A6XX_REGULAR_PERFCOUNTER_GROUP(VSC, vsc),
diff --git a/adreno_dispatch.c b/adreno_dispatch.c
index b52d4a9..bdf5598 100644
--- a/adreno_dispatch.c
+++ b/adreno_dispatch.c
@@ -1577,13 +1577,15 @@ static void adreno_fault_header(struct kgsl_device *device,
 	const char *type = fault & ADRENO_GMU_FAULT ? "gmu" : "gpu";
 
 	if (!gx_on) {
-		if (drawobj != NULL)
+		if (drawobj != NULL) {
 			pr_fault(device, drawobj,
 				"%s fault ctx %u ctx_type %s ts %u and GX is OFF\n",
 				type, drawobj->context->id,
 				kgsl_context_type(drawctxt->type),
 				drawobj->timestamp);
-		else
+			pr_fault(device, drawobj, "cmdline: %s\n",
+					drawctxt->base.proc_priv->cmdline);
+		} else
 			dev_err(device->dev, "RB[%d] : %s fault and GX is OFF\n",
 				id, type);
 
@@ -1616,6 +1618,9 @@ static void adreno_fault_header(struct kgsl_device *device,
 			drawobj->timestamp, status,
 			rptr, wptr, ib1base, ib1sz, ib2base, ib2sz);
 
+		pr_fault(device, drawobj, "cmdline: %s\n",
+				drawctxt->base.proc_priv->cmdline);
+
 		if (rb != NULL)
 			pr_fault(device, drawobj,
 				"%s fault rb %d rb sw r/w %4.4x/%4.4x\n",
diff --git a/adreno_gen7.c b/adreno_gen7.c
index 82f29ee..4a94c1e 100644
--- a/adreno_gen7.c
+++ b/adreno_gen7.c
@@ -1441,6 +1441,11 @@ int gen7_probe_common(struct platform_device *pdev,
 	debugfs_create_file("acd_calibrate", 0644, device->d_debugfs, device, &acd_cal_fops);
 
 	gen7_coresight_init(adreno_dev);
+
+	/* Dump additional AQE 16KB data on top of default 96KB(48(BR)+48(BV)) */
+	device->snapshot_ctxt_record_size = ADRENO_FEATURE(adreno_dev, ADRENO_AQE) ?
+					112 * SZ_1K : 96 * SZ_1K;
+
 	return 0;
 }
 
diff --git a/adreno_gen7.h b/adreno_gen7.h
index 445b9b6..be69d83 100644
--- a/adreno_gen7.h
+++ b/adreno_gen7.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef _ADRENO_GEN7_H_
@@ -23,7 +23,7 @@ struct gen7_snapshot_block_list;
 extern const struct adreno_power_ops gen7_gmu_power_ops;
 extern const struct adreno_power_ops gen7_hwsched_power_ops;
 extern const struct adreno_perfcounters adreno_gen7_perfcounters;
-extern const struct adreno_perfcounters adreno_gen7_2_0_perfcounters;
+extern const struct adreno_perfcounters adreno_gen7_hwsched_perfcounters;
 
 struct gen7_gpudev {
 	struct adreno_gpudev base;
@@ -118,6 +118,8 @@ struct adreno_gen7_core {
 	u32 preempt_level;
 	/** @qos_value: GPU qos value to set for each RB. */
 	const u32 *qos_value;
+	/** @acv_perfmode_vote: ACV vote for GPU perfmode */
+	u32 acv_perfmode_vote;
 };
 
 /**
@@ -191,6 +193,9 @@ struct gen7_cp_smmu_info {
 /* Size of the CP_INIT pm4 stream in dwords */
 #define GEN7_CP_INIT_DWORDS 10
 
+/* Size of the perf counter enable pm4 stream in dwords */
+#define GEN7_PERF_COUNTER_ENABLE_DWORDS 3
+
 #define GEN7_INT_MASK \
 	((1 << GEN7_INT_AHBERROR) |			\
 	 (1 << GEN7_INT_ATBASYNCFIFOOVERFLOW) |		\
diff --git a/adreno_gen7_gmu.c b/adreno_gen7_gmu.c
index 1b93f7d..6c19dc5 100644
--- a/adreno_gen7_gmu.c
+++ b/adreno_gen7_gmu.c
@@ -2153,7 +2153,9 @@ static int gen7_gmu_bus_set(struct adreno_device *adreno_dev, int buslevel,
 	struct kgsl_pwrctrl *pwr = &device->pwrctrl;
 	int ret = 0;
 
-	kgsl_icc_set_tag(pwr, buslevel);
+	/* Target gen7_9_0 votes for perfmode through ACV. Skip icc path for same */
+	if (!adreno_is_gen7_9_0(adreno_dev))
+		kgsl_icc_set_tag(pwr, buslevel);
 
 	if (buslevel == pwr->cur_buslevel)
 		buslevel = INVALID_DCVS_IDX;
diff --git a/adreno_gen7_hwsched.c b/adreno_gen7_hwsched.c
index 666b395..1dc9e05 100644
--- a/adreno_gen7_hwsched.c
+++ b/adreno_gen7_hwsched.c
@@ -1199,7 +1199,9 @@ static int gen7_hwsched_bus_set(struct adreno_device *adreno_dev, int buslevel,
 	struct kgsl_pwrctrl *pwr = &device->pwrctrl;
 	int ret = 0;
 
-	kgsl_icc_set_tag(pwr, buslevel);
+	/* Target gen7_9_0 votes for perfmode through ACV. Skip icc path for same */
+	if (!adreno_is_gen7_9_0(adreno_dev))
+		kgsl_icc_set_tag(pwr, buslevel);
 
 	if (buslevel == pwr->cur_buslevel)
 		buslevel = INVALID_DCVS_IDX;
diff --git a/adreno_gen7_hwsched_hfi.c b/adreno_gen7_hwsched_hfi.c
index d2e8e0e..c0d4b8f 100644
--- a/adreno_gen7_hwsched_hfi.c
+++ b/adreno_gen7_hwsched_hfi.c
@@ -3157,3 +3157,41 @@ void gen7_hwsched_context_destroy(struct adreno_device *adreno_dev,
 	if (drawctxt->gmu_hw_fence_queue.gmuaddr)
 		gen7_free_gmu_block(to_gen7_gmu(adreno_dev), &drawctxt->gmu_hw_fence_queue);
 }
+
+int gen7_hwsched_counter_inline_enable(struct adreno_device *adreno_dev,
+		const struct adreno_perfcount_group *group,
+		u32 counter, u32 countable)
+{
+	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
+	struct adreno_perfcount_register *reg = &group->regs[counter];
+	u32 val, cmds[GEN7_PERF_COUNTER_ENABLE_DWORDS + 1];
+	int ret;
+
+	if (group->flags & ADRENO_PERFCOUNTER_GROUP_RESTORE)
+		gen7_perfcounter_update(adreno_dev, reg, false,
+				FIELD_PREP(GENMASK(13, 12), PIPE_NONE));
+
+	cmds[0] = CREATE_MSG_HDR(H2F_MSG_ISSUE_CMD_RAW,
+		(GEN7_PERF_COUNTER_ENABLE_DWORDS + 1) << 2, HFI_MSG_CMD);
+
+	cmds[1] = cp_type7_packet(CP_WAIT_FOR_IDLE, 0);
+	cmds[2] = cp_type4_packet(reg->select, 1);
+	cmds[3] = countable;
+
+	ret = gen7_hfi_send_cmd_async(adreno_dev, cmds);
+	if (ret)
+		goto err;
+
+	/* Wait till the register is programmed with the countable */
+	ret = kgsl_regmap_read_poll_timeout(&device->regmap, reg->select, val,
+				val == countable, 100, ADRENO_IDLE_TIMEOUT);
+	if (!ret) {
+		reg->value = 0;
+		return ret;
+	}
+
+err:
+	dev_err(device->dev, "Perfcounter %s/%u/%u start via commands failed\n",
+			group->name, counter, countable);
+	return ret;
+}
diff --git a/adreno_gen7_hwsched_hfi.h b/adreno_gen7_hwsched_hfi.h
index 85751f4..6154a75 100644
--- a/adreno_gen7_hwsched_hfi.h
+++ b/adreno_gen7_hwsched_hfi.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef _ADRENO_GEN7_HWSCHED_HFI_H_
@@ -90,6 +90,20 @@ void gen7_hwsched_hfi_stop(struct adreno_device *adreno_dev);
 int gen7_hwsched_cp_init(struct adreno_device *adreno_dev);
 
 /**
+ * gen7_hwsched_counter_inline_enable - Configure a performance counter for a countable
+ * @adreno_dev -  Adreno device to configure
+ * @group - Desired performance counter group
+ * @counter - Desired performance counter in the group
+ * @countable - Desired countable
+ *
+ * Physically set up a counter within a group with the desired countable
+ * Return 0 on success or negative error on failure.
+ */
+int gen7_hwsched_counter_inline_enable(struct adreno_device *adreno_dev,
+		const struct adreno_perfcount_group *group,
+		u32 counter, u32 countable);
+
+/**
  * gen7_hfi_send_cmd_async - Send an hfi packet
  * @adreno_dev: Pointer to adreno device structure
  * @data: Data to be sent in the hfi packet
diff --git a/adreno_gen7_perfcounter.c b/adreno_gen7_perfcounter.c
index 2120c2c..74a7799 100644
--- a/adreno_gen7_perfcounter.c
+++ b/adreno_gen7_perfcounter.c
@@ -1,11 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include "adreno.h"
 #include "adreno_gen7.h"
+#include "adreno_gen7_hwsched_hfi.h"
 #include "adreno_perfcounter.h"
 #include "adreno_pm4types.h"
 #include "kgsl_device.h"
@@ -100,6 +101,16 @@ static int gen7_counter_enable(struct adreno_device *adreno_dev,
 	return ret;
 }
 
+static int gen7_hwsched_counter_enable(struct adreno_device *adreno_dev,
+		const struct adreno_perfcount_group *group,
+		u32 counter, u32 countable)
+{
+	if (!(KGSL_DEVICE(adreno_dev)->state == KGSL_STATE_ACTIVE))
+		return gen7_counter_enable(adreno_dev, group, counter, countable);
+
+	return gen7_hwsched_counter_inline_enable(adreno_dev, group, counter, countable);
+}
+
 static u64 gen7_counter_read(struct adreno_device *adreno_dev,
 		const struct adreno_perfcount_group *group,
 		unsigned int counter)
@@ -939,13 +950,13 @@ static struct adreno_perfcount_register gen7_perfcounters_alwayson[] = {
 	GEN7_BV_PERFCOUNTER_GROUP(offset, name, \
 		gen7_counter_enable, gen7_counter_read)
 
-static const struct adreno_perfcount_group gen7_2_0_perfcounter_groups
+static const struct adreno_perfcount_group gen7_hwsched_perfcounter_groups
 				[KGSL_PERFCOUNTER_GROUP_MAX] = {
 	GEN7_REGULAR_PERFCOUNTER_GROUP(CP, cp),
 	GEN7_PERFCOUNTER_GROUP_FLAGS(gen7, RBBM, rbbm, 0,
 		gen7_counter_enable, gen7_counter_read),
 	GEN7_REGULAR_PERFCOUNTER_GROUP(PC, pc),
-	GEN7_REGULAR_PERFCOUNTER_GROUP(VFD, vfd),
+	GEN7_PERFCOUNTER_GROUP(VFD, vfd, gen7_hwsched_counter_enable, gen7_counter_read),
 	GEN7_PERFCOUNTER_GROUP(HLSQ, hlsq, gen7_counter_br_enable, gen7_counter_read),
 	GEN7_REGULAR_PERFCOUNTER_GROUP(VPC, vpc),
 	GEN7_REGULAR_PERFCOUNTER_GROUP(CCU, ccu),
@@ -954,8 +965,8 @@ static const struct adreno_perfcount_group gen7_2_0_perfcounter_groups
 	GEN7_PERFCOUNTER_GROUP(RAS, ras, gen7_counter_br_enable, gen7_counter_read),
 	GEN7_PERFCOUNTER_GROUP(LRZ, lrz, gen7_counter_br_enable, gen7_counter_read),
 	GEN7_REGULAR_PERFCOUNTER_GROUP(UCHE, gen7_2_0_uche),
-	GEN7_REGULAR_PERFCOUNTER_GROUP(TP, tp),
-	GEN7_REGULAR_PERFCOUNTER_GROUP(SP, sp),
+	GEN7_PERFCOUNTER_GROUP(TP, tp, gen7_hwsched_counter_enable, gen7_counter_read),
+	GEN7_PERFCOUNTER_GROUP(SP, sp, gen7_hwsched_counter_enable, gen7_counter_read),
 	GEN7_REGULAR_PERFCOUNTER_GROUP(RB, rb),
 	GEN7_REGULAR_PERFCOUNTER_GROUP(VSC, vsc),
 	GEN7_PERFCOUNTER_GROUP_FLAGS(gen7, VBIF, gbif, 0,
@@ -1038,7 +1049,7 @@ const struct adreno_perfcounters adreno_gen7_perfcounters = {
 	ARRAY_SIZE(gen7_perfcounter_groups),
 };
 
-const struct adreno_perfcounters adreno_gen7_2_0_perfcounters = {
-	gen7_2_0_perfcounter_groups,
-	ARRAY_SIZE(gen7_2_0_perfcounter_groups),
+const struct adreno_perfcounters adreno_gen7_hwsched_perfcounters = {
+	gen7_hwsched_perfcounter_groups,
+	ARRAY_SIZE(gen7_hwsched_perfcounter_groups),
 };
diff --git a/adreno_gen7_rpmh.c b/adreno_gen7_rpmh.c
index 0c196ec..fd7c2fc 100644
--- a/adreno_gen7_rpmh.c
+++ b/adreno_gen7_rpmh.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include <linux/types.h>
@@ -118,8 +118,8 @@ static int setup_volt_dependency_tbl(u32 *votes,
 }
 
 /* Generate a set of bandwidth votes for the list of BCMs */
-static void tcs_cmd_data(struct bcm *bcms, int count, u32 ab, u32 ib,
-		u32 *data)
+static void tcs_cmd_data(struct bcm *bcms, int count,
+		u32 ab, u32 ib, u32 *data, u32 perfmode_vote, bool set_perfmode)
 {
 	int i;
 
@@ -135,7 +135,8 @@ static void tcs_cmd_data(struct bcm *bcms, int count, u32 ab, u32 ib,
 			if (!ab && !ib)
 				data[i] = BCM_TCS_CMD(commit, false, 0x0, 0x0);
 			else
-				data[i] = BCM_TCS_CMD(commit, true, 0x0, 0x8);
+				data[i] = BCM_TCS_CMD(commit, true, 0x0,
+							set_perfmode ? perfmode_vote : 0x0);
 			continue;
 		}
 
@@ -193,9 +194,10 @@ static void free_rpmh_bw_votes(struct rpmh_bw_votes *votes)
 
 /* Build the votes table from the specified bandwidth levels */
 static struct rpmh_bw_votes *build_rpmh_bw_votes(struct bcm *bcms,
-		int bcm_count, u32 *levels, int levels_count)
+		int bcm_count, u32 *levels, int levels_count, u32 perfmode_vote, u32 perfmode_lvl)
 {
 	struct rpmh_bw_votes *votes;
+	bool set_perfmode;
 	int i;
 
 	votes = kzalloc(sizeof(*votes), GFP_KERNEL);
@@ -243,7 +245,9 @@ static struct rpmh_bw_votes *build_rpmh_bw_votes(struct bcm *bcms,
 			return ERR_PTR(-ENOMEM);
 		}
 
-		tcs_cmd_data(bcms, bcm_count, levels[i], levels[i], votes->cmds[i]);
+		set_perfmode = (i >= perfmode_lvl) ? true : false;
+		tcs_cmd_data(bcms, bcm_count, levels[i], levels[i], votes->cmds[i],
+								perfmode_vote, set_perfmode);
 	}
 
 	return votes;
@@ -455,18 +459,27 @@ static void build_bw_table_cmd(struct hfi_bwtable_cmd *cmd,
 			cmd->cnoc_cmd_data[i][j] = (u32) cnoc->cmds[i][j];
 }
 
+#define GEN7_9_0_DDR_NOM_IDX 6
+
 static int build_bw_table(struct adreno_device *adreno_dev)
 {
 	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
+	const struct adreno_gen7_core *gen7_core = to_gen7_core(adreno_dev);
 	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
 	struct kgsl_pwrctrl *pwr = &device->pwrctrl;
 	struct rpmh_bw_votes *ddr, *cnoc = NULL;
+	u32 perfmode_vote = gen7_core->acv_perfmode_vote;
+	u32 perfmode_lvl = adreno_is_gen7_9_0(adreno_dev) ? GEN7_9_0_DDR_NOM_IDX : 1;
 	u32 *cnoc_table;
 	u32 count;
 	int ret;
 
+	/* If perfmode vote is not defined, use default value as 0x8 */
+	if (!perfmode_vote)
+		perfmode_vote = BIT(3);
+
 	ddr = build_rpmh_bw_votes(gen7_ddr_bcms, ARRAY_SIZE(gen7_ddr_bcms),
-		pwr->ddr_table, pwr->ddr_table_count);
+		pwr->ddr_table, pwr->ddr_table_count, perfmode_vote, perfmode_lvl);
 	if (IS_ERR(ddr))
 		return PTR_ERR(ddr);
 
@@ -475,7 +488,7 @@ static int build_bw_table(struct adreno_device *adreno_dev)
 
 	if (count > 0)
 		cnoc = build_rpmh_bw_votes(gen7_cnoc_bcms,
-			ARRAY_SIZE(gen7_cnoc_bcms), cnoc_table, count);
+			ARRAY_SIZE(gen7_cnoc_bcms), cnoc_table, count, 0, 0);
 
 	kfree(cnoc_table);
 
diff --git a/adreno_hwsched.c b/adreno_hwsched.c
index a03f2db..cae6a3c 100644
--- a/adreno_hwsched.c
+++ b/adreno_hwsched.c
@@ -1506,6 +1506,9 @@ static void do_fault_header_lpac(struct adreno_device *adreno_dev,
 		drawobj_lpac->context->gmu_dispatch_queue, lpac_rptr, lpac_wptr,
 		lpac_ib1base, lpac_ib1sz, lpac_ib2base, lpac_ib2sz);
 
+	pr_context(device, drawobj_lpac->context, "lpac cmdline: %s\n",
+			drawctxt_lpac->base.proc_priv->cmdline);
+
 	trace_adreno_gpu_fault(drawobj_lpac->context->id, drawobj_lpac->timestamp, status,
 		lpac_rptr, lpac_wptr, lpac_ib1base, lpac_ib1sz, lpac_ib2base, lpac_ib2sz,
 		adreno_get_level(drawobj_lpac->context));
@@ -1541,6 +1544,9 @@ static void do_fault_header(struct adreno_device *adreno_dev,
 		drawobj->context->gmu_dispatch_queue, rptr, wptr,
 		ib1base, ib1sz, ib2base, ib2sz);
 
+	pr_context(device, drawobj->context, "cmdline: %s\n",
+			drawctxt->base.proc_priv->cmdline);
+
 	trace_adreno_gpu_fault(drawobj->context->id, drawobj->timestamp, status,
 		rptr, wptr, ib1base, ib1sz, ib2base, ib2sz,
 		adreno_get_level(drawobj->context));
diff --git a/config/gki_khajedisp.conf b/config/gki_khajedisp.conf
index bc1590c..ddc889c 100644
--- a/config/gki_khajedisp.conf
+++ b/config/gki_khajedisp.conf
@@ -1,15 +1,21 @@
 CONFIG_DEVFREQ_GOV_QCOM_ADRENO_TZ = y
 CONFIG_DEVFREQ_GOV_QCOM_GPUBW_MON = y
 CONFIG_QCOM_KGSL_IDLE_TIMEOUT = 80
-CONFIG_QCOM_KGSL_SORT_POOL = y
 CONFIG_QCOM_KGSL_CONTEXT_DEBUG = y
-CONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT = y
 CONFIG_QCOM_ADRENO_DEFAULT_GOVERNOR = \"msm-adreno-tz\"
 
+ifneq ($(CONFIG_SHMEM),)
+	CONFIG_QCOM_KGSL_USE_SHMEM = y
+	CONFIG_QCOM_KGSL_PROCESS_RECLAIM = y
+endif
+
 ccflags-y += -DCONFIG_DEVFREQ_GOV_QCOM_ADRENO_TZ=1 \
 		-DCONFIG_DEVFREQ_GOV_QCOM_GPUBW_MON=1 \
 		-DCONFIG_QCOM_KGSL_IDLE_TIMEOUT=80 \
-		-DCONFIG_QCOM_KGSL_SORT_POOL=1 \
 		-DCONFIG_QCOM_KGSL_CONTEXT_DEBUG=1 \
-		-DCONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT=0 \
 		-DCONFIG_QCOM_ADRENO_DEFAULT_GOVERNOR=\"msm-adreno-tz\"
+
+ifneq ($(CONFIG_SHMEM),)
+	ccflags-y += -DCONFIG_QCOM_KGSL_USE_SHMEM=1 \
+			-DCONFIG_QCOM_KGSL_PROCESS_RECLAIM=1
+endif
diff --git a/kgsl.c b/kgsl.c
index 1d3e1aa..173018e 100644
--- a/kgsl.c
+++ b/kgsl.c
@@ -24,6 +24,7 @@
 #include <linux/qcom_dma_heap.h>
 #include <linux/security.h>
 #include <linux/sort.h>
+#include <linux/string_helpers.h>
 #include <soc/qcom/of_common.h>
 #include <soc/qcom/secure_buffer.h>
 #include <soc/qcom/boot_stats.h>
@@ -949,6 +950,7 @@ static void kgsl_destroy_process_private(struct kref *kref)
 	write_unlock(&kgsl_driver.proclist_lock);
 	mutex_unlock(&kgsl_driver.process_mutex);
 
+	kfree(private->cmdline);
 	put_pid(private->pid);
 	idr_destroy(&private->mem_idr);
 	idr_destroy(&private->syncsource_idr);
@@ -1167,6 +1169,7 @@ static struct kgsl_process_private *kgsl_process_private_new(
 	private->fd_count = 1;
 	private->pid = cur_pid;
 	get_task_comm(private->comm, current->group_leader);
+	private->cmdline = kstrdup_quotable_cmdline(current, GFP_KERNEL);
 
 	spin_lock_init(&private->mem_lock);
 	spin_lock_init(&private->syncsource_lock);
@@ -5064,11 +5067,6 @@ int kgsl_device_platform_probe(struct kgsl_device *device)
 
 	sched_set_fifo(device->events_worker->task);
 
-	/* This can return -EPROBE_DEFER */
-	status = kgsl_mmu_probe(device);
-	if (status != 0)
-		goto error_pwrctrl_close;
-
 	status = kgsl_reclaim_init();
 	if (status)
 		goto error_pwrctrl_close;
@@ -5121,12 +5119,6 @@ void kgsl_device_platform_remove(struct kgsl_device *device)
 
 	kgsl_device_events_remove(device);
 
-	kgsl_mmu_close(device);
-
-	/*
-	 * This needs to come after the MMU close so we can be sure all the
-	 * pagetables have been freed
-	 */
 	kgsl_free_globals(device);
 
 	kgsl_pwrctrl_close(device);
diff --git a/kgsl_device.h b/kgsl_device.h
index 9980f73..6b7464f 100644
--- a/kgsl_device.h
+++ b/kgsl_device.h
@@ -528,6 +528,10 @@ struct kgsl_process_private {
 	 * @private_mutex: Mutex lock to protect kgsl_process_private
 	 */
 	struct mutex private_mutex;
+	/**
+	 * @cmdline: Cmdline string of the process
+	 */
+	char *cmdline;
 };
 
 struct kgsl_device_private {
diff --git a/kgsl_gmu_core.c b/kgsl_gmu_core.c
index aa3c157..8dac6cc 100644
--- a/kgsl_gmu_core.c
+++ b/kgsl_gmu_core.c
@@ -34,7 +34,7 @@ void __init gmu_core_register(void)
 	of_node_put(node);
 }
 
-void __exit gmu_core_unregister(void)
+void gmu_core_unregister(void)
 {
 	const struct of_device_id *match;
 	struct device_node *node;
diff --git a/kgsl_gmu_core.h b/kgsl_gmu_core.h
index 7770e6e..2ef3ed0 100644
--- a/kgsl_gmu_core.h
+++ b/kgsl_gmu_core.h
@@ -263,7 +263,7 @@ extern struct platform_driver gen7_hwsched_driver;
 /* GMU core functions */
 
 void __init gmu_core_register(void);
-void __exit gmu_core_unregister(void);
+void gmu_core_unregister(void);
 
 bool gmu_core_gpmu_isenabled(struct kgsl_device *device);
 bool gmu_core_scales_bandwidth(struct kgsl_device *device);
diff --git a/kgsl_iommu.c b/kgsl_iommu.c
index adf3398..23fee14 100644
--- a/kgsl_iommu.c
+++ b/kgsl_iommu.c
@@ -1219,7 +1219,7 @@ int kgsl_set_smmu_aperture(struct kgsl_device *device,
 		ret = qcom_scm_kgsl_set_smmu_aperture(context->cb_num);
 
 	if (ret)
-		dev_err(device->dev, "Unable to set the SMMU aperture: %d. The aperture needs to be set to use per-process pagetables\n",
+		dev_err(&device->pdev->dev, "Unable to set the SMMU aperture: %d. The aperture needs to be set to use per-process pagetables\n",
 			ret);
 
 	return ret;
@@ -1238,7 +1238,7 @@ static int set_smmu_lpac_aperture(struct kgsl_device *device,
 		ret = qcom_scm_kgsl_set_smmu_lpac_aperture(context->cb_num);
 
 	if (ret)
-		dev_err(device->dev, "Unable to set the LPAC SMMU aperture: %d. The aperture needs to be set to use per-process pagetables\n",
+		dev_err(&device->pdev->dev, "Unable to set the LPAC SMMU aperture: %d. The aperture needs to be set to use per-process pagetables\n",
 			ret);
 
 	return ret;
@@ -1479,9 +1479,6 @@ static void kgsl_iommu_close(struct kgsl_mmu *mmu)
 		kgsl_guard_page = NULL;
 	}
 
-	of_platform_depopulate(&iommu->pdev->dev);
-	platform_device_put(iommu->pdev);
-
 	kmem_cache_destroy(addr_entry_cache);
 	addr_entry_cache = NULL;
 }
@@ -2136,6 +2133,7 @@ static int kgsl_iommu_setup_context(struct kgsl_mmu *mmu,
 {
 	struct device_node *node = of_find_node_by_name(parent, name);
 	struct platform_device *pdev;
+	struct kgsl_device *device = KGSL_MMU_DEVICE(mmu);
 	int ret;
 
 	if (!node)
@@ -2150,7 +2148,7 @@ static int kgsl_iommu_setup_context(struct kgsl_mmu *mmu,
 
 	context->cb_num = -1;
 	context->name = name;
-	context->kgsldev = KGSL_MMU_DEVICE(mmu);
+	context->kgsldev = device;
 	context->pdev = pdev;
 	ratelimit_default_init(&context->ratelimit);
 
@@ -2181,7 +2179,7 @@ static int kgsl_iommu_setup_context(struct kgsl_mmu *mmu,
 	if (context->cb_num >= 0)
 		return 0;
 
-	dev_err(KGSL_MMU_DEVICE(mmu)->dev, "Couldn't get the context bank for %s: %d\n",
+	dev_err(&device->pdev->dev, "Couldn't get the context bank for %s: %d\n",
 		context->name, context->cb_num);
 
 	iommu_detach_device(context->domain, &context->pdev->dev);
@@ -2306,7 +2304,7 @@ static int iommu_probe_secure_context(struct kgsl_device *device,
 
 	ret = qcom_iommu_set_secure_vmid(context->domain, secure_vmid);
 	if (ret) {
-		dev_err(device->dev, "Unable to set the secure VMID: %d\n", ret);
+		dev_err(&device->pdev->dev, "Unable to set the secure VMID: %d\n", ret);
 		iommu_domain_free(context->domain);
 		context->domain = NULL;
 
@@ -2382,20 +2380,15 @@ static void kgsl_iommu_check_config(struct kgsl_mmu *mmu,
 	of_node_put(node);
 }
 
-int kgsl_iommu_probe(struct kgsl_device *device)
+int kgsl_iommu_bind(struct kgsl_device *device, struct platform_device *pdev)
 {
 	u32 val[2];
 	int ret, i;
 	struct kgsl_iommu *iommu = KGSL_IOMMU(device);
-	struct platform_device *pdev;
 	struct kgsl_mmu *mmu = &device->mmu;
-	struct device_node *node;
+	struct device_node *node = pdev->dev.of_node;
 	struct kgsl_global_memdesc *md;
 
-	node = of_find_compatible_node(NULL, NULL, "qcom,kgsl-smmu-v2");
-	if (!node)
-		return -ENODEV;
-
 	/* Create a kmem cache for the pagetable address objects */
 	if (!addr_entry_cache) {
 		addr_entry_cache = KMEM_CACHE(kgsl_iommu_addr_entry, 0);
@@ -2407,7 +2400,7 @@ int kgsl_iommu_probe(struct kgsl_device *device)
 
 	ret = of_property_read_u32_array(node, "reg", val, 2);
 	if (ret) {
-		dev_err(device->dev,
+		dev_err(&device->pdev->dev,
 			"%pOF: Unable to read KGSL IOMMU register range\n",
 			node);
 		goto err;
@@ -2420,14 +2413,12 @@ int kgsl_iommu_probe(struct kgsl_device *device)
 		goto err;
 	}
 
-	pdev = of_find_device_by_node(node);
 	iommu->pdev = pdev;
 	iommu->num_clks = 0;
 
 	iommu->clks = devm_kcalloc(&pdev->dev, ARRAY_SIZE(kgsl_iommu_clocks),
 				sizeof(*iommu->clks), GFP_KERNEL);
 	if (!iommu->clks) {
-		platform_device_put(pdev);
 		ret = -ENOMEM;
 		goto err;
 	}
@@ -2451,9 +2442,6 @@ int kgsl_iommu_probe(struct kgsl_device *device)
 	mmu->type = KGSL_MMU_TYPE_IOMMU;
 	mmu->mmu_ops = &kgsl_iommu_ops;
 
-	/* Fill out the rest of the devices in the node */
-	of_platform_populate(node, NULL, NULL, &pdev->dev);
-
 	/* Peek at the phandle to set up configuration */
 	kgsl_iommu_check_config(mmu, node);
 
@@ -2461,13 +2449,11 @@ int kgsl_iommu_probe(struct kgsl_device *device)
 	ret = iommu_probe_user_context(device, node);
 	if (ret) {
 		of_platform_depopulate(&pdev->dev);
-		platform_device_put(pdev);
 		goto err;
 	}
 
 	/* Probe the secure pagetable (this is optional) */
 	iommu_probe_secure_context(device, node);
-	of_node_put(node);
 
 	/* Map any globals that might have been created early */
 	list_for_each_entry(md, &device->globals, node) {
@@ -2515,7 +2501,6 @@ err:
 	kmem_cache_destroy(addr_entry_cache);
 	addr_entry_cache = NULL;
 
-	of_node_put(node);
 	return ret;
 }
 
diff --git a/kgsl_mmu.c b/kgsl_mmu.c
index ac97d5e..8992d72 100644
--- a/kgsl_mmu.c
+++ b/kgsl_mmu.c
@@ -1,9 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (c) 2002,2007-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
+#include <linux/component.h>
+#include <linux/of_platform.h>
 #include <linux/slab.h>
 
 #include "kgsl_device.h"
@@ -499,14 +501,6 @@ void kgsl_mmu_map_global(struct kgsl_device *device,
 		mmu->mmu_ops->mmu_map_global(mmu, memdesc, padding);
 }
 
-void kgsl_mmu_close(struct kgsl_device *device)
-{
-	struct kgsl_mmu *mmu = &(device->mmu);
-
-	if (MMU_OP_VALID(mmu, mmu_close))
-		mmu->mmu_ops->mmu_close(mmu);
-}
-
 int kgsl_mmu_pagetable_get_context_bank(struct kgsl_pagetable *pagetable,
 	struct kgsl_context *context)
 {
@@ -610,16 +604,28 @@ static struct kgsl_mmu_ops kgsl_nommu_ops = {
 	.mmu_getpagetable = nommu_getpagetable,
 };
 
-int kgsl_mmu_probe(struct kgsl_device *device)
+static int kgsl_mmu_cb_bind(struct device *dev, struct device *master, void *data)
+{
+	return 0;
+}
+
+static void kgsl_mmu_cb_unbind(struct device *dev, struct device *master,
+		void *data)
+{
+}
+
+static int kgsl_mmu_bind(struct device *dev, struct device *master, void *data)
 {
+	struct kgsl_device *device = dev_get_drvdata(master);
 	struct kgsl_mmu *mmu = &device->mmu;
 	int ret;
 
 	/*
-	 * Try to probe for the IOMMU and if it doesn't exist for some reason
+	 * Try to bind the IOMMU and if it doesn't exist for some reason
 	 * go for the NOMMU option instead
 	 */
-	ret = kgsl_iommu_probe(device);
+	ret = kgsl_iommu_bind(device, to_platform_device(dev));
+
 	if (!ret || ret == -EPROBE_DEFER)
 		return ret;
 
@@ -627,3 +633,91 @@ int kgsl_mmu_probe(struct kgsl_device *device)
 	mmu->type = KGSL_MMU_TYPE_NONE;
 	return 0;
 }
+
+static void kgsl_mmu_unbind(struct device *dev, struct device *master,
+		void *data)
+{
+	struct kgsl_device *device = dev_get_drvdata(master);
+	struct kgsl_mmu *mmu = &device->mmu;
+
+	if (MMU_OP_VALID(mmu, mmu_close))
+		mmu->mmu_ops->mmu_close(mmu);
+}
+
+static const struct component_ops kgsl_mmu_cb_component_ops = {
+	.bind = kgsl_mmu_cb_bind,
+	.unbind = kgsl_mmu_cb_unbind,
+};
+
+static const struct component_ops kgsl_mmu_component_ops = {
+	.bind = kgsl_mmu_bind,
+	.unbind = kgsl_mmu_unbind,
+};
+
+static int kgsl_mmu_dev_probe(struct platform_device *pdev)
+{
+	/*
+	 * Add kgsl-smmu and context bank as a component device to establish
+	 * correct probe order with smmu driver.
+	 *
+	 * As context bank node in DT contains "iommus" property. fw_devlink
+	 * ensures that context bank is probed only after corresponding
+	 * supplier (smmu driver) probe is done.
+	 *
+	 * Adding context bank as a component device ensures master bind
+	 * (adreno_bind) is called only once component (kgsl-smmu and context
+	 * banks) probe is done thus ensuring correct probe order with smmu
+	 * driver.
+	 *
+	 * kgsl-smmu also need to be a component because we need kgsl-smmu
+	 * device info in order to initialize the context banks.
+	 */
+	if (of_device_is_compatible(pdev->dev.of_node,
+				"qcom,smmu-kgsl-cb")) {
+		return component_add(&pdev->dev, &kgsl_mmu_cb_component_ops);
+	}
+
+	/* Fill out the rest of the devices in the node */
+	of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
+
+	return component_add(&pdev->dev, &kgsl_mmu_component_ops);
+}
+
+static int kgsl_mmu_dev_remove(struct platform_device *pdev)
+{
+	if (of_device_is_compatible(pdev->dev.of_node,
+				"qcom,smmu-kgsl-cb")) {
+		component_del(&pdev->dev, &kgsl_mmu_cb_component_ops);
+		return 0;
+	}
+
+	component_del(&pdev->dev, &kgsl_mmu_component_ops);
+
+	of_platform_depopulate(&pdev->dev);
+	return 0;
+}
+
+static const struct of_device_id mmu_match_table[] = {
+	{ .compatible = "qcom,kgsl-smmu-v2" },
+	{ .compatible = "qcom,smmu-kgsl-cb" },
+	{},
+};
+
+static struct platform_driver kgsl_mmu_driver = {
+	.probe = kgsl_mmu_dev_probe,
+	.remove = kgsl_mmu_dev_remove,
+	.driver = {
+		.name = "kgsl-iommu",
+		.of_match_table = mmu_match_table,
+	}
+};
+
+int __init kgsl_mmu_init(void)
+{
+	return platform_driver_register(&kgsl_mmu_driver);
+}
+
+void kgsl_mmu_exit(void)
+{
+	platform_driver_unregister(&kgsl_mmu_driver);
+}
diff --git a/kgsl_mmu.h b/kgsl_mmu.h
index a1c9c1e..01be7c6 100644
--- a/kgsl_mmu.h
+++ b/kgsl_mmu.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2002,2007-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  */
 #ifndef __KGSL_MMU_H
 #define __KGSL_MMU_H
@@ -203,7 +203,9 @@ struct kgsl_mmu {
 
 #define KGSL_IOMMU(d) (&((d)->mmu.iommu))
 
-int kgsl_mmu_probe(struct kgsl_device *device);
+int __init kgsl_mmu_init(void);
+void kgsl_mmu_exit(void);
+
 int kgsl_mmu_start(struct kgsl_device *device);
 
 void kgsl_print_global_pt_entries(struct seq_file *s);
@@ -233,8 +235,6 @@ int kgsl_mmu_find_region(struct kgsl_pagetable *pagetable,
 		uint64_t region_start, uint64_t region_end,
 		uint64_t *gpuaddr, uint64_t size, unsigned int align);
 
-void kgsl_mmu_close(struct kgsl_device *device);
-
 uint64_t kgsl_mmu_find_svm_region(struct kgsl_pagetable *pagetable,
 		uint64_t start, uint64_t end, uint64_t size,
 		uint64_t alignment);
@@ -418,9 +418,9 @@ void kgsl_mmu_pagetable_init(struct kgsl_mmu *mmu,
 void kgsl_mmu_pagetable_add(struct kgsl_mmu *mmu, struct kgsl_pagetable *pagetable);
 
 #if IS_ENABLED(CONFIG_ARM_SMMU)
-int kgsl_iommu_probe(struct kgsl_device *device);
+int kgsl_iommu_bind(struct kgsl_device *device, struct platform_device *pdev);
 #else
-static inline int kgsl_iommu_probe(struct kgsl_device *device)
+static inline int kgsl_iommu_bind(struct kgsl_device *device, struct platform_device *pdev)
 {
 	return -ENODEV;
 }