Merge android13-gs-pixel-5.15 into android14-gs-pixel-5.15

Bug: 260173634
Change-Id: I0ff4522ec43ca9db260035d89b67123bcb857afb
Signed-off-by: Robin Peng <robinpeng@google.com>

7 files changed, 170 insertions, 58 deletions

diff --git a/drivers/edgetpu/edgetpu-kci.c b/drivers/edgetpu/edgetpu-kci.c
index caa80e7..194c9fb 100644
--- a/drivers/edgetpu/edgetpu-kci.c
+++ b/drivers/edgetpu/edgetpu-kci.c
@@ -213,10 +213,12 @@ int edgetpu_kci_init(struct edgetpu_mailbox_manager *mgr, struct edgetpu_kci *et
 	if (IS_ERR(mailbox))
 		return PTR_ERR(mailbox);
 
-	etkci->kci = devm_kzalloc(mgr->etdev->dev, sizeof(*etkci->kci), GFP_KERNEL);
 	if (!etkci->kci) {
-		ret = -ENOMEM;
-		goto err_mailbox_remove;
+		etkci->kci = devm_kzalloc(mgr->etdev->dev, sizeof(*etkci->kci), GFP_KERNEL);
+		if (!etkci->kci) {
+			ret = -ENOMEM;
+			goto err_mailbox_remove;
+		}
 	}
 
 	ret = edgetpu_kci_alloc_queue(mgr->etdev, mailbox, GCIP_MAILBOX_CMD_QUEUE, cmd_queue_mem);
diff --git a/drivers/edgetpu/edgetpu-mailbox.c b/drivers/edgetpu/edgetpu-mailbox.c
index f5ef44b..869fdec 100644
--- a/drivers/edgetpu/edgetpu-mailbox.c
+++ b/drivers/edgetpu/edgetpu-mailbox.c
@@ -244,8 +244,8 @@ edgetpu_mailbox_vii_add(struct edgetpu_mailbox_manager *mgr, uint id)
 
 /*
  * Every mailbox manager can allocate one mailbox for KCI to use.
- * -EBUSY is returned if the KCI mailbox is allocated and hasn't been removed
- * via edgetpu_mailbox_remove().
+ * Previously allocated KCI mailbox is returned if it hasn't been removed via
+ * edgetpu_mailbox_remove().
  */
 struct edgetpu_mailbox *edgetpu_mailbox_kci(struct edgetpu_mailbox_manager *mgr)
 {
@@ -254,7 +254,7 @@ struct edgetpu_mailbox *edgetpu_mailbox_kci(struct edgetpu_mailbox_manager *mgr)
 
 	write_lock_irqsave(&mgr->mailboxes_lock, flags);
 	if (mgr->mailboxes[KERNEL_MAILBOX_INDEX]) {
-		mailbox = ERR_PTR(-EBUSY);
+		mailbox = mgr->mailboxes[KERNEL_MAILBOX_INDEX];
 		goto out;
 	}
 
diff --git a/drivers/edgetpu/edgetpu-mobile-platform.c b/drivers/edgetpu/edgetpu-mobile-platform.c
index f9fe1fa..3ee70af 100644
--- a/drivers/edgetpu/edgetpu-mobile-platform.c
+++ b/drivers/edgetpu/edgetpu-mobile-platform.c
@@ -23,15 +23,7 @@
 #include "mobile-firmware.h"
 #include "mobile-pm.h"
 
-/*
- * Log and trace buffers at the beginning of the remapped region,
- * pool memory afterwards.
- */
-#define EDGETPU_POOL_MEM_OFFSET                                                                    \
-	((EDGETPU_TELEMETRY_LOG_BUFFER_SIZE + EDGETPU_TELEMETRY_TRACE_BUFFER_SIZE) *               \
-	 EDGETPU_NUM_CORES)
-
-static void get_telemetry_mem(struct edgetpu_mobile_platform_dev *etmdev,
+static void set_telemetry_mem(struct edgetpu_mobile_platform_dev *etmdev,
 			      enum gcip_telemetry_type type, struct edgetpu_coherent_mem *mem)
 {
 	int i, offset = type == GCIP_TELEMETRY_TRACE ? EDGETPU_TELEMETRY_LOG_BUFFER_SIZE : 0;
@@ -40,18 +32,18 @@ static void get_telemetry_mem(struct edgetpu_mobile_platform_dev *etmdev,
 
 	for (i = 0; i < etmdev->edgetpu_dev.num_cores; i++) {
 		mem[i].vaddr = etmdev->shared_mem_vaddr + offset;
-		mem[i].dma_addr = EDGETPU_REMAPPED_DATA_ADDR + offset;
-		mem[i].tpu_addr = EDGETPU_REMAPPED_DATA_ADDR + offset;
+		mem[i].dma_addr = etmdev->remapped_data_addr + offset;
+		mem[i].tpu_addr = etmdev->remapped_data_addr + offset;
 		mem[i].host_addr = 0;
 		mem[i].size = size;
 		offset += EDGETPU_TELEMETRY_LOG_BUFFER_SIZE + EDGETPU_TELEMETRY_TRACE_BUFFER_SIZE;
 	}
 }
 
-static void edgetpu_mobile_get_telemetry_mem(struct edgetpu_mobile_platform_dev *etmdev)
+void edgetpu_mobile_set_telemetry_mem(struct edgetpu_mobile_platform_dev *etmdev)
 {
-	get_telemetry_mem(etmdev, GCIP_TELEMETRY_LOG, etmdev->log_mem);
-	get_telemetry_mem(etmdev, GCIP_TELEMETRY_TRACE, etmdev->trace_mem);
+	set_telemetry_mem(etmdev, GCIP_TELEMETRY_LOG, etmdev->log_mem);
+	set_telemetry_mem(etmdev, GCIP_TELEMETRY_TRACE, etmdev->trace_mem);
 }
 
 static int edgetpu_platform_setup_fw_region(struct edgetpu_mobile_platform_dev *etmdev)
@@ -62,7 +54,7 @@ static int edgetpu_platform_setup_fw_region(struct edgetpu_mobile_platform_dev *
 	struct resource r;
 	struct device_node *np;
 	int err;
-	size_t region_map_size = EDGETPU_FW_SIZE_MAX + EDGETPU_REMAPPED_DATA_SIZE;
+	size_t region_map_size = EDGETPU_DEFAULT_FW_SIZE_MAX + EDGETPU_DEFAULT_REMAPPED_DATA_SIZE;
 
 	np = of_parse_phandle(dev->of_node, "memory-region", 0);
 	if (!np) {
@@ -99,17 +91,20 @@ static int edgetpu_platform_setup_fw_region(struct edgetpu_mobile_platform_dev *
 	}
 
 	etmdev->fw_region_paddr = r.start;
-	etmdev->fw_region_size = EDGETPU_FW_SIZE_MAX;
+	etmdev->fw_region_size = EDGETPU_DEFAULT_FW_SIZE_MAX;
+
+	etmdev->remapped_data_addr = EDGETPU_INSTRUCTION_REMAP_BASE + etmdev->fw_region_size;
+	etmdev->remapped_data_size = EDGETPU_DEFAULT_REMAPPED_DATA_SIZE;
 
-	etmdev->shared_mem_vaddr = memremap(r.start + EDGETPU_REMAPPED_DATA_OFFSET,
-					    EDGETPU_REMAPPED_DATA_SIZE, MEMREMAP_WC);
+	etmdev->shared_mem_vaddr = memremap(etmdev->fw_region_paddr + etmdev->fw_region_size,
+					    etmdev->remapped_data_size, MEMREMAP_WC);
 	if (!etmdev->shared_mem_vaddr) {
 		dev_err(dev, "Shared memory remap failed");
 		if (etmdev->gsa_dev)
 			put_device(etmdev->gsa_dev);
 		return -EINVAL;
 	}
-	etmdev->shared_mem_paddr = r.start + EDGETPU_REMAPPED_DATA_OFFSET;
+	etmdev->shared_mem_paddr = etmdev->fw_region_paddr + etmdev->fw_region_size;
 
 	return 0;
 }
@@ -124,6 +119,8 @@ static void edgetpu_platform_cleanup_fw_region(struct edgetpu_mobile_platform_de
 		return;
 	memunmap(etmdev->shared_mem_vaddr);
 	etmdev->shared_mem_vaddr = NULL;
+	etmdev->remapped_data_addr = 0;
+	etmdev->remapped_data_size = 0;
 }
 
 static int mobile_check_ext_mailbox_args(const char *func, struct edgetpu_dev *etdev,
@@ -351,13 +348,13 @@ static int edgetpu_mobile_platform_probe(struct platform_device *pdev,
 					 /* Base virtual address (kernel address space) */
 					 etmdev->shared_mem_vaddr + EDGETPU_POOL_MEM_OFFSET,
 					 /* Base DMA address */
-					 EDGETPU_REMAPPED_DATA_ADDR + EDGETPU_POOL_MEM_OFFSET,
+					 etmdev->remapped_data_addr + EDGETPU_POOL_MEM_OFFSET,
 					 /* Base TPU address */
-					 EDGETPU_REMAPPED_DATA_ADDR + EDGETPU_POOL_MEM_OFFSET,
+					 etmdev->remapped_data_addr + EDGETPU_POOL_MEM_OFFSET,
 					 /* Base physical address */
 					 etmdev->shared_mem_paddr + EDGETPU_POOL_MEM_OFFSET,
 					 /* Size */
-					 EDGETPU_REMAPPED_DATA_SIZE - EDGETPU_POOL_MEM_OFFSET,
+					 etmdev->remapped_data_size - EDGETPU_POOL_MEM_OFFSET,
 					 /* Granularity */
 					 PAGE_SIZE);
 	if (ret) {
@@ -398,7 +395,7 @@ static int edgetpu_mobile_platform_probe(struct platform_device *pdev,
 	}
 #endif
 
-	edgetpu_mobile_get_telemetry_mem(etmdev);
+	edgetpu_mobile_set_telemetry_mem(etmdev);
 	ret = edgetpu_telemetry_init(etdev, etmdev->log_mem, etmdev->trace_mem);
 	if (ret)
 		goto out_remove_irq;
diff --git a/drivers/edgetpu/edgetpu-mobile-platform.h b/drivers/edgetpu/edgetpu-mobile-platform.h
index 55f465f..eee53ac 100644
--- a/drivers/edgetpu/edgetpu-mobile-platform.h
+++ b/drivers/edgetpu/edgetpu-mobile-platform.h
@@ -19,6 +19,14 @@
 #include "edgetpu-internal.h"
 #include "mobile-debug-dump.h"
 
+/*
+ * Log and trace buffers at the beginning of the remapped region,
+ * pool memory afterwards.
+ */
+#define EDGETPU_POOL_MEM_OFFSET                                                                    \
+	((EDGETPU_TELEMETRY_LOG_BUFFER_SIZE + EDGETPU_TELEMETRY_TRACE_BUFFER_SIZE) *               \
+	 EDGETPU_NUM_CORES)
+
 #define to_mobile_dev(etdev) container_of(etdev, struct edgetpu_mobile_platform_dev, edgetpu_dev)
 
 struct edgetpu_mobile_platform_pwr {
@@ -49,6 +57,10 @@ struct edgetpu_mobile_platform_dev {
 	phys_addr_t fw_region_paddr;
 	/* Size of the firmware region */
 	size_t fw_region_size;
+	/* TPU address from which the TPU CPU can access data in the remapped region */
+	tpu_addr_t remapped_data_addr;
+	/* Size of remapped DRAM data region */
+	size_t remapped_data_size;
 	/* Virtual address of the memory region shared with firmware */
 	void *shared_mem_vaddr;
 	/* Physical address of the memory region shared with firmware */
@@ -94,4 +106,7 @@ struct edgetpu_mobile_platform_dev {
 	int (*after_probe)(struct edgetpu_mobile_platform_dev *etmdev);
 };
 
+/* Sets up telemetry buffer address and size. */
+void edgetpu_mobile_set_telemetry_mem(struct edgetpu_mobile_platform_dev *etmdev);
+
 #endif /* __EDGETPU_MOBILE_PLATFORM_H__ */
diff --git a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-image-config.h b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-image-config.h
index 6c3333e..a3539a0 100644
--- a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-image-config.h
+++ b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-image-config.h
@@ -11,7 +11,7 @@
 #include <linux/types.h>
 
 #define GCIP_FW_NUM_VERSIONS 4
-#define GCIP_IMG_CFG_MAX_IOMMU_MAPPINGS 23
+#define GCIP_IMG_CFG_MAX_IOMMU_MAPPINGS 22
 #define GCIP_IMG_CFG_MAX_NS_IOMMU_MAPPINGS 5
 
 #define GCIP_FW_PRIV_LEVEL_GSA 0
@@ -40,6 +40,8 @@ struct gcip_image_config {
 		 */
 		__u32 image_config_value;
 	} iommu_mappings[GCIP_IMG_CFG_MAX_IOMMU_MAPPINGS];
+	__u32 remapped_data_start;
+	__u32 remapped_data_size;
 	__u32 num_ns_iommu_mappings;
 	__u32 ns_iommu_mappings[GCIP_IMG_CFG_MAX_NS_IOMMU_MAPPINGS];
 } __packed;
diff --git a/drivers/edgetpu/mobile-firmware.c b/drivers/edgetpu/mobile-firmware.c
index b326156..de6b4cd 100644
--- a/drivers/edgetpu/mobile-firmware.c
+++ b/drivers/edgetpu/mobile-firmware.c
@@ -20,11 +20,13 @@
 #include "edgetpu-firmware.h"
 #include "edgetpu-firmware-util.h"
 #include "edgetpu-internal.h"
+#include "edgetpu-iremap-pool.h"
 #include "edgetpu-kci.h"
 #include "edgetpu-mailbox.h"
 #include "edgetpu-mmu.h"
 #include "edgetpu-mobile-platform.h"
 #include "edgetpu-soc.h"
+#include "edgetpu-telemetry.h"
 #include "mobile-firmware.h"
 
 static int image_config_map(void *data, dma_addr_t daddr, phys_addr_t paddr, size_t size,
@@ -188,14 +190,105 @@ static int mobile_firmware_gsa_authenticate(struct edgetpu_mobile_platform_dev *
 	return ret;
 }
 
+static int mobile_firmware_update_remapped_data_region(struct edgetpu_dev *etdev)
+{
+	struct edgetpu_mobile_platform_dev *etmdev = to_mobile_dev(etdev);
+	struct gcip_image_config *config = mobile_firmware_get_image_config(etdev);
+	tpu_addr_t remapped_data_addr = EDGETPU_INSTRUCTION_REMAP_BASE + etmdev->fw_region_size;
+	size_t remapped_data_size = config->remapped_data_size ? config->remapped_data_size :
+								 EDGETPU_DEFAULT_REMAPPED_DATA_SIZE;
+	int ret;
+
+	if (etmdev->remapped_data_addr == remapped_data_addr &&
+	    etmdev->remapped_data_size == remapped_data_size)
+		return 0;
+
+	if (remapped_data_addr < EDGETPU_INSTRUCTION_REMAP_BASE + config->firmware_size ||
+	    config->firmware_base + etmdev->fw_region_size + remapped_data_size >
+		    config->remapped_region_start)
+		return -EINVAL;
+
+	etdev_dbg(etdev, "Moving remapped data from %#llx to %#llx\n", etmdev->remapped_data_addr,
+		  remapped_data_addr);
+
+	if (etmdev->shared_mem_vaddr) {
+		/* No need to free the VII queues since allocated groups will block fw loading. */
+		edgetpu_kci_release(etdev, etdev->etkci);
+		edgetpu_telemetry_exit(etdev);
+		edgetpu_iremap_pool_destroy(etdev);
+		memunmap(etmdev->shared_mem_vaddr);
+	}
+
+	etmdev->remapped_data_addr = remapped_data_addr;
+	etmdev->remapped_data_size = remapped_data_size;
+
+	etmdev->shared_mem_paddr = etmdev->fw_region_paddr + etmdev->fw_region_size;
+	etmdev->shared_mem_vaddr =
+		memremap(etmdev->shared_mem_paddr, etmdev->remapped_data_size, MEMREMAP_WC);
+	if (!etmdev->shared_mem_vaddr) {
+		etdev_err(etdev, "Shared memory remap failed\n");
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	ret = edgetpu_iremap_pool_create(etdev,
+					 /* Base virtual address (kernel address space) */
+					 etmdev->shared_mem_vaddr + EDGETPU_POOL_MEM_OFFSET,
+					 /* Base DMA address */
+					 etmdev->remapped_data_addr + EDGETPU_POOL_MEM_OFFSET,
+					 /* Base TPU address */
+					 etmdev->remapped_data_addr + EDGETPU_POOL_MEM_OFFSET,
+					 /* Base physical address */
+					 etmdev->shared_mem_paddr + EDGETPU_POOL_MEM_OFFSET,
+					 /* Size */
+					 etmdev->remapped_data_size - EDGETPU_POOL_MEM_OFFSET,
+					 /* Granularity */
+					 PAGE_SIZE);
+	if (ret) {
+		etdev_err(etdev, "failed to initialize remapped memory pool: %d", ret);
+		goto out_memunmap;
+	}
+
+	edgetpu_mobile_set_telemetry_mem(etmdev);
+	ret = edgetpu_telemetry_init(etdev, etmdev->log_mem, etmdev->trace_mem);
+	if (ret)
+		goto out_iremap_pool_destroy;
+
+	ret = edgetpu_kci_init(etdev->mailbox_manager, etdev->etkci);
+	if (ret)
+		goto out_telemetry_exit;
+
+	return 0;
+
+out_telemetry_exit:
+	edgetpu_telemetry_exit(etdev);
+out_iremap_pool_destroy:
+	edgetpu_iremap_pool_destroy(etdev);
+out_memunmap:
+	memunmap(etmdev->shared_mem_vaddr);
+
+out:
+	etmdev->remapped_data_addr = 0;
+	etmdev->remapped_data_size = 0;
+	etmdev->shared_mem_paddr = 0;
+	etmdev->shared_mem_vaddr = NULL;
+
+	return ret;
+}
+
 static int mobile_firmware_prepare_run(struct edgetpu_firmware *et_fw,
 				       struct edgetpu_firmware_buffer *fw_buf)
 {
 	struct edgetpu_dev *etdev = et_fw->etdev;
+	int ret;
 
 	/* Reset KCI mailbox before starting f/w, don't process anything old.*/
 	edgetpu_mailbox_reset(etdev->etkci->mailbox);
 
+	ret = mobile_firmware_update_remapped_data_region(etdev);
+	if (ret)
+		return ret;
+
 	edgetpu_soc_prepare_firmware(etdev);
 
 	return edgetpu_mobile_firmware_reset_cpu(etdev, false);
@@ -235,18 +328,10 @@ static int mobile_firmware_setup_buffer(struct edgetpu_firmware *et_fw,
 		return -EINVAL;
 	}
 
-	image_vaddr = memremap(etmdev->fw_region_paddr,
-			       etmdev->fw_region_size, MEMREMAP_WC);
-	if (!image_vaddr) {
-		etdev_err(etdev, "memremap failed\n");
-		return -ENOMEM;
-	}
-
 	hdr = (struct mobile_image_header *)fw_buf->vaddr;
 	if (hdr->common.Magic != EDGETPU_MOBILE_FW_MAGIC) {
 		etdev_err(etdev, "Invalid firmware header magic value %#08x\n", hdr->common.Magic);
-		ret = -EINVAL;
-		goto out;
+		return -EINVAL;
 	}
 
 	switch (hdr->common.Generation) {
@@ -259,7 +344,19 @@ static int mobile_firmware_setup_buffer(struct edgetpu_firmware *et_fw,
 	default:
 		etdev_err(etdev, "Invalid header generation identifier (%d)\n",
 			  hdr->common.Generation);
-		goto out;
+		return -EINVAL;
+	}
+
+	etmdev->fw_region_paddr = image_config->firmware_base;
+	etmdev->fw_region_size =
+		image_config->remapped_data_start ?
+			image_config->remapped_data_start - image_config->firmware_base :
+			EDGETPU_DEFAULT_FW_SIZE_MAX;
+
+	image_vaddr = memremap(etmdev->fw_region_paddr, etmdev->fw_region_size, MEMREMAP_WC);
+	if (!image_vaddr) {
+		etdev_err(etdev, "FW region memremap failed\n");
+		return -ENOMEM;
 	}
 
 	memcpy(&etdev->fw_version, &image_config->firmware_versions, sizeof(etdev->fw_version));
@@ -309,13 +406,14 @@ out:
 static void program_iremap_csr(struct edgetpu_dev *etdev)
 {
 	const int ctx_id = 0, sid0 = 0x30, sid1 = 0x34;
-	phys_addr_t fw_paddr = EDGETPU_INSTRUCTION_REMAP_BASE;
 
 	edgetpu_dev_write_32(etdev, EDGETPU_REG_INSTRUCTION_REMAP_SECURITY, (ctx_id << 16) | sid0);
 	edgetpu_dev_write_32(etdev, EDGETPU_REG_INSTRUCTION_REMAP_SECURITY + 8,
 			     (ctx_id << 16) | sid1);
-	edgetpu_dev_write_32(etdev, EDGETPU_REG_INSTRUCTION_REMAP_NEW_BASE, fw_paddr);
-	edgetpu_dev_write_32(etdev, EDGETPU_REG_INSTRUCTION_REMAP_NEW_BASE + 8, fw_paddr);
+	edgetpu_dev_write_32(etdev, EDGETPU_REG_INSTRUCTION_REMAP_NEW_BASE,
+			     EDGETPU_INSTRUCTION_REMAP_BASE);
+	edgetpu_dev_write_32(etdev, EDGETPU_REG_INSTRUCTION_REMAP_NEW_BASE + 8,
+			     EDGETPU_INSTRUCTION_REMAP_BASE);
 
 	edgetpu_dev_write_32(etdev, EDGETPU_REG_INSTRUCTION_REMAP_LIMIT,
 			     EDGETPU_INSTRUCTION_REMAP_BASE + SZ_32M);
diff --git a/drivers/edgetpu/rio/config.h b/drivers/edgetpu/rio/config.h
index d358953..5b4f0f2 100644
--- a/drivers/edgetpu/rio/config.h
+++ b/drivers/edgetpu/rio/config.h
@@ -23,23 +23,17 @@
 /* Reserved VCID that uses the extra partition. */
 #define EDGETPU_VCID_EXTRA_PARTITION 0
 
+/* Pre-allocate 1 IOMMU domain per VCID */
+#define EDGETPU_NUM_PREALLOCATED_DOMAINS EDGETPU_NUM_VCIDS
+
 /* Placeholder value */
 #define EDGETPU_TZ_MAILBOX_ID 31
 
-/*
- * Size of the area in remapped DRAM reserved for firmware code and internal
- * data. This must match the firmware's linker file.
- */
-#define EDGETPU_FW_SIZE_MAX			0x100000
-
-/* Data in remapped DRAM starts after firmware code and internal data */
-#define EDGETPU_REMAPPED_DATA_OFFSET		EDGETPU_FW_SIZE_MAX
+/* Default size of the area in remapped DRAM reserved for firmware code and internal data. */
+#define EDGETPU_DEFAULT_FW_SIZE_MAX 0x100000
 
-/*
- * Size of remapped DRAM data region. This must match the firmware's linker
- * file
- */
-#define EDGETPU_REMAPPED_DATA_SIZE		0x100000
+/* Default size of remapped DRAM data region. */
+#define EDGETPU_DEFAULT_REMAPPED_DATA_SIZE 0x100000
 
 /*
  * Instruction remap registers make carveout memory appear at address
@@ -47,8 +41,12 @@
  */
 #define EDGETPU_INSTRUCTION_REMAP_BASE		0x10000000
 
-/* Address from which the TPU CPU can access data in the remapped region */
-#define EDGETPU_REMAPPED_DATA_ADDR (EDGETPU_INSTRUCTION_REMAP_BASE + EDGETPU_REMAPPED_DATA_OFFSET)
+/*
+ * Default address from which the TPU CPU can access data in the remapped region.
+ * Data in remapped DRAM starts after firmware code and internal data.
+ */
+#define EDGETPU_DEFAULT_REMAPPED_DATA_ADDR                                                         \
+	(EDGETPU_INSTRUCTION_REMAP_BASE + EDGETPU_DEFAULT_FW_SIZE_MAX)
 
 /*
  * Size of memory for FW accessible debug dump segments