[Copybara Auto Merge] Merge branch zuma into android14-gs-pixel-5.15-udc-qpr1

edgetpu: increase timeout of power OFF polling
Increase the total waiting time from 5ms * 50 to 5ms * 1000.
Bug: 293673694
(cherry picked from commit d2bafb6b23a9813c335f5f333b0696a57b86ffa9)
edgetpu: support 36-bit IOVAs
Roll-up the following CLs for support of 36-bit IOVAs:
ee216f610 edgetpu: stop using best-fit IOVA alloc algorithm
81ef0fa88 edgetpu: select 32-bit restricted IOVA for CC-accessible mapping
5a48fb7780 gcip: iommu: unmap all mapped DMA segments of > 32-bit sized mapping
5cb80704fd gcip: iommu fix DMA mapping scatterlists for > 32-bit buffer sizes
5a9496e877 gcip: iommu: optionally restrict IOVA allocation to 32-bit subrange
2a890c53b9 gcip: iommu parse DMA windows up to 64 bits
cbc55033ee gcip: include: iommu: add flag to restrict mappings to 32-bit window
Bug: 278822501

Signed-off-by: Zuma copybara merger <zuma-automerger@google.com>
GitOrigin-RevId: e16f5a97133424fa7cd7015d1d9b558087fdd4e7
Change-Id: I082980d340f199e9e17e18d20a350cc22007759c

4 files changed, 89 insertions, 30 deletions

diff --git a/drivers/edgetpu/edgetpu-google-iommu.c b/drivers/edgetpu/edgetpu-google-iommu.c
index a432783..1e018ab 100644
--- a/drivers/edgetpu/edgetpu-google-iommu.c
+++ b/drivers/edgetpu/edgetpu-google-iommu.c
@@ -242,7 +242,6 @@ int edgetpu_mmu_attach(struct edgetpu_dev *etdev, void *mmu_info)
 		etdev_err(etdev, "Unable create domain pool (%d)\n", ret);
 		goto err_free_etiommu;
 	}
-	gcip_iommu_domain_pool_enable_best_fit_algo(&etiommu->domain_pool);
 
 	idr_init(&etiommu->domain_id_pool);
 	mutex_init(&etiommu->pool_lock);
@@ -344,7 +343,8 @@ int edgetpu_mmu_map_sgt(struct edgetpu_dev *etdev, struct sg_table *sgt,
 	u64 gcip_map_flags =
 		GCIP_MAP_FLAGS_DMA_DIRECTION_TO_FLAGS(dir) |
 		GCIP_MAP_FLAGS_DMA_COHERENT_TO_FLAGS((mmu_flags & EDGETPU_MMU_COHERENT) != 0) |
-		GCIP_MAP_FLAGS_DMA_ATTR_TO_FLAGS(dma_attrs);
+		GCIP_MAP_FLAGS_DMA_ATTR_TO_FLAGS(dma_attrs) |
+		GCIP_MAP_FLAGS_RESTRICT_IOVA_TO_FLAGS(!(mmu_flags & EDGETPU_MMU_CC_NO_ACCESS));
 	int ret;
 
 	gdomain = get_domain_by_context_id(etdev, context_id);
diff --git a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c
index 50170c6..5fa3f87 100644
--- a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c
+++ b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c
@@ -11,6 +11,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/genalloc.h>
 #include <linux/iova.h>
+#include <linux/limits.h>
 #include <linux/log2.h>
 #include <linux/of.h>
 #include <linux/scatterlist.h>
@@ -32,6 +33,10 @@
 #define GCIP_MAP_FLAGS_GET_DMA_DIRECTION(flags) GCIP_MAP_FLAGS_GET_VALUE(DMA_DIRECTION, flags)
 #define GCIP_MAP_FLAGS_GET_DMA_COHERENT(flags) GCIP_MAP_FLAGS_GET_VALUE(DMA_COHERENT, flags)
 #define GCIP_MAP_FLAGS_GET_DMA_ATTR(flags) GCIP_MAP_FLAGS_GET_VALUE(DMA_ATTR, flags)
+#define GCIP_MAP_FLAGS_GET_RESTRICT_IOVA(flags) GCIP_MAP_FLAGS_GET_VALUE(RESTRICT_IOVA, flags)
+
+/* Restricted IOVA ceiling is for components with 32-bit DMA windows */
+#define GCIP_RESTRICT_IOVA_CEILING	UINT_MAX
 
 /**
  * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API
@@ -104,9 +109,14 @@ static void iovad_enable_best_fit_algo(struct gcip_iommu_domain *domain)
 #endif /* HAS_IOVAD_BEST_FIT_ALGO */
 }
 
-static dma_addr_t iovad_alloc_iova_space(struct gcip_iommu_domain *domain, size_t size)
+static dma_addr_t iovad_alloc_iova_space(struct gcip_iommu_domain *domain, size_t size,
+					 bool restrict_iova)
 {
-	unsigned long iova, shift = gcip_iommu_domain_shift(domain);
+	unsigned long iova_pfn, shift = gcip_iommu_domain_shift(domain);
+	dma_addr_t iova_ceiling =
+		restrict_iova ?
+		min_t(dma_addr_t, GCIP_RESTRICT_IOVA_CEILING, domain->domain_pool->last_daddr)
+		: domain->domain_pool->last_daddr;
 
 	size = size >> shift;
 
@@ -122,10 +132,8 @@ static dma_addr_t iovad_alloc_iova_space(struct gcip_iommu_domain *domain, size_
 		size = roundup_pow_of_two(size);
 #endif
 
-	iova = alloc_iova_fast(&domain->iova_space.iovad, size,
-			       domain->domain_pool->last_daddr >> shift, true);
-
-	return (dma_addr_t)iova << shift;
+	iova_pfn = alloc_iova_fast(&domain->iova_space.iovad, size, iova_ceiling >> shift, true);
+	return (dma_addr_t)iova_pfn << shift;
 }
 
 static void iovad_free_iova_space(struct gcip_iommu_domain *domain, dma_addr_t iova, size_t size)
@@ -145,10 +153,14 @@ static const struct gcip_iommu_domain_ops iovad_ops = {
 static int mem_pool_initialize_domain(struct gcip_iommu_domain *domain)
 {
 	struct gcip_iommu_domain_pool *dpool = domain->domain_pool;
+	size_t size = dpool->size;
 	int ret;
 
+	/* Restrict mem_pool IOVAs to 32 bits. */
+	if (dpool->base_daddr + size > UINT_MAX)
+		size = UINT_MAX - dpool->base_daddr;
 	ret = gcip_mem_pool_init(&domain->iova_space.mem_pool, dpool->dev, dpool->base_daddr,
-				 dpool->size, dpool->granule);
+				 size, dpool->granule);
 
 	return ret;
 }
@@ -163,8 +175,12 @@ static void mem_pool_enable_best_fit_algo(struct gcip_iommu_domain *domain)
 	gen_pool_set_algo(domain->iova_space.mem_pool.gen_pool, gen_pool_best_fit, NULL);
 }
 
-static dma_addr_t mem_pool_alloc_iova_space(struct gcip_iommu_domain *domain, size_t size)
+static dma_addr_t mem_pool_alloc_iova_space(struct gcip_iommu_domain *domain, size_t size,
+					    bool restrict_iova)
 {
+	/* mem pool IOVA allocs are currently always restricted. */
+	if (!restrict_iova)
+		dev_warn_once(domain->dev, "IOVA size always restricted to 32-bit");
 	return (dma_addr_t)gcip_mem_pool_alloc(&domain->iova_space.mem_pool, size);
 }
 
@@ -242,7 +258,8 @@ int gcip_iommu_domain_pool_init(struct gcip_iommu_domain_pool *pool, struct devi
 				dma_addr_t base_daddr, size_t iova_space_size, size_t granule,
 				unsigned int num_domains, enum gcip_iommu_domain_type domain_type)
 {
-	const __be32 *user_window;
+	const __be32 *user_window, *prop;
+	u32 cells;
 	int ret;
 
 	ret = gcip_domain_pool_init(dev, &pool->domain_pool, num_domains);
@@ -261,8 +278,17 @@ int gcip_iommu_domain_pool_init(struct gcip_iommu_domain_pool *pool, struct devi
 		if (!user_window) {
 			dev_warn(dev, "Failed to find gcip-dma-window property");
 		} else {
-			pool->base_daddr = of_read_number(user_window, 1);
-			pool->size = of_read_number(user_window + 1, 1);
+			prop = of_get_property(dev->of_node, "#dma-address-cells", NULL);
+			cells = prop ? be32_to_cpup(prop) : of_n_addr_cells(dev->of_node);
+			if (!cells)
+				cells = 1;
+			pool->base_daddr = of_read_number(user_window, cells);
+			user_window += cells;
+			prop = of_get_property(dev->of_node, "#dma-size-cells", NULL);
+			cells = prop ? be32_to_cpup(prop) : of_n_size_cells(dev->of_node);
+			if (!cells)
+				cells = 1;
+			pool->size = of_read_number(user_window, cells);
 		}
 	}
 
@@ -370,11 +396,13 @@ unsigned int gcip_iommu_domain_map_sg(struct gcip_iommu_domain *domain, struct s
 	enum dma_data_direction dir = GCIP_MAP_FLAGS_GET_DMA_DIRECTION(gcip_map_flags);
 	bool coherent = GCIP_MAP_FLAGS_GET_DMA_COHERENT(gcip_map_flags);
 	unsigned long attrs = GCIP_MAP_FLAGS_GET_DMA_ATTR(gcip_map_flags);
+	bool restrict_iova = GCIP_MAP_FLAGS_GET_RESTRICT_IOVA(gcip_map_flags);
 	int i, prot = dma_info_to_prot(dir, coherent, attrs);
 	struct scatterlist *sg;
 	dma_addr_t iova;
 	size_t iova_len = 0;
-	ssize_t ret;
+	ssize_t map_size;
+	int ret;
 
 	if (gcip_iommu_domain_is_legacy_mode(domain))
 		return dma_iommu_map_sg(domain, sgl, nents, dir, attrs, prot);
@@ -384,7 +412,8 @@ unsigned int gcip_iommu_domain_map_sg(struct gcip_iommu_domain *domain, struct s
 		iova_len += sg->length;
 
 	/* Allocates one continuous IOVA. */
-	iova = domain->ops->alloc_iova_space(domain, gcip_iommu_domain_align(domain, iova_len));
+	iova = domain->ops->alloc_iova_space(domain, gcip_iommu_domain_align(domain, iova_len),
+					     restrict_iova);
 	if (!iova)
 		return 0;
 
@@ -398,16 +427,30 @@ unsigned int gcip_iommu_domain_map_sg(struct gcip_iommu_domain *domain, struct s
 	 * Note: Before Linux 5.15, its return type was `size_t` and it returned 0 on failure.
 	 *       To make it compatible with those old versions, we should cast the return value.
 	 */
-	ret = (ssize_t)iommu_map_sg(domain->domain, iova, sgl, nents, prot);
-	if (ret < 0 || ret < iova_len)
+	map_size = (ssize_t)iommu_map_sg(domain->domain, iova, sgl, nents, prot);
+	if (map_size < 0 || map_size < iova_len)
 		goto err_free_iova;
 
-	/* Fills out the mapping information. */
-	sg_dma_address(sgl) = iova;
-	sg_dma_len(sgl) = iova_len;
+	/*
+	 * Fills out the mapping information. Each entry can be max UINT_MAX bytes, floored
+	 * to the pool granule size.
+	 */
+	ret = 0;
+	sg = sgl;
+	while (iova_len) {
+		size_t segment_len = min_t(size_t, iova_len,
+					   UINT_MAX & ~(domain->domain_pool->granule - 1));
+
+		sg_dma_address(sg) = iova;
+		sg_dma_len(sg) = segment_len;
+		iova += segment_len;
+		iova_len -= segment_len;
+		ret++;
+		sg = sg_next(sg);
+	}
 
-	/* As it put the whole mapping information to the first segment, it should return 1. */
-	return 1;
+	/* Return # of sg entries filled out above. */
+	return ret;
 
 err_free_iova:
 	domain->ops->free_iova_space(domain, iova, gcip_iommu_domain_align(domain, iova_len));
@@ -417,8 +460,10 @@ err_free_iova:
 void gcip_iommu_domain_unmap_sg(struct gcip_iommu_domain *domain, struct scatterlist *sgl,
 				int nents, u64 gcip_map_flags)
 {
-	dma_addr_t iova;
-	size_t iova_len;
+	dma_addr_t iova = sg_dma_address(sgl);
+	size_t iova_len = 0;
+	struct scatterlist *sg;
+	int i;
 
 	if (gcip_iommu_domain_is_legacy_mode(domain)) {
 		enum dma_data_direction dir = GCIP_MAP_FLAGS_GET_DMA_DIRECTION(gcip_map_flags);
@@ -428,8 +473,13 @@ void gcip_iommu_domain_unmap_sg(struct gcip_iommu_domain *domain, struct scatter
 		return;
 	}
 
-	iova = sg_dma_address(sgl);
-	iova_len = sg_dma_len(sgl);
+	for_each_sg(sgl, sg, nents, i) {
+		uint s_len = sg_dma_len(sg);
+
+		if (!s_len)
+			break;
+		iova_len += s_len;
+	}
 
 	iommu_unmap(domain->domain, iova, iova_len);
 	domain->ops->free_iova_space(domain, iova, gcip_iommu_domain_align(domain, iova_len));
diff --git a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-iommu.h b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-iommu.h
index 34f6efa..fa8fbff 100644
--- a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-iommu.h
+++ b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-iommu.h
@@ -57,6 +57,12 @@
 #define GCIP_MAP_FLAGS_DMA_ATTR_BIT_SIZE 10
 #define GCIP_MAP_FLAGS_DMA_ATTR_TO_FLAGS(attr) ((u64)(attr) << GCIP_MAP_FLAGS_DMA_ATTR_OFFSET)
 
+#define GCIP_MAP_FLAGS_RESTRICT_IOVA_OFFSET                                                        \
+	(GCIP_MAP_FLAGS_DMA_ATTR_OFFSET + GCIP_MAP_FLAGS_DMA_ATTR_BIT_SIZE)
+#define GCIP_MAP_FLAGS_RESTRICT_IOVA_BIT_SIZE 1
+#define GCIP_MAP_FLAGS_RESTRICT_IOVA_TO_FLAGS(restrict)                                            \
+	((u64)(restrict) << GCIP_MAP_FLAGS_RESTRICT_IOVA_OFFSET)
+
 struct gcip_iommu_domain_ops;
 
 /*
@@ -126,8 +132,9 @@ struct gcip_iommu_domain_ops {
 	 * Only domains which are allocated after calling this callback will be affected.
 	 */
 	void (*enable_best_fit_algo)(struct gcip_iommu_domain *domain);
-	/* Allocates @size of buffer and returns its IOVA. */
-	dma_addr_t (*alloc_iova_space)(struct gcip_iommu_domain *domain, size_t size);
+	/* Allocates @size of IOVA space, optionally restricted to 32 bits, returns start IOVA. */
+	dma_addr_t (*alloc_iova_space)(struct gcip_iommu_domain *domain, size_t size,
+				       bool restrict_iova);
 	/* Releases @size of buffer which was allocated to @iova. */
 	void (*free_iova_space)(struct gcip_iommu_domain *domain, dma_addr_t iova, size_t size);
 };
@@ -246,7 +253,9 @@ static inline bool gcip_iommu_domain_is_legacy_mode(struct gcip_iommu_domain *do
  *   [12:3]  - DMA_ATTR:
  *               Not used in the non-legacy mode.
  *               (See [REDACTED]
- *   [63:13] - RESERVED
+ *   [13:13] - RESTRICT_IOVA:
+ *               Restrict the IOVA assignment to 32 bit address window.
+ *   [63:14] - RESERVED
  *               Set RESERVED bits to 0 to ensure backwards compatibility.
  *
  * One can use `GCIP_MAP_FLAGS_DMA_*_TO_FLAGS` macros to generate a flag.
diff --git a/drivers/edgetpu/mobile-pm.c b/drivers/edgetpu/mobile-pm.c
index 53571e0..eb70c91 100644
--- a/drivers/edgetpu/mobile-pm.c
+++ b/drivers/edgetpu/mobile-pm.c
@@ -28,7 +28,7 @@
 #include "edgetpu-pm.c"
 #include "edgetpu-soc.h"
 
-#define BLOCK_DOWN_RETRY_TIMES 50
+#define BLOCK_DOWN_RETRY_TIMES 1000
 #define BLOCK_DOWN_MIN_DELAY_US 1000
 #define BLOCK_DOWN_MAX_DELAY_US 1500