[Copybara Auto Merge] Merge branch zuma into android14-gs-pixel-5.15

gcip: Fix version check for best-fit IOVA allocate
edgetpu: Handle missing scmversion
Bug: 277787303
gcip: Disable best-fit algo on kernel > 5.15
Bug: 277649169
gcip: check acquired lock is atomic
Bug: 275652066
gcip: Conditionally use alloc_iova_fast workaround
edgetpu: Pass down system properties to firmware
Bug: 273578844
gcip: check acquired lock is atomic
Bug: 275652066 (repeat)

Signed-off-by: Zuma copybara merger <zuma-automerger@google.com>
GitOrigin-RevId: 2874dde46f432d46dec6f58cefca5a1529f08f29
Change-Id: I684beb37926cf4bf1c265a9d79cf2ec788fcb523

12 files changed, 138 insertions, 19 deletions

diff --git a/drivers/edgetpu/edgetpu-core.c b/drivers/edgetpu/edgetpu-core.c
index dc6af23..fe441a5 100644
--- a/drivers/edgetpu/edgetpu-core.c
+++ b/drivers/edgetpu/edgetpu-core.c
@@ -452,6 +452,7 @@ int edgetpu_device_add(struct edgetpu_dev *etdev,
 	etdev->state = ETDEV_STATE_NOFW;
 	etdev->freq_count = 0;
 	mutex_init(&etdev->freq_lock);
+	mutex_init(&etdev->device_prop.lock);
 	ret = edgetpu_soc_init(etdev);
 	if (ret)
 		return ret;
diff --git a/drivers/edgetpu/edgetpu-firmware.c b/drivers/edgetpu/edgetpu-firmware.c
index 5e442d0..81e6cad 100644
--- a/drivers/edgetpu/edgetpu-firmware.c
+++ b/drivers/edgetpu/edgetpu-firmware.c
@@ -188,6 +188,10 @@ static int edgetpu_firmware_handshake(struct edgetpu_firmware *et_fw)
 	if (ret)
 		etdev_warn(etdev, "thermal restore error: %d", ret);
 
+	ret = edgetpu_kci_set_device_properties(etdev->etkci, &etdev->device_prop);
+	if (ret)
+		dev_warn(etdev->dev, "Failed to pass device_prop to fw: %d\n", ret);
+
 	/* Set debug dump buffer in FW */
 	edgetpu_get_debug_dump(etdev, 0);
 	return 0;
diff --git a/drivers/edgetpu/edgetpu-fs.c b/drivers/edgetpu/edgetpu-fs.c
index c626500..dac598b 100644
--- a/drivers/edgetpu/edgetpu-fs.c
+++ b/drivers/edgetpu/edgetpu-fs.c
@@ -655,6 +655,26 @@ static int edgetpu_ioctl_test_external(struct edgetpu_client *client,
 	return ret;
 }
 
+static int
+edgetpu_ioctl_set_device_properties(struct edgetpu_dev *etdev,
+				    struct edgetpu_set_device_properties_ioctl __user *argp)
+{
+	struct edgetpu_dev_prop *device_prop = &etdev->device_prop;
+	struct edgetpu_set_device_properties_ioctl ibuf;
+
+	if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
+		return -EFAULT;
+
+	mutex_lock(&device_prop->lock);
+
+	memcpy(&device_prop->opaque, &ibuf.opaque, sizeof(device_prop->opaque));
+	device_prop->initialized = true;
+
+	mutex_unlock(&device_prop->lock);
+
+	return 0;
+}
+
 long edgetpu_ioctl(struct file *file, uint cmd, ulong arg)
 {
 	struct edgetpu_client *client = file->private_data;
@@ -749,6 +769,9 @@ long edgetpu_ioctl(struct file *file, uint cmd, ulong arg)
 	case EDGETPU_TEST_EXTERNAL:
 		ret = edgetpu_ioctl_test_external(client, argp);
 		break;
+	case EDGETPU_SET_DEVICE_PROPERTIES:
+		ret = edgetpu_ioctl_set_device_properties(client->etdev, argp);
+		break;
 	default:
 		return -ENOTTY; /* unknown command */
 	}
diff --git a/drivers/edgetpu/edgetpu-internal.h b/drivers/edgetpu/edgetpu-internal.h
index 0ed3e55..747150f 100644
--- a/drivers/edgetpu/edgetpu-internal.h
+++ b/drivers/edgetpu/edgetpu-internal.h
@@ -163,6 +163,18 @@ enum edgetpu_dev_state {
 	ETDEV_STATE_BAD = 3,	/* firmware/device is in unusable state. */
 };
 
+/*
+ * struct edgetpu_dev_prop
+ * @lock:		Protects initialized and opaque.
+ * @initialized:	Set to true when this struct object is initialized.
+ * @opaque:		Device properties defined by runtime and firmware.
+ */
+struct edgetpu_dev_prop {
+	struct mutex lock;
+	bool initialized;
+	u8 opaque[EDGETPU_DEV_PROP_SIZE];
+};
+
 /* a mark to know whether we read valid versions from the firmware header */
 #define EDGETPU_INVALID_KCI_VERSION (~0u)
 
@@ -210,6 +222,8 @@ struct edgetpu_dev {
 	/* version read from the firmware binary file */
 	struct edgetpu_fw_version fw_version;
 	atomic_t job_count;	/* times joined to a device group */
+	/* To save device properties */
+	struct edgetpu_dev_prop device_prop;
 
 	/* counts of error events */
 	uint firmware_crash_count;
diff --git a/drivers/edgetpu/edgetpu-kci.c b/drivers/edgetpu/edgetpu-kci.c
index 44d8542..7001329 100644
--- a/drivers/edgetpu/edgetpu-kci.c
+++ b/drivers/edgetpu/edgetpu-kci.c
@@ -684,6 +684,28 @@ int edgetpu_kci_thermal_control(struct edgetpu_dev *etdev, bool enable)
 	return gcip_kci_send_cmd(etdev->etkci->kci, &cmd);
 }
 
+int edgetpu_kci_set_device_properties(struct edgetpu_kci *etkci, struct edgetpu_dev_prop *dev_prop)
+{
+	struct gcip_kci_command_element cmd = {
+		.code = GCIP_KCI_CODE_SET_DEVICE_PROPERTIES,
+	};
+	int ret = 0;
+
+	if (!etkci || !etkci->kci)
+		return -ENODEV;
+
+	mutex_lock(&dev_prop->lock);
+	if (!dev_prop->initialized)
+		goto out;
+
+	ret = edgetpu_kci_send_cmd_with_data(etkci, &cmd, &dev_prop->opaque,
+					     sizeof(dev_prop->opaque));
+
+out:
+	mutex_unlock(&dev_prop->lock);
+	return ret;
+}
+
 int edgetpu_kci_resp_rkci_ack(struct edgetpu_dev *etdev, struct gcip_kci_response_element *rkci_cmd)
 {
 	struct gcip_kci_command_element cmd = {
diff --git a/drivers/edgetpu/edgetpu-kci.h b/drivers/edgetpu/edgetpu-kci.h
index ec9154e..25674d7 100644
--- a/drivers/edgetpu/edgetpu-kci.h
+++ b/drivers/edgetpu/edgetpu-kci.h
@@ -209,6 +209,13 @@ int edgetpu_kci_firmware_tracing_level(void *data, unsigned long level,
 int edgetpu_kci_thermal_control(struct edgetpu_dev *etdev, bool enable);
 
 /*
+ * Sends device properties to firmware.
+ * The KCI command will be sent only when @device_prop is initialized.
+ */
+int edgetpu_kci_set_device_properties(struct edgetpu_kci *gkci,
+				      struct edgetpu_dev_prop *device_prop);
+
+/*
  * Send an ack to the FW after handling a reverse KCI request.
  *
  * The FW may wait for a response from the kernel for an RKCI request so a
diff --git a/drivers/edgetpu/edgetpu-mobile-platform.c b/drivers/edgetpu/edgetpu-mobile-platform.c
index 79a27d6..0147d9d 100644
--- a/drivers/edgetpu/edgetpu-mobile-platform.c
+++ b/drivers/edgetpu/edgetpu-mobile-platform.c
@@ -279,7 +279,7 @@ static void edgetpu_platform_remove_irq(struct edgetpu_mobile_platform_dev *etmd
 static inline const char *get_driver_commit(void)
 {
 #if IS_ENABLED(CONFIG_MODULE_SCMVERSION)
-	return THIS_MODULE->scmversion;
+	return THIS_MODULE->scmversion ?: "scmversion missing";
 #elif defined(GIT_REPO_TAG)
 	return GIT_REPO_TAG;
 #else
diff --git a/drivers/edgetpu/edgetpu.h b/drivers/edgetpu/edgetpu.h
index ad9a2ce..f5e3dec 100644
--- a/drivers/edgetpu/edgetpu.h
+++ b/drivers/edgetpu/edgetpu.h
@@ -608,4 +608,18 @@ struct edgetpu_test_ext_ioctl {
 #define EDGETPU_TEST_EXTERNAL \
 	_IOW(EDGETPU_IOCTL_BASE, 33, struct edgetpu_test_ext_ioctl)
 
+/* The size of device properties pre-agreed with firmware */
+#define EDGETPU_DEV_PROP_SIZE 256
+/*
+ * struct edgetpu_set_device_properties_ioctl
+ * @opaque:		Device properties defined by runtime and firmware.
+ */
+struct edgetpu_set_device_properties_ioctl {
+	__u8 opaque[EDGETPU_DEV_PROP_SIZE];
+};
+
+/* Registers device properties which will be passed down to firmware on boot. */
+#define EDGETPU_SET_DEVICE_PROPERTIES                                                              \
+	_IOW(EDGETPU_IOCTL_BASE, 34, struct edgetpu_set_device_properties_ioctl)
+
 #endif /* __EDGETPU_H__ */
diff --git a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c
index 979eb86..ab0ef51 100644
--- a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c
+++ b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c
@@ -16,12 +16,19 @@
 #include <linux/of.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
+#include <linux/version.h>
 
 #include <gcip/gcip-domain-pool.h>
 #include <gcip/gcip-iommu.h>
 #include <gcip/gcip-mem-pool.h>
 
-#define HAS_IOVAD_BEST_FIT_ALGO (IS_ENABLED(CONFIG_GCIP_TEST) || IS_ENABLED(CONFIG_ANDROID))
+/*
+ * TODO(b/277649169) Best fit IOVA allocator was removed in 6.1 GKI
+ * The API needs to either be upstreamed, integrated into this driver, or disabled for 6.1
+ * compatibility. For now, disable best-fit on all non-Android kernels and any GKI > 5.15.
+ */
+#define HAS_IOVAD_BEST_FIT_ALGO (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) && \
+				 (IS_ENABLED(CONFIG_GCIP_TEST) || IS_ENABLED(CONFIG_ANDROID)))
 
 /* Macros for manipulating @gcip_map_flags parameter. */
 #define GCIP_MAP_FLAGS_GET_VALUE(ATTR, flags)                                                      \
@@ -104,8 +111,17 @@ static dma_addr_t iovad_alloc_iova_space(struct gcip_iommu_domain *domain, size_
 
 	size = size >> shift;
 
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 17, 0)
+	/*
+	 * alloc_iova_fast() makes use of a cache of recently freed IOVA pages which does not
+	 * behave correctly for non-power-of-two amounts of pages. Round up the number of
+	 * pages being allocated to ensure it's a safe number of pages.
+	 *
+	 * This rounding is done automatically as of 5.17
+	 */
 	if (size < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
 		size = roundup_pow_of_two(size);
+#endif
 
 	iova = alloc_iova_fast(&domain->iova_space.iovad, size,
 			       domain->domain_pool->last_daddr >> shift, true);
diff --git a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-kci.c b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-kci.c
index 58852d5..33f8021 100644
--- a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-kci.c
+++ b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-kci.c
@@ -36,7 +36,7 @@ static void gcip_kci_inc_cmd_queue_tail(struct gcip_mailbox *mailbox, u32 inc)
 	kci->ops->inc_cmd_queue_tail(kci, inc);
 }
 
-static int gcip_kci_acquire_cmd_queue_lock(struct gcip_mailbox *mailbox, bool try)
+static int gcip_kci_acquire_cmd_queue_lock(struct gcip_mailbox *mailbox, bool try, bool *atomic)
 {
 	struct gcip_kci *kci = gcip_mailbox_get_data(mailbox);
 
@@ -102,10 +102,12 @@ static void gcip_kci_inc_resp_queue_head(struct gcip_mailbox *mailbox, u32 inc)
 	kci->ops->inc_resp_queue_head(kci, inc);
 }
 
-static int gcip_kci_acquire_resp_queue_lock(struct gcip_mailbox *mailbox, bool try)
+static int gcip_kci_acquire_resp_queue_lock(struct gcip_mailbox *mailbox, bool try, bool *atomic)
 {
 	struct gcip_kci *kci = gcip_mailbox_get_data(mailbox);
 
+	*atomic = true;
+
 	if (try)
 		return spin_trylock(&kci->resp_queue_lock);
 
diff --git a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c
index 6d20771..c7aa921 100644
--- a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c
+++ b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c
@@ -24,7 +24,8 @@
 #define GET_CMD_QUEUE_HEAD() mailbox->ops->get_cmd_queue_head(mailbox)
 #define GET_CMD_QUEUE_TAIL() mailbox->ops->get_cmd_queue_tail(mailbox)
 #define INC_CMD_QUEUE_TAIL(inc) mailbox->ops->inc_cmd_queue_tail(mailbox, inc)
-#define ACQUIRE_CMD_QUEUE_LOCK(try) mailbox->ops->acquire_cmd_queue_lock(mailbox, try)
+#define ACQUIRE_CMD_QUEUE_LOCK(try, atomic)                                                        \
+	mailbox->ops->acquire_cmd_queue_lock(mailbox, try, atomic)
 #define RELEASE_CMD_QUEUE_LOCK() mailbox->ops->release_cmd_queue_lock(mailbox)
 
 #define GET_CMD_ELEM_SEQ(cmd) mailbox->ops->get_cmd_elem_seq(mailbox, cmd)
@@ -35,7 +36,8 @@
 #define GET_RESP_QUEUE_HEAD() mailbox->ops->get_resp_queue_head(mailbox)
 #define INC_RESP_QUEUE_HEAD(inc) mailbox->ops->inc_resp_queue_head(mailbox, inc)
 #define GET_RESP_QUEUE_TAIL() mailbox->ops->get_resp_queue_tail(mailbox)
-#define ACQUIRE_RESP_QUEUE_LOCK(try) mailbox->ops->acquire_resp_queue_lock(mailbox, try)
+#define ACQUIRE_RESP_QUEUE_LOCK(try, atomic)                                                       \
+	mailbox->ops->acquire_resp_queue_lock(mailbox, try, atomic)
 #define RELEASE_RESP_QUEUE_LOCK() mailbox->ops->release_resp_queue_lock(mailbox)
 
 #define GET_RESP_ELEM_SEQ(resp) mailbox->ops->get_resp_elem_seq(mailbox, resp)
@@ -107,12 +109,13 @@ static void gcip_mailbox_del_wait_resp(struct gcip_mailbox *mailbox, void *resp)
  * Returns 0 on success, or -ENOMEM if failed on allocation.
  */
 static int gcip_mailbox_push_wait_resp(struct gcip_mailbox *mailbox, void *resp,
-				       struct gcip_mailbox_resp_awaiter *awaiter)
+				       struct gcip_mailbox_resp_awaiter *awaiter, bool atomic)
 {
-	struct gcip_mailbox_wait_list_elem *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
+	struct gcip_mailbox_wait_list_elem *entry;
 	unsigned long flags;
 	int ret;
 
+	entry = kzalloc(sizeof(*entry), atomic ? GFP_ATOMIC : GFP_KERNEL);
 	if (!entry)
 		return -ENOMEM;
 
@@ -148,8 +151,9 @@ static int gcip_mailbox_enqueue_cmd(struct gcip_mailbox *mailbox, void *cmd, voi
 {
 	int ret = 0;
 	u32 tail;
+	bool atomic = false;
 
-	ACQUIRE_CMD_QUEUE_LOCK(false);
+	ACQUIRE_CMD_QUEUE_LOCK(false, &atomic);
 
 	SET_CMD_ELEM_SEQ(cmd, mailbox->cur_seq);
 	/*
@@ -177,7 +181,7 @@ static int gcip_mailbox_enqueue_cmd(struct gcip_mailbox *mailbox, void *cmd, voi
 		/* Adds @resp to the wait_list only if the cmd can be pushed successfully. */
 		SET_RESP_ELEM_SEQ(resp, GET_CMD_ELEM_SEQ(cmd));
 		SET_RESP_ELEM_STATUS(resp, GCIP_MAILBOX_STATUS_WAITING_RESPONSE);
-		ret = gcip_mailbox_push_wait_resp(mailbox, resp, awaiter);
+		ret = gcip_mailbox_push_wait_resp(mailbox, resp, awaiter, atomic);
 		if (ret)
 			goto out;
 	}
@@ -324,9 +328,10 @@ static void *gcip_mailbox_fetch_responses(struct gcip_mailbox *mailbox, u32 *tot
 	const u32 elem_size = mailbox->resp_elem_size;
 	void *ret = NULL; /* Array of responses. */
 	void *prev_ptr = NULL; /* Temporary pointer to realloc ret. */
+	bool atomic = false;
 
 	/* Someone is working on consuming - we can leave early. */
-	if (!ACQUIRE_RESP_QUEUE_LOCK(true))
+	if (!ACQUIRE_RESP_QUEUE_LOCK(true, &atomic))
 		goto out;
 
 	head = GET_RESP_QUEUE_HEAD();
@@ -349,7 +354,8 @@ static void *gcip_mailbox_fetch_responses(struct gcip_mailbox *mailbox, u32 *tot
 			break;
 
 		prev_ptr = ret;
-		ret = krealloc(prev_ptr, (total + count) * elem_size, GFP_KERNEL);
+		ret = krealloc(prev_ptr, (total + count) * elem_size,
+			       atomic ? GFP_ATOMIC : GFP_KERNEL);
 		/*
 		 * Out-of-memory, we can return the previously fetched responses if any, or ENOMEM
 		 * otherwise.
@@ -387,8 +393,9 @@ static int gcip_mailbox_fetch_one_response(struct gcip_mailbox *mailbox, void *r
 {
 	u32 head;
 	u32 tail;
+	bool atomic;
 
-	if (!ACQUIRE_RESP_QUEUE_LOCK(true))
+	if (!ACQUIRE_RESP_QUEUE_LOCK(true, &atomic))
 		return 0;
 
 	head = GET_RESP_QUEUE_HEAD();
diff --git a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-mailbox.h b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-mailbox.h
index c88d2d7..835503f 100644
--- a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-mailbox.h
+++ b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-mailbox.h
@@ -133,14 +133,18 @@ struct gcip_mailbox_ops {
 	void (*inc_cmd_queue_tail)(struct gcip_mailbox *mailbox, u32 inc);
 	/*
 	 * Acquires the lock of cmd_queue. If @try is true, "_trylock" functions can be used, but
-	 * also it can be ignored. Returns 1 if succeed, 0 if failed. This callback will be called
-	 * in the following situations.
+	 * also it can be ignored. If the lock will make the context atomic, @atomic must be set
+	 * to true. Returns 1 if succeed, 0 if failed.
+	 *
+	 * This callback will be called in the following situations.
 	 * - Enqueue a command to the cmd_queue.
+	 *
 	 * The lock can be mutex lock or spin lock and it will be released by calling
 	 * `release_cmd_queue_lock` callback.
+	 *
 	 * Context: normal.
 	 */
-	int (*acquire_cmd_queue_lock)(struct gcip_mailbox *mailbox, bool try);
+	int (*acquire_cmd_queue_lock)(struct gcip_mailbox *mailbox, bool try, bool *atomic);
 	/*
 	 * Releases the lock of cmd_queue which is acquired by calling `acquire_cmd_queue_lock`.
 	 * Context: normal.
@@ -184,15 +188,20 @@ struct gcip_mailbox_ops {
 	void (*inc_resp_queue_head)(struct gcip_mailbox *mailbox, u32 inc);
 	/*
 	 * Acquires the lock of resp_queue. If @try is true, "_trylock" functions can be used, but
-	 * also it can be ignored. Returns 1 if succeed, 0 if failed. This callback will be called
-	 * in the following situations.
+	 * also it can be ignored. If the lock will make the context atomic, @atomic must be set
+	 * to true. Returns 1 if succeed, 0 if failed.
+	 *
+	 * This callback will be called in the following situations:
 	 * - Fetch response(s) from the resp_queue.
+	 *
 	 * The lock can be a mutex lock or a spin lock. However, if @try is considered and the
 	 * "_trylock" is used, it must be a spin lock only.
+	 *
 	 * The lock will be released by calling `release_resp_queue_lock` callback.
+	 *
 	 * Context: normal and in_interrupt().
 	 */
-	int (*acquire_resp_queue_lock)(struct gcip_mailbox *mailbox, bool try);
+	int (*acquire_resp_queue_lock)(struct gcip_mailbox *mailbox, bool try, bool *atomic);
 	/*
 	 * Releases the lock of resp_queue which is acquired by calling `acquire_resp_queue_lock`.
 	 * Context: normal and in_interrupt().