Bigo priority support

Change bigocean kernel driver to async mode by adding
a kernel worker thread. This change will allow kernel
to serve requests based on instance priority.

bug: 197665952

Signed-off-by: Ruofei Ma <ruofeim@google.com>
Change-Id: I801fd1e6fff0f9e795c01dee4d4cfa3802e3d47a

6 files changed, 297 insertions, 82 deletions

diff --git a/Makefile b/Makefile
index 0182c0f..2ce7a50 100644
--- a/Makefile
+++ b/Makefile
@@ -4,7 +4,7 @@
 #
 
 obj-$(CONFIG_BIGOCEAN) += bigocean.o
-bigocean-$(CONFIG_BIGOCEAN) += bigo.o bigo_pm.o bigo_io.o bigo_of.o bigo_iommu.o
+bigocean-$(CONFIG_BIGOCEAN) += bigo.o bigo_pm.o bigo_io.o bigo_of.o bigo_iommu.o bigo_prioq.o
 bigocean-$(CONFIG_SLC_PARTITION_MANAGER) += bigo_slc.o
 bigocean-$(CONFIG_DEBUG_FS) += bigo_debug.o
 
diff --git a/bigo.c b/bigo.c
index 08b7500..1935ecc 100644
--- a/bigo.c
+++ b/bigo.c
@@ -17,6 +17,7 @@
 #include <linux/uaccess.h>
 #include <linux/platform_data/sscoredump.h>
 #include <linux/soc/samsung/exynos-smc.h>
+#include <linux/kthread.h>
 
 #include "bigo_io.h"
 #include "bigo_iommu.h"
@@ -25,6 +26,7 @@
 #include "bigo_priv.h"
 #include "bigo_slc.h"
 #include "bigo_debug.h"
+#include "bigo_prioq.h"
 
 #define BIGO_DEVCLASS_NAME "video_codec"
 #define BIGO_CHRDEV_NAME "bigocean"
@@ -34,6 +36,8 @@
 #define DEFAULT_FPS 60
 #define BIGO_SMC_ID 0xd
 
+static int bigo_worker_thread(void *data);
+
 static struct sscd_platform_data bigo_sscd_platdata;
 
 static void bigo_sscd_release(struct device *dev)
@@ -72,27 +76,47 @@ static void bigo_coredump(struct bigo_core *core, const char *crash_info)
 
 static inline int on_first_instance_open(struct bigo_core *core)
 {
-	int rc = bigo_pt_client_enable(core);
+	int rc;
+
+	core->worker_thread = kthread_run(bigo_worker_thread, (void*)core,
+					"bigo_worker_thread");
+	if (IS_ERR(core->worker_thread)) {
+		rc = PTR_ERR(core->worker_thread);
+		core->worker_thread = NULL;
+		pr_err("failed to create worker thread rc = %d\n", rc);
+		goto exit;
+	}
 
-	if (rc)
+	rc = bigo_pt_client_enable(core);
+	if (rc) {
 		pr_info("failed to enable SLC");
+		kthread_stop(core->worker_thread);
+		goto exit;
+	}
 #if IS_ENABLED(CONFIG_PM)
 	rc = pm_runtime_get_sync(core->dev);
-	if (rc)
+	if (rc) {
 		pr_err("failed to resume: %d\n", rc);
+		kthread_stop(core->worker_thread);
+	}
 #endif
+
+exit:
 	return rc;
 }
 
 static inline void on_last_inst_close(struct bigo_core *core)
 {
+	int rc;
 #if IS_ENABLED(CONFIG_PM)
 	if (pm_runtime_put_sync_suspend(core->dev))
 		pr_warn("failed to suspend\n");
 #endif
 	bigo_pt_client_disable(core);
-	kfree(core->job.regs);
-	core->job.regs = NULL;
+
+	rc = kthread_stop(core->worker_thread);
+	if(rc)
+		pr_err("failed to stop worker thread rc = %d\n", rc);
 }
 
 static int bigo_open(struct inode *inode, struct file *file)
@@ -109,49 +133,73 @@ static int bigo_open(struct inode *inode, struct file *file)
 	}
 	INIT_LIST_HEAD(&inst->list);
 	INIT_LIST_HEAD(&inst->buffers);
+	kref_init(&inst->refcount);
 	mutex_init(&inst->lock);
+	init_completion(&inst->job_comp);
 	file->private_data = inst;
 	inst->height = DEFAULT_WIDTH;
 	inst->width = DEFAULT_HEIGHT;
 	inst->fps = DEFAULT_FPS;
 	inst->core = core;
+	inst->job.regs_size = core->regs_size;
+	inst->job.regs = kzalloc(core->regs_size, GFP_KERNEL);
+	if (!inst->job.regs) {
+		rc = -ENOMEM;
+		pr_err("Failed to alloc job regs\n");
+		goto err_first_inst;
+	}
 	mutex_lock(&core->lock);
 	if (list_empty(&core->instances)) {
 		rc = on_first_instance_open(core);
 		if (rc) {
 			pr_err("failed to setup first instance");
 			mutex_unlock(&core->lock);
-			goto err;
+			goto err_inst_open;
 		}
 	}
 	list_add_tail(&inst->list, &core->instances);
 	mutex_unlock(&core->lock);
 	bigo_update_qos(core);
-	pr_info("opened bigocean instance\n");
+	pr_info("opened instance\n");
+	return rc;
 
+err_inst_open:
+	kfree(inst->job.regs);
+err_first_inst:
+	kfree(inst);
 err:
 	return rc;
 }
 
-static int bigo_release(struct inode *inode, struct file *file)
+static void bigo_close(struct kref *ref)
 {
-	struct bigo_core *core =
-		container_of(inode->i_cdev, struct bigo_core, cdev);
-	struct bigo_inst *inst = file->private_data;
+	struct bigo_inst *inst = container_of(ref, struct bigo_inst, refcount);
+	struct bigo_core *core = inst->core;
 
 	if (!inst || !core) {
 		pr_err("No instance or core\n");
-		return -EINVAL;
+		return;
 	}
 	bigo_unmap_all(inst);
 	mutex_lock(&core->lock);
 	list_del(&inst->list);
+	kfree(inst->job.regs);
 	kfree(inst);
 	if (list_empty(&core->instances))
 		on_last_inst_close(core);
 	mutex_unlock(&core->lock);
 	bigo_update_qos(core);
-	pr_info("closed bigocean instance\n");
+	pr_info("closed instance\n");
+}
+
+static int bigo_release(struct inode *inode, struct file *file)
+{
+	struct bigo_inst *inst = file->private_data;
+
+	if (!inst)
+		return -EINVAL;
+
+	kref_put(&inst->refcount, bigo_close);
 	return 0;
 }
 
@@ -191,55 +239,6 @@ static int bigo_run_job(struct bigo_core *core, struct bigo_job *job)
 	return rc;
 }
 
-static int bigo_process(struct bigo_core *core, struct bigo_ioc_regs *desc)
-{
-	int rc = 0;
-	struct bigo_job *job = &core->job;
-
-	if (!desc) {
-		pr_err("Invalid input\n");
-		return -EINVAL;
-	}
-	if (desc->regs_size != core->regs_size) {
-		pr_err("Register size passed from userspace(%u) is different(%u)\n",
-		       (unsigned int)desc->regs_size, core->regs_size);
-		return -EINVAL;
-	}
-
-	mutex_lock(&core->lock);
-
-	if (!job->regs) {
-		job->regs = kzalloc(core->regs_size, GFP_KERNEL);
-		if (!job->regs) {
-			rc = -ENOMEM;
-			goto unlock;
-		}
-	}
-
-	if (copy_from_user(job->regs, (void *)desc->regs, core->regs_size)) {
-		pr_err("Failed to copy from user\n");
-		rc = -EFAULT;
-		goto unlock;
-	}
-
-	/*TODO(vinaykalia@): Replace this with EDF scheduler.*/
-	rc = bigo_run_job(core, job);
-	if (rc) {
-		pr_err("Error running job\n");
-		goto unlock;
-	}
-
-	if (copy_to_user((void *)desc->regs, job->regs, core->regs_size)) {
-		pr_err("Failed to copy to user\n");
-		rc = -EFAULT;
-		goto unlock;
-	}
-
-unlock:
-	mutex_unlock(&core->lock);
-	return rc;
-}
-
 inline void bigo_config_frmrate(struct bigo_inst *inst, __u32 frmrate)
 {
 	mutex_lock(&inst->lock);
@@ -265,6 +264,50 @@ inline void bigo_config_secure(struct bigo_inst *inst, __u32 is_secure)
 	mutex_unlock(&inst->lock);
 }
 
+inline void bigo_config_priority(struct bigo_inst *inst, __s32 priority)
+{
+	if (priority < 0 || priority >= BO_MAX_PRIO)
+		return;
+	mutex_lock(&inst->lock);
+	inst->priority = priority;
+	mutex_unlock(&inst->lock);
+}
+
+static int copy_regs_from_user(struct bigo_core *core,
+			struct bigo_ioc_regs *desc,
+			void __user *user_desc,
+			struct bigo_job *job)
+{
+	if (!core || !desc || !user_desc || !job)
+		return -EINVAL;
+
+	if (copy_from_user(desc, user_desc, sizeof(*desc)))
+		return -EFAULT;
+
+	if (desc->regs_size != core->regs_size) {
+		pr_err("Reg size of userspace(%u) is different(%u)\n",
+		(unsigned int)desc->regs_size, core->regs_size);
+		return -EINVAL;
+	}
+
+	if (copy_from_user(job->regs, (void *)desc->regs, desc->regs_size))
+		return -EFAULT;
+
+	return 0;
+}
+
+static int copy_regs_to_user(struct bigo_ioc_regs *desc,
+				struct bigo_job *job)
+{
+	if (!desc || !job)
+		return -EINVAL;
+
+	if (copy_to_user((void *)desc->regs, job->regs, desc->regs_size))
+		return -EFAULT;
+
+	return 0;
+}
+
 static long bigo_unlocked_ioctl(struct file *file, unsigned int cmd,
 				unsigned long arg)
 {
@@ -272,7 +315,6 @@ static long bigo_unlocked_ioctl(struct file *file, unsigned int cmd,
 	struct bigo_core *core =
 		container_of(file->f_inode->i_cdev, struct bigo_core, cdev);
 	void __user *user_desc = (void __user *)arg;
-	struct bigo_ioc_regs desc;
 	struct bigo_ioc_mapping mapping;
 	struct bigo_ioc_frmsize frmsize;
 	struct bigo_cache_info cinfo;
@@ -292,31 +334,41 @@ static long bigo_unlocked_ioctl(struct file *file, unsigned int cmd,
 	}
 	switch (cmd) {
 	case BIGO_IOCX_PROCESS:
-		if (copy_from_user(&desc, user_desc, sizeof(desc))) {
-			pr_err("Failed to copy from user\n");
+	{
+		struct bigo_ioc_regs desc;
+		struct bigo_job *job = &inst->job;
+		long ret;
+
+		if (copy_regs_from_user(core, &desc, user_desc, job)) {
+			pr_err("Failed to copy regs from user\n");
 			return -EFAULT;
 		}
 
-		if (inst->is_secure) {
-			rc = exynos_smc(SMC_PROTECTION_SET, 0, BIGO_SMC_ID,
-					SMC_PROTECTION_ENABLE);
-			if (rc) {
-				pr_err("failed to enable SMC_PROTECTION_SET: %d\n", rc);
-				break;
-			}
+		if(enqueue_prioq(core, inst)) {
+			pr_err("Failed enqueue frame\n");
+			return -EFAULT;
+		}
+
+		ret = wait_for_completion_timeout(
+			&inst->job_comp,
+			msecs_to_jiffies(JOB_COMPLETE_TIMEOUT_MS * 16));
+		if (!ret) {
+			pr_err("timed out waiting for HW: %d\n", rc);
+			rc = -ETIMEDOUT;
+		} else {
+			rc = (ret > 0) ? 0 : ret;
 		}
 
-		rc = bigo_process(core, &desc);
 		if (rc)
-			pr_err("Error processing data: %d\n", rc);
+			break;
 
-		if (inst->is_secure) {
-			rc = exynos_smc(SMC_PROTECTION_SET, 0, BIGO_SMC_ID,
-					SMC_PROTECTION_DISABLE);
-			if (rc)
-				pr_err("failed to disable SMC_PROTECTION_SET: %d\n", rc);
+		rc = job->status;
+		if(copy_regs_to_user(&desc, job)) {
+			pr_err("Failed to copy regs to user\n");
+			rc = -EFAULT;
 		}
 		break;
+	}
 	case BIGO_IOCX_MAP:
 		if (copy_from_user(&mapping, user_desc, sizeof(mapping))) {
 			pr_err("Failed to copy from user\n");
@@ -364,6 +416,11 @@ static long bigo_unlocked_ioctl(struct file *file, unsigned int cmd,
 		bigo_config_secure(inst, is_secure);
 		break;
 	}
+	case BIGO_IOCX_CONFIG_PRIORITY: {
+		s32 priority = (s32)arg;
+		bigo_config_priority(inst, priority);
+		break;
+	}
 	case BIGO_IOCX_ABORT:
 		break;
 	default:
@@ -461,9 +518,62 @@ static void deinit_chardev(struct bigo_core *core)
 	unregister_chrdev_region(core->devno, 1);
 }
 
+static int bigo_worker_thread(void *data)
+{
+	struct bigo_core *core = (struct bigo_core *)data;
+	struct bigo_inst *inst;
+	struct bigo_job *job;
+	bool should_stop;
+	int rc;
+
+	if (!core)
+		return -ENOMEM;
+
+	while(1) {
+		wait_event(core->worker,
+			dequeue_prioq(core, &job, &should_stop));
+		if(should_stop) {
+			pr_info("worker thread received stop signal, exit\n");
+			return 0;
+		}
+		if (!job)
+			continue;
+
+		inst = container_of(job, struct bigo_inst, job);
+		if (inst->is_secure) {
+			rc = exynos_smc(SMC_PROTECTION_SET, 0, BIGO_SMC_ID,
+					SMC_PROTECTION_ENABLE);
+			if (rc) {
+				pr_err("failed to enable SMC_PROTECTION_SET: %d\n", rc);
+				goto done;
+			}
+		}
+
+		rc = bigo_run_job(core, job);
+		if (rc) {
+			pr_err("Error running job\n");
+			goto done;
+		}
+
+		if (inst->is_secure) {
+			rc = exynos_smc(SMC_PROTECTION_SET, 0, BIGO_SMC_ID,
+					SMC_PROTECTION_DISABLE);
+			if (rc)
+				pr_err("failed to disable SMC_PROTECTION_SET: %d\n", rc);
+		}
+
+	done:
+		job->status = rc;
+		complete(&inst->job_comp);
+		kref_put(&inst->refcount, bigo_close);
+	}
+	return 0;
+}
+
 static int bigo_probe(struct platform_device *pdev)
 {
 	int rc = 0;
+	int i;
 	struct bigo_core *core;
 
 	core = devm_kzalloc(&pdev->dev, sizeof(struct bigo_core), GFP_KERNEL);
@@ -473,11 +583,15 @@ static int bigo_probe(struct platform_device *pdev)
 	}
 
 	mutex_init(&core->lock);
+	mutex_init(&core->prioq.lock);
 	INIT_LIST_HEAD(&core->instances);
 	INIT_LIST_HEAD(&core->pm.opps);
 	INIT_LIST_HEAD(&core->pm.bw);
+	for(i = 0; i < BO_MAX_PRIO; ++i)
+		INIT_LIST_HEAD(&core->prioq.queue[i]);
 	spin_lock_init(&core->status_lock);
 	init_completion(&core->frame_done);
+	init_waitqueue_head(&core->worker);
 	core->dev = &pdev->dev;
 	platform_set_drvdata(pdev, core);
 
diff --git a/bigo_prioq.c b/bigo_prioq.c
new file mode 100644
index 0000000..19da48a
--- /dev/null
+++ b/bigo_prioq.c
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Priority queue impletation with BigOcean
+ *
+ * Copyright 2021 Google LLC.
+ *
+ * Author: Ruofei Ma <ruofeim@google.com>
+ */
+
+#include <linux/kthread.h>
+#include <linux/module.h>
+
+#include "bigo_prioq.h"
+
+int enqueue_prioq(struct bigo_core *core, struct bigo_inst *inst)
+{
+	struct bigo_job *job = &inst->job;
+
+	if(!core || !inst)
+		return -EINVAL;
+
+	mutex_lock(&core->prioq.lock);
+	list_add_tail(&job->list, &core->prioq.queue[inst->priority]);
+	set_bit(inst->priority, &core->prioq.bitmap);
+	mutex_unlock(&core->prioq.lock);
+
+	kref_get(&inst->refcount);
+	wake_up(&core->worker);
+	return 0;
+}
+
+bool dequeue_prioq(struct bigo_core *core, struct bigo_job **job,
+			bool *should_stop)
+{
+	int high_prio;
+	struct bigo_job *j = NULL;
+	struct list_head *queue;
+	if (!core || !job || !should_stop)
+		return false;
+
+	*should_stop = false;
+	if(kthread_should_stop()) {
+		*should_stop = true;
+		return true;
+	}
+
+	mutex_lock(&core->prioq.lock);
+	high_prio = ffs(core->prioq.bitmap) - 1;
+	if (high_prio < 0)
+		goto exit;
+
+	queue = &core->prioq.queue[high_prio];
+	j = list_first_entry_or_null(queue, struct bigo_job, list);
+	if (j) {
+		list_del(&j->list);
+		if (list_empty(queue))
+			clear_bit(high_prio, &core->prioq.bitmap);
+	}
+
+exit:
+	mutex_unlock(&core->prioq.lock);
+	*job = j;
+	return *job != NULL;
+}
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ruofei Ma <ruofeim@google.com>");
diff --git a/bigo_prioq.h b/bigo_prioq.h
new file mode 100644
index 0000000..0a49290
--- /dev/null
+++ b/bigo_prioq.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright 2021 Google LLC.
+ *
+ * Author: Ruofei Ma <ruofeim@google.com>
+ */
+
+#ifndef _BIGO_PRIOQ_H_
+#define _BIGO_PRIOQ_H_
+
+#include "bigo_priv.h"
+
+bool dequeue_prioq(struct bigo_core *core, struct bigo_job **job,
+                        bool *should_stop);
+int enqueue_prioq(struct bigo_core *core, struct bigo_inst *inst);
+
+#endif //_BIGO_PRIOQ_H_
diff --git a/bigo_priv.h b/bigo_priv.h
index 649dbc6..0eb29cb 100644
--- a/bigo_priv.h
+++ b/bigo_priv.h
@@ -25,6 +25,7 @@
 #define AVG_CNT 30
 #define PEAK_CNT 5
 #define BUS_WIDTH 16
+#define BO_MAX_PRIO 2
 
 struct bufinfo {
 	struct list_head list;
@@ -69,8 +70,10 @@ struct slc_manager {
 };
 
 struct bigo_job {
+	struct list_head list;
 	void *regs;
 	size_t regs_size;
+	int status;
 };
 
 struct bigo_debugfs {
@@ -79,6 +82,12 @@ struct bigo_debugfs {
 	u32 trigger_ssr;
 };
 
+struct bigo_prio_array {
+	struct mutex lock;
+	unsigned long bitmap;
+	struct list_head queue[BO_MAX_PRIO];
+};
+
 struct bigo_core {
 	struct class *_class;
 	struct cdev cdev;
@@ -93,7 +102,6 @@ struct bigo_core {
 	struct list_head instances;
 	struct ion_client *mem_client;
 	u32 stat_with_irq;
-	struct bigo_job job;
 	struct power_manager pm;
 	struct slc_manager slc;
 	unsigned int regs_size;
@@ -101,6 +109,9 @@ struct bigo_core {
 	phys_addr_t paddr;
 	struct bigo_debugfs debugfs;
 	spinlock_t status_lock;
+	struct task_struct *worker_thread;
+	wait_queue_head_t worker;
+	struct bigo_prio_array prioq;
 };
 
 struct bigo_inst {
@@ -108,15 +119,19 @@ struct bigo_inst {
 	struct list_head buffers;
 	/* mutex protecting this data structure */
 	struct mutex lock;
+	struct kref refcount;
 	struct bigo_core *core;
 	u32 height;
 	u32 width;
 	u32 fps;
 	u32 is_secure;
+	int priority;
 	struct bigo_bw avg_bw[AVG_CNT];
 	struct bigo_bw pk_bw[AVG_CNT];
 	int job_cnt;
 	u32 hw_cycles[AVG_CNT];
+	struct completion job_comp;
+	struct bigo_job job;
 };
 
 inline void set_curr_inst(struct bigo_core *core, struct bigo_inst *inst);
diff --git a/include/uapi/linux/bigo.h b/include/uapi/linux/bigo.h
index 3ab10b8..53e743d 100644
--- a/include/uapi/linux/bigo.h
+++ b/include/uapi/linux/bigo.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
 /*
- * Copyright 2020 Google LLC.
+ * Copyright 2021 Google LLC.
  *
  * Author: Vinay Kalia <vinaykalia@google.com>
  */
@@ -68,6 +68,7 @@ enum bigo_cmd_id {
 	BIGO_CMD_CONFIG_FRMSIZE,
 	BIGO_CMD_GET_CACHE_INFO,
 	BIGO_CMD_CONFIG_SECURE,
+	BIGO_CMD_CONFIG_PRIORITY,
 	BIGO_CMD_MAXNR,
 };
 /* <END OF HELPERS> */
@@ -82,5 +83,6 @@ enum bigo_cmd_id {
 	_BIGO_IOR(BIGO_CMD_GET_CACHE_INFO, struct bigo_cache_info)
 #define BIGO_IOCX_ABORT _BIGO_IO(BIGO_CMD_ABORT)
 #define BIGO_IOCX_CONFIG_SECURE _BIGO_IOW(BIGO_CMD_CONFIG_SECURE, __u32)
+#define BIGO_IOCX_CONFIG_PRIORITY _BIGO_IOW(BIGO_CMD_CONFIG_PRIORITY, __s32)
 
 #endif /* _UAPI_BIGO_H_ */