ANDROID: trusty: Use high priority check WQ and regular nop WQ

Moving work items across different WQ invalidates non-reentrancy
guarantees.

So instead of having different WQs at different priorities, we only
maintain one WQ. High priority WQ for check work, and non-high priority
WQ for nop work.

Keep trusty-virtio "use_high_wq" toggle for user space compatibility.

Bug: 231003163
Test: echo 0 > /sys/module/trusty_core/parameters/use_high_wq
time /data/nativetest64/vendor/tipc-test/tipc-test -t echo -r 10000
Times do not regress.
Change-Id: I54b60cb370ffb727d2f56e28932c9ab841bd8366
Signed-off-by: Tri Vo <trong@google.com>

2 files changed, 4 insertions, 33 deletions

diff --git a/drivers/trusty/trusty-virtio.c b/drivers/trusty/trusty-virtio.c
index 8a7eb69..2df10d1 100644
--- a/drivers/trusty/trusty-virtio.c
+++ b/drivers/trusty/trusty-virtio.c
@@ -47,7 +47,6 @@ struct trusty_ctx {
 	struct mutex		mlock; /* protects vdev_list */
 	struct workqueue_struct	*kick_wq;
 	struct workqueue_struct	*check_wq;
-	struct workqueue_struct	*check_wq_high;
 };
 
 struct trusty_vring {
@@ -102,10 +101,7 @@ static int trusty_call_notify(struct notifier_block *nb,
 		return NOTIFY_DONE;
 
 	tctx = container_of(nb, struct trusty_ctx, call_notifier);
-	if (use_high_wq)
-		queue_work(tctx->check_wq_high, &tctx->check_vqs);
-	else
-		queue_work(tctx->check_wq, &tctx->check_vqs);
+	queue_work(tctx->check_wq, &tctx->check_vqs);
 
 	return NOTIFY_OK;
 }
@@ -742,7 +738,8 @@ static int trusty_virtio_probe(struct platform_device *pdev)
 
 	set_dma_ops(&pdev->dev, &trusty_virtio_dma_map_ops);
 
-	tctx->check_wq = alloc_workqueue("trusty-check-wq", WQ_UNBOUND, 0);
+	tctx->check_wq = alloc_workqueue("trusty-check-wq",
+					 WQ_UNBOUND | WQ_HIGHPRI, 0);
 	if (!tctx->check_wq) {
 		ret = -ENODEV;
 		dev_err(&pdev->dev, "Failed create trusty-check-wq\n");
@@ -757,14 +754,6 @@ static int trusty_virtio_probe(struct platform_device *pdev)
 		goto err_create_kick_wq;
 	}
 
-	tctx->check_wq_high = alloc_workqueue("trusty-check-wq-high",
-					      WQ_UNBOUND | WQ_HIGHPRI, 0);
-	if (!tctx->check_wq_high) {
-		ret = -ENODEV;
-		dev_err(&pdev->dev, "Failed create trusty-check-wq-high\n");
-		goto err_create_check_wq_high;
-	}
-
 	ret = trusty_virtio_add_devices(tctx);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to add virtio devices\n");
@@ -775,8 +764,6 @@ static int trusty_virtio_probe(struct platform_device *pdev)
 	return 0;
 
 err_add_devices:
-	destroy_workqueue(tctx->check_wq_high);
-err_create_check_wq_high:
 	destroy_workqueue(tctx->kick_wq);
 err_create_kick_wq:
 	destroy_workqueue(tctx->check_wq);
@@ -802,7 +789,6 @@ static int trusty_virtio_remove(struct platform_device *pdev)
 	/* destroy workqueues */
 	destroy_workqueue(tctx->kick_wq);
 	destroy_workqueue(tctx->check_wq);
-	destroy_workqueue(tctx->check_wq_high);
 
 	/* notify remote that shared area goes away */
 	trusty_virtio_stop(tctx, tctx->shared_id, tctx->shared_sz);
diff --git a/drivers/trusty/trusty.c b/drivers/trusty/trusty.c
index c773beb..d1d7054 100644
--- a/drivers/trusty/trusty.c
+++ b/drivers/trusty/trusty.c
@@ -41,7 +41,6 @@ struct trusty_state {
 	bool trusty_panicked;
 	struct device *dev;
 	struct workqueue_struct *nop_wq;
-	struct workqueue_struct *nop_wq_high;
 	struct trusty_work __percpu *nop_works;
 	struct list_head nop_queue;
 	spinlock_t nop_lock; /* protects nop_queue */
@@ -824,10 +823,7 @@ void trusty_enqueue_nop(struct device *dev, struct trusty_nop *nop)
 			list_add_tail(&nop->node, &s->nop_queue);
 		spin_unlock_irqrestore(&s->nop_lock, flags);
 	}
-	if (use_high_wq)
-		queue_work(s->nop_wq_high, &tw->work);
-	else
-		queue_work(s->nop_wq, &tw->work);
+	queue_work(s->nop_wq, &tw->work);
 	preempt_enable();
 }
 EXPORT_SYMBOL(trusty_enqueue_nop);
@@ -901,14 +897,6 @@ static int trusty_probe(struct platform_device *pdev)
 		goto err_create_nop_wq;
 	}
 
-	s->nop_wq_high = alloc_workqueue("trusty-nop-wq-high", WQ_HIGHPRI |
-					 WQ_CPU_INTENSIVE, 0);
-	if (!s->nop_wq_high) {
-		ret = -ENODEV;
-		dev_err(&pdev->dev, "Failed create trusty-nop-wq-high\n");
-		goto err_create_nop_wq_high;
-	}
-
 	s->nop_works = alloc_percpu(struct trusty_work);
 	if (!s->nop_works) {
 		ret = -ENOMEM;
@@ -944,8 +932,6 @@ err_add_children:
 	}
 	free_percpu(s->nop_works);
 err_alloc_works:
-	destroy_workqueue(s->nop_wq_high);
-err_create_nop_wq_high:
 	destroy_workqueue(s->nop_wq);
 err_create_nop_wq:
 	trusty_free_msg_buf(s, &pdev->dev);
@@ -975,7 +961,6 @@ static int trusty_remove(struct platform_device *pdev)
 	}
 	free_percpu(s->nop_works);
 	destroy_workqueue(s->nop_wq);
-	destroy_workqueue(s->nop_wq_high);
 
 	mutex_destroy(&s->share_memory_msg_lock);
 	mutex_destroy(&s->smc_lock);