Merge "msm: kgsl: Add support to handle hardware fence timeout"

5 files changed, 482 insertions, 20 deletions

diff --git a/adreno_gen7_hwsched_hfi.c b/adreno_gen7_hwsched_hfi.c
index 0cfe3e7..d2e8e0e 100644
--- a/adreno_gen7_hwsched_hfi.c
+++ b/adreno_gen7_hwsched_hfi.c
@@ -254,6 +254,93 @@ static u32 get_payload_rb_key_legacy(struct adreno_device *adreno_dev,
 	return 0;
 }
 
+struct syncobj_flags {
+	unsigned long mask;
+	const char *name;
+};
+
+static void _get_syncobj_string(char *str, u32 max_size, struct hfi_syncobj *syncobj, u32 index)
+{
+	u32 count = scnprintf(str, max_size, "syncobj[%d] ctxt_id:%lu seqno:%lu flags:", index,
+			syncobj->ctxt_id, syncobj->seq_no);
+	u32 i;
+	bool first = true;
+	static const struct syncobj_flags _flags[] = {
+		GMU_SYNCOBJ_FLAGS, { -1, NULL }};
+
+	for (i = 0; _flags[i].name; i++) {
+		if (!(syncobj->flags & _flags[i].mask))
+			continue;
+
+		if (first) {
+			count += scnprintf(str + count, max_size - count, "%s", _flags[i].name);
+			first = false;
+		} else {
+			count += scnprintf(str + count, max_size - count, "|%s", _flags[i].name);
+		}
+	}
+}
+
+static void log_syncobj(struct gen7_gmu_device *gmu, struct hfi_submit_syncobj *cmd)
+{
+	struct hfi_syncobj *syncobj = (struct hfi_syncobj *)&cmd[1];
+	char str[128];
+	u32 i = 0;
+
+	for (i = 0; i < cmd->num_syncobj; i++) {
+		_get_syncobj_string(str, sizeof(str), syncobj, i);
+		dev_err(&gmu->pdev->dev, "%s\n", str);
+		syncobj++;
+	}
+}
+
+static void find_timeout_syncobj(struct adreno_device *adreno_dev, u32 ctxt_id, u32 ts)
+{
+	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
+	struct kgsl_context *context = NULL;
+	struct adreno_context *drawctxt;
+	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
+	struct gmu_context_queue_header *hdr;
+	struct hfi_submit_syncobj *cmd;
+	u32 *queue, i;
+	int ret;
+
+	/* We want to get the context even if it is detached */
+	read_lock(&device->context_lock);
+	context = idr_find(&device->context_idr, ctxt_id);
+	ret = _kgsl_context_get(context);
+	read_unlock(&device->context_lock);
+
+	if (!ret)
+		return;
+
+	drawctxt = ADRENO_CONTEXT(context);
+
+	hdr = drawctxt->gmu_context_queue.hostptr;
+	queue = (u32 *)(drawctxt->gmu_context_queue.hostptr + sizeof(*hdr));
+
+	for (i = hdr->read_index; i != hdr->write_index;) {
+		if (MSG_HDR_GET_ID(queue[i]) != H2F_MSG_ISSUE_SYNCOBJ) {
+			i = (i + MSG_HDR_GET_SIZE(queue[i])) % hdr->queue_size;
+			continue;
+		}
+
+		cmd = (struct hfi_submit_syncobj *)&queue[i];
+
+		if (cmd->timestamp == ts) {
+			log_syncobj(gmu, cmd);
+			break;
+		}
+		i = (i + MSG_HDR_GET_SIZE(queue[i])) % hdr->queue_size;
+	}
+
+	if (i == hdr->write_index)
+		dev_err(&gmu->pdev->dev, "Couldn't find unsignaled syncobj ctx:%d ts:%d\n",
+			ctxt_id, ts);
+
+	kgsl_context_put(context);
+}
+
 static void log_gpu_fault_legacy(struct adreno_device *adreno_dev)
 {
 	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
@@ -382,6 +469,11 @@ static void log_gpu_fault_legacy(struct adreno_device *adreno_dev)
 	case GMU_GPU_AQE1_ILLEGAL_INST_ERROR:
 		dev_crit_ratelimited(dev, "AQE1 Illegal instruction error\n");
 		break;
+	case GMU_SYNCOBJ_TIMEOUT_ERROR:
+		dev_crit_ratelimited(dev, "syncobj timeout ctx %d ts %u\n",
+			cmd->ctxt_id, cmd->ts);
+		find_timeout_syncobj(adreno_dev, cmd->ctxt_id, cmd->ts);
+		break;
 	case GMU_CP_UNKNOWN_ERROR:
 		fallthrough;
 	default:
@@ -610,6 +702,11 @@ static void log_gpu_fault(struct adreno_device *adreno_dev)
 	case GMU_GPU_AQE1_ILLEGAL_INST_ERROR:
 		dev_crit_ratelimited(dev, "AQE1 Illegal instruction error\n");
 		break;
+	case GMU_SYNCOBJ_TIMEOUT_ERROR:
+		dev_crit_ratelimited(dev, "syncobj timeout ctx %d ts %u\n",
+			cmd->gc.ctxt_id, cmd->gc.ts);
+		find_timeout_syncobj(adreno_dev, cmd->gc.ctxt_id, cmd->gc.ts);
+		break;
 	case GMU_CP_UNKNOWN_ERROR:
 		fallthrough;
 	default:
@@ -649,6 +746,191 @@ static void process_ctx_bad(struct adreno_device *adreno_dev)
 	adreno_hwsched_fault(adreno_dev, ADRENO_HARD_FAULT);
 }
 
+#define GET_QUERIED_FENCE_INDEX(x) (x / BITS_PER_SYNCOBJ_QUERY)
+#define GET_QUERIED_FENCE_BIT(x) (x % BITS_PER_SYNCOBJ_QUERY)
+
+static bool fence_is_queried(struct hfi_syncobj_query_cmd *cmd, u32 fence_index)
+{
+	u32 index = GET_QUERIED_FENCE_INDEX(fence_index);
+	u32 bit = GET_QUERIED_FENCE_BIT(fence_index);
+
+	return (cmd->queries[index].query_bitmask & BIT(bit));
+}
+
+static void set_fence_signal_bit(struct adreno_device *adreno_dev,
+	struct hfi_syncobj_query_cmd *reply, struct dma_fence *fence, u32 fence_index,
+	char *name)
+{
+	u32 index = GET_QUERIED_FENCE_INDEX(fence_index);
+	u32 bit = GET_QUERIED_FENCE_BIT(fence_index);
+	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
+	u64 flags = ADRENO_HW_FENCE_SW_STATUS_PENDING;
+	char value[32] = "unknown";
+
+	if (fence->ops->timeline_value_str)
+		fence->ops->timeline_value_str(fence, value, sizeof(value));
+
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) {
+		dev_err(&gmu->pdev->dev,
+			"GMU is waiting for signaled fence(ctx:%ld seqno:%ld value:%s)\n",
+			fence->context, fence->seqno, value);
+		reply->queries[index].query_bitmask |= BIT(bit);
+		flags = ADRENO_HW_FENCE_SW_STATUS_SIGNALED;
+	}
+	trace_adreno_hw_fence_query(fence->context, fence->seqno, flags, name, value);
+}
+
+static void gen7_syncobj_query_reply(struct adreno_device *adreno_dev,
+	struct kgsl_drawobj *drawobj, struct hfi_syncobj_query_cmd *cmd)
+{
+	struct hfi_syncobj_query_cmd reply = {0};
+	struct gen7_hfi *hfi = to_gen7_hfi(adreno_dev);
+	int i, j, fence_index = 0;
+	struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj);
+	const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
+
+	for (i = 0; i < syncobj->numsyncs; i++) {
+		struct kgsl_drawobj_sync_event *event = &syncobj->synclist[i];
+		struct kgsl_sync_fence_cb *kcb = event->handle;
+		struct dma_fence **fences;
+		struct dma_fence_array *array;
+		struct event_fence_info *info = event->priv;
+		u32 num_fences;
+
+		array = to_dma_fence_array(kcb->fence);
+		if (array != NULL) {
+			num_fences = array->num_fences;
+			fences = array->fences;
+		} else {
+			num_fences = 1;
+			fences = &kcb->fence;
+		}
+
+		for (j = 0; j < num_fences; j++, fence_index++) {
+			if (!fence_is_queried(cmd, fence_index))
+				continue;
+
+			set_fence_signal_bit(adreno_dev, &reply, fences[j], fence_index,
+				info ? info->fences[j].name : "unknown");
+		}
+	}
+
+	reply.hdr = CREATE_MSG_HDR(F2H_MSG_SYNCOBJ_QUERY, sizeof(reply),
+			HFI_MSG_CMD);
+	reply.hdr = MSG_HDR_SET_SEQNUM(reply.hdr,
+			atomic_inc_return(&hfi->seqnum));
+	reply.gmu_ctxt_id = cmd->gmu_ctxt_id;
+	reply.sync_obj_ts = cmd->sync_obj_ts;
+
+	trace_adreno_syncobj_query_reply(reply.gmu_ctxt_id, reply.sync_obj_ts,
+		gpudev->read_alwayson(adreno_dev));
+
+	gen7_hfi_send_cmd_async(adreno_dev, &reply);
+}
+
+struct syncobj_query_work {
+	/** @cmd: The query command to be processed */
+	struct hfi_syncobj_query_cmd cmd;
+	/** @context: kgsl context that is waiting for this sync object */
+	struct kgsl_context *context;
+	/** @work: The work structure to execute syncobj query reply */
+	struct kthread_work work;
+};
+
+static void gen7_process_syncobj_query_work(struct kthread_work *work)
+{
+	struct syncobj_query_work *query_work = container_of(work,
+						struct syncobj_query_work, work);
+	struct hfi_syncobj_query_cmd *cmd = (struct hfi_syncobj_query_cmd *)&query_work->cmd;
+	struct kgsl_context *context = query_work->context;
+	struct kgsl_device *device = context->device;
+	struct adreno_device *adreno_dev = ADRENO_DEVICE(device);
+	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
+	struct cmd_list_obj *obj;
+	bool missing = true;
+
+	mutex_lock(&hwsched->mutex);
+	mutex_lock(&device->mutex);
+
+	list_for_each_entry(obj, &hwsched->cmd_list, node) {
+		struct kgsl_drawobj *drawobj = obj->drawobj;
+
+		if ((drawobj->type & SYNCOBJ_TYPE) == 0)
+			continue;
+
+		if ((drawobj->context->id == cmd->gmu_ctxt_id) &&
+			(drawobj->timestamp == cmd->sync_obj_ts)) {
+			gen7_syncobj_query_reply(adreno_dev, drawobj, cmd);
+			missing = false;
+			break;
+		}
+	}
+
+	if (missing) {
+		struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
+		struct adreno_context *drawctxt = ADRENO_CONTEXT(context);
+		struct gmu_context_queue_header *hdr = drawctxt->gmu_context_queue.hostptr;
+
+		/*
+		 * If the sync object is not found, it can only mean that the sync object was
+		 * retired by the GMU in the meanwhile. However, if that is not the case, then
+		 * we have a problem.
+		 */
+		if (timestamp_cmp(cmd->sync_obj_ts, hdr->sync_obj_ts) > 0) {
+			dev_err(&gmu->pdev->dev, "Missing sync object ctx:%d ts:%d retired:%d\n",
+				context->id, cmd->sync_obj_ts, hdr->sync_obj_ts);
+			gmu_core_fault_snapshot(device);
+			adreno_hwsched_fault(adreno_dev, ADRENO_HARD_FAULT);
+		}
+	}
+
+	mutex_unlock(&device->mutex);
+	mutex_unlock(&hwsched->mutex);
+
+	kgsl_context_put(context);
+	kfree(query_work);
+}
+
+static void gen7_trigger_syncobj_query(struct adreno_device *adreno_dev,
+	u32 *rcvd)
+{
+	struct syncobj_query_work *query_work;
+	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
+	struct hfi_syncobj_query_cmd *cmd = (struct hfi_syncobj_query_cmd *)rcvd;
+	struct kgsl_context *context = NULL;
+	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
+	const struct adreno_gpudev *gpudev = ADRENO_GPU_DEVICE(adreno_dev);
+	int ret;
+
+	trace_adreno_syncobj_query(cmd->gmu_ctxt_id, cmd->sync_obj_ts,
+		gpudev->read_alwayson(adreno_dev));
+
+	/*
+	 * We need the context even if it is detached. Hence, we can't use kgsl_context_get here.
+	 * We must make sure that this context id doesn't get destroyed (to avoid re-use) until GMU
+	 * has ack'd the query reply.
+	 */
+	read_lock(&device->context_lock);
+	context = idr_find(&device->context_idr, cmd->gmu_ctxt_id);
+	ret = _kgsl_context_get(context);
+	read_unlock(&device->context_lock);
+
+	if (!ret)
+		return;
+
+	query_work = kzalloc(sizeof(*query_work), GFP_KERNEL);
+	if (!query_work) {
+		kgsl_context_put(context);
+		return;
+	}
+
+	kthread_init_work(&query_work->work, gen7_process_syncobj_query_work);
+	memcpy(&query_work->cmd, cmd, sizeof(*cmd));
+	query_work->context = context;
+
+	kthread_queue_work(hwsched->worker, &query_work->work);
+}
+
 static void gen7_hwsched_process_msgq(struct adreno_device *adreno_dev)
 {
 	struct gen7_gmu_device *gmu = to_gen7_gmu(adreno_dev);
@@ -683,6 +965,8 @@ static void gen7_hwsched_process_msgq(struct adreno_device *adreno_dev)
 		} else if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_TS_RETIRE) {
 			log_profiling_info(adreno_dev, rcvd);
 			adreno_hwsched_trigger(adreno_dev);
+		} else if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_SYNCOBJ_QUERY) {
+			gen7_trigger_syncobj_query(adreno_dev, rcvd);
 		} else if (MSG_HDR_GET_ID(rcvd[0]) == F2H_MSG_GMU_CNTR_RELEASE) {
 			struct hfi_gmu_cntr_release_cmd *cmd =
 				(struct hfi_gmu_cntr_release_cmd *) rcvd;
@@ -2188,12 +2472,12 @@ static void populate_kgsl_fence(struct hfi_syncobj *obj,
 	struct kgsl_sync_timeline *ktimeline = kfence->parent;
 	unsigned long flags;
 
-	obj->flags |= GMU_SYNCOBJ_KGSL_FENCE;
+	obj->flags |= BIT(GMU_SYNCOBJ_FLAG_KGSL_FENCE_BIT);
 
 	spin_lock_irqsave(&ktimeline->lock, flags);
 	/* This means that the context is going away. Mark the fence as triggered */
 	if (!ktimeline->context) {
-		obj->flags |= GMU_SYNCOBJ_RETIRED;
+		obj->flags |= BIT(GMU_SYNCOBJ_FLAG_SIGNALED_BIT);
 		spin_unlock_irqrestore(&ktimeline->lock, flags);
 		return;
 	}
@@ -2258,8 +2542,9 @@ static int _submit_hw_fence(struct adreno_device *adreno_dev,
 					return ret;
 				}
 
-				if (test_bit(MSM_HW_FENCE_FLAG_SIGNALED_BIT, &fences[j]->flags))
-					obj->flags |= GMU_SYNCOBJ_RETIRED;
+				if (test_bit(MSM_HW_FENCE_FLAG_SIGNALED_BIT, &fences[j]->flags) ||
+					test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fences[j]->flags))
+					obj->flags |= BIT(GMU_SYNCOBJ_FLAG_SIGNALED_BIT);
 
 				obj->ctxt_id = fences[j]->context;
 				obj->seq_no =  fences[j]->seqno;
diff --git a/adreno_hfi.h b/adreno_hfi.h
index 437f842..46d7311 100644
--- a/adreno_hfi.h
+++ b/adreno_hfi.h
@@ -453,6 +453,7 @@ struct hfi_queue_table {
 #define F2H_MSG_CONTEXT_BAD		150
 #define H2F_MSG_HW_FENCE_INFO		151
 #define H2F_MSG_ISSUE_SYNCOBJ		152
+#define F2H_MSG_SYNCOBJ_QUERY		153
 
 enum gmu_ret_type {
 	GMU_SUCCESS = 0,
@@ -759,9 +760,22 @@ struct hfi_ts_notify_cmd {
 
 
 /* This indicates that the SYNCOBJ is kgsl output fence */
-#define GMU_SYNCOBJ_KGSL_FENCE  BIT(0)
-/* This indicates that the SYNCOBJ is already retired */
-#define GMU_SYNCOBJ_RETIRED     BIT(1)
+#define GMU_SYNCOBJ_FLAG_KGSL_FENCE_BIT		0
+/* This indicates that the SYNCOBJ is signaled */
+#define GMU_SYNCOBJ_FLAG_SIGNALED_BIT		1
+/* This indicates that the SYNCOBJ's software status is queried */
+#define GMU_SYNCOBJ_FLAG_QUERY_SW_STATUS_BIT	2
+/* This indicates that the SYNCOBJ's software status is signaled */
+#define GMU_SYNCOBJ_FLAG_SW_STATUS_SIGNALED_BIT	3
+/* This indicates that the SYNCOBJ's software status is pending */
+#define GMU_SYNCOBJ_FLAG_SW_STATUS_PENDING_BIT	4
+
+#define GMU_SYNCOBJ_FLAGS  \
+	{ BIT(GMU_SYNCOBJ_FLAG_KGSL_FENCE_BIT), "KGSL"}, \
+	{ BIT(GMU_SYNCOBJ_FLAG_SIGNALED_BIT), "SIGNALED"}, \
+	{ BIT(GMU_SYNCOBJ_FLAG_QUERY_SW_STATUS_BIT), "QUERIED"}, \
+	{ BIT(GMU_SYNCOBJ_FLAG_SW_STATUS_SIGNALED_BIT), "SW_SIGNALED"}, \
+	{ BIT(GMU_SYNCOBJ_FLAG_SW_STATUS_PENDING_BIT), "SW_PENDING"}
 
 /* F2H */
 struct hfi_ts_retire_cmd {
@@ -883,6 +897,37 @@ struct hfi_hw_fence_info {
 	u64 hash_index;
 } __packed;
 
+/* The software fence corresponding to the queried hardware fence has not signaled */
+#define ADRENO_HW_FENCE_SW_STATUS_PENDING  BIT(0)
+/* The software fence corresponding to the queried hardware fence has signaled */
+#define ADRENO_HW_FENCE_SW_STATUS_SIGNALED BIT(1)
+
+struct hfi_syncobj_query {
+	/**
+	 * @query_bitmask: Bitmask representing the sync object descriptors to be queried. For
+	 * example, to query the second sync object descriptor(index=1) in a sync object,
+	 * bit(1) should be set in this bitmask.
+	 */
+	u32 query_bitmask;
+} __packed;
+
+#define MAX_SYNCOBJ_QUERY_BITS	128
+#define BITS_PER_SYNCOBJ_QUERY	32
+#define MAX_SYNCOBJ_QUERY_DWORDS (MAX_SYNCOBJ_QUERY_BITS / BITS_PER_SYNCOBJ_QUERY)
+
+struct hfi_syncobj_query_cmd {
+	/** @hdr: Header for the fence info packet */
+	u32 hdr;
+	/** @version: Version of the fence info packet */
+	u32 version;
+	/** @gmu_ctxt_id: GMU Context id to which this SYNC object belongs */
+	u32 gmu_ctxt_id;
+	/** @sync_obj_ts: Timestamp of this SYNC object */
+	u32 sync_obj_ts;
+	/** @queries: Array of query bitmasks */
+	struct hfi_syncobj_query queries[MAX_SYNCOBJ_QUERY_DWORDS];
+} __packed;
+
 /**
  * struct pending_cmd - data structure to track outstanding HFI
  *	command messages
@@ -1022,7 +1067,8 @@ struct payload_section {
 #define GMU_GPU_AQE1_UCODE_ERROR 627
 #define GMU_GPU_AQE1_HW_FAULT_ERROR 628
 #define GMU_GPU_AQE1_ILLEGAL_INST_ERROR 629
-
+/* GMU encountered a sync object which is signaled via software but not via hardware */
+#define GMU_SYNCOBJ_TIMEOUT_ERROR 630
 /* GPU encountered an unknown CP error */
 #define GMU_CP_UNKNOWN_ERROR 700
 
diff --git a/adreno_hwsched.c b/adreno_hwsched.c
index 6497ed4..a03f2db 100644
--- a/adreno_hwsched.c
+++ b/adreno_hwsched.c
@@ -5,6 +5,7 @@
  */
 
 #include <dt-bindings/soc/qcom,ipcc.h>
+#include <linux/dma-fence-array.h>
 #include <linux/soc/qcom/msm_hw_fence.h>
 #include <soc/qcom/msm_performance.h>
 
@@ -16,14 +17,6 @@
 #include "kgsl_timeline.h"
 #include <linux/msm_kgsl.h>
 
-/* This structure represents inflight command object */
-struct cmd_list_obj {
-	/** @drawobj: Handle to the draw object */
-	struct kgsl_drawobj *drawobj;
-	/** @node: List node to put it in the list of inflight commands */
-	struct list_head node;
-};
-
 /*
  * Number of commands that can be queued in a context before it sleeps
  *
@@ -1707,6 +1700,62 @@ static bool context_is_throttled(struct kgsl_device *device,
 	return false;
 }
 
+static void _print_syncobj(struct adreno_device *adreno_dev, struct kgsl_drawobj *drawobj)
+{
+	int i, j, fence_index = 0;
+	struct kgsl_drawobj_sync *syncobj = SYNCOBJ(drawobj);
+	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
+
+	for (i = 0; i < syncobj->numsyncs; i++) {
+		struct kgsl_drawobj_sync_event *event = &syncobj->synclist[i];
+		struct kgsl_sync_fence_cb *kcb = event->handle;
+		struct dma_fence **fences;
+		struct dma_fence_array *array;
+		u32 num_fences;
+
+		array = to_dma_fence_array(kcb->fence);
+		if (array != NULL) {
+			num_fences = array->num_fences;
+			fences = array->fences;
+		} else {
+			num_fences = 1;
+			fences = &kcb->fence;
+		}
+
+		for (j = 0; j < num_fences; j++, fence_index++) {
+			bool kgsl = is_kgsl_fence(fences[j]);
+			bool signaled = test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fences[j]->flags);
+			char value[32] = "unknown";
+
+			if (fences[j]->ops->timeline_value_str)
+				fences[j]->ops->timeline_value_str(fences[j], value, sizeof(value));
+
+			dev_err(device->dev,
+				"dma fence[%d] signaled:%d kgsl:%d ctx:%lu seqno:%lu value:%s\n",
+				fence_index, signaled, kgsl, fences[j]->context, fences[j]->seqno,
+				value);
+		}
+	}
+
+}
+
+static void print_fault_syncobj(struct adreno_device *adreno_dev,
+				u32 ctxt_id, u32 ts)
+{
+	struct adreno_hwsched *hwsched = &adreno_dev->hwsched;
+	struct cmd_list_obj *obj;
+
+	list_for_each_entry(obj, &hwsched->cmd_list, node) {
+		struct kgsl_drawobj *drawobj = obj->drawobj;
+
+		if (drawobj->type == SYNCOBJ_TYPE) {
+			if ((ctxt_id == drawobj->context->id) &&
+			(ts == drawobj->timestamp))
+				_print_syncobj(adreno_dev, drawobj);
+		}
+	}
+}
+
 static void adreno_hwsched_reset_and_snapshot_legacy(struct adreno_device *adreno_dev, int fault)
 {
 	struct kgsl_drawobj *drawobj = NULL;
@@ -1723,6 +1772,12 @@ static void adreno_hwsched_reset_and_snapshot_legacy(struct adreno_device *adren
 	if (hwsched->recurring_cmdobj)
 		srcu_notifier_call_chain(&device->nh, GPU_SSR_BEGIN, NULL);
 
+	if (cmd->error == GMU_SYNCOBJ_TIMEOUT_ERROR) {
+		print_fault_syncobj(adreno_dev, cmd->ctxt_id, cmd->ts);
+		kgsl_device_snapshot(device, NULL, NULL, true);
+		goto done;
+	}
+
 	/*
 	 * First, try to see if the faulted command object is marked
 	 * in case there was a context bad hfi. But, with stall-on-fault,
@@ -1795,6 +1850,12 @@ static void adreno_hwsched_reset_and_snapshot(struct adreno_device *adreno_dev,
 	if (hwsched->recurring_cmdobj)
 		srcu_notifier_call_chain(&device->nh, GPU_SSR_BEGIN, NULL);
 
+	if (cmd->error == GMU_SYNCOBJ_TIMEOUT_ERROR) {
+		print_fault_syncobj(adreno_dev, cmd->gc.ctxt_id, cmd->gc.ts);
+		kgsl_device_snapshot(device, NULL, NULL, true);
+		goto done;
+	}
+
 	/*
 	 * First, try to see if the faulted command object is marked
 	 * in case there was a context bad hfi. But, with stall-on-fault,
diff --git a/adreno_hwsched.h b/adreno_hwsched.h
index c9bb3d0..962f044 100644
--- a/adreno_hwsched.h
+++ b/adreno_hwsched.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef _ADRENO_HWSCHED_H_
@@ -11,6 +11,14 @@
 
 #include "kgsl_sync.h"
 
+/* This structure represents inflight command object */
+struct cmd_list_obj {
+	/** @drawobj: Handle to the draw object */
+	struct kgsl_drawobj *drawobj;
+	/** @node: List node to put it in the list of inflight commands */
+	struct list_head node;
+};
+
 /**
  * struct adreno_hw_fence_entry - A structure to store hardware fence and the context
  */
diff --git a/adreno_trace.h b/adreno_trace.h
index ff91810..8e7c355 100644
--- a/adreno_trace.h
+++ b/adreno_trace.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2013-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #if !defined(_ADRENO_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
@@ -56,6 +56,33 @@ TRACE_EVENT(adreno_cmdbatch_queued,
 	)
 );
 
+TRACE_EVENT(adreno_hw_fence_query,
+	TP_PROTO(u64 context, u64 seqno, u32 flags, const char *name, const char *val),
+	TP_ARGS(context, seqno, flags, name, val),
+	TP_STRUCT__entry(
+		__field(u64, context)
+		__field(u64, seqno)
+		__field(u32, flags)
+		__string(fence_name, name)
+		__string(val, val)
+	),
+	TP_fast_assign(
+		__entry->context = context;
+		__entry->seqno = seqno;
+		__entry->flags = flags;
+		__assign_str(fence_name, name);
+		__assign_str(val, val);
+	),
+	TP_printk(
+		"id=%lld seqno=%lld sw_status=%s name=%s val=%s",
+			__entry->context, __entry->seqno,
+			__entry->flags ? __print_flags(__entry->flags, "|",
+				{ ADRENO_HW_FENCE_SW_STATUS_PENDING, "PENDING" },
+				{ ADRENO_HW_FENCE_SW_STATUS_SIGNALED, "SIGNALED" }) : "none",
+			__get_str(fence_name),
+			__get_str(val))
+);
+
 TRACE_EVENT(adreno_input_hw_fence,
 	TP_PROTO(u32 id, u64 context, u64 seqno, u64 flags, const char *name),
 	TP_ARGS(id, context, seqno, flags, name),
@@ -77,11 +104,46 @@ TRACE_EVENT(adreno_input_hw_fence,
 		"ctx=%u id=%lld seqno=%lld flags=%s name=%s",
 			__entry->id,  __entry->context, __entry->seqno,
 			__entry->flags ? __print_flags(__entry->flags, "|",
-				{ GMU_SYNCOBJ_KGSL_FENCE, "KGSL_FENCE" },
-				{ GMU_SYNCOBJ_RETIRED, "RETIRED" }) : "none",
+				GMU_SYNCOBJ_FLAGS) : "none",
 			__get_str(fence_name))
 );
 
+TRACE_EVENT(adreno_syncobj_query_reply,
+	TP_PROTO(u32 id, u32 timestamp, uint64_t ticks),
+	TP_ARGS(id, timestamp, ticks),
+	TP_STRUCT__entry(
+		__field(u32, id)
+		__field(u32, timestamp)
+		__field(uint64_t, ticks)
+	),
+	TP_fast_assign(
+		__entry->id = id;
+		__entry->timestamp = timestamp;
+		__entry->ticks = ticks;
+	),
+	TP_printk(
+		"ctx=%u ts=%u ticks=%lld",
+			__entry->id, __entry->timestamp, __entry->ticks)
+);
+
+TRACE_EVENT(adreno_syncobj_query,
+	TP_PROTO(u32 id, u32 timestamp, uint64_t ticks),
+	TP_ARGS(id, timestamp, ticks),
+	TP_STRUCT__entry(
+		__field(u32, id)
+		__field(u32, timestamp)
+		__field(uint64_t, ticks)
+	),
+	TP_fast_assign(
+		__entry->id = id;
+		__entry->timestamp = timestamp;
+		__entry->ticks = ticks;
+	),
+	TP_printk(
+		"ctx=%u ts=%u ticks=%lld",
+			__entry->id, __entry->timestamp, __entry->ticks)
+);
+
 TRACE_EVENT(adreno_syncobj_submitted,
 	TP_PROTO(u32 id, u32 timestamp, u32 num_syncobj,
 		uint64_t ticks),