diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-05-23 20:10:02 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-05-23 20:10:02 +0000 |
commit | c7629bd6ee571a4c25ab0d28bb7c985343b5390d (patch) | |
tree | 770bbfc55b6f2c6ac533b47f6ba4838baa6b0736 | |
parent | f364597eb683e2b774b96af4383079153a243048 (diff) | |
parent | 0c19c9fa8f8f829376ac7d07d14ae935f98c2c7e (diff) | |
download | lwis-android-gs-lynx-5.10-u-beta3.tar.gz |
Snap for 10128799 from 0c19c9fa8f8f829376ac7d07d14ae935f98c2c7e to android13-gs-pixel-5.10-releaseandroid-u-beta-3_r0.5android-u-beta-3_r0.4android-gs-pantah-5.10-u-beta3android-gs-lynx-5.10-u-beta3
Change-Id: If1f5ec20e8b860b1ddfc38a2b64c337d2df8588e
52 files changed, 4634 insertions, 1534 deletions
@@ -3,6 +3,7 @@ lwis-objs += lwis_device_dpm.o lwis-objs += lwis_device_i2c.o lwis-objs += lwis_device_ioreg.o lwis-objs += lwis_device_slc.o +lwis-objs += lwis_device_test.o lwis-objs += lwis_device_top.o lwis-objs += lwis_clock.o lwis-objs += lwis_gpio.o @@ -22,6 +23,7 @@ lwis-objs += lwis_debug.o lwis-objs += lwis_io_entry.o lwis-objs += lwis_allocator.o lwis-objs += lwis_version.o +lwis-objs += lwis_fence.o # Anchorage specific files ifeq ($(CONFIG_SOC_GS101), y) @@ -42,4 +44,4 @@ endif obj-$(CONFIG_LWIS) += lwis.o -ccflags-y = -I$(abspath $(KERNEL_SRC)/$(M)) -I$(abspath $(KBUILD_SRC)/drivers/soc/google) +ccflags-y += -I$(abspath $(KERNEL_SRC)/$(M)) -I$(abspath $(KBUILD_SRC)/drivers/soc/google) diff --git a/lwis_allocator.c b/lwis_allocator.c index 0250d04..0e4d55d 100644 --- a/lwis_allocator.c +++ b/lwis_allocator.c @@ -13,7 +13,6 @@ #include <linux/mm.h> #include <linux/mutex.h> -#include <linux/preempt.h> #include <linux/slab.h> #include "lwis_allocator.h" #include "lwis_commands.h" @@ -118,6 +117,9 @@ allocator_get_block_pool(struct lwis_allocator_block_mgr *block_mgr, int idx) struct lwis_allocator_block_pool *block_pool; switch (idx) { + case 12: + block_pool = &block_mgr->pool_4k; + break; case 13: block_pool = &block_mgr->pool_8k; break; @@ -179,14 +181,15 @@ int lwis_allocator_init(struct lwis_device *lwis_dev) hash_init(block_mgr->allocated_blocks); /* Initialize block pools */ - strlcpy(block_mgr->pool_8k.name, "lwis-block-8k", LWIS_MAX_NAME_STRING_LEN); - strlcpy(block_mgr->pool_16k.name, "lwis-block-16k", LWIS_MAX_NAME_STRING_LEN); - strlcpy(block_mgr->pool_32k.name, "lwis-block-32k", LWIS_MAX_NAME_STRING_LEN); - strlcpy(block_mgr->pool_64k.name, "lwis-block-64k", LWIS_MAX_NAME_STRING_LEN); - strlcpy(block_mgr->pool_128k.name, "lwis-block-128k", LWIS_MAX_NAME_STRING_LEN); - strlcpy(block_mgr->pool_256k.name, "lwis-block-256k", LWIS_MAX_NAME_STRING_LEN); - strlcpy(block_mgr->pool_512k.name, "lwis-block-512k", LWIS_MAX_NAME_STRING_LEN); - strlcpy(block_mgr->pool_large.name, "lwis-block-large", LWIS_MAX_NAME_STRING_LEN); + strscpy(block_mgr->pool_4k.name, "lwis-block-4k", LWIS_MAX_NAME_STRING_LEN); + strscpy(block_mgr->pool_8k.name, "lwis-block-8k", LWIS_MAX_NAME_STRING_LEN); + strscpy(block_mgr->pool_16k.name, "lwis-block-16k", LWIS_MAX_NAME_STRING_LEN); + strscpy(block_mgr->pool_32k.name, "lwis-block-32k", LWIS_MAX_NAME_STRING_LEN); + strscpy(block_mgr->pool_64k.name, "lwis-block-64k", LWIS_MAX_NAME_STRING_LEN); + strscpy(block_mgr->pool_128k.name, "lwis-block-128k", LWIS_MAX_NAME_STRING_LEN); + strscpy(block_mgr->pool_256k.name, "lwis-block-256k", LWIS_MAX_NAME_STRING_LEN); + strscpy(block_mgr->pool_512k.name, "lwis-block-512k", LWIS_MAX_NAME_STRING_LEN); + strscpy(block_mgr->pool_large.name, "lwis-block-large", LWIS_MAX_NAME_STRING_LEN); /* Initialize reference count */ block_mgr->ref_count = 1; @@ -220,6 +223,7 @@ void lwis_allocator_release(struct lwis_device *lwis_dev) return; } + allocator_block_pool_free_locked(lwis_dev, &block_mgr->pool_4k); allocator_block_pool_free_locked(lwis_dev, &block_mgr->pool_8k); allocator_block_pool_free_locked(lwis_dev, &block_mgr->pool_16k); allocator_block_pool_free_locked(lwis_dev, &block_mgr->pool_32k); @@ -233,7 +237,7 @@ void lwis_allocator_release(struct lwis_device *lwis_dev) mutex_unlock(&lwis_dev->client_lock); } -void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size) +void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size, gfp_t gfp_flags) { struct lwis_allocator_block_mgr *block_mgr; struct lwis_allocator_block_pool *block_pool; @@ -253,15 +257,6 @@ void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size) } /* - * Linux already has slab allocator to cache the allocated memory within a page. - * The default page size is 4K. We can leverage linux's slab implementation for - * small size memory recycling. - */ - if (size <= 4 * 1024) { - return kmalloc(size, GFP_KERNEL); - } - - /* fls() has better performance profile, it's currently used to mimic the behavior of kmalloc_index(). @@ -292,6 +287,11 @@ void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size) */ idx = fls(size - 1); + /* Set 4K as the minimal block size */ + if (idx < 12) { + idx = 12; + } + /* * For the large size memory allocation, we usually use kvmalloc() to allocate * the memory, but kvmalloc() does not take advantage of slab. For this case, @@ -301,7 +301,7 @@ void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size) * memory on hand. */ if (idx > 19) { - block = kmalloc(sizeof(struct lwis_allocator_block), GFP_KERNEL); + block = kmalloc(sizeof(struct lwis_allocator_block), gfp_flags); if (block == NULL) { dev_err(lwis_dev->dev, "Allocate failed\n"); return NULL; @@ -309,7 +309,7 @@ void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size) block->type = idx; block->next = NULL; block->prev = NULL; - block->ptr = kvmalloc(size, GFP_KERNEL); + block->ptr = kvmalloc(size, gfp_flags); if (block->ptr == NULL) { dev_err(lwis_dev->dev, "Allocate failed\n"); kfree(block); @@ -336,7 +336,7 @@ void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size) } /* Allocate new block */ - block = kmalloc(sizeof(struct lwis_allocator_block), GFP_KERNEL); + block = kmalloc(sizeof(struct lwis_allocator_block), gfp_flags); if (block == NULL) { dev_err(lwis_dev->dev, "Allocate failed\n"); return NULL; @@ -345,7 +345,7 @@ void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size) block->next = NULL; block->prev = NULL; block_size = 1 << idx; - block->ptr = kvmalloc(block_size, GFP_KERNEL); + block->ptr = kvmalloc(block_size, gfp_flags); if (block->ptr == NULL) { dev_err(lwis_dev->dev, "Allocate failed\n"); kfree(block); @@ -390,6 +390,7 @@ void lwis_allocator_free(struct lwis_device *lwis_dev, void *ptr) } if (block == NULL) { + dev_err(lwis_dev->dev, "Allocator free ptr not found\n"); kfree(ptr); return; } diff --git a/lwis_allocator.h b/lwis_allocator.h index c85f88b..b72f0ba 100644 --- a/lwis_allocator.h +++ b/lwis_allocator.h @@ -34,6 +34,7 @@ struct lwis_allocator_block_pool { struct lwis_allocator_block_mgr { spinlock_t lock; + struct lwis_allocator_block_pool pool_4k; struct lwis_allocator_block_pool pool_8k; struct lwis_allocator_block_pool pool_16k; struct lwis_allocator_block_pool pool_32k; @@ -61,7 +62,7 @@ void lwis_allocator_release(struct lwis_device *lwis_dev); /* * lwis_allocator_allocate: Allocate a block from the recycling memory allocator */ -void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size); +void *lwis_allocator_allocate(struct lwis_device *lwis_dev, size_t size, gfp_t gfp_flags); /* * lwis_allocator_free: Free a block to the recycling memory allocator diff --git a/lwis_clock.c b/lwis_clock.c index 4a9dc18..e03fe9d 100644 --- a/lwis_clock.c +++ b/lwis_clock.c @@ -82,7 +82,7 @@ int lwis_clock_get(struct lwis_clock_list *list, char *name, struct device *dev, /* Make sure clock exists */ clk = devm_clk_get(dev, name); - if (IS_ERR(clk)) { + if (IS_ERR_OR_NULL(clk)) { pr_err("Clock %s not found\n", name); return PTR_ERR(clk); } diff --git a/lwis_commands.h b/lwis_commands.h index 2e1232c..ba0db3a 100644 --- a/lwis_commands.h +++ b/lwis_commands.h @@ -33,6 +33,16 @@ extern "C" { */ /* + * Device tree strings have a maximum length of 31, according to specs. + * Adding 1 byte for the null character. + */ +#define LWIS_MAX_NAME_STRING_LEN 32 +/* Maximum clock number defined in device tree. */ +#define LWIS_MAX_CLOCK_NUM 20 +/* Maximum number of register blocks per device */ +#define LWIS_MAX_REG_NUM 20 + +/* * lwis_device_types * top : top level device that overlooks all the LWIS devices. Will be used to * list the information of the other LWIS devices in the system. @@ -40,31 +50,29 @@ extern "C" { * ioreg: for controlling mapped register I/O devices * slc : for configuring system level cache partitions * dpm : for dynamic power manager requests update. + * test : for test-specific devices. */ -#define DEVICE_TYPE_UNKNOWN -1 -#define DEVICE_TYPE_TOP 0 -#define DEVICE_TYPE_I2C 1 -#define DEVICE_TYPE_IOREG 2 -#define DEVICE_TYPE_SLC 3 -#define DEVICE_TYPE_DPM 4 -#define NUM_DEVICE_TYPES 5 - -// Qos clock family. -#define CLOCK_FAMILY_INVALID -1 -#define CLOCK_FAMILY_CAM 0 -#define CLOCK_FAMILY_INTCAM 1 -#define CLOCK_FAMILY_TNR 2 -#define CLOCK_FAMILY_MIF 3 -#define CLOCK_FAMILY_INT 4 -#define NUM_CLOCK_FAMILY 5 - -/* Device tree strings have a maximum length of 31, according to specs. - Adding 1 byte for the null character. */ -#define LWIS_MAX_NAME_STRING_LEN 32 -/* Maximum clock number defined in device tree. */ -#define LWIS_MAX_CLOCK_NUM 20 -/* Maximum reg number defined in device tree. */ -#define LWIS_MAX_REG_NUM 20 +enum lwis_device_types { + DEVICE_TYPE_UNKNOWN = -1, + DEVICE_TYPE_TOP, + DEVICE_TYPE_I2C, + DEVICE_TYPE_IOREG, + DEVICE_TYPE_SLC, + DEVICE_TYPE_DPM, + DEVICE_TYPE_TEST, + NUM_DEVICE_TYPES +}; + +/* Qos clock family. */ +enum lwis_clock_family { + CLOCK_FAMILY_INVALID = -1, + CLOCK_FAMILY_CAM, + CLOCK_FAMILY_INTCAM, + CLOCK_FAMILY_TNR, + CLOCK_FAMILY_MIF, + CLOCK_FAMILY_INT, + NUM_CLOCK_FAMILY +}; struct lwis_clk_setting { // clock name defined in device tree. @@ -240,6 +248,13 @@ struct lwis_echo { #define LWIS_TRANSACTION_FAILURE_EVENT_FLAG (1ULL << 62) #define LWIS_HW_IRQ_EVENT_FLAG (1ULL << 61) #define LWIS_PERIODIC_IO_EVENT_FLAG (1ULL << 60) +#define LWIS_OVERFLOW_IRQ_EVENT_FLAG (1ULL << 59) + +// Status code for unsignaled LWIS fence +#define LWIS_FENCE_STATUS_NOT_SIGNALED -1 + +// Interval in ms for the Heartbeat Event if enabled +#define LWIS_HEARTBEAT_EVENT_INTERVAL_MS 10 struct lwis_event_info { // IOCTL Inputs @@ -268,10 +283,49 @@ struct lwis_event_control_list { struct lwis_event_control *event_controls; }; +enum lwis_transaction_trigger_node_types { + LWIS_TRIGGER_EVENT, + LWIS_TRIGGER_FENCE, + LWIS_TRIGGER_FENCE_PLACEHOLDER +}; + +struct lwis_transaction_trigger_event { + int64_t id; + int64_t counter; + int32_t precondition_fence_fd; +}; + +struct lwis_transaction_trigger_node { + int32_t type; //lwis_transaction_trigger_node_types + union { + int32_t fence_fd; + struct lwis_transaction_trigger_event event; + }; +}; + +enum lwis_transaction_trigger_node_operator { + LWIS_TRIGGER_NODE_OPERATOR_INVALID = -1, + LWIS_TRIGGER_NODE_OPERATOR_NONE, + LWIS_TRIGGER_NODE_OPERATOR_AND, + LWIS_TRIGGER_NODE_OPERATOR_OR, +}; + +#define LWIS_TRIGGER_NODES_MAX_NUM 16 +struct lwis_transaction_trigger_condition { + size_t num_nodes; + int32_t operator_type; //lwis_transaction_trigger_node_operator + struct lwis_transaction_trigger_node trigger_nodes[LWIS_TRIGGER_NODES_MAX_NUM]; +}; + +// Status code for completion fences +#define LWIS_NO_COMPLETION_FENCE -1 +#define LWIS_CREATE_COMPLETION_FENCE -2 + // Invalid ID for Transaction id and Periodic IO id #define LWIS_ID_INVALID (-1LL) #define LWIS_EVENT_COUNTER_ON_NEXT_OCCURRENCE (-1LL) #define LWIS_EVENT_COUNTER_EVERY_TIME (-2LL) + struct lwis_transaction_info { // Input int64_t trigger_event_id; @@ -279,10 +333,33 @@ struct lwis_transaction_info { size_t num_io_entries; struct lwis_io_entry *io_entries; bool run_in_event_context; - bool run_at_real_time; + // Use reserved to keep the original interface + bool reserved; + int64_t emit_success_event_id; + int64_t emit_error_event_id; + bool is_level_triggered; + // Output + int64_t id; + // Only will be set if trigger_event_id is specified. + // Otherwise, the value is -1. + int64_t current_trigger_event_counter; + int64_t submission_timestamp_ns; +}; + +struct lwis_transaction_info_v2 { + // Input + int64_t trigger_event_id; + int64_t trigger_event_counter; + struct lwis_transaction_trigger_condition trigger_condition; + int32_t completion_fence_fd; + size_t num_io_entries; + struct lwis_io_entry *io_entries; + bool run_in_event_context; + // Use reserved to keep the original interface + bool reserved; int64_t emit_success_event_id; int64_t emit_error_event_id; - bool allow_counter_eq; + bool is_level_triggered; // Output int64_t id; // Only will be set if trigger_event_id is specified. @@ -356,6 +433,25 @@ struct lwis_qos_setting { int64_t rt_bw; }; +struct lwis_qos_setting_v2 { + // Frequency in hz. + int64_t frequency_hz; + // Device id for this vote. + int32_t device_id; + // Target clock family. + int32_t clock_family; + // read BW + int64_t read_bw; + // write BW + int64_t write_bw; + // peak BW + int64_t peak_bw; + // RT BW (total peak) + int64_t rt_bw; + // Bts client name + char bts_block_name[LWIS_MAX_NAME_STRING_LEN]; +}; + struct lwis_dpm_qos_requirements { // qos entities from user. struct lwis_qos_setting *qos_settings; @@ -363,39 +459,178 @@ struct lwis_dpm_qos_requirements { size_t num_settings; }; +struct lwis_dpm_qos_requirements_v2 { + // qos entities from user. + struct lwis_qos_setting_v2 *qos_settings; + // number of qos_settings. + size_t num_settings; +}; + +enum lwis_cmd_id { + LWIS_CMD_ID_ECHO = 0x100, + LWIS_CMD_ID_TIME_QUERY = 0x200, + + LWIS_CMD_ID_GET_DEVICE_INFO = 0x10000, + LWIS_CMD_ID_DEVICE_ENABLE = 0x10100, + LWIS_CMD_ID_DEVICE_DISABLE = 0x10200, + LWIS_CMD_ID_DEVICE_RESET = 0x10300, + LWIS_CMD_ID_DEVICE_SUSPEND = 0x10400, + LWIS_CMD_ID_DEVICE_RESUME = 0x10500, + LWIS_CMD_ID_DUMP_DEBUG_STATE = 0x10600, + + LWIS_CMD_ID_DMA_BUFFER_ENROLL = 0x20000, + LWIS_CMD_ID_DMA_BUFFER_DISENROLL = 0x20100, + LWIS_CMD_ID_DMA_BUFFER_CPU_ACCESS = 0x20200, + LWIS_CMD_ID_DMA_BUFFER_ALLOC = 0x20300, + LWIS_CMD_ID_DMA_BUFFER_FREE = 0x20400, + + LWIS_CMD_ID_REG_IO = 0x30000, + + LWIS_CMD_ID_EVENT_CONTROL_GET = 0x40000, + LWIS_CMD_ID_EVENT_CONTROL_SET = 0x40100, + LWIS_CMD_ID_EVENT_DEQUEUE = 0x40200, + + LWIS_CMD_ID_TRANSACTION_SUBMIT = 0x50000, + LWIS_CMD_ID_TRANSACTION_SUBMIT_V2, + LWIS_CMD_ID_TRANSACTION_CANCEL = 0x50100, + LWIS_CMD_ID_TRANSACTION_REPLACE = 0x50200, + LWIS_CMD_ID_TRANSACTION_REPLACE_V2, + + LWIS_CMD_ID_PERIODIC_IO_SUBMIT = 0x60000, + LWIS_CMD_ID_PERIODIC_IO_CANCEL = 0x60100, + + LWIS_CMD_ID_DPM_CLK_UPDATE = 0x70000, + LWIS_CMD_ID_DPM_QOS_UPDATE = 0x70100, + LWIS_CMD_ID_DPM_QOS_UPDATE_V2, + LWIS_CMD_ID_DPM_GET_CLOCK = 0x70200, + + LWIS_CMD_ID_FENCE_CREATE = 0x80000, + + LWIS_CMD_ID_EVENT_INJECTION = 0x90000 +}; + +struct lwis_cmd_pkt { + uint32_t cmd_id; + int32_t ret_code; + struct lwis_cmd_pkt *next; +}; + +struct lwis_cmd_echo { + struct lwis_cmd_pkt header; + struct lwis_echo msg; +}; + +struct lwis_cmd_time_query { + struct lwis_cmd_pkt header; + int64_t timestamp_ns; +}; + +struct lwis_cmd_device_info { + struct lwis_cmd_pkt header; + struct lwis_device_info info; +}; + +struct lwis_cmd_io_entries { + struct lwis_cmd_pkt header; + struct lwis_io_entries io; +}; + +struct lwis_cmd_dma_buffer_enroll { + struct lwis_cmd_pkt header; + struct lwis_buffer_info info; +}; + +struct lwis_cmd_dma_buffer_disenroll { + struct lwis_cmd_pkt header; + struct lwis_enrolled_buffer_info info; +}; + +struct lwis_cmd_dma_buffer_cpu_access { + struct lwis_cmd_pkt header; + struct lwis_buffer_cpu_access_op op; +}; + +struct lwis_cmd_dma_buffer_alloc { + struct lwis_cmd_pkt header; + struct lwis_alloc_buffer_info info; +}; + +struct lwis_cmd_dma_buffer_free { + struct lwis_cmd_pkt header; + int32_t fd; +}; + +struct lwis_cmd_event_control_get { + struct lwis_cmd_pkt header; + struct lwis_event_control ctl; +}; + +struct lwis_cmd_event_control_set { + struct lwis_cmd_pkt header; + struct lwis_event_control_list list; +}; + +struct lwis_cmd_event_dequeue { + struct lwis_cmd_pkt header; + struct lwis_event_info info; +}; + +struct lwis_cmd_transaction_info { + struct lwis_cmd_pkt header; + struct lwis_transaction_info info; +}; + +struct lwis_cmd_transaction_info_v2 { + struct lwis_cmd_pkt header; + struct lwis_transaction_info_v2 info; +}; + +struct lwis_cmd_transaction_cancel { + struct lwis_cmd_pkt header; + int64_t id; +}; + +struct lwis_cmd_periodic_io_info { + struct lwis_cmd_pkt header; + struct lwis_periodic_io_info info; +}; + +struct lwis_cmd_periodic_io_cancel { + struct lwis_cmd_pkt header; + int64_t id; +}; + +struct lwis_cmd_dpm_clk_update { + struct lwis_cmd_pkt header; + struct lwis_dpm_clk_settings settings; +}; + +struct lwis_cmd_dpm_qos_update { + struct lwis_cmd_pkt header; + struct lwis_dpm_qos_requirements reqs; +}; + +struct lwis_cmd_dpm_qos_update_v2 { + struct lwis_cmd_pkt header; + struct lwis_dpm_qos_requirements_v2 reqs; +}; + +struct lwis_cmd_dpm_clk_get { + struct lwis_cmd_pkt header; + struct lwis_qos_setting setting; +}; + +struct lwis_cmd_fence_create { + struct lwis_cmd_pkt header; + int32_t fd; +}; + /* * IOCTL Commands */ #define LWIS_IOC_TYPE 'L' - -#define LWIS_GET_DEVICE_INFO _IOWR(LWIS_IOC_TYPE, 1, struct lwis_device_info) -#define LWIS_BUFFER_ENROLL _IOWR(LWIS_IOC_TYPE, 2, struct lwis_buffer_info) -#define LWIS_BUFFER_DISENROLL _IOWR(LWIS_IOC_TYPE, 3, struct lwis_enrolled_buffer_info) -#define LWIS_BUFFER_CPU_ACCESS _IOWR(LWIS_IOC_TYPE, 4, struct lwis_buffer_cpu_access_op) -#define LWIS_DEVICE_ENABLE _IO(LWIS_IOC_TYPE, 6) -#define LWIS_DEVICE_DISABLE _IO(LWIS_IOC_TYPE, 7) -#define LWIS_BUFFER_ALLOC _IOWR(LWIS_IOC_TYPE, 8, struct lwis_alloc_buffer_info) -#define LWIS_BUFFER_FREE _IOWR(LWIS_IOC_TYPE, 9, int32_t) -#define LWIS_TIME_QUERY _IOWR(LWIS_IOC_TYPE, 10, int64_t) -#define LWIS_REG_IO _IOWR(LWIS_IOC_TYPE, 11, struct lwis_io_entries) -#define LWIS_ECHO _IOWR(LWIS_IOC_TYPE, 12, struct lwis_echo) -#define LWIS_DEVICE_RESET _IOWR(LWIS_IOC_TYPE, 13, struct lwis_io_entries) - -#define LWIS_EVENT_CONTROL_GET _IOWR(LWIS_IOC_TYPE, 20, struct lwis_event_control) -#define LWIS_EVENT_CONTROL_SET _IOW(LWIS_IOC_TYPE, 21, struct lwis_event_control_list) -#define LWIS_EVENT_DEQUEUE _IOWR(LWIS_IOC_TYPE, 22, struct lwis_event_info) - -#define LWIS_TRANSACTION_SUBMIT _IOWR(LWIS_IOC_TYPE, 30, struct lwis_transaction_info) -#define LWIS_TRANSACTION_CANCEL _IOWR(LWIS_IOC_TYPE, 31, int64_t) -#define LWIS_TRANSACTION_REPLACE _IOWR(LWIS_IOC_TYPE, 32, struct lwis_transaction_info) - -#define LWIS_PERIODIC_IO_SUBMIT _IOWR(LWIS_IOC_TYPE, 40, struct lwis_periodic_io_info) -#define LWIS_PERIODIC_IO_CANCEL _IOWR(LWIS_IOC_TYPE, 41, int64_t) - -#define LWIS_DPM_CLK_UPDATE _IOW(LWIS_IOC_TYPE, 50, struct lwis_dpm_clk_settings) -#define LWIS_DPM_QOS_UPDATE _IOW(LWIS_IOC_TYPE, 51, struct lwis_dpm_qos_requirements) -#define LWIS_DPM_GET_CLOCK _IOW(LWIS_IOC_TYPE, 52, struct lwis_qos_setting) +#define LWIS_CMD_PACKET _IOWR(LWIS_IOC_TYPE, 100, struct lwis_cmd_pkt) /* * Event payloads diff --git a/lwis_debug.c b/lwis_debug.c index 882f9f7..8e5927e 100644 --- a/lwis_debug.c +++ b/lwis_debug.c @@ -99,10 +99,17 @@ static void list_transactions(struct lwis_client *client, char *k_buf, size_t k_ trans_hist->info.emit_success_event_id, trans_hist->info.emit_error_event_id); strlcat(k_buf, tmp_buf, k_buf_size); - scnprintf(tmp_buf, sizeof(tmp_buf), - " Num Entries: %zu Processed @ %lld for %lldns\n", - trans_hist->info.num_io_entries, trans_hist->process_timestamp, - trans_hist->process_duration_ns); + /* Process timestamp not recorded */ + if (trans_hist->process_timestamp == -1) { + scnprintf(tmp_buf, sizeof(tmp_buf), " Num Entries: %zu\n", + trans_hist->info.num_io_entries); + } else { + scnprintf(tmp_buf, sizeof(tmp_buf), + " Num Entries: %zu Processed @ %lld for %lldns\n", + trans_hist->info.num_io_entries, + trans_hist->process_timestamp, + trans_hist->process_duration_ns); + } strlcat(k_buf, tmp_buf, k_buf_size); } hist_idx++; @@ -177,7 +184,7 @@ static int generate_device_info(struct lwis_device *lwis_dev, char *buffer, size } static int generate_event_states_info(struct lwis_device *lwis_dev, char *buffer, - size_t buffer_size) + size_t buffer_size, int lwis_event_dump_cnt) { /* Temporary buffer to be concatenated to the main buffer. */ char tmp_buf[96] = {}; @@ -186,13 +193,27 @@ static int generate_event_states_info(struct lwis_device *lwis_dev, char *buffer unsigned long flags; struct lwis_device_event_state *state; bool enabled_event_present = false; + int traverse_last_events_size; if (lwis_dev == NULL) { pr_err("Unknown LWIS device pointer\n"); return -EINVAL; } - scnprintf(buffer, buffer_size, "=== LWIS EVENT STATES INFO: %s ===\n", lwis_dev->name); + if (lwis_event_dump_cnt >= 0 && lwis_event_dump_cnt <= EVENT_DEBUG_HISTORY_SIZE) { + scnprintf(tmp_buf, sizeof(tmp_buf), "=== LWIS DUMP LAST %d Received Events ===\n", + lwis_event_dump_cnt); + strlcat(buffer, tmp_buf, buffer_size); + traverse_last_events_size = lwis_event_dump_cnt; + } else if (lwis_event_dump_cnt > EVENT_DEBUG_HISTORY_SIZE) { + pr_err("lwis_event_dump_cnt (%d) exceed EVENT_DEBUG_HISTORY_SIZE (%d) \n", + lwis_event_dump_cnt, EVENT_DEBUG_HISTORY_SIZE); + return -EINVAL; + } else { + scnprintf(buffer, buffer_size, "=== LWIS EVENT STATES INFO: %s ===\n", + lwis_dev->name); + traverse_last_events_size = EVENT_DEBUG_HISTORY_SIZE; + } spin_lock_irqsave(&lwis_dev->lock, flags); if (hash_empty(lwis_dev->event_states)) { @@ -211,9 +232,18 @@ static int generate_event_states_info(struct lwis_device *lwis_dev, char *buffer if (!enabled_event_present) { strlcat(buffer, "No enabled events\n", buffer_size); } - strlcat(buffer, "Last Events:\n", buffer_size); + if (lwis_event_dump_cnt < 0) { + strlcat(buffer, "Last Events:\n", buffer_size); + } + idx = lwis_dev->debug_info.cur_event_hist_idx; - for (i = 0; i < EVENT_DEBUG_HISTORY_SIZE; ++i) { + for (i = 0; i < traverse_last_events_size; ++i) { + if (lwis_event_dump_cnt >= 0) { + if (idx == 0) { + idx = EVENT_DEBUG_HISTORY_SIZE; + } + idx--; + } state = &lwis_dev->debug_info.event_hist[idx].state; /* Skip uninitialized entries */ if (state->event_id != 0) { @@ -223,9 +253,11 @@ static int generate_event_states_info(struct lwis_device *lwis_dev, char *buffer lwis_dev->debug_info.event_hist[idx].timestamp); strlcat(buffer, tmp_buf, buffer_size); } - idx++; - if (idx >= EVENT_DEBUG_HISTORY_SIZE) { - idx = 0; + if (lwis_event_dump_cnt < 0) { + idx++; + if (idx >= EVENT_DEBUG_HISTORY_SIZE) { + idx = 0; + } } } @@ -295,6 +327,101 @@ static int generate_buffer_info(struct lwis_device *lwis_dev, char *buffer, size return 0; } +static int generate_register_io_history(struct lwis_device *lwis_dev, char *buffer, + size_t buffer_size) +{ + /* Temporary buffer to be concatenated to the main buffer. */ + char tmp_buf[128] = {}; + struct lwis_register_io_info *reg_io; + int i, hist_idx; + + if (lwis_dev == NULL) { + pr_err("Unknown LWIS device pointer\n"); + return -EINVAL; + } + + scnprintf(buffer, buffer_size, "=== LWIS REGISTER IO INFO: %s ===\n", lwis_dev->name); + strlcat(buffer, "Last register read/writes:\n", buffer_size); + hist_idx = lwis_dev->debug_info.cur_io_entry_hist_idx; + for (i = 0; i < IO_ENTRY_DEBUG_HISTORY_SIZE; ++i) { + reg_io = &lwis_dev->debug_info.io_entry_hist[hist_idx]; + /* Skip uninitialized entries */ + if (reg_io->start_timestamp != 0) { + if (reg_io->io_entry.type == LWIS_IO_ENTRY_READ) { + scnprintf( + tmp_buf, sizeof(tmp_buf), + "READ: bid %d, offset %llu, val %llu, access_size %lu, start_timestamp %llu\n", + reg_io->io_entry.rw.bid, reg_io->io_entry.rw.offset, + reg_io->io_entry.rw.val, reg_io->access_size, + reg_io->start_timestamp); + strlcat(buffer, tmp_buf, buffer_size); + } else if (reg_io->io_entry.type == LWIS_IO_ENTRY_READ_BATCH) { + scnprintf( + tmp_buf, sizeof(tmp_buf), + "READ_BATCH: bid %d, offset %llu, size_in_bytes %lu, access_size %lu, start_timestamp %llu\n", + reg_io->io_entry.rw_batch.bid, + reg_io->io_entry.rw_batch.offset, + reg_io->io_entry.rw_batch.size_in_bytes, + reg_io->access_size, reg_io->start_timestamp); + strlcat(buffer, tmp_buf, buffer_size); + } else if (reg_io->io_entry.type == LWIS_IO_ENTRY_WRITE) { + scnprintf( + tmp_buf, sizeof(tmp_buf), + "WRITE: bid %d, offset %llu, val %llu, access_size %lu, start_timestamp %llu\n", + reg_io->io_entry.rw.bid, reg_io->io_entry.rw.offset, + reg_io->io_entry.rw.val, reg_io->access_size, + reg_io->start_timestamp); + strlcat(buffer, tmp_buf, buffer_size); + } else if (reg_io->io_entry.type == LWIS_IO_ENTRY_WRITE_BATCH) { + scnprintf( + tmp_buf, sizeof(tmp_buf), + "WRITE_BATCH: bid %d, offset %llu, size_in_bytes %lu, access_size %lu, start_timestamp %llu\n", + reg_io->io_entry.rw_batch.bid, + reg_io->io_entry.rw_batch.offset, + reg_io->io_entry.rw_batch.size_in_bytes, + reg_io->access_size, reg_io->start_timestamp); + strlcat(buffer, tmp_buf, buffer_size); + } else if (reg_io->io_entry.type == LWIS_IO_ENTRY_MODIFY) { + scnprintf( + tmp_buf, sizeof(tmp_buf), + "MODIFY: bid %d, offset %llu, access_size %lu, start_timestamp %llu\n", + reg_io->io_entry.mod.bid, reg_io->io_entry.mod.offset, + reg_io->access_size, reg_io->start_timestamp); + strlcat(buffer, tmp_buf, buffer_size); + } + } + hist_idx++; + if (hist_idx >= IO_ENTRY_DEBUG_HISTORY_SIZE) { + hist_idx = 0; + } + } + + return 0; +} + +int lwis_debug_print_register_io_history(struct lwis_device *lwis_dev) +{ + int ret = 0; + /* Buffer to store information */ + const size_t buffer_size = 10240; + char *buffer = kzalloc(buffer_size, GFP_KERNEL); + if (!buffer) { + dev_err(lwis_dev->dev, "Failed to allocate register io history log buffer\n"); + return -ENOMEM; + } + + ret = generate_register_io_history(lwis_dev, buffer, buffer_size); + if (ret) { + dev_err(lwis_dev->dev, "Failed to generate register io history"); + goto exit; + } + print_to_log(buffer); + +exit: + kfree(buffer); + return 0; +} + int lwis_debug_print_device_info(struct lwis_device *lwis_dev) { int ret = 0; @@ -311,7 +438,7 @@ int lwis_debug_print_device_info(struct lwis_device *lwis_dev) return 0; } -int lwis_debug_print_event_states_info(struct lwis_device *lwis_dev) +int lwis_debug_print_event_states_info(struct lwis_device *lwis_dev, int lwis_event_dump_cnt) { int ret = 0; /* Buffer to store information */ @@ -322,7 +449,7 @@ int lwis_debug_print_event_states_info(struct lwis_device *lwis_dev) return -ENOMEM; } - ret = generate_event_states_info(lwis_dev, buffer, buffer_size); + ret = generate_event_states_info(lwis_dev, buffer, buffer_size, lwis_event_dump_cnt); if (ret) { dev_err(lwis_dev->dev, "Failed to generate event states info"); goto exit; @@ -410,7 +537,7 @@ static ssize_t event_states_read(struct file *fp, char __user *user_buf, size_t return -ENOMEM; } - ret = generate_event_states_info(lwis_dev, buffer, buffer_size); + ret = generate_event_states_info(lwis_dev, buffer, buffer_size, /*lwis_event_dump_cnt=*/-1); if (ret) { dev_err(lwis_dev->dev, "Failed to generate event states info"); goto exit; @@ -471,6 +598,31 @@ exit: return ret; } +static ssize_t register_io_history_read(struct file *fp, char __user *user_buf, size_t count, + loff_t *position) +{ + int ret = 0; + /* Buffer to store information */ + const size_t buffer_size = 10240; + struct lwis_device *lwis_dev = fp->f_inode->i_private; + char *buffer = kzalloc(buffer_size, GFP_KERNEL); + if (!buffer) { + dev_err(lwis_dev->dev, "Failed to allocate register io history log buffer\n"); + return -ENOMEM; + } + + ret = generate_register_io_history(lwis_dev, buffer, buffer_size); + if (ret) { + dev_err(lwis_dev->dev, "Failed to generate register io history"); + goto exit; + } + + ret = simple_read_from_buffer(user_buf, count, position, buffer, strlen(buffer)); +exit: + kfree(buffer); + return ret; +} + static struct file_operations dev_info_fops = { .owner = THIS_MODULE, .read = dev_info_read, @@ -491,6 +643,11 @@ static struct file_operations buffer_info_fops = { .read = buffer_info_read, }; +static struct file_operations register_io_history_fops = { + .owner = THIS_MODULE, + .read = register_io_history_read, +}; + int lwis_device_debugfs_setup(struct lwis_device *lwis_dev, struct dentry *dbg_root) { struct dentry *dbg_dir; @@ -498,6 +655,7 @@ int lwis_device_debugfs_setup(struct lwis_device *lwis_dev, struct dentry *dbg_r struct dentry *dbg_event_file; struct dentry *dbg_transaction_file; struct dentry *dbg_buffer_file; + struct dentry *dbg_reg_io_file; /* DebugFS not present, just return */ if (dbg_root == NULL) { @@ -543,11 +701,20 @@ int lwis_device_debugfs_setup(struct lwis_device *lwis_dev, struct dentry *dbg_r dbg_buffer_file = NULL; } + dbg_reg_io_file = debugfs_create_file("io_history", 0444, dbg_dir, lwis_dev, + ®ister_io_history_fops); + if (IS_ERR_OR_NULL(dbg_reg_io_file)) { + dev_warn(lwis_dev->dev, "Failed to create DebugFS io_history - %ld", + PTR_ERR(dbg_reg_io_file)); + dbg_reg_io_file = NULL; + } + lwis_dev->dbg_dir = dbg_dir; lwis_dev->dbg_dev_info_file = dbg_dev_info_file; lwis_dev->dbg_event_file = dbg_event_file; lwis_dev->dbg_transaction_file = dbg_transaction_file; lwis_dev->dbg_buffer_file = dbg_buffer_file; + lwis_dev->dbg_reg_io_file = dbg_reg_io_file; return 0; } @@ -564,6 +731,7 @@ int lwis_device_debugfs_cleanup(struct lwis_device *lwis_dev) lwis_dev->dbg_event_file = NULL; lwis_dev->dbg_transaction_file = NULL; lwis_dev->dbg_buffer_file = NULL; + lwis_dev->dbg_reg_io_file = NULL; return 0; } diff --git a/lwis_debug.h b/lwis_debug.h index 9c94de0..85996a3 100644 --- a/lwis_debug.h +++ b/lwis_debug.h @@ -10,9 +10,11 @@ #include "lwis_device.h" +/* Functions to print debugging info */ int lwis_debug_print_device_info(struct lwis_device *lwis_dev); -int lwis_debug_print_event_states_info(struct lwis_device *lwis_dev); +int lwis_debug_print_event_states_info(struct lwis_device *lwis_dev, int lwis_event_dump_cnt); int lwis_debug_print_transaction_info(struct lwis_device *lwis_dev); +int lwis_debug_print_register_io_history(struct lwis_device *lwis_dev); int lwis_debug_print_buffer_info(struct lwis_device *lwis_dev); /* DebugFS specific functions */ diff --git a/lwis_device.c b/lwis_device.c index 56ceafc..b4c8a44 100644 --- a/lwis_device.c +++ b/lwis_device.c @@ -28,6 +28,7 @@ #include "lwis_device.h" #include "lwis_device_dpm.h" #include "lwis_device_slc.h" +#include "lwis_device_test.h" #include "lwis_dt.h" #include "lwis_event.h" #include "lwis_gpio.h" @@ -37,7 +38,9 @@ #include "lwis_pinctrl.h" #include "lwis_platform.h" #include "lwis_transaction.h" +#include "lwis_util.h" #include "lwis_version.h" +#include "lwis_trace.h" #ifdef CONFIG_OF #include "lwis_dt.h" @@ -71,6 +74,17 @@ static struct file_operations lwis_fops = { }; /* + * transaction_work_func: + * Function to be called by transaction worker thread to direct it the correct client + * to process their queues + */ +static void transaction_work_func(struct kthread_work *work) +{ + struct lwis_client *client = container_of(work, struct lwis_client, transaction_work); + lwis_process_worker_queue(client); +} + +/* * lwis_open: Opening an instance of a LWIS device */ static int lwis_open(struct inode *node, struct file *fp) @@ -120,6 +134,8 @@ static int lwis_open(struct inode *node, struct file *fp) /* Initialize the allocator */ lwis_allocator_init(lwis_dev); + kthread_init_work(&lwis_client->transaction_work, transaction_work_func); + /* Start transaction processor task */ lwis_transaction_init(lwis_client); @@ -158,7 +174,7 @@ static int lwis_cleanup_client(struct lwis_client *lwis_client) /* Clean up all periodic io state for the client */ lwis_periodic_io_client_cleanup(lwis_client); - /* Cancel all pending transactions for the client */ + /* Cancel all pending transactions for the client and destory workqueue*/ lwis_transaction_clear(lwis_client); /* Run cleanup transactions. */ @@ -221,6 +237,7 @@ static int lwis_release(struct inode *node, struct file *fp) struct lwis_client *lwis_client = fp->private_data; struct lwis_device *lwis_dev = lwis_client->lwis_dev; int rc = 0; + int __maybe_unused i; bool is_client_enabled = lwis_client->is_enabled; dev_info(lwis_dev->dev, "Closing instance %d\n", iminor(node)); @@ -233,17 +250,22 @@ static int lwis_release(struct inode *node, struct file *fp) mutex_lock(&lwis_dev->client_lock); /* Release power if client closed without power down called */ if (is_client_enabled && lwis_dev->enabled > 0) { + lwis_device_crash_info_dump(lwis_dev); lwis_dev->enabled--; if (lwis_dev->enabled == 0) { dev_info(lwis_dev->dev, "No more client, power down\n"); rc = lwis_dev_power_down_locked(lwis_dev); + lwis_dev->is_suspended = false; } } if (lwis_dev->enabled == 0) { - if (lwis_dev->bts_index != BTS_UNSUPPORTED) { - lwis_platform_update_bts(lwis_dev, /*bw_peak=*/0, - /*bw_read=*/0, /*bw_write=*/0, /*bw_rt=*/0); + for (i = 0; i < lwis_dev->bts_block_num; i++) { + if (lwis_dev->bts_indexes[i] != BTS_UNSUPPORTED) { + lwis_platform_update_bts(lwis_dev, i, /*bw_peak=*/0, + /*bw_read=*/0, /*bw_write=*/0, + /*bw_rt=*/0); + } } /* remove voted qos */ lwis_platform_remove_qos(lwis_dev); @@ -366,9 +388,180 @@ static void lwis_assign_top_to_other(struct lwis_device *top_dev) mutex_unlock(&core.lock); } -static int process_power_sequence(struct lwis_device *lwis_dev, - struct lwis_device_power_sequence_list *list, bool set_active, - bool skip_error) +static bool need_to_power_up(struct lwis_device *lwis_dev) +{ + int i; + + if (lwis_dev->power_seq_handler == NULL) { + return true; + } + + mutex_lock(&core.lock); + for (i = 0; i < MAX_UNIFIED_POWER_DEVICE; i++) { + if (core.unified_dev_pwr_map[i].dev_node_seq == NULL) { + break; + } + if (core.unified_dev_pwr_map[i].dev_node_seq == lwis_dev->power_seq_handler) { + if (core.unified_dev_pwr_map[i].count == 0) { + break; + } + mutex_unlock(&core.lock); +#ifdef LWIS_PWR_SEQ_DEBUG + dev_info(lwis_dev->dev, "%s: Already power up\n", __func__); +#endif + return false; + } + } + mutex_unlock(&core.lock); +#ifdef LWIS_PWR_SEQ_DEBUG + dev_info(lwis_dev->dev, "%s: Need power up\n", __func__); +#endif + return true; +} + +static bool need_to_power_down(struct lwis_device *lwis_dev) +{ + int i; + + if (lwis_dev->power_seq_handler == NULL) { + return true; + } + + mutex_lock(&core.lock); + for (i = 0; i < MAX_UNIFIED_POWER_DEVICE; i++) { + if (core.unified_dev_pwr_map[i].dev_node_seq == NULL) { + break; + } + if (core.unified_dev_pwr_map[i].dev_node_seq == lwis_dev->power_seq_handler) { + if (core.unified_dev_pwr_map[i].count == 1) { + break; + } + mutex_unlock(&core.lock); +#ifdef LWIS_PWR_SEQ_DEBUG + dev_info(lwis_dev->dev, "%s: No need power down\n", __func__); +#endif + return false; + } + } + mutex_unlock(&core.lock); +#ifdef LWIS_PWR_SEQ_DEBUG + dev_info(lwis_dev->dev, "%s: Ready to power down\n", __func__); +#endif + return true; +} + +static int increase_unified_power_count(struct lwis_device *lwis_dev) +{ + int i; + + if (lwis_dev->power_seq_handler == NULL) { + return 0; + } + + mutex_lock(&core.lock); + for (i = 0; i < MAX_UNIFIED_POWER_DEVICE; i++) { + if (core.unified_dev_pwr_map[i].dev_node_seq == NULL) { + break; + } + if (core.unified_dev_pwr_map[i].dev_node_seq == lwis_dev->power_seq_handler) { + core.unified_dev_pwr_map[i].count++; + if (core.unified_dev_pwr_map[i].count == 1) { + core.unified_dev_pwr_map[i].hold_dev = lwis_dev; + } + mutex_unlock(&core.lock); +#ifdef LWIS_PWR_SEQ_DEBUG + dev_info(lwis_dev->dev, "%s: power counter = %d\n", __func__, + core.unified_dev_pwr_map[i].count); +#endif + return 0; + } + } + if (i >= MAX_UNIFIED_POWER_DEVICE) { + dev_err(lwis_dev->dev, "Unified power sequence map overflow\n"); + mutex_unlock(&core.lock); + return -EOVERFLOW; + } + + core.unified_dev_pwr_map[i].dev_node_seq = lwis_dev->power_seq_handler; + core.unified_dev_pwr_map[i].hold_dev = lwis_dev; + core.unified_dev_pwr_map[i].count++; + mutex_unlock(&core.lock); + +#ifdef LWIS_PWR_SEQ_DEBUG + dev_info(lwis_dev->dev, "%s: power counter = %d\n", __func__, + core.unified_dev_pwr_map[i].count); +#endif + return 0; +} + +static int decrease_unified_power_count(struct lwis_device *lwis_dev) +{ + int i; + + if (lwis_dev->power_seq_handler == NULL) { + return 0; + } + + mutex_lock(&core.lock); + for (i = 0; i < MAX_UNIFIED_POWER_DEVICE; i++) { + if (core.unified_dev_pwr_map[i].dev_node_seq == NULL) { + break; + } + if (core.unified_dev_pwr_map[i].dev_node_seq == lwis_dev->power_seq_handler) { + if (core.unified_dev_pwr_map[i].count > 0) { + core.unified_dev_pwr_map[i].count--; + if (core.unified_dev_pwr_map[i].count == 0) { + core.unified_dev_pwr_map[i].hold_dev = NULL; + } + } + mutex_unlock(&core.lock); +#ifdef LWIS_PWR_SEQ_DEBUG + dev_info(lwis_dev->dev, "%s: power counter = %d\n", __func__, + core.unified_dev_pwr_map[i].count); +#endif + return 0; + } + } + mutex_unlock(&core.lock); + dev_err(lwis_dev->dev, "Unified power sequence not found\n"); + return -ENODEV; +} + +static struct lwis_device *get_power_down_dev(struct lwis_device *lwis_dev) +{ + int i; + + if (lwis_dev->power_seq_handler == NULL) { + return lwis_dev; + } + + mutex_lock(&core.lock); + for (i = 0; i < MAX_UNIFIED_POWER_DEVICE; i++) { + if (core.unified_dev_pwr_map[i].dev_node_seq == NULL) { + break; + } + if (core.unified_dev_pwr_map[i].dev_node_seq == lwis_dev->power_seq_handler) { + mutex_unlock(&core.lock); +#ifdef LWIS_PWR_SEQ_DEBUG + dev_info(lwis_dev->dev, "%s: power dev = %s\n", __func__, + core.unified_dev_pwr_map[i].hold_dev->name); +#endif + return core.unified_dev_pwr_map[i].hold_dev; + } + } + if (i >= MAX_UNIFIED_POWER_DEVICE) { + dev_err(lwis_dev->dev, "Unified power sequence not found\n"); + mutex_unlock(&core.lock); + return lwis_dev; + } + mutex_unlock(&core.lock); + + return lwis_dev; +} + +int lwis_dev_process_power_sequence(struct lwis_device *lwis_dev, + struct lwis_device_power_sequence_list *list, bool set_active, + bool skip_error) { int ret = 0; int last_error = 0; @@ -383,7 +576,7 @@ static int process_power_sequence(struct lwis_device *lwis_dev, dev_err(lwis_dev->dev, "No power_up_sequence defined\n"); return -EINVAL; } - + LWIS_ATRACE_FUNC_BEGIN(lwis_dev, "lwis_dev_process_power_sequence"); for (i = 0; i < list->count; ++i) { #ifdef LWIS_PWR_SEQ_DEBUG dev_info(lwis_dev->dev, "%s: %d - type:%s name:%s delay_us:%d", __func__, i, @@ -411,6 +604,8 @@ static int process_power_sequence(struct lwis_device *lwis_dev, if (ret) { dev_err(lwis_dev->dev, "Error set regulators (%d)\n", ret); if (!skip_error) { + LWIS_ATRACE_FUNC_END(lwis_dev, + "lwis_dev_process_power_sequence"); return ret; } last_error = ret; @@ -422,11 +617,13 @@ static int process_power_sequence(struct lwis_device *lwis_dev, gpios_info = lwis_gpios_get_info_by_name(lwis_dev->gpios_list, list->seq_info[i].name); - if (IS_ERR(gpios_info)) { + if (IS_ERR_OR_NULL(gpios_info)) { dev_err(lwis_dev->dev, "Get %s gpios info failed\n", list->seq_info[i].name); ret = PTR_ERR(gpios_info); if (!skip_error) { + LWIS_ATRACE_FUNC_END(lwis_dev, + "lwis_dev_process_power_sequence"); return ret; } else { last_error = ret; @@ -451,6 +648,9 @@ static int process_power_sequence(struct lwis_device *lwis_dev, dev_err(lwis_dev->dev, "Failed to obtain gpio list (%d)\n", ret); if (!skip_error) { + LWIS_ATRACE_FUNC_END( + lwis_dev, + "lwis_dev_process_power_sequence"); return ret; } else { last_error = ret; @@ -469,6 +669,9 @@ static int process_power_sequence(struct lwis_device *lwis_dev, list->seq_info[i].name); ret = -ENODEV; if (!skip_error) { + LWIS_ATRACE_FUNC_END( + lwis_dev, + "lwis_dev_process_power_sequence"); return ret; } else { last_error = ret; @@ -490,7 +693,7 @@ static int process_power_sequence(struct lwis_device *lwis_dev, gpios_info_it = lwis_gpios_get_info_by_name( lwis_dev_it->gpios_list, list->seq_info[i].name); - if (IS_ERR(gpios_info_it)) { + if (IS_ERR_OR_NULL(gpios_info_it)) { continue; } if (gpios_info_it->id == gpios_info->id && @@ -519,6 +722,9 @@ static int process_power_sequence(struct lwis_device *lwis_dev, if (ret) { dev_err(lwis_dev->dev, "Error set GPIO pins (%d)\n", ret); if (!skip_error) { + LWIS_ATRACE_FUNC_END( + lwis_dev, + "lwis_dev_process_power_sequence"); return ret; } last_error = ret; @@ -529,6 +735,8 @@ static int process_power_sequence(struct lwis_device *lwis_dev, if (ret) { dev_err(lwis_dev->dev, "Error set GPIO pins (%d)\n", ret); if (!skip_error) { + LWIS_ATRACE_FUNC_END(lwis_dev, + "lwis_dev_process_power_sequence"); return ret; } last_error = ret; @@ -544,11 +752,14 @@ static int process_power_sequence(struct lwis_device *lwis_dev, if (set_active) { lwis_dev->mclk_ctrl = devm_pinctrl_get(&lwis_dev->plat_dev->dev); - if (IS_ERR(lwis_dev->mclk_ctrl)) { + if (IS_ERR_OR_NULL(lwis_dev->mclk_ctrl)) { dev_err(lwis_dev->dev, "Failed to get mclk\n"); ret = PTR_ERR(lwis_dev->mclk_ctrl); lwis_dev->mclk_ctrl = NULL; if (!skip_error) { + LWIS_ATRACE_FUNC_END( + lwis_dev, + "lwis_dev_process_power_sequence"); return ret; } else { last_error = ret; @@ -560,6 +771,9 @@ static int process_power_sequence(struct lwis_device *lwis_dev, dev_err(lwis_dev->dev, "No pinctrl defined\n"); ret = -ENODEV; if (!skip_error) { + LWIS_ATRACE_FUNC_END( + lwis_dev, + "lwis_dev_process_power_sequence"); return ret; } else { last_error = ret; @@ -608,6 +822,9 @@ static int process_power_sequence(struct lwis_device *lwis_dev, lwis_dev->mclk_ctrl = NULL; } if (!skip_error) { + LWIS_ATRACE_FUNC_END( + lwis_dev, + "lwis_dev_process_power_sequence"); return ret; } else { last_error = ret; @@ -624,9 +841,10 @@ static int process_power_sequence(struct lwis_device *lwis_dev, } if (last_error) { + LWIS_ATRACE_FUNC_END(lwis_dev, "lwis_dev_process_power_sequence"); return last_error; } - + LWIS_ATRACE_FUNC_END(lwis_dev, "lwis_dev_process_power_sequence"); return ret; } @@ -705,7 +923,7 @@ static int lwis_dev_power_up_by_default(struct lwis_device *lwis_dev) bool activate_mclk = true; lwis_dev->mclk_ctrl = devm_pinctrl_get(&lwis_dev->plat_dev->dev); - if (IS_ERR(lwis_dev->mclk_ctrl)) { + if (IS_ERR_OR_NULL(lwis_dev->mclk_ctrl)) { dev_err(lwis_dev->dev, "Failed to get mclk\n"); ret = PTR_ERR(lwis_dev->mclk_ctrl); lwis_dev->mclk_ctrl = NULL; @@ -816,15 +1034,20 @@ int lwis_dev_power_up_locked(struct lwis_device *lwis_dev) mutex_lock(i2c_dev->group_i2c_lock); } if (lwis_dev->power_up_sequence) { - ret = process_power_sequence(lwis_dev, lwis_dev->power_up_sequence, - /*set_active=*/true, /*skip_error=*/false); - if (ret) { - dev_err(lwis_dev->dev, "Error process_power_sequence (%d)\n", ret); - if (lwis_dev->type == DEVICE_TYPE_I2C) { - mutex_unlock(i2c_dev->group_i2c_lock); + if (need_to_power_up(lwis_dev)) { + ret = lwis_dev_process_power_sequence(lwis_dev, lwis_dev->power_up_sequence, + /*set_active=*/true, + /*skip_error=*/false); + if (ret) { + dev_err(lwis_dev->dev, + "Error lwis_dev_process_power_sequence (%d)\n", ret); + if (lwis_dev->type == DEVICE_TYPE_I2C) { + mutex_unlock(i2c_dev->group_i2c_lock); + } + goto error_power_up; } - goto error_power_up; } + increase_unified_power_count(lwis_dev); } else { ret = lwis_dev_power_up_by_default(lwis_dev); if (ret) { @@ -1009,12 +1232,19 @@ int lwis_dev_power_down_locked(struct lwis_device *lwis_dev) mutex_lock(i2c_dev->group_i2c_lock); } if (lwis_dev->power_down_sequence) { - ret = process_power_sequence(lwis_dev, lwis_dev->power_down_sequence, - /*set_active=*/false, /*skip_error=*/true); - if (ret) { - dev_err(lwis_dev->dev, "Error process_power_sequence (%d)\n", ret); - last_error = ret; + if (need_to_power_down(lwis_dev)) { + struct lwis_device *power_dev = get_power_down_dev(lwis_dev); + ret = lwis_dev_process_power_sequence(power_dev, + power_dev->power_down_sequence, + /*set_active=*/false, + /*skip_error=*/true); + if (ret) { + dev_err(lwis_dev->dev, + "Error lwis_dev_process_power_sequence (%d)\n", ret); + last_error = ret; + } } + decrease_unified_power_count(lwis_dev); } else { ret = lwis_dev_power_down_by_default(lwis_dev); if (ret) { @@ -1111,10 +1341,9 @@ void lwis_dev_power_seq_list_print(struct lwis_device_power_sequence_list *list) } } -static struct lwis_device *find_top_dev() +static struct lwis_device *find_top_dev(void) { struct lwis_device *lwis_dev; - mutex_lock(&core.lock); list_for_each_entry (lwis_dev, &core.lwis_dev_list, dev_list) { if (lwis_dev->type == DEVICE_TYPE_TOP) { @@ -1132,10 +1361,9 @@ static void event_heartbeat_timer(struct timer_list *t) int64_t event_id = LWIS_EVENT_ID_HEARTBEAT | (int64_t)lwis_dev->id << LWIS_EVENT_ID_EVENT_CODE_LEN; - lwis_device_event_emit(lwis_dev, event_id, NULL, 0, - /*in_irq=*/false); + lwis_device_event_emit(lwis_dev, event_id, NULL, 0); - mod_timer(t, jiffies + msecs_to_jiffies(1000)); + mod_timer(t, jiffies + msecs_to_jiffies(LWIS_HEARTBEAT_EVENT_INTERVAL_MS)); } /* @@ -1200,6 +1428,34 @@ void lwis_device_info_dump(const char *name, void (*func)(struct lwis_device *)) mutex_unlock(&core.lock); } +void lwis_device_crash_info_dump(struct lwis_device *lwis_dev) +{ + int dump_cnt = 5; + int64_t timestamp; + + pr_info("LWIS Device (%s) Crash Info Dump:\n", lwis_dev->name); + + /* Dump Current kernel timestamp && Last 5 Received Event*/ + timestamp = ktime_to_ns(lwis_get_time()); + dev_info(lwis_dev->dev, " AT %lld Dump Last %d Received Events:\n\n", timestamp, dump_cnt); + lwis_debug_print_event_states_info(lwis_dev, /*lwis_event_dump_cnt=*/dump_cnt); +} + +void lwis_save_register_io_info(struct lwis_device *lwis_dev, struct lwis_io_entry *io_entry, + size_t access_size) +{ + lwis_dev->debug_info.io_entry_hist[lwis_dev->debug_info.cur_io_entry_hist_idx].io_entry = + *io_entry; + lwis_dev->debug_info.io_entry_hist[lwis_dev->debug_info.cur_io_entry_hist_idx].access_size = + access_size; + lwis_dev->debug_info.io_entry_hist[lwis_dev->debug_info.cur_io_entry_hist_idx] + .start_timestamp = ktime_to_ns(lwis_get_time()); + lwis_dev->debug_info.cur_io_entry_hist_idx++; + if (lwis_dev->debug_info.cur_io_entry_hist_idx >= IO_ENTRY_DEBUG_HISTORY_SIZE) { + lwis_dev->debug_info.cur_io_entry_hist_idx = 0; + } +} + /* * lwis_base_probe: Create a device instance for each of the LWIS device. */ @@ -1214,12 +1470,13 @@ int lwis_base_probe(struct lwis_device *lwis_dev, struct platform_device *plat_d if (ret >= 0) { lwis_dev->id = ret; } else { - pr_err("Unable to allocate minor ID (%d)\n", ret); + dev_err(&plat_dev->dev, "Unable to allocate minor ID (%d)\n", ret); return ret; } /* Initialize enabled state */ lwis_dev->enabled = 0; + lwis_dev->is_suspended = false; lwis_dev->clock_family = CLOCK_FAMILY_INVALID; /* Initialize client mutex */ @@ -1259,7 +1516,7 @@ int lwis_base_probe(struct lwis_device *lwis_dev, struct platform_device *plat_d /* Upon success initialization, create device for this instance */ lwis_dev->dev = device_create(core.dev_class, NULL, MKDEV(core.device_major, lwis_dev->id), lwis_dev, LWIS_DEVICE_NAME "-%s", lwis_dev->name); - if (IS_ERR(lwis_dev->dev)) { + if (IS_ERR_OR_NULL(lwis_dev->dev)) { pr_err("Failed to create device\n"); ret = PTR_ERR(lwis_dev->dev); goto error_init; @@ -1271,7 +1528,6 @@ int lwis_base_probe(struct lwis_device *lwis_dev, struct platform_device *plat_d lwis_platform_probe(lwis_dev); lwis_device_debugfs_setup(lwis_dev, core.dbg_root); - memset(&lwis_dev->debug_info, 0, sizeof(lwis_dev->debug_info)); timer_setup(&lwis_dev->heartbeat_timer, event_heartbeat_timer, 0); @@ -1346,7 +1602,7 @@ void lwis_base_unprobe(struct lwis_device *unprobe_lwis_dev) lwis_dev->irq_gpios_info.gpios = NULL; } /* Destroy device */ - if (!IS_ERR(lwis_dev->dev)) { + if (!IS_ERR_OR_NULL(lwis_dev->dev)) { device_destroy(core.dev_class, MKDEV(core.device_major, lwis_dev->id)); } @@ -1392,7 +1648,7 @@ static int __init lwis_register_base_device(void) /* Create a device class*/ core.dev_class = class_create(THIS_MODULE, LWIS_CLASS_NAME); - if (IS_ERR(core.dev_class)) { + if (IS_ERR_OR_NULL(core.dev_class)) { pr_err("Failed to create device class\n"); ret = PTR_ERR(core.dev_class); goto error_class_create; @@ -1517,8 +1773,16 @@ static int __init lwis_base_device_init(void) goto dpm_failure; } + ret = lwis_test_device_init(); + if (ret) { + pr_err("Failed to lwis_test_device_init (%d)\n", ret); + goto test_failure; + } + return 0; +test_failure: + lwis_test_device_deinit(); dpm_failure: lwis_slc_device_deinit(); slc_failure: @@ -1614,6 +1878,7 @@ static void __exit lwis_driver_exit(void) } /* Deinit device classes */ + lwis_test_device_deinit(); lwis_dpm_device_deinit(); lwis_slc_device_deinit(); lwis_i2c_device_deinit(); @@ -1624,9 +1889,16 @@ static void __exit lwis_driver_exit(void) lwis_unregister_base_device(); } +void lwis_process_worker_queue(struct lwis_client *client) +{ + lwis_process_transactions_in_queue(client); + lwis_process_periodic_io_in_queue(client); +} + subsys_initcall(lwis_base_device_init); module_exit(lwis_driver_exit); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Google-ACMA"); MODULE_DESCRIPTION("LWIS Base Device Driver"); +MODULE_IMPORT_NS(DMA_BUF); diff --git a/lwis_device.h b/lwis_device.h index 42c0bdb..2f30aa5 100644 --- a/lwis_device.h +++ b/lwis_device.h @@ -22,6 +22,7 @@ #include <linux/mutex.h> #include <linux/platform_device.h> #include <linux/poll.h> +#include <linux/workqueue.h> #include "lwis_clock.h" #include "lwis_commands.h" @@ -37,12 +38,14 @@ #define LWIS_IOREG_DEVICE_COMPAT "google,lwis-ioreg-device" #define LWIS_SLC_DEVICE_COMPAT "google,lwis-slc-device" #define LWIS_DPM_DEVICE_COMPAT "google,lwis-dpm-device" +#define LWIS_TEST_DEVICE_COMPAT "google,lwis-test-device" #define EVENT_HASH_BITS 8 #define BUFFER_HASH_BITS 8 #define TRANSACTION_HASH_BITS 8 #define PERIODIC_IO_HASH_BITS 8 #define BTS_UNSUPPORTED -1 +#define MAX_UNIFIED_POWER_DEVICE 8 /* Forward declaration for lwis_device. This is needed for the declaration for lwis_device_subclass_operations data struct. */ @@ -57,6 +60,16 @@ int lwis_allocator_init(struct lwis_device *lwis_dev); void lwis_allocator_release(struct lwis_device *lwis_dev); /* + * struct lwis_dev_pwr_ref_cnt + * This struct is to store the power up/down sequence reference count + */ +struct lwis_dev_pwr_ref_cnt { + struct device_node *dev_node_seq; + struct lwis_device *hold_dev; + int count; +}; + +/* * struct lwis_core * This struct applies to all LWIS devices that are defined in the * device tree. @@ -69,6 +82,7 @@ struct lwis_core { dev_t lwis_devt; int device_major; struct list_head lwis_dev_list; + struct lwis_dev_pwr_ref_cnt unified_dev_pwr_map[MAX_UNIFIED_POWER_DEVICE]; struct dentry *dbg_root; }; @@ -118,7 +132,7 @@ struct lwis_event_subscribe_operations { /* Notify subscriber when an event is happening */ void (*notify_event_subscriber)(struct lwis_device *lwis_dev, int64_t trigger_event_id, int64_t trigger_event_count, - int64_t trigger_event_timestamp, bool in_irq); + int64_t trigger_event_timestamp); /* Clean up event subscription hash table when unloading top device */ void (*release)(struct lwis_device *lwis_dev); }; @@ -153,20 +167,34 @@ struct lwis_client_debug_info { int cur_transaction_hist_idx; }; +/* + * struct lwis_register_io_info + * This struct is to store the register read write info + */ +struct lwis_register_io_info { + struct lwis_io_entry io_entry; + size_t access_size; + int64_t start_timestamp; +}; + /* struct lwis_device_debug_info * This struct applies to each of the LWIS devices, and the purpose is to * store information in help debugability. */ #define EVENT_DEBUG_HISTORY_SIZE 16 +#define IO_ENTRY_DEBUG_HISTORY_SIZE 8 struct lwis_device_debug_info { struct lwis_device_event_state_history event_hist[EVENT_DEBUG_HISTORY_SIZE]; int cur_event_hist_idx; + struct lwis_register_io_info io_entry_hist[IO_ENTRY_DEBUG_HISTORY_SIZE]; + int cur_io_entry_hist_idx; }; /* * struct lwis_device * This struct applies to each of the LWIS devices, e.g. /dev/lwis* */ +#define MAX_BTS_BLOCK_NUM 4 struct lwis_device { struct lwis_platform *platform; int id; @@ -192,6 +220,8 @@ struct lwis_device { /* Enabled state of the device */ int enabled; + /* Mark if the client called device suspend */ + bool is_suspended; /* Mutex used to synchronize access between clients */ struct mutex client_lock; /* Spinlock used to synchronize access to the device struct */ @@ -217,23 +247,34 @@ struct lwis_device { struct dentry *dbg_event_file; struct dentry *dbg_transaction_file; struct dentry *dbg_buffer_file; + struct dentry *dbg_reg_io_file; #endif /* Structure to store info to help debugging device data */ struct lwis_device_debug_info debug_info; /* clock family this device belongs to */ int clock_family; - /* index to bandwidth traffic shaper */ - int bts_index; + /* number of BTS blocks */ + int bts_block_num; + /* BTS block names*/ + const char *bts_block_names[MAX_BTS_BLOCK_NUM]; + /* indexes to bandwidth traffic shaper */ + int bts_indexes[MAX_BTS_BLOCK_NUM]; /* BTS scenario name */ const char *bts_scenario_name; /* BTS scenario index */ unsigned int bts_scenario; + /* Power sequence handler */ + struct device_node *power_seq_handler; /* Power up sequence information */ struct lwis_device_power_sequence_list *power_up_sequence; /* Power down sequence information */ struct lwis_device_power_sequence_list *power_down_sequence; + /* Suspend sequence information */ + struct lwis_device_power_sequence_list *suspend_sequence; + /* Resume sequence information */ + struct lwis_device_power_sequence_list *resume_sequence; /* GPIOs list */ struct lwis_gpios_list *gpios_list; /* GPIO interrupts list */ @@ -246,7 +287,6 @@ struct lwis_device { bool is_read_only; /* Adjust thread priority */ u32 transaction_thread_priority; - u32 periodic_io_thread_priority; /* LWIS allocator block manager */ struct lwis_allocator_block_mgr *block_mgr; @@ -254,10 +294,6 @@ struct lwis_device { /* Worker thread */ struct kthread_worker transaction_worker; struct task_struct *transaction_worker_thread; - struct kthread_worker periodic_io_worker; - struct task_struct *periodic_io_worker_thread; - struct kthread_worker subscribe_worker; - struct task_struct *subscribe_worker_thread; }; /* @@ -291,11 +327,12 @@ struct lwis_client { struct list_head transaction_process_queue; /* Transaction counter, which also provides transacton ID */ int64_t transaction_counter; + /* Hash table of pending transactions keyed by transaction id */ + DECLARE_HASHTABLE(pending_transactions, TRANSACTION_HASH_BITS); /* Hash table of hrtimer keyed by time out duration */ DECLARE_HASHTABLE(timer_list, PERIODIC_IO_HASH_BITS); /* Work item */ struct kthread_work transaction_work; - struct kthread_work periodic_io_work; /* Spinlock used to synchronize access to periodic io data structs */ spinlock_t periodic_io_lock; /* Queue of all periodic_io pending processing */ @@ -345,6 +382,13 @@ int lwis_dev_power_up_locked(struct lwis_device *lwis_dev); int lwis_dev_power_down_locked(struct lwis_device *lwis_dev); /* + * Process lwis device power sequence list + */ +int lwis_dev_process_power_sequence(struct lwis_device *lwis_dev, + struct lwis_device_power_sequence_list *list, bool set_active, + bool skip_error); + +/* * lwis_dev_power_seq_list_alloc: * Allocate an instance of the lwis_device_power_sequence_info * and initialize the data structures according to the number of @@ -370,4 +414,25 @@ void lwis_dev_power_seq_list_print(struct lwis_device_power_sequence_list *list) */ void lwis_device_info_dump(const char *name, void (*func)(struct lwis_device *)); +/* + * lwis_device_crash_info_dump: + * Use the customized function handle to print information from each device registered in LWIS + * when usersapce crash. + */ +void lwis_device_crash_info_dump(struct lwis_device *lwis_dev); + +/* + * lwis_save_register_io_info: Saves the register io info in a history buffer + * for better debugability. + */ +void lwis_save_register_io_info(struct lwis_device *lwis_dev, struct lwis_io_entry *io_entry, + size_t access_size); + +/* + * lwis_process_worker_queue: + * Function to process the transaction process queue and + * periodic io queue on the transaction thread + */ +void lwis_process_worker_queue(struct lwis_client *client); + #endif /* LWIS_DEVICE_H_ */ diff --git a/lwis_device_dpm.c b/lwis_device_dpm.c index 78d264d..deaaca3 100644 --- a/lwis_device_dpm.c +++ b/lwis_device_dpm.c @@ -38,12 +38,38 @@ static struct lwis_event_subscribe_operations dpm_subscribe_ops = { .release = NULL, }; +static int find_bts_block(struct lwis_device *lwis_dev, struct lwis_device *target_dev, + struct lwis_qos_setting_v2 *qos_setting) +{ + int i; + + if (strcmp(qos_setting->bts_block_name, "") == 0) { + if (target_dev->bts_block_num != 1) { + dev_err(lwis_dev->dev, + "Device %s has %d bts blocks but no block name specified in qos setting\n", + target_dev->name, target_dev->bts_block_num); + return -EINVAL; + } + return 0; + } else { + for (i = 0; i < target_dev->bts_block_num; i++) { + if (strcmp(target_dev->bts_block_names[i], qos_setting->bts_block_name) == + 0) { + return i; + } + } + dev_err(lwis_dev->dev, "Failed to find block name matching %s for device %s\n", + qos_setting->bts_block_name, target_dev->name); + return -EINVAL; + } +} + /* * lwis_dpm_update_qos: update qos requirement for lwis device. */ -int lwis_dpm_update_qos(struct lwis_device *lwis_dev, struct lwis_qos_setting *qos_setting) +int lwis_dpm_update_qos(struct lwis_device *lwis_dev, struct lwis_qos_setting_v2 *qos_setting) { - int ret = 0; + int ret = 0, bts_block = -1; int64_t peak_bw = 0; int64_t read_bw = 0; int64_t write_bw = 0; @@ -78,15 +104,22 @@ int lwis_dpm_update_qos(struct lwis_device *lwis_dev, struct lwis_qos_setting *q qos_setting->clock_family); } } else { + bts_block = find_bts_block(lwis_dev, target_dev, qos_setting); + if (bts_block < 0) { + return bts_block; + } + read_bw = qos_setting->read_bw; write_bw = qos_setting->write_bw; peak_bw = (qos_setting->peak_bw > 0) ? - qos_setting->peak_bw : - ((read_bw > write_bw) ? read_bw : write_bw) / 4; + qos_setting->peak_bw : + ((read_bw > write_bw) ? read_bw : write_bw) / 4; rt_bw = (qos_setting->rt_bw > 0) ? qos_setting->rt_bw : 0; - ret = lwis_platform_update_bts(target_dev, peak_bw, read_bw, write_bw, rt_bw); + ret = lwis_platform_update_bts(target_dev, bts_block, peak_bw, read_bw, + write_bw, rt_bw); if (ret < 0) { - dev_err(lwis_dev->dev, "Failed to update bandwidth to bts, ret: %d\n", ret); + dev_err(lwis_dev->dev, + "Failed to update bandwidth to bts, ret: %d\n", ret); } } break; @@ -134,7 +167,7 @@ int lwis_dpm_update_clock(struct lwis_device *lwis_dev, struct lwis_clk_setting goto out; } - if (IS_ERR(lwis_dev->clocks->clk[clk_index].clk)) { + if (IS_ERR_OR_NULL(lwis_dev->clocks->clk[clk_index].clk)) { dev_err(lwis_dev->dev, "%s clk is invalid\n", lwis_dev->name); ret = -EINVAL; goto out; diff --git a/lwis_device_dpm.h b/lwis_device_dpm.h index 077c056..0aeba1f 100644 --- a/lwis_device_dpm.h +++ b/lwis_device_dpm.h @@ -32,7 +32,7 @@ int lwis_dpm_update_clock(struct lwis_device *lwis_dev, struct lwis_clk_setting /* * lwis_dpm_update_qos: update qos requirement from dpm client. */ -int lwis_dpm_update_qos(struct lwis_device *lwis_dev, struct lwis_qos_setting *qos_setting); +int lwis_dpm_update_qos(struct lwis_device *lwis_dev, struct lwis_qos_setting_v2 *qos_setting); /* * lwis_dpm_read_clock: read current IP core clock for given lwis device. diff --git a/lwis_device_i2c.c b/lwis_device_i2c.c index 3adfa57..0135b05 100644 --- a/lwis_device_i2c.c +++ b/lwis_device_i2c.c @@ -19,16 +19,17 @@ #include <linux/module.h> #include <linux/platform_device.h> #include <linux/pm.h> -#include <linux/preempt.h> #include <linux/sched.h> #include <linux/sched/types.h> #include <linux/slab.h> #include <uapi/linux/sched/types.h> +#include "lwis_device.h" #include "lwis_i2c.h" #include "lwis_init.h" #include "lwis_periodic_io.h" #include "lwis_util.h" +#include "lwis_trace.h" #ifdef CONFIG_OF #include "lwis_dt.h" @@ -72,11 +73,10 @@ static int lwis_i2c_device_enable(struct lwis_device *lwis_dev) /* Enable the I2C bus */ mutex_lock(i2c_dev->group_i2c_lock); - + LWIS_ATRACE_FUNC_BEGIN(lwis_dev, "lwis_i2c_device_enable"); #if IS_ENABLED(CONFIG_INPUT_STMVL53L1) if (is_shared_i2c_with_stmvl53l1(i2c_dev->state_pinctrl)) - ret = shared_i2c_set_state(&i2c_dev->client->dev, - i2c_dev->state_pinctrl, + ret = shared_i2c_set_state(&i2c_dev->client->dev, i2c_dev->state_pinctrl, I2C_ON_STRING); else ret = lwis_i2c_set_state(i2c_dev, I2C_ON_STRING); @@ -85,6 +85,7 @@ static int lwis_i2c_device_enable(struct lwis_device *lwis_dev) #endif mutex_unlock(i2c_dev->group_i2c_lock); + LWIS_ATRACE_FUNC_END(lwis_dev, "lwis_i2c_device_enable"); if (ret) { dev_err(lwis_dev->dev, "Error enabling i2c bus (%d)\n", ret); return ret; @@ -109,13 +110,11 @@ static int lwis_i2c_device_disable(struct lwis_device *lwis_dev) if (is_shared_i2c_with_stmvl53l1(i2c_dev->state_pinctrl)) { /* Disable the shared i2c bus */ mutex_lock(i2c_dev->group_i2c_lock); - ret = shared_i2c_set_state(&i2c_dev->client->dev, - i2c_dev->state_pinctrl, + ret = shared_i2c_set_state(&i2c_dev->client->dev, i2c_dev->state_pinctrl, I2C_OFF_STRING); mutex_unlock(i2c_dev->group_i2c_lock); if (ret) { - dev_err(lwis_dev->dev, "Error disabling i2c bus (%d)\n", - ret); + dev_err(lwis_dev->dev, "Error disabling i2c bus (%d)\n", ret); } return ret; } @@ -124,8 +123,10 @@ static int lwis_i2c_device_disable(struct lwis_device *lwis_dev) if (!lwis_i2c_dev_is_in_use(lwis_dev)) { /* Disable the I2C bus */ mutex_lock(i2c_dev->group_i2c_lock); + LWIS_ATRACE_FUNC_BEGIN(lwis_dev, "lwis_i2c_device_disable"); ret = lwis_i2c_set_state(i2c_dev, I2C_OFF_STRING); mutex_unlock(i2c_dev->group_i2c_lock); + LWIS_ATRACE_FUNC_END(lwis_dev, "lwis_i2c_device_disable"); if (ret) { dev_err(lwis_dev->dev, "Error disabling i2c bus (%d)\n", ret); return ret; @@ -145,7 +146,9 @@ static int lwis_i2c_register_io(struct lwis_device *lwis_dev, struct lwis_io_ent if (in_interrupt()) { return -EAGAIN; } - return lwis_i2c_io_entry_rw(i2c_dev, entry); + lwis_save_register_io_info(lwis_dev, entry, access_size); + + return lwis_i2c_io_entry_rw(i2c_dev, entry, lwis_dev); } static int lwis_i2c_addr_matcher(struct device *dev, void *data) @@ -184,7 +187,6 @@ static int lwis_i2c_device_setup(struct lwis_i2c_device *i2c_dev) /* Initialize device i2c lock */ i2c_dev->group_i2c_lock = &group_i2c_lock[i2c_dev->i2c_lock_group_id]; - info.addr = i2c_dev->address; i2c_dev->client = i2c_new_client_device(i2c_dev->adapter, &info); @@ -211,7 +213,7 @@ static int lwis_i2c_device_setup(struct lwis_i2c_device *i2c_dev) pinctrl's are defined */ /* TODO: Need to figure out why this is parent's parent */ pinctrl = devm_pinctrl_get(dev->parent->parent); - if (IS_ERR(pinctrl)) { + if (IS_ERR_OR_NULL(pinctrl)) { dev_err(i2c_dev->base_dev.dev, "Cannot instantiate pinctrl instance (%lu)\n", PTR_ERR(pinctrl)); i2c_dev->state_pinctrl = NULL; @@ -220,11 +222,11 @@ static int lwis_i2c_device_setup(struct lwis_i2c_device *i2c_dev) /* Verify that on_i2c or off_i2c strings are present */ i2c_dev->pinctrl_default_state_only = false; - if (IS_ERR(pinctrl_lookup_state(pinctrl, I2C_OFF_STRING)) || - IS_ERR(pinctrl_lookup_state(pinctrl, I2C_ON_STRING))) { + if (IS_ERR_OR_NULL(pinctrl_lookup_state(pinctrl, I2C_OFF_STRING)) || + IS_ERR_OR_NULL(pinctrl_lookup_state(pinctrl, I2C_ON_STRING))) { state = pinctrl_lookup_state(pinctrl, I2C_DEFAULT_STATE_STRING); /* Default option also missing, return error */ - if (IS_ERR(state)) { + if (IS_ERR_OR_NULL(state)) { dev_err(i2c_dev->base_dev.dev, "Pinctrl states {%s, %s, %s} not found (%lu)\n", I2C_OFF_STRING, I2C_ON_STRING, I2C_DEFAULT_STATE_STRING, PTR_ERR(state)); @@ -275,31 +277,18 @@ static int lwis_i2c_device_probe(struct platform_device *plat_dev) /* Create associated kworker threads */ ret = lwis_create_kthread_workers(&i2c_dev->base_dev); if (ret) { - dev_err(i2c_dev->base_dev.dev,"Failed to create lwis_i2c_kthread"); + dev_err(i2c_dev->base_dev.dev, "Failed to create lwis_i2c_kthread"); lwis_base_unprobe(&i2c_dev->base_dev); goto error_probe; } if (i2c_dev->base_dev.transaction_thread_priority != 0) { ret = lwis_set_kthread_priority(&i2c_dev->base_dev, - i2c_dev->base_dev.transaction_worker_thread, - i2c_dev->base_dev.transaction_thread_priority); - if (ret) { - dev_err(i2c_dev->base_dev.dev, - "Failed to set LWIS I2C transaction kthread priority (%d)", - ret); - lwis_base_unprobe(&i2c_dev->base_dev); - goto error_probe; - } - } - if (i2c_dev->base_dev.periodic_io_thread_priority != 0) { - ret = lwis_set_kthread_priority(&i2c_dev->base_dev, - i2c_dev->base_dev.periodic_io_worker_thread, - i2c_dev->base_dev.periodic_io_thread_priority); + i2c_dev->base_dev.transaction_worker_thread, + i2c_dev->base_dev.transaction_thread_priority); if (ret) { dev_err(i2c_dev->base_dev.dev, - "Failed to set LWIS I2C periodic io kthread priority (%d)", - ret); + "Failed to set LWIS I2C transaction kthread priority (%d)", ret); lwis_base_unprobe(&i2c_dev->base_dev); goto error_probe; } @@ -320,7 +309,7 @@ static int lwis_i2c_device_suspend(struct device *dev) struct lwis_device *lwis_dev = dev_get_drvdata(dev); if (lwis_dev->pm_hibernation == 0) { - /* TODO(b/265688764): Cleaning up system deep sleep for flash driver. */ + /* Allow the device to enter PM hibernation, e.g., flash driver. */ return 0; } diff --git a/lwis_device_i2c.h b/lwis_device_i2c.h index 1ae8294..7d52d1f 100644 --- a/lwis_device_i2c.h +++ b/lwis_device_i2c.h @@ -43,8 +43,7 @@ int lwis_i2c_device_deinit(void); * two APIs in stmvl53l1 driver to well handle the enabling and disabling. */ extern bool is_shared_i2c_with_stmvl53l1(struct pinctrl *pinctrl); -extern int shared_i2c_set_state(struct device *dev, struct pinctrl *pinctrl, - const char *state_str); +extern int shared_i2c_set_state(struct device *dev, struct pinctrl *pinctrl, const char *state_str); #endif #endif /* LWIS_DEVICE_I2C_H_ */ diff --git a/lwis_device_ioreg.c b/lwis_device_ioreg.c index 5474430..36d2d98 100644 --- a/lwis_device_ioreg.c +++ b/lwis_device_ioreg.c @@ -68,6 +68,7 @@ static int lwis_ioreg_device_disable(struct lwis_device *lwis_dev) static int lwis_ioreg_register_io(struct lwis_device *lwis_dev, struct lwis_io_entry *entry, int access_size) { + lwis_save_register_io_info(lwis_dev, entry, access_size); return lwis_ioreg_io_entry_rw((struct lwis_ioreg_device *)lwis_dev, entry, access_size); } @@ -137,24 +138,11 @@ static int lwis_ioreg_device_probe(struct platform_device *plat_dev) if (ioreg_dev->base_dev.transaction_thread_priority != 0) { ret = lwis_set_kthread_priority(&ioreg_dev->base_dev, - ioreg_dev->base_dev.transaction_worker_thread, - ioreg_dev->base_dev.transaction_thread_priority); + ioreg_dev->base_dev.transaction_worker_thread, + ioreg_dev->base_dev.transaction_thread_priority); if (ret) { dev_err(ioreg_dev->base_dev.dev, - "Failed to set LWIS IOREG transaction kthread priority (%d)", - ret); - lwis_base_unprobe(&ioreg_dev->base_dev); - goto error_probe; - } - } - if (ioreg_dev->base_dev.periodic_io_thread_priority != 0) { - ret = lwis_set_kthread_priority(&ioreg_dev->base_dev, - ioreg_dev->base_dev.periodic_io_worker_thread, - ioreg_dev->base_dev.periodic_io_thread_priority); - if (ret) { - dev_err(ioreg_dev->base_dev.dev, - "Failed to set LWIS IOREG periodic io kthread priority (%d)", - ret); + "Failed to set LWIS IOREG transaction kthread priority (%d)", ret); lwis_base_unprobe(&ioreg_dev->base_dev); goto error_probe; } diff --git a/lwis_device_slc.c b/lwis_device_slc.c index 5b0dd53..5debf65 100644 --- a/lwis_device_slc.c +++ b/lwis_device_slc.c @@ -93,7 +93,7 @@ static int lwis_slc_enable(struct lwis_device *lwis_dev) /* Initialize SLC partitions and get a handle */ slc_dev->partition_handle = pt_client_register(node, NULL, NULL); - if (IS_ERR(slc_dev->partition_handle)) { + if (IS_ERR_OR_NULL(slc_dev->partition_handle)) { ret = PTR_ERR(slc_dev->partition_handle); dev_err(lwis_dev->dev, "Failed to register PT client (%d)\n", ret); slc_dev->partition_handle = NULL; diff --git a/lwis_device_test.c b/lwis_device_test.c new file mode 100644 index 0000000..31b9ce5 --- /dev/null +++ b/lwis_device_test.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Google LWIS Test Device Driver + * + * Copyright (c) 2022 Google, LLC + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME "-test-dev: " fmt + +#include "lwis_device_test.h" + +#include <linux/module.h> +#include <linux/platform_device.h> + +#include "lwis_commands.h" +#include "lwis_device.h" +#include "lwis_init.h" +#include "lwis_platform.h" + +#ifdef CONFIG_OF +#include "lwis_dt.h" +#endif + +#define LWIS_DRIVER_NAME "lwis-test" + +static int lwis_test_device_enable(struct lwis_device *lwis_dev); +static int lwis_test_device_disable(struct lwis_device *lwis_dev); +static int lwis_test_register_io(struct lwis_device *lwis_dev, struct lwis_io_entry *entry, + int access_size); + +static struct lwis_device_subclass_operations test_vops = { + .register_io = lwis_test_register_io, + .register_io_barrier = NULL, + .device_enable = lwis_test_device_enable, + .device_disable = lwis_test_device_disable, + .event_enable = NULL, + .event_flags_updated = NULL, + .close = NULL, +}; + +static struct lwis_event_subscribe_operations test_subscribe_ops = { + .subscribe_event = NULL, + .unsubscribe_event = NULL, + .notify_event_subscriber = NULL, + .release = NULL, +}; + +static int lwis_test_device_enable(struct lwis_device *lwis_dev) +{ + return 0; +} + +static int lwis_test_device_disable(struct lwis_device *lwis_dev) +{ + return 0; +} + +static int lwis_test_register_io(struct lwis_device *lwis_dev, struct lwis_io_entry *entry, + int access_size) +{ + struct lwis_test_device *test_dev = + container_of(lwis_dev, struct lwis_test_device, base_dev); + struct lwis_io_entry_rw_batch *rw_batch; + int i; + uint64_t reg_value; + + if (!entry) { + dev_err(test_dev->base_dev.dev, "IO entry is NULL.\n"); + return -EINVAL; + } + + lwis_save_register_io_info(lwis_dev, entry, access_size); + + if (entry->type == LWIS_IO_ENTRY_READ) { + if (entry->rw.offset >= SCRATCH_TEST_DEV_MEMORY_SIZE) { + dev_err(test_dev->base_dev.dev, "Offset (%llu) must be < %d\n", + entry->rw.offset, SCRATCH_TEST_DEV_MEMORY_SIZE); + return -EINVAL; + } + entry->rw.val = test_dev->scratch_mem[entry->rw.offset]; + } else if (entry->type == LWIS_IO_ENTRY_READ_BATCH) { + rw_batch = &entry->rw_batch; + if (rw_batch->offset >= SCRATCH_TEST_DEV_MEMORY_SIZE || + SCRATCH_TEST_DEV_MEMORY_SIZE - rw_batch->offset < rw_batch->size_in_bytes) { + dev_err(test_dev->base_dev.dev, + "Read range[offset(%llu) + size_in_bytes(%zu)] exceeds scratch memory (%d)\n", + rw_batch->offset, rw_batch->size_in_bytes, + SCRATCH_TEST_DEV_MEMORY_SIZE); + return -EINVAL; + } + for (i = 0; i < rw_batch->size_in_bytes; ++i) { + rw_batch->buf[i] = test_dev->scratch_mem[rw_batch->offset + i]; + } + } else if (entry->type == LWIS_IO_ENTRY_WRITE) { + if (entry->rw.offset >= SCRATCH_TEST_DEV_MEMORY_SIZE) { + dev_err(test_dev->base_dev.dev, "Offset (%llu) must be < %d\n", + entry->rw.offset, SCRATCH_TEST_DEV_MEMORY_SIZE); + return -EINVAL; + } + test_dev->scratch_mem[entry->rw.offset] = entry->rw.val; + } else if (entry->type == LWIS_IO_ENTRY_WRITE_BATCH) { + rw_batch = &entry->rw_batch; + if (rw_batch->offset >= SCRATCH_TEST_DEV_MEMORY_SIZE || + SCRATCH_TEST_DEV_MEMORY_SIZE - rw_batch->offset < rw_batch->size_in_bytes) { + dev_err(test_dev->base_dev.dev, + "Write range[offset(%llu) + size_in_bytes(%zu)] exceeds scratch memory (%d)\n", + rw_batch->offset, rw_batch->size_in_bytes, + SCRATCH_TEST_DEV_MEMORY_SIZE); + return -EINVAL; + } + for (i = 0; i < rw_batch->size_in_bytes; ++i) { + test_dev->scratch_mem[rw_batch->offset + i] = rw_batch->buf[i]; + } + } else if (entry->type == LWIS_IO_ENTRY_MODIFY) { + if (entry->mod.offset >= SCRATCH_TEST_DEV_MEMORY_SIZE) { + dev_err(test_dev->base_dev.dev, "Offset (%llu) must be < %d\n", + entry->mod.offset, SCRATCH_TEST_DEV_MEMORY_SIZE); + return -EINVAL; + } + reg_value = test_dev->scratch_mem[entry->mod.offset]; + reg_value &= ~entry->mod.val_mask; + reg_value |= entry->mod.val_mask & entry->mod.val; + test_dev->scratch_mem[entry->rw.offset] = reg_value; + } else { + dev_err(test_dev->base_dev.dev, "Invalid IO entry type: %d\n", entry->type); + return -EINVAL; + } + + return 0; +} + +static int lwis_test_device_setup(struct lwis_test_device *test_dev) +{ + int ret = 0; + +#ifdef CONFIG_OF + /* Parse device tree for device configurations */ + ret = lwis_test_device_parse_dt(test_dev); + if (ret) { + dev_err(test_dev->base_dev.dev, "Failed to parse device tree\n"); + } +#else + /* Non-device-tree init: Save for future implementation */ + ret = -ENOSYS; +#endif + + return ret; +} + +static int lwis_test_device_probe(struct platform_device *plat_dev) +{ + int ret = 0; + struct lwis_test_device *test_dev; + struct device *dev = &plat_dev->dev; + + /* Allocate test device specific data construct */ + test_dev = devm_kzalloc(dev, sizeof(struct lwis_test_device), GFP_KERNEL); + if (!test_dev) { + dev_err(dev, "Failed to allocate TEST device structure\n"); + return -ENOMEM; + } + + test_dev->base_dev.type = DEVICE_TYPE_TEST; + test_dev->base_dev.vops = test_vops; + test_dev->base_dev.subscribe_ops = test_subscribe_ops; + + /* Call the base device probe function */ + ret = lwis_base_probe(&test_dev->base_dev, plat_dev); + if (ret) { + dev_err(dev, "TEST device: Error in lwis base probe: %d\n", ret); + goto error_probe; + } + + /* Call TEST device specific setup function */ + ret = lwis_test_device_setup(test_dev); + if (ret) { + dev_err(test_dev->base_dev.dev, "Error in TEST device initialization\n"); + lwis_base_unprobe(&test_dev->base_dev); + goto error_probe; + } + + dev_info(test_dev->base_dev.dev, "TEST Device Probe: Success\n"); + + return 0; + +error_probe: + return ret; +} + +#ifdef CONFIG_OF +static const struct of_device_id lwis_id_match[] = { + { .compatible = LWIS_TEST_DEVICE_COMPAT }, + {}, +}; +// MODULE_DEVICE_TABLE(of, lwis_id_match); + +static struct platform_driver lwis_driver = { + .probe = lwis_test_device_probe, + .driver = { + .name = LWIS_DRIVER_NAME, + .owner = THIS_MODULE, + .of_match_table = lwis_id_match, + }, +}; + +#else /* CONFIG_OF not defined */ +static struct platform_device_id lwis_driver_id[] = { + { + .name = LWIS_DRIVER_NAME, + .driver_data = 0, + }, + {}, +}; +MODULE_DEVICE_TABLE(platform, lwis_driver_id); + +static struct platform_driver lwis_driver = { .probe = lwis_test_device_probe, + .id_table = lwis_driver_id, + .driver = { + .name = LWIS_DRIVER_NAME, + .owner = THIS_MODULE, + } }; +#endif /* CONFIG_OF */ + +/* + * lwis_test_device_init: Init function that will be called by the kernel + * initialization routines. + */ +int __init lwis_test_device_init(void) +{ + int ret = 0; + + pr_info("TEST device initialization\n"); + + ret = platform_driver_register(&lwis_driver); + if (ret) + pr_err("platform_driver_register failed: %d\n", ret); + + return ret; +} + +int lwis_test_device_deinit(void) +{ + platform_driver_unregister(&lwis_driver); + return 0; +} diff --git a/lwis_device_test.h b/lwis_device_test.h new file mode 100644 index 0000000..93b19b7 --- /dev/null +++ b/lwis_device_test.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Google LWIS Test Device Driver + * + * Copyright (c) 2022 Google, LLC + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef LWIS_DEVICE_TEST_H_ +#define LWIS_DEVICE_TEST_H_ + +#include "lwis_commands.h" +#include "lwis_device.h" + +#define SCRATCH_TEST_DEV_MEMORY_SIZE 32 +#define TEST_DEVICE_IRQ_CNT 1 +#define TEST_DEVICE_FAKE_INJECTION_IRQ 999 + +/* + * struct lwis_test_device + * The device majorly control/handle requests from test clients. + */ +struct lwis_test_device { + struct lwis_device base_dev; + /* + * For testing purposes, scratch memory is used as register space in + * test device. + */ + uint8_t scratch_mem[SCRATCH_TEST_DEV_MEMORY_SIZE]; +}; + +int lwis_test_device_deinit(void); + +#endif /* LWIS_DEVICE_TEST_H_ */ diff --git a/lwis_device_top.c b/lwis_device_top.c index 6c72371..3a3b0c3 100644 --- a/lwis_device_top.c +++ b/lwis_device_top.c @@ -11,9 +11,6 @@ #define pr_fmt(fmt) KBUILD_MODNAME "-top-dev: " fmt #include "lwis_device_top.h" -#include "lwis_event.h" -#include "lwis_init.h" -#include "lwis_util.h" #include <linux/device.h> #include <linux/init.h> @@ -23,13 +20,22 @@ #include <linux/platform_device.h> #include <linux/slab.h> +#include "lwis_device.h" +#include "lwis_event.h" +#include "lwis_init.h" +#include "lwis_util.h" + #ifdef CONFIG_OF #include "lwis_dt.h" #endif #define LWIS_DRIVER_NAME "lwis-top" #define LWIS_SUBSCRIBER_THREAD_NAME "lwis_s_top" -#define LWIS_SUBSCRIBER_THREAD_PRIORITY 99 +/* + * RT priority needed because events need to be transferred to + * another device in low latency. +*/ +#define SUBSCRIBE_THREAD_PRIORITY 99 static int lwis_top_register_io(struct lwis_device *lwis_dev, struct lwis_io_entry *entry, int access_size); @@ -49,8 +55,7 @@ static int lwis_top_event_subscribe(struct lwis_device *lwis_dev, int64_t trigge static int lwis_top_event_unsubscribe(struct lwis_device *lwis_dev, int64_t trigger_event_id, int subscriber_device_id); static void lwis_top_event_notify(struct lwis_device *lwis_dev, int64_t trigger_event_id, - int64_t trigger_event_count, int64_t trigger_event_timestamp, - bool in_irq); + int64_t trigger_event_count, int64_t trigger_event_timestamp); static void lwis_top_event_subscribe_release(struct lwis_device *lwis_dev); static struct lwis_event_subscribe_operations top_subscribe_ops = { .subscribe_event = lwis_top_event_subscribe, @@ -161,8 +166,7 @@ static void subscribe_work_func(struct kthread_work *work) lwis_device_external_event_emit(subscribe_info->subscriber_dev, trigger_event->trigger_event_id, trigger_event->trigger_event_count, - trigger_event->trigger_event_timestamp, - false); + trigger_event->trigger_event_timestamp); } kfree(trigger_event); } @@ -170,8 +174,7 @@ static void subscribe_work_func(struct kthread_work *work) } static void lwis_top_event_notify(struct lwis_device *lwis_dev, int64_t trigger_event_id, - int64_t trigger_event_count, int64_t trigger_event_timestamp, - bool in_irq) + int64_t trigger_event_count, int64_t trigger_event_timestamp) { struct lwis_top_device *lwis_top_dev = container_of(lwis_dev, struct lwis_top_device, base_dev); @@ -192,7 +195,7 @@ static void lwis_top_event_notify(struct lwis_device *lwis_dev, int64_t trigger_ list_add_tail(&trigger_event->node, &lwis_top_dev->emitted_event_list_work); spin_unlock_irqrestore(&lwis_top_dev->base_dev.lock, flags); /* Schedule deferred subscribed events */ - kthread_queue_work(&lwis_top_dev->base_dev.subscribe_worker, &lwis_top_dev->subscribe_work); + kthread_queue_work(&lwis_top_dev->subscribe_worker, &lwis_top_dev->subscribe_work); } static int lwis_top_event_subscribe(struct lwis_device *lwis_dev, int64_t trigger_event_id, @@ -366,8 +369,8 @@ static void lwis_top_event_subscribe_clear(struct lwis_top_device *lwis_top_dev) } spin_unlock_irqrestore(&lwis_top_dev->base_dev.lock, flags); - if (lwis_top_dev->base_dev.subscribe_worker_thread) { - kthread_flush_worker(&lwis_top_dev->base_dev.subscribe_worker); + if (lwis_top_dev->subscribe_worker_thread) { + kthread_flush_worker(&lwis_top_dev->subscribe_worker); } } @@ -391,6 +394,7 @@ static int lwis_top_register_io(struct lwis_device *lwis_dev, struct lwis_io_ent dev_err(top_dev->base_dev.dev, "IO entry is NULL.\n"); return -EINVAL; } + lwis_save_register_io_info(lwis_dev, entry, access_size); if (entry->type == LWIS_IO_ENTRY_READ) { if (entry->rw.offset >= SCRATCH_MEMORY_SIZE) { @@ -482,6 +486,7 @@ static int lwis_top_device_probe(struct platform_device *plat_dev) /* Allocate top device specific data construct */ top_dev = kzalloc(sizeof(struct lwis_top_device), GFP_KERNEL); if (!top_dev) { + pr_err("Failed to allocate top device structure\n"); return -ENOMEM; } @@ -492,6 +497,7 @@ static int lwis_top_device_probe(struct platform_device *plat_dev) /* Call the base device probe function */ ret = lwis_base_probe(&top_dev->base_dev, plat_dev); if (ret) { + pr_err("Error in lwis base probe\n"); goto error_probe; } @@ -505,32 +511,33 @@ static int lwis_top_device_probe(struct platform_device *plat_dev) lwis_top_event_subscribe_init(top_dev); - kthread_init_worker(&top_dev->base_dev.subscribe_worker); - top_dev->base_dev.subscribe_worker_thread = kthread_run(kthread_worker_fn, - &top_dev->base_dev.subscribe_worker, LWIS_SUBSCRIBER_THREAD_NAME); - if (IS_ERR(top_dev->base_dev.subscribe_worker_thread)) { + kthread_init_worker(&top_dev->subscribe_worker); + top_dev->subscribe_worker_thread = kthread_run( + kthread_worker_fn, &top_dev->subscribe_worker, LWIS_SUBSCRIBER_THREAD_NAME); + if (IS_ERR_OR_NULL(top_dev->subscribe_worker_thread)) { dev_err(top_dev->base_dev.dev, "subscribe kthread_run failed\n"); goto error_probe; } - /* Create associated kworker threads */ - ret = lwis_create_kthread_workers(&top_dev->base_dev); + ret = lwis_set_kthread_priority(&top_dev->base_dev, top_dev->subscribe_worker_thread, + SUBSCRIBE_THREAD_PRIORITY); if (ret) { - dev_err(top_dev->base_dev.dev, "Failed to create lwis_top_kthread"); + dev_err(top_dev->base_dev.dev, + "Failed to set LWIS top subscription kthread priority (%d)", ret); lwis_base_unprobe(&top_dev->base_dev); goto error_probe; } - ret = lwis_set_kthread_priority(&top_dev->base_dev, - top_dev->base_dev.subscribe_worker_thread, - LWIS_SUBSCRIBER_THREAD_PRIORITY); + /* Create associated kworker threads */ + ret = lwis_create_kthread_workers(&top_dev->base_dev); if (ret) { - dev_err(top_dev->base_dev.dev, - "Failed to set LWIS top subscriber kthread priority (%d)", ret); + dev_err(top_dev->base_dev.dev, "Failed to create lwis_top_kthread"); lwis_base_unprobe(&top_dev->base_dev); goto error_probe; } + dev_info(top_dev->base_dev.dev, "Top Device Probe: Success\n"); + return 0; error_probe: diff --git a/lwis_device_top.h b/lwis_device_top.h index 11bbebc..8d0a324 100644 --- a/lwis_device_top.h +++ b/lwis_device_top.h @@ -31,6 +31,10 @@ struct lwis_top_device { /* Subscription work */ struct kthread_work subscribe_work; struct list_head emitted_event_list_work; + + /* Subscription thread */ + struct kthread_worker subscribe_worker; + struct task_struct *subscribe_worker_thread; }; int lwis_top_device_deinit(void); @@ -50,7 +50,7 @@ static int parse_gpios(struct lwis_device *lwis_dev, char *name, bool *is_presen } list = lwis_gpio_list_get(dev, name); - if (IS_ERR(list)) { + if (IS_ERR_OR_NULL(list)) { pr_err("Error parsing GPIO list %s (%ld)\n", name, PTR_ERR(list)); return PTR_ERR(list); } @@ -67,6 +67,7 @@ static int parse_irq_gpios(struct lwis_device *lwis_dev) int count; int name_count; int event_count; + int type_count; int ret; struct device *dev; struct device_node *dev_node; @@ -74,10 +75,11 @@ static int parse_irq_gpios(struct lwis_device *lwis_dev) const char *name; char *irq_gpios_names = NULL; u64 *irq_gpios_events = NULL; + u32 *irq_gpios_types = NULL; int i; /* Initialize the data structure */ - strlcpy(lwis_dev->irq_gpios_info.name, "irq", LWIS_MAX_NAME_STRING_LEN); + strscpy(lwis_dev->irq_gpios_info.name, "irq", LWIS_MAX_NAME_STRING_LEN); lwis_dev->irq_gpios_info.gpios = NULL; lwis_dev->irq_gpios_info.irq_list = NULL; lwis_dev->irq_gpios_info.is_shared = false; @@ -93,18 +95,28 @@ static int parse_irq_gpios(struct lwis_device *lwis_dev) dev_node = dev->of_node; name_count = of_property_count_strings(dev_node, "irq-gpios-names"); event_count = of_property_count_elems_of_size(dev_node, "irq-gpios-events", sizeof(u64)); - if (count != event_count || count != name_count) { + type_count = of_property_count_elems_of_size(dev_node, "irq-gpios-types", sizeof(u32)); + + if (count != event_count || count != name_count || count != type_count) { pr_err("Count of irq-gpios-* is not match\n"); return -EINVAL; } gpios = lwis_gpio_list_get(dev, "irq"); - if (IS_ERR(gpios)) { + if (IS_ERR_OR_NULL(gpios)) { pr_err("Error parsing irq GPIO list (%ld)\n", PTR_ERR(gpios)); return PTR_ERR(gpios); } lwis_dev->irq_gpios_info.gpios = gpios; + lwis_dev->irq_gpios_info.irq_list = lwis_interrupt_list_alloc(lwis_dev, gpios->ndescs); + if (IS_ERR_OR_NULL(lwis_dev->irq_gpios_info.irq_list)) { + ret = -ENOMEM; + lwis_dev->irq_gpios_info.irq_list = NULL; + pr_err("Failed to allocate irq list\n"); + goto error_parse_irq_gpios; + } + irq_gpios_names = kmalloc(LWIS_MAX_NAME_STRING_LEN * name_count, GFP_KERNEL); if (IS_ERR_OR_NULL(irq_gpios_names)) { pr_err("Allocating event list failed\n"); @@ -118,13 +130,23 @@ static int parse_irq_gpios(struct lwis_device *lwis_dev) pr_err("Error get GPIO irq name list (%d)\n", ret); goto error_parse_irq_gpios; } - strlcpy(irq_gpios_names + i * LWIS_MAX_NAME_STRING_LEN, name, + strscpy(irq_gpios_names + i * LWIS_MAX_NAME_STRING_LEN, name, LWIS_MAX_NAME_STRING_LEN); } - ret = lwis_gpio_list_to_irqs(lwis_dev, &lwis_dev->irq_gpios_info, irq_gpios_names); - if (ret) { - pr_err("Error get GPIO irq list (%d)\n", ret); + irq_gpios_types = kmalloc(sizeof(u32) * type_count, GFP_KERNEL); + if (IS_ERR_OR_NULL(irq_gpios_types)) { + pr_err("Allocating irq_gpios_types list failed\n"); + ret = -ENOMEM; + goto error_parse_irq_gpios; + } + + type_count = of_property_read_variable_u32_array(dev_node, "irq-gpios-types", + irq_gpios_types, type_count, type_count); + + if (type_count != count) { + pr_err("Error getting irq-gpios-types: %d\n", type_count); + ret = type_count; goto error_parse_irq_gpios; } @@ -152,8 +174,23 @@ static int parse_irq_gpios(struct lwis_device *lwis_dev) } } + for (i = 0; i < gpios->ndescs; ++i) { + char *name; + int irq; + irq = gpiod_to_irq(gpios->desc[i]); + if (irq < 0) { + pr_err("gpio to irq failed (%d)\n", irq); + lwis_interrupt_list_free(lwis_dev->irq_gpios_info.irq_list); + return irq; + } + name = irq_gpios_names + i * LWIS_MAX_NAME_STRING_LEN; + lwis_interrupt_get_gpio_irq(lwis_dev->irq_gpios_info.irq_list, i, name, irq, + irq_gpios_types[i]); + } + kfree(irq_gpios_names); kfree(irq_gpios_events); + kfree(irq_gpios_types); return 0; error_parse_irq_gpios: @@ -161,12 +198,13 @@ error_parse_irq_gpios: lwis_gpio_list_put(lwis_dev->irq_gpios_info.gpios, dev); lwis_dev->irq_gpios_info.gpios = NULL; } - if (irq_gpios_names) { - kfree(irq_gpios_names); - } - if (irq_gpios_events) { - kfree(irq_gpios_events); + if (lwis_dev->irq_gpios_info.irq_list) { + lwis_interrupt_list_free(lwis_dev->irq_gpios_info.irq_list); + lwis_dev->irq_gpios_info.irq_list = NULL; } + kfree(irq_gpios_names); + kfree(irq_gpios_events); + kfree(irq_gpios_types); return ret; } @@ -214,9 +252,11 @@ static int parse_regulators(struct lwis_device *lwis_dev) of_property_count_elems_of_size(dev_node, "regulator-voltages", sizeof(u32)); lwis_dev->regulators = lwis_regulator_list_alloc(count); - if (IS_ERR(lwis_dev->regulators)) { + if (IS_ERR_OR_NULL(lwis_dev->regulators)) { pr_err("Cannot allocate regulator list\n"); - return PTR_ERR(lwis_dev->regulators); + ret = PTR_ERR(lwis_dev->regulators); + lwis_dev->regulators = NULL; + return ret; } /* Parse regulator list and acquire the regulator pointers */ @@ -253,6 +293,7 @@ static int parse_clocks(struct lwis_device *lwis_dev) int i; int ret = 0; int count; + int __maybe_unused bts_count; struct device *dev; struct device_node *dev_node; const char *name; @@ -271,9 +312,11 @@ static int parse_clocks(struct lwis_device *lwis_dev) } lwis_dev->clocks = lwis_clock_list_alloc(count); - if (IS_ERR(lwis_dev->clocks)) { + if (IS_ERR_OR_NULL(lwis_dev->clocks)) { pr_err("Cannot allocate clocks list\n"); - return PTR_ERR(lwis_dev->clocks); + ret = PTR_ERR(lwis_dev->clocks); + lwis_dev->clocks = NULL; + return ret; } /* Parse and acquire clock pointers and frequencies, if applicable */ @@ -294,6 +337,24 @@ static int parse_clocks(struct lwis_device *lwis_dev) ret = of_property_read_u32(dev_node, "clock-family", &clock_family); lwis_dev->clock_family = (ret == 0) ? clock_family : CLOCK_FAMILY_INVALID; + /* Parse the BTS block names */ + bts_count = of_property_count_strings(dev_node, "bts-block-names"); + if (bts_count > 0) { + lwis_dev->bts_block_num = bts_count; + for (i = 0; i < bts_count; ++i) { + of_property_read_string_index(dev_node, "bts-block-names", i, &name); + lwis_dev->bts_block_names[i] = (const char *)name; + } + } else { + lwis_dev->bts_block_num = 1; + lwis_dev->bts_block_names[0] = lwis_dev->name; + } + + /* Initialize all the BTS indexes */ + for (i = 0; i < MAX_BTS_BLOCK_NUM; ++i) { + lwis_dev->bts_indexes[i] = BTS_UNSUPPORTED; + } + #ifdef LWIS_DT_DEBUG pr_info("%s: clock family %d", lwis_dev->name, lwis_dev->clock_family); lwis_clock_print(lwis_dev->clocks); @@ -332,13 +393,13 @@ static int parse_pinctrls(struct lwis_device *lwis_dev, char *expected_state) /* Set up pinctrl */ pc = devm_pinctrl_get(dev); - if (IS_ERR(pc)) { + if (IS_ERR_OR_NULL(pc)) { pr_err("Cannot allocate pinctrl\n"); return PTR_ERR(pc); } pinctrl_state = pinctrl_lookup_state(pc, expected_state); - if (IS_ERR(pinctrl_state)) { + if (IS_ERR_OR_NULL(pinctrl_state)) { pr_err("Cannot find pinctrl state %s\n", expected_state); devm_pinctrl_put(pc); return PTR_ERR(pinctrl_state); @@ -356,7 +417,37 @@ static int parse_pinctrls(struct lwis_device *lwis_dev, char *expected_state) return 0; } -static int parse_critical_irq_events(struct device_node *event_info, u64** irq_events) +static int parse_irq_reg_bits(struct device_node *info, int *bits_num_result, u32 **reg_bits_result) +{ + int int_reg_bits_num; + u32 *int_reg_bits; + + int_reg_bits_num = of_property_count_elems_of_size(info, "int-reg-bits", 4); + if (int_reg_bits_num <= 0) { + pr_err("Error getting int-reg-bits: %d\n", int_reg_bits_num); + return -EINVAL; + } + + int_reg_bits = kmalloc(sizeof(u32) * int_reg_bits_num, GFP_KERNEL); + if (IS_ERR_OR_NULL(int_reg_bits)) { + pr_err("Failed to allocate memory for irq regiater bits\n"); + return -ENOMEM; + } + + *bits_num_result = int_reg_bits_num; + int_reg_bits_num = of_property_read_variable_u32_array(info, "int-reg-bits", int_reg_bits, + int_reg_bits_num, int_reg_bits_num); + if (*bits_num_result != int_reg_bits_num) { + pr_err("Error getting int-reg-bits: %d\n", int_reg_bits_num); + kfree(int_reg_bits); + return int_reg_bits_num; + } + *reg_bits_result = int_reg_bits; + + return 0; +} + +static int parse_critical_irq_events(struct device_node *event_info, u64 **irq_events) { int ret; int critical_irq_events_num; @@ -391,6 +482,150 @@ static int parse_critical_irq_events(struct device_node *event_info, u64** irq_e return critical_irq_events_num; } +static int parse_interrupts_event_info(struct lwis_interrupt_list *list, int index, + struct device_node *event_info) +{ + int irq_events_num; + int int_reg_bits_num = 0; + int critical_events_num = 0; + u64 *irq_events = NULL; + u32 *int_reg_bits = NULL; + u64 *critical_events = NULL; + int ret = 0; + + ret = parse_irq_reg_bits(event_info, &int_reg_bits_num, &int_reg_bits); + if (ret) { + return ret; + } + + irq_events_num = of_property_count_elems_of_size(event_info, "irq-events", 8); + if (irq_events_num != int_reg_bits_num || irq_events_num <= 0) { + pr_err("Error getting irq-events: %d\n", irq_events_num); + ret = -EINVAL; + goto event_info_exit; + } + + irq_events = kmalloc(sizeof(u64) * irq_events_num, GFP_KERNEL); + if (IS_ERR_OR_NULL(irq_events)) { + ret = -ENOMEM; + goto event_info_exit; + } + + irq_events_num = of_property_read_variable_u64_array(event_info, "irq-events", irq_events, + irq_events_num, irq_events_num); + if (irq_events_num != int_reg_bits_num) { + pr_err("Error getting irq-events: %d\n", irq_events_num); + ret = irq_events_num; + goto event_info_exit; + } + + critical_events_num = parse_critical_irq_events(event_info, &critical_events); + + ret = lwis_interrupt_set_event_info(list, index, (int64_t *)irq_events, irq_events_num, + int_reg_bits, int_reg_bits_num, + (int64_t *)critical_events, critical_events_num); + if (ret) { + pr_err("Error setting event info for interrupt %d %d\n", index, ret); + goto event_info_exit; + } + +event_info_exit: + kfree(critical_events); + kfree(irq_events); + kfree(int_reg_bits); + return ret; +} + +static int find_irq_index_by_name(struct lwis_interrupt_list *list, const char *irq_name) +{ + int i; + + for (i = 0; i < list->count; ++i) { + if (strncmp(irq_name, list->irq[i].name, IRQ_FULL_NAME_LENGTH - 1) == 0) { + return i; + } + } + return -ENOENT; +} + +static int parse_interrupt_leaf_nodes(struct lwis_interrupt_list *list, int index, + struct device_node *leaf_info) +{ + int irq_leaves_num; + int int_reg_bits_num; + u32 *int_reg_bits = NULL; + struct of_phandle_iterator it; + int i = 0, ret = 0; + + ret = parse_irq_reg_bits(leaf_info, &int_reg_bits_num, &int_reg_bits); + if (ret) { + return ret; + } + + irq_leaves_num = of_property_count_elems_of_size(leaf_info, "irq-leaf-nodes", 4); + if (irq_leaves_num != int_reg_bits_num || irq_leaves_num <= 0) { + pr_err("Error getting irq-leaf-nodes: %d\n", irq_leaves_num); + ret = -EINVAL; + kfree(int_reg_bits); + return ret; + } + + i = 0; + of_for_each_phandle (&it, ret, leaf_info, "irq-leaf-nodes", 0, 0) { + struct device_node *irq_group_node = of_node_get(it.node); + int leaf_interrupts_count; + const char *leaf_interrupt_name; + int32_t *leaf_indexes = NULL; + int j = 0; + + leaf_interrupts_count = + of_property_count_strings(irq_group_node, "leaf-interrupt-names"); + if (leaf_interrupts_count == -ENODATA) { + /* Does not have a value means no leaf interrupt is configured for this */ + /* leaf node */ + continue; + } else if (leaf_interrupts_count < 0) { + pr_err("Error counting leaf-interrupt-names for : %d\n", + leaf_interrupts_count); + ret = -EINVAL; + goto leaf_error; + } + + leaf_indexes = kmalloc(sizeof(int32_t) * leaf_interrupts_count, GFP_KERNEL); + if (IS_ERR_OR_NULL(leaf_indexes)) { + ret = -ENOMEM; + goto leaf_error; + } + + for (j = 0; j < leaf_interrupts_count; ++j) { + of_property_read_string_index(irq_group_node, "leaf-interrupt-names", j, + &leaf_interrupt_name); + leaf_indexes[j] = find_irq_index_by_name(list, leaf_interrupt_name); + if (leaf_indexes[j] < 0) { + ret = leaf_indexes[j]; + pr_err("Cannot find leaf irq %s\n", leaf_interrupt_name); + kfree(leaf_indexes); + goto leaf_error; + } + } + + ret = lwis_interrupt_add_leaf(list, index, int_reg_bits[i], leaf_interrupts_count, + leaf_indexes); + if (ret) { + pr_err("Error setting event info for interrupt %d %d\n", index, ret); + kfree(leaf_indexes); + goto leaf_error; + } + i++; + } + + return 0; +leaf_error: + lwis_interrupt_free_leaves(&list->irq[index]); + kfree(int_reg_bits); + return ret; +} + static int parse_interrupts(struct lwis_device *lwis_dev) { int i; @@ -404,7 +639,12 @@ static int parse_interrupts(struct lwis_device *lwis_dev) plat_dev = lwis_dev->plat_dev; dev_node = plat_dev->dev.of_node; - count = platform_irq_count(plat_dev); + /* Test device type DEVICE_TYPE_TEST used for test, platform independent. */ + if (lwis_dev->type == DEVICE_TYPE_TEST) { + count = TEST_DEVICE_IRQ_CNT; + } else { + count = platform_irq_count(plat_dev); + } /* No interrupts found, just return */ if (count <= 0) { @@ -413,16 +653,22 @@ static int parse_interrupts(struct lwis_device *lwis_dev) } lwis_dev->irqs = lwis_interrupt_list_alloc(lwis_dev, count); - if (IS_ERR(lwis_dev->irqs)) { - pr_err("Failed to allocate IRQ list\n"); - return PTR_ERR(lwis_dev->irqs); + if (IS_ERR_OR_NULL(lwis_dev->irqs)) { + if (lwis_dev->type == DEVICE_TYPE_TEST) { + pr_err("Failed to allocate injection\n"); + } else { + pr_err("Failed to allocate IRQ list\n"); + } + ret = PTR_ERR(lwis_dev->irqs); + lwis_dev->irqs = NULL; + return ret; } for (i = 0; i < count; ++i) { of_property_read_string_index(dev_node, "interrupt-names", i, &name); - ret = lwis_interrupt_get(lwis_dev->irqs, i, (char *)name, plat_dev); + ret = lwis_interrupt_init(lwis_dev->irqs, i, (char *)name); if (ret) { - pr_err("Cannot set irq %s\n", name); + pr_err("Cannot initialize irq %s\n", name); goto error_get_irq; } } @@ -437,78 +683,23 @@ static int parse_interrupts(struct lwis_device *lwis_dev) /* Get event infos */ i = 0; of_for_each_phandle (&it, ret, dev_node, "interrupt-event-infos", 0, 0) { - const char *irq_reg_space = NULL; + const char *irq_reg_space = NULL, *irq_type_str = NULL; bool irq_mask_reg_toggle; u64 irq_src_reg; u64 irq_reset_reg; u64 irq_mask_reg; - int irq_events_num; - int int_reg_bits_num; - int critical_events_num = 0; - u64 *irq_events; - u32 *int_reg_bits; - u64 *critical_events = NULL; + u64 irq_overflow_reg = 0; int irq_reg_bid = -1; int irq_reg_bid_count; /* To match default value of reg-addr/value-bitwidth. */ u32 irq_reg_bitwidth = 32; + int32_t irq_type = REGULAR_INTERRUPT; int j; struct device_node *event_info = of_node_get(it.node); - irq_events_num = of_property_count_elems_of_size(event_info, "irq-events", 8); - if (irq_events_num <= 0) { - pr_err("Error getting irq-events: %d\n", irq_events_num); - ret = -EINVAL; - goto error_event_infos; - } - - int_reg_bits_num = of_property_count_elems_of_size(event_info, "int-reg-bits", 4); - if (irq_events_num != int_reg_bits_num || int_reg_bits_num <= 0) { - pr_err("Error getting int-reg-bits: %d\n", int_reg_bits_num); - ret = -EINVAL; - goto error_event_infos; - } - - irq_events = kmalloc(sizeof(u64) * irq_events_num, GFP_KERNEL); - if (IS_ERR_OR_NULL(irq_events)) { - ret = -ENOMEM; - goto error_event_infos; - } - - int_reg_bits = kmalloc(sizeof(u32) * int_reg_bits_num, GFP_KERNEL); - if (IS_ERR_OR_NULL(int_reg_bits)) { - ret = -ENOMEM; - kfree(irq_events); - goto error_event_infos; - } - - irq_events_num = of_property_read_variable_u64_array( - event_info, "irq-events", irq_events, irq_events_num, irq_events_num); - if (irq_events_num != int_reg_bits_num) { - pr_err("Error getting irq-events: %d\n", irq_events_num); - ret = irq_events_num; - kfree(irq_events); - kfree(int_reg_bits); - goto error_event_infos; - } - - int_reg_bits_num = - of_property_read_variable_u32_array(event_info, "int-reg-bits", - int_reg_bits, int_reg_bits_num, - int_reg_bits_num); - if (irq_events_num != int_reg_bits_num) { - pr_err("Error getting int-reg-bits: %d\n", int_reg_bits_num); - ret = int_reg_bits_num; - kfree(irq_events); - kfree(int_reg_bits); - goto error_event_infos; - } - ret = of_property_read_string(event_info, "irq-reg-space", &irq_reg_space); if (ret) { pr_err("Error getting irq-reg-space from dt: %d\n", ret); - kfree(irq_events); - kfree(int_reg_bits); goto error_event_infos; } @@ -516,8 +707,6 @@ static int parse_interrupts(struct lwis_device *lwis_dev) if (irq_reg_bid_count <= 0) { pr_err("Error getting reg-names from dt: %d\n", irq_reg_bid_count); - kfree(irq_events); - kfree(int_reg_bits); goto error_event_infos; } for (j = 0; j < irq_reg_bid_count; j++) { @@ -534,63 +723,93 @@ static int parse_interrupts(struct lwis_device *lwis_dev) } if (irq_reg_bid < 0) { pr_err("Could not find a reg bid for %s\n", irq_reg_space); - kfree(irq_events); - kfree(int_reg_bits); goto error_event_infos; } ret = of_property_read_u64(event_info, "irq-src-reg", &irq_src_reg); if (ret) { pr_err("Error getting irq-src-reg from dt: %d\n", ret); - kfree(irq_events); - kfree(int_reg_bits); goto error_event_infos; } ret = of_property_read_u64(event_info, "irq-reset-reg", &irq_reset_reg); if (ret) { pr_err("Error getting irq-reset-reg from dt: %d\n", ret); - kfree(irq_events); - kfree(int_reg_bits); goto error_event_infos; } ret = of_property_read_u64(event_info, "irq-mask-reg", &irq_mask_reg); if (ret) { pr_err("Error getting irq-mask-reg from dt: %d\n", ret); - kfree(irq_events); - kfree(int_reg_bits); goto error_event_infos; } + of_property_read_u64(event_info, "irq-overflow-reg", &irq_overflow_reg); + irq_mask_reg_toggle = of_property_read_bool(event_info, "irq-mask-reg-toggle"); of_property_read_u32(event_info, "irq-reg-bitwidth", &irq_reg_bitwidth); - critical_events_num = parse_critical_irq_events(event_info, &critical_events); + ret = of_property_read_string(event_info, "irq-type", &irq_type_str); + if (ret && ret != -EINVAL) { + pr_err("Error getting irq-type from dt: %d\n", ret); + goto error_event_infos; + } else if (ret && ret == -EINVAL) { + /* The property does not exist, which means regular*/ + irq_type = REGULAR_INTERRUPT; + } else { + if (strcmp(irq_type_str, "regular") == 0) { + irq_type = REGULAR_INTERRUPT; + } else if (strcmp(irq_type_str, "aggregate") == 0) { + irq_type = AGGREGATE_INTERRUPT; + } else if (strcmp(irq_type_str, "leaf") == 0) { + irq_type = LEAF_INTERRUPT; + } else if (strcmp(irq_type_str, "injection") == 0) { + irq_type = FAKEEVENT_INTERRUPT; + } else { + pr_err("Invalid irq-type from dt: %s\n", irq_type_str); + goto error_event_infos; + } + } - ret = lwis_interrupt_set_event_info( - lwis_dev->irqs, i, irq_reg_space, irq_reg_bid, (int64_t *)irq_events, - irq_events_num, int_reg_bits, int_reg_bits_num, irq_src_reg, irq_reset_reg, - irq_mask_reg, irq_mask_reg_toggle, irq_reg_bitwidth, - (int64_t *)critical_events, critical_events_num); - if (ret) { - pr_err("Error setting event info for interrupt %d %d\n", i, ret); - if (critical_events) { - kfree(critical_events); + lwis_interrupt_set_basic_info(lwis_dev->irqs, i, irq_reg_space, irq_reg_bid, + irq_src_reg, irq_reset_reg, irq_mask_reg, + irq_overflow_reg, irq_mask_reg_toggle, + irq_reg_bitwidth, irq_type); + + /* Register IRQ handler only for aggregate and regular interrupts */ + if (irq_type == AGGREGATE_INTERRUPT || irq_type == REGULAR_INTERRUPT) { + ret = lwis_interrupt_get(lwis_dev->irqs, i, plat_dev); + if (ret) { + pr_err("Cannot set irq %s\n", name); + goto error_event_infos; } - kfree(irq_events); - kfree(int_reg_bits); + } else if (irq_type == FAKEEVENT_INTERRUPT) { + /* + * Hardcode the fake injection irq number to + * TEST_DEVICE_FAKE_INJECTION_IRQ + */ + lwis_dev->irqs->irq[i].irq = TEST_DEVICE_FAKE_INJECTION_IRQ; + } + + /* Parse event info */ + ret = parse_interrupts_event_info(lwis_dev->irqs, i, event_info); + if (ret) { + pr_err("Cannot set event info %s\n", name); goto error_event_infos; } + /* Parse leaf nodes if it's an aggregate interrupt */ + if (irq_type == AGGREGATE_INTERRUPT) { + ret = parse_interrupt_leaf_nodes(lwis_dev->irqs, i, event_info); + if (ret) { + pr_err("Error setting leaf nodes for interrupt %d %d\n", i, ret); + goto error_event_infos; + } + } + of_node_put(event_info); i++; - if (critical_events) { - kfree(critical_events); - } - kfree(irq_events); - kfree(int_reg_bits); } #ifdef LWIS_DT_DEBUG @@ -629,9 +848,11 @@ static int parse_phys(struct lwis_device *lwis_dev) } lwis_dev->phys = lwis_phy_list_alloc(count); - if (IS_ERR(lwis_dev->phys)) { + if (IS_ERR_OR_NULL(lwis_dev->phys)) { pr_err("Failed to allocate PHY list\n"); - return PTR_ERR(lwis_dev->phys); + ret = PTR_ERR(lwis_dev->phys); + lwis_dev->phys = NULL; + return ret; } for (i = 0; i < count; ++i) { @@ -683,7 +904,8 @@ static void parse_bitwidths(struct lwis_device *lwis_dev) } static int parse_power_seqs(struct lwis_device *lwis_dev, const char *seq_name, - struct lwis_device_power_sequence_list **list) + struct lwis_device_power_sequence_list **list, + struct device_node *dev_node_seq) { char str_seq_name[LWIS_MAX_NAME_STRING_LEN]; char str_seq_type[LWIS_MAX_NAME_STRING_LEN]; @@ -708,6 +930,9 @@ static int parse_power_seqs(struct lwis_device *lwis_dev, const char *seq_name, dev = &lwis_dev->plat_dev->dev; dev_node = dev->of_node; *list = NULL; + if (dev_node_seq) { + dev_node = dev_node_seq; + } power_seq_count = of_property_count_strings(dev_node, str_seq_name); power_seq_type_count = of_property_count_strings(dev_node, str_seq_type); @@ -724,9 +949,11 @@ static int parse_power_seqs(struct lwis_device *lwis_dev, const char *seq_name, } *list = lwis_dev_power_seq_list_alloc(power_seq_count); - if (IS_ERR(*list)) { + if (IS_ERR_OR_NULL(*list)) { pr_err("Failed to allocate power sequence list\n"); - return PTR_ERR(*list); + ret = PTR_ERR(*list); + *list = NULL; + return ret; } for (i = 0; i < power_seq_count; ++i) { @@ -763,7 +990,7 @@ static int parse_power_seqs(struct lwis_device *lwis_dev, const char *seq_name, if (type_gpio_count > 0 && lwis_dev->gpios_list == NULL) { lwis_dev->gpios_list = lwis_gpios_list_alloc(type_gpio_count); - if (IS_ERR(lwis_dev->gpios_list)) { + if (IS_ERR_OR_NULL(lwis_dev->gpios_list)) { pr_err("Failed to allocate gpios list\n"); ret = PTR_ERR(lwis_dev->gpios_list); goto error_parse_power_seqs; @@ -784,7 +1011,7 @@ static int parse_power_seqs(struct lwis_device *lwis_dev, const char *seq_name, seq_item_name = (*list)->seq_info[i].name; dev = &lwis_dev->plat_dev->dev; descs = lwis_gpio_list_get(dev, seq_item_name); - if (IS_ERR(descs)) { + if (IS_ERR_OR_NULL(descs)) { pr_err("Error parsing GPIO list %s (%ld)\n", seq_item_name, PTR_ERR(descs)); ret = PTR_ERR(descs); @@ -800,7 +1027,7 @@ static int parse_power_seqs(struct lwis_device *lwis_dev, const char *seq_name, gpios_info->gpios = NULL; gpios_info->irq_list = NULL; - strlcpy(gpios_info->name, seq_item_name, LWIS_MAX_NAME_STRING_LEN); + strscpy(gpios_info->name, seq_item_name, LWIS_MAX_NAME_STRING_LEN); if (strncmp(SHARED_STRING, seq_item_name, strlen(SHARED_STRING)) == 0) { gpios_info->is_shared = true; @@ -818,7 +1045,7 @@ static int parse_power_seqs(struct lwis_device *lwis_dev, const char *seq_name, if (type_regulator_count > 0 && lwis_dev->regulators == NULL) { lwis_dev->regulators = lwis_regulator_list_alloc(type_regulator_count); - if (IS_ERR(lwis_dev->regulators)) { + if (IS_ERR_OR_NULL(lwis_dev->regulators)) { pr_err("Failed to allocate regulator list\n"); ret = PTR_ERR(lwis_dev->regulators); goto error_parse_power_seqs; @@ -859,6 +1086,49 @@ error_parse_power_seqs: return ret; } +static int parse_unified_power_seqs(struct lwis_device *lwis_dev) +{ + struct device *dev; + struct device_node *dev_node; + struct device_node *dev_node_seq; + int count; + int ret = 0; + + dev = &lwis_dev->plat_dev->dev; + dev_node = dev->of_node; + + count = of_property_count_elems_of_size(dev_node, "power-seq", sizeof(u32)); + + /* No power-seq found, or entry does not exist, just return */ + if (count <= 0) { + lwis_dev->power_seq_handler = NULL; + return 0; + } + + dev_node_seq = of_parse_phandle(dev_node, "power-seq", 0); + if (!dev_node_seq) { + pr_err("Can't get power-seq node\n"); + return -EINVAL; + } + + ret = parse_power_seqs(lwis_dev, "power-up", &lwis_dev->power_up_sequence, dev_node_seq); + if (ret) { + pr_err("Error parsing power-up-seqs\n"); + return ret; + } + + ret = parse_power_seqs(lwis_dev, "power-down", &lwis_dev->power_down_sequence, + dev_node_seq); + if (ret) { + pr_err("Error parsing power-down-seqs\n"); + return ret; + } + + lwis_dev->power_seq_handler = dev_node_seq; + + return ret; +} + static int parse_pm_hibernation(struct lwis_device *lwis_dev) { struct device_node *dev_node; @@ -888,12 +1158,9 @@ static int parse_thread_priority(struct lwis_device *lwis_dev) dev_node = lwis_dev->plat_dev->dev.of_node; lwis_dev->transaction_thread_priority = 0; - lwis_dev->periodic_io_thread_priority = 0; of_property_read_u32(dev_node, "transaction-thread-priority", &lwis_dev->transaction_thread_priority); - of_property_read_u32(dev_node, "periodic-io-thread-priority", - &lwis_dev->periodic_io_thread_priority); return 0; } @@ -944,7 +1211,7 @@ int lwis_base_parse_dt(struct lwis_device *lwis_dev) pr_err("Error parsing node name\n"); return -EINVAL; } - strlcpy(lwis_dev->name, name_str, LWIS_MAX_NAME_STRING_LEN); + strscpy(lwis_dev->name, name_str, LWIS_MAX_NAME_STRING_LEN); pr_debug("Device tree entry [%s] - begin\n", lwis_dev->name); @@ -972,15 +1239,38 @@ int lwis_base_parse_dt(struct lwis_device *lwis_dev) return ret; } - ret = parse_power_seqs(lwis_dev, "power-up", &lwis_dev->power_up_sequence); + ret = parse_unified_power_seqs(lwis_dev); if (ret) { - pr_err("Error parsing power-up-seqs\n"); + pr_err("Error parse_unified_power_seqs\n"); return ret; } - ret = parse_power_seqs(lwis_dev, "power-down", &lwis_dev->power_down_sequence); + if (lwis_dev->power_up_sequence == NULL) { + ret = parse_power_seqs(lwis_dev, "power-up", &lwis_dev->power_up_sequence, NULL); + if (ret) { + pr_err("Error parsing power-up-seqs\n"); + return ret; + } + } + + if (lwis_dev->power_down_sequence == NULL) { + ret = parse_power_seqs(lwis_dev, "power-down", &lwis_dev->power_down_sequence, + NULL); + if (ret) { + pr_err("Error parsing power-down-seqs\n"); + return ret; + } + } + + ret = parse_power_seqs(lwis_dev, "suspend", &lwis_dev->suspend_sequence, NULL); if (ret) { - pr_err("Error parsing power-down-seqs\n"); + pr_err("Error parsing suspend-seqs\n"); + return ret; + } + + ret = parse_power_seqs(lwis_dev, "resume", &lwis_dev->resume_sequence, NULL); + if (ret) { + pr_err("Error parsing resume-seqs\n"); return ret; } @@ -1125,3 +1415,9 @@ int lwis_top_device_parse_dt(struct lwis_top_device *top_dev) /* To be implemented */ return 0; } + +int lwis_test_device_parse_dt(struct lwis_test_device *test_dev) +{ + /* To be implemented */ + return 0; +} @@ -16,6 +16,7 @@ #include "lwis_device.h" #include "lwis_device_i2c.h" #include "lwis_device_ioreg.h" +#include "lwis_device_test.h" #include "lwis_device_top.h" /* @@ -42,4 +43,10 @@ int lwis_ioreg_device_parse_dt(struct lwis_ioreg_device *ioreg_dev); */ int lwis_top_device_parse_dt(struct lwis_top_device *top_dev); +/* + * lwis_test_device_parse_dt: Parse device configurations specifically for + * TEST devices. + */ +int lwis_test_device_parse_dt(struct lwis_test_device *test_dev); + #endif /* LWIS_DT_H_ */ diff --git a/lwis_event.c b/lwis_event.c index 128280a..971d70a 100644 --- a/lwis_event.c +++ b/lwis_event.c @@ -24,13 +24,13 @@ /* Exposes the device id embedded in the event id */ #define EVENT_OWNER_DEVICE_ID(x) ((x >> LWIS_EVENT_ID_EVENT_CODE_LEN) & 0xFFFF) -#define lwis_dev_err_ratelimited(dev, fmt, ...) \ - { \ - static int64_t timestamp = 0; \ - if (ktime_to_ns(lwis_get_time()) - timestamp > 200000000LL) { \ - dev_err(dev, fmt, ##__VA_ARGS__); \ - timestamp = ktime_to_ns(lwis_get_time()); \ - } \ +#define lwis_dev_err_ratelimited(dev, fmt, ...) \ + { \ + static int64_t timestamp = 0; \ + if (ktime_to_ns(lwis_get_time()) - timestamp > 200000000LL) { \ + dev_err(dev, fmt, ##__VA_ARGS__); \ + timestamp = ktime_to_ns(lwis_get_time()); \ + } \ } /* @@ -47,11 +47,15 @@ lwis_client_event_state_find_locked(struct lwis_client *lwis_client, int64_t eve { /* Our hash iterator */ struct lwis_client_event_state *p; + int64_t new_event_id; + new_event_id = (event_id & LWIS_OVERFLOW_IRQ_EVENT_FLAG) ? + (event_id ^ LWIS_OVERFLOW_IRQ_EVENT_FLAG) : + event_id; /* Iterate through the hash bucket for this event_id */ - hash_for_each_possible (lwis_client->event_states, p, node, event_id) { + hash_for_each_possible (lwis_client->event_states, p, node, new_event_id) { /* If it's indeed the right one, return it */ - if (p->event_control.event_id == event_id) { + if (p->event_control.event_id == new_event_id) { return p; } } @@ -162,11 +166,15 @@ lwis_device_event_state_find_locked(struct lwis_device *lwis_dev, int64_t event_ { /* Our hash iterator */ struct lwis_device_event_state *p; + int64_t new_event_id; + new_event_id = (event_id & LWIS_OVERFLOW_IRQ_EVENT_FLAG) ? + (event_id ^ LWIS_OVERFLOW_IRQ_EVENT_FLAG) : + event_id; /* Iterate through the hash bucket for this event_id */ - hash_for_each_possible (lwis_dev->event_states, p, node, event_id) { + hash_for_each_possible (lwis_dev->event_states, p, node, new_event_id) { /* If it's indeed the right one, return it */ - if (p->event_id == event_id) { + if (p->event_id == new_event_id) { return p; } } @@ -569,7 +577,8 @@ static int lwis_client_event_push_back(struct lwis_client *lwis_client, list_first_entry(&lwis_client->event_queue, struct lwis_event_entry, node); current_timestamp = lwis_get_time(); timestamp_diff = ktime_sub(current_timestamp, first_event->event_info.timestamp_ns); - lwis_dev_err_ratelimited(lwis_client->lwis_dev->dev, + lwis_dev_err_ratelimited( + lwis_client->lwis_dev->dev, "First event in queue ID: 0x%llx, current timestamp %lld ns, diff: %lld ns\n", event->event_info.event_id, current_timestamp, timestamp_diff); spin_unlock_irqrestore(&lwis_client->event_lock, flags); @@ -717,7 +726,7 @@ int lwis_device_event_flags_updated(struct lwis_device *lwis_dev, int64_t event_ /* Call our handler dispatcher */ ret = lwis_device_event_enable(lwis_dev, event_id, event_enabled); if (ret) { - dev_err(lwis_dev->dev, "Failed to %s event: %lld (err:%d)\n", + dev_err(lwis_dev->dev, "Failed to %s event: 0x%llx (err:%d)\n", event_enabled ? "enable" : "disable", event_id, ret); return ret; } @@ -785,6 +794,16 @@ int lwis_device_event_enable(struct lwis_device *lwis_dev, int64_t event_id, boo err = ret; } } + + if (lwis_dev->irq_gpios_info.irq_list) { + ret = lwis_interrupt_event_enable(lwis_dev->irq_gpios_info.irq_list, + event_id, enabled); + if (ret && ret != -EINVAL) { + dev_err(lwis_dev->dev, "Failed to %s GPIO IRQ event: %lld (e:%d)\n", + enabled ? "enable" : "disable", event_id, ret); + err = ret; + } + } } /* Check if our specialization cares about event updates */ if (!err && lwis_dev->vops.event_enable) { @@ -800,7 +819,7 @@ int lwis_device_event_enable(struct lwis_device *lwis_dev, int64_t event_id, boo static int lwis_device_event_emit_impl(struct lwis_device *lwis_dev, int64_t event_id, void *payload, size_t payload_size, - struct list_head *pending_events, bool in_irq) + struct list_head *pending_events) { struct lwis_client_event_state *client_event_state; struct lwis_device_event_state *device_event_state; @@ -820,7 +839,7 @@ static int lwis_device_event_emit_impl(struct lwis_device *lwis_dev, int64_t eve device_event_state = lwis_device_event_state_find_locked(lwis_dev, event_id); if (IS_ERR_OR_NULL(device_event_state)) { - dev_err(lwis_dev->dev, "Device event state not found %llx\n", event_id); + dev_err(lwis_dev->dev, "Device event state not found 0x%llx\n", event_id); spin_unlock_irqrestore(&lwis_dev->lock, flags); return -EINVAL; } @@ -842,7 +861,7 @@ static int lwis_device_event_emit_impl(struct lwis_device *lwis_dev, int64_t eve /* Emit event to subscriber via top device */ if (has_subscriber) { lwis_dev->top_dev->subscribe_ops.notify_event_subscriber( - lwis_dev->top_dev, event_id, event_counter, timestamp, in_irq); + lwis_dev->top_dev, event_id, event_counter, timestamp); } /* Run internal handler if any */ @@ -892,7 +911,8 @@ static int lwis_device_event_emit_impl(struct lwis_device *lwis_dev, int64_t eve } ret = lwis_client_event_push_back(lwis_client, event); if (ret) { - lwis_dev_err_ratelimited(lwis_dev->dev, + lwis_dev_err_ratelimited( + lwis_dev->dev, "Failed to push event to queue: ID 0x%llx Counter %lld\n", event_id, event_counter); kfree(event); @@ -903,7 +923,7 @@ static int lwis_device_event_emit_impl(struct lwis_device *lwis_dev, int64_t eve /* Trigger transactions, if there's any that matches this event ID and counter */ if (lwis_transaction_event_trigger(lwis_client, event_id, event_counter, - pending_events, in_irq)) { + pending_events)) { dev_warn(lwis_dev->dev, "Failed to process transactions: Event ID: 0x%llx Counter: %lld\n", event_id, event_counter); @@ -914,7 +934,7 @@ static int lwis_device_event_emit_impl(struct lwis_device *lwis_dev, int64_t eve } int lwis_device_event_emit(struct lwis_device *lwis_dev, int64_t event_id, void *payload, - size_t payload_size, bool in_irq) + size_t payload_size) { int ret; struct list_head pending_events; @@ -925,16 +945,16 @@ int lwis_device_event_emit(struct lwis_device *lwis_dev, int64_t event_id, void /* Emit the original event */ ret = lwis_device_event_emit_impl(lwis_dev, event_id, payload, payload_size, - &pending_events, in_irq); + &pending_events); if (ret) { lwis_dev_err_ratelimited(lwis_dev->dev, - "lwis_device_event_emit_impl failed: event ID 0x%llx\n", - event_id); + "lwis_device_event_emit_impl failed: event ID 0x%llx\n", + event_id); return ret; } /* Emit pending events */ - return lwis_pending_events_emit(lwis_dev, &pending_events, in_irq); + return lwis_pending_events_emit(lwis_dev, &pending_events); } int lwis_pending_event_push(struct list_head *pending_events, int64_t event_id, void *payload, @@ -962,23 +982,22 @@ int lwis_pending_event_push(struct list_head *pending_events, int64_t event_id, return 0; } -int lwis_pending_events_emit(struct lwis_device *lwis_dev, struct list_head *pending_events, - bool in_irq) +int lwis_pending_events_emit(struct lwis_device *lwis_dev, struct list_head *pending_events) { int return_val = 0, emit_result = 0; struct lwis_event_entry *event; while (!list_empty(pending_events)) { event = list_first_entry(pending_events, struct lwis_event_entry, node); - emit_result = lwis_device_event_emit_impl(lwis_dev, event->event_info.event_id, - event->event_info.payload_buffer, - event->event_info.payload_size, - pending_events, in_irq); + emit_result = + lwis_device_event_emit_impl(lwis_dev, event->event_info.event_id, + event->event_info.payload_buffer, + event->event_info.payload_size, pending_events); if (emit_result) { return_val = emit_result; dev_warn_ratelimited(lwis_dev->dev, - "lwis_device_pending_event_emit error on ID 0x%llx\n", - event->event_info.event_id); + "lwis_device_pending_event_emit error on ID 0x%llx\n", + event->event_info.event_id); } list_del(&event->node); kfree(event); @@ -1011,7 +1030,7 @@ out: } void lwis_device_external_event_emit(struct lwis_device *lwis_dev, int64_t event_id, - int64_t event_counter, int64_t timestamp, bool in_irq) + int64_t event_counter, int64_t timestamp) { struct lwis_client_event_state *client_event_state; struct lwis_device_event_state *device_event_state; @@ -1071,7 +1090,8 @@ void lwis_device_external_event_emit(struct lwis_device *lwis_dev, int64_t event event->event_info.payload_size = 0; event->event_info.payload_buffer = NULL; if (lwis_client_event_push_back(lwis_client, event)) { - lwis_dev_err_ratelimited(lwis_dev->dev, + lwis_dev_err_ratelimited( + lwis_dev->dev, "Failed to push event to queue: ID 0x%llx Counter %lld\n", event_id, event_counter); kfree(event); @@ -1080,14 +1100,13 @@ void lwis_device_external_event_emit(struct lwis_device *lwis_dev, int64_t event } if (lwis_transaction_event_trigger(lwis_client, event_id, event_counter, - &pending_events, in_irq)) + &pending_events)) dev_warn( lwis_dev->dev, "Failed to process transactions: external event ID: 0x%llx counter: %lld\n", event_id, event_counter); } - - lwis_pending_events_emit(lwis_dev, &pending_events, in_irq); + lwis_pending_events_emit(lwis_dev, &pending_events); } void lwis_device_error_event_emit(struct lwis_device *lwis_dev, int64_t event_id, void *payload, @@ -1130,8 +1149,8 @@ void lwis_device_error_event_emit(struct lwis_device *lwis_dev, int64_t event_id } if (lwis_client_error_event_push_back(lwis_client, event)) { lwis_dev_err_ratelimited(lwis_dev->dev, - "Failed to push error event to queue: ID 0x%llx\n", - event_id); + "Failed to push error event to queue: ID 0x%llx\n", + event_id); kfree(event); return; } diff --git a/lwis_event.h b/lwis_event.h index c8e2642..86f8fa5 100644 --- a/lwis_event.h +++ b/lwis_event.h @@ -232,7 +232,7 @@ int lwis_device_event_enable(struct lwis_device *lwis_dev, int64_t event_id, boo * Returns: 0 on success */ int lwis_device_event_emit(struct lwis_device *lwis_dev, int64_t event_id, void *payload, - size_t payload_size, bool in_irq); + size_t payload_size); /* * lwis_device_external_event_emit: Emits an subscribed event to device. @@ -243,7 +243,7 @@ int lwis_device_event_emit(struct lwis_device *lwis_dev, int64_t event_id, void * 4. Not supported chain transaction */ void lwis_device_external_event_emit(struct lwis_device *lwis_dev, int64_t event_id, - int64_t event_counter, int64_t timestamp, bool in_irq); + int64_t event_counter, int64_t timestamp); /* * lwis_device_error_event_emit: Emits an error event for all clients. @@ -311,8 +311,7 @@ int lwis_pending_event_push(struct list_head *pending_events, int64_t event_id, * * Returns: 0 on success */ -int lwis_pending_events_emit(struct lwis_device *lwis_dev, struct list_head *pending_events, - bool in_irq); +int lwis_pending_events_emit(struct lwis_device *lwis_dev, struct list_head *pending_events); /* * lwis_device_event_update_subscriber: The function to notify an event has been subscribed/unsubscribed. diff --git a/lwis_fence.c b/lwis_fence.c new file mode 100644 index 0000000..184dab7 --- /dev/null +++ b/lwis_fence.c @@ -0,0 +1,660 @@ +/* + * Google LWIS Fence + * + * Copyright (c) 2022 Google, LLC + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/anon_inodes.h> +#include <linux/file.h> +#include <linux/mm.h> +#include <linux/poll.h> + +#include "lwis_device_top.h" +#include "lwis_commands.h" +#include "lwis_fence.h" +#include "lwis_transaction.h" + +#define HASH_CLIENT(x) hash_ptr(x, LWIS_CLIENTS_HASH_BITS) + +bool lwis_fence_debug; +module_param(lwis_fence_debug, bool, 0644); + +static int lwis_fence_release(struct inode *node, struct file *fp); +static ssize_t lwis_fence_get_status(struct file *fp, char __user *user_buffer, size_t len, + loff_t *offset); +static ssize_t lwis_fence_write_status(struct file *fp, const char __user *user_buffer, size_t len, + loff_t *offset); +static unsigned int lwis_fence_poll(struct file *fp, poll_table *wait); + +static const struct file_operations fence_file_ops = { + .owner = THIS_MODULE, + .release = lwis_fence_release, + .read = lwis_fence_get_status, + .write = lwis_fence_write_status, + .poll = lwis_fence_poll, +}; + +static int get_fence_status(struct lwis_fence *lwis_fence) +{ + int status; + unsigned long flags; + spin_lock_irqsave(&lwis_fence->lock, flags); + status = lwis_fence->status; + spin_unlock_irqrestore(&lwis_fence->lock, flags); + return status; +} + +/* + * lwis_fence_release: Closing an instance of a LWIS fence + */ +static int lwis_fence_release(struct inode *node, struct file *fp) +{ + struct lwis_fence *lwis_fence = fp->private_data; + struct lwis_fence_trigger_transaction_list *tx_list; + struct lwis_pending_transaction_id *transaction_id; + /* Temporary vars for traversal */ + struct hlist_node *n; + struct list_head *it_tran, *it_tran_tmp; + int i; + + if (lwis_fence_debug) { + dev_info(lwis_fence->lwis_top_dev->dev, "Releasing lwis_fence fd-%d", + lwis_fence->fd); + } + + if (lwis_fence->status == LWIS_FENCE_STATUS_NOT_SIGNALED) { + dev_err(lwis_fence->lwis_top_dev->dev, + "lwis_fence fd-%d release without being signaled", lwis_fence->fd); + } + + if (!hash_empty(lwis_fence->transaction_list)) { + hash_for_each_safe (lwis_fence->transaction_list, i, n, tx_list, node) { + if (!list_empty(&tx_list->list)) { + list_for_each_safe (it_tran, it_tran_tmp, &tx_list->list) { + transaction_id = + list_entry(it_tran, + struct lwis_pending_transaction_id, + list_node); + list_del(&transaction_id->list_node); + kfree(transaction_id); + } + } + hash_del(&tx_list->node); + kfree(tx_list); + } + } + + kfree(lwis_fence); + return 0; +} + +/* + * lwis_fence_get_status: Read the LWIS fence's status + */ +static ssize_t lwis_fence_get_status(struct file *fp, char __user *user_buffer, size_t len, + loff_t *offset) +{ + int status = 0; + struct lwis_fence *lwis_fence = fp->private_data; + int max_len, read_len; + + if (!lwis_fence) { + dev_err(lwis_fence->lwis_top_dev->dev, "Cannot find lwis_fence instance\n"); + return -EFAULT; + } + + max_len = sizeof(status) - *offset; + if (len > max_len) { + len = max_len; + } + + status = get_fence_status(lwis_fence); + read_len = len - copy_to_user((void __user *)user_buffer, (void *)&status + *offset, len); + + return read_len; +} + +/* + * lwis_fence_write_status: Signal fence with the error code from user + */ +static ssize_t lwis_fence_write_status(struct file *fp, const char __user *user_buffer, size_t len, + loff_t *offset) +{ + int ret = 0; + int status = 0; + struct lwis_fence *lwis_fence = fp->private_data; + + if (!lwis_fence) { + dev_err(lwis_fence->lwis_top_dev->dev, "Cannot find lwis_fence instance\n"); + return -EFAULT; + } + + if (len != sizeof(lwis_fence->status)) { + dev_err(lwis_fence->lwis_top_dev->dev, + "Signal lwis_fence fd-%d with incorrect buffer length\n", lwis_fence->fd); + return -EINVAL; + } + + /* Set lwis_fence's status if not signaled */ + len = len - copy_from_user(&status, (void __user *)user_buffer, len); + ret = lwis_fence_signal(lwis_fence, status); + if (ret) { + return ret; + } + + return len; +} + +int lwis_fence_signal(struct lwis_fence *lwis_fence, int status) +{ + unsigned long flags; + struct lwis_fence_trigger_transaction_list *tx_list; + /* Temporary vars for hash table traversal */ + struct hlist_node *n; + int i; + + spin_lock_irqsave(&lwis_fence->lock, flags); + + if (lwis_fence->status != LWIS_FENCE_STATUS_NOT_SIGNALED) { + /* Return error if fence is already signaled */ + dev_err(lwis_fence->lwis_top_dev->dev, + "Cannot signal a lwis_fence fd-%d already signaled, status is %d\n", + lwis_fence->fd, lwis_fence->status); + spin_unlock_irqrestore(&lwis_fence->lock, flags); + return -EINVAL; + } + lwis_fence->status = status; + spin_unlock_irqrestore(&lwis_fence->lock, flags); + + wake_up_interruptible(&lwis_fence->status_wait_queue); + + hash_for_each_safe (lwis_fence->transaction_list, i, n, tx_list, node) { + hash_del(&tx_list->node); + lwis_transaction_fence_trigger(tx_list->owner, lwis_fence, &tx_list->list); + if (!list_empty(&tx_list->list)) { + dev_err(lwis_fence->lwis_top_dev->dev, + "Fail to trigger all transactions\n"); + } + kfree(tx_list); + } + + return 0; +} + +/* + * lwis_fence_poll: Poll status function of LWIS fence + */ +static unsigned int lwis_fence_poll(struct file *fp, poll_table *wait) +{ + unsigned long flags; + int status = 0; + struct lwis_fence *lwis_fence = fp->private_data; + if (!lwis_fence) { + dev_err(lwis_fence->lwis_top_dev->dev, "Cannot find lwis_fence instance\n"); + return POLLERR; + } + + poll_wait(fp, &lwis_fence->status_wait_queue, wait); + + spin_lock_irqsave(&lwis_fence->lock, flags); + status = lwis_fence->status; + spin_unlock_irqrestore(&lwis_fence->lock, flags); + + /* Check if the fence is already signaled */ + if (status != LWIS_FENCE_STATUS_NOT_SIGNALED) { + return POLLIN; + } + + return 0; +} + +int lwis_fence_create(struct lwis_device *lwis_dev) +{ + int fd_or_err; + struct lwis_fence *new_fence; + + /* Allocate a new instance of lwis_fence struct */ + new_fence = kmalloc(sizeof(struct lwis_fence), GFP_ATOMIC); + if (!new_fence) { + dev_err(lwis_dev->dev, "Failed to allocate lwis_fence at creating new fence\n"); + return -ENOMEM; + } + + /* Open a new fd for the new fence */ + fd_or_err = + anon_inode_getfd("lwis_fence_file", &fence_file_ops, new_fence, O_RDWR | O_CLOEXEC); + if (fd_or_err < 0) { + kfree(new_fence); + dev_err(lwis_dev->dev, "Failed to create a new file instance for lwis_fence\n"); + return fd_or_err; + } + + new_fence->fd = fd_or_err; + new_fence->lwis_top_dev = lwis_dev->top_dev; + new_fence->status = LWIS_FENCE_STATUS_NOT_SIGNALED; + spin_lock_init(&new_fence->lock); + init_waitqueue_head(&new_fence->status_wait_queue); + if (lwis_fence_debug) { + dev_info(lwis_dev->dev, "lwis_fence created new LWIS fence fd: %d", new_fence->fd); + } + return fd_or_err; +} + +static struct lwis_fence_trigger_transaction_list *transaction_list_find(struct lwis_fence *fence, + struct lwis_client *owner) +{ + int hash_key = HASH_CLIENT(owner); + struct lwis_fence_trigger_transaction_list *tx_list; + hash_for_each_possible (fence->transaction_list, tx_list, node, hash_key) { + if (tx_list->owner == owner) { + return tx_list; + } + } + return NULL; +} + +static struct lwis_fence_trigger_transaction_list * +transaction_list_create(struct lwis_fence *fence, struct lwis_client *owner) +{ + struct lwis_fence_trigger_transaction_list *tx_list = + kmalloc(sizeof(struct lwis_fence_trigger_transaction_list), GFP_ATOMIC); + if (!tx_list) { + dev_err(fence->lwis_top_dev->dev, "Cannot allocate new event list\n"); + return NULL; + } + tx_list->owner = owner; + INIT_LIST_HEAD(&tx_list->list); + hash_add(fence->transaction_list, &tx_list->node, HASH_CLIENT(owner)); + return tx_list; +} + +static struct lwis_fence_trigger_transaction_list * +transaction_list_find_or_create(struct lwis_fence *fence, struct lwis_client *owner) +{ + struct lwis_fence_trigger_transaction_list *list = transaction_list_find(fence, owner); + return (list == NULL) ? transaction_list_create(fence, owner) : list; +} + +static int lwis_trigger_fence_add_transaction(int fence_fd, struct lwis_client *client, + struct lwis_transaction *transaction) +{ + unsigned long flags; + struct file *fp; + struct lwis_fence *lwis_fence; + struct lwis_pending_transaction_id *pending_transaction_id; + struct lwis_fence_trigger_transaction_list *tx_list; + int ret = 0; + + if (transaction->num_trigger_fences >= LWIS_TRIGGER_NODES_MAX_NUM) { + dev_err(client->lwis_dev->dev, + "Invalid num_trigger_fences value in transaction %d\n", fence_fd); + return -EINVAL; + } + + fp = fget(fence_fd); + if (fp == NULL) { + dev_err(client->lwis_dev->dev, "Failed to find lwis_fence with fd %d\n", fence_fd); + return -EBADF; + } + lwis_fence = fp->private_data; + if (lwis_fence->fd != fence_fd) { + fput(fp); + dev_err(client->lwis_dev->dev, + "Invalid lwis_fence with fd %d. Contains stale data \n", fence_fd); + return -EBADF; + } + + pending_transaction_id = kmalloc(sizeof(struct lwis_pending_transaction_id), GFP_ATOMIC); + if (!pending_transaction_id) { + fput(fp); + dev_err(client->lwis_dev->dev, + "Failed to allocate lwis_pending_transaction_id at adding transactions to fence\n"); + return -ENOMEM; + } + pending_transaction_id->id = transaction->info.id; + + spin_lock_irqsave(&lwis_fence->lock, flags); + if (lwis_fence->status == LWIS_FENCE_STATUS_NOT_SIGNALED) { + transaction->trigger_fence_fps[transaction->num_trigger_fences++] = fp; + tx_list = transaction_list_find_or_create(lwis_fence, client); + list_add(&pending_transaction_id->list_node, &tx_list->list); + if (lwis_fence_debug) { + dev_info(client->lwis_dev->dev, + "lwis_fence transaction id %llu added to its trigger fence fd %d ", + transaction->info.id, lwis_fence->fd); + } + } else { + kfree(pending_transaction_id); + if (lwis_fence_debug) { + dev_info( + client->lwis_dev->dev, + "lwis_fence fd-%d not added to transaction id %llu, fence already signaled with error code %d \n", + fence_fd, transaction->info.id, lwis_fence->status); + } + if (!transaction->info.is_level_triggered) { + /* If level triggering is disabled, return an error. */ + fput(fp); + ret = -EINVAL; + } else { + transaction->trigger_fence_fps[transaction->num_trigger_fences++] = fp; + /* If the transaction's trigger_condition evaluates to true, queue the + * transaction to be executed immediately. + */ + if (lwis_fence_triggered_condition_ready(transaction, lwis_fence->status)) { + if (lwis_fence->status != 0) { + transaction->resp->error_code = -ECANCELED; + } + transaction->queue_immediately = true; + } + } + } + spin_unlock_irqrestore(&lwis_fence->lock, flags); + + return ret; +} + +bool lwis_triggered_by_condition(struct lwis_transaction *transaction) +{ + return (transaction->info.trigger_condition.num_nodes > 0); +} + +bool lwis_event_triggered_condition_ready(struct lwis_transaction *transaction, + struct lwis_transaction *weak_transaction, + int64_t event_id, int64_t event_counter) +{ + int32_t operator_type; + size_t all_signaled; + struct lwis_transaction_info_v2 *info = &transaction->info; + int i; + struct lwis_fence *lwis_fence; + bool is_node_signaled = false; + + operator_type = info->trigger_condition.operator_type; + all_signaled = info->trigger_condition.num_nodes; + + /* + * Three scenarios to consider a node signaled: + * 1) Event ID and event counter match, + * 2) Event ID match, event counter not specified but precondition fence signaled, or, + * 3) Event ID match, event counter and precondition fence not specified. + */ + for (i = 0; i < info->trigger_condition.num_nodes; i++) { + is_node_signaled = false; + if (info->trigger_condition.trigger_nodes[i].type == LWIS_TRIGGER_EVENT && + info->trigger_condition.trigger_nodes[i].event.id == event_id) { + if (info->trigger_condition.trigger_nodes[i].event.counter == + event_counter || + (info->trigger_condition.trigger_nodes[i].event.counter == + LWIS_EVENT_COUNTER_ON_NEXT_OCCURRENCE && + weak_transaction->precondition_fence_fp == NULL)) { + is_node_signaled = true; + } else if (info->trigger_condition.trigger_nodes[i].event.counter == + LWIS_EVENT_COUNTER_ON_NEXT_OCCURRENCE) { + lwis_fence = weak_transaction->precondition_fence_fp->private_data; + if (lwis_fence != NULL && get_fence_status(lwis_fence) == 0) { + is_node_signaled = true; + if (lwis_fence_debug) { + pr_info("TransactionId %lld: event 0x%llx (%lld), precondition fence %d signaled", + info->id, event_id, event_counter, + info->trigger_condition.trigger_nodes[i] + .event.precondition_fence_fd); + } + } else { + if (lwis_fence_debug) { + pr_info("TransactionId %lld: event 0x%llx (%lld), precondition fence %d NOT signaled yet", + info->id, event_id, event_counter, + info->trigger_condition.trigger_nodes[i] + .event.precondition_fence_fd); + } + } + } + + if (is_node_signaled) { + transaction->signaled_count++; + list_del(&weak_transaction->event_list_node); + if (weak_transaction->precondition_fence_fp) { + fput(weak_transaction->precondition_fence_fp); + } + kfree(weak_transaction); + /* The break here assumes that this event ID only appears once in + the trigger expression. Might need to revisit this. */ + break; + } + } + } + + if (i >= info->trigger_condition.num_nodes) { + /* No event counter is matched */ + return false; + } + + if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_AND && + transaction->signaled_count == all_signaled) { + return true; + } else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_OR) { + return true; + } else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_NONE) { + return true; + } + + return false; +} + +bool lwis_fence_triggered_condition_ready(struct lwis_transaction *transaction, int fence_status) +{ + int32_t operator_type; + size_t all_signaled; + + operator_type = transaction->info.trigger_condition.operator_type; + all_signaled = transaction->info.trigger_condition.num_nodes; + + transaction->signaled_count++; + if ((operator_type == LWIS_TRIGGER_NODE_OPERATOR_AND || + operator_type == LWIS_TRIGGER_NODE_OPERATOR_OR) && + transaction->signaled_count == all_signaled) { + return true; + } else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_AND && fence_status != 0) { + /* + This condition is ready to cancel transaction as long as there is + an error condition from fence with operator type "AND". + No matter whether all condition nodes are signaled. + */ + return true; + } else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_OR && fence_status == 0) { + return true; + } else if (operator_type == LWIS_TRIGGER_NODE_OPERATOR_NONE) { + return true; + } + + return false; +} + +int lwis_parse_trigger_condition(struct lwis_client *client, struct lwis_transaction *transaction) +{ + struct lwis_transaction_info_v2 *info; + struct lwis_device *lwis_dev; + int i, ret; + + if (!transaction || !client) { + dev_err(client->lwis_dev->dev, "Invalid lwis transaction\n"); + return -EINVAL; + } + + info = &transaction->info; + lwis_dev = client->lwis_dev; + + if (info->trigger_condition.num_nodes > LWIS_TRIGGER_NODES_MAX_NUM) { + dev_err(lwis_dev->dev, + "Trigger condition contains %lu node, more than the limit of %d\n", + info->trigger_condition.num_nodes, LWIS_TRIGGER_NODES_MAX_NUM); + return -EINVAL; + } + + for (i = 0; i < info->trigger_condition.num_nodes; i++) { + if (info->trigger_condition.trigger_nodes[i].type == LWIS_TRIGGER_EVENT) { + ret = lwis_trigger_event_add_weak_transaction( + client, info->id, info->trigger_condition.trigger_nodes[i].event.id, + info->trigger_condition.trigger_nodes[i] + .event.precondition_fence_fd); + } else { + ret = lwis_trigger_fence_add_transaction( + info->trigger_condition.trigger_nodes[i].fence_fd, client, + transaction); + } + if (ret) { + return ret; + } + } + + return 0; +} + +int ioctl_lwis_fence_create(struct lwis_device *lwis_dev, int32_t __user *msg) +{ + int32_t fd_or_err; + + fd_or_err = lwis_fence_create(lwis_dev); + if (fd_or_err < 0) { + return fd_or_err; + } + + if (copy_to_user((void __user *)msg, &fd_or_err, sizeof(int32_t))) { + dev_err(lwis_dev->dev, "failed to copy to user\n"); + return -EFAULT; + } + + return 0; +} + +int lwis_initialize_transaction_fences(struct lwis_client *client, + struct lwis_transaction *transaction) +{ + struct lwis_transaction_info_v2 *info = &transaction->info; + struct lwis_device *lwis_dev = client->lwis_dev; + int i; + int fd_or_err; + + if (!transaction || !client) { + dev_err(client->lwis_dev->dev, "Invalid lwis transaction\n"); + return -EINVAL; + } + + /* If triggered by trigger_condition */ + if (lwis_triggered_by_condition(transaction)) { + /* Initialize all placeholder fences in the trigger_condition */ + for (i = 0; i < info->trigger_condition.num_nodes; i++) { + if (info->trigger_condition.trigger_nodes[i].type == + LWIS_TRIGGER_FENCE_PLACEHOLDER) { + fd_or_err = lwis_fence_create(lwis_dev); + if (fd_or_err < 0) { + return fd_or_err; + } + info->trigger_condition.trigger_nodes[i].fence_fd = fd_or_err; + } + } + } + + /* Initialize completion fence if one is requested */ + if (info->completion_fence_fd == LWIS_CREATE_COMPLETION_FENCE) { + fd_or_err = lwis_fence_create(client->lwis_dev); + if (fd_or_err < 0) { + return fd_or_err; + } + info->completion_fence_fd = fd_or_err; + } + + return 0; +} + +int lwis_add_completion_fence(struct lwis_client *client, struct lwis_transaction *transaction) +{ + struct file *fp; + struct lwis_fence *lwis_fence; + struct lwis_fence_pending_signal *fence_pending_signal; + struct lwis_device *lwis_dev = client->lwis_dev; + int fence_fd = transaction->info.completion_fence_fd; + + /* If completion fence is not requested, we can safely return */ + if (fence_fd == LWIS_NO_COMPLETION_FENCE) { + return 0; + } + + /* If completion fence is requested but not initialized, we cannot continue */ + if (fence_fd == LWIS_CREATE_COMPLETION_FENCE) { + dev_err(lwis_dev->dev, + "Cannot add uninitialized completion fence to transaction\n"); + return -EPERM; + } + + fp = fget(fence_fd); + if (fp == NULL) { + dev_err(lwis_dev->dev, "Failed to find lwis_fence with fd %d\n", fence_fd); + return -EBADF; + } + + lwis_fence = fp->private_data; + fence_pending_signal = lwis_fence_pending_signal_create(lwis_fence, fp); + if (fence_pending_signal == NULL) { + return -ENOMEM; + } + list_add(&fence_pending_signal->node, &transaction->completion_fence_list); + if (lwis_fence_debug) { + dev_info(client->lwis_dev->dev, + "lwis_fence transaction id %llu add completion fence fd %d ", + transaction->info.id, lwis_fence->fd); + } + return 0; +} + +struct lwis_fence_pending_signal *lwis_fence_pending_signal_create(struct lwis_fence *fence, + struct file *fp) +{ + struct lwis_fence_pending_signal *pending_fence_signal = + kmalloc(sizeof(struct lwis_fence_pending_signal), GFP_ATOMIC); + if (!pending_fence_signal) { + dev_err(fence->lwis_top_dev->dev, + "Cannot allocate new fence pending signal list\n"); + return NULL; + } + pending_fence_signal->fp = fp; + pending_fence_signal->fence = fence; + pending_fence_signal->pending_status = LWIS_FENCE_STATUS_NOT_SIGNALED; + return pending_fence_signal; +} + +void lwis_fences_pending_signal_emit(struct lwis_device *lwis_device, + struct list_head *pending_fences) +{ + int ret; + struct lwis_fence_pending_signal *pending_fence; + struct list_head *it_fence, *it_fence_tmp; + + list_for_each_safe (it_fence, it_fence_tmp, pending_fences) { + pending_fence = list_entry(it_fence, struct lwis_fence_pending_signal, node); + ret = lwis_fence_signal(pending_fence->fence, pending_fence->pending_status); + if (ret) { + dev_err(lwis_device->dev, "Failed signaling fence with fd %d", + pending_fence->fence->fd); + } + list_del(&pending_fence->node); + fput(pending_fence->fp); + kfree(pending_fence); + } +} + +void lwis_pending_fences_move_all(struct lwis_device *lwis_device, + struct lwis_transaction *transaction, + struct list_head *pending_fences, int error_code) +{ + struct lwis_fence_pending_signal *pending_fence, *temp; + + /* For each fence in transaction's signal list, move to pending_fences for signaling */ + list_for_each_entry_safe (pending_fence, temp, &transaction->completion_fence_list, node) { + pending_fence->pending_status = error_code; + list_move_tail(&pending_fence->node, pending_fences); + } +} diff --git a/lwis_fence.h b/lwis_fence.h new file mode 100644 index 0000000..a79fc06 --- /dev/null +++ b/lwis_fence.h @@ -0,0 +1,105 @@ +/* + * Google LWIS Fence + * + * Copyright (c) 2022 Google, LLC + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef LWIS_FENCE_H_ +#define LWIS_FENCE_H_ + +#include <linux/hashtable.h> +#include <linux/list.h> + +#include "lwis_device.h" + +#define LWIS_CLIENTS_HASH_BITS 8 + +extern bool lwis_fence_debug; + +struct lwis_fence { + int fd; + int status; + spinlock_t lock; + /* Top device for printing logs */ + struct lwis_device *lwis_top_dev; + /* Status wait queue for waking up userspace */ + wait_queue_head_t status_wait_queue; + /* Hash table of transactions that's triggered by this fence */ + DECLARE_HASHTABLE(transaction_list, LWIS_CLIENTS_HASH_BITS); +}; + +struct lwis_fence_trigger_transaction_list { + struct lwis_client *owner; + struct list_head list; + struct hlist_node node; +}; + +struct lwis_fence_pending_signal { + struct file *fp; + struct lwis_fence *fence; + int pending_status; + struct list_head node; +}; + +/* + * lwis_fence_create: Create a new lwis_fence. + */ +int lwis_fence_create(struct lwis_device *lwis_dev); + +int ioctl_lwis_fence_create(struct lwis_device *lwis_dev, int32_t __user *msg); + +/* + * lwis_fence_get: Get the lwis_fence associated with the fd. + */ +struct lwis_device *lwis_fence_get(int fd); + +/* Creates all fences that do not currently exist */ +int lwis_initialize_transaction_fences(struct lwis_client *client, + struct lwis_transaction *transaction); + +bool lwis_triggered_by_condition(struct lwis_transaction *transaction); + +bool lwis_event_triggered_condition_ready(struct lwis_transaction *transaction, + struct lwis_transaction *weak_transaction, + int64_t event_id, int64_t event_counter); + +bool lwis_fence_triggered_condition_ready(struct lwis_transaction *transaction, int fence_status); + +/* + * lwis_parse_trigger_condition: Add the transaction to the associated trigger + * fence and event lists. + */ +int lwis_parse_trigger_condition(struct lwis_client *client, struct lwis_transaction *transaction); + +/* + * lwis_fence_signal: Signals the lwis_fence with the provided error code. + */ +int lwis_fence_signal(struct lwis_fence *lwis_fence, int status); + +/* + * lwis_add_completion_fence: Adds the fence with the given fd as a completion fence to this transaction. + */ +int lwis_add_completion_fence(struct lwis_client *client, struct lwis_transaction *transaction); + +/* lwis_fence_pending_signal_create: Creates and returns a lwis_fence_pending_signal list entry */ +struct lwis_fence_pending_signal *lwis_fence_pending_signal_create(struct lwis_fence *fence, + struct file *fp); + +/* + * lwis_fences_pending_signal_emit: Signal all lwis_fence_pending_signals in the pending_fences list + */ +void lwis_fences_pending_signal_emit(struct lwis_device *lwis_device, + struct list_head *pending_fences); + +/* + * lwis_pending_fences_move_all: Move all lwis_fence_pending_signal from the transaction to pending_fences. + */ +void lwis_pending_fences_move_all(struct lwis_device *lwis_device, + struct lwis_transaction *transaction, + struct list_head *pending_fences, int error_code); + +#endif /* LWIS_IOCTL_H_ */ diff --git a/lwis_gpio.c b/lwis_gpio.c index 0d68658..764b075 100644 --- a/lwis_gpio.c +++ b/lwis_gpio.c @@ -102,44 +102,6 @@ int lwis_gpio_list_set_input(struct gpio_descs *gpios) return 0; } -int lwis_gpio_list_to_irqs(struct lwis_device *lwis_dev, struct lwis_gpios_info *gpios_info, - char *irq_gpios_names) -{ - struct gpio_descs *gpios; - struct lwis_interrupt_list *irq_list; - int i; - int irq; - - if (!lwis_dev || !gpios_info) { - return -EINVAL; - } - gpios = gpios_info->gpios; - if (!gpios) { - return 0; - } - - irq_list = lwis_interrupt_list_alloc(lwis_dev, gpios->ndescs); - if (IS_ERR(irq_list)) { - pr_err("Failed to allocate irq list\n"); - return PTR_ERR(irq_list); - } - - for (i = 0; i < gpios->ndescs; ++i) { - char *name; - irq = gpiod_to_irq(gpios->desc[i]); - if (irq < 0) { - pr_err("gpio to irq failed (%d)\n", irq); - lwis_interrupt_list_free(irq_list); - return irq; - } - name = irq_gpios_names + i * LWIS_MAX_NAME_STRING_LEN; - lwis_interrupt_get_gpio_irq(irq_list, i, name, irq); - } - - gpios_info->irq_list = irq_list; - return 0; -} - struct lwis_gpios_list *lwis_gpios_list_alloc(int count) { struct lwis_gpios_list *list; diff --git a/lwis_gpio.h b/lwis_gpio.h index bbd77ff..52817e7 100644 --- a/lwis_gpio.h +++ b/lwis_gpio.h @@ -75,12 +75,6 @@ int lwis_gpio_list_set_output_value_raw(struct gpio_descs *gpios, int value); int lwis_gpio_list_set_input(struct gpio_descs *gpios); /* - * Get the IRQ list corresponding to the GPIO list - */ -int lwis_gpio_list_to_irqs(struct lwis_device *lwis_dev, struct lwis_gpios_info *gpios_info, - char *irq_gpios_names); - -/* * Allocate an instance of the lwis_gpios_info and initialize * the data structures according to the number of lwis_gpios_info * specified. @@ -11,6 +11,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME "-i2c: " fmt #include "lwis_i2c.h" +#include "lwis_trace.h" #include <linux/bits.h> #include <linux/kernel.h> @@ -65,7 +66,7 @@ static uint64_t buf_to_value(uint8_t *buf, int buf_size) } static int perform_read_transfer(struct i2c_client *client, struct i2c_msg *msg, uint64_t offset, - int offset_size_bytes) + int offset_size_bytes, struct lwis_device *lwis_dev) { int ret = 0; u8 *wbuf = msg[0].buf; @@ -73,12 +74,15 @@ static int perform_read_transfer(struct i2c_client *client, struct i2c_msg *msg, const int num_msg = 2; value_to_buf(offset, wbuf, offset_size_bytes); + LWIS_ATRACE_FUNC_BEGIN(lwis_dev, "i2c_read"); ret = i2c_transfer(client->adapter, msg, num_msg); + LWIS_ATRACE_FUNC_END(lwis_dev, "i2c_read"); return (ret == num_msg) ? 0 : ret; } static int perform_write_transfer(struct i2c_client *client, struct i2c_msg *msg, uint64_t offset, - int offset_size_bytes, int value_size_bytes, uint64_t value) + int offset_size_bytes, int value_size_bytes, uint64_t value, + struct lwis_device *lwis_dev) { int ret = 0; u8 *buf = msg->buf; @@ -87,14 +91,16 @@ static int perform_write_transfer(struct i2c_client *client, struct i2c_msg *msg value_to_buf(offset, buf, offset_size_bytes); value_to_buf(value, buf + offset_size_bytes, value_size_bytes); - + LWIS_ATRACE_FUNC_BEGIN(lwis_dev, "i2c_write"); ret = i2c_transfer(client->adapter, msg, num_msg); + LWIS_ATRACE_FUNC_END(lwis_dev, "i2c_write"); return (ret == num_msg) ? 0 : ret; } static int perform_write_batch_transfer(struct i2c_client *client, struct i2c_msg *msg, uint64_t offset, int offset_size_bytes, - int value_size_bytes, uint8_t *value_buf) + int value_size_bytes, uint8_t *value_buf, + struct lwis_device *lwis_dev) { int ret = 0; u8 *buf = msg->buf; @@ -104,7 +110,9 @@ static int perform_write_batch_transfer(struct i2c_client *client, struct i2c_ms value_to_buf(offset, buf, offset_size_bytes); memcpy(buf + offset_size_bytes, value_buf, value_size_bytes); + LWIS_ATRACE_FUNC_BEGIN(lwis_dev, "i2c_write_batch"); ret = i2c_transfer(client->adapter, msg, num_msg); + LWIS_ATRACE_FUNC_END(lwis_dev, "i2c_write_batch"); return (ret == num_msg) ? 0 : ret; } @@ -122,7 +130,7 @@ int lwis_i2c_set_state(struct lwis_i2c_device *i2c, const char *state_str) state_to_set = i2c->pinctrl_default_state_only ? "default" : state_str; state = pinctrl_lookup_state(i2c->state_pinctrl, state_to_set); - if (IS_ERR(state)) { + if (IS_ERR_OR_NULL(state)) { dev_err(i2c->base_dev.dev, "State %s not found (%ld)\n", state_str, PTR_ERR(state)); return PTR_ERR(state); } @@ -136,7 +144,8 @@ int lwis_i2c_set_state(struct lwis_i2c_device *i2c, const char *state_str) return 0; } -static int i2c_read(struct lwis_i2c_device *i2c, uint64_t offset, uint64_t *value) +static int i2c_read(struct lwis_i2c_device *i2c, uint64_t offset, uint64_t *value, + struct lwis_device *lwis_dev) { int ret = 0; u8 *wbuf; @@ -191,7 +200,7 @@ static int i2c_read(struct lwis_i2c_device *i2c, uint64_t offset, uint64_t *valu msg[1].len = value_bytes; msg[1].buf = rbuf; - ret = perform_read_transfer(client, msg, offset, offset_bytes); + ret = perform_read_transfer(client, msg, offset, offset_bytes, lwis_dev); if (ret) { dev_err(i2c->base_dev.dev, "I2C Read failed: Offset 0x%llx (%d)\n", offset, ret); @@ -208,7 +217,8 @@ error_rbuf_alloc: return ret; } -static int i2c_write(struct lwis_i2c_device *i2c, uint64_t offset, uint64_t value) +static int i2c_write(struct lwis_i2c_device *i2c, uint64_t offset, uint64_t value, + struct lwis_device *lwis_dev) { int ret; u8 *buf; @@ -257,7 +267,8 @@ static int i2c_write(struct lwis_i2c_device *i2c, uint64_t offset, uint64_t valu msg.buf = buf; msg.len = msg_bytes; - ret = perform_write_transfer(client, &msg, offset, offset_bytes, value_bytes, value); + ret = perform_write_transfer(client, &msg, offset, offset_bytes, value_bytes, value, + lwis_dev); if (ret) { dev_err(i2c->base_dev.dev, "I2C Write failed: Offset 0x%llx Value 0x%llx (%d)\n", @@ -270,7 +281,7 @@ static int i2c_write(struct lwis_i2c_device *i2c, uint64_t offset, uint64_t valu } static int i2c_read_batch(struct lwis_i2c_device *i2c, uint64_t start_offset, uint8_t *read_buf, - int read_buf_size) + int read_buf_size, struct lwis_device *lwis_dev) { int ret = 0; uint8_t *wbuf; @@ -308,7 +319,7 @@ static int i2c_read_batch(struct lwis_i2c_device *i2c, uint64_t start_offset, ui msg[1].len = read_buf_size; msg[1].buf = read_buf; - ret = perform_read_transfer(client, msg, start_offset, offset_bytes); + ret = perform_read_transfer(client, msg, start_offset, offset_bytes, lwis_dev); if (ret) { dev_err(i2c->base_dev.dev, "I2C Read Batch failed: Start Offset 0x%llx (%d)\n", @@ -320,7 +331,7 @@ static int i2c_read_batch(struct lwis_i2c_device *i2c, uint64_t start_offset, ui } static int i2c_write_batch(struct lwis_i2c_device *i2c, uint64_t start_offset, uint8_t *write_buf, - int write_buf_size) + int write_buf_size, struct lwis_device *lwis_dev) { int ret; uint8_t *buf; @@ -361,7 +372,7 @@ static int i2c_write_batch(struct lwis_i2c_device *i2c, uint64_t start_offset, u msg.len = msg_bytes; ret = perform_write_batch_transfer(client, &msg, start_offset, offset_bytes, write_buf_size, - write_buf); + write_buf, lwis_dev); if (ret) { dev_err(i2c->base_dev.dev, "I2C Write Batch failed: Start Offset 0x%llx (%d)\n", @@ -373,7 +384,8 @@ static int i2c_write_batch(struct lwis_i2c_device *i2c, uint64_t start_offset, u return ret; } -int lwis_i2c_io_entry_rw(struct lwis_i2c_device *i2c, struct lwis_io_entry *entry) +int lwis_i2c_io_entry_rw(struct lwis_i2c_device *i2c, struct lwis_io_entry *entry, + struct lwis_device *lwis_dev) { int ret; uint64_t reg_value; @@ -384,27 +396,27 @@ int lwis_i2c_io_entry_rw(struct lwis_i2c_device *i2c, struct lwis_io_entry *entr } if (entry->type == LWIS_IO_ENTRY_READ) { - return i2c_read(i2c, entry->rw.offset, &entry->rw.val); + return i2c_read(i2c, entry->rw.offset, &entry->rw.val, lwis_dev); } if (entry->type == LWIS_IO_ENTRY_WRITE) { - return i2c_write(i2c, entry->rw.offset, entry->rw.val); + return i2c_write(i2c, entry->rw.offset, entry->rw.val, lwis_dev); } if (entry->type == LWIS_IO_ENTRY_MODIFY) { - ret = i2c_read(i2c, entry->mod.offset, ®_value); + ret = i2c_read(i2c, entry->mod.offset, ®_value, lwis_dev); if (ret) { return ret; } reg_value &= ~entry->mod.val_mask; reg_value |= entry->mod.val_mask & entry->mod.val; - return i2c_write(i2c, entry->mod.offset, reg_value); + return i2c_write(i2c, entry->mod.offset, reg_value, lwis_dev); } if (entry->type == LWIS_IO_ENTRY_READ_BATCH) { return i2c_read_batch(i2c, entry->rw_batch.offset, entry->rw_batch.buf, - entry->rw_batch.size_in_bytes); + entry->rw_batch.size_in_bytes, lwis_dev); } if (entry->type == LWIS_IO_ENTRY_WRITE_BATCH) { return i2c_write_batch(i2c, entry->rw_batch.offset, entry->rw_batch.buf, - entry->rw_batch.size_in_bytes); + entry->rw_batch.size_in_bytes, lwis_dev); } dev_err(i2c->base_dev.dev, "Invalid IO entry type: %d\n", entry->type); return -EINVAL; @@ -33,6 +33,7 @@ int lwis_i2c_set_state(struct lwis_i2c_device *i2c, const char *state_str); * lwis_i2c_io_entry_rw: Read/Write from i2c bus via io_entry request. * The readback values will be stored in the entry. */ -int lwis_i2c_io_entry_rw(struct lwis_i2c_device *i2c, struct lwis_io_entry *entry); +int lwis_i2c_io_entry_rw(struct lwis_i2c_device *i2c, struct lwis_io_entry *entry, + struct lwis_device *lwis_dev); #endif /* LWIS_I2C_H_ */ diff --git a/lwis_init.h b/lwis_init.h index 673ac8c..b84caf8 100644 --- a/lwis_init.h +++ b/lwis_init.h @@ -26,4 +26,7 @@ int lwis_slc_device_init(void); /* lwis_device_dpm.c */ int lwis_dpm_device_init(void); +/* lwis_device_test.c */ +int lwis_test_device_init(void); + #endif // LWIS_INIT_H_ diff --git a/lwis_interrupt.c b/lwis_interrupt.c index f01a353..62dd3ad 100644 --- a/lwis_interrupt.c +++ b/lwis_interrupt.c @@ -36,7 +36,8 @@ struct lwis_single_event_info { struct list_head node_enabled; }; -static irqreturn_t lwis_interrupt_event_isr(int irq_number, void *data); +static irqreturn_t lwis_interrupt_regular_isr(int irq_number, void *data); +static irqreturn_t lwis_interrupt_aggregate_isr(int irq_number, void *data); static irqreturn_t lwis_interrupt_gpios_event_isr(int irq_number, void *data); struct lwis_interrupt_list *lwis_interrupt_list_alloc(struct lwis_device *lwis_dev, int count) @@ -67,6 +68,24 @@ struct lwis_interrupt_list *lwis_interrupt_list_alloc(struct lwis_device *lwis_d return list; } +void lwis_interrupt_free_leaves(struct lwis_interrupt *irq) +{ + struct lwis_interrupt_leaf_node *leaf_node; + struct list_head *it_leaf, *it_tmp; + + if (!irq || irq->irq_type != AGGREGATE_INTERRUPT || list_empty(&irq->leaf_nodes)) { + // Nothing to clean + return; + } + + list_for_each_safe (it_leaf, it_tmp, &irq->leaf_nodes) { + leaf_node = list_entry(it_leaf, struct lwis_interrupt_leaf_node, node); + list_del(&leaf_node->node); + kfree(leaf_node->leaf_irq_indexes); + kfree(leaf_node); + } +} + void lwis_interrupt_list_free(struct lwis_interrupt_list *list) { int i; @@ -80,53 +99,67 @@ void lwis_interrupt_list_free(struct lwis_interrupt_list *list) } for (i = 0; i < list->count; ++i) { + lwis_interrupt_free_leaves(&list->irq[i]); free_irq(list->irq[i].irq, &list->irq[i]); } kfree(list->irq); } -int lwis_interrupt_get(struct lwis_interrupt_list *list, int index, char *name, - struct platform_device *plat_dev) +int lwis_interrupt_init(struct lwis_interrupt_list *list, int index, char *name) { - int irq; - int ret = 0; - if (!list || index < 0 || index >= list->count) { return -EINVAL; } - irq = platform_get_irq(plat_dev, index); - if (irq <= 0) { - pr_err("Error retriving interrupt %s at %d\n", name, index); - return -EINVAL; - } - /* Initialize the spinlock */ spin_lock_init(&list->irq[index].lock); - list->irq[index].irq = irq; - strlcpy(list->irq[index].name, name, IRQ_FULL_NAME_LENGTH); + strscpy(list->irq[index].name, name, IRQ_FULL_NAME_LENGTH); snprintf(list->irq[index].full_name, IRQ_FULL_NAME_LENGTH, "lwis-%s:%s", list->lwis_dev->name, name); list->irq[index].has_events = false; list->irq[index].lwis_dev = list->lwis_dev; + return 0; +} - ret = request_irq(irq, lwis_interrupt_event_isr, IRQF_SHARED, list->irq[index].full_name, - &list->irq[index]); +int lwis_interrupt_get(struct lwis_interrupt_list *list, int index, + struct platform_device *plat_dev) +{ + int irq; + int ret = 0; + unsigned long flags; + + irq = platform_get_irq(plat_dev, index); + if (irq <= 0) { + pr_err("Error retrieving interrupt %s at %d\n", list->irq[index].full_name, index); + return -EINVAL; + } + + if (list->irq[index].irq_type == AGGREGATE_INTERRUPT) { + ret = request_irq(irq, lwis_interrupt_aggregate_isr, IRQF_SHARED, + list->irq[index].full_name, &list->irq[index]); + } else if (list->irq[index].irq_type == REGULAR_INTERRUPT) { + ret = request_irq(irq, lwis_interrupt_regular_isr, IRQF_SHARED, + list->irq[index].full_name, &list->irq[index]); + } if (ret) { dev_err(list->lwis_dev->dev, "Failed to request IRQ %d\n", irq); return ret; } - if (lwis_plaform_set_default_irq_affinity(list->irq[index].irq) != 0) { + if (lwis_plaform_set_default_irq_affinity(irq) != 0) { dev_warn(list->lwis_dev->dev, "Interrupt %s cannot set affinity.\n", list->irq[index].full_name); } + spin_lock_irqsave(&list->irq[index].lock, flags); + list->irq[index].irq = irq; + spin_unlock_irqrestore(&list->irq[index].lock, flags); + return 0; } int lwis_interrupt_get_gpio_irq(struct lwis_interrupt_list *list, int index, char *name, - int gpio_irq) + int gpio_irq, int32_t irq_gpios_types) { int ret = 0; @@ -137,14 +170,17 @@ int lwis_interrupt_get_gpio_irq(struct lwis_interrupt_list *list, int index, cha /* Initialize the spinlock */ spin_lock_init(&list->irq[index].lock); list->irq[index].irq = gpio_irq; - strlcpy(list->irq[index].name, name, IRQ_FULL_NAME_LENGTH); + strscpy(list->irq[index].name, name, IRQ_FULL_NAME_LENGTH); snprintf(list->irq[index].full_name, IRQ_FULL_NAME_LENGTH, "lwis-%s:%s", list->lwis_dev->name, name); list->irq[index].has_events = false; list->irq[index].lwis_dev = list->lwis_dev; + list->irq[index].irq_gpios_types = irq_gpios_types; + list->irq[index].irq_type = GPIO_HW_INTERRUPT; - ret = request_irq(gpio_irq, lwis_interrupt_gpios_event_isr, IRQF_SHARED, - list->irq[index].full_name, &list->irq[index]); + ret = request_irq(gpio_irq, lwis_interrupt_gpios_event_isr, + list->irq[index].irq_gpios_types, list->irq[index].full_name, + &list->irq[index]); if (ret) { dev_err(list->lwis_dev->dev, "Failed to request GPIO IRQ\n"); return ret; @@ -215,53 +251,90 @@ static int lwis_interrupt_set_mask(struct lwis_interrupt *irq, int int_reg_bit, return ret; } -static irqreturn_t lwis_interrupt_event_isr(int irq_number, void *data) +static int lwis_interrupt_read_and_clear_src_reg(struct lwis_interrupt *irq, uint64_t *source_value, + uint64_t *overflow_value) { int ret; - struct lwis_interrupt *irq = (struct lwis_interrupt *)data; - struct lwis_client_event_state *event_state; - struct lwis_single_event_info *event; - struct list_head *p; - uint64_t source_value, reset_value = 0; - struct lwis_client *lwis_client; - struct list_head *t, *n; -#ifdef LWIS_INTERRUPT_DEBUG - uint64_t mask_value; -#endif - unsigned long flags; - /* Read the mask register */ + /* Read IRQ status register */ ret = lwis_device_single_register_read(irq->lwis_dev, irq->irq_reg_bid, irq->irq_src_reg, - &source_value, irq->irq_reg_access_size); + source_value, irq->irq_reg_access_size); if (ret) { dev_err(irq->lwis_dev->dev, "%s: Failed to read IRQ status register: %d\n", irq->name, ret); - goto error; + return ret; } /* Write back to the reset register */ ret = lwis_device_single_register_write(irq->lwis_dev, irq->irq_reg_bid, irq->irq_reset_reg, - source_value, irq->irq_reg_access_size); + *source_value, irq->irq_reg_access_size); if (ret) { dev_err(irq->lwis_dev->dev, "%s: Failed to write IRQ reset register: %d\n", irq->name, ret); - goto error; + return ret; } - /* Nothing is triggered, just return */ - if (source_value == 0) { - return IRQ_HANDLED; + if (irq->irq_overflow_reg) { + /* Read the overflow register */ + ret = lwis_device_single_register_read(irq->lwis_dev, irq->irq_reg_bid, + irq->irq_overflow_reg, overflow_value, + irq->irq_reg_access_size); + if (ret) { + dev_err(irq->lwis_dev->dev, + "%s: Failed to read IRQ overflow register: %d\n", irq->name, ret); + return ret; + } + + /* Overflow is triggered */ + if (*overflow_value != 0) { + dev_warn(irq->lwis_dev->dev, + "IRQ(%s) overflow register(0x%llx) value(%lld) is detected\n", + irq->name, irq->irq_overflow_reg, *overflow_value); + /* Write back to the overflow register */ + ret = lwis_device_single_register_write(irq->lwis_dev, irq->irq_reg_bid, + irq->irq_overflow_reg, + *overflow_value, + irq->irq_reg_access_size); + if (ret) { + dev_err(irq->lwis_dev->dev, + "%s: Failed to write IRQ overflow register: %d\n", + irq->name, ret); + return ret; + } + } } + return 0; +} + +static void lwis_interrupt_emit_events(struct lwis_interrupt *irq, uint64_t source_value, + uint64_t overflow_value) +{ + struct lwis_client_event_state *event_state; + struct lwis_single_event_info *event; + struct list_head *p; + uint64_t reset_value = 0; + struct lwis_client *lwis_client; + struct list_head *t, *n; +#ifdef LWIS_INTERRUPT_DEBUG + uint64_t mask_value; +#endif + unsigned long flags; + spin_lock_irqsave(&irq->lock, flags); list_for_each (p, &irq->enabled_event_infos) { event = list_entry(p, struct lwis_single_event_info, node_enabled); /* Check if this event needs to be emitted */ if ((source_value >> event->int_reg_bit) & 0x1) { - /* Emit the event */ - lwis_device_event_emit(irq->lwis_dev, event->event_id, NULL, 0, - /*in_irq=*/true); + lwis_device_event_emit(irq->lwis_dev, event->event_id, NULL, 0); + /* Check if this overflow event needs to combine event id + overflow flag */ + if ((overflow_value >> event->int_reg_bit) & 0x1) { + lwis_device_event_emit( + irq->lwis_dev, + event->event_id | LWIS_OVERFLOW_IRQ_EVENT_FLAG, NULL, 0); + } + /* Clear this interrupt */ reset_value |= (1ULL << event->int_reg_bit); @@ -280,7 +353,8 @@ static irqreturn_t lwis_interrupt_event_isr(int irq_number, void *data) event->event_id && event_state->event_control.flags & LWIS_EVENT_CONTROL_FLAG_IRQ_ENABLE_ONCE) { - dev_err(irq->lwis_dev->dev, + dev_err_ratelimited( + irq->lwis_dev->dev, "IRQ(%s) event(0x%llx) enabled once\n", irq->name, event->event_id); lwis_interrupt_set_mask(irq, event->int_reg_bit, @@ -319,6 +393,112 @@ static irqreturn_t lwis_interrupt_event_isr(int irq_number, void *data) } } #endif +} + +static irqreturn_t lwis_interrupt_regular_isr(int irq_number, void *data) +{ + int ret; + struct lwis_interrupt *irq = (struct lwis_interrupt *)data; + uint64_t source_value, overflow_value; + + ret = lwis_interrupt_read_and_clear_src_reg(irq, &source_value, &overflow_value); + if (ret) { + goto error; + } + + /* Nothing is triggered, just return */ + if (source_value == 0) { + return IRQ_HANDLED; + } + + lwis_interrupt_emit_events(irq, source_value, overflow_value); +error: + return IRQ_HANDLED; +} + +int lwis_fake_event_inject(void *data) +{ + struct lwis_interrupt *irq = (struct lwis_interrupt *)data; + uint64_t source_value = 0x00000020ll, overflow_value = 0; + + lwis_interrupt_emit_events(irq, source_value, overflow_value); + + return irq->irq; +} + +static int lwis_interrupt_handle_aggregation(struct lwis_interrupt *irq, uint64_t source_value) +{ + struct lwis_interrupt_leaf_node *leaf; + struct lwis_interrupt *leaf_irq = NULL; + int leaf_irq_index = 0; + struct list_head *p; + uint64_t reset_value = 0; + struct lwis_device *lwis_dev = irq->lwis_dev; + int i; + + list_for_each (p, &irq->leaf_nodes) { + leaf = list_entry(p, struct lwis_interrupt_leaf_node, node); + /* Check if this leaf has signal */ + if ((source_value >> leaf->int_reg_bit) & 0x1) { + for (i = 0; i < leaf->count; ++i) { + leaf_irq_index = leaf->leaf_irq_indexes[i]; + if (leaf_irq_index < 0 || leaf_irq_index >= lwis_dev->irqs->count) { + dev_err(lwis_dev->dev, + "%s: Contain invalid leaf irq index: %d\n", + irq->name, leaf_irq_index); + return -EINVAL; + } + + leaf_irq = &lwis_dev->irqs->irq[leaf_irq_index]; + if (leaf_irq->irq_type != LEAF_INTERRUPT) { + dev_err(lwis_dev->dev, + "%s: Contain leaf irq %s type is: %d, which is not LEAF\n", + irq->name, leaf_irq->name, leaf_irq->irq_type); + return -EINVAL; + } + + /* Call the leaf-level handler if there's any event enabled */ + if (!list_empty(&leaf_irq->enabled_event_infos)) { + lwis_interrupt_regular_isr(leaf_irq->irq, leaf_irq); + } + } + /* Clear this leaf */ + reset_value |= (1ULL << leaf->int_reg_bit); + } + + /* All leaves are handled */ + if (source_value == reset_value) { + break; + } + } + return 0; +} + +static irqreturn_t lwis_interrupt_aggregate_isr(int irq_number, void *data) +{ + int ret; + struct lwis_interrupt *irq = (struct lwis_interrupt *)data; + uint64_t source_value, overflow_value; + + ret = lwis_interrupt_read_and_clear_src_reg(irq, &source_value, &overflow_value); + if (ret) { + goto error; + } + + /* Nothing is triggered, just return */ + if (source_value == 0) { + return IRQ_HANDLED; + } + + /* Handle leaf interrupt */ + ret = lwis_interrupt_handle_aggregation(irq, source_value); + if (ret) { + dev_warn(irq->lwis_dev->dev, "Aggregate IRQ(%s) fail to handle leaf nodes\n", + irq->name); + goto error; + } + + lwis_interrupt_emit_events(irq, source_value, overflow_value); error: return IRQ_HANDLED; } @@ -334,29 +514,20 @@ static irqreturn_t lwis_interrupt_gpios_event_isr(int irq_number, void *data) list_for_each (p, &irq->enabled_event_infos) { event = list_entry(p, struct lwis_single_event_info, node_enabled); /* Emit the event */ - lwis_device_event_emit(irq->lwis_dev, event->event_id, NULL, 0, /*in_irq=*/true); + lwis_device_event_emit(irq->lwis_dev, event->event_id, NULL, 0); } spin_unlock_irqrestore(&irq->lock, flags); return IRQ_HANDLED; } -int lwis_interrupt_set_event_info(struct lwis_interrupt_list *list, int index, - const char *irq_reg_space, int irq_reg_bid, int64_t *irq_events, - size_t irq_events_num, uint32_t *int_reg_bits, - size_t int_reg_bits_num, int64_t irq_src_reg, - int64_t irq_reset_reg, int64_t irq_mask_reg, bool mask_toggled, - int irq_reg_access_size, int64_t *critical_events, - size_t critical_events_num) +void lwis_interrupt_set_basic_info(struct lwis_interrupt_list *list, int index, + const char *irq_reg_space, int irq_reg_bid, int64_t irq_src_reg, + int64_t irq_reset_reg, int64_t irq_mask_reg, + int64_t irq_overflow_reg, bool mask_toggled, + int irq_reg_access_size, int32_t irq_type) { - int i, j; unsigned long flags; - bool is_critical = false; - - if (int_reg_bits_num != irq_events_num) { - pr_err("reg bits num != irq event num.\n"); - return -EINVAL; - } /* Protect the structure */ spin_lock_irqsave(&list->irq[index].lock, flags); @@ -365,13 +536,32 @@ int lwis_interrupt_set_event_info(struct lwis_interrupt_list *list, int index, list->irq[index].irq_src_reg = irq_src_reg; list->irq[index].irq_reset_reg = irq_reset_reg; list->irq[index].irq_mask_reg = irq_mask_reg; + list->irq[index].irq_overflow_reg = irq_overflow_reg; list->irq[index].mask_toggled = mask_toggled; list->irq[index].irq_reg_access_size = irq_reg_access_size; + list->irq[index].irq_type = irq_type; /* Empty hash table for event infos */ hash_init(list->irq[index].event_infos); /* Initialize an empty list for enabled events */ INIT_LIST_HEAD(&list->irq[index].enabled_event_infos); + /* Initialize an empty list for leaf nodes */ + INIT_LIST_HEAD(&list->irq[index].leaf_nodes); spin_unlock_irqrestore(&list->irq[index].lock, flags); +} + +int lwis_interrupt_set_event_info(struct lwis_interrupt_list *list, int index, int64_t *irq_events, + size_t irq_events_num, uint32_t *int_reg_bits, + size_t int_reg_bits_num, int64_t *critical_events, + size_t critical_events_num) +{ + int i, j; + unsigned long flags; + bool is_critical = false; + + if (int_reg_bits_num != irq_events_num) { + pr_err("reg bits num != irq event num.\n"); + return -EINVAL; + } /* Build the hash table of events we can emit */ for (i = 0; i < irq_events_num; i++) { @@ -425,6 +615,26 @@ int lwis_interrupt_set_event_info(struct lwis_interrupt_list *list, int index, return 0; } +int lwis_interrupt_add_leaf(struct lwis_interrupt_list *list, int index, uint32_t int_reg_bit, + int count, int32_t *leaf_indexes) +{ + struct lwis_interrupt_leaf_node *new_leaf_node; + unsigned long flags; + + new_leaf_node = kmalloc(sizeof(struct lwis_interrupt_leaf_node), GFP_KERNEL); + if (IS_ERR_OR_NULL(new_leaf_node)) { + return -ENOMEM; + } + + new_leaf_node->int_reg_bit = int_reg_bit; + new_leaf_node->count = count; + new_leaf_node->leaf_irq_indexes = leaf_indexes; + spin_lock_irqsave(&list->irq[index].lock, flags); + list_add(&new_leaf_node->node, &list->irq[index].leaf_nodes); + spin_unlock_irqrestore(&list->irq[index].lock, flags); + return 0; +} + int lwis_interrupt_set_gpios_event_info(struct lwis_interrupt_list *list, int index, int64_t irq_event) { @@ -503,7 +713,10 @@ static int lwis_interrupt_single_event_enable_locked(struct lwis_interrupt *irq, /* If mask_toggled is set, reverse the enable/disable logic. */ is_set = (!irq->mask_toggled) ? enabled : !enabled; - ret = lwis_interrupt_set_mask(irq, event->int_reg_bit, is_set); + /* GPIO HW interrupt doesn't support to set interrupt mask */ + if (irq->irq_type != GPIO_HW_INTERRUPT) { + ret = lwis_interrupt_set_mask(irq, event->int_reg_bit, is_set); + } return ret; } diff --git a/lwis_interrupt.h b/lwis_interrupt.h index 3824e8b..d6106bc 100644 --- a/lwis_interrupt.h +++ b/lwis_interrupt.h @@ -18,6 +18,26 @@ #define EVENT_INFO_HASH_BITS 8 #define IRQ_FULL_NAME_LENGTH 32 +#define LEAF_NODE_HASH_BITS 8 + +enum lwis_interrupt_types { + REGULAR_INTERRUPT, + AGGREGATE_INTERRUPT, + LEAF_INTERRUPT, + GPIO_HW_INTERRUPT, + FAKEEVENT_INTERRUPT +}; + +/* + * struct lwis_interrupt_leaf_node + * This is to represent aggregate interrupt's leaf node + */ +struct lwis_interrupt_leaf_node { + uint32_t int_reg_bit; + int count; + int32_t *leaf_irq_indexes; + struct list_head node; +}; struct lwis_interrupt { int irq; @@ -40,17 +60,26 @@ struct lwis_interrupt { int64_t irq_reset_reg; /* Offset of the mask register */ int64_t irq_mask_reg; + /* Offset of the overflow register */ + int64_t irq_overflow_reg; /* IRQ register access size, in case it is different from the bus * bitwidth */ int irq_reg_access_size; /* If mask_reg actually disable the interrupts. */ bool mask_toggled; + /* Type of the interrupt */ + int32_t irq_type; /* Hash table of event info */ /* GUARDED_BY(lock) */ DECLARE_HASHTABLE(event_infos, EVENT_INFO_HASH_BITS); /* List of enabled events */ /* GUARDED_BY(lock) */ struct list_head enabled_event_infos; + /* Select the interrupt line behavior */ + int irq_gpios_types; + /* List of aggregate interrupt leaf nodes */ + /* GUARDED_BY(lock) */ + struct list_head leaf_nodes; }; /* @@ -72,15 +101,25 @@ struct lwis_interrupt_list { struct lwis_interrupt_list *lwis_interrupt_list_alloc(struct lwis_device *lwis_dev, int count); /* + * lwis_interrupt_free_leaves: Deallocate the leaf_nodes in lwis_interrupt structure. + */ +void lwis_interrupt_free_leaves(struct lwis_interrupt *irq); + +/* * lwis_interrupt_list_free: Deallocate the lwis_interrupt_list structure. */ void lwis_interrupt_list_free(struct lwis_interrupt_list *list); /* + * lwis_interrupt_init: Initialize the interrupt by index. + */ +int lwis_interrupt_init(struct lwis_interrupt_list *list, int index, char *name); + +/* * lwis_interrupt_get: Register the interrupt by index. * Returns: 0 if success, -ve if error */ -int lwis_interrupt_get(struct lwis_interrupt_list *list, int index, char *name, +int lwis_interrupt_get(struct lwis_interrupt_list *list, int index, struct platform_device *plat_dev); /* @@ -88,7 +127,17 @@ int lwis_interrupt_get(struct lwis_interrupt_list *list, int index, char *name, * Returns: 0 if success, -ve if error */ int lwis_interrupt_get_gpio_irq(struct lwis_interrupt_list *list, int index, char *name, - int gpio_irq); + int gpio_irq, int32_t irq_gpios_types); + +/* + * lwis_interrupt_set_basic_info: Provides basic register info for a given + * interrupt based on index + */ +void lwis_interrupt_set_basic_info(struct lwis_interrupt_list *list, int index, + const char *irq_reg_space, int irq_reg_bid, int64_t irq_src_reg, + int64_t irq_reset_reg, int64_t irq_mask_reg, + int64_t irq_overflow_reg, bool mask_toggled, + int irq_reg_access_size, int32_t irq_type); /* * lwis_interrupt_set_event_info: Provides event-info structure for a given @@ -98,15 +147,20 @@ int lwis_interrupt_get_gpio_irq(struct lwis_interrupt_list *list, int index, cha * Does not free irq_reg_space * Returns: 0 on success */ -int lwis_interrupt_set_event_info(struct lwis_interrupt_list *list, int index, - const char *irq_reg_space, int irq_reg_bid, int64_t *irq_events, +int lwis_interrupt_set_event_info(struct lwis_interrupt_list *list, int index, int64_t *irq_events, size_t irq_events_num, uint32_t *int_reg_bits, - size_t int_reg_bits_num, int64_t irq_src_reg, - int64_t irq_reset_reg, int64_t irq_mask_reg, bool mask_toggled, - int irq_reg_access_size, int64_t *critical_events, + size_t int_reg_bits_num, int64_t *critical_events, size_t critical_events_num); /* + * lwis_interrupt_add_leaf: Provides one leaf node information for the aggregate interrupt + * + * Returns: 0 on success + */ +int lwis_interrupt_add_leaf(struct lwis_interrupt_list *list, int index, uint32_t int_reg_bit, + int count, int32_t *leaf_indexes); + +/* * lwis_interrupt_set_gpios_event_info: Provides event-info structure for a given * interrupt based on index * @@ -132,4 +186,6 @@ int lwis_interrupt_event_enable(struct lwis_interrupt_list *list, int64_t event_ */ void lwis_interrupt_print(struct lwis_interrupt_list *list); +int lwis_fake_event_inject(void *data); + #endif /* LWIS_INTERRUPT_H_ */ diff --git a/lwis_io_entry.c b/lwis_io_entry.c index ab7d5bb..df50f96 100644 --- a/lwis_io_entry.c +++ b/lwis_io_entry.c @@ -11,18 +11,19 @@ #define pr_fmt(fmt) KBUILD_MODNAME "-ioentry: " fmt #include <linux/delay.h> +#include <linux/preempt.h> #include "lwis_io_entry.h" #include "lwis_util.h" -int lwis_io_entry_poll(struct lwis_device *lwis_dev, struct lwis_io_entry *entry, bool non_blocking) +int lwis_io_entry_poll(struct lwis_device *lwis_dev, struct lwis_io_entry *entry) { uint64_t val, start; uint64_t timeout_ms = entry->read_assert.timeout_ms; int ret = 0; - /* Only read and check once if non_blocking */ - if (non_blocking) { + /* Only read and check once if in_irq() */ + if (in_irq()) { timeout_ms = 0; } diff --git a/lwis_io_entry.h b/lwis_io_entry.h index a2679da..ae1b2cc 100644 --- a/lwis_io_entry.h +++ b/lwis_io_entry.h @@ -18,8 +18,7 @@ * lwis_io_entry_poll: * Polls a register for a specified time or until it reaches the expected value. */ -int lwis_io_entry_poll(struct lwis_device *lwis_dev, struct lwis_io_entry *entry, - bool non_blocking); +int lwis_io_entry_poll(struct lwis_device *lwis_dev, struct lwis_io_entry *entry); /* * lwis_io_entry_read_assert: diff --git a/lwis_ioctl.c b/lwis_ioctl.c index 003010a..221dd98 100644 --- a/lwis_ioctl.c +++ b/lwis_ioctl.c @@ -24,7 +24,9 @@ #include "lwis_device_dpm.h" #include "lwis_device_i2c.h" #include "lwis_device_ioreg.h" +#include "lwis_device_test.h" #include "lwis_event.h" +#include "lwis_fence.h" #include "lwis_i2c.h" #include "lwis_io_entry.h" #include "lwis_ioreg.h" @@ -45,100 +47,12 @@ static void lwis_ioctl_pr_err(struct lwis_device *lwis_dev, unsigned int ioctl_t size_t exp_size; switch (type) { - case IOCTL_TO_ENUM(LWIS_GET_DEVICE_INFO): - strlcpy(type_name, STRINGIFY(LWIS_GET_DEVICE_INFO), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_GET_DEVICE_INFO); - break; - case IOCTL_TO_ENUM(LWIS_BUFFER_ALLOC): - strlcpy(type_name, STRINGIFY(LWIS_BUFFER_ALLOC), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_BUFFER_ALLOC); - break; - case IOCTL_TO_ENUM(LWIS_BUFFER_FREE): - strlcpy(type_name, STRINGIFY(LWIS_BUFFER_FREE), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_BUFFER_FREE); - break; - case IOCTL_TO_ENUM(LWIS_BUFFER_ENROLL): - strlcpy(type_name, STRINGIFY(LWIS_BUFFER_ENROLL), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_BUFFER_ENROLL); - break; - case IOCTL_TO_ENUM(LWIS_BUFFER_DISENROLL): - strlcpy(type_name, STRINGIFY(LWIS_BUFFER_DISENROLL), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_BUFFER_DISENROLL); - break; - case IOCTL_TO_ENUM(LWIS_BUFFER_CPU_ACCESS): - strlcpy(type_name, STRINGIFY(LWIS_BUFFER_CPU_ACCESS), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_BUFFER_CPU_ACCESS); - break; - case IOCTL_TO_ENUM(LWIS_REG_IO): - strlcpy(type_name, STRINGIFY(LWIS_REG_IO), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_REG_IO); - break; - case IOCTL_TO_ENUM(LWIS_DEVICE_ENABLE): - strlcpy(type_name, STRINGIFY(LWIS_DEVICE_ENABLE), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_DEVICE_ENABLE); - break; - case IOCTL_TO_ENUM(LWIS_DEVICE_DISABLE): - strlcpy(type_name, STRINGIFY(LWIS_DEVICE_DISABLE), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_DEVICE_DISABLE); - break; - case IOCTL_TO_ENUM(LWIS_DEVICE_RESET): - strlcpy(type_name, STRINGIFY(LWIS_DEVICE_RESET), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_DEVICE_RESET); - break; - case IOCTL_TO_ENUM(LWIS_EVENT_CONTROL_GET): - strlcpy(type_name, STRINGIFY(LWIS_EVENT_CONTROL_GET), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_EVENT_CONTROL_GET); - break; - case IOCTL_TO_ENUM(LWIS_EVENT_CONTROL_SET): - strlcpy(type_name, STRINGIFY(LWIS_EVENT_CONTROL_SET), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_EVENT_CONTROL_SET); - break; - case IOCTL_TO_ENUM(LWIS_EVENT_DEQUEUE): - strlcpy(type_name, STRINGIFY(LWIS_EVENT_DEQUEUE), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_EVENT_DEQUEUE); - break; - case IOCTL_TO_ENUM(LWIS_TIME_QUERY): - strlcpy(type_name, STRINGIFY(LWIS_TIME_QUERY), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_TIME_QUERY); - break; - case IOCTL_TO_ENUM(LWIS_TRANSACTION_SUBMIT): - strlcpy(type_name, STRINGIFY(LWIS_TRANSACTION_SUBMIT), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_TRANSACTION_SUBMIT); - break; - case IOCTL_TO_ENUM(LWIS_TRANSACTION_CANCEL): - strlcpy(type_name, STRINGIFY(LWIS_TRANSACTION_CANCEL), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_TRANSACTION_CANCEL); - break; - case IOCTL_TO_ENUM(LWIS_TRANSACTION_REPLACE): - strlcpy(type_name, STRINGIFY(LWIS_TRANSACTION_REPLACE), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_TRANSACTION_REPLACE); - break; - case IOCTL_TO_ENUM(LWIS_DPM_CLK_UPDATE): - strlcpy(type_name, STRINGIFY(LWIS_DPM_CLK_UPDATE), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_DPM_CLK_UPDATE); - break; - case IOCTL_TO_ENUM(LWIS_ECHO): - strlcpy(type_name, STRINGIFY(LWIS_ECHO), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_ECHO); - break; - case IOCTL_TO_ENUM(LWIS_DPM_QOS_UPDATE): - strlcpy(type_name, STRINGIFY(LWIS_DPM_QOS_UPDATE), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_DPM_QOS_UPDATE); - break; - case IOCTL_TO_ENUM(LWIS_DPM_GET_CLOCK): - strlcpy(type_name, STRINGIFY(LWIS_DPM_GET_CLOCK), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_DPM_GET_CLOCK); - break; - case IOCTL_TO_ENUM(LWIS_PERIODIC_IO_SUBMIT): - strlcpy(type_name, STRINGIFY(LWIS_PERIODIC_IO_SUBMIT), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_PERIODIC_IO_SUBMIT); - break; - case IOCTL_TO_ENUM(LWIS_PERIODIC_IO_CANCEL): - strlcpy(type_name, STRINGIFY(LWIS_PERIODIC_IO_CANCEL), sizeof(type_name)); - exp_size = IOCTL_ARG_SIZE(LWIS_PERIODIC_IO_CANCEL); + case IOCTL_TO_ENUM(LWIS_CMD_PACKET): + strscpy(type_name, STRINGIFY(LWIS_CMD_PACKET), sizeof(type_name)); + exp_size = IOCTL_ARG_SIZE(LWIS_CMD_PACKET); break; default: - strlcpy(type_name, "UNDEFINED", sizeof(type_name)); + strscpy(type_name, "UNDEFINED", sizeof(type_name)); exp_size = 0; break; }; @@ -154,68 +68,6 @@ static void lwis_ioctl_pr_err(struct lwis_device *lwis_dev, unsigned int ioctl_t } } -static int ioctl_get_device_info(struct lwis_device *lwis_dev, struct lwis_device_info *msg) -{ - int i; - struct lwis_device_info k_info = { .id = lwis_dev->id, - .type = lwis_dev->type, - .num_clks = 0, - .num_regs = 0, - .transaction_worker_thread_pid = -1, - .periodic_io_thread_pid = -1 }; - strlcpy(k_info.name, lwis_dev->name, LWIS_MAX_NAME_STRING_LEN); - - if (lwis_dev->clocks) { - k_info.num_clks = lwis_dev->clocks->count; - for (i = 0; i < lwis_dev->clocks->count; i++) { - if (i >= LWIS_MAX_CLOCK_NUM) { - dev_err(lwis_dev->dev, - "Clock count larger than LWIS_MAX_CLOCK_NUM\n"); - break; - } - strlcpy(k_info.clks[i].name, lwis_dev->clocks->clk[i].name, - LWIS_MAX_NAME_STRING_LEN); - k_info.clks[i].clk_index = i; - k_info.clks[i].frequency = 0; - } - } - - if (lwis_dev->type == DEVICE_TYPE_IOREG) { - struct lwis_ioreg_device *ioreg_dev; - ioreg_dev = container_of(lwis_dev, struct lwis_ioreg_device, base_dev); - if (ioreg_dev->reg_list.count > 0) { - k_info.num_regs = ioreg_dev->reg_list.count; - for (i = 0; i < ioreg_dev->reg_list.count; i++) { - if (i >= LWIS_MAX_REG_NUM) { - dev_err(lwis_dev->dev, - "Reg count larger than LWIS_MAX_REG_NUM\n"); - break; - } - strlcpy(k_info.regs[i].name, ioreg_dev->reg_list.block[i].name, - LWIS_MAX_NAME_STRING_LEN); - k_info.regs[i].reg_index = i; - k_info.regs[i].start = ioreg_dev->reg_list.block[i].start; - k_info.regs[i].size = ioreg_dev->reg_list.block[i].size; - } - } - } - - if (lwis_dev->transaction_worker_thread) { - k_info.transaction_worker_thread_pid = lwis_dev->transaction_worker_thread->pid; - } - - if (lwis_dev->periodic_io_worker_thread) { - k_info.periodic_io_thread_pid = lwis_dev->periodic_io_worker_thread->pid; - } - - if (copy_to_user((void __user *)msg, &k_info, sizeof(k_info))) { - dev_err(lwis_dev->dev, "Failed to copy device info to userspace\n"); - return -EFAULT; - } - - return 0; -} - static int register_read(struct lwis_device *lwis_dev, struct lwis_io_entry *read_entry, struct lwis_io_entry *user_msg) { @@ -228,8 +80,8 @@ static int register_read(struct lwis_device *lwis_dev, struct lwis_io_entry *rea /* Save the userspace buffer address */ user_buf = read_entry->rw_batch.buf; /* Allocate read buffer */ - read_entry->rw_batch.buf = - lwis_allocator_allocate(lwis_dev, read_entry->rw_batch.size_in_bytes); + read_entry->rw_batch.buf = lwis_allocator_allocate( + lwis_dev, read_entry->rw_batch.size_in_bytes, GFP_KERNEL); if (!read_entry->rw_batch.buf) { dev_err_ratelimited(lwis_dev->dev, "Failed to allocate register read buffer\n"); @@ -284,8 +136,8 @@ static int register_write(struct lwis_device *lwis_dev, struct lwis_io_entry *wr /* Save the userspace buffer address */ user_buf = write_entry->rw_batch.buf; /* Allocate write buffer and copy contents from userspace */ - write_entry->rw_batch.buf = - lwis_allocator_allocate(lwis_dev, write_entry->rw_batch.size_in_bytes); + write_entry->rw_batch.buf = lwis_allocator_allocate( + lwis_dev, write_entry->rw_batch.size_in_bytes, GFP_KERNEL); if (!write_entry->rw_batch.buf) { dev_err_ratelimited(lwis_dev->dev, "Failed to allocate register write buffer\n"); @@ -330,45 +182,6 @@ static int register_modify(struct lwis_device *lwis_dev, struct lwis_io_entry *m return ret; } -static int copy_io_entries(struct lwis_device *lwis_dev, struct lwis_io_entries *user_msg, - struct lwis_io_entries *k_msg, struct lwis_io_entry **k_entries) -{ - int ret = 0; - struct lwis_io_entry *io_entries; - uint32_t buf_size; - - /* Register io is not supported for the lwis device, return */ - if (!lwis_dev->vops.register_io) { - dev_err(lwis_dev->dev, "Register IO not supported on this LWIS device\n"); - return -EINVAL; - } - - /* Copy io_entries from userspace */ - if (copy_from_user(k_msg, (void __user *)user_msg, sizeof(*k_msg))) { - dev_err(lwis_dev->dev, "Failed to copy io_entries header from userspace.\n"); - return -EFAULT; - } - buf_size = sizeof(struct lwis_io_entry) * k_msg->num_io_entries; - if (buf_size / sizeof(struct lwis_io_entry) != k_msg->num_io_entries) { - dev_err(lwis_dev->dev, "Failed to copy io_entries due to integer overflow.\n"); - return -EOVERFLOW; - } - io_entries = lwis_allocator_allocate(lwis_dev, buf_size); - if (!io_entries) { - dev_err(lwis_dev->dev, "Failed to allocate io_entries buffer\n"); - return -ENOMEM; - } - if (copy_from_user(io_entries, (void __user *)k_msg->io_entries, buf_size)) { - ret = -EFAULT; - lwis_allocator_free(lwis_dev, io_entries); - dev_err(lwis_dev->dev, "Failed to copy io_entries from userspace.\n"); - return ret; - } - *k_entries = io_entries; - - return 0; -} - static int synchronous_process_io_entries(struct lwis_device *lwis_dev, int num_io_entries, struct lwis_io_entry *io_entries, struct lwis_io_entry *user_msg) @@ -396,7 +209,7 @@ static int synchronous_process_io_entries(struct lwis_device *lwis_dev, int num_ ret = register_write(lwis_dev, &io_entries[i]); break; case LWIS_IO_ENTRY_POLL: - ret = lwis_io_entry_poll(lwis_dev, &io_entries[i], /*non_blocking=*/false); + ret = lwis_io_entry_poll(lwis_dev, &io_entries[i]); break; case LWIS_IO_ENTRY_READ_ASSERT: ret = lwis_io_entry_read_assert(lwis_dev, &io_entries[i]); @@ -421,208 +234,222 @@ exit: return ret; } -static int ioctl_reg_io(struct lwis_device *lwis_dev, struct lwis_io_entries *user_msg) +static int construct_io_entry(struct lwis_client *client, struct lwis_io_entry *user_entries, + size_t num_io_entries, struct lwis_io_entry **io_entries) { + int i; int ret = 0; - struct lwis_io_entries k_msg; - struct lwis_io_entry *k_entries = NULL; - - ret = copy_io_entries(lwis_dev, user_msg, &k_msg, &k_entries); - if (ret) { - goto reg_io_exit; - } - - /* Walk through and execute the entries */ - ret = synchronous_process_io_entries(lwis_dev, k_msg.num_io_entries, k_entries, - k_msg.io_entries); + int last_buf_alloc_idx = -1; + size_t entry_size; + struct lwis_io_entry *k_entries; + uint8_t *user_buf; + uint8_t *k_buf; + struct lwis_device *lwis_dev = client->lwis_dev; -reg_io_exit: - if (k_entries) { - lwis_allocator_free(lwis_dev, k_entries); + entry_size = num_io_entries * sizeof(struct lwis_io_entry); + if (entry_size / sizeof(struct lwis_io_entry) != num_io_entries) { + dev_err(lwis_dev->dev, "Failed to prepare io entries due to integer overflow\n"); + return -EOVERFLOW; } - return ret; -} - -static int ioctl_buffer_alloc(struct lwis_client *lwis_client, - struct lwis_alloc_buffer_info __user *msg) -{ - unsigned long ret = 0; - struct lwis_alloc_buffer_info alloc_info; - struct lwis_allocated_buffer *buffer; - struct lwis_device *lwis_dev = lwis_client->lwis_dev; - - buffer = kmalloc(sizeof(*buffer), GFP_KERNEL); - if (!buffer) { - dev_err(lwis_dev->dev, "Failed to allocated lwis_allocated_buffer\n"); + k_entries = lwis_allocator_allocate(lwis_dev, entry_size, GFP_KERNEL); + if (!k_entries) { + dev_err(lwis_dev->dev, "Failed to allocate io entries\n"); return -ENOMEM; } - if (copy_from_user((void *)&alloc_info, (void __user *)msg, sizeof(alloc_info))) { + if (copy_from_user((void *)k_entries, (void __user *)user_entries, entry_size)) { ret = -EFAULT; - dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", sizeof(alloc_info)); - goto error_alloc; - } - - ret = lwis_buffer_alloc(lwis_client, &alloc_info, buffer); - if (ret) { - dev_err(lwis_dev->dev, "Failed to allocate buffer\n"); - goto error_alloc; + dev_err(lwis_dev->dev, "Failed to copy io entries from user\n"); + goto error_free_entries; } - if (copy_to_user((void __user *)msg, (void *)&alloc_info, sizeof(alloc_info))) { - ret = -EFAULT; - dev_err(lwis_dev->dev, "Failed to copy %zu bytes to user\n", sizeof(alloc_info)); - lwis_buffer_free(lwis_client, buffer); - goto error_alloc; + /* + * For batch writes, need to allocate kernel buffers to deep copy the + * write values. Don't need to do this for batch reads because memory + * will be allocated in the form of lwis_io_result in io processing. + */ + for (i = 0; i < num_io_entries; ++i) { + if (k_entries[i].type == LWIS_IO_ENTRY_WRITE_BATCH) { + user_buf = k_entries[i].rw_batch.buf; + k_buf = lwis_allocator_allocate( + lwis_dev, k_entries[i].rw_batch.size_in_bytes, GFP_KERNEL); + if (!k_buf) { + dev_err_ratelimited(lwis_dev->dev, + "Failed to allocate io write buffer\n"); + ret = -ENOMEM; + goto error_free_buf; + } + last_buf_alloc_idx = i; + k_entries[i].rw_batch.buf = k_buf; + if (copy_from_user(k_buf, (void __user *)user_buf, + k_entries[i].rw_batch.size_in_bytes)) { + ret = -EFAULT; + dev_err_ratelimited( + lwis_dev->dev, + "Failed to copy io write buffer from userspace\n"); + goto error_free_buf; + } + } } + *io_entries = k_entries; return 0; -error_alloc: - kfree(buffer); +error_free_buf: + for (i = 0; i <= last_buf_alloc_idx; ++i) { + if (k_entries[i].type == LWIS_IO_ENTRY_WRITE_BATCH) { + lwis_allocator_free(lwis_dev, k_entries[i].rw_batch.buf); + k_entries[i].rw_batch.buf = NULL; + } + } +error_free_entries: + lwis_allocator_free(lwis_dev, k_entries); + *io_entries = NULL; return ret; } -static int ioctl_buffer_free(struct lwis_client *lwis_client, int __user *msg) +static int copy_pkt_to_user(struct lwis_device *lwis_dev, void __user *u_msg, void *k_msg, + size_t size) { - int ret = 0; - int fd; - struct lwis_allocated_buffer *buffer; - struct lwis_device *lwis_dev = lwis_client->lwis_dev; - - if (copy_from_user((void *)&fd, (void __user *)msg, sizeof(fd))) { - dev_err(lwis_dev->dev, "Failed to copy file descriptor from user\n"); + if (copy_to_user(u_msg, k_msg, size)) { + dev_err(lwis_dev->dev, "Failed to copy %zu bytes to user\n", size); return -EFAULT; } - buffer = lwis_client_allocated_buffer_find(lwis_client, fd); - if (!buffer) { - dev_err(lwis_dev->dev, "Cannot find allocated buffer FD %d\n", fd); - return -ENOENT; - } - - ret = lwis_buffer_free(lwis_client, buffer); - if (ret) { - dev_err(lwis_dev->dev, "Failed to free buffer FD %d\n", fd); - return ret; - } - - kfree(buffer); - return 0; } -static int ioctl_buffer_enroll(struct lwis_client *lwis_client, struct lwis_buffer_info __user *msg) +static int cmd_echo(struct lwis_device *lwis_dev, struct lwis_cmd_pkt *header, + struct lwis_cmd_echo __user *u_msg) { - unsigned long ret = 0; - struct lwis_enrolled_buffer *buffer; - struct lwis_device *lwis_dev = lwis_client->lwis_dev; + struct lwis_cmd_echo echo_msg; + char *buffer = NULL; - buffer = kmalloc(sizeof(struct lwis_enrolled_buffer), GFP_KERNEL); - if (!buffer) { - dev_err(lwis_dev->dev, "Failed to allocate lwis_enrolled_buffer struct\n"); - return -ENOMEM; + if (copy_from_user((void *)&echo_msg, (void __user *)u_msg, sizeof(echo_msg))) { + dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", sizeof(echo_msg)); + return -EFAULT; } - if (copy_from_user((void *)&buffer->info, (void __user *)msg, sizeof(buffer->info))) { - ret = -EFAULT; - dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", - sizeof(buffer->info)); - goto error_enroll; + if (echo_msg.msg.size == 0) { + header->ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } - ret = lwis_buffer_enroll(lwis_client, buffer); - if (ret) { - dev_err(lwis_dev->dev, "Failed to enroll buffer\n"); - goto error_enroll; + buffer = kmalloc(echo_msg.msg.size + 1, GFP_KERNEL); + if (!buffer) { + dev_err(lwis_dev->dev, "Failed to allocate buffer for echo message\n"); + header->ret_code = -ENOMEM; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } - - if (copy_to_user((void __user *)msg, (void *)&buffer->info, sizeof(buffer->info))) { - ret = -EFAULT; - dev_err(lwis_dev->dev, "Failed to copy %zu bytes to user\n", sizeof(buffer->info)); - lwis_buffer_disenroll(lwis_client, buffer); - goto error_enroll; + if (copy_from_user(buffer, (void __user *)echo_msg.msg.msg, echo_msg.msg.size)) { + dev_err(lwis_dev->dev, "Failed to copy %zu bytes echo message from user\n", + echo_msg.msg.size); + kfree(buffer); + header->ret_code = -EFAULT; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } + buffer[echo_msg.msg.size] = '\0'; - return 0; - -error_enroll: + if (echo_msg.msg.kernel_log) { + dev_info(lwis_dev->dev, "LWIS_ECHO: %s\n", buffer); + } kfree(buffer); - return ret; + + header->ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_buffer_disenroll(struct lwis_client *lwis_client, - struct lwis_enrolled_buffer_info __user *msg) +static int cmd_time_query(struct lwis_device *lwis_dev, struct lwis_cmd_pkt *header, + struct lwis_cmd_time_query __user *u_msg) { - unsigned long ret = 0; - struct lwis_enrolled_buffer_info info; - struct lwis_enrolled_buffer *buffer; - struct lwis_device *lwis_dev = lwis_client->lwis_dev; - - if (copy_from_user((void *)&info, (void __user *)msg, sizeof(info))) { - dev_err(lwis_dev->dev, "Failed to copy DMA virtual address from user\n"); - return -EFAULT; - } - - buffer = lwis_client_enrolled_buffer_find(lwis_client, info.fd, info.dma_vaddr); - - if (!buffer) { - dev_err(lwis_dev->dev, "Failed to find dma buffer for fd %d vaddr %pad\n", info.fd, - &info.dma_vaddr); - return -ENOENT; - } - - ret = lwis_buffer_disenroll(lwis_client, buffer); - if (ret) { - dev_err(lwis_dev->dev, "Failed to disenroll dma buffer for fd %d vaddr %pad\n", - info.fd, &info.dma_vaddr); - return ret; - } - - kfree(buffer); + struct lwis_cmd_time_query time_query; + time_query.timestamp_ns = ktime_to_ns(lwis_get_time()); + time_query.header.cmd_id = header->cmd_id; + time_query.header.next = header->next; + time_query.header.ret_code = 0; - return 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&time_query, sizeof(time_query)); } -static int ioctl_buffer_cpu_access(struct lwis_client *lwis_client, - struct lwis_buffer_cpu_access_op __user *msg) +static int cmd_get_device_info(struct lwis_device *lwis_dev, struct lwis_cmd_pkt *header, + struct lwis_cmd_device_info __user *u_msg) { - int ret = 0; - struct lwis_buffer_cpu_access_op op; - struct lwis_device *lwis_dev = lwis_client->lwis_dev; + int i; + struct lwis_cmd_device_info k_info = { .header.cmd_id = header->cmd_id, + .header.next = header->next, + .info.id = lwis_dev->id, + .info.type = lwis_dev->type, + .info.num_clks = 0, + .info.num_regs = 0, + .info.transaction_worker_thread_pid = -1, + .info.periodic_io_thread_pid = -1 }; + strscpy(k_info.info.name, lwis_dev->name, LWIS_MAX_NAME_STRING_LEN); - if (copy_from_user((void *)&op, (void __user *)msg, sizeof(op))) { - dev_err(lwis_dev->dev, "Failed to copy buffer CPU access operation from user\n"); - return -EFAULT; + if (lwis_dev->clocks) { + k_info.info.num_clks = lwis_dev->clocks->count; + for (i = 0; i < lwis_dev->clocks->count; i++) { + if (i >= LWIS_MAX_CLOCK_NUM) { + dev_err(lwis_dev->dev, + "Clock count larger than LWIS_MAX_CLOCK_NUM\n"); + break; + } + strscpy(k_info.info.clks[i].name, lwis_dev->clocks->clk[i].name, + LWIS_MAX_NAME_STRING_LEN); + k_info.info.clks[i].clk_index = i; + k_info.info.clks[i].frequency = 0; + } } - ret = lwis_buffer_cpu_access(lwis_client, &op); - if (ret) { - dev_err(lwis_dev->dev, "Failed to prepare for cpu access for fd %d\n", op.fd); - return ret; + if (lwis_dev->type == DEVICE_TYPE_IOREG) { + struct lwis_ioreg_device *ioreg_dev; + ioreg_dev = container_of(lwis_dev, struct lwis_ioreg_device, base_dev); + if (ioreg_dev->reg_list.count > 0) { + k_info.info.num_regs = ioreg_dev->reg_list.count; + for (i = 0; i < ioreg_dev->reg_list.count; i++) { + if (i >= LWIS_MAX_REG_NUM) { + dev_err(lwis_dev->dev, + "Reg count larger than LWIS_MAX_REG_NUM\n"); + break; + } + strscpy(k_info.info.regs[i].name, ioreg_dev->reg_list.block[i].name, + LWIS_MAX_NAME_STRING_LEN); + k_info.info.regs[i].reg_index = i; + k_info.info.regs[i].start = ioreg_dev->reg_list.block[i].start; + k_info.info.regs[i].size = ioreg_dev->reg_list.block[i].size; + } + } } - return 0; + if (lwis_dev->transaction_worker_thread) { + k_info.info.transaction_worker_thread_pid = + lwis_dev->transaction_worker_thread->pid; + } + + k_info.header.ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&k_info, sizeof(k_info)); } -static int ioctl_device_enable(struct lwis_client *lwis_client) +static int cmd_device_enable(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_pkt __user *u_msg) { int ret = 0; struct lwis_device *lwis_dev = lwis_client->lwis_dev; if (lwis_client->is_enabled) { - return ret; + header->ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } + mutex_lock(&lwis_dev->client_lock); if (lwis_dev->enabled > 0 && lwis_dev->enabled < INT_MAX) { lwis_dev->enabled++; lwis_client->is_enabled = true; - mutex_unlock(&lwis_dev->client_lock); - return 0; + ret = 0; + goto exit_locked; } else if (lwis_dev->enabled == INT_MAX) { dev_err(lwis_dev->dev, "Enable counter overflow\n"); ret = -EINVAL; - goto error_locked; + goto exit_locked; } /* Clear event queues to make sure there is no stale event from @@ -633,24 +460,28 @@ static int ioctl_device_enable(struct lwis_client *lwis_client) ret = lwis_dev_power_up_locked(lwis_dev); if (ret < 0) { dev_err(lwis_dev->dev, "Failed to power up device\n"); - goto error_locked; + goto exit_locked; } lwis_dev->enabled++; lwis_client->is_enabled = true; + lwis_dev->is_suspended = false; dev_info(lwis_dev->dev, "Device enabled\n"); -error_locked: +exit_locked: mutex_unlock(&lwis_dev->client_lock); - return ret; + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_device_disable(struct lwis_client *lwis_client) +static int cmd_device_disable(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_pkt __user *u_msg) { int ret = 0; struct lwis_device *lwis_dev = lwis_client->lwis_dev; if (!lwis_client->is_enabled) { - return ret; + header->ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } mutex_lock(&lwis_dev->client_lock); @@ -677,73 +508,81 @@ static int ioctl_device_disable(struct lwis_client *lwis_client) if (lwis_dev->enabled > 1) { lwis_dev->enabled--; lwis_client->is_enabled = false; - mutex_unlock(&lwis_dev->client_lock); - return 0; + ret = 0; + goto exit_locked; } else if (lwis_dev->enabled <= 0) { dev_err(lwis_dev->dev, "Disabling a device that is already disabled\n"); ret = -EINVAL; - goto error_locked; + goto exit_locked; } ret = lwis_dev_power_down_locked(lwis_dev); if (ret < 0) { dev_err(lwis_dev->dev, "Failed to power down device\n"); - goto error_locked; + goto exit_locked; } lwis_device_event_states_clear_locked(lwis_dev); lwis_dev->enabled--; lwis_client->is_enabled = false; + lwis_dev->is_suspended = false; dev_info(lwis_dev->dev, "Device disabled\n"); -error_locked: +exit_locked: mutex_unlock(&lwis_dev->client_lock); - return ret; + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_echo(struct lwis_device *lwis_dev, struct lwis_echo __user *msg) +static int copy_io_entries_from_cmd(struct lwis_device *lwis_dev, + struct lwis_cmd_io_entries __user *u_msg, + struct lwis_cmd_io_entries *k_msg, + struct lwis_io_entry **k_entries) { - struct lwis_echo echo_msg; - char *buffer; + struct lwis_io_entry *io_entries; + uint32_t buf_size; - if (copy_from_user((void *)&echo_msg, (void __user *)msg, sizeof(echo_msg))) { - dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", sizeof(echo_msg)); - return -EFAULT; + /* Register io is not supported for the lwis device, return */ + if (!lwis_dev->vops.register_io) { + dev_err(lwis_dev->dev, "Register IO not supported on this LWIS device\n"); + return -EINVAL; } - if (echo_msg.size == 0) { - return 0; + /* Copy io_entries from userspace */ + if (copy_from_user(k_msg, (void __user *)u_msg, sizeof(*k_msg))) { + dev_err(lwis_dev->dev, "Failed to copy io_entries header from userspace.\n"); + return -EFAULT; } - - buffer = kmalloc(echo_msg.size + 1, GFP_KERNEL); - if (!buffer) { - dev_err(lwis_dev->dev, "Failed to allocate buffer for echo message\n"); + buf_size = sizeof(struct lwis_io_entry) * k_msg->io.num_io_entries; + if (buf_size / sizeof(struct lwis_io_entry) != k_msg->io.num_io_entries) { + dev_err(lwis_dev->dev, "Failed to copy io_entries due to integer overflow.\n"); + return -EOVERFLOW; + } + io_entries = lwis_allocator_allocate(lwis_dev, buf_size, GFP_KERNEL); + if (!io_entries) { + dev_err(lwis_dev->dev, "Failed to allocate io_entries buffer\n"); return -ENOMEM; } - if (copy_from_user(buffer, (void __user *)echo_msg.msg, echo_msg.size)) { - dev_err(lwis_dev->dev, "Failed to copy %zu bytes echo message from user\n", - echo_msg.size); - kfree(buffer); + if (copy_from_user(io_entries, (void __user *)k_msg->io.io_entries, buf_size)) { + dev_err(lwis_dev->dev, "Failed to copy io_entries from userspace.\n"); + lwis_allocator_free(lwis_dev, io_entries); return -EFAULT; } - buffer[echo_msg.size] = '\0'; + *k_entries = io_entries; - if (echo_msg.kernel_log) { - dev_info(lwis_dev->dev, "LWIS_ECHO: %s\n", buffer); - } - kfree(buffer); return 0; } -static int ioctl_device_reset(struct lwis_client *lwis_client, struct lwis_io_entries *user_msg) +static int cmd_device_reset(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_io_entries __user *u_msg) { int ret = 0; struct lwis_device *lwis_dev = lwis_client->lwis_dev; - struct lwis_io_entries k_msg; + struct lwis_cmd_io_entries k_msg; struct lwis_io_entry *k_entries = NULL; unsigned long flags; bool device_enabled = false; - ret = copy_io_entries(lwis_dev, user_msg, &k_msg, &k_entries); + ret = copy_io_entries_from_cmd(lwis_dev, u_msg, &k_msg, &k_entries); if (ret) { goto soft_reset_exit; } @@ -770,8 +609,8 @@ static int ioctl_device_reset(struct lwis_client *lwis_client, struct lwis_io_en /* Perform reset routine defined by the io_entries */ if (device_enabled) { - ret = synchronous_process_io_entries(lwis_dev, k_msg.num_io_entries, k_entries, - k_msg.io_entries); + ret = synchronous_process_io_entries(lwis_dev, k_msg.io.num_io_entries, k_entries, + k_msg.io.io_entries); } else { dev_warn(lwis_dev->dev, "Device is not enabled, IoEntries will not be executed in DEVICE_RESET\n"); @@ -784,95 +623,410 @@ soft_reset_exit: if (k_entries) { lwis_allocator_free(lwis_dev, k_entries); } + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); +} + +static int cmd_device_suspend(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_pkt __user *u_msg) +{ + int ret = 0; + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + + if (!lwis_dev->suspend_sequence) { + dev_err(lwis_dev->dev, "No suspend sequence defined\n"); + header->ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + if (!lwis_client->is_enabled) { + dev_err(lwis_dev->dev, "Trying to suspend a disabled device\n"); + header->ret_code = -EINVAL; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + if (lwis_dev->is_suspended) { + header->ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + mutex_lock(&lwis_dev->client_lock); + /* Clear event states for this client */ + lwis_client_event_states_clear(lwis_client); + mutex_unlock(&lwis_dev->client_lock); + + /* Flush all periodic io to complete */ + ret = lwis_periodic_io_client_flush(lwis_client); + if (ret) { + dev_err(lwis_dev->dev, "Failed to wait for in-process periodic io to complete\n"); + } + + /* Flush all pending transactions */ + ret = lwis_transaction_client_flush(lwis_client); + if (ret) { + dev_err(lwis_dev->dev, "Failed to flush pending transactions\n"); + } + + /* Run cleanup transactions. */ + lwis_transaction_client_cleanup(lwis_client); + + mutex_lock(&lwis_dev->client_lock); + ret = lwis_dev_process_power_sequence(lwis_dev, lwis_dev->suspend_sequence, + /*set_active=*/false, /*skip_error=*/false); + if (ret) { + dev_err(lwis_dev->dev, "Error lwis_dev_process_power_sequence (%d)\n", ret); + goto exit_locked; + } + + lwis_device_event_states_clear_locked(lwis_dev); + + lwis_dev->is_suspended = true; + dev_info(lwis_dev->dev, "Device suspended\n"); +exit_locked: + mutex_unlock(&lwis_dev->client_lock); + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); +} + +static int cmd_device_resume(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_pkt __user *u_msg) +{ + int ret = 0; + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + + if (!lwis_dev->resume_sequence) { + dev_err(lwis_dev->dev, "No resume sequence defined\n"); + header->ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + if (!lwis_dev->is_suspended) { + header->ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + mutex_lock(&lwis_dev->client_lock); + /* Clear event queues to make sure there is no stale event from + * previous session */ + lwis_client_event_queue_clear(lwis_client); + lwis_client_error_event_queue_clear(lwis_client); + + ret = lwis_dev_process_power_sequence(lwis_dev, lwis_dev->resume_sequence, + /*set_active=*/true, /*skip_error=*/false); + if (ret) { + dev_err(lwis_dev->dev, "Error lwis_dev_process_power_sequence (%d)\n", ret); + goto exit_locked; + } + + lwis_dev->is_suspended = false; + dev_info(lwis_dev->dev, "Device resumed\n"); +exit_locked: + mutex_unlock(&lwis_dev->client_lock); + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); +} + +static int cmd_dump_debug_state(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_pkt __user *u_msg) +{ + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + + mutex_lock(&lwis_dev->client_lock); + /* Dump lwis device crash info */ + lwis_device_crash_info_dump(lwis_dev); + mutex_unlock(&lwis_dev->client_lock); + + header->ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); +} + +static int cmd_dma_buffer_enroll(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_dma_buffer_enroll __user *u_msg) +{ + int ret = 0; + struct lwis_cmd_dma_buffer_enroll buf_info; + struct lwis_enrolled_buffer *buffer; + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + + buffer = kmalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) { + dev_err(lwis_dev->dev, "Failed to allocate lwis_enrolled_buffer struct\n"); + header->ret_code = -ENOMEM; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + if (copy_from_user((void *)&buf_info, (void __user *)u_msg, sizeof(buf_info))) { + dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", sizeof(buf_info)); + ret = -EFAULT; + goto error_enroll; + } + + buffer->info.fd = buf_info.info.fd; + buffer->info.dma_read = buf_info.info.dma_read; + buffer->info.dma_write = buf_info.info.dma_write; + + ret = lwis_buffer_enroll(lwis_client, buffer); + if (ret) { + dev_err(lwis_dev->dev, "Failed to enroll buffer\n"); + goto error_enroll; + } + + buf_info.info.dma_vaddr = buffer->info.dma_vaddr; + buf_info.header.cmd_id = header->cmd_id; + buf_info.header.next = header->next; + buf_info.header.ret_code = ret; + ret = copy_pkt_to_user(lwis_dev, u_msg, (void *)&buf_info, sizeof(buf_info)); + if (ret) { + lwis_buffer_disenroll(lwis_client, buffer); + goto error_enroll; + } + return ret; + +error_enroll: + kfree(buffer); + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_event_control_get(struct lwis_client *lwis_client, - struct lwis_event_control __user *msg) +static int cmd_dma_buffer_disenroll(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_dma_buffer_disenroll __user *u_msg) { - unsigned long ret = 0; - struct lwis_event_control control; + int ret = 0; + struct lwis_cmd_dma_buffer_disenroll info; + struct lwis_enrolled_buffer *buffer; struct lwis_device *lwis_dev = lwis_client->lwis_dev; - if (copy_from_user((void *)&control, (void __user *)msg, sizeof(control))) { - dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", sizeof(control)); + if (copy_from_user((void *)&info, (void __user *)u_msg, sizeof(info))) { + dev_err(lwis_dev->dev, "Failed to copy DMA virtual address from user\n"); return -EFAULT; } - ret = lwis_client_event_control_get(lwis_client, control.event_id, &control); + buffer = lwis_client_enrolled_buffer_find(lwis_client, info.info.fd, info.info.dma_vaddr); + if (!buffer) { + dev_err(lwis_dev->dev, "Failed to find dma buffer for fd %d vaddr %pad\n", + info.info.fd, &info.info.dma_vaddr); + header->ret_code = -ENOENT; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + ret = lwis_buffer_disenroll(lwis_client, buffer); if (ret) { - dev_err(lwis_dev->dev, "Failed to get event: %lld (err:%ld)\n", control.event_id, - ret); - return -EINVAL; + dev_err(lwis_dev->dev, "Failed to disenroll dma buffer for fd %d vaddr %pad\n", + info.info.fd, &info.info.dma_vaddr); + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + kfree(buffer); + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); +} + +static int cmd_dma_buffer_cpu_access(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_dma_buffer_cpu_access __user *u_msg) +{ + int ret = 0; + struct lwis_cmd_dma_buffer_cpu_access op; + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + + if (copy_from_user((void *)&op, (void __user *)u_msg, sizeof(op))) { + dev_err(lwis_dev->dev, "Failed to copy buffer CPU access operation from user\n"); + return -EFAULT; + } + + ret = lwis_buffer_cpu_access(lwis_client, &op.op); + if (ret) { + dev_err(lwis_dev->dev, "Failed to prepare for cpu access for fd %d\n", op.op.fd); + } + + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); +} + +static int cmd_dma_buffer_alloc(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_dma_buffer_alloc __user *u_msg) +{ + int ret = 0; + struct lwis_cmd_dma_buffer_alloc alloc_info; + struct lwis_allocated_buffer *buffer; + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + + buffer = kmalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) { + dev_err(lwis_dev->dev, "Failed to allocated lwis_allocated_buffer\n"); + return -ENOMEM; + } + + if (copy_from_user((void *)&alloc_info, (void __user *)u_msg, sizeof(alloc_info))) { + dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", sizeof(alloc_info)); + ret = -EFAULT; + goto error_alloc; + } + + ret = lwis_buffer_alloc(lwis_client, &alloc_info.info, buffer); + if (ret) { + dev_err(lwis_dev->dev, "Failed to allocate buffer\n"); + goto error_alloc; + } + + alloc_info.header.ret_code = 0; + ret = copy_pkt_to_user(lwis_dev, u_msg, (void *)&alloc_info, sizeof(alloc_info)); + if (ret) { + lwis_buffer_free(lwis_client, buffer); + ret = -EFAULT; + goto error_alloc; } - if (copy_to_user((void __user *)msg, (void *)&control, sizeof(control))) { - dev_err(lwis_dev->dev, "Failed to copy %zu bytes to user\n", sizeof(control)); + return ret; + +error_alloc: + kfree(buffer); + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); +} + +static int cmd_dma_buffer_free(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_dma_buffer_free __user *u_msg) +{ + int ret = 0; + struct lwis_cmd_dma_buffer_free info; + struct lwis_allocated_buffer *buffer; + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + + if (copy_from_user((void *)&info, (void __user *)u_msg, sizeof(info))) { + dev_err(lwis_dev->dev, "Failed to copy file descriptor from user\n"); return -EFAULT; } - return 0; + buffer = lwis_client_allocated_buffer_find(lwis_client, info.fd); + if (!buffer) { + dev_err(lwis_dev->dev, "Cannot find allocated buffer FD %d\n", info.fd); + header->ret_code = -ENOENT; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + ret = lwis_buffer_free(lwis_client, buffer); + if (ret) { + dev_err(lwis_dev->dev, "Failed to free buffer FD %d\n", info.fd); + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + kfree(buffer); + + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); +} + +static int cmd_reg_io(struct lwis_device *lwis_dev, struct lwis_cmd_pkt *header, + struct lwis_cmd_io_entries __user *u_msg) +{ + int ret = 0; + struct lwis_cmd_io_entries k_msg; + struct lwis_io_entry *k_entries = NULL; + + ret = copy_io_entries_from_cmd(lwis_dev, u_msg, &k_msg, &k_entries); + if (ret) { + goto reg_io_exit; + } + + /* Walk through and execute the entries */ + ret = synchronous_process_io_entries(lwis_dev, k_msg.io.num_io_entries, k_entries, + k_msg.io.io_entries); + +reg_io_exit: + if (k_entries) { + lwis_allocator_free(lwis_dev, k_entries); + } + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_event_control_set(struct lwis_client *lwis_client, - struct lwis_event_control_list __user *msg) +static int cmd_event_control_get(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_event_control_get __user *u_msg) { - struct lwis_event_control_list k_msg; + int ret = 0; + struct lwis_cmd_event_control_get control; + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + + if (copy_from_user((void *)&control, (void __user *)u_msg, sizeof(control))) { + dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", sizeof(control)); + return -EFAULT; + } + + ret = lwis_client_event_control_get(lwis_client, control.ctl.event_id, &control.ctl); + if (ret) { + dev_err(lwis_dev->dev, "Failed to get event: %lld (err:%d)\n", control.ctl.event_id, + ret); + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); + } + + control.header.ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&control, sizeof(control)); +} + +static int cmd_event_control_set(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_event_control_set __user *u_msg) +{ + struct lwis_cmd_event_control_set k_msg; struct lwis_event_control *k_event_controls; struct lwis_device *lwis_dev = lwis_client->lwis_dev; int ret = 0; int i; size_t buf_size; - if (copy_from_user((void *)&k_msg, (void __user *)msg, - sizeof(struct lwis_event_control_list))) { - ret = -EFAULT; + if (copy_from_user((void *)&k_msg, (void __user *)u_msg, sizeof(k_msg))) { dev_err(lwis_dev->dev, "Failed to copy ioctl message from user\n"); - return ret; + return -EFAULT; } /* Copy event controls from user buffer. */ - buf_size = sizeof(struct lwis_event_control) * k_msg.num_event_controls; - if (buf_size / sizeof(struct lwis_event_control) != k_msg.num_event_controls) { + buf_size = sizeof(struct lwis_event_control) * k_msg.list.num_event_controls; + if (buf_size / sizeof(struct lwis_event_control) != k_msg.list.num_event_controls) { dev_err(lwis_dev->dev, "Failed to copy event controls due to integer overflow.\n"); - return -EOVERFLOW; + header->ret_code = -EOVERFLOW; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } k_event_controls = kmalloc(buf_size, GFP_KERNEL); if (!k_event_controls) { dev_err(lwis_dev->dev, "Failed to allocate event controls\n"); - return -ENOMEM; + header->ret_code = -ENOMEM; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } - if (copy_from_user(k_event_controls, (void __user *)k_msg.event_controls, buf_size)) { - ret = -EFAULT; + if (copy_from_user(k_event_controls, (void __user *)k_msg.list.event_controls, buf_size)) { dev_err(lwis_dev->dev, "Failed to copy event controls from user\n"); - goto out; + ret = -EFAULT; + goto exit; } - for (i = 0; i < k_msg.num_event_controls; i++) { + for (i = 0; i < k_msg.list.num_event_controls; i++) { ret = lwis_client_event_control_set(lwis_client, &k_event_controls[i]); if (ret) { dev_err(lwis_dev->dev, "Failed to apply event control 0x%llx\n", k_event_controls[i].event_id); - goto out; + goto exit; } } -out: +exit: kfree(k_event_controls); - return ret; + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_event_dequeue(struct lwis_client *lwis_client, struct lwis_event_info __user *msg) +static int cmd_event_dequeue(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_event_dequeue __user *u_msg) { - unsigned long ret = 0; - unsigned long err = 0; - struct lwis_event_entry *event; - struct lwis_event_info info_user; + struct lwis_cmd_event_dequeue info; struct lwis_device *lwis_dev = lwis_client->lwis_dev; + struct lwis_event_entry *event; + int ret = 0; + int err = 0; bool is_error_event = false; - if (copy_from_user((void *)&info_user, (void __user *)msg, sizeof(info_user))) { - dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", sizeof(info_user)); + if (copy_from_user((void *)&info, (void __user *)u_msg, sizeof(info))) { + dev_err(lwis_dev->dev, "Failed to copy %zu bytes from user\n", sizeof(info)); return -EFAULT; } @@ -882,45 +1036,39 @@ static int ioctl_event_dequeue(struct lwis_client *lwis_client, struct lwis_even if (ret == 0) { is_error_event = true; } else if (ret != -ENOENT) { - dev_err(lwis_dev->dev, "Error dequeueing error event: %ld\n", ret); + dev_err(lwis_dev->dev, "Error dequeueing error event: %d\n", ret); mutex_unlock(&lwis_dev->client_lock); - return ret; + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } else { /* Nothing at error event queue, continue to check normal * event queue */ ret = lwis_client_event_peek_front(lwis_client, &event); if (ret) { if (ret != -ENOENT) { - dev_err(lwis_dev->dev, "Error dequeueing event: %ld\n", ret); + dev_err(lwis_dev->dev, "Error dequeueing event: %d\n", ret); } mutex_unlock(&lwis_dev->client_lock); - return ret; + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } } /* We need to check if we have an adequate payload buffer */ - if (event->event_info.payload_size > info_user.payload_buffer_size) { + if (event->event_info.payload_size > info.info.payload_buffer_size) { /* Nope, we don't. Let's inform the user and bail */ - info_user.payload_size = event->event_info.payload_size; + info.info.payload_size = event->event_info.payload_size; err = -EAGAIN; } else { - /* - * Let's save the IOCTL inputs because they'll get overwritten - */ - size_t user_buffer_size = info_user.payload_buffer_size; - void *user_buffer = info_user.payload_buffer; - - /* Copy over the rest of the info */ - memcpy(&info_user, &event->event_info, sizeof(info_user)); - - /* Restore the IOCTL inputs */ - info_user.payload_buffer_size = user_buffer_size; - info_user.payload_buffer = user_buffer; + info.info.event_id = event->event_info.event_id; + info.info.event_counter = event->event_info.event_counter; + info.info.timestamp_ns = event->event_info.timestamp_ns; + info.info.payload_size = event->event_info.payload_size; /* Here we have a payload and the buffer is big enough */ - if (event->event_info.payload_size > 0 && info_user.payload_buffer) { + if (event->event_info.payload_size > 0 && info.info.payload_buffer) { /* Copy over the payload buffer to userspace */ - if (copy_to_user((void __user *)info_user.payload_buffer, + if (copy_to_user((void __user *)info.info.payload_buffer, (void *)event->event_info.payload_buffer, event->event_info.payload_size)) { dev_err(lwis_dev->dev, "Failed to copy %zu bytes to user\n", @@ -942,125 +1090,95 @@ static int ioctl_event_dequeue(struct lwis_client *lwis_client, struct lwis_even ret = lwis_client_event_pop_front(lwis_client, NULL); } if (ret) { - dev_err(lwis_dev->dev, "Error dequeueing event: %ld\n", ret); + dev_err(lwis_dev->dev, "Error dequeueing event: %d\n", ret); mutex_unlock(&lwis_dev->client_lock); - return ret; + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } } mutex_unlock(&lwis_dev->client_lock); /* Now let's copy the actual info struct back to user */ - if (copy_to_user((void __user *)msg, (void *)&info_user, sizeof(info_user))) { - dev_err(lwis_dev->dev, "Failed to copy %zu bytes to user\n", sizeof(info_user)); - return -EFAULT; - } - return err; -} - -static int ioctl_time_query(struct lwis_client *client, int64_t __user *msg) -{ - int ret = 0; - int64_t timestamp = ktime_to_ns(lwis_get_time()); - - if (copy_to_user((void __user *)msg, ×tamp, sizeof(timestamp))) { - ret = -EFAULT; - dev_err(client->lwis_dev->dev, "Failed to copy timestamp to userspace\n"); - } - - return ret; + info.header.ret_code = err; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&info, sizeof(info)); } -static int construct_io_entry(struct lwis_client *client, struct lwis_io_entry *user_entries, - size_t num_io_entries, struct lwis_io_entry **io_entries) +static int cmd_fake_event_inject(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_pkt __user *u_msg) { - int i; int ret = 0; - int last_buf_alloc_idx = -1; - size_t entry_size; - struct lwis_io_entry *k_entries; - uint8_t *user_buf; - uint8_t *k_buf; - struct lwis_device *lwis_dev = client->lwis_dev; + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + struct lwis_interrupt_list *list = lwis_dev->irqs; + int rt_irq; - entry_size = num_io_entries * sizeof(struct lwis_io_entry); - if (entry_size / sizeof(struct lwis_io_entry) != num_io_entries) { - dev_err(lwis_dev->dev, "Failed to prepare io entries due to integer overflow\n"); - return -EOVERFLOW; - } - k_entries = lwis_allocator_allocate(lwis_dev, entry_size); - if (!k_entries) { - dev_err(lwis_dev->dev, "Failed to allocate io entries\n"); - return -ENOMEM; + if (lwis_dev->type != DEVICE_TYPE_TEST || list->count != TEST_DEVICE_IRQ_CNT) { + return -EINVAL; } - if (copy_from_user((void *)k_entries, (void __user *)user_entries, entry_size)) { - ret = -EFAULT; - dev_err(lwis_dev->dev, "Failed to copy io entries from user\n"); - goto error_free_entries; + /* Fake Event Injection */ + rt_irq = lwis_fake_event_inject(&list->irq[0]); + if (rt_irq != TEST_DEVICE_FAKE_INJECTION_IRQ) { + dev_err(lwis_dev->dev, "Error fake injection: rt_irq = %d, expect rt_irq = %d\n", + rt_irq, TEST_DEVICE_FAKE_INJECTION_IRQ); + ret = -1; } - /* For batch writes, ened to allocate kernel buffers to deep copy the - * write values. Don't need to do this for batch reads because memory - * will be allocated in the form of lwis_io_result in io processing. - */ - for (i = 0; i < num_io_entries; ++i) { - if (k_entries[i].type == LWIS_IO_ENTRY_WRITE_BATCH) { - user_buf = k_entries[i].rw_batch.buf; - k_buf = lwis_allocator_allocate(lwis_dev, - k_entries[i].rw_batch.size_in_bytes); - if (!k_buf) { - dev_err_ratelimited(lwis_dev->dev, - "Failed to allocate io write buffer\n"); - ret = -ENOMEM; - goto error_free_buf; - } - last_buf_alloc_idx = i; - k_entries[i].rw_batch.buf = k_buf; - if (copy_from_user(k_buf, (void __user *)user_buf, - k_entries[i].rw_batch.size_in_bytes)) { - ret = -EFAULT; - dev_err_ratelimited(lwis_dev->dev, - "Failed to copy io write buffer from userspace\n"); - goto error_free_buf; - } - } - } - - *io_entries = k_entries; - return 0; - -error_free_buf: - for (i = 0; i <= last_buf_alloc_idx; ++i) { - if (k_entries[i].type == LWIS_IO_ENTRY_WRITE_BATCH) { - lwis_allocator_free(lwis_dev, k_entries[i].rw_batch.buf); - k_entries[i].rw_batch.buf = NULL; - } - } -error_free_entries: - lwis_allocator_free(lwis_dev, k_entries); - *io_entries = NULL; - return ret; + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int construct_transaction(struct lwis_client *client, - struct lwis_transaction_info __user *msg, - struct lwis_transaction **transaction) +static int construct_transaction_from_cmd(struct lwis_client *client, uint32_t cmd_id, + struct lwis_cmd_pkt __user *u_msg, + struct lwis_transaction **transaction) { int ret; + struct lwis_cmd_transaction_info k_info_v1; + struct lwis_cmd_transaction_info_v2 k_info_v2; struct lwis_transaction *k_transaction; - struct lwis_transaction_info *user_transaction; struct lwis_device *lwis_dev = client->lwis_dev; - k_transaction = kmalloc(sizeof(struct lwis_transaction), GFP_KERNEL); + k_transaction = kmalloc(sizeof(*k_transaction), GFP_KERNEL); if (!k_transaction) { dev_err(lwis_dev->dev, "Failed to allocate transaction info\n"); return -ENOMEM; } - user_transaction = (struct lwis_transaction_info *)msg; - if (copy_from_user((void *)&k_transaction->info, (void __user *)user_transaction, - sizeof(struct lwis_transaction_info))) { - ret = -EFAULT; - dev_err(lwis_dev->dev, "Failed to copy transaction info from user\n"); + if (cmd_id == LWIS_CMD_ID_TRANSACTION_SUBMIT_V2 || + cmd_id == LWIS_CMD_ID_TRANSACTION_REPLACE_V2) { + if (copy_from_user((void *)&k_info_v2, (void __user *)u_msg, sizeof(k_info_v2))) { + dev_err(lwis_dev->dev, "Failed to copy transaction info from user\n"); + ret = -EFAULT; + goto error_free_transaction; + } + memcpy(&k_transaction->info, &k_info_v2.info, sizeof(k_transaction->info)); + } else if (cmd_id == LWIS_CMD_ID_TRANSACTION_SUBMIT || + cmd_id == LWIS_CMD_ID_TRANSACTION_REPLACE) { + if (copy_from_user((void *)&k_info_v1, (void __user *)u_msg, sizeof(k_info_v1))) { + dev_err(lwis_dev->dev, "Failed to copy transaction info from user\n"); + ret = -EFAULT; + goto error_free_transaction; + } + k_transaction->info.trigger_event_id = k_info_v1.info.trigger_event_id; + k_transaction->info.trigger_event_counter = k_info_v1.info.trigger_event_counter; + k_transaction->info.num_io_entries = k_info_v1.info.num_io_entries; + k_transaction->info.io_entries = k_info_v1.info.io_entries; + k_transaction->info.run_in_event_context = k_info_v1.info.run_in_event_context; + k_transaction->info.reserved = k_info_v1.info.reserved; + k_transaction->info.emit_success_event_id = k_info_v1.info.emit_success_event_id; + k_transaction->info.emit_error_event_id = k_info_v1.info.emit_error_event_id; + k_transaction->info.is_level_triggered = k_info_v1.info.is_level_triggered; + k_transaction->info.id = k_info_v1.info.id; + k_transaction->info.current_trigger_event_counter = + k_info_v1.info.current_trigger_event_counter; + k_transaction->info.submission_timestamp_ns = + k_info_v1.info.submission_timestamp_ns; + + k_transaction->info.trigger_condition.num_nodes = 0; + k_transaction->info.trigger_condition.operator_type = + LWIS_TRIGGER_NODE_OPERATOR_INVALID; + k_transaction->info.completion_fence_fd = LWIS_NO_COMPLETION_FENCE; + } else { + dev_err(lwis_dev->dev, "Invalid command id for transaction\n"); + ret = -EINVAL; goto error_free_transaction; } @@ -1073,8 +1191,10 @@ static int construct_transaction(struct lwis_client *client, } k_transaction->resp = NULL; + k_transaction->is_weak_transaction = false; INIT_LIST_HEAD(&k_transaction->event_list_node); INIT_LIST_HEAD(&k_transaction->process_queue_node); + INIT_LIST_HEAD(&k_transaction->completion_fence_list); *transaction = k_transaction; return 0; @@ -1084,107 +1204,180 @@ error_free_transaction: return ret; } -static int ioctl_transaction_submit(struct lwis_client *client, - struct lwis_transaction_info __user *msg) +static int copy_transaction_info_v2_to_v1_locked(struct lwis_transaction_info_v2 *info_v2, + struct lwis_transaction_info *info_v1) +{ + if (!info_v2 || !info_v1) { + return -EINVAL; + } + + info_v1->trigger_event_id = info_v2->trigger_event_id; + info_v1->trigger_event_counter = info_v2->trigger_event_counter; + info_v1->num_io_entries = info_v2->num_io_entries; + info_v1->io_entries = info_v2->io_entries; + info_v1->run_in_event_context = info_v2->run_in_event_context; + info_v1->reserved = info_v2->reserved; + info_v1->emit_success_event_id = info_v2->emit_success_event_id; + info_v1->emit_error_event_id = info_v2->emit_error_event_id; + info_v1->is_level_triggered = info_v2->is_level_triggered; + info_v1->id = info_v2->id; + info_v1->current_trigger_event_counter = info_v2->current_trigger_event_counter; + info_v1->submission_timestamp_ns = info_v2->submission_timestamp_ns; + + return 0; +} + +static int cmd_transaction_submit(struct lwis_client *client, struct lwis_cmd_pkt *header, + struct lwis_cmd_pkt __user *u_msg) { - int ret = 0; - unsigned long flags; struct lwis_transaction *k_transaction = NULL; - struct lwis_transaction_info k_transaction_info; + struct lwis_cmd_transaction_info k_cmd_transaction_info_v1; + struct lwis_cmd_transaction_info_v2 k_cmd_transaction_info_v2; + struct lwis_cmd_pkt *resp_header = NULL; struct lwis_device *lwis_dev = client->lwis_dev; + int ret = 0; + unsigned long flags; - if (lwis_dev->type == DEVICE_TYPE_SLC) { + if (lwis_dev->type == DEVICE_TYPE_SLC || lwis_dev->type == DEVICE_TYPE_DPM) { dev_err(lwis_dev->dev, "not supported device type: %d\n", lwis_dev->type); - return -EINVAL; + ret = -EINVAL; + goto err_exit; + } + + ret = construct_transaction_from_cmd(client, header->cmd_id, u_msg, &k_transaction); + if (ret) { + goto err_exit; } - ret = construct_transaction(client, msg, &k_transaction); + ret = lwis_initialize_transaction_fences(client, k_transaction); if (ret) { - return ret; + lwis_transaction_free(lwis_dev, k_transaction); + goto err_exit; } spin_lock_irqsave(&client->transaction_lock, flags); ret = lwis_transaction_submit_locked(client, k_transaction); - k_transaction_info = k_transaction->info; + if (header->cmd_id == LWIS_CMD_ID_TRANSACTION_SUBMIT_V2) { + resp_header = &k_cmd_transaction_info_v2.header; + k_cmd_transaction_info_v2.info = k_transaction->info; + } else if (header->cmd_id == LWIS_CMD_ID_TRANSACTION_SUBMIT) { + resp_header = &k_cmd_transaction_info_v1.header; + ret = copy_transaction_info_v2_to_v1_locked(&k_transaction->info, + &k_cmd_transaction_info_v1.info); + } spin_unlock_irqrestore(&client->transaction_lock, flags); - if (ret) { - k_transaction_info.id = LWIS_ID_INVALID; + k_cmd_transaction_info_v1.info.id = LWIS_ID_INVALID; + k_cmd_transaction_info_v2.info.id = LWIS_ID_INVALID; lwis_transaction_free(lwis_dev, k_transaction); } - if (copy_to_user((void __user *)msg, &k_transaction_info, - sizeof(struct lwis_transaction_info))) { - ret = -EFAULT; - dev_err_ratelimited(lwis_dev->dev, - "Failed to copy transaction results to userspace\n"); + resp_header->cmd_id = header->cmd_id; + resp_header->next = header->next; + resp_header->ret_code = ret; + if (header->cmd_id == LWIS_CMD_ID_TRANSACTION_SUBMIT_V2) { + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&k_cmd_transaction_info_v2, + sizeof(k_cmd_transaction_info_v2)); + } else if (header->cmd_id == LWIS_CMD_ID_TRANSACTION_SUBMIT) { + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&k_cmd_transaction_info_v1, + sizeof(k_cmd_transaction_info_v1)); } - return ret; + ret = -EINVAL; + +err_exit: + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_transaction_replace(struct lwis_client *client, - struct lwis_transaction_info __user *msg) +static int cmd_transaction_cancel(struct lwis_client *client, struct lwis_cmd_pkt *header, + struct lwis_cmd_transaction_cancel __user *u_msg) { int ret = 0; - unsigned long flags; - struct lwis_transaction *k_transaction = NULL; - struct lwis_transaction_info k_transaction_info; + struct lwis_cmd_transaction_cancel k_msg; struct lwis_device *lwis_dev = client->lwis_dev; - ret = construct_transaction(client, msg, &k_transaction); - if (ret) { - return ret; + if (copy_from_user((void *)&k_msg, (void __user *)u_msg, sizeof(k_msg))) { + dev_err(lwis_dev->dev, "Failed to copy transaction ID from user\n"); + return -EFAULT; } - spin_lock_irqsave(&client->transaction_lock, flags); - ret = lwis_transaction_replace_locked(client, k_transaction); - k_transaction_info = k_transaction->info; - spin_unlock_irqrestore(&client->transaction_lock, flags); - + ret = lwis_transaction_cancel(client, k_msg.id); if (ret) { - k_transaction_info.id = LWIS_ID_INVALID; - lwis_transaction_free(lwis_dev, k_transaction); - } - - if (copy_to_user((void __user *)msg, &k_transaction_info, - sizeof(struct lwis_transaction_info))) { - ret = -EFAULT; - dev_err_ratelimited(lwis_dev->dev, - "Failed to copy transaction results to userspace\n"); + dev_info_ratelimited( + lwis_dev->dev, + "Transaction id 0x%llx does not exist or is already done, not available for cancel(%d)\n", + k_msg.id, ret); } - return ret; + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_transaction_cancel(struct lwis_client *client, int64_t __user *msg) +static int cmd_transaction_replace(struct lwis_client *client, struct lwis_cmd_pkt *header, + struct lwis_cmd_pkt __user *u_msg) { - int ret = 0; - int64_t id; + struct lwis_transaction *k_transaction = NULL; + struct lwis_cmd_transaction_info k_cmd_transaction_info_v1; + struct lwis_cmd_transaction_info_v2 k_cmd_transaction_info_v2; + struct lwis_cmd_pkt *resp_header = NULL; struct lwis_device *lwis_dev = client->lwis_dev; + int ret = 0; + unsigned long flags; - if (copy_from_user((void *)&id, (void __user *)msg, sizeof(id))) { - dev_err(lwis_dev->dev, "Failed to copy transaction ID from user\n"); - return -EFAULT; + ret = construct_transaction_from_cmd(client, header->cmd_id, u_msg, &k_transaction); + if (ret) { + goto err_exit; } - ret = lwis_transaction_cancel(client, id); + ret = lwis_initialize_transaction_fences(client, k_transaction); if (ret) { - dev_warn_ratelimited(lwis_dev->dev, "Failed to cancel transaction id 0x%llx (%d)\n", - id, ret); - return ret; + lwis_transaction_free(lwis_dev, k_transaction); + goto err_exit; } - return 0; + spin_lock_irqsave(&client->transaction_lock, flags); + ret = lwis_transaction_replace_locked(client, k_transaction); + if (header->cmd_id == LWIS_CMD_ID_TRANSACTION_REPLACE_V2) { + resp_header = &k_cmd_transaction_info_v2.header; + k_cmd_transaction_info_v2.info = k_transaction->info; + } else if (header->cmd_id == LWIS_CMD_ID_TRANSACTION_REPLACE) { + resp_header = &k_cmd_transaction_info_v1.header; + ret = copy_transaction_info_v2_to_v1_locked(&k_transaction->info, + &k_cmd_transaction_info_v1.info); + } + spin_unlock_irqrestore(&client->transaction_lock, flags); + if (ret) { + k_cmd_transaction_info_v1.info.id = LWIS_ID_INVALID; + k_cmd_transaction_info_v2.info.id = LWIS_ID_INVALID; + lwis_transaction_free(lwis_dev, k_transaction); + } + + resp_header->cmd_id = header->cmd_id; + resp_header->next = header->next; + resp_header->ret_code = ret; + if (header->cmd_id == LWIS_CMD_ID_TRANSACTION_REPLACE_V2) { + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&k_cmd_transaction_info_v2, + sizeof(k_cmd_transaction_info_v2)); + } else if (header->cmd_id == LWIS_CMD_ID_TRANSACTION_REPLACE) { + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&k_cmd_transaction_info_v1, + sizeof(k_cmd_transaction_info_v1)); + } + + ret = -EINVAL; + +err_exit: + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int construct_periodic_io(struct lwis_client *client, - struct lwis_periodic_io_info __user *msg, - struct lwis_periodic_io **periodic_io) +static int construct_periodic_io_from_cmd(struct lwis_client *client, + struct lwis_cmd_periodic_io_info __user *u_msg, + struct lwis_periodic_io **periodic_io) { int ret = 0; struct lwis_periodic_io *k_periodic_io; - struct lwis_periodic_io_info *user_periodic_io; + struct lwis_cmd_periodic_io_info k_info; struct lwis_device *lwis_dev = client->lwis_dev; k_periodic_io = kmalloc(sizeof(struct lwis_periodic_io), GFP_KERNEL); @@ -1193,14 +1386,14 @@ static int construct_periodic_io(struct lwis_client *client, return -ENOMEM; } - user_periodic_io = (struct lwis_periodic_io_info *)msg; - if (copy_from_user((void *)&k_periodic_io->info, (void __user *)user_periodic_io, - sizeof(struct lwis_periodic_io_info))) { - ret = -EFAULT; + if (copy_from_user((void *)&k_info, (void __user *)u_msg, sizeof(k_info))) { dev_err(lwis_dev->dev, "Failed to copy periodic io info from user\n"); + ret = -EFAULT; goto error_free_periodic_io; } + memcpy(&k_periodic_io->info, &k_info.info, sizeof(k_periodic_io->info)); + ret = construct_io_entry(client, k_periodic_io->info.io_entries, k_periodic_io->info.num_io_entries, &k_periodic_io->info.io_entries); @@ -1220,99 +1413,104 @@ error_free_periodic_io: return ret; } -static int ioctl_periodic_io_submit(struct lwis_client *client, - struct lwis_periodic_io_info __user *msg) +static int cmd_periodic_io_submit(struct lwis_client *client, struct lwis_cmd_pkt *header, + struct lwis_cmd_periodic_io_info __user *u_msg) { int ret = 0; + struct lwis_cmd_periodic_io_info k_periodic_io_info; struct lwis_periodic_io *k_periodic_io = NULL; struct lwis_device *lwis_dev = client->lwis_dev; - ret = construct_periodic_io(client, msg, &k_periodic_io); + ret = construct_periodic_io_from_cmd(client, u_msg, &k_periodic_io); if (ret) { - return ret; + goto err_exit; } ret = lwis_periodic_io_submit(client, k_periodic_io); + k_periodic_io_info.info = k_periodic_io->info; if (ret) { - k_periodic_io->info.id = LWIS_ID_INVALID; - if (copy_to_user((void __user *)msg, &k_periodic_io->info, - sizeof(struct lwis_periodic_io_info))) { - dev_err_ratelimited(lwis_dev->dev, "Failed to return info to userspace\n"); - } + k_periodic_io_info.info.id = LWIS_ID_INVALID; lwis_periodic_io_free(lwis_dev, k_periodic_io); - return ret; + goto err_exit; } - if (copy_to_user((void __user *)msg, &k_periodic_io->info, - sizeof(struct lwis_periodic_io_info))) { - dev_err_ratelimited(lwis_dev->dev, - "Failed to copy periodic io results to userspace\n"); - return -EFAULT; - } + k_periodic_io_info.header.cmd_id = header->cmd_id; + k_periodic_io_info.header.next = header->next; + k_periodic_io_info.header.ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&k_periodic_io_info, + sizeof(k_periodic_io_info)); - return ret; +err_exit: + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_periodic_io_cancel(struct lwis_client *client, int64_t __user *msg) +static int cmd_periodic_io_cancel(struct lwis_client *client, struct lwis_cmd_pkt *header, + struct lwis_cmd_periodic_io_cancel __user *u_msg) { int ret = 0; - int64_t id; + struct lwis_cmd_periodic_io_cancel k_msg; struct lwis_device *lwis_dev = client->lwis_dev; - if (copy_from_user((void *)&id, (void __user *)msg, sizeof(id))) { + if (copy_from_user((void *)&k_msg, (void __user *)u_msg, sizeof(k_msg))) { dev_err(lwis_dev->dev, "Failed to copy periodic io ID from user\n"); return -EFAULT; } - ret = lwis_periodic_io_cancel(client, id); + ret = lwis_periodic_io_cancel(client, k_msg.id); if (ret) { - dev_err_ratelimited(lwis_dev->dev, "Failed to clear periodic io id 0x%llx\n", id); - return ret; + dev_err_ratelimited(lwis_dev->dev, "Failed to clear periodic io id 0x%llx\n", + k_msg.id); } - return 0; + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_dpm_clk_update(struct lwis_device *lwis_dev, - struct lwis_dpm_clk_settings __user *msg) +static int cmd_dpm_clk_update(struct lwis_device *lwis_dev, struct lwis_cmd_pkt *header, + struct lwis_cmd_dpm_clk_update __user *u_msg) { int ret; - struct lwis_dpm_clk_settings k_msg; + struct lwis_cmd_dpm_clk_update k_msg; struct lwis_clk_setting *clk_settings; size_t buf_size; - if (copy_from_user((void *)&k_msg, (void __user *)msg, - sizeof(struct lwis_dpm_clk_settings))) { + if (copy_from_user((void *)&k_msg, (void __user *)u_msg, sizeof(k_msg))) { dev_err(lwis_dev->dev, "Failed to copy ioctl message from user\n"); return -EFAULT; } - buf_size = sizeof(struct lwis_clk_setting) * k_msg.num_settings; - if (buf_size / sizeof(struct lwis_clk_setting) != k_msg.num_settings) { + buf_size = sizeof(struct lwis_clk_setting) * k_msg.settings.num_settings; + if (buf_size / sizeof(struct lwis_clk_setting) != k_msg.settings.num_settings) { dev_err(lwis_dev->dev, "Failed to copy clk settings due to integer overflow.\n"); - return -EOVERFLOW; + ret = -EOVERFLOW; + goto exit; } clk_settings = kmalloc(buf_size, GFP_KERNEL); if (!clk_settings) { dev_err(lwis_dev->dev, "Failed to allocate clock settings\n"); - return -ENOMEM; + ret = -ENOMEM; + goto exit; } - if (copy_from_user(clk_settings, (void __user *)k_msg.settings, buf_size)) { + if (copy_from_user(clk_settings, (void __user *)k_msg.settings.settings, buf_size)) { dev_err(lwis_dev->dev, "Failed to copy clk settings from user\n"); kfree(clk_settings); - return -EFAULT; + ret = -EFAULT; + goto exit; } - ret = lwis_dpm_update_clock(lwis_dev, clk_settings, k_msg.num_settings); + ret = lwis_dpm_update_clock(lwis_dev, clk_settings, k_msg.settings.num_settings); kfree(clk_settings); - return ret; +exit: + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_dpm_qos_update(struct lwis_device *lwis_dev, - struct lwis_dpm_qos_requirements __user *msg) +static int cmd_dpm_qos_update(struct lwis_device *lwis_dev, struct lwis_cmd_pkt *header, + struct lwis_cmd_dpm_qos_update __user *u_msg) { - struct lwis_dpm_qos_requirements k_msg; + struct lwis_cmd_dpm_qos_update k_msg; struct lwis_qos_setting *k_qos_settings; int ret = 0; int i; @@ -1320,191 +1518,414 @@ static int ioctl_dpm_qos_update(struct lwis_device *lwis_dev, if (lwis_dev->type != DEVICE_TYPE_DPM) { dev_err(lwis_dev->dev, "not supported device type: %d\n", lwis_dev->type); - return -EINVAL; + ret = -EINVAL; + goto exit; } - if (copy_from_user((void *)&k_msg, (void __user *)msg, - sizeof(struct lwis_dpm_qos_requirements))) { + if (copy_from_user((void *)&k_msg, (void __user *)u_msg, sizeof(k_msg))) { dev_err(lwis_dev->dev, "Failed to copy ioctl message from user\n"); return -EFAULT; } // Copy qos settings from user buffer. - buf_size = sizeof(struct lwis_qos_setting) * k_msg.num_settings; - if (buf_size / sizeof(struct lwis_qos_setting) != k_msg.num_settings) { + buf_size = sizeof(struct lwis_qos_setting) * k_msg.reqs.num_settings; + if (buf_size / sizeof(struct lwis_qos_setting) != k_msg.reqs.num_settings) { dev_err(lwis_dev->dev, "Failed to copy qos settings due to integer overflow.\n"); - return -EOVERFLOW; + ret = -EOVERFLOW; + goto exit; } k_qos_settings = kmalloc(buf_size, GFP_KERNEL); if (!k_qos_settings) { dev_err(lwis_dev->dev, "Failed to allocate qos settings\n"); - return -ENOMEM; + ret = -ENOMEM; + goto exit; } - if (copy_from_user(k_qos_settings, (void __user *)k_msg.qos_settings, buf_size)) { + if (copy_from_user(k_qos_settings, (void __user *)k_msg.reqs.qos_settings, buf_size)) { + dev_err(lwis_dev->dev, "Failed to copy clk settings from user\n"); + kfree(k_qos_settings); ret = -EFAULT; + goto exit; + } + + for (i = 0; i < k_msg.reqs.num_settings; i++) { + if (sizeof(struct lwis_qos_setting) != sizeof(struct lwis_qos_setting_v2)) { + struct lwis_qos_setting_v2 k_qos_setting_v2; + memcpy(&k_qos_setting_v2, &k_qos_settings[i], + sizeof(struct lwis_qos_setting)); + k_qos_setting_v2.bts_block_name[0] = '\0'; + ret = lwis_dpm_update_qos(lwis_dev, &k_qos_setting_v2); + } else { + ret = lwis_dpm_update_qos(lwis_dev, + (struct lwis_qos_setting_v2 *)&k_qos_settings[i]); + } + if (ret) { + dev_err(lwis_dev->dev, "Failed to apply qos setting, ret: %d\n", ret); + kfree(k_qos_settings); + goto exit; + } + } + kfree(k_qos_settings); +exit: + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); +} + +static int cmd_dpm_qos_update_v2(struct lwis_device *lwis_dev, struct lwis_cmd_pkt *header, + struct lwis_cmd_dpm_qos_update_v2 __user *u_msg) +{ + struct lwis_cmd_dpm_qos_update_v2 k_msg; + struct lwis_qos_setting_v2 *k_qos_settings; + int ret = 0; + int i; + size_t buf_size; + + if (lwis_dev->type != DEVICE_TYPE_DPM) { + dev_err(lwis_dev->dev, "not supported device type: %d\n", lwis_dev->type); + ret = -EINVAL; + goto exit; + } + + if (copy_from_user((void *)&k_msg, (void __user *)u_msg, sizeof(k_msg))) { + dev_err(lwis_dev->dev, "Failed to copy ioctl message from user\n"); + return -EFAULT; + } + + // Copy qos settings from user buffer. + buf_size = sizeof(struct lwis_qos_setting_v2) * k_msg.reqs.num_settings; + if (buf_size / sizeof(struct lwis_qos_setting_v2) != k_msg.reqs.num_settings) { + dev_err(lwis_dev->dev, "Failed to copy qos settings due to integer overflow.\n"); + ret = -EOVERFLOW; + goto exit; + } + k_qos_settings = kmalloc(buf_size, GFP_KERNEL); + if (!k_qos_settings) { + dev_err(lwis_dev->dev, "Failed to allocate qos settings\n"); + ret = -ENOMEM; + goto exit; + } + if (copy_from_user(k_qos_settings, (void __user *)k_msg.reqs.qos_settings, buf_size)) { dev_err(lwis_dev->dev, "Failed to copy clk settings from user\n"); - goto out; + kfree(k_qos_settings); + ret = -EFAULT; + goto exit; } - for (i = 0; i < k_msg.num_settings; i++) { + for (i = 0; i < k_msg.reqs.num_settings; i++) { ret = lwis_dpm_update_qos(lwis_dev, &k_qos_settings[i]); if (ret) { dev_err(lwis_dev->dev, "Failed to apply qos setting, ret: %d\n", ret); - goto out; + kfree(k_qos_settings); + goto exit; } } -out: kfree(k_qos_settings); - return ret; +exit: + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -static int ioctl_dpm_get_clock(struct lwis_device *lwis_dev, struct lwis_qos_setting __user *msg) +static int cmd_dpm_get_clock(struct lwis_device *lwis_dev, struct lwis_cmd_pkt *header, + struct lwis_cmd_dpm_clk_get __user *u_msg) { - struct lwis_qos_setting current_setting; + struct lwis_cmd_dpm_clk_get current_setting; struct lwis_device *target_device; + int ret = 0; if (lwis_dev->type != DEVICE_TYPE_DPM) { dev_err(lwis_dev->dev, "not supported device type: %d\n", lwis_dev->type); - return -EINVAL; + ret = -EINVAL; + goto err_exit; } - if (copy_from_user((void *)¤t_setting, (void __user *)msg, - sizeof(struct lwis_qos_setting))) { + if (copy_from_user((void *)¤t_setting, (void __user *)u_msg, + sizeof(current_setting))) { dev_err(lwis_dev->dev, "failed to copy from user\n"); return -EFAULT; } - target_device = lwis_find_dev_by_id(current_setting.device_id); + target_device = lwis_find_dev_by_id(current_setting.setting.device_id); if (!target_device) { dev_err(lwis_dev->dev, "could not find lwis device by id %d\n", - current_setting.device_id); - return -ENODEV; + current_setting.setting.device_id); + ret = -ENODEV; + goto err_exit; } if (target_device->enabled == 0 && target_device->type != DEVICE_TYPE_DPM) { dev_warn(target_device->dev, "%s disabled, can't get clk\n", target_device->name); - return -EPERM; + ret = -EPERM; + goto err_exit; } - current_setting.frequency_hz = (int64_t)lwis_dpm_read_clock(target_device); - if (copy_to_user((void __user *)msg, ¤t_setting, sizeof(struct lwis_qos_setting))) { - dev_err(lwis_dev->dev, "failed to copy to user\n"); - return -EFAULT; - } + current_setting.setting.frequency_hz = (int64_t)lwis_dpm_read_clock(target_device); + current_setting.header.ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)¤t_setting, sizeof(current_setting)); - return 0; +err_exit: + header->ret_code = ret; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } -int lwis_ioctl_handler(struct lwis_client *lwis_client, unsigned int type, unsigned long param) +static int cmd_fence_create(struct lwis_device *lwis_dev, struct lwis_cmd_pkt *header, + struct lwis_cmd_fence_create __user *u_msg) { - int ret = 0; - bool device_disabled; - struct lwis_device *lwis_dev = lwis_client->lwis_dev; + int32_t fd_or_err; + struct lwis_cmd_fence_create fence_create; - // Skip the lock for LWIS_EVENT_DEQUEUE because we want to emit events ASAP. The internal - // handler function of LWIS_EVENT_DEQUEUE will acquire the necessary lock. - if (type != LWIS_EVENT_DEQUEUE) { - mutex_lock(&lwis_client->lock); + if (copy_from_user((void *)&fence_create, (void __user *)u_msg, sizeof(fence_create))) { + dev_err(lwis_dev->dev, "failed to copy from user\n"); + return -EFAULT; } - mutex_lock(&lwis_dev->client_lock); - device_disabled = (lwis_dev->enabled == 0); - mutex_unlock(&lwis_dev->client_lock); - /* Buffer dis/enroll is added here temporarily. Will need a proper - fix to ensure buffer enrollment when device is enabled. */ - if (lwis_dev->type != DEVICE_TYPE_TOP && device_disabled && type != LWIS_GET_DEVICE_INFO && - type != LWIS_DEVICE_ENABLE && type != LWIS_DEVICE_RESET && - type != LWIS_EVENT_CONTROL_GET && type != LWIS_TIME_QUERY && - type != LWIS_EVENT_DEQUEUE && type != LWIS_BUFFER_ENROLL && - type != LWIS_BUFFER_DISENROLL && type != LWIS_BUFFER_FREE && - type != LWIS_DPM_QOS_UPDATE && type != LWIS_DPM_GET_CLOCK) { - ret = -EBADFD; - dev_err_ratelimited(lwis_dev->dev, "Unsupported IOCTL on disabled device.\n"); - goto out; + fd_or_err = lwis_fence_create(lwis_dev); + if (fd_or_err < 0) { + header->ret_code = fd_or_err; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)header, sizeof(*header)); } - switch (type) { - case LWIS_GET_DEVICE_INFO: - ret = ioctl_get_device_info(lwis_dev, (struct lwis_device_info *)param); + fence_create.fd = fd_or_err; + fence_create.header.ret_code = 0; + return copy_pkt_to_user(lwis_dev, u_msg, (void *)&fence_create, sizeof(fence_create)); +} + +static int handle_cmd_pkt(struct lwis_client *lwis_client, struct lwis_cmd_pkt *header, + struct lwis_cmd_pkt __user *user_msg) +{ + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + int ret = 0; + + switch (header->cmd_id) { + case LWIS_CMD_ID_ECHO: + ret = cmd_echo(lwis_dev, header, (struct lwis_cmd_echo __user *)user_msg); break; - case LWIS_BUFFER_ALLOC: - ret = ioctl_buffer_alloc(lwis_client, (struct lwis_alloc_buffer_info *)param); + case LWIS_CMD_ID_TIME_QUERY: + ret = cmd_time_query(lwis_dev, header, + (struct lwis_cmd_time_query __user *)user_msg); break; - case LWIS_BUFFER_FREE: - ret = ioctl_buffer_free(lwis_client, (int *)param); + case LWIS_CMD_ID_GET_DEVICE_INFO: + mutex_lock(&lwis_client->lock); + ret = cmd_get_device_info(lwis_dev, header, + (struct lwis_cmd_device_info __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_BUFFER_ENROLL: - ret = ioctl_buffer_enroll(lwis_client, (struct lwis_buffer_info *)param); + case LWIS_CMD_ID_DEVICE_ENABLE: + mutex_lock(&lwis_client->lock); + ret = cmd_device_enable(lwis_client, header, + (struct lwis_cmd_pkt __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_BUFFER_DISENROLL: - ret = ioctl_buffer_disenroll(lwis_client, - (struct lwis_enrolled_buffer_info *)param); + case LWIS_CMD_ID_DEVICE_DISABLE: + mutex_lock(&lwis_client->lock); + ret = cmd_device_disable(lwis_client, header, + (struct lwis_cmd_pkt __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_BUFFER_CPU_ACCESS: - ret = ioctl_buffer_cpu_access(lwis_client, - (struct lwis_buffer_cpu_access_op *)param); + case LWIS_CMD_ID_DEVICE_RESET: + mutex_lock(&lwis_client->lock); + ret = cmd_device_reset(lwis_client, header, + (struct lwis_cmd_io_entries __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_REG_IO: - ret = ioctl_reg_io(lwis_dev, (struct lwis_io_entries *)param); + case LWIS_CMD_ID_DEVICE_SUSPEND: + mutex_lock(&lwis_client->lock); + ret = cmd_device_suspend(lwis_client, header, + (struct lwis_cmd_pkt __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_DEVICE_ENABLE: - ret = ioctl_device_enable(lwis_client); + case LWIS_CMD_ID_DEVICE_RESUME: + mutex_lock(&lwis_client->lock); + ret = cmd_device_resume(lwis_client, header, + (struct lwis_cmd_pkt __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_DEVICE_DISABLE: - ret = ioctl_device_disable(lwis_client); + case LWIS_CMD_ID_DUMP_DEBUG_STATE: + ret = cmd_dump_debug_state(lwis_client, header, + (struct lwis_cmd_pkt __user *)user_msg); break; - case LWIS_ECHO: - ret = ioctl_echo(lwis_dev, (struct lwis_echo *)param); + case LWIS_CMD_ID_DMA_BUFFER_ENROLL: + mutex_lock(&lwis_client->lock); + ret = cmd_dma_buffer_enroll(lwis_client, header, + (struct lwis_cmd_dma_buffer_enroll __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_DEVICE_RESET: - ret = ioctl_device_reset(lwis_client, (struct lwis_io_entries *)param); + case LWIS_CMD_ID_DMA_BUFFER_DISENROLL: + mutex_lock(&lwis_client->lock); + ret = cmd_dma_buffer_disenroll( + lwis_client, header, + (struct lwis_cmd_dma_buffer_disenroll __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_EVENT_CONTROL_GET: - ret = ioctl_event_control_get(lwis_client, (struct lwis_event_control *)param); + case LWIS_CMD_ID_DMA_BUFFER_CPU_ACCESS: + mutex_lock(&lwis_client->lock); + ret = cmd_dma_buffer_cpu_access( + lwis_client, header, + (struct lwis_cmd_dma_buffer_cpu_access __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_EVENT_CONTROL_SET: - ret = ioctl_event_control_set(lwis_client, (struct lwis_event_control_list *)param); + case LWIS_CMD_ID_DMA_BUFFER_ALLOC: + mutex_lock(&lwis_client->lock); + ret = cmd_dma_buffer_alloc(lwis_client, header, + (struct lwis_cmd_dma_buffer_alloc __user *)user_msg); + mutex_unlock(&lwis_client->lock); + break; + case LWIS_CMD_ID_DMA_BUFFER_FREE: + mutex_lock(&lwis_client->lock); + ret = cmd_dma_buffer_free(lwis_client, header, + (struct lwis_cmd_dma_buffer_free __user *)user_msg); + mutex_unlock(&lwis_client->lock); + break; + case LWIS_CMD_ID_REG_IO: + mutex_lock(&lwis_client->lock); + ret = cmd_reg_io(lwis_dev, header, (struct lwis_cmd_io_entries __user *)user_msg); + mutex_unlock(&lwis_client->lock); + break; + case LWIS_CMD_ID_EVENT_CONTROL_GET: + mutex_lock(&lwis_client->lock); + ret = cmd_event_control_get(lwis_client, header, + (struct lwis_cmd_event_control_get __user *)user_msg); + mutex_unlock(&lwis_client->lock); + break; + case LWIS_CMD_ID_EVENT_CONTROL_SET: + mutex_lock(&lwis_client->lock); + ret = cmd_event_control_set(lwis_client, header, + (struct lwis_cmd_event_control_set __user *)user_msg); + mutex_unlock(&lwis_client->lock); + break; + case LWIS_CMD_ID_EVENT_DEQUEUE: + ret = cmd_event_dequeue(lwis_client, header, + (struct lwis_cmd_event_dequeue __user *)user_msg); + break; + case LWIS_CMD_ID_TRANSACTION_SUBMIT: + case LWIS_CMD_ID_TRANSACTION_SUBMIT_V2: + mutex_lock(&lwis_client->lock); + ret = cmd_transaction_submit(lwis_client, header, + (struct lwis_cmd_pkt __user *)user_msg); + mutex_unlock(&lwis_client->lock); + break; + case LWIS_CMD_ID_TRANSACTION_CANCEL: + mutex_lock(&lwis_client->lock); + ret = cmd_transaction_cancel(lwis_client, header, + (struct lwis_cmd_transaction_cancel __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_EVENT_DEQUEUE: - ret = ioctl_event_dequeue(lwis_client, (struct lwis_event_info *)param); + case LWIS_CMD_ID_TRANSACTION_REPLACE: + case LWIS_CMD_ID_TRANSACTION_REPLACE_V2: + mutex_lock(&lwis_client->lock); + ret = cmd_transaction_replace(lwis_client, header, + (struct lwis_cmd_pkt __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_TIME_QUERY: - ret = ioctl_time_query(lwis_client, (int64_t *)param); + case LWIS_CMD_ID_PERIODIC_IO_SUBMIT: + mutex_lock(&lwis_client->lock); + ret = cmd_periodic_io_submit(lwis_client, header, + (struct lwis_cmd_periodic_io_info __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_TRANSACTION_SUBMIT: - ret = ioctl_transaction_submit(lwis_client, (struct lwis_transaction_info *)param); + case LWIS_CMD_ID_PERIODIC_IO_CANCEL: + mutex_lock(&lwis_client->lock); + ret = cmd_periodic_io_cancel(lwis_client, header, + (struct lwis_cmd_periodic_io_cancel __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_TRANSACTION_CANCEL: - ret = ioctl_transaction_cancel(lwis_client, (int64_t *)param); + case LWIS_CMD_ID_DPM_CLK_UPDATE: + mutex_lock(&lwis_client->lock); + ret = cmd_dpm_clk_update(lwis_dev, header, + (struct lwis_cmd_dpm_clk_update __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_TRANSACTION_REPLACE: - ret = ioctl_transaction_replace(lwis_client, (struct lwis_transaction_info *)param); + case LWIS_CMD_ID_DPM_QOS_UPDATE: + mutex_lock(&lwis_client->lock); + ret = cmd_dpm_qos_update(lwis_dev, header, + (struct lwis_cmd_dpm_qos_update __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_PERIODIC_IO_SUBMIT: - ret = ioctl_periodic_io_submit(lwis_client, (struct lwis_periodic_io_info *)param); + case LWIS_CMD_ID_DPM_QOS_UPDATE_V2: + mutex_lock(&lwis_client->lock); + ret = cmd_dpm_qos_update_v2(lwis_dev, header, + (struct lwis_cmd_dpm_qos_update_v2 __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_PERIODIC_IO_CANCEL: - ret = ioctl_periodic_io_cancel(lwis_client, (int64_t *)param); + case LWIS_CMD_ID_DPM_GET_CLOCK: + mutex_lock(&lwis_client->lock); + ret = cmd_dpm_get_clock(lwis_dev, header, + (struct lwis_cmd_dpm_clk_get __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_DPM_CLK_UPDATE: - ret = ioctl_dpm_clk_update(lwis_dev, (struct lwis_dpm_clk_settings *)param); + case LWIS_CMD_ID_FENCE_CREATE: + ret = cmd_fence_create(lwis_dev, header, + (struct lwis_cmd_fence_create __user *)user_msg); break; - case LWIS_DPM_QOS_UPDATE: - ret = ioctl_dpm_qos_update(lwis_dev, (struct lwis_dpm_qos_requirements *)param); + case LWIS_CMD_ID_EVENT_INJECTION: + mutex_lock(&lwis_client->lock); + ret = cmd_fake_event_inject(lwis_client, header, + (struct lwis_cmd_pkt __user *)user_msg); + mutex_unlock(&lwis_client->lock); break; - case LWIS_DPM_GET_CLOCK: - ret = ioctl_dpm_get_clock(lwis_dev, (struct lwis_qos_setting *)param); + default: + dev_err_ratelimited(lwis_dev->dev, "Unknown command id 0x%x\n", header->cmd_id); + header->ret_code = -ENOSYS; + ret = copy_pkt_to_user(lwis_dev, user_msg, (void *)header, sizeof(*header)); + } + + return ret; +} + +static int lwis_ioctl_handle_cmd_pkt(struct lwis_client *lwis_client, + struct lwis_cmd_pkt __user *user_msg) +{ + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + struct lwis_cmd_pkt header; + int ret = 0; + bool device_disabled; + + while (user_msg) { + /* Copy cmd packet header from userspace */ + if (copy_from_user(&header, (void __user *)user_msg, sizeof(header))) { + dev_err(lwis_dev->dev, + "Failed to copy cmd packet header from userspace.\n"); + return -EFAULT; + } + + mutex_lock(&lwis_dev->client_lock); + device_disabled = (lwis_dev->enabled == 0); + mutex_unlock(&lwis_dev->client_lock); + if (lwis_dev->type != DEVICE_TYPE_TOP && device_disabled && + (header.cmd_id == LWIS_CMD_ID_DMA_BUFFER_ALLOC || + header.cmd_id == LWIS_CMD_ID_REG_IO || + header.cmd_id == LWIS_CMD_ID_TRANSACTION_SUBMIT || + header.cmd_id == LWIS_CMD_ID_TRANSACTION_REPLACE || + header.cmd_id == LWIS_CMD_ID_PERIODIC_IO_SUBMIT)) { + dev_err_ratelimited(lwis_dev->dev, + "Unsupported IOCTL on disabled device.\n"); + header.ret_code = -EBADFD; + return copy_pkt_to_user(lwis_dev, user_msg, (void *)&header, + sizeof(header)); + } + + ret = handle_cmd_pkt(lwis_client, &header, user_msg); + if (ret) { + return ret; + } + user_msg = header.next; + } + + return ret; +} + +int lwis_ioctl_handler(struct lwis_client *lwis_client, unsigned int type, unsigned long param) +{ + int ret = 0; + struct lwis_device *lwis_dev = lwis_client->lwis_dev; + + switch (type) { + case LWIS_CMD_PACKET: + ret = lwis_ioctl_handle_cmd_pkt(lwis_client, (struct lwis_cmd_pkt *)param); break; default: dev_err_ratelimited(lwis_dev->dev, "Unknown IOCTL operation\n"); ret = -EINVAL; }; -out: - if (type != LWIS_EVENT_DEQUEUE) { - mutex_unlock(&lwis_client->lock); - } - if (ret && ret != -ENOENT && ret != -ETIMEDOUT && ret != -EAGAIN) { lwis_ioctl_pr_err(lwis_dev, type, ret); } diff --git a/lwis_ioreg.c b/lwis_ioreg.c index 0a8478c..ce8ee75 100644 --- a/lwis_ioreg.c +++ b/lwis_ioreg.c @@ -17,6 +17,7 @@ #include "lwis_device.h" #include "lwis_ioreg.h" +#include "lwis_util.h" static int find_block_idx_by_name(struct lwis_ioreg_list *list, char *name) { @@ -270,28 +271,28 @@ static int ioreg_write_batch_internal(void __iomem *base, uint64_t offset, int v case 8: for (i = 0; i < size_in_bytes; ++i) { writeb_relaxed(*(buf + i), is_offset_fixed ? (void __iomem *)(addr) : - (void __iomem *)(addr + i)); + (void __iomem *)(addr + i)); } break; case 16: for (i = 0; i < size_in_bytes; i += 2) { writew_relaxed(*(uint16_t *)(buf + i), is_offset_fixed ? (void __iomem *)(addr) : - (void __iomem *)(addr + i)); + (void __iomem *)(addr + i)); } break; case 32: for (i = 0; i < size_in_bytes; i += 4) { writel_relaxed(*(uint32_t *)(buf + i), is_offset_fixed ? (void __iomem *)(addr) : - (void __iomem *)(addr + i)); + (void __iomem *)(addr + i)); } break; case 64: for (i = 0; i < size_in_bytes; i += 8) { writeq_relaxed(*(uint64_t *)(buf + i), is_offset_fixed ? (void __iomem *)(addr) : - (void __iomem *)(addr + i)); + (void __iomem *)(addr + i)); } break; default: @@ -589,4 +590,4 @@ int lwis_ioreg_set_io_barrier(struct lwis_ioreg_device *ioreg_dev, bool use_read dma_wmb(); } return 0; -} +}
\ No newline at end of file diff --git a/lwis_periodic_io.c b/lwis_periodic_io.c index 699a7fe..442153e 100644 --- a/lwis_periodic_io.c +++ b/lwis_periodic_io.c @@ -15,6 +15,7 @@ #include <linux/completion.h> #include <linux/kthread.h> #include <linux/slab.h> +#include <linux/workqueue.h> #include "lwis_allocator.h" #include "lwis_event.h" @@ -42,8 +43,8 @@ static enum hrtimer_restart periodic_io_timer_func(struct hrtimer *timer) list_for_each_safe (it_period, it_period_tmp, &periodic_io_list->list) { periodic_io = list_entry(it_period, struct lwis_periodic_io, timer_list_node); if (periodic_io->active) { - periodic_io_proxy = - kmalloc(sizeof(struct lwis_periodic_io_proxy), GFP_NOWAIT); + periodic_io_proxy = lwis_allocator_allocate( + client->lwis_dev, sizeof(*periodic_io_proxy), GFP_ATOMIC); if (!periodic_io_proxy) { /* Non-fatal, skip this period */ pr_warn("Cannot allocate new periodic io proxy.\n"); @@ -56,8 +57,8 @@ static enum hrtimer_restart periodic_io_timer_func(struct hrtimer *timer) } } if (active_periodic_io_present) { - kthread_queue_work(&client->lwis_dev->periodic_io_worker, - &client->periodic_io_work); + kthread_queue_work(&client->lwis_dev->transaction_worker, + &client->transaction_work); } spin_unlock_irqrestore(&client->periodic_io_lock, flags); if (!active_periodic_io_present) { @@ -187,8 +188,7 @@ static int process_io_entries(struct lwis_client *client, /* Abort if periodic io is deactivated during processing. * Abort can only apply to <= 1 write entries to prevent partial writes, * or we just started the process. */ - if (!periodic_io->active && - (i == 0 || !periodic_io->contains_multiple_writes)) { + if (!periodic_io->active && (i == 0 || !periodic_io->contains_multiple_writes)) { resp->error_code = -ECANCELED; goto event_push; } @@ -233,7 +233,7 @@ static int process_io_entries(struct lwis_client *client, read_buf += sizeof(struct lwis_periodic_io_result) + io_result->io_result.num_value_bytes; } else if (entry->type == LWIS_IO_ENTRY_POLL) { - ret = lwis_io_entry_poll(lwis_dev, entry, /*non_blocking=*/false); + ret = lwis_io_entry_poll(lwis_dev, entry); if (ret) { resp->error_code = ret; goto event_push; @@ -297,14 +297,13 @@ event_push: return ret; } -static void periodic_io_work_func(struct kthread_work *work) +void lwis_process_periodic_io_in_queue(struct lwis_client *client) { int error_code; unsigned long flags; struct lwis_periodic_io *periodic_io; struct lwis_periodic_io_proxy *periodic_io_proxy; struct list_head *it_period, *it_period_tmp; - struct lwis_client *client = container_of(work, struct lwis_client, periodic_io_work); struct list_head pending_events; INIT_LIST_HEAD(&pending_events); @@ -317,7 +316,7 @@ static void periodic_io_work_func(struct kthread_work *work) /* Error indicates the cancellation of the periodic io */ if (periodic_io->resp->error_code || !periodic_io->active) { error_code = periodic_io->resp->error_code ? periodic_io->resp->error_code : - -ECANCELED; + -ECANCELED; push_periodic_io_error_event_locked(periodic_io, error_code, &pending_events); } else { @@ -325,12 +324,10 @@ static void periodic_io_work_func(struct kthread_work *work) process_io_entries(client, periodic_io_proxy, &pending_events); spin_lock_irqsave(&client->periodic_io_lock, flags); } - kfree(periodic_io_proxy); + lwis_allocator_free(client->lwis_dev, periodic_io_proxy); } spin_unlock_irqrestore(&client->periodic_io_lock, flags); - - lwis_pending_events_emit(client->lwis_dev, &pending_events, - /*in_irq=*/false); + lwis_pending_events_emit(client->lwis_dev, &pending_events); } static int prepare_emit_events(struct lwis_client *client, struct lwis_periodic_io *periodic_io) @@ -445,7 +442,6 @@ void lwis_periodic_io_free(struct lwis_device *lwis_dev, struct lwis_periodic_io int lwis_periodic_io_init(struct lwis_client *client) { INIT_LIST_HEAD(&client->periodic_io_process_queue); - kthread_init_work(&client->periodic_io_work, periodic_io_work_func); client->periodic_io_counter = 0; hash_init(client->timer_list); return 0; @@ -513,8 +509,8 @@ int lwis_periodic_io_client_flush(struct lwis_client *client) } /* Wait until all workload in process queue are processed */ - if (client->lwis_dev->periodic_io_worker_thread) { - kthread_flush_worker(&client->lwis_dev->periodic_io_worker); + if (client->lwis_dev->transaction_worker_thread) { + kthread_flush_worker(&client->lwis_dev->transaction_worker); } spin_lock_irqsave(&client->periodic_io_lock, flags); @@ -561,7 +557,8 @@ static int mark_periodic_io_resp_error_locked(struct lwis_periodic_io *periodic_ } /* Calling this function requires holding the client's periodic_io_lock */ -static struct lwis_periodic_io * periodic_io_find_locked(struct lwis_client *client, int64_t id) { +static struct lwis_periodic_io *periodic_io_find_locked(struct lwis_client *client, int64_t id) +{ int i; struct hlist_node *tmp; struct list_head *it_period, *it_period_tmp; diff --git a/lwis_periodic_io.h b/lwis_periodic_io.h index dbec999..8661638 100644 --- a/lwis_periodic_io.h +++ b/lwis_periodic_io.h @@ -82,5 +82,6 @@ int lwis_periodic_io_client_cleanup(struct lwis_client *client); int lwis_periodic_io_submit(struct lwis_client *client, struct lwis_periodic_io *periodic_io); int lwis_periodic_io_cancel(struct lwis_client *client, int64_t id); void lwis_periodic_io_free(struct lwis_device *lwis_dev, struct lwis_periodic_io *periodic_io); +void lwis_process_periodic_io_in_queue(struct lwis_client *client); #endif /* LWIS_PERIODIC_IO_H_ */
\ No newline at end of file @@ -83,7 +83,7 @@ int lwis_phy_get(struct lwis_phy_list *list, char *name, struct device *dev) /* Make sure PHY exists */ phy = devm_phy_get(dev, name); - if (IS_ERR(phy)) { + if (IS_ERR_OR_NULL(phy)) { pr_err("PHY %s not found\n", name); return PTR_ERR(phy); } diff --git a/lwis_pinctrl.c b/lwis_pinctrl.c index f8e758d..170912e 100644 --- a/lwis_pinctrl.c +++ b/lwis_pinctrl.c @@ -25,7 +25,7 @@ int lwis_pinctrl_set_state(struct pinctrl *pc, char *state_str) } state = pinctrl_lookup_state(pc, state_str); - if (IS_ERR(state)) { + if (IS_ERR_OR_NULL(state)) { pr_err("Cannot find mclk state %s\n", state_str); return PTR_ERR(state); } diff --git a/lwis_platform.h b/lwis_platform.h index 168cefd..ad4f75b 100644 --- a/lwis_platform.h +++ b/lwis_platform.h @@ -47,7 +47,7 @@ int lwis_platform_remove_qos(struct lwis_device *lwis_dev); * lwis_platform_update_bts: handles platform-specific parts of * updating bts requirement. */ -int lwis_platform_update_bts(struct lwis_device *lwis_dev, unsigned int bw_kb_peak, +int lwis_platform_update_bts(struct lwis_device *lwis_dev, int block, unsigned int bw_kb_peak, unsigned int bw_kb_read, unsigned int bw_kb_write, unsigned int bw_kb_rt); diff --git a/lwis_regulator.c b/lwis_regulator.c index fac08d9..56d9d92 100644 --- a/lwis_regulator.c +++ b/lwis_regulator.c @@ -81,12 +81,12 @@ int lwis_regulator_get(struct lwis_regulator_list *list, char *name, int voltage /* Make sure regulator exists */ reg = devm_regulator_get(dev, name); - if (IS_ERR(reg)) { + if (IS_ERR_OR_NULL(reg)) { return PTR_ERR(reg); } list->reg[index].reg = reg; - strlcpy(list->reg[index].name, name, LWIS_MAX_NAME_STRING_LEN); + strscpy(list->reg[index].name, name, LWIS_MAX_NAME_STRING_LEN); list->reg[index].voltage = voltage; return index; diff --git a/lwis_trace.h b/lwis_trace.h index 2f263e2..e1f14a6 100644 --- a/lwis_trace.h +++ b/lwis_trace.h @@ -19,45 +19,33 @@ #include "lwis_commands.h" #include "lwis_device.h" -#define LWIS_DEVICE_NAME_ENTRY \ - __array(char, lwis_name, LWIS_MAX_NAME_STRING_LEN) -#define LWIS_DEVICE_NAME_ASSIGN \ - strlcpy(__entry->lwis_name, lwis_dev->name, LWIS_MAX_NAME_STRING_LEN) -#define LWIS_DEVICE_NAME __entry->lwis_name +#define LWIS_DEVICE_NAME_ENTRY __array(char, lwis_name, LWIS_MAX_NAME_STRING_LEN) +#define LWIS_DEVICE_NAME_ASSIGN \ + strscpy(__entry->lwis_name, lwis_dev->name, LWIS_MAX_NAME_STRING_LEN) +#define LWIS_TRACE_DEVICE_NAME __entry->lwis_name TRACE_EVENT(tracing_mark_write, - TP_PROTO(struct lwis_device *lwis_dev, char type, - int pid, const char *func_name, int64_t value), - TP_ARGS(lwis_dev, type, pid, func_name, value), - TP_STRUCT__entry( - LWIS_DEVICE_NAME_ENTRY - __field(char, type) - __field(int, pid) - __string(func_name, func_name) - __field(int64_t, value) - ), - TP_fast_assign( - LWIS_DEVICE_NAME_ASSIGN; - __entry->type = type; - __entry->pid = pid; - __assign_str(func_name, func_name); - __entry->value = value; - ), - TP_printk("%c|%d|lwis-%s:%s|%lld", - __entry->type, __entry->pid, LWIS_DEVICE_NAME, __get_str(func_name), __entry->value) -); - -#define LWIS_ATRACE_BEGIN(lwis_dev, func_name) \ + TP_PROTO(struct lwis_device *lwis_dev, char type, int pid, const char *func_name, + int64_t value), + TP_ARGS(lwis_dev, type, pid, func_name, value), + TP_STRUCT__entry(LWIS_DEVICE_NAME_ENTRY __field(char, type) __field(int, pid) + __string(func_name, func_name) __field(int64_t, value)), + TP_fast_assign(LWIS_DEVICE_NAME_ASSIGN; __entry->type = type; __entry->pid = pid; + __assign_str(func_name, func_name); __entry->value = value;), + TP_printk("%c|%d|lwis-%s:%s|%lld", __entry->type, __entry->pid, LWIS_TRACE_DEVICE_NAME, + __get_str(func_name), __entry->value)); + +#define LWIS_ATRACE_BEGIN(lwis_dev, func_name) \ trace_tracing_mark_write(lwis_dev, 'B', current->tgid, func_name, 0) -#define LWIS_ATRACE_FUNC_BEGIN(lwis_dev) LWIS_ATRACE_BEGIN(lwis_dev, __func__) +#define LWIS_ATRACE_FUNC_BEGIN(lwis_dev, func_name) LWIS_ATRACE_BEGIN(lwis_dev, func_name) -#define LWIS_ATRACE_END(lwis_dev, func_name) \ +#define LWIS_ATRACE_END(lwis_dev, func_name) \ trace_tracing_mark_write(lwis_dev, 'E', current->tgid, func_name, 0) -#define LWIS_ATRACE_FUNC_END(lwis_dev) LWIS_ATRACE_END(lwis_dev, __func__) +#define LWIS_ATRACE_FUNC_END(lwis_dev, func_name) LWIS_ATRACE_END(lwis_dev, func_name) -#define LWIS_ATRACE_INT_PID(lwis_dev, func_name, value, pid) \ +#define LWIS_ATRACE_INT_PID(lwis_dev, func_name, value, pid) \ trace_tracing_mark_write(lwis_dev, 'C', pid, func_name, value) -#define LWIS_ATRACE_INT(lwis_dev, func_name, value) \ +#define LWIS_ATRACE_INT(lwis_dev, func_name, value) \ LWIS_ATRACE_INT_PID(lwis_dev, func_name, value, current->tgid) #endif /* _TRACE_LWIS_H_ */ diff --git a/lwis_transaction.c b/lwis_transaction.c index 434846c..1016af5 100644 --- a/lwis_transaction.c +++ b/lwis_transaction.c @@ -15,11 +15,15 @@ #include <linux/delay.h> #include <linux/kernel.h> #include <linux/mm.h> +#include <linux/preempt.h> #include <linux/slab.h> +#include <linux/workqueue.h> #include "lwis_allocator.h" +#include "lwis_commands.h" #include "lwis_device.h" #include "lwis_event.h" +#include "lwis_fence.h" #include "lwis_io_entry.h" #include "lwis_ioreg.h" #include "lwis_util.h" @@ -30,6 +34,9 @@ #define EXPLICIT_EVENT_COUNTER(x) \ ((x) != LWIS_EVENT_COUNTER_ON_NEXT_OCCURRENCE && (x) != LWIS_EVENT_COUNTER_EVERY_TIME) +bool lwis_transaction_debug; +module_param(lwis_transaction_debug, bool, 0644); + static struct lwis_transaction_event_list *event_list_find(struct lwis_client *client, int64_t event_id) { @@ -64,8 +71,34 @@ static struct lwis_transaction_event_list *event_list_find_or_create(struct lwis return (list == NULL) ? event_list_create(client, event_id) : list; } +static void add_pending_transaction(struct lwis_client *client, + struct lwis_transaction *transaction) +{ + hash_add(client->pending_transactions, &transaction->pending_map_node, + transaction->info.id); + if (lwis_fence_debug) { + dev_info(client->lwis_dev->dev, + "lwis_fence add transaction id %llu to lwis_client pending map", + transaction->info.id); + } +} + +static struct lwis_transaction *pending_transaction_peek(struct lwis_client *client, + int64_t transaction_id) +{ + struct hlist_node *tmp; + struct lwis_transaction *transaction; + hash_for_each_possible_safe (client->pending_transactions, transaction, tmp, + pending_map_node, transaction_id) { + if (transaction->info.id == transaction_id) { + return transaction; + } + } + return NULL; +} + static void save_transaction_to_history(struct lwis_client *client, - struct lwis_transaction_info *trans_info, + struct lwis_transaction_info_v2 *trans_info, int64_t process_timestamp, int64_t process_duration_ns) { client->debug_info.transaction_hist[client->debug_info.cur_transaction_hist_idx].info = @@ -83,6 +116,28 @@ static void save_transaction_to_history(struct lwis_client *client, void lwis_transaction_free(struct lwis_device *lwis_dev, struct lwis_transaction *transaction) { int i; + struct lwis_fence_pending_signal *pending_fence; + struct list_head *it_fence, *it_fence_tmp; + + if (transaction->is_weak_transaction) { + kfree(transaction); + return; + } + + if (!list_empty(&transaction->completion_fence_list)) { + list_for_each_safe (it_fence, it_fence_tmp, &transaction->completion_fence_list) { + pending_fence = + list_entry(it_fence, struct lwis_fence_pending_signal, node); + list_del(&pending_fence->node); + fput(pending_fence->fp); + kfree(pending_fence); + } + } + + for (i = 0; i < transaction->num_trigger_fences; i++) { + fput(transaction->trigger_fence_fps[i]); + transaction->trigger_fence_fps[i] = NULL; + } for (i = 0; i < transaction->info.num_io_entries; ++i) { if (transaction->info.io_entries[i].type == LWIS_IO_ENTRY_WRITE_BATCH) { @@ -98,22 +153,28 @@ void lwis_transaction_free(struct lwis_device *lwis_dev, struct lwis_transaction } static int process_transaction(struct lwis_client *client, struct lwis_transaction *transaction, - struct list_head *pending_events, bool in_irq, bool skip_err) + struct list_head *pending_events, struct list_head *pending_fences, + bool skip_err) { int i; int ret = 0; + int pending_status; struct lwis_io_entry *entry = NULL; struct lwis_device *lwis_dev = client->lwis_dev; - struct lwis_transaction_info *info = &transaction->info; + struct lwis_transaction_info_v2 *info = &transaction->info; struct lwis_transaction_response_header *resp = transaction->resp; size_t resp_size; uint8_t *read_buf; struct lwis_io_result *io_result; const int reg_value_bytewidth = lwis_dev->native_value_bitwidth / 8; - int64_t process_duration_ns = 0; - int64_t process_timestamp = ktime_to_ns(lwis_get_time()); + int64_t process_duration_ns = -1; + int64_t process_timestamp = -1; + unsigned long flags; + + if (lwis_transaction_debug) { + process_timestamp = ktime_to_ns(lwis_get_time()); + } - LWIS_ATRACE_FUNC_BEGIN(lwis_dev); resp_size = sizeof(struct lwis_transaction_response_header) + resp->results_size_bytes; read_buf = (uint8_t *)resp + sizeof(struct lwis_transaction_response_header); resp->completion_index = -1; @@ -136,9 +197,10 @@ static int process_transaction(struct lwis_client *client, struct lwis_transacti if (ret) { resp->error_code = ret; if (skip_err) { - dev_warn(lwis_dev->dev, - "transaction type %d processing failed, skip this error and run the next command\n", - entry->type); + dev_warn( + lwis_dev->dev, + "transaction type %d processing failed, skip this error and run the next command\n", + entry->type); continue; } break; @@ -153,9 +215,10 @@ static int process_transaction(struct lwis_client *client, struct lwis_transacti if (ret) { resp->error_code = ret; if (skip_err) { - dev_warn(lwis_dev->dev, - "transaction type %d processing failed, skip this error and run the next command\n", - entry->type); + dev_warn( + lwis_dev->dev, + "transaction type %d processing failed, skip this error and run the next command\n", + entry->type); continue; } break; @@ -173,22 +236,24 @@ static int process_transaction(struct lwis_client *client, struct lwis_transacti if (ret) { resp->error_code = ret; if (skip_err) { - dev_warn(lwis_dev->dev, - "transaction type %d processing failed, skip this error and run the next command\n", - entry->type); + dev_warn( + lwis_dev->dev, + "transaction type %d processing failed, skip this error and run the next command\n", + entry->type); continue; } break; } read_buf += sizeof(struct lwis_io_result) + io_result->num_value_bytes; } else if (entry->type == LWIS_IO_ENTRY_POLL) { - ret = lwis_io_entry_poll(lwis_dev, entry, in_irq); + ret = lwis_io_entry_poll(lwis_dev, entry); if (ret) { resp->error_code = ret; if (skip_err) { - dev_warn(lwis_dev->dev, - "transaction type %d processing failed, skip this error and run the next command\n", - entry->type); + dev_warn( + lwis_dev->dev, + "transaction type %d processing failed, skip this error and run the next command\n", + entry->type); continue; } break; @@ -198,9 +263,10 @@ static int process_transaction(struct lwis_client *client, struct lwis_transacti if (ret) { resp->error_code = ret; if (skip_err) { - dev_warn(lwis_dev->dev, - "transaction type %d processing failed, skip this error and run the next command\n", - entry->type); + dev_warn( + lwis_dev->dev, + "transaction type %d processing failed, skip this error and run the next command\n", + entry->type); continue; } break; @@ -209,9 +275,10 @@ static int process_transaction(struct lwis_client *client, struct lwis_transacti dev_err(lwis_dev->dev, "Unrecognized io_entry command\n"); resp->error_code = -EINVAL; if (skip_err) { - dev_warn(lwis_dev->dev, - "transaction type %d processing failed, skip this error and run the next command\n", - entry->type); + dev_warn( + lwis_dev->dev, + "transaction type %d processing failed, skip this error and run the next command\n", + entry->type); continue; } break; @@ -219,7 +286,9 @@ static int process_transaction(struct lwis_client *client, struct lwis_transacti resp->completion_index = i; } - process_duration_ns = ktime_to_ns(lwis_get_time() - process_timestamp); + if (lwis_transaction_debug) { + process_duration_ns = ktime_to_ns(lwis_get_time() - process_timestamp); + } /* Use read memory barrier at the end of I/O entries if the access protocol * allows it */ @@ -230,7 +299,7 @@ static int process_transaction(struct lwis_client *client, struct lwis_transacti if (pending_events) { lwis_pending_event_push(pending_events, resp->error_code ? info->emit_error_event_id : - info->emit_success_event_id, + info->emit_success_event_id, (void *)resp, resp_size); } else { /* No pending events indicates it's cleanup io_entries. */ @@ -240,6 +309,13 @@ static int process_transaction(struct lwis_client *client, struct lwis_transacti resp->error_code, transaction->info.id, i, entry->type); } } + + spin_lock_irqsave(&client->transaction_lock, flags); + if (pending_fences) { + /* Convert -ECANCELED error code to userspace Cancellation error code */ + pending_status = resp->error_code == -ECANCELED ? 1 : resp->error_code; + lwis_pending_fences_move_all(lwis_dev, transaction, pending_fences, pending_status); + } save_transaction_to_history(client, info, process_timestamp, process_duration_ns); if (info->trigger_event_counter == LWIS_EVENT_COUNTER_EVERY_TIME) { /* Only clean the transaction struct for this iteration. The @@ -249,14 +325,17 @@ static int process_transaction(struct lwis_client *client, struct lwis_transacti } else { lwis_transaction_free(lwis_dev, transaction); } - LWIS_ATRACE_FUNC_END(lwis_dev); + spin_unlock_irqrestore(&client->transaction_lock, flags); + return ret; } static void cancel_transaction(struct lwis_device *lwis_dev, struct lwis_transaction *transaction, - int error_code, struct list_head *pending_events) + int error_code, struct list_head *pending_events, + struct list_head *pending_fences) { - struct lwis_transaction_info *info = &transaction->info; + int pending_status; + struct lwis_transaction_info_v2 *info = &transaction->info; struct lwis_transaction_response_header resp; resp.id = info->id; resp.error_code = error_code; @@ -264,22 +343,33 @@ static void cancel_transaction(struct lwis_device *lwis_dev, struct lwis_transac resp.results_size_bytes = 0; resp.completion_index = -1; + if (transaction->is_weak_transaction) { + lwis_transaction_free(lwis_dev, transaction); + return; + } + if (pending_events) { lwis_pending_event_push(pending_events, info->emit_error_event_id, &resp, sizeof(resp)); } + if (pending_fences) { + /* Convert -ECANCELED error code to userspace Cancellation error code */ + pending_status = error_code == -ECANCELED ? 1 : error_code; + lwis_pending_fences_move_all(lwis_dev, transaction, pending_fences, pending_status); + } lwis_transaction_free(lwis_dev, transaction); } -static void transaction_work_func(struct kthread_work *work) +void lwis_process_transactions_in_queue(struct lwis_client *client) { unsigned long flags; - struct lwis_client *client = container_of(work, struct lwis_client, transaction_work); struct list_head *it_tran, *it_tran_tmp; struct list_head pending_events; + struct list_head pending_fences; struct lwis_transaction *transaction; INIT_LIST_HEAD(&pending_events); + INIT_LIST_HEAD(&pending_fences); spin_lock_irqsave(&client->transaction_lock, flags); list_for_each_safe (it_tran, it_tran_tmp, &client->transaction_process_queue) { @@ -287,27 +377,27 @@ static void transaction_work_func(struct kthread_work *work) list_del(&transaction->process_queue_node); if (transaction->resp->error_code) { cancel_transaction(client->lwis_dev, transaction, - transaction->resp->error_code, &pending_events); + transaction->resp->error_code, &pending_events, + &pending_fences); } else { spin_unlock_irqrestore(&client->transaction_lock, flags); - process_transaction(client, transaction, &pending_events, - /*in_irq=*/false, + process_transaction(client, transaction, &pending_events, &pending_fences, /*skip_err=*/false); spin_lock_irqsave(&client->transaction_lock, flags); } } spin_unlock_irqrestore(&client->transaction_lock, flags); - - lwis_pending_events_emit(client->lwis_dev, &pending_events, /*in_irq=*/false); + lwis_pending_events_emit(client->lwis_dev, &pending_events); + lwis_fences_pending_signal_emit(client->lwis_dev, &pending_fences); } int lwis_transaction_init(struct lwis_client *client) { spin_lock_init(&client->transaction_lock); INIT_LIST_HEAD(&client->transaction_process_queue); - kthread_init_work(&client->transaction_work, transaction_work_func); client->transaction_counter = 0; hash_init(client->transaction_list); + hash_init(client->pending_transactions); return 0; } @@ -325,7 +415,7 @@ int lwis_transaction_clear(struct lwis_client *client) } static void cancel_all_transactions_in_queue_locked(struct lwis_client *client, - struct list_head *transaction_queue) + struct list_head *transaction_queue) { struct lwis_transaction *transaction; struct list_head *it_tran, *it_tran_tmp; @@ -336,7 +426,7 @@ static void cancel_all_transactions_in_queue_locked(struct lwis_client *client, transaction = list_entry(it_tran, struct lwis_transaction, process_queue_node); list_del(&transaction->process_queue_node); - cancel_transaction(client->lwis_dev, transaction, -ECANCELED, NULL); + cancel_transaction(client->lwis_dev, transaction, -ECANCELED, NULL, NULL); } } } @@ -364,11 +454,15 @@ int lwis_transaction_client_flush(struct lwis_client *client) list_for_each_safe (it_tran, it_tran_tmp, &it_evt_list->list) { transaction = list_entry(it_tran, struct lwis_transaction, event_list_node); list_del(&transaction->event_list_node); - cancel_transaction(client->lwis_dev, transaction, -ECANCELED, NULL); + cancel_transaction(client->lwis_dev, transaction, -ECANCELED, NULL, NULL); } hash_del(&it_evt_list->node); kfree(it_evt_list); } + hash_for_each_safe (client->pending_transactions, i, tmp, transaction, pending_map_node) { + cancel_transaction(client->lwis_dev, transaction, -ECANCELED, NULL, NULL); + hash_del(&transaction->pending_map_node); + } spin_unlock_irqrestore(&client->transaction_lock, flags); if (client->lwis_dev->transaction_worker_thread) @@ -383,36 +477,12 @@ int lwis_transaction_client_flush(struct lwis_client *client) return 0; } -static void print_buffer(struct lwis_client *client, const char *buf, int size) { - char output_string[80], temp_string[8]; - int i; - - output_string[0] = '\0'; - for (i = 0; i < size; i++) { - if (i % 16 == 0 && i != 0) { - dev_info(client->lwis_dev->dev, "%s\n", output_string); - output_string[0] = '\0'; - } - sprintf(temp_string, "%02X ", buf[i]); - if (i % 4 == 3) { - strcat(temp_string, " "); - } - strcat(output_string, temp_string); - } - dev_info(client->lwis_dev->dev, "%s\n", output_string); -} - int lwis_transaction_client_cleanup(struct lwis_client *client) { unsigned long flags; struct list_head *it_tran, *it_tran_tmp; struct lwis_transaction *transaction; struct lwis_transaction_event_list *it_evt_list; - struct list_head pending_events; - bool in_irq = false; - struct lwis_event_entry *event; - - INIT_LIST_HEAD(&pending_events); spin_lock_irqsave(&client->transaction_lock, flags); /* Perform client defined clean-up routine. */ @@ -426,12 +496,20 @@ int lwis_transaction_client_cleanup(struct lwis_client *client) list_for_each_safe (it_tran, it_tran_tmp, &it_evt_list->list) { transaction = list_entry(it_tran, struct lwis_transaction, event_list_node); + if (!list_empty(&transaction->completion_fence_list)) { + dev_warn( + client->lwis_dev->dev, + "Cleanup transaction with id %llu has tailing fences; cleanup transactions should not have tailing fences", + transaction->info.id); + } list_del(&transaction->event_list_node); if (transaction->resp->error_code || client->lwis_dev->enabled == 0) { - cancel_transaction(client->lwis_dev, transaction, -ECANCELED, NULL); + cancel_transaction(client->lwis_dev, transaction, -ECANCELED, NULL, NULL); } else { spin_unlock_irqrestore(&client->transaction_lock, flags); - process_transaction(client, transaction, &pending_events, in_irq, + process_transaction(client, transaction, + /*pending_events=*/NULL, + /*pending_fences=*/NULL, /*skip_err=*/true); spin_lock_irqsave(&client->transaction_lock, flags); } @@ -440,52 +518,55 @@ int lwis_transaction_client_cleanup(struct lwis_client *client) kfree(it_evt_list); spin_unlock_irqrestore(&client->transaction_lock, flags); + return 0; +} - while (!list_empty(&pending_events)) { - struct lwis_transaction_response_header *resp; - struct lwis_io_result *result; - char *results_buf; - int i; - - event = list_first_entry(&pending_events, struct lwis_event_entry, node); - /* - If there is no read result, the payload size will less than - sizeof(struct lwis_transaction_response_header). - We can skip it. - */ - if (event->event_info.payload_size <= - sizeof(struct lwis_transaction_response_header)) { - list_del(&event->node); - kfree(event); - continue; - } - resp = (struct lwis_transaction_response_header *) - event->event_info.payload_buffer; - dev_info(client->lwis_dev->dev, "event_id = %llx, num_entries = %ld\n", - event->event_info.event_id, resp->num_entries); - results_buf = (char *)event->event_info.payload_buffer + - sizeof(struct lwis_transaction_response_header); - for (i = 0; i < resp->num_entries; i++) { - dev_info(client->lwis_dev->dev, "entry-%d\n", i); - result = (struct lwis_io_result *)results_buf; - print_buffer(client, result->values, result->num_value_bytes); - results_buf += sizeof(struct lwis_io_result) + - result->num_value_bytes; - } +int lwis_trigger_event_add_weak_transaction(struct lwis_client *client, int64_t transaction_id, + int64_t event_id, int32_t precondition_fence_fd) +{ + struct lwis_transaction *weak_transaction; + struct lwis_transaction_event_list *event_list; - list_del(&event->node); - kfree(event); + weak_transaction = kmalloc(sizeof(struct lwis_transaction), GFP_ATOMIC); + if (!weak_transaction) { + dev_err(client->lwis_dev->dev, "Cannot allocate weak transaction\n"); + return -ENOMEM; + } + weak_transaction->is_weak_transaction = true; + weak_transaction->id = transaction_id; + if (precondition_fence_fd >= 0) { + weak_transaction->precondition_fence_fp = fget(precondition_fence_fd); + if (weak_transaction->precondition_fence_fp == NULL) { + dev_err(client->lwis_dev->dev, + "Precondition fence %d results in NULL file pointer", + precondition_fence_fd); + return -EINVAL; + } + } else { + weak_transaction->precondition_fence_fp = NULL; } + event_list = event_list_find_or_create(client, event_id); + if (!event_list) { + dev_err(client->lwis_dev->dev, "Cannot create transaction event list\n"); + return -EINVAL; + } + list_add_tail(&weak_transaction->event_list_node, &event_list->list); + if (lwis_fence_debug) { + dev_info( + client->lwis_dev->dev, + "lwis_fence add weak transaction for event id-%lld triggered transaction id %llu", + event_id, transaction_id); + } return 0; } static int check_transaction_param_locked(struct lwis_client *client, struct lwis_transaction *transaction, - bool allow_counter_eq) + bool is_level_triggered) { struct lwis_device_event_state *event_state; - struct lwis_transaction_info *info = &transaction->info; + struct lwis_transaction_info_v2 *info = &transaction->info; struct lwis_device *lwis_dev = client->lwis_dev; if (!client) { @@ -498,6 +579,8 @@ static int check_transaction_param_locked(struct lwis_client *client, return -ENODEV; } + /* Assign the transaction id and increment transaction counter */ + info->id = client->transaction_counter++; /* Initialize event counter return value */ info->current_trigger_event_counter = -1LL; @@ -512,6 +595,9 @@ static int check_transaction_param_locked(struct lwis_client *client, /* Event found, return current counter to userspace */ info->current_trigger_event_counter = event_state->event_counter; } + } else if (!lwis_triggered_by_condition(transaction)) { + /* Otherwise it's an immediate transaction */ + transaction->queue_immediately = true; } /* Both trigger event ID and counter are defined */ @@ -519,10 +605,10 @@ static int check_transaction_param_locked(struct lwis_client *client, EXPLICIT_EVENT_COUNTER(info->trigger_event_counter)) { /* Check if event has happened already */ if (info->trigger_event_counter == info->current_trigger_event_counter) { - if (allow_counter_eq) { + if (is_level_triggered) { /* Convert this transaction into an immediate * one */ - info->trigger_event_id = LWIS_EVENT_ID_NONE; + transaction->queue_immediately = true; } else { return -ENOENT; } @@ -547,17 +633,34 @@ static int check_transaction_param_locked(struct lwis_client *client, return 0; } +static int prepare_transaction_fences_locked(struct lwis_client *client, + struct lwis_transaction *transaction) +{ + int ret = 0; + + /* If triggered by trigger_condition */ + if (lwis_triggered_by_condition(transaction)) { + ret = lwis_parse_trigger_condition(client, transaction); + if (ret) { + return ret; + } + } + + /* If transaction contains completion fences, add them to the transaction */ + ret = lwis_add_completion_fence(client, transaction); + + return ret; +} + static int prepare_response_locked(struct lwis_client *client, struct lwis_transaction *transaction) { - struct lwis_transaction_info *info = &transaction->info; + struct lwis_transaction_info_v2 *info = &transaction->info; int i; size_t resp_size; size_t read_buf_size = 0; int read_entries = 0; const int reg_value_bytewidth = client->lwis_dev->native_value_bitwidth / 8; - info->id = client->transaction_counter; - for (i = 0; i < info->num_io_entries; ++i) { struct lwis_io_entry *entry = &info->io_entries[i]; if (entry->type == LWIS_IO_ENTRY_READ) { @@ -595,13 +698,16 @@ static int queue_transaction_locked(struct lwis_client *client, struct lwis_transaction *transaction) { struct lwis_transaction_event_list *event_list; - struct lwis_transaction_info *info = &transaction->info; + struct lwis_transaction_info_v2 *info = &transaction->info; - if (info->trigger_event_id == LWIS_EVENT_ID_NONE) { + if (transaction->queue_immediately) { /* Immediate trigger. */ list_add_tail(&transaction->process_queue_node, &client->transaction_process_queue); kthread_queue_work(&client->lwis_dev->transaction_worker, &client->transaction_work); + } else if (lwis_triggered_by_condition(transaction)) { + /* Trigger by trigger conditions. */ + add_pending_transaction(client, transaction); } else { /* Trigger by event. */ event_list = event_list_find_or_create(client, info->trigger_event_id); @@ -613,18 +719,16 @@ static int queue_transaction_locked(struct lwis_client *client, list_add_tail(&transaction->event_list_node, &event_list->list); } info->submission_timestamp_ns = ktime_to_ns(ktime_get()); - client->transaction_counter++; return 0; } int lwis_transaction_submit_locked(struct lwis_client *client, struct lwis_transaction *transaction) { int ret; - struct lwis_transaction_info *info = &transaction->info; + struct lwis_transaction_info_v2 *info = &transaction->info; ret = check_transaction_param_locked(client, transaction, - /*allow_counter_eq=*/info->allow_counter_eq); - + /*is_level_triggered=*/info->is_level_triggered); if (ret) { return ret; } @@ -634,6 +738,11 @@ int lwis_transaction_submit_locked(struct lwis_client *client, struct lwis_trans return ret; } + ret = prepare_transaction_fences_locked(client, transaction); + if (ret) { + return ret; + } + ret = queue_transaction_locked(client, transaction); return ret; } @@ -652,7 +761,7 @@ new_repeating_transaction_iteration(struct lwis_client *client, "Failed to allocate repeating transaction instance\n"); return NULL; } - memcpy(&new_instance->info, &transaction->info, sizeof(struct lwis_transaction_info)); + memcpy(&new_instance->info, &transaction->info, sizeof(transaction->info)); /* Allocate response buffer */ resp_buf = kmalloc(sizeof(struct lwis_transaction_response_header) + @@ -669,14 +778,15 @@ new_repeating_transaction_iteration(struct lwis_client *client, INIT_LIST_HEAD(&new_instance->event_list_node); INIT_LIST_HEAD(&new_instance->process_queue_node); + INIT_LIST_HEAD(&new_instance->completion_fence_list); return new_instance; } static void defer_transaction_locked(struct lwis_client *client, struct lwis_transaction *transaction, - struct list_head *pending_events, bool in_irq, - bool del_event_list_node) + struct list_head *pending_events, + struct list_head *pending_fences, bool del_event_list_node) { unsigned long flags = 0; if (del_event_list_node) { @@ -684,14 +794,14 @@ static void defer_transaction_locked(struct lwis_client *client, } /* I2C read/write cannot be executed in IRQ context */ - if (in_irq && client->lwis_dev->type == DEVICE_TYPE_I2C) { + if (in_irq() && client->lwis_dev->type == DEVICE_TYPE_I2C) { list_add_tail(&transaction->process_queue_node, &client->transaction_process_queue); return; } if (transaction->info.run_in_event_context) { spin_unlock_irqrestore(&client->transaction_lock, flags); - process_transaction(client, transaction, pending_events, in_irq, + process_transaction(client, transaction, pending_events, pending_fences, /*skip_err=*/false); spin_lock_irqsave(&client->transaction_lock, flags); } else { @@ -700,18 +810,23 @@ static void defer_transaction_locked(struct lwis_client *client, } int lwis_transaction_event_trigger(struct lwis_client *client, int64_t event_id, - int64_t event_counter, struct list_head *pending_events, - bool in_irq) + int64_t event_counter, struct list_head *pending_events) { unsigned long flags; struct lwis_transaction_event_list *event_list; struct list_head *it_tran, *it_tran_tmp; - struct lwis_transaction *transaction; + struct lwis_transaction *transaction, *weak_transaction = NULL; struct lwis_transaction *new_instance; int64_t trigger_counter = 0; + struct list_head pending_fences; + + INIT_LIST_HEAD(&pending_fences); /* Find event list that matches the trigger event ID. */ spin_lock_irqsave(&client->transaction_lock, flags); + if (event_id & LWIS_OVERFLOW_IRQ_EVENT_FLAG) { + event_id = event_id ^ LWIS_OVERFLOW_IRQ_EVENT_FLAG; + } event_list = event_list_find(client, event_id); /* No event found, just return. */ if (event_list == NULL || list_empty(&event_list->list)) { @@ -722,6 +837,33 @@ int lwis_transaction_event_trigger(struct lwis_client *client, int64_t event_id, /* Go through all transactions under the chosen event list. */ list_for_each_safe (it_tran, it_tran_tmp, &event_list->list) { transaction = list_entry(it_tran, struct lwis_transaction, event_list_node); + + if (transaction->is_weak_transaction) { + weak_transaction = transaction; + transaction = pending_transaction_peek(client, weak_transaction->id); + if (transaction == NULL) { + /* It means the transaction is already executed or is canceled */ + list_del(&weak_transaction->event_list_node); + kfree(weak_transaction); + continue; + } + + if (lwis_event_triggered_condition_ready(transaction, weak_transaction, + event_id, event_counter)) { + if (lwis_fence_debug) { + dev_info( + client->lwis_dev->dev, + "lwis_fence event id-%lld counter-%lld triggered transaction id %llu", + event_id, event_counter, transaction->info.id); + } + hash_del(&transaction->pending_map_node); + defer_transaction_locked(client, transaction, pending_events, + &pending_fences, + /* del_event_list_node */ false); + } + continue; + } + if (transaction->resp->error_code) { list_add_tail(&transaction->process_queue_node, &client->transaction_process_queue); @@ -734,8 +876,8 @@ int lwis_transaction_event_trigger(struct lwis_client *client, int64_t event_id, trigger_counter = transaction->info.trigger_event_counter; if (trigger_counter == LWIS_EVENT_COUNTER_ON_NEXT_OCCURRENCE || trigger_counter == event_counter) { - defer_transaction_locked(client, transaction, pending_events, in_irq, - /* del_event_list_node */ true); + defer_transaction_locked(client, transaction, pending_events, + &pending_fences, /* del_event_list_node */ true); } else if (trigger_counter == LWIS_EVENT_COUNTER_EVERY_TIME) { new_instance = new_repeating_transaction_iteration(client, transaction); if (!new_instance) { @@ -745,8 +887,8 @@ int lwis_transaction_event_trigger(struct lwis_client *client, int64_t event_id, list_del(&transaction->event_list_node); continue; } - defer_transaction_locked(client, new_instance, pending_events, in_irq, - /* del_event_list_node */ false); + defer_transaction_locked(client, new_instance, pending_events, + &pending_fences, /* del_event_list_node */ false); } } @@ -758,9 +900,77 @@ int lwis_transaction_event_trigger(struct lwis_client *client, int64_t event_id, spin_unlock_irqrestore(&client->transaction_lock, flags); + lwis_fences_pending_signal_emit(client->lwis_dev, &pending_fences); + return 0; } +void lwis_transaction_fence_trigger(struct lwis_client *client, struct lwis_fence *fence, + struct list_head *transaction_list) +{ + unsigned long flags = 0; + struct lwis_pending_transaction_id *transaction_id; + struct lwis_transaction *transaction; + struct list_head *it_tran, *it_tran_tmp; + struct list_head pending_events; + struct list_head pending_fences; + + if (list_empty(transaction_list)) { + return; + } + + INIT_LIST_HEAD(&pending_events); + INIT_LIST_HEAD(&pending_fences); + + spin_lock_irqsave(&client->transaction_lock, flags); + list_for_each_safe (it_tran, it_tran_tmp, transaction_list) { + transaction_id = list_entry(it_tran, struct lwis_pending_transaction_id, list_node); + list_del(&transaction_id->list_node); + + transaction = pending_transaction_peek(client, transaction_id->id); + if (transaction == NULL) { + /* It means the transaction is already executed or is canceled */ + if (lwis_fence_debug) { + dev_info( + client->lwis_dev->dev, + "lwis_fence fd-%d did NOT triggered transaction id %llu, seems already triggered", + fence->fd, transaction_id->id); + } + } else { + if (lwis_fence_triggered_condition_ready(transaction, fence->status)) { + hash_del(&transaction->pending_map_node); + if (fence->status == 0) { + list_add_tail(&transaction->process_queue_node, + &client->transaction_process_queue); + if (lwis_fence_debug) { + dev_info( + client->lwis_dev->dev, + "lwis_fence fd-%d triggered transaction id %llu", + fence->fd, transaction->info.id); + } + } else { + cancel_transaction(client->lwis_dev, transaction, + -ECANCELED, &pending_events, + &pending_fences); + } + } + } + + kfree(transaction_id); + } + + /* Schedule deferred transactions */ + if (!list_empty(&client->transaction_process_queue)) { + kthread_queue_work(&client->lwis_dev->transaction_worker, + &client->transaction_work); + } + + spin_unlock_irqrestore(&client->transaction_lock, flags); + + lwis_pending_events_emit(client->lwis_dev, &pending_events); + lwis_fences_pending_signal_emit(client->lwis_dev, &pending_fences); +} + /* Calling this function requires holding the client's transaction_lock. */ static int cancel_waiting_transaction_locked(struct lwis_client *client, int64_t id) { @@ -770,6 +980,7 @@ static int cancel_waiting_transaction_locked(struct lwis_client *client, int64_t struct lwis_transaction_event_list *it_evt_list; struct lwis_transaction *transaction; + /* Search transactions triggered by events */ hash_for_each_safe (client->transaction_list, i, tmp, it_evt_list, node) { list_for_each_safe (it_tran, it_tran_tmp, &it_evt_list->list) { transaction = list_entry(it_tran, struct lwis_transaction, event_list_node); @@ -779,6 +990,16 @@ static int cancel_waiting_transaction_locked(struct lwis_client *client, int64_t } } } + + /* Search transactions triggered by trigger_condition */ + hash_for_each_possible_safe (client->pending_transactions, transaction, tmp, + pending_map_node, id) { + if (transaction->info.id == id) { + transaction->resp->error_code = -ECANCELED; + return 0; + } + } + return -ENOENT; } @@ -798,14 +1019,15 @@ int lwis_transaction_replace_locked(struct lwis_client *client, struct lwis_transaction *transaction) { int ret; + int64_t old_transaction_id = transaction->info.id; ret = check_transaction_param_locked(client, transaction, - /*allow_counter_eq=*/false); + /*is_level_triggered=*/false); if (ret) { return ret; } - ret = cancel_waiting_transaction_locked(client, transaction->info.id); + ret = cancel_waiting_transaction_locked(client, old_transaction_id); if (ret) { return ret; } @@ -815,6 +1037,11 @@ int lwis_transaction_replace_locked(struct lwis_client *client, return ret; } + ret = prepare_transaction_fences_locked(client, transaction); + if (ret) { + return ret; + } + ret = queue_transaction_locked(client, transaction); return ret; } diff --git a/lwis_transaction.h b/lwis_transaction.h index 2d6157e..340257a 100644 --- a/lwis_transaction.h +++ b/lwis_transaction.h @@ -16,6 +16,7 @@ /* LWIS forward declarations */ struct lwis_device; struct lwis_client; +struct lwis_fence; /* Transaction entry. Each entry belongs to two queues: * 1) Event list: Transactions are sorted by event IDs. This is to search for @@ -24,17 +25,35 @@ struct lwis_client; * into a queue. */ struct lwis_transaction { - struct lwis_transaction_info info; + struct lwis_transaction_info_v2 info; struct lwis_transaction_response_header *resp; struct list_head event_list_node; struct list_head process_queue_node; + struct hlist_node pending_map_node; + int signaled_count; + /* Flag used for level trigger conditions, indicating the transaction + * should be queued right after creation + */ + bool queue_immediately; + /* temporary variables to add supports for mixing events and fences in + * trigger_condition. Will be removed and refacter into an union soon. + */ + bool is_weak_transaction; + int64_t id; + /* List of fences's fp that's referenced by the transaction */ + int num_trigger_fences; + struct file *trigger_fence_fps[LWIS_TRIGGER_NODES_MAX_NUM]; + /* Parameters for completion fences */ + struct list_head completion_fence_list; + /* Precondition fence file pointer */ + struct file *precondition_fence_fp; }; /* For debugging purposes, keeps track of the transaction information, as * well as the time it executes and the time it took to execute. */ struct lwis_transaction_history { - struct lwis_transaction_info info; + struct lwis_transaction_info_v2 info; int64_t process_timestamp; int64_t process_duration_ns; }; @@ -45,14 +64,21 @@ struct lwis_transaction_event_list { struct hlist_node node; }; +struct lwis_pending_transaction_id { + int64_t id; + struct list_head list_node; +}; + int lwis_transaction_init(struct lwis_client *client); int lwis_transaction_clear(struct lwis_client *client); int lwis_transaction_client_flush(struct lwis_client *client); int lwis_transaction_client_cleanup(struct lwis_client *client); int lwis_transaction_event_trigger(struct lwis_client *client, int64_t event_id, - int64_t event_counter, struct list_head *pending_events, - bool in_irq); + int64_t event_counter, struct list_head *pending_events); +void lwis_transaction_fence_trigger(struct lwis_client *client, struct lwis_fence *fence, + struct list_head *transaction_list); + int lwis_transaction_cancel(struct lwis_client *client, int64_t id); void lwis_transaction_free(struct lwis_device *lwis_dev, struct lwis_transaction *transaction); @@ -64,4 +90,9 @@ int lwis_transaction_submit_locked(struct lwis_client *client, int lwis_transaction_replace_locked(struct lwis_client *client, struct lwis_transaction *transaction); +int lwis_trigger_event_add_weak_transaction(struct lwis_client *client, int64_t transaction_id, + int64_t event_id, int32_t precondition_fence_fd); + +void lwis_process_transactions_in_queue(struct lwis_client *client); + #endif /* LWIS_TRANSACTION_H_ */ diff --git a/lwis_util.c b/lwis_util.c index 43cd266..bb6e211 100644 --- a/lwis_util.c +++ b/lwis_util.c @@ -90,16 +90,30 @@ const char *lwis_device_type_to_string(int32_t type) return "IOREG"; case DEVICE_TYPE_SLC: return "SLC"; + case DEVICE_TYPE_TEST: + return "TEST"; case DEVICE_TYPE_UNKNOWN: default: return "UNKNOWN"; } } +const char *trigger_condition_node_operator_to_string(int32_t type) +{ + switch (type) { + case LWIS_TRIGGER_NODE_OPERATOR_AND: + return "AND"; + case LWIS_TRIGGER_NODE_OPERATOR_OR: + return "OR"; + case LWIS_TRIGGER_NODE_OPERATOR_NONE: + default: + return "NONE"; + } +} + int lwis_create_kthread_workers(struct lwis_device *lwis_dev) { char t_name[LWIS_MAX_NAME_STRING_LEN]; - char p_name[LWIS_MAX_NAME_STRING_LEN]; if (!lwis_dev) { pr_err("lwis_create_kthread_workers: lwis_dev is NULL\n"); @@ -107,37 +121,27 @@ int lwis_create_kthread_workers(struct lwis_device *lwis_dev) } scnprintf(t_name, LWIS_MAX_NAME_STRING_LEN, "lwis_t_%s", lwis_dev->name); - scnprintf(p_name, LWIS_MAX_NAME_STRING_LEN, "lwis_p_%s", lwis_dev->name); kthread_init_worker(&lwis_dev->transaction_worker); - lwis_dev->transaction_worker_thread = kthread_run(kthread_worker_fn, - &lwis_dev->transaction_worker, t_name); - if (IS_ERR(lwis_dev->transaction_worker_thread)) { + lwis_dev->transaction_worker_thread = + kthread_run(kthread_worker_fn, &lwis_dev->transaction_worker, t_name); + if (IS_ERR_OR_NULL(lwis_dev->transaction_worker_thread)) { dev_err(lwis_dev->dev, "transaction kthread_run failed\n"); return -EINVAL; } - kthread_init_worker(&lwis_dev->periodic_io_worker); - lwis_dev->periodic_io_worker_thread = kthread_run(kthread_worker_fn, - &lwis_dev->periodic_io_worker, p_name); - if (IS_ERR(lwis_dev->periodic_io_worker_thread)) { - dev_err(lwis_dev->dev, "periodic_io kthread_run failed\n"); - return -EINVAL; - } - return 0; } -int lwis_set_kthread_priority(struct lwis_device *lwis_dev, struct task_struct *task, - u32 priority) +int lwis_set_kthread_priority(struct lwis_device *lwis_dev, struct task_struct *task, u32 priority) { int policy; struct sched_param param; int ret; if (priority >= MAX_PRIO) { - dev_err(lwis_dev->dev, "transaction_thread_priority(%d) >= Max(%d)", - priority, MAX_PRIO); + dev_err(lwis_dev->dev, "transaction_thread_priority(%d) >= Max(%d)", priority, + MAX_PRIO); return -EINVAL; } if (priority < MAX_RT_PRIO) { diff --git a/lwis_util.h b/lwis_util.h index 1b1456f..29dca82 100644 --- a/lwis_util.h +++ b/lwis_util.h @@ -53,13 +53,19 @@ int lwis_device_single_register_read(struct lwis_device *lwis_dev, int bid, uint const char *lwis_device_type_to_string(int32_t type); /* + * trigger_condition_node_operator_to_string: Converts the trigger condition + * node type into a human-readable string. Useful for debug logging. + */ +const char *trigger_condition_node_operator_to_string(int32_t type); + +/* * lwis_get_time: Returns time since boot, this uses CLOCK_BOOTTIME which * does not stop during system suspend. * * This wrapper is created to encourage consistent usage of clock source * throughout LWIS implementations. */ -static inline ktime_t lwis_get_time() +static inline ktime_t lwis_get_time(void) { return ktime_get_boottime(); } @@ -72,7 +78,6 @@ int lwis_create_kthread_workers(struct lwis_device *lwis_dev); /* * lwis_set_kthread_priority: Set kthread priority. */ -int lwis_set_kthread_priority(struct lwis_device *lwis_dev, struct task_struct *task, - u32 priority); +int lwis_set_kthread_priority(struct lwis_device *lwis_dev, struct task_struct *task, u32 priority); #endif // LWIS_UTIL_H_ diff --git a/lwis_version.c b/lwis_version.c index a284cbb..001d6a8 100644 --- a/lwis_version.c +++ b/lwis_version.c @@ -24,11 +24,27 @@ void lwis_get_feature_flags(char *buffer, size_t buffer_size) * Feature flags start here: */ - /* core: + /* + * core: * All features we have shipped with in our initial version of LWIS, the basic fundamental * functions of LWIS. */ strlcat(buffer, "core", buffer_size); + /* + * cmd-pkt: + * A forward and backward compatible interface for LWIS commands. It uses a single IOCTL + * interface with different command packets to replace the current multiple IOCTL + * interfaces for different commands. It resolves the kernel version mismatch error when + * interface(s) change. + */ + strlcat(buffer, " cmd-pkt", buffer_size); + + /* + * fence: + * Support fence feature + */ + strlcat(buffer, " fence", buffer_size); + strlcat(buffer, "\n", buffer_size); }
\ No newline at end of file diff --git a/platform/anchorage/lwis_platform_anchorage.c b/platform/anchorage/lwis_platform_anchorage.c index 2eb8d9b..36fc0ba 100644 --- a/platform/anchorage/lwis_platform_anchorage.c +++ b/platform/anchorage/lwis_platform_anchorage.c @@ -27,6 +27,7 @@ int lwis_platform_probe(struct lwis_device *lwis_dev) { struct lwis_platform *platform; + int i; if (!lwis_dev) { return -ENODEV; @@ -41,16 +42,19 @@ int lwis_platform_probe(struct lwis_device *lwis_dev) /* Enable runtime power management for the platform device */ pm_runtime_enable(&lwis_dev->plat_dev->dev); - lwis_dev->bts_index = BTS_UNSUPPORTED; /* Only IOREG devices will access DMA resources */ if (lwis_dev->type != DEVICE_TYPE_IOREG) { return 0; } + /* Register to bts */ - lwis_dev->bts_index = bts_get_bwindex(lwis_dev->name); - if (lwis_dev->bts_index < 0) { - dev_err(lwis_dev->dev, "Failed to register to BTS, ret: %d\n", lwis_dev->bts_index); - lwis_dev->bts_index = BTS_UNSUPPORTED; + for (i = 0; i < lwis_dev->bts_block_num; i++) { + lwis_dev->bts_indexes[i] = bts_get_bwindex(lwis_dev->bts_block_names[i]); + if (lwis_dev->bts_indexes[i] < 0) { + dev_err(lwis_dev->dev, "Failed to register to BTS, ret: %d\n", + lwis_dev->bts_indexes[i]); + lwis_dev->bts_indexes[i] = BTS_UNSUPPORTED; + } } return 0; @@ -80,7 +84,9 @@ static int lwis_iommu_fault_handler(struct iommu_fault *fault, void *param) pr_err("\n"); lwis_debug_print_transaction_info(lwis_dev); pr_err("\n"); - lwis_debug_print_event_states_info(lwis_dev); + lwis_debug_print_register_io_history(lwis_dev); + pr_err("\n"); + lwis_debug_print_event_states_info(lwis_dev, /*lwis_event_dump_cnt=*/-1); pr_err("\n"); lwis_debug_print_buffer_info(lwis_dev); pr_err("\n"); @@ -98,6 +104,18 @@ static int lwis_iommu_fault_handler(struct iommu_fault *fault, void *param) #endif /* ENABLE_PAGE_FAULT_PANIC */ } +static bool lwis_device_support_bts(struct lwis_device *lwis_dev) +{ + int i; + + for (i = 0; i < lwis_dev->bts_block_num; i++) { + if (lwis_dev->bts_indexes[i] != BTS_UNSUPPORTED) { + return true; + } + } + return false; +} + int lwis_platform_device_enable(struct lwis_device *lwis_dev) { int ret; @@ -163,7 +181,7 @@ int lwis_platform_device_enable(struct lwis_device *lwis_dev) } } - if (lwis_dev->bts_scenario_name) { + if (lwis_device_support_bts(lwis_dev) && lwis_dev->bts_scenario_name) { lwis_dev->bts_scenario = bts_get_scenindex(lwis_dev->bts_scenario_name); if (!lwis_dev->bts_scenario) { dev_err(lwis_dev->dev, "Failed to get default camera BTS scenario.\n"); @@ -188,7 +206,7 @@ int lwis_platform_device_disable(struct lwis_device *lwis_dev) return -ENODEV; } - if (lwis_dev->bts_scenario_name) { + if (lwis_device_support_bts(lwis_dev) && lwis_dev->bts_scenario_name) { bts_del_scenario(lwis_dev->bts_scenario); } @@ -207,8 +225,7 @@ int lwis_platform_device_disable(struct lwis_device *lwis_dev) return pm_runtime_put_sync(&lwis_dev->plat_dev->dev); } -int lwis_platform_update_qos(struct lwis_device *lwis_dev, int value, - int32_t clock_family) +int lwis_platform_update_qos(struct lwis_device *lwis_dev, int value, int32_t clock_family) { struct lwis_platform *platform; struct exynos_pm_qos_request *qos_req; @@ -303,30 +320,38 @@ int lwis_platform_remove_qos(struct lwis_device *lwis_dev) return 0; } -int lwis_platform_update_bts(struct lwis_device *lwis_dev, unsigned int bw_kb_peak, +int lwis_platform_update_bts(struct lwis_device *lwis_dev, int block, unsigned int bw_kb_peak, unsigned int bw_kb_read, unsigned int bw_kb_write, unsigned int bw_kb_rt) { - int ret = 0; + int ret = 0, bts_index = lwis_dev->bts_indexes[block]; + const char *block_name = lwis_dev->bts_block_names[block]; struct bts_bw bts_request; - if (lwis_dev->bts_index == BTS_UNSUPPORTED) { - dev_info(lwis_dev->dev, "%s doesn't support bts\n", lwis_dev->name); - return ret; + if (block >= lwis_dev->bts_block_num) { + dev_err(lwis_dev->dev, "Invalid block index %d, %s only has %d bts blocks\n", block, + lwis_dev->name, lwis_dev->bts_block_num); + return -EINVAL; + } + + if (bts_index == BTS_UNSUPPORTED) { + dev_err(lwis_dev->dev, "%s block %s doesn't support bts\n", lwis_dev->name, + block_name); + return -EINVAL; } bts_request.peak = bw_kb_peak; bts_request.read = bw_kb_read; bts_request.write = bw_kb_write; bts_request.rt = bw_kb_rt; - ret = bts_update_bw(lwis_dev->bts_index, bts_request); + ret = bts_update_bw(bts_index, bts_request); if (ret < 0) { dev_err(lwis_dev->dev, "Failed to update bandwidth to bts, ret: %d\n", ret); } else { dev_info( lwis_dev->dev, - "Updated bandwidth to bts for device %s: peak: %u, read: %u, write: %u, rt: %u\n", - lwis_dev->name, bw_kb_peak, bw_kb_read, bw_kb_write, bw_kb_rt); + "Updated bandwidth to bts for device %s block %s: peak: %u, read: %u, write: %u, rt: %u\n", + lwis_dev->name, block_name, bw_kb_peak, bw_kb_read, bw_kb_write, bw_kb_rt); } return ret; } diff --git a/platform/anchorage/lwis_platform_anchorage_dma.c b/platform/anchorage/lwis_platform_anchorage_dma.c index 401275c..22b4257 100644 --- a/platform/anchorage/lwis_platform_anchorage_dma.c +++ b/platform/anchorage/lwis_platform_anchorage_dma.c @@ -37,8 +37,8 @@ struct dma_buf *lwis_platform_dma_buffer_alloc(size_t len, unsigned int flags) dmabuf = dma_heap_buffer_alloc(heap, len, O_RDWR, 0); if (IS_ERR_OR_NULL(dmabuf)) { - pr_err("DMA-BUF heap failed to alloc %#zx bytes. Error code %lu\n", - len, PTR_ERR(dmabuf)); + pr_err("DMA-BUF heap failed to alloc %#zx bytes. Error code %lu\n", len, + PTR_ERR(dmabuf)); dmabuf = NULL; } diff --git a/platform/busan/lwis_platform_busan.c b/platform/busan/lwis_platform_busan.c index b9fe7c0..19588a9 100644 --- a/platform/busan/lwis_platform_busan.c +++ b/platform/busan/lwis_platform_busan.c @@ -24,6 +24,7 @@ int lwis_platform_probe(struct lwis_device *lwis_dev) { struct lwis_platform *platform; + int i; if (!lwis_dev) { return -ENODEV; @@ -38,16 +39,19 @@ int lwis_platform_probe(struct lwis_device *lwis_dev) /* Enable runtime power management for the platform device */ pm_runtime_enable(&lwis_dev->plat_dev->dev); - lwis_dev->bts_index = BTS_UNSUPPORTED; /* Only IOREG devices will access DMA resources */ if (lwis_dev->type != DEVICE_TYPE_IOREG) { return 0; } + /* Register to bts */ - lwis_dev->bts_index = bts_get_bwindex(lwis_dev->name); - if (lwis_dev->bts_index < 0) { - dev_err(lwis_dev->dev, "Failed to register to BTS, ret: %d\n", lwis_dev->bts_index); - lwis_dev->bts_index = BTS_UNSUPPORTED; + for (i = 0; i < lwis_dev->bts_block_num; i++) { + lwis_dev->bts_indexes[i] = bts_get_bwindex(lwis_dev->bts_block_names[i]); + if (lwis_dev->bts_indexes[i] < 0) { + dev_err(lwis_dev->dev, "Failed to register to BTS, ret: %d\n", + lwis_dev->bts_indexes[i]); + lwis_dev->bts_indexes[i] = BTS_UNSUPPORTED; + } } return 0; @@ -77,7 +81,9 @@ static int lwis_iommu_fault_handler(struct iommu_fault *fault, void *param) pr_err("\n"); lwis_debug_print_transaction_info(lwis_dev); pr_err("\n"); - lwis_debug_print_event_states_info(lwis_dev); + lwis_debug_print_register_io_history(lwis_dev); + pr_err("\n"); + lwis_debug_print_event_states_info(lwis_dev, /*lwis_event_dump_cnt=*/-1); pr_err("\n"); lwis_debug_print_buffer_info(lwis_dev); pr_err("\n"); @@ -95,6 +101,18 @@ static int lwis_iommu_fault_handler(struct iommu_fault *fault, void *param) #endif /* ENABLE_PAGE_FAULT_PANIC */ } +static bool lwis_device_support_bts(struct lwis_device *lwis_dev) +{ + int i; + + for (i = 0; i < lwis_dev->bts_block_num; i++) { + if (lwis_dev->bts_indexes[i] != BTS_UNSUPPORTED) { + return true; + } + } + return false; +} + int lwis_platform_device_enable(struct lwis_device *lwis_dev) { int ret; @@ -131,17 +149,6 @@ int lwis_platform_device_enable(struct lwis_device *lwis_dev) } } - /* - * PM_QOS_CPU_ONLINE_MIN is not defined in 5.4 branch, will need to - * revisit and see if a replacement is needed. - */ -#if 0 - /* Set hardcoded DVFS levels */ - if (!exynos_pm_qos_request_active(&platform->pm_qos_hpg)) { - exynos_pm_qos_add_request(&platform->pm_qos_hpg, - PM_QOS_CPU_ONLINE_MIN, hpg_qos); - } -#endif if (lwis_dev->clock_family != CLOCK_FAMILY_INVALID && lwis_dev->clock_family < NUM_CLOCK_FAMILY) { ret = lwis_platform_update_qos(lwis_dev, core_clock_qos, lwis_dev->clock_family); @@ -160,7 +167,7 @@ int lwis_platform_device_enable(struct lwis_device *lwis_dev) } } - if (lwis_dev->bts_index != BTS_UNSUPPORTED && lwis_dev->bts_scenario_name) { + if (lwis_device_support_bts(lwis_dev) && lwis_dev->bts_scenario_name) { lwis_dev->bts_scenario = bts_get_scenindex(lwis_dev->bts_scenario_name); if (!lwis_dev->bts_scenario) { dev_err(lwis_dev->dev, "Failed to get default camera BTS scenario.\n"); @@ -185,7 +192,7 @@ int lwis_platform_device_disable(struct lwis_device *lwis_dev) return -ENODEV; } - if (lwis_dev->bts_index != BTS_UNSUPPORTED && lwis_dev->bts_scenario_name) { + if (lwis_device_support_bts(lwis_dev) && lwis_dev->bts_scenario_name) { bts_del_scenario(lwis_dev->bts_scenario); } @@ -204,8 +211,7 @@ int lwis_platform_device_disable(struct lwis_device *lwis_dev) return pm_runtime_put_sync(&lwis_dev->plat_dev->dev); } -int lwis_platform_update_qos(struct lwis_device *lwis_dev, int value, - int32_t clock_family) +int lwis_platform_update_qos(struct lwis_device *lwis_dev, int value, int32_t clock_family) { struct lwis_platform *platform; struct exynos_pm_qos_request *qos_req; @@ -279,15 +285,6 @@ int lwis_platform_remove_qos(struct lwis_device *lwis_dev) exynos_pm_qos_remove_request(&platform->pm_qos_mem); } - /* - * pm_qos_hpg is not being used, see comments above regarding - * PM_QOS_CPU_ONLINE_MIN - */ -#if 0 - if (exynos_pm_qos_request_active(&platform->pm_qos_hpg)) { - exynos_pm_qos_remove_request(&platform->pm_qos_hpg); - } -#endif if (exynos_pm_qos_request_active(&platform->pm_qos_int_cam)) { exynos_pm_qos_remove_request(&platform->pm_qos_int_cam); } @@ -300,30 +297,38 @@ int lwis_platform_remove_qos(struct lwis_device *lwis_dev) return 0; } -int lwis_platform_update_bts(struct lwis_device *lwis_dev, unsigned int bw_kb_peak, +int lwis_platform_update_bts(struct lwis_device *lwis_dev, int block, unsigned int bw_kb_peak, unsigned int bw_kb_read, unsigned int bw_kb_write, unsigned int bw_kb_rt) { - int ret = 0; + int ret = 0, bts_index = lwis_dev->bts_indexes[block]; + const char *block_name = lwis_dev->bts_block_names[block]; struct bts_bw bts_request; - if (lwis_dev->bts_index == BTS_UNSUPPORTED) { - dev_info(lwis_dev->dev, "%s doesn't support bts\n", lwis_dev->name); - return ret; + if (block >= lwis_dev->bts_block_num) { + dev_err(lwis_dev->dev, "Invalid block index %d, %s only has %d bts blocks\n", block, + lwis_dev->name, lwis_dev->bts_block_num); + return -EINVAL; + } + + if (bts_index == BTS_UNSUPPORTED) { + dev_err(lwis_dev->dev, "%s block %s doesn't support bts\n", lwis_dev->name, + block_name); + return -EINVAL; } bts_request.peak = bw_kb_peak; bts_request.read = bw_kb_read; bts_request.write = bw_kb_write; bts_request.rt = bw_kb_rt; - ret = bts_update_bw(lwis_dev->bts_index, bts_request); + ret = bts_update_bw(bts_index, bts_request); if (ret < 0) { dev_err(lwis_dev->dev, "Failed to update bandwidth to bts, ret: %d\n", ret); } else { dev_info( lwis_dev->dev, - "Updated bandwidth to bts for device %s: peak: %u, read: %u, write: %u, rt: %u\n", - lwis_dev->name, bw_kb_peak, bw_kb_read, bw_kb_write, bw_kb_rt); + "Updated bandwidth to bts for device %s block %s: peak: %u, read: %u, write: %u, rt: %u\n", + lwis_dev->name, block_name, bw_kb_peak, bw_kb_read, bw_kb_write, bw_kb_rt); } return ret; } diff --git a/platform/busan/lwis_platform_busan_dma.c b/platform/busan/lwis_platform_busan_dma.c index 1fc5dc8..2d82735 100644 --- a/platform/busan/lwis_platform_busan_dma.c +++ b/platform/busan/lwis_platform_busan_dma.c @@ -34,8 +34,8 @@ struct dma_buf *lwis_platform_dma_buffer_alloc(size_t len, unsigned int flags) dmabuf = dma_heap_buffer_alloc(heap, len, O_RDWR, 0); if (IS_ERR_OR_NULL(dmabuf)) { - pr_err("DMA-BUF heap failed to alloc %#zx bytes. Error code %lu\n", - len, PTR_ERR(dmabuf)); + pr_err("DMA-BUF heap failed to alloc %#zx bytes. Error code %lu\n", len, + PTR_ERR(dmabuf)); dmabuf = NULL; } |