diff options
| author | Nishant Prajapati <nishantpjpt@google.com> | 2021-07-20 10:48:52 +0530 |
|---|---|---|
| committer | Nishant Prajapati <nishantpjpt@google.com> | 2021-07-20 13:57:05 +0530 |
| commit | 22bce3194998638e9df4bc051386f66554bf7865 (patch) | |
| tree | ce9a0127395082a18872e1e4f97ee12969379df7 | |
| parent | 393f516340ee24138a1ce819416ae9875c3090e3 (diff) | |
| download | janeiro-22bce3194998638e9df4bc051386f66554bf7865.tar.gz | |
Merge remote-tracking branch 'pro' into android-gs-cloudripper-5.10
* origin/darwinn-2.0: (65 commits)
edgetpu: add support to create coherent/non-coherent mapping
edgetpu: add sysfs attr clients to dump client and wakelock state
edgetpu: janeiro: fix device cleanup sequence
edgetpu: add support to track device removal
edgetpu: PM log clients holding TPU wakelocks at suspend reject time
edgetpu: add list of clients per device
edgetpu: rename group client list fields and iterators
edgetpu: cleanup host DMA direction modification
edgetpu: janeiro bcl port tpu clock divider ratio
edgetpu: fix kernel paging error in edgetpu_mmu_attach_domain
edgetpu: janeiro: use DT property to configure shareability
edgetpu: remove remaining references to emulators
edgetpu: add "is a mobile style device" feature flag
edgetpu: organize objects into mobile vs. mcp groupings
...
Signed-off-by: Nishant Prajapati <nishantpjpt@google.com>
Change-Id: I4b79d9d5b493c8cdb4fa132354e2dca42b125fa7
35 files changed, 693 insertions, 531 deletions
diff --git a/drivers/edgetpu/Kbuild b/drivers/edgetpu/Kbuild index 408ebae..170a69e 100644 --- a/drivers/edgetpu/Kbuild +++ b/drivers/edgetpu/Kbuild @@ -10,9 +10,12 @@ else ccflags-y += -DGIT_REPO_TAG=\"Not\ a\ git\ repository\" endif -edgetpu-fw-objs := edgetpu-firmware.o edgetpu-firmware-util.o edgetpu-shared-fw.o -edgetpu-objs := edgetpu-mailbox.o edgetpu-kci.o edgetpu-telemetry.o edgetpu-mapping.o edgetpu-dmabuf.o edgetpu-async.o edgetpu-iremap-pool.o edgetpu-sw-watchdog.o $(edgetpu-fw-objs) -janeiro-y := janeiro-device.o janeiro-device-group.o janeiro-fs.o janeiro-core.o janeiro-platform.o janeiro-firmware.o janeiro-pm.o janeiro-debug-dump.o janeiro-usage-stats.o janeiro-iommu.o janeiro-wakelock.o $(edgetpu-objs) +edgetpu-objs := edgetpu-mailbox.o edgetpu-kci.o edgetpu-telemetry.o edgetpu-mapping.o edgetpu-dmabuf.o edgetpu-async.o edgetpu-iremap-pool.o edgetpu-sw-watchdog.o edgetpu-firmware.o edgetpu-firmware-util.o + +edgetpu-mobile-objs := edgetpu-mobile-firmware.o + +janeiro-y := janeiro-device.o janeiro-device-group.o janeiro-fs.o janeiro-core.o janeiro-platform.o janeiro-firmware.o janeiro-pm.o janeiro-debug-dump.o janeiro-usage-stats.o janeiro-iommu.o janeiro-wakelock.o $(edgetpu-mobile-objs) $(edgetpu-objs) + CFLAGS_janeiro-fs.o := -DCONFIG_JANEIRO=1 CFLAGS_janeiro-core.o := -DCONFIG_JANEIRO=1 CFLAGS_janeiro-device.o := -DCONFIG_JANEIRO=1 diff --git a/drivers/edgetpu/Kconfig b/drivers/edgetpu/Kconfig index a035b30..0436086 100644 --- a/drivers/edgetpu/Kconfig +++ b/drivers/edgetpu/Kconfig @@ -60,14 +60,6 @@ config EDGETPU_EXTERNAL_WRAPPER_CLASS external classes that wrap the EdgeTPU core driver and is not intended for interactive use. -config EDGETPU_FPGA - bool "Build for EdgeTPU chip FPGA emulation" - depends on EDGETPU_FRAMEWORK - default n - help - Say Y to build for HAPS/Palladium/etc. FPGA emulators, or N to build - silicon with full number of tiles. - config EDGETPU_TELEMETRY_TRACE bool "Build EdgeTPU driver with firmware tracing support" depends on EDGETPU_FRAMEWORK diff --git a/drivers/edgetpu/Makefile b/drivers/edgetpu/Makefile index 89e3da3..caf01c0 100644 --- a/drivers/edgetpu/Makefile +++ b/drivers/edgetpu/Makefile @@ -13,17 +13,18 @@ else ccflags-y += -DGIT_REPO_TAG=\"Not\ a\ git\ repository\" endif -edgetpu-fw-objs := edgetpu-firmware-util.o edgetpu-firmware.o edgetpu-shared-fw.o edgetpu-objs := edgetpu-async.o edgetpu-dmabuf.o edgetpu-iremap-pool.o \ edgetpu-kci.o edgetpu-mailbox.o edgetpu-mapping.o \ edgetpu-sw-watchdog.o edgetpu-telemetry.o \ - $(edgetpu-fw-objs) + edgetpu-firmware-util.o edgetpu-firmware.o + +edgetpu-mobile-objs := edgetpu-mobile-firmware.o janeiro-objs := janeiro-core.o janeiro-debug-dump.o janeiro-device-group.o \ janeiro-device.o janeiro-firmware.o janeiro-fs.o \ janeiro-iommu.o janeiro-platform.o janeiro-pm.o \ janeiro-usage-stats.o janeiro-wakelock.o \ - $(edgetpu-objs) + $(edgetpu-objs) $(edgetpu-mobile-objs) modules modules_install clean: $(MAKE) -C $(KERNEL_SRC) M=$(M) W=1 $(KBUILD_OPTIONS) $(@) diff --git a/drivers/edgetpu/edgetpu-core.c b/drivers/edgetpu/edgetpu-core.c index 7701a71..e7dbaf2 100644 --- a/drivers/edgetpu/edgetpu-core.c +++ b/drivers/edgetpu/edgetpu-core.c @@ -371,6 +371,8 @@ int edgetpu_device_add(struct edgetpu_dev *etdev, INIT_LIST_HEAD(&etdev->groups); etdev->n_groups = 0; etdev->group_join_lockout = false; + mutex_init(&etdev->clients_lock); + INIT_LIST_HEAD(&etdev->clients); etdev->vcid_pool = (1u << EDGETPU_NUM_VCIDS) - 1; mutex_init(&etdev->state_lock); etdev->state = ETDEV_STATE_NOFW; @@ -457,13 +459,19 @@ void edgetpu_device_remove(struct edgetpu_dev *etdev) struct edgetpu_client *edgetpu_client_add(struct edgetpu_dev *etdev) { struct edgetpu_client *client; + struct edgetpu_list_device_client *l = kmalloc(sizeof(*l), GFP_KERNEL); + if (!l) + return ERR_PTR(-ENOMEM); client = kzalloc(sizeof(*client), GFP_KERNEL); - if (!client) + if (!client) { + kfree(l); return ERR_PTR(-ENOMEM); + } client->wakelock = edgetpu_wakelock_alloc(etdev); if (!client->wakelock) { kfree(client); + kfree(l); return ERR_PTR(-ENOMEM); } @@ -474,6 +482,10 @@ struct edgetpu_client *edgetpu_client_add(struct edgetpu_dev *etdev) /* equivalent to edgetpu_client_get() */ refcount_set(&client->count, 1); client->perdie_events = 0; + mutex_lock(&etdev->clients_lock); + l->client = client; + list_add_tail(&l->list, &etdev->clients); + mutex_unlock(&etdev->clients_lock); return client; } @@ -494,14 +506,27 @@ void edgetpu_client_put(struct edgetpu_client *client) void edgetpu_client_remove(struct edgetpu_client *client) { struct edgetpu_dev *etdev; + struct edgetpu_list_device_client *lc; if (IS_ERR_OR_NULL(client)) return; etdev = client->etdev; + mutex_lock(&etdev->clients_lock); + /* remove the client from the device list */ + for_each_list_device_client(etdev, lc) { + if (lc->client == client) { + list_del(&lc->list); + kfree(lc); + break; + } + } + mutex_unlock(&etdev->clients_lock); /* * A quick check without holding client->group_lock. * - * If client doesn't belong to a group then we are fine to not proceed. + * If client doesn't belong to a group then we are fine to not remove + * from groups. + * * If there is a race that the client belongs to a group but is removing * by another process - this will be detected by the check with holding * client->group_lock later. @@ -549,7 +574,8 @@ int edgetpu_alloc_coherent(struct edgetpu_dev *etdev, size_t size, struct edgetpu_coherent_mem *mem, enum edgetpu_context_id context_id) { - const u32 flags = EDGETPU_MMU_DIE | EDGETPU_MMU_32 | EDGETPU_MMU_HOST; + const u32 flags = EDGETPU_MMU_DIE | EDGETPU_MMU_32 | EDGETPU_MMU_HOST | + EDGETPU_MMU_COHERENT; mem->vaddr = dma_alloc_coherent(etdev->dev, size, &mem->dma_addr, GFP_KERNEL); diff --git a/drivers/edgetpu/edgetpu-device-group.c b/drivers/edgetpu/edgetpu-device-group.c index 6172b2c..f51119c 100644 --- a/drivers/edgetpu/edgetpu-device-group.c +++ b/drivers/edgetpu/edgetpu-device-group.c @@ -12,6 +12,7 @@ #include <linux/eventfd.h> #include <linux/iommu.h> #include <linux/kconfig.h> +#include <linux/kernel.h> #include <linux/list.h> #include <linux/mm.h> #include <linux/refcount.h> @@ -40,10 +41,10 @@ #include "edgetpu-p2p-mailbox.h" #endif -#define for_each_list_client(c, group) \ +#define for_each_list_group_client(c, group) \ list_for_each_entry(c, &group->clients, list) -#define for_each_list_client_safe(c, n, group) \ +#define for_each_list_group_client_safe(c, n, group) \ list_for_each_entry_safe(c, n, &group->clients, list) /* Records the mapping and other fields needed for a host buffer mapping */ @@ -311,20 +312,20 @@ static struct edgetpu_client *edgetpu_device_group_leader( { if (group->n_clients < 1 || edgetpu_device_group_is_disbanded(group)) return NULL; - return list_first_entry(&group->clients, struct edgetpu_list_client, - list)->client; + return list_first_entry(&group->clients, + struct edgetpu_list_group_client, list)->client; } static int group_alloc_members(struct edgetpu_device_group *group) { - struct edgetpu_list_client *c; + struct edgetpu_list_group_client *c; int i = 0; group->members = kcalloc(group->n_clients, sizeof(*group->members), GFP_KERNEL); if (!group->members) return -ENOMEM; - for_each_list_client(c, group) { + for_each_list_group_client(c, group) { group->members[i] = c->client; i++; } @@ -607,7 +608,7 @@ void edgetpu_device_group_leave(struct edgetpu_client *client) { struct edgetpu_device_group *group; struct edgetpu_list_group *l; - struct edgetpu_list_client *cur, *nxt; + struct edgetpu_list_group_client *cur, *nxt; bool will_disband = false; mutex_lock(&client->group_lock); @@ -633,8 +634,8 @@ void edgetpu_device_group_leave(struct edgetpu_client *client) /* release the group before removing any members */ edgetpu_device_group_release(group); - /* removes the client from the list */ - for_each_list_client_safe(cur, nxt, group) { + /* removes the client from the group list */ + for_each_list_group_client_safe(cur, nxt, group) { if (cur->client == client) { list_del(&cur->list); kfree(cur); @@ -704,18 +705,11 @@ edgetpu_device_group_alloc(struct edgetpu_client *client, group->mbox_attr = *attr; if (attr->priority & EDGETPU_PRIORITY_DETACHABLE) group->mailbox_detachable = true; - /* adds @client as the first entry */ - ret = edgetpu_device_group_add(group, client); - if (ret) { - etdev_dbg(group->etdev, "%s: group %u add failed ret=%d", - __func__, group->workload_id, ret); - goto error_put_group; - } etdomain = edgetpu_mmu_alloc_domain(group->etdev); if (!etdomain) { ret = -ENOMEM; - goto error_leave_group; + goto error_put_group; } group->etdomain = etdomain; if (etdomain->token != EDGETPU_DOMAIN_TOKEN_END) @@ -723,10 +717,18 @@ edgetpu_device_group_alloc(struct edgetpu_client *client, EDGETPU_CONTEXT_DOMAIN_TOKEN | etdomain->token; else group->context_id = EDGETPU_CONTEXT_INVALID; + + /* adds @client as the first entry */ + ret = edgetpu_device_group_add(group, client); + if (ret) { + etdev_dbg(group->etdev, "%s: group %u add failed ret=%d", + __func__, group->workload_id, ret); + goto error_free_mmu_domain; + } return group; -error_leave_group: - edgetpu_device_group_leave(client); +error_free_mmu_domain: + edgetpu_mmu_free_domain(group->etdev, group->etdomain); error_put_group: edgetpu_device_group_put(group); error: @@ -736,7 +738,7 @@ error: int edgetpu_device_group_add(struct edgetpu_device_group *group, struct edgetpu_client *client) { - struct edgetpu_list_client *c; + struct edgetpu_list_group_client *c; int ret = 0; mutex_lock(&client->group_lock); @@ -751,7 +753,7 @@ int edgetpu_device_group_add(struct edgetpu_device_group *group, goto out; } - for_each_list_client(c, group) { + for_each_list_group_client(c, group) { if (!edgetpu_clients_groupable(c->client, client)) { ret = -EINVAL; goto out; @@ -940,6 +942,7 @@ static int edgetpu_map_iova_sgt_worker(struct iova_mapping_worker_param *param) edgetpu_mmu_reserve(etdev, map->alloc_iova, map->alloc_size); ret = edgetpu_mmu_map_iova_sgt(etdev, map->device_address, &hmap->sg_tables[i], map->dir, + map_to_mmu_flags(map->flags), ctx_id); if (ret) edgetpu_mmu_free(etdev, map->alloc_iova, map->alloc_size); @@ -1112,7 +1115,7 @@ static void edgetpu_host_map_show(struct edgetpu_mapping *map, seq_printf(s, " die %u: ", map->die_index); seq_printf(s, "0x%llx %lu %s 0x%llx %pap %pad\n", map->device_address + cur_offset, - sg_dma_len(sg) / PAGE_SIZE, + DIV_ROUND_UP(sg_dma_len(sg), PAGE_SIZE), edgetpu_dma_dir_rw_s(map->dir), map->host_address + cur_offset, &phys_addr, &dma_addr); @@ -1132,7 +1135,6 @@ static struct page **edgetpu_pin_user_pages(struct edgetpu_device_group *group, { u64 host_addr = untagged_addr(arg->host_address); u64 size = arg->size; - const enum dma_data_direction dir = arg->flags & EDGETPU_MAP_DIR_MASK; uint num_pages; ulong offset; struct edgetpu_dev *etdev = group->etdev; @@ -1148,12 +1150,8 @@ static struct page **edgetpu_pin_user_pages(struct edgetpu_device_group *group, /* overflow check */ if (unlikely((size + offset) / PAGE_SIZE >= UINT_MAX - 1 || size + offset < size)) return ERR_PTR(-ENOMEM); - num_pages = (size + offset) / PAGE_SIZE; - if ((size + offset) % PAGE_SIZE) - num_pages++; - - etdev_dbg(etdev, "%s: hostaddr=0x%llx pages=%u dir=%x", __func__, - host_addr, num_pages, dir); + num_pages = DIV_ROUND_UP((size + offset), PAGE_SIZE); + etdev_dbg(etdev, "%s: hostaddr=0x%llx pages=%u", __func__, host_addr, num_pages); /* * "num_pages" is decided from user-space arguments, don't show warnings * when facing malicious input. @@ -1216,7 +1214,6 @@ alloc_mapping_from_useraddr(struct edgetpu_device_group *group, u64 host_addr, { struct edgetpu_dev *etdev = group->etdev; struct edgetpu_host_map *hmap; - const enum dma_data_direction dir = flags & EDGETPU_MAP_DIR_MASK; int n; struct sg_table *sgt; int i; @@ -1229,7 +1226,7 @@ alloc_mapping_from_useraddr(struct edgetpu_device_group *group, u64 host_addr, } hmap->map.host_address = host_addr; - hmap->map.dir = dir; + hmap->map.dir = map_flag_to_host_dma_dir(flags); hmap->map.priv = edgetpu_device_group_get(group); hmap->map.release = edgetpu_unmap_node; hmap->map.show = edgetpu_host_map_show; @@ -1528,6 +1525,10 @@ int edgetpu_device_group_sync_buffer(struct edgetpu_device_group *group, struct edgetpu_mapping *map; int ret = 0; tpu_addr_t tpu_addr = arg->device_address; + /* + * Sync operations don't care the data correctness of prefetch by TPU CPU if they mean to + * sync FROM_DEVICE only, so @dir here doesn't need to be wrapped with host_dma_dir(). + */ enum dma_data_direction dir = arg->flags & EDGETPU_MAP_DIR_MASK; struct edgetpu_host_map *hmap; @@ -1600,12 +1601,14 @@ void edgetpu_group_mappings_show(struct edgetpu_device_group *group, seq_puts(s, "VII queues:\n"); seq_printf(s, " 0x%llx %lu cmdq 0x%llx %pad\n", group->vii.cmd_queue_mem.tpu_addr, - group->vii.cmd_queue_mem.size / PAGE_SIZE, + DIV_ROUND_UP(group->vii.cmd_queue_mem.size, + PAGE_SIZE), group->vii.cmd_queue_mem.host_addr, &group->vii.cmd_queue_mem.dma_addr); seq_printf(s, " 0x%llx %lu rspq 0x%llx %pad\n", group->vii.resp_queue_mem.tpu_addr, - group->vii.resp_queue_mem.size / PAGE_SIZE, + DIV_ROUND_UP(group->vii.resp_queue_mem.size, + PAGE_SIZE), group->vii.resp_queue_mem.host_addr, &group->vii.resp_queue_mem.dma_addr); } diff --git a/drivers/edgetpu/edgetpu-device-group.h b/drivers/edgetpu/edgetpu-device-group.h index 7b20dd0..7ec262f 100644 --- a/drivers/edgetpu/edgetpu-device-group.h +++ b/drivers/edgetpu/edgetpu-device-group.h @@ -24,7 +24,7 @@ #include "edgetpu.h" /* entry of edgetpu_device_group#clients */ -struct edgetpu_list_client { +struct edgetpu_list_group_client { struct list_head list; struct edgetpu_client *client; }; diff --git a/drivers/edgetpu/edgetpu-dmabuf.c b/drivers/edgetpu/edgetpu-dmabuf.c index 1c89178..b0dc9a1 100644 --- a/drivers/edgetpu/edgetpu-dmabuf.c +++ b/drivers/edgetpu/edgetpu-dmabuf.c @@ -10,6 +10,7 @@ #include <linux/dma-direction.h> #include <linux/dma-fence.h> #include <linux/dma-mapping.h> +#include <linux/kernel.h> #include <linux/ktime.h> #include <linux/list.h> #include <linux/seq_file.h> @@ -91,10 +92,11 @@ static const struct dma_fence_ops edgetpu_dma_fence_ops; static int etdev_add_translations(struct edgetpu_dev *etdev, tpu_addr_t tpu_addr, struct dmabuf_map_entry *entry, + u32 mmu_flags, enum dma_data_direction dir, enum edgetpu_context_id ctx_id) { - const int prot = __dma_dir_to_iommu_prot(dir, etdev->dev); + int prot = mmu_flag_to_iommu_prot(mmu_flags, etdev->dev, dir); uint i; u64 offset = 0; int ret; @@ -145,7 +147,7 @@ static void etdev_remove_translations(struct edgetpu_dev *etdev, */ static int etdev_map_dmabuf(struct edgetpu_dev *etdev, struct edgetpu_dmabuf_map *dmap, - enum dma_data_direction dir, tpu_addr_t *tpu_addr_p) + tpu_addr_t *tpu_addr_p) { struct edgetpu_device_group *group = dmap->map.priv; const enum edgetpu_context_id ctx_id = @@ -164,7 +166,7 @@ static int etdev_map_dmabuf(struct edgetpu_dev *etdev, return -ENOSPC; } else { tpu_addr = - edgetpu_mmu_tpu_map_sgt(etdev, &entry->shrunk_sgt, dir, + edgetpu_mmu_tpu_map_sgt(etdev, &entry->shrunk_sgt, dmap->map.dir, ctx_id, dmap->mmu_flags); if (!tpu_addr) return -ENOSPC; @@ -202,7 +204,7 @@ static void etdev_unmap_dmabuf(struct edgetpu_dev *etdev, */ static int group_map_dmabuf(struct edgetpu_device_group *group, struct edgetpu_dmabuf_map *dmap, - enum dma_data_direction dir, tpu_addr_t *tpu_addr_p) + tpu_addr_t *tpu_addr_p) { const enum edgetpu_context_id ctx_id = edgetpu_group_context_id_locked(group); @@ -211,7 +213,7 @@ static int group_map_dmabuf(struct edgetpu_device_group *group, uint i; int ret; - ret = etdev_map_dmabuf(etdev, dmap, dir, &tpu_addr); + ret = etdev_map_dmabuf(etdev, dmap, &tpu_addr); if (ret) return ret; for (i = 1; i < group->n_clients; i++) { @@ -221,7 +223,7 @@ static int group_map_dmabuf(struct edgetpu_device_group *group, continue; } ret = etdev_add_translations(etdev, tpu_addr, &dmap->entries[i], - dir, ctx_id); + dmap->mmu_flags, dmap->map.dir, ctx_id); if (ret) goto err_remove; } @@ -276,7 +278,7 @@ static void dmabuf_map_callback_release(struct edgetpu_mapping *map) struct edgetpu_dmabuf_map *dmap = container_of(map, struct edgetpu_dmabuf_map, map); struct edgetpu_device_group *group = map->priv; - const enum dma_data_direction dir = edgetpu_host_dma_dir(map->dir); + const enum dma_data_direction dir = map->dir; tpu_addr_t tpu_addr = map->device_address; struct edgetpu_dev *etdev; uint i; @@ -337,12 +339,14 @@ static void dmabuf_map_callback_show(struct edgetpu_mapping *map, if (IS_MIRRORED(dmap->map.flags)) seq_printf(s, " <%s> mirrored: iova=0x%llx pages=%llu %s", - dmap->dmabufs[0]->exp_name, map->device_address, dmap->size / PAGE_SIZE, + dmap->dmabufs[0]->exp_name, map->device_address, + DIV_ROUND_UP(dmap->size, PAGE_SIZE), edgetpu_dma_dir_rw_s(map->dir)); else seq_printf(s, " <%s> die %u: iova=0x%llx pages=%llu %s", dmap->dmabufs[0]->exp_name, map->die_index, map->device_address, - dmap->size / PAGE_SIZE, edgetpu_dma_dir_rw_s(map->dir)); + DIV_ROUND_UP(dmap->size, PAGE_SIZE), + edgetpu_dma_dir_rw_s(map->dir)); edgetpu_device_dram_dmabuf_info_show(dmap->dmabufs[0], s); seq_puts(s, " dma="); @@ -403,7 +407,7 @@ static void dmabuf_bulk_map_callback_release(struct edgetpu_mapping *map) struct edgetpu_dmabuf_map *bmap = container_of(map, struct edgetpu_dmabuf_map, map); struct edgetpu_device_group *group = map->priv; - const enum dma_data_direction dir = edgetpu_host_dma_dir(map->dir); + const enum dma_data_direction dir = map->dir; const tpu_addr_t tpu_addr = map->device_address; int i; @@ -414,8 +418,7 @@ static void dmabuf_bulk_map_callback_release(struct edgetpu_mapping *map) sg_free_table(&entry->shrunk_sgt); if (entry->sgt) - dma_buf_unmap_attachment(entry->attachment, entry->sgt, - dir); + dma_buf_unmap_attachment(entry->attachment, entry->sgt, dir); if (entry->attachment) dma_buf_detach(bmap->dmabufs[i], entry->attachment); if (bmap->dmabufs[i]) @@ -435,7 +438,7 @@ static void dmabuf_bulk_map_callback_show(struct edgetpu_mapping *map, int i; seq_printf(s, " bulk: iova=0x%llx pages=%llu %s\n", - map->device_address, bmap->size / PAGE_SIZE, + map->device_address, DIV_ROUND_UP(bmap->size, PAGE_SIZE), edgetpu_dma_dir_rw_s(map->dir)); for (i = 0; i < bmap->num_entries; i++) { if (!bmap->dmabufs[i]) { @@ -633,8 +636,7 @@ int edgetpu_map_dmabuf(struct edgetpu_device_group *group, struct dma_buf *dmabuf; edgetpu_map_flag_t flags = arg->flags; u64 size; - const enum dma_data_direction dir = - edgetpu_host_dma_dir(flags & EDGETPU_MAP_DIR_MASK); + const enum dma_data_direction dir = map_flag_to_host_dma_dir(flags); struct edgetpu_dev *etdev; struct edgetpu_dmabuf_map *dmap; tpu_addr_t tpu_addr; @@ -681,7 +683,7 @@ int edgetpu_map_dmabuf(struct edgetpu_device_group *group, goto err_release_map; } } - ret = group_map_dmabuf(group, dmap, dir, &tpu_addr); + ret = group_map_dmabuf(group, dmap, &tpu_addr); if (ret) { etdev_dbg(group->etdev, "%s: group_map_dmabuf returns %d\n", @@ -705,7 +707,7 @@ int edgetpu_map_dmabuf(struct edgetpu_device_group *group, __func__, ret); goto err_release_map; } - ret = etdev_map_dmabuf(etdev, dmap, dir, &tpu_addr); + ret = etdev_map_dmabuf(etdev, dmap, &tpu_addr); if (ret) { etdev_dbg(group->etdev, "%s: etdev_map_dmabuf returns %d\n", @@ -771,8 +773,7 @@ out_unlock: int edgetpu_map_bulk_dmabuf(struct edgetpu_device_group *group, struct edgetpu_map_bulk_dmabuf_ioctl *arg) { - const enum dma_data_direction dir = - edgetpu_host_dma_dir(arg->flags & EDGETPU_MAP_DIR_MASK); + const enum dma_data_direction dir = map_flag_to_host_dma_dir(arg->flags); int ret = -EINVAL; struct edgetpu_dmabuf_map *bmap; struct dma_buf *dmabuf; @@ -820,7 +821,7 @@ int edgetpu_map_bulk_dmabuf(struct edgetpu_device_group *group, if (ret) goto err_release_bmap; } - ret = group_map_dmabuf(group, bmap, dir, &tpu_addr); + ret = group_map_dmabuf(group, bmap, &tpu_addr); if (ret) goto err_release_bmap; bmap->map.device_address = tpu_addr; diff --git a/drivers/edgetpu/edgetpu-firmware.c b/drivers/edgetpu/edgetpu-firmware.c index 2a1e577..f2d6a0c 100644 --- a/drivers/edgetpu/edgetpu-firmware.c +++ b/drivers/edgetpu/edgetpu-firmware.c @@ -8,6 +8,7 @@ #include <linux/delay.h> #include <linux/device.h> #include <linux/firmware.h> +#include <linux/module.h> #include <linux/mutex.h> #include <linux/seq_file.h> #include <linux/slab.h> @@ -21,32 +22,14 @@ #include "edgetpu-internal.h" #include "edgetpu-kci.h" #include "edgetpu-pm.h" -#include "edgetpu-shared-fw.h" #include "edgetpu-sw-watchdog.h" #include "edgetpu-telemetry.h" -/* - * Descriptor for loaded firmware, either in shared buffer mode or legacy mode - * (non-shared, custom allocated memory). - */ -struct edgetpu_firmware_desc { - /* - * Mode independent buffer information. This is either passed into or - * updated by handlers. - */ - struct edgetpu_firmware_buffer buf; - /* - * Shared firmware buffer when we're using shared buffer mode. This - * pointer to keep and release the reference count on unloading this - * shared firmware buffer. - * - * This is NULL when firmware is loaded in legacy mode. - */ - struct edgetpu_shared_fw_buffer *shared_buf; -}; +static char *firmware_name; +module_param(firmware_name, charp, 0660); struct edgetpu_firmware_private { - const struct edgetpu_firmware_handlers *handlers; + const struct edgetpu_firmware_chip_data *chip_fw; void *data; /* for edgetpu_firmware_(set/get)_data */ struct mutex fw_desc_lock; @@ -66,120 +49,19 @@ void *edgetpu_firmware_get_data(struct edgetpu_firmware *et_fw) return et_fw->p->data; } -static int edgetpu_firmware_legacy_load_locked( - struct edgetpu_firmware *et_fw, - struct edgetpu_firmware_desc *fw_desc, const char *name) -{ - int ret; - struct edgetpu_dev *etdev = et_fw->etdev; - struct device *dev = etdev->dev; - const struct firmware *fw; - size_t aligned_size; - - ret = request_firmware(&fw, name, dev); - if (ret) { - etdev_dbg(etdev, - "%s: request '%s' failed: %d\n", __func__, name, ret); - return ret; - } - - aligned_size = ALIGN(fw->size, fw_desc->buf.used_size_align); - if (aligned_size > fw_desc->buf.alloc_size) { - etdev_dbg(etdev, - "%s: firmware buffer too small: alloc size=0x%zx, required size=0x%zx\n", - __func__, fw_desc->buf.alloc_size, aligned_size); - ret = -ENOSPC; - goto out_release_firmware; - } - - memcpy(fw_desc->buf.vaddr, fw->data, fw->size); - fw_desc->buf.used_size = aligned_size; - fw_desc->buf.name = kstrdup(name, GFP_KERNEL); - -out_release_firmware: - release_firmware(fw); - return ret; -} - -static void edgetpu_firmware_legacy_unload_locked( - struct edgetpu_firmware *et_fw, - struct edgetpu_firmware_desc *fw_desc) -{ - kfree(fw_desc->buf.name); - fw_desc->buf.name = NULL; - fw_desc->buf.used_size = 0; -} - -static int edgetpu_firmware_shared_load_locked( - struct edgetpu_firmware *et_fw, - struct edgetpu_firmware_desc *fw_desc, const char *name) -{ - int ret; - struct edgetpu_dev *etdev = et_fw->etdev; - struct edgetpu_shared_fw_buffer *shared_buf; - - shared_buf = edgetpu_shared_fw_load(name, etdev); - if (IS_ERR(shared_buf)) { - ret = PTR_ERR(shared_buf); - etdev_dbg(etdev, "shared buffer loading failed: %d\n", ret); - return ret; - } - fw_desc->shared_buf = shared_buf; - fw_desc->buf.vaddr = edgetpu_shared_fw_buffer_vaddr(shared_buf); - fw_desc->buf.alloc_size = edgetpu_shared_fw_buffer_size(shared_buf); - fw_desc->buf.used_size = fw_desc->buf.alloc_size; - fw_desc->buf.name = edgetpu_shared_fw_buffer_name(shared_buf); - return 0; -} - -static void edgetpu_firmware_shared_unload_locked( - struct edgetpu_firmware *et_fw, - struct edgetpu_firmware_desc *fw_desc) -{ - fw_desc->buf.vaddr = NULL; - fw_desc->buf.alloc_size = 0; - fw_desc->buf.used_size = 0; - fw_desc->buf.name = NULL; - edgetpu_shared_fw_put(fw_desc->shared_buf); - fw_desc->shared_buf = NULL; -} - -static int edgetpu_firmware_do_load_locked( - struct edgetpu_firmware *et_fw, - struct edgetpu_firmware_desc *fw_desc, const char *name) -{ - /* Use shared firmware from host if not allocated a buffer space. */ - if (!fw_desc->buf.vaddr) - return edgetpu_firmware_shared_load_locked(et_fw, fw_desc, - name); - else - return edgetpu_firmware_legacy_load_locked(et_fw, fw_desc, - name); -} - -static void edgetpu_firmware_do_unload_locked( - struct edgetpu_firmware *et_fw, - struct edgetpu_firmware_desc *fw_desc) -{ - if (fw_desc->shared_buf) - edgetpu_firmware_shared_unload_locked(et_fw, fw_desc); - else - edgetpu_firmware_legacy_unload_locked(et_fw, fw_desc); -} - static int edgetpu_firmware_load_locked( struct edgetpu_firmware *et_fw, struct edgetpu_firmware_desc *fw_desc, const char *name, enum edgetpu_firmware_flags flags) { - const struct edgetpu_firmware_handlers *handlers = et_fw->p->handlers; + const struct edgetpu_firmware_chip_data *chip_fw = et_fw->p->chip_fw; struct edgetpu_dev *etdev = et_fw->etdev; int ret; fw_desc->buf.flags = flags; - if (handlers && handlers->alloc_buffer) { - ret = handlers->alloc_buffer(et_fw, &fw_desc->buf); + if (chip_fw->alloc_buffer) { + ret = chip_fw->alloc_buffer(et_fw, &fw_desc->buf); if (ret) { etdev_err(etdev, "handler alloc_buffer failed: %d\n", ret); @@ -187,28 +69,28 @@ static int edgetpu_firmware_load_locked( } } - ret = edgetpu_firmware_do_load_locked(et_fw, fw_desc, name); + ret = edgetpu_firmware_chip_load_locked(et_fw, fw_desc, name); if (ret) { etdev_err(etdev, "firmware request failed: %d\n", ret); goto out_free_buffer; } - if (handlers && handlers->setup_buffer) { - ret = handlers->setup_buffer(et_fw, &fw_desc->buf); + if (chip_fw->setup_buffer) { + ret = chip_fw->setup_buffer(et_fw, &fw_desc->buf); if (ret) { etdev_err(etdev, "handler setup_buffer failed: %d\n", ret); - goto out_do_unload_locked; + goto out_unload_locked; } } return 0; -out_do_unload_locked: - edgetpu_firmware_do_unload_locked(et_fw, fw_desc); +out_unload_locked: + edgetpu_firmware_chip_unload_locked(et_fw, fw_desc); out_free_buffer: - if (handlers && handlers->free_buffer) - handlers->free_buffer(et_fw, &fw_desc->buf); + if (chip_fw->free_buffer) + chip_fw->free_buffer(et_fw, &fw_desc->buf); return ret; } @@ -216,19 +98,19 @@ static void edgetpu_firmware_unload_locked( struct edgetpu_firmware *et_fw, struct edgetpu_firmware_desc *fw_desc) { - const struct edgetpu_firmware_handlers *handlers = et_fw->p->handlers; + const struct edgetpu_firmware_chip_data *chip_fw = et_fw->p->chip_fw; /* * Platform specific implementation for cleaning up allocated buffer. */ - if (handlers && handlers->teardown_buffer) - handlers->teardown_buffer(et_fw, &fw_desc->buf); - edgetpu_firmware_do_unload_locked(et_fw, fw_desc); + if (chip_fw->teardown_buffer) + chip_fw->teardown_buffer(et_fw, &fw_desc->buf); + edgetpu_firmware_chip_unload_locked(et_fw, fw_desc); /* * Platform specific implementation for freeing allocated buffer. */ - if (handlers && handlers->free_buffer) - handlers->free_buffer(et_fw, &fw_desc->buf); + if (chip_fw->free_buffer) + chip_fw->free_buffer(et_fw, &fw_desc->buf); } static char *fw_flavor_str(enum edgetpu_fw_flavor fw_flavor) @@ -445,7 +327,7 @@ int edgetpu_firmware_run_locked(struct edgetpu_firmware *et_fw, const char *name, enum edgetpu_firmware_flags flags) { - const struct edgetpu_firmware_handlers *handlers = et_fw->p->handlers; + const struct edgetpu_firmware_chip_data *chip_fw = et_fw->p->chip_fw; struct edgetpu_firmware_desc new_fw_desc; int ret; bool is_bl1_run = (flags & FW_BL1); @@ -460,9 +342,9 @@ int edgetpu_firmware_run_locked(struct edgetpu_firmware *et_fw, goto out_failed; etdev_dbg(et_fw->etdev, "run fw %s flags=0x%x", name, flags); - if (handlers && handlers->prepare_run) { + if (chip_fw->prepare_run) { /* Note this may recursively call us to run BL1 */ - ret = handlers->prepare_run(et_fw, &new_fw_desc.buf); + ret = chip_fw->prepare_run(et_fw, &new_fw_desc.buf); if (ret) goto out_unload_new_fw; } @@ -486,18 +368,18 @@ int edgetpu_firmware_run_locked(struct edgetpu_firmware *et_fw, if (!ret && !is_bl1_run && et_fw->p->fw_info.fw_flavor != FW_FLAVOR_BL1) edgetpu_sw_wdt_start(et_fw->etdev); - if (!ret && !is_bl1_run && handlers && handlers->launch_complete) - handlers->launch_complete(et_fw); - else if (ret && handlers && handlers->launch_failed) - handlers->launch_failed(et_fw, ret); + if (!ret && !is_bl1_run && chip_fw->launch_complete) + chip_fw->launch_complete(et_fw); + else if (ret && chip_fw->launch_failed) + chip_fw->launch_failed(et_fw, ret); edgetpu_firmware_set_state(et_fw, ret); return ret; out_unload_new_fw: edgetpu_firmware_unload_locked(et_fw, &new_fw_desc); out_failed: - if (handlers && handlers->launch_failed) - handlers->launch_failed(et_fw, ret); + if (chip_fw->launch_failed) + chip_fw->launch_failed(et_fw, ret); edgetpu_firmware_set_state(et_fw, ret); return ret; } @@ -528,6 +410,31 @@ int edgetpu_firmware_run(struct edgetpu_dev *etdev, const char *name, return ret; } +int edgetpu_firmware_run_default_locked(struct edgetpu_dev *etdev) +{ + struct edgetpu_firmware *et_fw = etdev->firmware; + const char *run_firmware_name = + et_fw->p->chip_fw->default_firmware_name; + + if (firmware_name && *firmware_name) + run_firmware_name = firmware_name; + + return edgetpu_firmware_run_locked(etdev->firmware, run_firmware_name, + FW_DEFAULT); +} + +int edgetpu_firmware_run_default(struct edgetpu_dev *etdev) +{ + struct edgetpu_firmware *et_fw = etdev->firmware; + const char *run_firmware_name = + et_fw->p->chip_fw->default_firmware_name; + + if (firmware_name && *firmware_name) + run_firmware_name = firmware_name; + + return edgetpu_firmware_run(etdev, run_firmware_name, FW_DEFAULT); +} + bool edgetpu_firmware_is_loading(struct edgetpu_dev *etdev) { struct edgetpu_firmware *et_fw = etdev->firmware; @@ -558,10 +465,10 @@ edgetpu_firmware_set_status_locked(struct edgetpu_dev *etdev, } /* Caller must hold firmware lock for loading. */ -int edgetpu_firmware_restart_locked(struct edgetpu_dev *etdev) +int edgetpu_firmware_restart_locked(struct edgetpu_dev *etdev, bool force_reset) { struct edgetpu_firmware *et_fw = etdev->firmware; - const struct edgetpu_firmware_handlers *handlers = et_fw->p->handlers; + const struct edgetpu_firmware_chip_data *chip_fw = et_fw->p->chip_fw; int ret = -1; edgetpu_firmware_set_loading(et_fw); @@ -570,10 +477,10 @@ int edgetpu_firmware_restart_locked(struct edgetpu_dev *etdev) * Try restarting the firmware first, fall back to normal firmware start * if this fails. */ - if (handlers && handlers->restart) - ret = handlers->restart(et_fw); - if (ret && handlers && handlers->prepare_run) { - ret = handlers->prepare_run(et_fw, &et_fw->p->fw_desc.buf); + if (chip_fw->restart) + ret = chip_fw->restart(et_fw, force_reset); + if (ret && chip_fw->prepare_run) { + ret = chip_fw->prepare_run(et_fw, &et_fw->p->fw_desc.buf); if (ret) goto out; } @@ -637,7 +544,7 @@ static ssize_t load_firmware_store( return PTR_ERR(name); etdev_info(etdev, "loading firmware %s\n", name); - ret = edgetpu_chip_firmware_run(etdev, name, 0); + ret = edgetpu_firmware_run(etdev, name, 0); kfree(name); @@ -726,14 +633,14 @@ static void edgetpu_firmware_wdt_timeout_action(void *data) ret = edgetpu_firmware_pm_get(et_fw); if (!ret) { - ret = edgetpu_firmware_restart_locked(etdev); + ret = edgetpu_firmware_restart_locked(etdev, true); edgetpu_pm_put(etdev->pm); } edgetpu_firmware_unlock(etdev); } int edgetpu_firmware_create(struct edgetpu_dev *etdev, - const struct edgetpu_firmware_handlers *handlers) + const struct edgetpu_firmware_chip_data *chip_fw) { struct edgetpu_firmware *et_fw; int ret; @@ -751,7 +658,7 @@ int edgetpu_firmware_create(struct edgetpu_dev *etdev, ret = -ENOMEM; goto out_kfree_et_fw; } - et_fw->p->handlers = handlers; + et_fw->p->chip_fw = chip_fw; mutex_init(&et_fw->p->fw_desc_lock); @@ -759,8 +666,8 @@ int edgetpu_firmware_create(struct edgetpu_dev *etdev, if (ret) goto out_kfree_et_fw_p; - if (handlers && handlers->after_create) { - ret = handlers->after_create(et_fw); + if (chip_fw->after_create) { + ret = chip_fw->after_create(et_fw); if (ret) { etdev_dbg(etdev, "%s: after create handler failed: %d\n", @@ -791,20 +698,20 @@ out_kfree_et_fw: void edgetpu_firmware_destroy(struct edgetpu_dev *etdev) { struct edgetpu_firmware *et_fw = etdev->firmware; - const struct edgetpu_firmware_handlers *handlers; + const struct edgetpu_firmware_chip_data *chip_fw; if (!et_fw) return; edgetpu_sw_wdt_destroy(etdev); if (et_fw->p) { - handlers = et_fw->p->handlers; + chip_fw = et_fw->p->chip_fw; /* * Platform specific implementation, which includes stop * running firmware. */ - if (handlers && handlers->before_destroy) - handlers->before_destroy(et_fw); + if (chip_fw->before_destroy) + chip_fw->before_destroy(et_fw); } device_remove_group(etdev->dev, &edgetpu_firmware_attr_group); @@ -839,6 +746,6 @@ void edgetpu_firmware_mappings_show(struct edgetpu_dev *etdev, fw_iova_target = fw_buf->dram_tpa ? fw_buf->dram_tpa : fw_buf->dma_addr; iova = edgetpu_chip_firmware_iova(etdev); seq_printf(s, " 0x%lx %lu fw - %pad %s\n", iova, - fw_buf->alloc_size / PAGE_SIZE, &fw_iova_target, + DIV_ROUND_UP(fw_buf->alloc_size, PAGE_SIZE), &fw_iova_target, fw_buf->flags & FW_ONDEV ? "dev" : ""); } diff --git a/drivers/edgetpu/edgetpu-firmware.h b/drivers/edgetpu/edgetpu-firmware.h index 3b784c5..1cdfaa1 100644 --- a/drivers/edgetpu/edgetpu-firmware.h +++ b/drivers/edgetpu/edgetpu-firmware.h @@ -95,10 +95,34 @@ struct edgetpu_firmware_buffer { }; /* - * Each handler returns 0 to indicate success, non-zero value to - * indicate error. + * Descriptor for loaded firmware, either in shared buffer mode or carveout mode + * (non-shared, custom allocated memory). */ -struct edgetpu_firmware_handlers { +struct edgetpu_firmware_desc { + /* + * Mode independent buffer information. This is either passed into or + * updated by handlers. + */ + struct edgetpu_firmware_buffer buf; + /* + * Shared firmware buffer when we're using shared buffer mode. This + * pointer to keep and release the reference count on unloading this + * shared firmware buffer. + * + * This is NULL when firmware is loaded in carveout mode. + */ + struct edgetpu_shared_fw_buffer *shared_buf; +}; + +struct edgetpu_firmware_chip_data { + /* Name of default firmware image for this chip. */ + const char *default_firmware_name; + + /* + * Chip handlers called by common firmware processing. + * Each handler returns 0 to indicate success, non-zero value to + * indicate error. + */ int (*after_create)(struct edgetpu_firmware *et_fw); /* * Release resource used in platform specific implementation, @@ -154,19 +178,22 @@ struct edgetpu_firmware_handlers { * Optional platform-specific handler to restart an already loaded * firmware. */ - int (*restart)(struct edgetpu_firmware *et_fw); + int (*restart)(struct edgetpu_firmware *et_fw, bool force_reset); }; /* - * Top-level chip-specific run firmware routine. - * Calls edgetpu_firmware_run() one or more times as appropriate for chip- - * specific one- or two-stage bootloader processing. - * - * @name: the name passed into underlying request_firmware API - * @flags: edgetpu_firmware_flags for the image + * Chip-dependent (actually chip family dependent, mobile vs. MCP) calls + * for loading/unloading firmware images. These handle chip-specified carveout + * buffers vs. shared firmware handling for multi-chip platforms. Used by the + * common firmware layer. */ -int edgetpu_chip_firmware_run(struct edgetpu_dev *etdev, const char *name, - enum edgetpu_firmware_flags flags); +int edgetpu_firmware_chip_load_locked( + struct edgetpu_firmware *et_fw, + struct edgetpu_firmware_desc *fw_desc, const char *name); +void edgetpu_firmware_chip_unload_locked( + struct edgetpu_firmware *et_fw, + struct edgetpu_firmware_desc *fw_desc); + /* * Returns the chip-specific IOVA where the firmware is mapped. * @@ -175,13 +202,20 @@ int edgetpu_chip_firmware_run(struct edgetpu_dev *etdev, const char *name, unsigned long edgetpu_chip_firmware_iova(struct edgetpu_dev *etdev); /* - * Load and run firmware. Called by edgetpu_chip_firmware_run(). + * Load and run firmware. * @name: the name passed into underlying request_firmware API * @flags: edgetpu_firmware_flags for the image + * Used internally by the sysfs load interface and by unit tests. */ int edgetpu_firmware_run(struct edgetpu_dev *etdev, const char *name, enum edgetpu_firmware_flags flags); +/* Load and run the default firmware name for the chip. */ +int edgetpu_firmware_run_default(struct edgetpu_dev *etdev); + +/* Runs default firmware for the chip, caller holds FW/PM locks */ +int edgetpu_firmware_run_default_locked(struct edgetpu_dev *etdev); + /* * Private data set and used by handlers. It is expected to * allocate and set the data on after_create() and release on @@ -191,7 +225,7 @@ void edgetpu_firmware_set_data(struct edgetpu_firmware *et_fw, void *data); void *edgetpu_firmware_get_data(struct edgetpu_firmware *et_fw); int edgetpu_firmware_create(struct edgetpu_dev *etdev, - const struct edgetpu_firmware_handlers *handlers); + const struct edgetpu_firmware_chip_data *chip_fw); void edgetpu_firmware_destroy(struct edgetpu_dev *etdev); void edgetpu_firmware_mappings_show(struct edgetpu_dev *etdev, struct seq_file *s); @@ -223,15 +257,16 @@ edgetpu_firmware_set_status_locked(struct edgetpu_dev *etdev, /* * Restarts the last firmware image loaded * Intended for power managed devices to re-run the firmware without a full - * reload from the file system + * reload from the file system. + * Optionally, force a CPU reset to recover from a bad firmware state. */ -int edgetpu_firmware_restart_locked(struct edgetpu_dev *etdev); +int edgetpu_firmware_restart_locked(struct edgetpu_dev *etdev, + bool force_reset); /* * Loads and runs the specified firmware assuming the required locks have been - * acquired + * acquired. Used to run second-stage bootloader. */ - int edgetpu_firmware_run_locked(struct edgetpu_firmware *et_fw, const char *name, enum edgetpu_firmware_flags flags); diff --git a/drivers/edgetpu/edgetpu-fs.c b/drivers/edgetpu/edgetpu-fs.c index 6fbd642..c16c964 100644 --- a/drivers/edgetpu/edgetpu-fs.c +++ b/drivers/edgetpu/edgetpu-fs.c @@ -625,6 +625,7 @@ static int edgetpu_ioctl_acquire_wakelock(struct edgetpu_client *client) error_release: edgetpu_wakelock_release(client->wakelock); edgetpu_wakelock_unlock(client->wakelock); + etdev_err(client->etdev, "PID: %d failed to acquire wakelock", client->pid); return ret; } @@ -646,11 +647,15 @@ edgetpu_ioctl_acquire_ext_mailbox(struct edgetpu_client *client, struct edgetpu_ext_mailbox_ioctl __user *argp) { struct edgetpu_ext_mailbox_ioctl ext_mailbox; + int ret; if (copy_from_user(&ext_mailbox, argp, sizeof(ext_mailbox))) return -EFAULT; - return edgetpu_chip_acquire_ext_mailbox(client, &ext_mailbox); + ret = edgetpu_chip_acquire_ext_mailbox(client, &ext_mailbox); + if (ret) + etdev_err(client->etdev, "PID: %d failed to acquire ext mailbox", client->pid); + return ret; } static int @@ -998,9 +1003,32 @@ static ssize_t watchdog_timeout_count_show( } static DEVICE_ATTR_RO(watchdog_timeout_count); +static ssize_t clients_show( + struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct edgetpu_dev *etdev = dev_get_drvdata(dev); + struct edgetpu_list_device_client *lc; + ssize_t ret = 0; + + mutex_lock(&etdev->clients_lock); + for_each_list_device_client(etdev, lc) { + ret += scnprintf(buf, PAGE_SIZE - ret, + "pid %d tgid %d wakelock %d\n", + lc->client->pid, lc->client->tgid, + NO_WAKELOCK(lc->client->wakelock) ? + 0 : lc->client->wakelock->req_count); + buf += ret; + } + mutex_unlock(&etdev->clients_lock); + return ret; +} +static DEVICE_ATTR_RO(clients); + static struct attribute *edgetpu_dev_attrs[] = { &dev_attr_firmware_crash_count.attr, &dev_attr_watchdog_timeout_count.attr, + &dev_attr_clients.attr, NULL, }; @@ -1055,6 +1083,7 @@ void edgetpu_fs_remove(struct edgetpu_dev *etdev) { device_remove_group(etdev->dev, &edgetpu_attr_group); device_destroy(edgetpu_class, etdev->devno); + etdev->etcdev = NULL; cdev_del(&etdev->cdev); debugfs_remove_recursive(etdev->d_entry); } diff --git a/drivers/edgetpu/edgetpu-google-iommu.c b/drivers/edgetpu/edgetpu-google-iommu.c index 851a326..f15b61a 100644 --- a/drivers/edgetpu/edgetpu-google-iommu.c +++ b/drivers/edgetpu/edgetpu-google-iommu.c @@ -320,11 +320,11 @@ int edgetpu_mmu_reattach(struct edgetpu_dev *etdev) static int get_iommu_map_params(struct edgetpu_dev *etdev, struct edgetpu_mapping *map, enum edgetpu_context_id context_id, - struct edgetpu_iommu_map_params *params) + struct edgetpu_iommu_map_params *params, u32 mmu_flags) { struct edgetpu_iommu *etiommu = etdev->mmu_cookie; size_t size = 0; - int prot = __dma_dir_to_iommu_prot(map->dir, etdev->dev); + int prot = mmu_flag_to_iommu_prot(mmu_flags, etdev->dev, map->dir); struct iommu_domain *domain; int i; struct scatterlist *sg; @@ -357,7 +357,7 @@ int edgetpu_mmu_map(struct edgetpu_dev *etdev, struct edgetpu_mapping *map, struct iommu_domain *default_domain = iommu_get_domain_for_dev(etdev->dev); - ret = get_iommu_map_params(etdev, map, context_id, ¶ms); + ret = get_iommu_map_params(etdev, map, context_id, ¶ms, mmu_flags); if (ret) return ret; @@ -367,8 +367,7 @@ int edgetpu_mmu_map(struct edgetpu_dev *etdev, struct edgetpu_mapping *map, "%s: 64-bit addressing is not supported", __func__); - ret = dma_map_sg_attrs(etdev->dev, map->sgt.sgl, map->sgt.nents, - edgetpu_host_dma_dir(map->dir), map->dma_attrs); + ret = dma_map_sg_attrs(etdev->dev, map->sgt.sgl, map->sgt.nents, map->dir, map->dma_attrs); if (!ret) return -EINVAL; map->sgt.nents = ret; @@ -383,9 +382,7 @@ int edgetpu_mmu_map(struct edgetpu_dev *etdev, struct edgetpu_mapping *map, if (!iommu_map_sg(params.domain, iova, map->sgt.sgl, map->sgt.orig_nents, params.prot)) { /* Undo the mapping in the default domain */ - dma_unmap_sg_attrs(etdev->dev, map->sgt.sgl, - map->sgt.orig_nents, - edgetpu_host_dma_dir(map->dir), + dma_unmap_sg_attrs(etdev->dev, map->sgt.sgl, map->sgt.orig_nents, map->dir, DMA_ATTR_SKIP_CPU_SYNC); return -ENOMEM; } @@ -403,7 +400,7 @@ void edgetpu_mmu_unmap(struct edgetpu_dev *etdev, struct edgetpu_mapping *map, struct iommu_domain *default_domain = iommu_get_domain_for_dev(etdev->dev); - ret = get_iommu_map_params(etdev, map, context_id, ¶ms); + ret = get_iommu_map_params(etdev, map, context_id, ¶ms, 0); if (!ret && params.domain != default_domain) { /* * If this is a per-context mapping, it was mirrored in the @@ -413,15 +410,15 @@ void edgetpu_mmu_unmap(struct edgetpu_dev *etdev, struct edgetpu_mapping *map, } /* Undo the mapping in the default domain */ - dma_unmap_sg_attrs(etdev->dev, map->sgt.sgl, map->sgt.orig_nents, - edgetpu_host_dma_dir(map->dir), map->dma_attrs); + dma_unmap_sg_attrs(etdev->dev, map->sgt.sgl, map->sgt.orig_nents, map->dir, map->dma_attrs); } int edgetpu_mmu_map_iova_sgt(struct edgetpu_dev *etdev, tpu_addr_t iova, struct sg_table *sgt, enum dma_data_direction dir, + u32 mmu_flags, enum edgetpu_context_id context_id) { - const int prot = __dma_dir_to_iommu_prot(edgetpu_host_dma_dir(dir), etdev->dev); + const int prot = mmu_flag_to_iommu_prot(mmu_flags, etdev->dev, dir); const tpu_addr_t orig_iova = iova; struct scatterlist *sg; int i; @@ -510,7 +507,7 @@ tpu_addr_t edgetpu_mmu_tpu_map(struct edgetpu_dev *etdev, dma_addr_t down_addr, struct iommu_domain *default_domain = iommu_get_domain_for_dev(etdev->dev); phys_addr_t paddr; - int prot = __dma_dir_to_iommu_prot(dir, etdev->dev); + int prot = mmu_flag_to_iommu_prot(mmu_flags, etdev->dev, dir); domain = get_domain_by_context_id(etdev, context_id); /* @@ -551,8 +548,7 @@ void edgetpu_mmu_tpu_unmap(struct edgetpu_dev *etdev, tpu_addr_t tpu_addr, } tpu_addr_t edgetpu_mmu_tpu_map_sgt(struct edgetpu_dev *etdev, - struct sg_table *sgt, - enum dma_data_direction dir, + struct sg_table *sgt, enum dma_data_direction dir, enum edgetpu_context_id context_id, u32 mmu_flags) { @@ -562,7 +558,7 @@ tpu_addr_t edgetpu_mmu_tpu_map_sgt(struct edgetpu_dev *etdev, phys_addr_t paddr; dma_addr_t iova, cur_iova; size_t size; - int prot = __dma_dir_to_iommu_prot(dir, etdev->dev); + int prot = mmu_flag_to_iommu_prot(mmu_flags, etdev->dev, dir); struct scatterlist *sg; int ret; int i; diff --git a/drivers/edgetpu/edgetpu-internal.h b/drivers/edgetpu/edgetpu-internal.h index 7c4966e..d352fe1 100644 --- a/drivers/edgetpu/edgetpu-internal.h +++ b/drivers/edgetpu/edgetpu-internal.h @@ -35,29 +35,27 @@ #include "edgetpu-thermal.h" #include "edgetpu-usage-stats.h" -#define etdev_err(etdev, fmt, ...) dev_err((etdev)->etcdev, fmt, ##__VA_ARGS__) +#define get_dev_for_logging(etdev) ((etdev)->etcdev ? (etdev)->etcdev : (etdev)->dev) + +#define etdev_err(etdev, fmt, ...) dev_err(get_dev_for_logging(etdev), fmt, ##__VA_ARGS__) #define etdev_warn(etdev, fmt, ...) \ - dev_warn((etdev)->etcdev, fmt, ##__VA_ARGS__) + dev_warn(get_dev_for_logging(etdev), fmt, ##__VA_ARGS__) #define etdev_info(etdev, fmt, ...) \ - dev_info((etdev)->etcdev, fmt, ##__VA_ARGS__) -#define etdev_dbg(etdev, fmt, ...) dev_dbg((etdev)->etcdev, fmt, ##__VA_ARGS__) + dev_info(get_dev_for_logging(etdev), fmt, ##__VA_ARGS__) +#define etdev_dbg(etdev, fmt, ...) dev_dbg(get_dev_for_logging(etdev), fmt, ##__VA_ARGS__) #define etdev_err_ratelimited(etdev, fmt, ...) \ - dev_err_ratelimited((etdev)->etcdev, fmt, ##__VA_ARGS__) + dev_err_ratelimited(get_dev_for_logging(etdev), fmt, ##__VA_ARGS__) #define etdev_warn_ratelimited(etdev, fmt, ...) \ - dev_warn_ratelimited((etdev)->etcdev, fmt, ##__VA_ARGS__) + dev_warn_ratelimited(get_dev_for_logging(etdev), fmt, ##__VA_ARGS__) #define etdev_info_ratelimited(etdev, fmt, ...) \ - dev_info_ratelimited((etdev)->etcdev, fmt, ##__VA_ARGS__) + dev_info_ratelimited(get_dev_for_logging(etdev), fmt, ##__VA_ARGS__) #define etdev_dbg_ratelimited(etdev, fmt, ...) \ - dev_dbg_ratelimited((etdev)->etcdev, fmt, ##__VA_ARGS__) + dev_dbg_ratelimited(get_dev_for_logging(etdev), fmt, ##__VA_ARGS__) #define etdev_warn_once(etdev, fmt, ...) \ - dev_warn_once((etdev)->etcdev, fmt, ##__VA_ARGS__) + dev_warn_once(get_dev_for_logging(etdev), fmt, ##__VA_ARGS__) /* The number of TPU tiles in an edgetpu chip */ -#ifdef CONFIG_EDGETPU_FPGA -#define EDGETPU_NTILES 4 -#else #define EDGETPU_NTILES 16 -#endif /* Up to 7 concurrent device groups / workloads per device. */ #define EDGETPU_NGROUPS 7 @@ -129,6 +127,16 @@ struct edgetpu_client { u64 perdie_events; }; +/* edgetpu_dev#clients list entry. */ +struct edgetpu_list_device_client { + struct list_head list; + struct edgetpu_client *client; +}; + +/* Macro to loop through etdev->clients (hold clients_lock prior). */ +#define for_each_list_device_client(etdev, c) \ + list_for_each_entry(c, &etdev->clients, list) + struct edgetpu_mapping; struct edgetpu_mailbox_manager; struct edgetpu_kci; @@ -177,6 +185,8 @@ struct edgetpu_dev { /* end of fields protected by @groups_lock */ + struct mutex clients_lock; /* protects clients */ + struct list_head clients; void *mmu_cookie; /* mmu driver private data */ void *dram_cookie; /* on-device DRAM private data */ struct edgetpu_mailbox_manager *mailbox_manager; @@ -205,6 +215,8 @@ struct edgetpu_dev { /* debug dump handlers */ edgetpu_debug_dump_handlers *debug_dump_handlers; struct work_struct debug_dump_work; + /* status about if device going to be removed from system */ + bool on_exit; }; /* Firmware crash_type codes */ diff --git a/drivers/edgetpu/edgetpu-kci.c b/drivers/edgetpu/edgetpu-kci.c index 73a47cc..b54fed1 100644 --- a/drivers/edgetpu/edgetpu-kci.c +++ b/drivers/edgetpu/edgetpu-kci.c @@ -10,14 +10,15 @@ #include <linux/circ_buf.h> #include <linux/device.h> #include <linux/errno.h> +#include <linux/kernel.h> #include <linux/seq_file.h> #include <linux/slab.h> #include <linux/string.h> /* memcpy */ #include "edgetpu-firmware.h" #include "edgetpu-internal.h" -#include "edgetpu-kci.h" #include "edgetpu-iremap-pool.h" +#include "edgetpu-kci.h" #include "edgetpu-mmu.h" #include "edgetpu-telemetry.h" #include "edgetpu-usage-stats.h" @@ -29,10 +30,7 @@ #define QUEUE_SIZE MAX_QUEUE_SIZE /* Timeout for KCI responses from the firmware (milliseconds) */ -#if IS_ENABLED(CONFIG_EDGETPU_FPGA) -/* Set extra ludicrously high to 60 seconds for (slow) Palladium emulation. */ -#define KCI_TIMEOUT (60000) -#elif IS_ENABLED(CONFIG_EDGETPU_TEST) +#if IS_ENABLED(CONFIG_EDGETPU_TEST) /* fake-firmware could respond in a short time */ #define KCI_TIMEOUT (200) #else @@ -40,6 +38,14 @@ #define KCI_TIMEOUT (5000) #endif +/* A macro for KCIs to leave early when the device state is known to be bad. */ +#define RETURN_ERRNO_IF_ETDEV_NOT_GOOD(kci) \ + do { \ + int ret = edgetpu_get_state_errno_locked(kci->mailbox->etdev); \ + if (ret) \ + return ret; \ + } while (0) + static inline u32 edgetpu_kci_queue_element_size(enum mailbox_queue_type type) { if (type == MAILBOX_CMD_QUEUE) @@ -781,6 +787,7 @@ int edgetpu_kci_join_group(struct edgetpu_kci *kci, u8 n_dies, u8 vid) if (!kci) return -ENODEV; + RETURN_ERRNO_IF_ETDEV_NOT_GOOD(kci); return edgetpu_kci_send_cmd_with_data(kci, &cmd, &detail, sizeof(detail)); } @@ -792,6 +799,7 @@ int edgetpu_kci_leave_group(struct edgetpu_kci *kci) if (!kci) return -ENODEV; + RETURN_ERRNO_IF_ETDEV_NOT_GOOD(kci); return edgetpu_kci_send_cmd(kci, &cmd); } @@ -946,14 +954,18 @@ void edgetpu_kci_mappings_show(struct edgetpu_dev *etdev, struct seq_file *s) seq_printf(s, "kci context %u:\n", EDGETPU_CONTEXT_KCI); seq_printf(s, " 0x%llx %lu cmdq - %pad\n", kci->cmd_queue_mem.tpu_addr, - QUEUE_SIZE * - edgetpu_kci_queue_element_size(MAILBOX_CMD_QUEUE) - / PAGE_SIZE, &kci->cmd_queue_mem.dma_addr); + DIV_ROUND_UP( + QUEUE_SIZE * + edgetpu_kci_queue_element_size(MAILBOX_CMD_QUEUE), + PAGE_SIZE), + &kci->cmd_queue_mem.dma_addr); seq_printf(s, " 0x%llx %lu rspq - %pad\n", kci->resp_queue_mem.tpu_addr, - QUEUE_SIZE * - edgetpu_kci_queue_element_size(MAILBOX_RESP_QUEUE) - / PAGE_SIZE, &kci->resp_queue_mem.dma_addr); + DIV_ROUND_UP( + QUEUE_SIZE * + edgetpu_kci_queue_element_size(MAILBOX_RESP_QUEUE), + PAGE_SIZE), + &kci->resp_queue_mem.dma_addr); edgetpu_telemetry_mappings_show(etdev, s); edgetpu_firmware_mappings_show(etdev, s); } @@ -1001,6 +1013,7 @@ int edgetpu_kci_open_device(struct edgetpu_kci *kci, u32 mailbox_id, s16 vcid, b if (!kci) return -ENODEV; + RETURN_ERRNO_IF_ETDEV_NOT_GOOD(kci); if (vcid < 0) return edgetpu_kci_send_cmd(kci, &cmd); return edgetpu_kci_send_cmd_with_data(kci, &cmd, &detail, sizeof(detail)); @@ -1017,6 +1030,7 @@ int edgetpu_kci_close_device(struct edgetpu_kci *kci, u32 mailbox_id) if (!kci) return -ENODEV; + RETURN_ERRNO_IF_ETDEV_NOT_GOOD(kci); return edgetpu_kci_send_cmd(kci, &cmd); } @@ -1032,11 +1046,8 @@ int edgetpu_kci_notify_throttling(struct edgetpu_dev *etdev, u32 level) if (!etdev->kci) return -ENODEV; - if (!edgetpu_pm_get_if_powered(etdev->pm)) - return -EAGAIN; ret = edgetpu_kci_send_cmd(etdev->kci, &cmd); - edgetpu_pm_put(etdev->pm); return ret; } diff --git a/drivers/edgetpu/edgetpu-kci.h b/drivers/edgetpu/edgetpu-kci.h index deb258d..2893f20 100644 --- a/drivers/edgetpu/edgetpu-kci.h +++ b/drivers/edgetpu/edgetpu-kci.h @@ -24,7 +24,7 @@ * Maximum number of outstanding KCI requests from firmware * This is used to size a circular buffer, so it must be a power of 2 */ -#define REVERSE_KCI_BUFFER_SIZE (8) +#define REVERSE_KCI_BUFFER_SIZE (32) /* * The status field in a firmware response is set to this by us when the diff --git a/drivers/edgetpu/edgetpu-mailbox.c b/drivers/edgetpu/edgetpu-mailbox.c index cf996f7..9bda7d6 100644 --- a/drivers/edgetpu/edgetpu-mailbox.c +++ b/drivers/edgetpu/edgetpu-mailbox.c @@ -18,6 +18,7 @@ #include "edgetpu-kci.h" #include "edgetpu-mailbox.h" #include "edgetpu-mmu.h" +#include "edgetpu-sw-watchdog.h" #include "edgetpu-wakelock.h" #include "edgetpu.h" @@ -506,10 +507,6 @@ int edgetpu_mailbox_alloc_queue(struct edgetpu_dev *etdev, u32 size = unit * queue_size; int ret; - /* checks integer overflow */ - if (queue_size > SIZE_MAX / unit) - return -ENOMEM; - /* Align queue size to page size for TPU MMU map. */ size = __ALIGN_KERNEL(size, PAGE_SIZE); ret = edgetpu_iremap_alloc(etdev, size, mem, @@ -992,7 +989,7 @@ static int edgetpu_mailbox_external_alloc_enable(struct edgetpu_client *client, for (i = 0; i < ext_mailbox->count; i++) { id = ext_mailbox->descriptors[i].mailbox->mailbox_id; etdev_dbg(group->etdev, "Enabling mailbox: %d\n", id); - ret = edgetpu_mailbox_activate(group->etdev, id, vcid, false); + ret = edgetpu_mailbox_activate(group->etdev, id, vcid, true); if (ret) { etdev_err(group->etdev, "Activate mailbox %d failed: %d", id, ret); break; @@ -1132,6 +1129,13 @@ int edgetpu_mailbox_activate(struct edgetpu_dev *etdev, u32 mailbox_id, s16 vcid eh->fw_state |= bit; } mutex_unlock(&eh->lock); + /* + * We are observing OPEN_DEVICE KCI fails while other KCIs (usage update / shutdown) still + * succeed and no firmware crash is reported. Kick off the firmware restart when we are + * facing this and hope this can rescue the device from the bad state. + */ + if (ret == -ETIMEDOUT) + edgetpu_watchdog_bite(etdev, false); return ret; } diff --git a/drivers/edgetpu/edgetpu-mailbox.h b/drivers/edgetpu/edgetpu-mailbox.h index 1ae4889..3fd961f 100644 --- a/drivers/edgetpu/edgetpu-mailbox.h +++ b/drivers/edgetpu/edgetpu-mailbox.h @@ -487,7 +487,12 @@ static inline void edgetpu_mailbox_enable(struct edgetpu_mailbox *mailbox) /* Disables a mailbox by setting CSR. */ static inline void edgetpu_mailbox_disable(struct edgetpu_mailbox *mailbox) { - EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, context_enable, 0); + /* + * if device is being removed, no need to access device CSR since it + * is going to be powered-off anyway + */ + if (!mailbox->etdev->on_exit) + EDGETPU_MAILBOX_CONTEXT_WRITE(mailbox, context_enable, 0); } #endif /* __EDGETPU_MAILBOX_H__ */ diff --git a/drivers/edgetpu/edgetpu-mapping.h b/drivers/edgetpu/edgetpu-mapping.h index e3a0dc9..8ad165b 100644 --- a/drivers/edgetpu/edgetpu-mapping.h +++ b/drivers/edgetpu/edgetpu-mapping.h @@ -21,6 +21,7 @@ #endif #include "edgetpu-internal.h" +#include "edgetpu-mmu.h" struct edgetpu_mapping_root { struct rb_root rb; @@ -144,24 +145,33 @@ void edgetpu_mapping_clear(struct edgetpu_mapping_root *mappings); void edgetpu_mappings_show(struct edgetpu_mapping_root *mappings, struct seq_file *s); -static inline int __dma_dir_to_iommu_prot(enum dma_data_direction dir, struct device *dev) +static inline int __dma_dir_to_iommu_prot(enum dma_data_direction dir) { -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) - int prot = dev_is_dma_coherent(dev) ? IOMMU_CACHE : 0; -#else - int prot = 0; /* hardcode to non-dma-coherent for prior kernels */ -#endif - switch (dir) { case DMA_BIDIRECTIONAL: - return prot | IOMMU_READ | IOMMU_WRITE; + return IOMMU_READ | IOMMU_WRITE; case DMA_TO_DEVICE: - return prot | IOMMU_READ; + return IOMMU_READ; case DMA_FROM_DEVICE: - return prot | IOMMU_WRITE; + return IOMMU_WRITE; default: return 0; } } +/* Returns iommu prot based on @flags and @dir */ +static inline int mmu_flag_to_iommu_prot(u32 mmu_flags, struct device *dev, + enum dma_data_direction dir) +{ + int prot = 0; /* hardcode to non-dma-coherent for prior kernels */ + + if (mmu_flags & EDGETPU_MMU_COHERENT) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 10, 0) + prot = dev_is_dma_coherent(dev) ? IOMMU_CACHE : 0; +#endif + } + prot |= __dma_dir_to_iommu_prot(dir); + return prot; +} + #endif /* __EDGETPU_MAPPING_H__ */ diff --git a/drivers/edgetpu/edgetpu-mmu.h b/drivers/edgetpu/edgetpu-mmu.h index 7cc9ffa..812a05c 100644 --- a/drivers/edgetpu/edgetpu-mmu.h +++ b/drivers/edgetpu/edgetpu-mmu.h @@ -37,6 +37,8 @@ #define EDGETPU_MMU_DEVICE (1 << 2) #define EDGETPU_MMU_DMABUF (2 << 2) +#define EDGETPU_MMU_COHERENT (1 << 4) + /* * The max possible value of token is (EDGETPU_DOMAIN_TOKEN_END - 1), which * shouldn't equal or exceed the bit mask EDGETPU_CONTEXT_DOMAIN_TOKEN. @@ -75,6 +77,11 @@ edgetpu_host_dma_dir(enum dma_data_direction target_dir) } } +static inline enum dma_data_direction map_flag_to_host_dma_dir(edgetpu_map_flag_t flags) +{ + return edgetpu_host_dma_dir(flags & EDGETPU_MAP_DIR_MASK); +} + static inline u32 map_to_mmu_flags(edgetpu_map_flag_t flags) { u32 ret = 0; @@ -82,6 +89,7 @@ static inline u32 map_to_mmu_flags(edgetpu_map_flag_t flags) ret |= IS_MIRRORED(flags) ? EDGETPU_MMU_VDG : EDGETPU_MMU_DIE; ret |= (flags & EDGETPU_MAP_CPU_NONACCESSIBLE) ? EDGETPU_MMU_64 : EDGETPU_MMU_32; + ret |= (flags & EDGETPU_MAP_COHERENT) ? EDGETPU_MMU_COHERENT : 0; return ret; } @@ -145,7 +153,7 @@ void edgetpu_mmu_unmap(struct edgetpu_dev *dev, struct edgetpu_mapping *map, */ int edgetpu_mmu_map_iova_sgt(struct edgetpu_dev *etdev, tpu_addr_t iova, struct sg_table *sgt, enum dma_data_direction dir, - enum edgetpu_context_id context_id); + u32 mmu_flags, enum edgetpu_context_id context_id); void edgetpu_mmu_unmap_iova_sgt_attrs(struct edgetpu_dev *etdev, tpu_addr_t iova, struct sg_table *sgt, enum dma_data_direction dir, diff --git a/drivers/edgetpu/edgetpu-mobile-firmware.c b/drivers/edgetpu/edgetpu-mobile-firmware.c new file mode 100644 index 0000000..407a698 --- /dev/null +++ b/drivers/edgetpu/edgetpu-mobile-firmware.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Edge TPU firmware management for mobile chipsets. + * + * Copyright (C) 2021 Google, Inc. + */ + +#include <linux/device.h> +#include <linux/firmware.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/types.h> + +#include "edgetpu-firmware.h" +#include "edgetpu-firmware-util.h" +#include "edgetpu-internal.h" +#include "mobile-firmware.h" + +/* Load firmware for chips that use carveout memory for a single chip. */ +int edgetpu_firmware_chip_load_locked( + struct edgetpu_firmware *et_fw, + struct edgetpu_firmware_desc *fw_desc, const char *name) +{ + int ret; + struct edgetpu_dev *etdev = et_fw->etdev; + struct device *dev = etdev->dev; + const struct firmware *fw; + size_t aligned_size; + + ret = request_firmware(&fw, name, dev); + if (ret) { + etdev_dbg(etdev, + "%s: request '%s' failed: %d\n", __func__, name, ret); + return ret; + } + + aligned_size = ALIGN(fw->size, fw_desc->buf.used_size_align); + if (aligned_size > fw_desc->buf.alloc_size) { + etdev_dbg(etdev, + "%s: firmware buffer too small: alloc size=0x%zx, required size=0x%zx\n", + __func__, fw_desc->buf.alloc_size, aligned_size); + ret = -ENOSPC; + goto out_release_firmware; + } + + memcpy(fw_desc->buf.vaddr, fw->data, fw->size); + fw_desc->buf.used_size = aligned_size; + /* May return NULL on out of memory, driver must handle properly */ + fw_desc->buf.name = kstrdup(name, GFP_KERNEL); + +out_release_firmware: + release_firmware(fw); + return ret; +} + +void edgetpu_firmware_chip_unload_locked( + struct edgetpu_firmware *et_fw, + struct edgetpu_firmware_desc *fw_desc) +{ + kfree(fw_desc->buf.name); + fw_desc->buf.name = NULL; + fw_desc->buf.used_size = 0; +} diff --git a/drivers/edgetpu/edgetpu-pm.c b/drivers/edgetpu/edgetpu-pm.c index df1c179..872149f 100644 --- a/drivers/edgetpu/edgetpu-pm.c +++ b/drivers/edgetpu/edgetpu-pm.c @@ -15,6 +15,7 @@ #include "edgetpu-mailbox.h" #include "edgetpu-pm.h" #include "edgetpu-sw-watchdog.h" +#include "edgetpu-wakelock.h" #if IS_ENABLED(CONFIG_EDGETPU_TEST) #include "unittests/factory/fake-edgetpu-firmware.h" @@ -310,15 +311,28 @@ void edgetpu_pchannel_power_up(struct edgetpu_dev *etdev) int edgetpu_pm_suspend(struct edgetpu_dev *etdev) { struct edgetpu_pm *etpm = etdev->pm; + struct edgetpu_list_device_client *lc; - if (etpm && etpm->p->power_up_count) { - etdev_warn_ratelimited( - etdev, "cannot suspend with power up count = %d\n", - etpm->p->power_up_count); + if (!etpm || !etpm->p->power_up_count) + return 0; + + etdev_warn_ratelimited( + etdev, "cannot suspend with power up count = %d\n", + etpm->p->power_up_count); + + if (!mutex_trylock(&etdev->clients_lock)) return -EAGAIN; + for_each_list_device_client(etdev, lc) { + if (NO_WAKELOCK(lc->client->wakelock) || + !lc->client->wakelock->req_count) + continue; + etdev_warn_ratelimited(etdev, "pid %d tgid %d count %d\n", + lc->client->pid, + lc->client->tgid, + lc->client->wakelock->req_count); } - - return 0; + mutex_unlock(&etdev->clients_lock); + return -EAGAIN; } int edgetpu_pm_resume(struct edgetpu_dev *etdev) diff --git a/drivers/edgetpu/edgetpu-shared-fw.c b/drivers/edgetpu/edgetpu-shared-fw.c index 9abee03..c3c4e48 100644 --- a/drivers/edgetpu/edgetpu-shared-fw.c +++ b/drivers/edgetpu/edgetpu-shared-fw.c @@ -13,6 +13,7 @@ #include <linux/slab.h> #include <linux/string.h> +#include "edgetpu-firmware.h" #include "edgetpu-firmware-util.h" #include "edgetpu-internal.h" #include "edgetpu-shared-fw.h" @@ -96,17 +97,11 @@ edgetpu_shared_fw_find_locked(const char *name) return NULL; } -int +void edgetpu_shared_fw_init(const struct edgetpu_shared_fw_init_data *init_data) { - if (!list_empty(&global.firmware_list)) { - pr_err("%s: already initialized with firmware loaded.\n", - __func__); - return -EINVAL; - } - - global.init_data = *init_data; - return 0; + if (list_empty(&global.firmware_list)) + global.init_data = *init_data; } void edgetpu_shared_fw_exit(void) @@ -121,12 +116,6 @@ void edgetpu_shared_fw_exit(void) for_each_shared_fw_buffer_safe(cur_buf, nxt_buf) list_del(&cur_buf->list); - /* - * TODO(b/152573549): release all firmwares besides clearing the list. - * We have to add a handler for forced stop/unload on loading/getting - * firmware. - */ - mutex_unlock(&global.lock); } @@ -235,6 +224,40 @@ struct edgetpu_shared_fw_buffer *edgetpu_shared_fw_load( return buffer; } +int edgetpu_firmware_chip_load_locked( + struct edgetpu_firmware *et_fw, + struct edgetpu_firmware_desc *fw_desc, const char *name) +{ + int ret; + struct edgetpu_dev *etdev = et_fw->etdev; + struct edgetpu_shared_fw_buffer *shared_buf; + + shared_buf = edgetpu_shared_fw_load(name, etdev); + if (IS_ERR(shared_buf)) { + ret = PTR_ERR(shared_buf); + etdev_dbg(etdev, "shared buffer loading failed: %d\n", ret); + return ret; + } + fw_desc->shared_buf = shared_buf; + fw_desc->buf.vaddr = edgetpu_shared_fw_buffer_vaddr(shared_buf); + fw_desc->buf.alloc_size = edgetpu_shared_fw_buffer_size(shared_buf); + fw_desc->buf.used_size = fw_desc->buf.alloc_size; + fw_desc->buf.name = edgetpu_shared_fw_buffer_name(shared_buf); + return 0; +} + +void edgetpu_firmware_chip_unload_locked( + struct edgetpu_firmware *et_fw, + struct edgetpu_firmware_desc *fw_desc) +{ + fw_desc->buf.vaddr = NULL; + fw_desc->buf.alloc_size = 0; + fw_desc->buf.used_size = 0; + fw_desc->buf.name = NULL; + edgetpu_shared_fw_put(fw_desc->shared_buf); + fw_desc->shared_buf = NULL; +} + static void edgetpu_shared_fw_put_locked(struct edgetpu_shared_fw_buffer *buffer) { @@ -259,102 +282,3 @@ void edgetpu_shared_fw_put(struct edgetpu_shared_fw_buffer *buffer) edgetpu_shared_fw_put_locked(buffer); mutex_unlock(&global.lock); } - -static ssize_t shared_fw_load_store(struct device_driver *drv, - const char *buf, size_t count) -{ - struct edgetpu_shared_fw_buffer *buffer; - char *name; - ssize_t ret; - - name = edgetpu_fwutil_name_from_attr_buf(buf); - if (IS_ERR(name)) - return PTR_ERR(name); - - mutex_lock(&global.lock); - - /* - * TODO(b/152573549): reload firmware to read the latest firmware on - * filesystem. - */ - - buffer = edgetpu_shared_fw_load_locked(name, NULL); - if (IS_ERR(buffer)) { - ret = PTR_ERR(buffer); - goto out_mutex_unlock; - } - - if (buffer->is_sysfs_loaded) - edgetpu_shared_fw_put_locked(buffer); - else - buffer->is_sysfs_loaded = true; - - ret = count; - -out_mutex_unlock: - mutex_unlock(&global.lock); - kfree(name); - return ret; -} -static DRIVER_ATTR_WO(shared_fw_load); - -static ssize_t shared_fw_unload_store(struct device_driver *drv, - const char *buf, size_t count) -{ - struct edgetpu_shared_fw_buffer *buffer; - char *name; - ssize_t ret; - - name = edgetpu_fwutil_name_from_attr_buf(buf); - if (IS_ERR(name)) - return PTR_ERR(name); - - mutex_lock(&global.lock); - - buffer = edgetpu_shared_fw_find_locked(name); - if (!buffer || !buffer->is_sysfs_loaded) { - ret = -ENOENT; - goto out_mutex_unlock; - } - buffer->is_sysfs_loaded = false; - edgetpu_shared_fw_put_locked(buffer); - ret = count; - -out_mutex_unlock: - mutex_unlock(&global.lock); - kfree(name); - return ret; -} -static DRIVER_ATTR_WO(shared_fw_unload); - -static struct driver_attribute *driver_attrs[] = { - &driver_attr_shared_fw_load, - &driver_attr_shared_fw_unload, - NULL, -}; - -int edgetpu_shared_fw_add_driver_attrs(struct device_driver *driver) -{ - struct driver_attribute **driver_attr; - int ret; - - for (driver_attr = driver_attrs; *driver_attr; driver_attr++) { - ret = driver_create_file(driver, *driver_attr); - if (ret) - goto out_remove_driver_attrs; - } - return 0; - -out_remove_driver_attrs: - while (--driver_attr >= driver_attrs) - driver_remove_file(driver, *driver_attr); - return ret; -} - -void edgetpu_shared_fw_remove_driver_attrs(struct device_driver *driver) -{ - struct driver_attribute **driver_attr; - - for (driver_attr = driver_attrs; *driver_attr; driver_attr++) - driver_remove_file(driver, *driver_attr); -} diff --git a/drivers/edgetpu/edgetpu-shared-fw.h b/drivers/edgetpu/edgetpu-shared-fw.h index 101ad79..28033e3 100644 --- a/drivers/edgetpu/edgetpu-shared-fw.h +++ b/drivers/edgetpu/edgetpu-shared-fw.h @@ -32,7 +32,7 @@ struct edgetpu_shared_fw_init_data { }; /* Initializes structures for shared firmware management. */ -int +void edgetpu_shared_fw_init(const struct edgetpu_shared_fw_init_data *init_data); /* Finalizes structures for shared firmware management. */ void edgetpu_shared_fw_exit(void); @@ -71,11 +71,4 @@ edgetpu_shared_fw_get_by_name(const char *name); * reference count reaches 0. */ void edgetpu_shared_fw_put(struct edgetpu_shared_fw_buffer *buffer); - -/* - * (Add/Remove) driver-wide sysfs attributes for development and debug. - */ -int edgetpu_shared_fw_add_driver_attrs(struct device_driver *driver); -void edgetpu_shared_fw_remove_driver_attrs(struct device_driver *driver); - #endif /* __EDGETPU_SHARED_FW_H__ */ diff --git a/drivers/edgetpu/edgetpu-telemetry.c b/drivers/edgetpu/edgetpu-telemetry.c index abc9095..4e1053e 100644 --- a/drivers/edgetpu/edgetpu-telemetry.c +++ b/drivers/edgetpu/edgetpu-telemetry.c @@ -8,6 +8,7 @@ #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/errno.h> +#include <linux/kernel.h> #include <linux/mm.h> #include <linux/mutex.h> #include <linux/slab.h> @@ -244,7 +245,7 @@ static void telemetry_mappings_show(struct edgetpu_telemetry *tel, seq_printf(s, " 0x%llx %lu %s 0x%llx %pad\n", tel->coherent_mem.tpu_addr, - tel->coherent_mem.size / PAGE_SIZE, tel->name, + DIV_ROUND_UP(tel->coherent_mem.size, PAGE_SIZE), tel->name, tel->coherent_mem.host_addr, &tel->coherent_mem.dma_addr); } diff --git a/drivers/edgetpu/edgetpu-usage-stats.c b/drivers/edgetpu/edgetpu-usage-stats.c index 0001210..f796117 100644 --- a/drivers/edgetpu/edgetpu-usage-stats.c +++ b/drivers/edgetpu/edgetpu-usage-stats.c @@ -8,42 +8,14 @@ #include <linux/slab.h> #include <linux/sysfs.h> +#include "edgetpu-config.h" #include "edgetpu-internal.h" #include "edgetpu-kci.h" #include "edgetpu-usage-stats.h" -#if IS_ENABLED(CONFIG_ABROLHOS) -#include "abrolhos-pm.h" - -static enum tpu_pwr_state tpu_states_arr[] = { - TPU_ACTIVE_UUD, - TPU_ACTIVE_SUD, - TPU_ACTIVE_UD, - TPU_ACTIVE_NOM, - TPU_ACTIVE_OD, -}; - -#else /* CONFIG_HERMOSA */ - -static uint32_t tpu_states_arr[] = { - 4, /* kActiveMinPower, kActiveVeryLowPower: 400MHz */ - 5, /* kActiveLowPower: 800MHz */ - 6, /* kActive: 950MHz */ -}; - -static uint32_t tpu_states_display[] = { - 400, - 800, - 950, -}; - -#endif /* CONFIG_ABROLHOS */ - -#define NUM_TPU_STATES ARRAY_SIZE(tpu_states_arr) - struct uid_entry { int32_t uid; - uint64_t time_in_state[NUM_TPU_STATES]; + uint64_t time_in_state[EDGETPU_NUM_STATES]; struct hlist_node node; }; @@ -51,8 +23,8 @@ static int tpu_state_map(uint32_t state) { int i; - for (i = (NUM_TPU_STATES - 1); i >= 0; i--) { - if (state >= tpu_states_arr[i]) + for (i = (EDGETPU_NUM_STATES - 1); i >= 0; i--) { + if (state >= edgetpu_active_states[i]) return i; } @@ -322,13 +294,9 @@ static ssize_t tpu_usage_show(struct device *dev, /* uid: state0speed state1speed ... */ ret += scnprintf(buf, PAGE_SIZE, "uid:"); - for (i = 0; i < NUM_TPU_STATES; i++) + for (i = 0; i < EDGETPU_NUM_STATES; i++) ret += scnprintf(buf + ret, PAGE_SIZE - ret, " %d", -#if IS_ENABLED(CONFIG_ABROLHOS) - tpu_states_arr[i]); -#else - tpu_states_display[i]); -#endif + edgetpu_states_display[i]); ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); @@ -338,7 +306,7 @@ static ssize_t tpu_usage_show(struct device *dev, ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%d:", uid_entry->uid); - for (i = 0; i < NUM_TPU_STATES; i++) + for (i = 0; i < EDGETPU_NUM_STATES; i++) ret += scnprintf(buf + ret, PAGE_SIZE - ret, " %lld", uid_entry->time_in_state[i]); diff --git a/drivers/edgetpu/edgetpu.h b/drivers/edgetpu/edgetpu.h index db6b6b8..3f43bab 100644 --- a/drivers/edgetpu/edgetpu.h +++ b/drivers/edgetpu/edgetpu.h @@ -40,6 +40,8 @@ typedef __u32 edgetpu_map_flag_t; /* Offset and mask to set the PBHA bits of IOMMU mappings */ #define EDGETPU_MAP_ATTR_PBHA_SHIFT 5 #define EDGETPU_MAP_ATTR_PBHA_MASK 0xf +/* Create coherent mapping of the buffer */ +#define EDGETPU_MAP_COHERENT (1u << 9) /* External mailbox types */ #define EDGETPU_EXT_MAILBOX_TYPE_TZ 1 @@ -78,7 +80,12 @@ struct edgetpu_map_ioctl { * 1 = Skip CPU sync. * Note: This bit is ignored on the map call. * [8:5] - Value of PBHA bits for IOMMU mappings. For Abrolhos only. - * [31:9] - RESERVED + * [9:9] - Coherent Mapping: + * 0 = Create non-coherent mappings of the buffer. + * 1 = Create coherent mappings of the buffer. + * Note: this attribute may be ignored on platforms where + * the TPU is not I/O coherent. + * [31:10] - RESERVED */ edgetpu_map_flag_t flags; /* diff --git a/drivers/edgetpu/janeiro-device.c b/drivers/edgetpu/janeiro-device.c index 5fdf792..265d5e8 100644 --- a/drivers/edgetpu/janeiro-device.c +++ b/drivers/edgetpu/janeiro-device.c @@ -13,7 +13,7 @@ #include "edgetpu-mailbox.h" #include "edgetpu-telemetry.h" #include "janeiro-platform.h" -#include "janeiro-pm.h" +#include "mobile-pm.h" static irqreturn_t janeiro_mailbox_handle_irq(struct edgetpu_dev *etdev, int irq) @@ -85,23 +85,15 @@ struct edgetpu_dumpregs_range edgetpu_chip_tile_statusregs_ranges[] = { int edgetpu_chip_tile_statusregs_nranges = ARRAY_SIZE(edgetpu_chip_tile_statusregs_ranges); -static void edgetpu_chip_set_pm_qos(struct edgetpu_dev *etdev, u32 value) -{ -} - -static void edgetpu_chip_set_bts(struct edgetpu_dev *etdev, u32 value) -{ -} - void edgetpu_chip_handle_reverse_kci(struct edgetpu_dev *etdev, struct edgetpu_kci_response_element *resp) { switch (resp->code) { case RKCI_CODE_PM_QOS: - edgetpu_chip_set_pm_qos(etdev, resp->retval); + mobile_pm_set_pm_qos(etdev, resp->retval); break; case RKCI_CODE_BTS: - edgetpu_chip_set_bts(etdev, resp->retval); + mobile_pm_set_bts(etdev, resp->retval); break; default: etdev_warn(etdev, "%s: Unrecognized KCI request: %u\n", @@ -123,7 +115,7 @@ int edgetpu_chip_acquire_ext_mailbox(struct edgetpu_client *client, req.count = args->count; req.start = JANEIRO_EXT_DSP_MAILBOX_START; req.end = JANEIRO_EXT_DSP_MAILBOX_END; - return edgetpu_mailbox_enable_ext(client, -1, &req); + return edgetpu_mailbox_enable_ext(client, EDGETPU_MAILBOX_ID_USE_ASSOC, &req); } return -ENODEV; } @@ -132,7 +124,7 @@ int edgetpu_chip_release_ext_mailbox(struct edgetpu_client *client, struct edgetpu_ext_mailbox_ioctl *args) { if (args->type == EDGETPU_EXT_MAILBOX_TYPE_DSP) - return edgetpu_mailbox_disable_ext(client, -1); + return edgetpu_mailbox_disable_ext(client, EDGETPU_MAILBOX_ID_USE_ASSOC); return -ENODEV; } diff --git a/drivers/edgetpu/janeiro-firmware.c b/drivers/edgetpu/janeiro-firmware.c index 4f0ce84..3ce8880 100644 --- a/drivers/edgetpu/janeiro-firmware.c +++ b/drivers/edgetpu/janeiro-firmware.c @@ -79,7 +79,8 @@ static void janeiro_firmware_before_destroy(struct edgetpu_firmware *et_fw) u32 i, tpu_addr, size; struct edgetpu_dev *etdev = et_fw->etdev; - tpu_cpu_reset(et_fw->etdev, 1); + if (!etdev->on_exit) + tpu_cpu_reset(et_fw->etdev, 1); /* TODO(b/189906347): Remove when GSA/TZ support is available. */ /* Remove mappings created by setup_buffer() */ data = edgetpu_firmware_get_data(et_fw); @@ -247,7 +248,8 @@ static int janeiro_firmware_prepare_run(struct edgetpu_firmware *et_fw, return ret; } -static const struct edgetpu_firmware_handlers janeiro_firmware_handlers = { +static const struct edgetpu_firmware_chip_data janeiro_firmware_chip_data = { + .default_firmware_name = EDGETPU_DEFAULT_FIRMWARE_NAME, .after_create = janeiro_firmware_after_create, .before_destroy = janeiro_firmware_before_destroy, .alloc_buffer = janeiro_firmware_alloc_buffer, @@ -259,7 +261,7 @@ static const struct edgetpu_firmware_handlers janeiro_firmware_handlers = { int mobile_edgetpu_firmware_create(struct edgetpu_dev *etdev) { - return edgetpu_firmware_create(etdev, &janeiro_firmware_handlers); + return edgetpu_firmware_create(etdev, &janeiro_firmware_chip_data); } void mobile_edgetpu_firmware_destroy(struct edgetpu_dev *etdev) @@ -267,12 +269,6 @@ void mobile_edgetpu_firmware_destroy(struct edgetpu_dev *etdev) edgetpu_firmware_destroy(etdev); } -int edgetpu_chip_firmware_run(struct edgetpu_dev *etdev, const char *name, - enum edgetpu_firmware_flags flags) -{ - return edgetpu_firmware_run(etdev, name, flags); -} - unsigned long edgetpu_chip_firmware_iova(struct edgetpu_dev *etdev) { /* diff --git a/drivers/edgetpu/janeiro-platform.c b/drivers/edgetpu/janeiro-platform.c index 81241f1..6f956c1 100644 --- a/drivers/edgetpu/janeiro-platform.c +++ b/drivers/edgetpu/janeiro-platform.c @@ -20,8 +20,8 @@ #include "edgetpu-mmu.h" #include "edgetpu-telemetry.h" #include "janeiro-platform.h" -#include "janeiro-pm.h" #include "mobile-firmware.h" +#include "mobile-pm.h" static const struct of_device_id edgetpu_of_match[] = { /* TODO(b/190677977): remove */ @@ -120,10 +120,9 @@ void edgetpu_chip_remove_mmu(struct edgetpu_dev *etdev) /* * Set shareability for enabling IO coherency in Janeiro */ -//TODO(b/185301967): check if sysreg is to be moved under TrustZone -static int janeiro_mmu_set_shareability(struct device *dev) +static int janeiro_mmu_set_shareability(struct device *dev, u32 reg_base) { - void __iomem *addr = ioremap(EDGETPU_SYSREG_TPU_BASE, PAGE_SIZE); + void __iomem *addr = ioremap(reg_base, PAGE_SIZE); if (!addr) { dev_err(dev, "sysreg ioremap failed\n"); @@ -137,6 +136,27 @@ static int janeiro_mmu_set_shareability(struct device *dev) return 0; } +static int janeiro_parse_dt(struct device *dev) +{ + int ret; + u32 reg; + + if (of_find_property(dev->of_node, "edgetpu,shareability", NULL)) { + ret = of_property_read_u32_index(dev->of_node, "edgetpu,shareability", 0, ®); + if (ret) + return ret; + } else { + /* + * TODO(b/193593081): Remove compatibility code + * Fallback for older driver versions until DT property support is + * widely adopted. + */ + reg = EDGETPU_SYSREG_TPU_BASE; + } + + return janeiro_mmu_set_shareability(dev, reg); +} + static int edgetpu_platform_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -167,7 +187,7 @@ static int edgetpu_platform_probe(struct platform_device *pdev) return -ENODEV; } - ret = janeiro_pm_create(&edgetpu_pdev->edgetpu_dev); + ret = mobile_pm_create(&edgetpu_pdev->edgetpu_dev); if (ret) { dev_err(dev, "Failed to initialize PM interface (%d)\n", ret); return ret; @@ -207,7 +227,11 @@ static int edgetpu_platform_probe(struct platform_device *pdev) for (i = 0; i < EDGETPU_NCONTEXTS; i++) edgetpu_pdev->irq[i] = platform_get_irq(pdev, i); - janeiro_mmu_set_shareability(dev); + ret = janeiro_parse_dt(dev); + + if (ret) + dev_warn(dev, "%s failed to enable shareability: %d\n", + DRIVER_NAME, ret); ret = edgetpu_device_add(&edgetpu_pdev->edgetpu_dev, ®s); @@ -279,18 +303,17 @@ static int edgetpu_platform_remove(struct platform_device *pdev) /* TODO(b/189906347): Use edgetpu_device_remove() for cleanup after * having GSA/TZ support. */ + etdev->on_exit = true; edgetpu_pm_get(etdev->pm); - for (i = 0; i < EDGETPU_NCONTEXTS; i++) { if (janeiro_pdev->irq[i] >= 0) edgetpu_unregister_irq(etdev, janeiro_pdev->irq[i]); } - + mobile_edgetpu_firmware_destroy(etdev); edgetpu_telemetry_exit(etdev); edgetpu_chip_exit(etdev); edgetpu_debug_dump_exit(etdev); edgetpu_mailbox_remove_all(etdev->mailbox_manager); - mobile_edgetpu_firmware_destroy(etdev); edgetpu_pm_put(etdev->pm); edgetpu_pm_shutdown(etdev, true); edgetpu_usage_stats_exit(etdev); @@ -298,7 +321,7 @@ static int edgetpu_platform_remove(struct platform_device *pdev) edgetpu_fs_remove(etdev); edgetpu_iremap_pool_destroy(etdev); janeiro_platform_cleanup_fw_region(janeiro_pdev); - janeiro_pm_destroy(etdev); + mobile_pm_destroy(etdev); return 0; } diff --git a/drivers/edgetpu/janeiro-platform.h b/drivers/edgetpu/janeiro-platform.h index a051f44..09991cf 100644 --- a/drivers/edgetpu/janeiro-platform.h +++ b/drivers/edgetpu/janeiro-platform.h @@ -14,7 +14,6 @@ #include <soc/google/bcl.h> #include "edgetpu-internal.h" -#include "janeiro-pm.h" #define to_janeiro_dev(etdev) \ container_of(etdev, struct janeiro_platform_dev, edgetpu_dev) diff --git a/drivers/edgetpu/janeiro-pm.c b/drivers/edgetpu/janeiro-pm.c index acccf9e..569858d 100644 --- a/drivers/edgetpu/janeiro-pm.c +++ b/drivers/edgetpu/janeiro-pm.c @@ -13,17 +13,20 @@ #include <soc/google/bcl.h> #include <linux/version.h> -#include "edgetpu-config.h" #include "edgetpu-firmware.h" #include "edgetpu-internal.h" #include "edgetpu-kci.h" #include "edgetpu-mailbox.h" #include "edgetpu-pm.h" #include "janeiro-platform.h" -#include "janeiro-pm.h" +#include "mobile-pm.h" #include "edgetpu-pm.c" +#define SHUTDOWN_DELAY_US_MIN 20 +#define SHUTDOWN_DELAY_US_MAX 20 +#define SHUTDOWN_MAX_DELAY_COUNT 20 + /* Default power state */ static int power_state = TPU_ACTIVE_NOM; @@ -31,6 +34,15 @@ module_param(power_state, int, 0660); static struct dentry *janeiro_pwr_debugfs_dir; +enum edgetpu_pwr_state edgetpu_active_states[EDGETPU_NUM_STATES] = { + TPU_ACTIVE_UUD, + TPU_ACTIVE_SUD, + TPU_ACTIVE_UD, + TPU_ACTIVE_NOM, +}; + +uint32_t *edgetpu_states_display = edgetpu_active_states; + static int janeiro_pwr_state_init(struct device *dev) { int ret; @@ -234,7 +246,9 @@ static int janeiro_set_lpm(struct edgetpu_dev *etdev) static int janeiro_power_up(struct edgetpu_pm *etpm) { struct edgetpu_dev *etdev = etpm->etdev; +#if IS_ENABLED(CONFIG_GOOGLE_BCL) struct janeiro_platform_dev *edgetpu_pdev = to_janeiro_dev(etdev); +#endif int ret = 0; ret = janeiro_pwr_state_set( @@ -258,7 +272,7 @@ static int janeiro_power_up(struct edgetpu_pm *etpm) edgetpu_mailbox_reset_vii(etdev->mailbox_manager); } - if (!etdev->firmware) + if (!etdev->firmware || etdev->on_exit) return 0; /* @@ -282,12 +296,10 @@ static int janeiro_power_up(struct edgetpu_pm *etpm) /* attempt firmware run */ switch (edgetpu_firmware_status_locked(etdev)) { case FW_VALID: - ret = edgetpu_firmware_restart_locked(etdev); + ret = edgetpu_firmware_restart_locked(etdev, false); break; case FW_INVALID: - ret = edgetpu_firmware_run_locked(etdev->firmware, - EDGETPU_DEFAULT_FIRMWARE_NAME, - FW_DEFAULT); + ret = edgetpu_firmware_run_default_locked(etdev); break; default: break; @@ -326,6 +338,7 @@ static void janeiro_power_down(struct edgetpu_pm *etpm) struct edgetpu_dev *etdev = etpm->etdev; struct janeiro_platform_dev *edgetpu_pdev = to_janeiro_dev(etdev); u64 val; + int timeout_cnt = 0; etdev_info(etdev, "Powering down\n"); @@ -344,7 +357,17 @@ static void janeiro_power_down(struct edgetpu_pm *etpm) janeiro_pm_shutdown_firmware(edgetpu_pdev, etdev); edgetpu_kci_cancel_work_queues(etdev->kci); } - + do { + /* Manually delay 20us per retry till LPM shutdown finished */ + usleep_range(SHUTDOWN_DELAY_US_MIN, SHUTDOWN_DELAY_US_MAX); + val = edgetpu_dev_read_32_sync(etdev, EDGETPU_REG_LPM_CONTROL); + if ((val & 0x1000) || (val == 0)) + break; + timeout_cnt++; + } while (timeout_cnt < SHUTDOWN_MAX_DELAY_COUNT); + if (timeout_cnt == SHUTDOWN_MAX_DELAY_COUNT) + // Log the issue then continue to perform the shutdown forcefully. + etdev_err(etdev, "LPM shutdown failure, performing BLK shutdown\n"); janeiro_pwr_state_set(etdev, TPU_OFF); } @@ -392,12 +415,20 @@ static struct edgetpu_pm_handlers janeiro_pm_handlers = { .power_down = janeiro_power_down, }; -int janeiro_pm_create(struct edgetpu_dev *etdev) +int mobile_pm_create(struct edgetpu_dev *etdev) { return edgetpu_pm_create(etdev, &janeiro_pm_handlers); } -void janeiro_pm_destroy(struct edgetpu_dev *etdev) +void mobile_pm_destroy(struct edgetpu_dev *etdev) { edgetpu_pm_destroy(etdev); } + +void mobile_pm_set_pm_qos(struct edgetpu_dev *etdev, u32 pm_qos_val) +{ +} + +void mobile_pm_set_bts(struct edgetpu_dev *etdev, u32 bts_val) +{ +} diff --git a/drivers/edgetpu/janeiro/config-pwr-state.h b/drivers/edgetpu/janeiro/config-pwr-state.h new file mode 100644 index 0000000..cdea172 --- /dev/null +++ b/drivers/edgetpu/janeiro/config-pwr-state.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Chip-dependent power configuration and states. + * + * Copyright (C) 2021 Google, Inc. + */ + +#ifndef __JANEIRO_CONFIG_PWR_STATE_H__ +#define __JANEIRO_CONFIG_PWR_STATE_H__ + +/* + * TPU Power States: + * 0: Off + * 227000 Ultra Underdrive @227MHz + * 625000: Super Underdrive @625MHz + * 845000: Underdrive @845MHz + * 1066000: Nominal @1066MHz + */ +enum edgetpu_pwr_state { + TPU_OFF = 0, + TPU_ACTIVE_UUD = 227000, + TPU_ACTIVE_SUD = 625000, + TPU_ACTIVE_UD = 845000, + TPU_ACTIVE_NOM = 1066000, +}; + +#define EDGETPU_NUM_STATES 4 + +extern enum edgetpu_pwr_state edgetpu_active_states[]; + +extern uint32_t *edgetpu_states_display; + +#define TPU_POLICY_MAX TPU_ACTIVE_NOM + +#define TPU_ACPM_DOMAIN 7 + +#endif /* __JANEIRO_CONFIG_PWR_STATE_H__ */ diff --git a/drivers/edgetpu/janeiro/config-tpu-cpu.h b/drivers/edgetpu/janeiro/config-tpu-cpu.h index 1c84523..6cd2844 100644 --- a/drivers/edgetpu/janeiro/config-tpu-cpu.h +++ b/drivers/edgetpu/janeiro/config-tpu-cpu.h @@ -16,7 +16,8 @@ #define EDGETPU_REG_INSTRUCTION_REMAP_NEW_BASE 0x190068 #define EDGETPU_REG_INSTRUCTION_REMAP_SECURITY 0x190070 #define EDGETPU_REG_SECURITY 0x190048 -#define EDGETPU_REG_POWER_CONTROL 0x1A0008 +#define EDGETPU_REG_POWER_CONTROL 0x1A0008 +#define EDGETPU_REG_LPM_CONTROL 0x1D0000 #define PSTATE_SHIFT 1 #define PSTATE (1 << PSTATE_SHIFT) #define PREQ (1 << 2) diff --git a/drivers/edgetpu/janeiro/config.h b/drivers/edgetpu/janeiro/config.h index 4dc759f..add13dc 100644 --- a/drivers/edgetpu/janeiro/config.h +++ b/drivers/edgetpu/janeiro/config.h @@ -18,6 +18,8 @@ /* Reserved VCID that uses the extra partition. */ #define EDGETPU_VCID_EXTRA_PARTITION 0 +/* Is a "mobile" style device. */ +#define EDGETPU_FEATURE_MOBILE #define EDGETPU_HAS_WAKELOCK /* @@ -52,6 +54,7 @@ #define EDGETPU_REMAPPED_DATA_ADDR \ (EDGETPU_INSTRUCTION_REMAP_BASE + EDGETPU_REMAPPED_DATA_OFFSET) #include "config-mailbox.h" +#include "config-pwr-state.h" #include "config-tpu-cpu.h" #include "csrs.h" diff --git a/drivers/edgetpu/mobile-firmware.h b/drivers/edgetpu/mobile-firmware.h index e0c8dd8..691eaf5 100644 --- a/drivers/edgetpu/mobile-firmware.h +++ b/drivers/edgetpu/mobile-firmware.h @@ -49,6 +49,4 @@ struct mobile_image_header { int mobile_edgetpu_firmware_create(struct edgetpu_dev *etdev); void mobile_edgetpu_firmware_destroy(struct edgetpu_dev *etdev); -int mobile_edgetpu_firmware_run_default(struct edgetpu_dev *etdev); - #endif /* __MOBILE_FIRMWARE_H__ */ diff --git a/drivers/edgetpu/mobile-pm.h b/drivers/edgetpu/mobile-pm.h new file mode 100644 index 0000000..a04b6bb --- /dev/null +++ b/drivers/edgetpu/mobile-pm.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Power management header for mobile chipsets. + * + * Copyright (C) 2021 Google, Inc. + */ + +#ifndef __MOBILE_PM_H__ +#define __MOBILE_PM_H__ + +#include "edgetpu-internal.h" +#include "edgetpu-kci.h" + +/* Can't build out of tree with acpm_dvfs unless kernel supports ACPM */ +#if IS_ENABLED(CONFIG_ACPM_DVFS) || IS_ENABLED(CONFIG_EDGETPU_TEST) + +#include <linux/acpm_dvfs.h> + +#else + +static unsigned long exynos_acpm_rate; +static inline int exynos_acpm_set_rate(unsigned int id, unsigned long rate) +{ + exynos_acpm_rate = rate; + return 0; +} +static inline int exynos_acpm_set_init_freq(unsigned int dfs_id, + unsigned long freq) +{ + return 0; +} +static inline unsigned long exynos_acpm_get_rate(unsigned int id, + unsigned long dbg_val) +{ + return exynos_acpm_rate; +} +static inline int exynos_acpm_set_policy(unsigned int id, unsigned long policy) +{ + return 0; +} +#endif /* IS_ENABLED(CONFIG_ACPM_DVFS) || IS_ENABLED(CONFIG_EDGETPU_TEST) */ + +/* + * Request codes from firmware + * Values must match with firmware code base + */ +enum mobile_reverse_kci_code { + RKCI_CODE_PM_QOS = RKCI_CHIP_CODE_FIRST + 1, + RKCI_CODE_BTS = RKCI_CHIP_CODE_FIRST + 2, +}; + +/* + * Initialize a power management interface for an edgetpu device on mobile + * chipsets. + */ +int mobile_pm_create(struct edgetpu_dev *etdev); + +/* + * Destroy power management interface for an edgetpu device on mobile chipsets. + */ +void mobile_pm_destroy(struct edgetpu_dev *etdev); + +/* Set required QoS value for the edgetpu device. */ +void mobile_pm_set_pm_qos(struct edgetpu_dev *etdev, u32 pm_qos_val); + +/* Set BTS value for the edgetpu device. */ +void mobile_pm_set_bts(struct edgetpu_dev *etdev, u32 bts_val); + +#endif /* __MOBILE_PM_H__ */ |