From 8b939d935ca255d445f37054c6b0a054e2121497 Mon Sep 17 00:00:00 2001 From: Zuma copybara merger Date: Sat, 29 Apr 2023 03:12:23 +0000 Subject: [Copybara Auto Merge] Merge branch zuma into android14-gs-pixel-5.15 edgetpu: return -EIO for operations on device with bad firmware state gcip: Move HAS_IOVAD_BEST_FIT_ALGO to header Bug: 278833993 gcip: Add HAS_IOVAD_BEST_FIT_ALGO Bug: 278833993 (repeat) edgetpu: Enable iommu dma best fit algorithm Bug: 278833993 (repeat) edgetpu: Add client_priv to edgetpu_mailbox_attr Bug: 279805034 edgetpu: adopt gcip-usage-stats gcip: remove useless written variable gcip: fix false not found freq warning log gcip: decide subcomponent after update_usage_kci gcip: support unsorted default DVFS freqs gcip: add missing component type Bug: 279866789 gcip: add missing thread IDs Bug: 279866789 (repeat) gcip: manage response status internally Bug: 278819094 gcip: add status field to struct gcip_mailbox_async_resp Bug: 278819094 (repeat) gcip: introduce struct gcip_mailbox_async_resp Bug: 278819094 (repeat) gcip: Conditionally include dma-iommu.h gcip: add a new line to dvfs_freqs show gcip: remove GCIP_KCI_STATUS_* Bug: 278819094 (repeat) gcip: manage response status internally Bug: 278819094 (repeat) gcip: add status field to struct gcip_mailbox_async_resp Bug: 278819094 (repeat) gcip: introduce struct gcip_mailbox_async_resp Bug: 278819094 (repeat) Signed-off-by: Zuma copybara merger GitOrigin-RevId: f1ea5702d8b432368f0705e7ed5b29746b9c565e Change-Id: I9d9e17aee270bd45d967fe2a45899492f9fb1212 --- drivers/edgetpu/edgetpu-core.c | 7 +- drivers/edgetpu/edgetpu-device-group.c | 3 +- drivers/edgetpu/edgetpu-external.c | 3 +- drivers/edgetpu/edgetpu-firmware.c | 2 +- drivers/edgetpu/edgetpu-internal.h | 4 - drivers/edgetpu/edgetpu-kci.c | 27 +- drivers/edgetpu/edgetpu-kci.h | 7 +- drivers/edgetpu/edgetpu-mailbox.c | 26 +- drivers/edgetpu/edgetpu-mailbox.h | 10 +- drivers/edgetpu/edgetpu-mobile-platform.c | 11 +- drivers/edgetpu/edgetpu-usage-stats.c | 1068 ++++---------------- drivers/edgetpu/edgetpu-usage-stats.h | 242 +---- drivers/edgetpu/edgetpu.h | 1 + .../gcip-kernel-driver/drivers/gcip/gcip-iommu.c | 11 +- .../gcip-kernel-driver/drivers/gcip/gcip-kci.c | 16 - .../gcip-kernel-driver/drivers/gcip/gcip-mailbox.c | 65 +- .../drivers/gcip/gcip-usage-stats.c | 48 +- .../gcip-kernel-driver/include/gcip/gcip-iommu.h | 10 + .../gcip-kernel-driver/include/gcip/gcip-kci.h | 22 +- .../gcip-kernel-driver/include/gcip/gcip-mailbox.h | 23 +- .../include/gcip/gcip-usage-stats.h | 6 + 21 files changed, 346 insertions(+), 1266 deletions(-) diff --git a/drivers/edgetpu/edgetpu-core.c b/drivers/edgetpu/edgetpu-core.c index fe441a5..18bffa2 100644 --- a/drivers/edgetpu/edgetpu-core.c +++ b/drivers/edgetpu/edgetpu-core.c @@ -384,11 +384,10 @@ int edgetpu_get_state_errno_locked(struct edgetpu_dev *etdev) { switch (etdev->state) { case ETDEV_STATE_BAD: - return -ENODEV; + case ETDEV_STATE_NOFW: + return -EIO; case ETDEV_STATE_FWLOADING: return -EAGAIN; - case ETDEV_STATE_NOFW: - return -EINVAL; default: break; } @@ -450,8 +449,6 @@ int edgetpu_device_add(struct edgetpu_dev *etdev, etdev->vcid_pool = (1u << EDGETPU_NUM_VCIDS) - 1; mutex_init(&etdev->state_lock); etdev->state = ETDEV_STATE_NOFW; - etdev->freq_count = 0; - mutex_init(&etdev->freq_lock); mutex_init(&etdev->device_prop.lock); ret = edgetpu_soc_init(etdev); if (ret) diff --git a/drivers/edgetpu/edgetpu-device-group.c b/drivers/edgetpu/edgetpu-device-group.c index 1c1aec5..cb7d9d9 100644 --- a/drivers/edgetpu/edgetpu-device-group.c +++ b/drivers/edgetpu/edgetpu-device-group.c @@ -83,7 +83,8 @@ static int edgetpu_group_activate(struct edgetpu_device_group *group) return 0; mailbox_id = edgetpu_group_context_id_locked(group); - ret = edgetpu_mailbox_activate(group->etdev, mailbox_id, group->vcid, !group->activated); + ret = edgetpu_mailbox_activate(group->etdev, mailbox_id, group->mbox_attr.client_priv, + group->vcid, !group->activated); if (ret) { etdev_err(group->etdev, "activate mailbox for VCID %d failed with %d", group->vcid, ret); diff --git a/drivers/edgetpu/edgetpu-external.c b/drivers/edgetpu/edgetpu-external.c index 301f565..fb452c7 100644 --- a/drivers/edgetpu/edgetpu-external.c +++ b/drivers/edgetpu/edgetpu-external.c @@ -109,7 +109,8 @@ static int edgetpu_external_mailbox_alloc(struct device *edgetpu_dev, req.attr = group->mbox_attr; } - ret = edgetpu_mailbox_enable_ext(client, EDGETPU_MAILBOX_ID_USE_ASSOC, &req); + ret = edgetpu_mailbox_enable_ext(client, EDGETPU_MAILBOX_ID_USE_ASSOC, &req, + group->mbox_attr.client_priv); if (ret) goto error_put_group; mutex_lock(&group->lock); diff --git a/drivers/edgetpu/edgetpu-firmware.c b/drivers/edgetpu/edgetpu-firmware.c index 81e6cad..86a4f16 100644 --- a/drivers/edgetpu/edgetpu-firmware.c +++ b/drivers/edgetpu/edgetpu-firmware.c @@ -375,7 +375,7 @@ static int edgetpu_firmware_run_locked(struct edgetpu_firmware *et_fw, edgetpu_firmware_set_state(et_fw, ret); /* If previous firmware was metrics v1-only reset that flag and probe this again. */ if (etdev->usage_stats) - etdev->usage_stats->use_metrics_v1 = false; + etdev->usage_stats->ustats.version = EDGETPU_USAGE_METRIC_VERSION; return ret; out_unload_new_fw: diff --git a/drivers/edgetpu/edgetpu-internal.h b/drivers/edgetpu/edgetpu-internal.h index 747150f..17dc48c 100644 --- a/drivers/edgetpu/edgetpu-internal.h +++ b/drivers/edgetpu/edgetpu-internal.h @@ -233,10 +233,6 @@ struct edgetpu_dev { /* debug dump handlers */ edgetpu_debug_dump_handlers *debug_dump_handlers; struct work_struct debug_dump_work; - - struct mutex freq_lock; /* protects below freq_* variables */ - uint32_t *freq_table; /* Array to record reported frequencies by f/w */ - uint32_t freq_count; /* Number of entries in freq_table */ }; struct edgetpu_dev_iface { diff --git a/drivers/edgetpu/edgetpu-kci.c b/drivers/edgetpu/edgetpu-kci.c index 7001329..7a90346 100644 --- a/drivers/edgetpu/edgetpu-kci.c +++ b/drivers/edgetpu/edgetpu-kci.c @@ -17,6 +17,7 @@ #include #include +#include #include "edgetpu-firmware.h" #include "edgetpu-internal.h" @@ -47,12 +48,15 @@ #endif /* A macro for KCIs to leave early when the device state is known to be bad. */ -#define RETURN_ERRNO_IF_ETDEV_NOT_GOOD(etkci) \ +#define RETURN_ERRNO_IF_ETDEV_NOT_GOOD(etkci, opstring) \ do { \ struct edgetpu_mailbox *mailbox = etkci->mailbox; \ int ret = edgetpu_get_state_errno_locked(mailbox->etdev); \ - if (ret) \ + if (ret) { \ + etdev_err(mailbox->etdev, "%s failed: device state %u (%d)", \ + opstring, mailbox->etdev->state, ret); \ return ret; \ + } \ } while (0) static int edgetpu_kci_alloc_queue(struct edgetpu_dev *etdev, struct edgetpu_mailbox *mailbox, @@ -501,16 +505,17 @@ int edgetpu_kci_update_usage_locked(struct edgetpu_dev *etdev) /* TODO(b/271372136): remove v1 when v1 firmware no longer in use. */ retry_v1: - if (etdev->usage_stats && etdev->usage_stats->use_metrics_v1) + if (etdev->usage_stats && etdev->usage_stats->ustats.version == GCIP_USAGE_STATS_V1) cmd.code = GCIP_KCI_CODE_GET_USAGE_V1; cmd.dma.address = mem.tpu_addr; cmd.dma.size = EDGETPU_USAGE_BUFFER_SIZE; - memset(mem.vaddr, 0, sizeof(struct edgetpu_usage_header)); + memset(mem.vaddr, 0, sizeof(struct gcip_usage_stats_header)); ret = gcip_kci_send_cmd_return_resp(etdev->etkci->kci, &cmd, &resp); if (ret == GCIP_KCI_ERROR_UNIMPLEMENTED || ret == GCIP_KCI_ERROR_UNAVAILABLE) { - if (etdev->usage_stats && !etdev->usage_stats->use_metrics_v1) { - etdev->usage_stats->use_metrics_v1 = true; + if (etdev->usage_stats && + etdev->usage_stats->ustats.version != GCIP_USAGE_STATS_V1) { + etdev->usage_stats->ustats.version = GCIP_USAGE_STATS_V1; goto retry_v1; } etdev_dbg(etdev, "firmware does not report usage\n"); @@ -578,10 +583,11 @@ int edgetpu_kci_get_debug_dump(struct edgetpu_kci *etkci, tpu_addr_t tpu_addr, s return gcip_kci_send_cmd(etkci->kci, &cmd); } -int edgetpu_kci_open_device(struct edgetpu_kci *etkci, u32 mailbox_map, s16 vcid, bool first_open) +int edgetpu_kci_open_device(struct edgetpu_kci *etkci, u32 mailbox_map, u32 client_priv, s16 vcid, + bool first_open) { const struct edgetpu_kci_open_device_detail detail = { - .mailbox_map = mailbox_map, + .client_priv = client_priv, .vcid = vcid, .flags = (mailbox_map << 1) | first_open, }; @@ -595,8 +601,7 @@ int edgetpu_kci_open_device(struct edgetpu_kci *etkci, u32 mailbox_map, s16 vcid if (!etkci || !etkci->kci) return -ENODEV; - RETURN_ERRNO_IF_ETDEV_NOT_GOOD(etkci); - + RETURN_ERRNO_IF_ETDEV_NOT_GOOD(etkci, "open device"); if (vcid < 0) return gcip_kci_send_cmd(etkci->kci, &cmd); @@ -615,7 +620,7 @@ int edgetpu_kci_close_device(struct edgetpu_kci *etkci, u32 mailbox_map) if (!etkci || !etkci->kci) return -ENODEV; - RETURN_ERRNO_IF_ETDEV_NOT_GOOD(etkci); + RETURN_ERRNO_IF_ETDEV_NOT_GOOD(etkci, "close device"); return gcip_kci_send_cmd(etkci->kci, &cmd); } diff --git a/drivers/edgetpu/edgetpu-kci.h b/drivers/edgetpu/edgetpu-kci.h index 25674d7..13e6c8d 100644 --- a/drivers/edgetpu/edgetpu-kci.h +++ b/drivers/edgetpu/edgetpu-kci.h @@ -56,8 +56,8 @@ struct edgetpu_kci_device_group_detail { }; struct edgetpu_kci_open_device_detail { - /* The bit map of mailboxes to be opened. */ - u16 mailbox_map; + /* The client privilege level. */ + u16 client_priv; /* * Virtual context ID @mailbox_id is associated to. * For device groups with @mailbox_detachable attribute the mailbox attached to the group @@ -168,7 +168,8 @@ int edgetpu_kci_get_debug_dump(struct edgetpu_kci *etkci, tpu_addr_t tpu_addr, s * You usually shouldn't call this directly - consider using * edgetpu_mailbox_activate() or edgetpu_mailbox_activate_bulk() instead. */ -int edgetpu_kci_open_device(struct edgetpu_kci *etkci, u32 mailbox_map, s16 vcid, bool first_open); +int edgetpu_kci_open_device(struct edgetpu_kci *etkci, u32 mailbox_map, u32 client_priv, s16 vcid, + bool first_open); /* * Inform the firmware that the VII mailboxes included in @mailbox_map are closed. diff --git a/drivers/edgetpu/edgetpu-mailbox.c b/drivers/edgetpu/edgetpu-mailbox.c index 37ee227..80c98d2 100644 --- a/drivers/edgetpu/edgetpu-mailbox.c +++ b/drivers/edgetpu/edgetpu-mailbox.c @@ -1006,7 +1006,8 @@ void edgetpu_mailbox_external_disable_free_locked(struct edgetpu_device_group *g edgetpu_mailbox_external_free(group); } -static int edgetpu_mailbox_external_enable_by_id(struct edgetpu_client *client, int mailbox_id) +static int edgetpu_mailbox_external_enable_by_id(struct edgetpu_client *client, int mailbox_id, + u32 client_priv) { int ret; @@ -1019,7 +1020,7 @@ static int edgetpu_mailbox_external_enable_by_id(struct edgetpu_client *client, etdev_dbg(client->etdev, "Enabling mailbox: %d\n", mailbox_id); - ret = edgetpu_mailbox_activate(client->etdev, mailbox_id, -1, false); + ret = edgetpu_mailbox_activate(client->etdev, mailbox_id, client_priv, -1, false); if (ret) etdev_err(client->etdev, "Activate mailbox %d failed: %d", mailbox_id, ret); else @@ -1066,7 +1067,8 @@ int edgetpu_mailbox_activate_external_mailbox(struct edgetpu_device_group *group for (i = 0; i < ext_mailbox->count; i++) mbox_map |= BIT(ext_mailbox->descriptors[i].mailbox->mailbox_id); - ret = edgetpu_mailbox_activate_bulk(ext_mailbox->etdev, mbox_map, vcid, false); + ret = edgetpu_mailbox_activate_bulk(ext_mailbox->etdev, mbox_map, + group->mbox_attr.client_priv, vcid, false); if (ret) etdev_err(group->etdev, "Activate mailbox bulk failed: %d", ret); @@ -1102,12 +1104,13 @@ void edgetpu_mailbox_deactivate_external_mailbox(struct edgetpu_device_group *gr } int edgetpu_mailbox_enable_ext(struct edgetpu_client *client, int mailbox_id, - struct edgetpu_external_mailbox_req *ext_mailbox_req) + struct edgetpu_external_mailbox_req *ext_mailbox_req, + u32 client_priv) { if (mailbox_id == EDGETPU_MAILBOX_ID_USE_ASSOC) return edgetpu_mailbox_external_alloc_enable(client, ext_mailbox_req); else - return edgetpu_mailbox_external_enable_by_id(client, mailbox_id); + return edgetpu_mailbox_external_enable_by_id(client, mailbox_id, client_priv); } int edgetpu_mailbox_disable_ext(struct edgetpu_client *client, int mailbox_id) @@ -1118,16 +1121,16 @@ int edgetpu_mailbox_disable_ext(struct edgetpu_client *client, int mailbox_id) return edgetpu_mailbox_external_disable_by_id(client, mailbox_id); } -int edgetpu_mailbox_activate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map, s16 vcid, - bool first_open) +int edgetpu_mailbox_activate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map, u32 client_priv, + s16 vcid, bool first_open) { struct edgetpu_handshake *eh = &etdev->mailbox_manager->open_devices; int ret = 0; mutex_lock(&eh->lock); if (mailbox_map & ~eh->fw_state) - ret = edgetpu_kci_open_device(etdev->etkci, mailbox_map & ~eh->fw_state, vcid, - first_open); + ret = edgetpu_kci_open_device(etdev->etkci, mailbox_map & ~eh->fw_state, + client_priv, vcid, first_open); if (!ret) { eh->state |= mailbox_map; eh->fw_state |= mailbox_map; @@ -1144,9 +1147,10 @@ int edgetpu_mailbox_activate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map, s1 } -int edgetpu_mailbox_activate(struct edgetpu_dev *etdev, u32 mailbox_id, s16 vcid, bool first_open) +int edgetpu_mailbox_activate(struct edgetpu_dev *etdev, u32 mailbox_id, u32 client_priv, s16 vcid, + bool first_open) { - return edgetpu_mailbox_activate_bulk(etdev, BIT(mailbox_id), vcid, first_open); + return edgetpu_mailbox_activate_bulk(etdev, BIT(mailbox_id), client_priv, vcid, first_open); } void edgetpu_mailbox_deactivate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map) diff --git a/drivers/edgetpu/edgetpu-mailbox.h b/drivers/edgetpu/edgetpu-mailbox.h index 36194cf..80f6e10 100644 --- a/drivers/edgetpu/edgetpu-mailbox.h +++ b/drivers/edgetpu/edgetpu-mailbox.h @@ -329,7 +329,8 @@ void edgetpu_mailbox_restore_active_mailbox_queues(struct edgetpu_dev *etdev); * Otherwise, activate the external mailbox with id @mailbox_id. */ int edgetpu_mailbox_enable_ext(struct edgetpu_client *client, int mailbox_id, - struct edgetpu_external_mailbox_req *ext_mailbox_req); + struct edgetpu_external_mailbox_req *ext_mailbox_req, + u32 client_priv); /* * Notify firmware of an external mailboxes becoming inactive. @@ -342,8 +343,8 @@ int edgetpu_mailbox_disable_ext(struct edgetpu_client *client, int mailbox_id); * Returns what edgetpu_kci_open_device() returned. * Caller ensures device is powered on. */ -int edgetpu_mailbox_activate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map, s16 vcid, - bool first_open); +int edgetpu_mailbox_activate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map, u32 client_priv, + s16 vcid, bool first_open); /* * Activates @mailbox_id, OPEN_DEVICE KCI will be sent. @@ -354,7 +355,8 @@ int edgetpu_mailbox_activate_bulk(struct edgetpu_dev *etdev, u32 mailbox_map, s1 * Returns what edgetpu_kci_open_device() returned. * Caller ensures device is powered on. */ -int edgetpu_mailbox_activate(struct edgetpu_dev *etdev, u32 mailbox_id, s16 vcid, bool first_open); +int edgetpu_mailbox_activate(struct edgetpu_dev *etdev, u32 mailbox_id, u32 client_priv, s16 vcid, + bool first_open); /* * Similar to edgetpu_mailbox_activate_bulk() but sends CLOSE_DEVICE KCI with the @mailbox_map diff --git a/drivers/edgetpu/edgetpu-mobile-platform.c b/drivers/edgetpu/edgetpu-mobile-platform.c index 0147d9d..db66dab 100644 --- a/drivers/edgetpu/edgetpu-mobile-platform.c +++ b/drivers/edgetpu/edgetpu-mobile-platform.c @@ -14,6 +14,11 @@ #include #include +#include + +#if HAS_IOVAD_BEST_FIT_ALGO +#include +#endif #include "edgetpu-config.h" #include "edgetpu-dmabuf.h" @@ -161,7 +166,7 @@ int edgetpu_chip_acquire_ext_mailbox(struct edgetpu_client *client, mutex_unlock(&etmdev->tz_mailbox_lock); return -EBUSY; } - ret = edgetpu_mailbox_enable_ext(client, EDGETPU_TZ_MAILBOX_ID, NULL); + ret = edgetpu_mailbox_enable_ext(client, EDGETPU_TZ_MAILBOX_ID, NULL, 0); if (!ret) etmdev->secure_client = client; mutex_unlock(&etmdev->tz_mailbox_lock); @@ -349,6 +354,10 @@ static int edgetpu_mobile_platform_probe(struct platform_device *pdev, goto out_cleanup_fw; } +#if HAS_IOVAD_BEST_FIT_ALGO + iommu_dma_enable_best_fit_algo(dev); +#endif + INIT_LIST_HEAD(&etmdev->fw_ctx_list); mutex_init(&etmdev->fw_ctx_list_lock); diff --git a/drivers/edgetpu/edgetpu-usage-stats.c b/drivers/edgetpu/edgetpu-usage-stats.c index 9934ca6..16e6892 100644 --- a/drivers/edgetpu/edgetpu-usage-stats.c +++ b/drivers/edgetpu/edgetpu-usage-stats.c @@ -2,448 +2,58 @@ /* * EdgeTPU usage stats * - * Copyright (C) 2020 Google, Inc. + * Copyright (C) 2020-2023 Google, Inc. */ -#include -#include +#include +#include +#include +#include + +#include #include "edgetpu-config.h" #include "edgetpu-internal.h" #include "edgetpu-kci.h" #include "edgetpu-usage-stats.h" -/* Max number of frequencies to support */ -#define EDGETPU_MAX_STATES 10 - -struct uid_entry { - int32_t uid; - uint64_t time_in_state[EDGETPU_MAX_STATES]; - struct hlist_node node; -}; - -static int tpu_state_map(struct edgetpu_dev *etdev, uint32_t state) -{ - int i, idx = 0; - - mutex_lock(&etdev->freq_lock); - /* Use frequency table if f/w already reported via usage_stats */ - if (etdev->freq_table) { - for (i = etdev->freq_count - 1; i >= 0; i--) { - if (state == etdev->freq_table[i]) - idx = i; - } - mutex_unlock(&etdev->freq_lock); - return idx; - } - - mutex_unlock(&etdev->freq_lock); - - /* - * use predefined state table in case of no f/w reported supported - * frequencies. - */ - for (i = (EDGETPU_NUM_STATES - 1); i >= 0; i--) { - if (state >= edgetpu_active_states[i]) - return i; - } - - return 0; -} - -/* Caller must hold usage_stats lock */ -static struct uid_entry * -find_uid_entry_locked(int32_t uid, struct edgetpu_usage_stats *ustats) -{ - struct uid_entry *uid_entry; - - hash_for_each_possible(ustats->uid_hash_table, uid_entry, node, uid) { - if (uid_entry->uid == uid) - return uid_entry; - } - - return NULL; -} - -int edgetpu_usage_add(struct edgetpu_dev *etdev, struct tpu_usage *tpu_usage) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - struct uid_entry *uid_entry; - int state = tpu_state_map(etdev, tpu_usage->power_state); - - if (!ustats) - return 0; - - /* Note: as of metrics v2 the cluster_id is always zero and is ignored. */ - etdev_dbg(etdev, "%s: uid=%u state=%u dur=%u", __func__, - tpu_usage->uid, tpu_usage->power_state, - tpu_usage->duration_us); - mutex_lock(&ustats->usage_stats_lock); - - /* Find the uid in uid_hash_table first */ - uid_entry = find_uid_entry_locked(tpu_usage->uid, ustats); - if (uid_entry) { - uid_entry->time_in_state[state] += tpu_usage->duration_us; - mutex_unlock(&ustats->usage_stats_lock); - return 0; - } - - /* Allocate memory for this uid */ - uid_entry = kzalloc(sizeof(*uid_entry), GFP_KERNEL); - if (!uid_entry) { - mutex_unlock(&ustats->usage_stats_lock); - return -ENOMEM; - } - - uid_entry->uid = tpu_usage->uid; - uid_entry->time_in_state[state] += tpu_usage->duration_us; - - /* Add uid_entry to the uid_hash_table */ - hash_add(ustats->uid_hash_table, &uid_entry->node, tpu_usage->uid); - - mutex_unlock(&ustats->usage_stats_lock); - - return 0; -} - -static void edgetpu_utilization_update( - struct edgetpu_dev *etdev, - struct edgetpu_component_activity *activity) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - - if (!ustats) - return; - - etdev_dbg(etdev, "%s: comp=%d utilized %d%%\n", __func__, - activity->component, activity->utilization); - - mutex_lock(&ustats->usage_stats_lock); - if (activity->utilization && activity->component >= 0 && - activity->component < EDGETPU_USAGE_COMPONENT_COUNT) - ustats->component_utilization[activity->component] = - activity->utilization; - mutex_unlock(&ustats->usage_stats_lock); -} - -static void edgetpu_counter_update(struct edgetpu_dev *etdev, struct edgetpu_usage_counter *counter, - uint version) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - uint component = version > 1 ? counter->component_id : 0; - - if (!ustats) - return; - - etdev_dbg(etdev, "%s: type=%d value=%llu comp=%u\n", __func__, counter->type, - counter->value, component); - - mutex_lock(&ustats->usage_stats_lock); - if (counter->type >= 0 && counter->type < EDGETPU_COUNTER_COUNT) - ustats->counter[counter->type][component] += counter->value; - mutex_unlock(&ustats->usage_stats_lock); -} - -static void edgetpu_counter_clear(struct edgetpu_dev *etdev, - enum edgetpu_usage_counter_type counter_type) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - int i; - - if (counter_type >= EDGETPU_COUNTER_COUNT) - return; - - mutex_lock(&ustats->usage_stats_lock); - for (i = 0; i < EDGETPU_TPU_CLUSTER_COUNT; i++) - ustats->counter[counter_type][i] = 0; - mutex_unlock(&ustats->usage_stats_lock); -} - -static void edgetpu_max_watermark_update(struct edgetpu_dev *etdev, - struct edgetpu_usage_max_watermark *max_watermark, - uint version) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - uint component = version > 1 ? max_watermark->component_id : 0; - - if (!ustats) - return; - - etdev_dbg(etdev, "%s: type=%d value=%llu comp=%u\n", __func__, max_watermark->type, - max_watermark->value, component); - - if (max_watermark->type < 0 || - max_watermark->type >= EDGETPU_MAX_WATERMARK_TYPE_COUNT) - return; - - mutex_lock(&ustats->usage_stats_lock); - if (max_watermark->value > ustats->max_watermark[max_watermark->type][component]) - ustats->max_watermark[max_watermark->type][component] = - max_watermark->value; - mutex_unlock(&ustats->usage_stats_lock); -} - -static void edgetpu_max_watermark_clear(struct edgetpu_dev *etdev, - enum edgetpu_usage_max_watermark_type max_watermark_type) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - int i; - - if (max_watermark_type < 0 || max_watermark_type >= EDGETPU_MAX_WATERMARK_TYPE_COUNT) - return; - - mutex_lock(&ustats->usage_stats_lock); - for (i = 0; i < EDGETPU_TPU_CLUSTER_COUNT; i++) - ustats->max_watermark[max_watermark_type][i] = 0; - mutex_unlock(&ustats->usage_stats_lock); -} - -static void edgetpu_thread_stats_update( - struct edgetpu_dev *etdev, - struct edgetpu_thread_stats *thread_stats) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - - if (!ustats) - return; - - etdev_dbg(etdev, "%s: id=%d stackmax=%u\n", __func__, - thread_stats->thread_id, thread_stats->max_stack_usage_bytes); - - if (thread_stats->thread_id < 0 || - thread_stats->thread_id >= EDGETPU_FW_THREAD_COUNT) - return; - - mutex_lock(&ustats->usage_stats_lock); - if (thread_stats->max_stack_usage_bytes > - ustats->thread_stack_max[thread_stats->thread_id]) - ustats->thread_stack_max[thread_stats->thread_id] = - thread_stats->max_stack_usage_bytes; - mutex_unlock(&ustats->usage_stats_lock); -} - -/* Record new supported frequencies if reported by firmware */ -static void edgetpu_dvfs_frequency_update(struct edgetpu_dev *etdev, uint32_t frequency) -{ - uint32_t *freq_table, i; - - mutex_lock(&etdev->freq_lock); - if (!etdev->freq_table) { - freq_table = kvmalloc(EDGETPU_MAX_STATES * sizeof(uint32_t), GFP_KERNEL); - if (!freq_table) { - etdev_warn(etdev, "Unable to create supported frequencies table"); - goto out; - } - etdev->freq_count = 0; - etdev->freq_table = freq_table; - } - - freq_table = etdev->freq_table; - - for (i = 0; i < etdev->freq_count; i++) { - if (freq_table[i] == frequency) - goto out; - } - - if (etdev->freq_count >= EDGETPU_MAX_STATES) { - etdev_warn(etdev, "Unable to record supported frequencies"); - goto out; - } - - freq_table[etdev->freq_count++] = frequency; -out: - mutex_unlock(&etdev->freq_lock); -} - -void edgetpu_usage_stats_process_buffer(struct edgetpu_dev *etdev, void *buf) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - struct edgetpu_usage_metric *metric; - uint metric_size; - uint num_metrics; - uint version; - int i; - - if (!ustats) - return; - - /* TODO(b/271372136): remove v1 when v1 firmware no longer in use. */ - if (ustats->use_metrics_v1) { - struct edgetpu_usage_header_v1 *header = buf; - - metric_size = header->metric_size; - num_metrics = header->num_metrics; - version = 1; - metric = (struct edgetpu_usage_metric *)(header + 1); - } else { - struct edgetpu_usage_header *header = buf; - - metric_size = header->metric_size; - num_metrics = header->num_metrics; - version = header->version; - metric = (struct edgetpu_usage_metric *)((char *)header + header->header_bytes); - } - - etdev_dbg(etdev, "%s: v=%u n=%u sz=%u", __func__, version, num_metrics, metric_size); - if (metric_size < EDGETPU_USAGE_METRIC_SIZE_V1) { - etdev_warn_once(etdev, "fw metric size %u less than minimum %u", - metric_size, EDGETPU_USAGE_METRIC_SIZE_V1); - return; - } - - if (metric_size > sizeof(struct edgetpu_usage_metric)) - etdev_dbg(etdev, "fw metrics are later version with unknown fields"); - - for (i = 0; i < num_metrics; i++) { - switch (metric->type) { - case EDGETPU_METRIC_TYPE_TPU_USAGE: - edgetpu_usage_add(etdev, &metric->tpu_usage); - break; - case EDGETPU_METRIC_TYPE_COMPONENT_ACTIVITY: - edgetpu_utilization_update( - etdev, &metric->component_activity); - break; - case EDGETPU_METRIC_TYPE_COUNTER: - edgetpu_counter_update(etdev, &metric->counter, version); - break; - case EDGETPU_METRIC_TYPE_MAX_WATERMARK: - edgetpu_max_watermark_update(etdev, &metric->max_watermark, version); - break; - case EDGETPU_METRIC_TYPE_THREAD_STATS: - edgetpu_thread_stats_update(etdev, &metric->thread_stats); - break; - case EDGETPU_METRIC_TYPE_DVFS_FREQUENCY_INFO: - edgetpu_dvfs_frequency_update(etdev, metric->dvfs_frequency_info); - break; - default: - etdev_dbg(etdev, "%s: %d: skip unknown type=%u", - __func__, i, metric->type); - break; - } - - metric = (struct edgetpu_usage_metric *)((char *)metric + metric_size); - } -} - -int edgetpu_usage_get_utilization(struct edgetpu_dev *etdev, - enum edgetpu_usage_component component) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - int32_t val; - - if (component >= EDGETPU_USAGE_COMPONENT_COUNT) - return -1; - edgetpu_kci_update_usage(etdev); - mutex_lock(&ustats->usage_stats_lock); - val = ustats->component_utilization[component]; - ustats->component_utilization[component] = 0; - mutex_unlock(&ustats->usage_stats_lock); - return val; -} - -/* - * Resyncs firmware stats and formats the requested counter in the supplied buffer. - * - * If @report_per_cluster is true, and if the firmware implements metrics V2 or higher, - * then one value is formatted per cluster (for chips with only one cluster only one value is - * formatted). - * - * Returns the number of bytes written to buf. - */ -static ssize_t edgetpu_usage_format_counter(struct edgetpu_dev *etdev, char *buf, - enum edgetpu_usage_counter_type counter_type, - bool report_per_cluster) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - uint ncomponents = report_per_cluster && !etdev->usage_stats->use_metrics_v1 ? - EDGETPU_TPU_CLUSTER_COUNT : 1; - uint i; - ssize_t ret = 0; - - if (counter_type >= EDGETPU_COUNTER_COUNT) - return 0; - edgetpu_kci_update_usage(etdev); - mutex_lock(&ustats->usage_stats_lock); - for (i = 0; i < ncomponents; i++) { - if (i) - ret += scnprintf(buf + ret, PAGE_SIZE - ret, " "); - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%llu", - ustats->counter[counter_type][i]); - } - mutex_unlock(&ustats->usage_stats_lock); - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); - return ret; -} - /* - * Resyncs firmware stats and formats the requested max watermark in the supplied buffer. - * - * If @report_per_cluster is true, and if the firmware implements metrics V2 or higher, - * then one value is formatted per cluster (for chips with only one cluster only one value is - * formatted). - * - * Returns the number of bytes written to buf. + * TODO(b/279844328): Use the default show callback of GCIP (i.e., remove this callback and pass + * NULL to the show callback of gcip_usage_stats_attr_tpu_usage) after we are ready to update the + * logging format of this metric. */ -static ssize_t edgetpu_usage_format_max_watermark( - struct edgetpu_dev *etdev, char *buf, - enum edgetpu_usage_max_watermark_type max_watermark_type, bool report_per_cluster) -{ - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - uint ncomponents = report_per_cluster && !etdev->usage_stats->use_metrics_v1 ? - EDGETPU_TPU_CLUSTER_COUNT : 1; - uint i; - ssize_t ret = 0; - - if (max_watermark_type >= EDGETPU_MAX_WATERMARK_TYPE_COUNT) - return 0; - edgetpu_kci_update_usage(etdev); - mutex_lock(&ustats->usage_stats_lock); - for (i = 0; i < ncomponents; i++) { - if (i) - ret += scnprintf(buf + ret, PAGE_SIZE - ret, " "); - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%llu", - ustats->max_watermark[max_watermark_type][i]); - } - mutex_unlock(&ustats->usage_stats_lock); - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); - return ret; -} - -static ssize_t tpu_usage_show(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t tpu_usage_show(struct device *dev, struct gcip_usage_stats_attr *attr, char *buf, + void *data) { struct edgetpu_dev *etdev = dev_get_drvdata(dev); - struct edgetpu_usage_stats *ustats = etdev->usage_stats; + struct gcip_usage_stats *ustats = &etdev->usage_stats->ustats; int i; int ret = 0; unsigned int bkt; - struct uid_entry *uid_entry; + struct gcip_usage_stats_core_usage_uid_entry *uid_entry; edgetpu_kci_update_usage(etdev); /* uid: state0speed state1speed ... */ ret += scnprintf(buf, PAGE_SIZE, "uid:"); - mutex_lock(&etdev->freq_lock); - if (!etdev->freq_table) { - mutex_unlock(&etdev->freq_lock); + mutex_lock(&ustats->dvfs_freqs_lock); + if (!ustats->dvfs_freqs_num) { + mutex_unlock(&ustats->dvfs_freqs_lock); for (i = 0; i < EDGETPU_NUM_STATES; i++) ret += scnprintf(buf + ret, PAGE_SIZE - ret, " %d", edgetpu_states_display[i]); } else { - for (i = 0; i < etdev->freq_count; i++) - ret += scnprintf(buf + ret, PAGE_SIZE - ret, " %d", - etdev->freq_table[i]); - mutex_unlock(&etdev->freq_lock); + for (i = 0; i < ustats->dvfs_freqs_num; i++) + ret += scnprintf(buf + ret, PAGE_SIZE - ret, " %d", ustats->dvfs_freqs[i]); + mutex_unlock(&ustats->dvfs_freqs_lock); } ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n"); mutex_lock(&ustats->usage_stats_lock); - hash_for_each(ustats->uid_hash_table, bkt, uid_entry, node) { - ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%d:", - uid_entry->uid); + hash_for_each (ustats->core_usage_htable[attr->subcomponent], bkt, uid_entry, node) { + ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%d:", uid_entry->uid); for (i = 0; i < EDGETPU_NUM_STATES; i++) ret += scnprintf(buf + ret, PAGE_SIZE - ret, " %lld", @@ -457,521 +67,197 @@ static ssize_t tpu_usage_show(struct device *dev, return ret; } -static void usage_stats_remove_uids(struct edgetpu_usage_stats *ustats) -{ - unsigned int bkt; - struct uid_entry *uid_entry; - struct hlist_node *tmp; - - mutex_lock(&ustats->usage_stats_lock); - - hash_for_each_safe(ustats->uid_hash_table, bkt, tmp, uid_entry, node) { - hash_del(&uid_entry->node); - kfree(uid_entry); - } - - mutex_unlock(&ustats->usage_stats_lock); -} - -/* Write to clear all entries in uid_hash_table */ -static ssize_t tpu_usage_clear(struct device *dev, - struct device_attribute *attr, - const char *buf, - size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - - usage_stats_remove_uids(ustats); - - return count; -} - -static DEVICE_ATTR(tpu_usage, 0664, tpu_usage_show, tpu_usage_clear); - -static ssize_t device_utilization_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - int32_t val; - - val = edgetpu_usage_get_utilization( - etdev, EDGETPU_USAGE_COMPONENT_DEVICE); - return scnprintf(buf, PAGE_SIZE, "%d\n", val); -} -static DEVICE_ATTR_RO(device_utilization); - -static ssize_t tpu_utilization_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - int32_t val; - - val = edgetpu_usage_get_utilization( - etdev, EDGETPU_USAGE_COMPONENT_TPU); - return scnprintf(buf, PAGE_SIZE, "%d\n", val); -} -static DEVICE_ATTR_RO(tpu_utilization); - -static ssize_t tpu_active_cycle_count_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); +/* Core usage. */ +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_CORE_USAGE, 0, 0, tpu_usage, + tpu_usage_show, NULL); - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_TPU_ACTIVE_CYCLES, false); -} +/* Component utilization. */ +static GCIP_USAGE_STATS_ATTR_RO(GCIP_USAGE_STATS_METRIC_TYPE_COMPONENT_UTILIZATION, + GCIP_USAGE_STATS_COMPONENT_UTILIZATION_IP, 0, device_utilization, + NULL); -static ssize_t tpu_active_cycle_count_store(struct device *dev, - struct device_attribute *attr, - const char *buf, - size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); +static GCIP_USAGE_STATS_ATTR_RO(GCIP_USAGE_STATS_METRIC_TYPE_COMPONENT_UTILIZATION, + GCIP_USAGE_STATS_COMPONENT_UTILIZATION_CORES, 0, tpu_utilization, + NULL); - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_TPU_ACTIVE_CYCLES); - return count; -} -static DEVICE_ATTR(tpu_active_cycle_count, 0664, tpu_active_cycle_count_show, - tpu_active_cycle_count_store); +/* Counter. */ +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_TPU_ACTIVIY_CYCLES, 0, + tpu_active_cycle_count, NULL, NULL); -static ssize_t tpu_throttle_stall_count_show(struct device *dev, - struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_TPU_THROTTLE_STALLS, 0, + tpu_throttle_stall_count, NULL, NULL); - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_TPU_THROTTLE_STALLS, false); -} +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_WORKLOAD, + GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS, inference_count, NULL, + NULL); -static ssize_t tpu_throttle_stall_count_store(struct device *dev, - struct device_attribute *attr, - const char *buf, - size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_TPU_THROTTLE_STALLS); - return count; -} -static DEVICE_ATTR(tpu_throttle_stall_count, 0664, - tpu_throttle_stall_count_show, - tpu_throttle_stall_count_store); +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_TPU_OP, + GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS, tpu_op_count, NULL, NULL); -static ssize_t inference_count_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_PARAM_CACHING_HIT, 0, + param_cache_hit_count, NULL, NULL); - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_INFERENCES, true); -} +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_PARAM_CACHING_MISS, 0, + param_cache_miss_count, NULL, NULL); -static ssize_t inference_count_store(struct device *dev, - struct device_attribute *attr, - const char *buf, - size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_CONTEXT_PREEMPTIONS, + GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS, context_preempt_count, + NULL, NULL); - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_INFERENCES); - return count; -} -static DEVICE_ATTR(inference_count, 0664, inference_count_show, - inference_count_store); +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_HW_PREEMPTIONS, + GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS, hardware_preempt_count, + NULL, NULL); -static ssize_t tpu_op_count_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_TOTAL_HW_CONTEXT_SAVE_TIME, + GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS, hardware_ctx_save_time, + NULL, NULL); - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_TPU_OPS, true); -} +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_TOTAL_SCALAR_FENCE_WAIT_TIME, + GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS, scalar_fence_wait_time, + NULL, NULL); -static ssize_t tpu_op_count_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_TPU_OPS); - return count; -} -static DEVICE_ATTR(tpu_op_count, 0664, tpu_op_count_show, tpu_op_count_store); - -static ssize_t param_cache_hit_count_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_PARAM_CACHE_HITS, false); -} - -static ssize_t param_cache_hit_count_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_PARAM_CACHE_HITS); - return count; -} -static DEVICE_ATTR(param_cache_hit_count, 0664, param_cache_hit_count_show, - param_cache_hit_count_store); - -static ssize_t param_cache_miss_count_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_PARAM_CACHE_MISSES, false); -} - -static ssize_t param_cache_miss_count_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_PARAM_CACHE_MISSES); - return count; -} -static DEVICE_ATTR(param_cache_miss_count, 0664, param_cache_miss_count_show, - param_cache_miss_count_store); - -static ssize_t context_preempt_count_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_CONTEXT_PREEMPTS, true); -} - -static ssize_t context_preempt_count_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_CONTEXT_PREEMPTS); - return count; -} -static DEVICE_ATTR(context_preempt_count, 0664, context_preempt_count_show, - context_preempt_count_store); - -static ssize_t hardware_preempt_count_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_HARDWARE_PREEMPTS, true); -} - -static ssize_t hardware_preempt_count_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_HARDWARE_PREEMPTS); - return count; -} -static DEVICE_ATTR(hardware_preempt_count, 0664, hardware_preempt_count_show, - hardware_preempt_count_store); - -static ssize_t hardware_ctx_save_time_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_HARDWARE_CTX_SAVE_TIME_US, - true); -} - -static ssize_t hardware_ctx_save_time_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_HARDWARE_CTX_SAVE_TIME_US); - return count; -} -static DEVICE_ATTR(hardware_ctx_save_time, 0664, hardware_ctx_save_time_show, - hardware_ctx_save_time_store); - -static ssize_t scalar_fence_wait_time_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_SCALAR_FENCE_WAIT_TIME_US, - true); -} - -static ssize_t scalar_fence_wait_time_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_SCALAR_FENCE_WAIT_TIME_US); - return count; -} -static DEVICE_ATTR(scalar_fence_wait_time, 0664, scalar_fence_wait_time_show, - scalar_fence_wait_time_store); - -static ssize_t long_suspend_count_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_LONG_SUSPEND, false); -} - -static ssize_t long_suspend_count_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_LONG_SUSPEND); - return count; -} -static DEVICE_ATTR(long_suspend_count, 0664, long_suspend_count_show, - long_suspend_count_store); +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_NUM_OF_LONG_SUSPENDS, 0, + long_suspend_count, NULL, NULL); #if EDGETPU_TPU_CLUSTER_COUNT > 1 -static ssize_t reconfigurations_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_RECONFIGURATIONS, false); -} - -static ssize_t reconfigurations_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_RECONFIGURATIONS); - return count; -} -static DEVICE_ATTR(reconfigurations, 0664, reconfigurations_show, reconfigurations_store); - -static ssize_t preempt_reconfigurations_show(struct device *dev, struct device_attribute *attr, - char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_counter(etdev, buf, EDGETPU_COUNTER_PREEMPT_RECONFIGURATIONS, - false); -} - -static ssize_t preempt_reconfigurations_store(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_counter_clear(etdev, EDGETPU_COUNTER_PREEMPT_RECONFIGURATIONS); - return count; -} -static DEVICE_ATTR(preempt_reconfigurations, 0664, preempt_reconfigurations_show, - preempt_reconfigurations_store); -#endif /* EDGETPU_TPU_CLUSTER_COUNT > 1 */ - - -static ssize_t outstanding_commands_max_show( - struct device *dev, struct device_attribute *attr, char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_max_watermark(etdev, buf, EDGETPU_MAX_WATERMARK_OUT_CMDS, - false); -} - -static ssize_t outstanding_commands_max_store( - struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_max_watermark_clear(etdev, EDGETPU_MAX_WATERMARK_OUT_CMDS); - return count; -} -static DEVICE_ATTR(outstanding_commands_max, 0664, - outstanding_commands_max_show, - outstanding_commands_max_store); - -static ssize_t preempt_depth_max_show( - struct device *dev, struct device_attribute *attr, char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_max_watermark(etdev, buf, EDGETPU_MAX_WATERMARK_PREEMPT_DEPTH, - true); -} - -static ssize_t preempt_depth_max_store( - struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_max_watermark_clear(etdev, EDGETPU_MAX_WATERMARK_PREEMPT_DEPTH); - return count; -} -static DEVICE_ATTR(preempt_depth_max, 0664, preempt_depth_max_show, - preempt_depth_max_store); - -static ssize_t hardware_ctx_save_time_max_show( - struct device *dev, struct device_attribute *attr, char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_max_watermark(etdev, buf, - EDGETPU_MAX_WATERMARK_HARDWARE_CTX_SAVE_TIME_US, - true); -} - -static ssize_t hardware_ctx_save_time_max_store( - struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_max_watermark_clear(etdev, EDGETPU_MAX_WATERMARK_HARDWARE_CTX_SAVE_TIME_US); - return count; -} -static DEVICE_ATTR(hardware_ctx_save_time_max, 0664, hardware_ctx_save_time_max_show, - hardware_ctx_save_time_max_store); +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_NUM_OF_RECONFIGURATIONS, 0, + reconfigurations, NULL, NULL); -static ssize_t scalar_fence_wait_time_max_show( - struct device *dev, struct device_attribute *attr, char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_COUNTER, + GCIP_USAGE_STATS_COUNTER_NUM_OF_RECONFIGURATIONS_BY_PREEMPTION, 0, + preempt_reconfigurations, NULL, NULL); +#endif - return edgetpu_usage_format_max_watermark( - etdev, buf, EDGETPU_MAX_WATERMARK_SCALAR_FENCE_WAIT_TIME_US, true); -} +/* Max watermark. */ +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_MAX_WATERMARK, + GCIP_USAGE_STATS_MAX_WATERMARK_OUTSTANDING_CMDS, 0, + outstanding_commands_max, NULL, NULL); + +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_MAX_WATERMARK, + GCIP_USAGE_STATS_MAX_WATERMARK_PREEMPTION_DEPTH, + GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS, preempt_depth_max, NULL, + NULL); + +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_MAX_WATERMARK, + GCIP_USAGE_STATS_MAX_WATERMARK_MAX_HW_CONTEXT_SAVE_TIME, + GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS, hardware_ctx_save_time_max, + NULL, NULL); + +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_MAX_WATERMARK, + GCIP_USAGE_STATS_MAX_WATERMARK_MAX_SCALAR_FENCE_WAIT_TIME, + GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS, scalar_fence_wait_time_max, + NULL, NULL); + +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_MAX_WATERMARK, + GCIP_USAGE_STATS_MAX_WATERMARK_MAX_SUSPEND_TIME, 0, + suspend_time_max, NULL, NULL); + +/* Thread statistics. */ +static GCIP_USAGE_STATS_ATTR_RW(GCIP_USAGE_STATS_METRIC_TYPE_THREAD_STATS, 0, 0, fw_thread_stats, + NULL, NULL); + +/* TODO(b/279844328): Add `scaling_available_frequencies` attribute which prints DVFS freqs. */ + +static struct gcip_usage_stats_attr *attrs[] = { + &gcip_usage_stats_attr_tpu_usage, + &gcip_usage_stats_attr_device_utilization, + &gcip_usage_stats_attr_tpu_utilization, + &gcip_usage_stats_attr_tpu_active_cycle_count, + &gcip_usage_stats_attr_tpu_throttle_stall_count, + &gcip_usage_stats_attr_inference_count, + &gcip_usage_stats_attr_tpu_op_count, + &gcip_usage_stats_attr_param_cache_hit_count, + &gcip_usage_stats_attr_param_cache_miss_count, + &gcip_usage_stats_attr_context_preempt_count, + &gcip_usage_stats_attr_hardware_preempt_count, + &gcip_usage_stats_attr_hardware_ctx_save_time, + &gcip_usage_stats_attr_scalar_fence_wait_time, + &gcip_usage_stats_attr_long_suspend_count, +#if EDGETPU_TPU_CLUSTER_COUNT > 1 + &gcip_usage_stats_attr_reconfigurations, + &gcip_usage_stats_attr_preempt_reconfigurations, +#endif + &gcip_usage_stats_attr_outstanding_commands_max, + &gcip_usage_stats_attr_preempt_depth_max, + &gcip_usage_stats_attr_hardware_ctx_save_time_max, + &gcip_usage_stats_attr_scalar_fence_wait_time_max, + &gcip_usage_stats_attr_suspend_time_max, + &gcip_usage_stats_attr_fw_thread_stats, +}; -static ssize_t scalar_fence_wait_time_max_store( - struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +static int update_usage_kci(void *data) { - struct edgetpu_dev *etdev = dev_get_drvdata(dev); + struct edgetpu_usage_stats *ustats = data; - edgetpu_max_watermark_clear(etdev, EDGETPU_MAX_WATERMARK_SCALAR_FENCE_WAIT_TIME_US); - return count; + return edgetpu_kci_update_usage(ustats->etdev); } -static DEVICE_ATTR(scalar_fence_wait_time_max, 0664, scalar_fence_wait_time_max_show, - scalar_fence_wait_time_max_store); -static ssize_t suspend_time_max_show( - struct device *dev, struct device_attribute *attr, char *buf) +static int get_default_dvfs_freqs_num(void *data) { - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - return edgetpu_usage_format_max_watermark(etdev, buf, EDGETPU_MAX_WATERMARK_SUSPEND_TIME_US, - false); + return EDGETPU_NUM_STATES; } -static ssize_t suspend_time_max_store( - struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +static int get_default_dvfs_freq(int idx, void *data) { - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - - edgetpu_max_watermark_clear(etdev, EDGETPU_MAX_WATERMARK_SUSPEND_TIME_US); - return count; + if (idx >= EDGETPU_NUM_STATES) + return 0; + return edgetpu_states_display[idx]; } -static DEVICE_ATTR(suspend_time_max, 0664, suspend_time_max_show, - suspend_time_max_store); - -static ssize_t fw_thread_stats_show( - struct device *dev, struct device_attribute *attr, char *buf) -{ - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - int i; - ssize_t ret = 0; - - edgetpu_kci_update_usage(etdev); - mutex_lock(&ustats->usage_stats_lock); - - for (i = 0; i < EDGETPU_FW_THREAD_COUNT; i++) { - if (!ustats->thread_stack_max[i]) - continue; - ret += scnprintf(buf + ret, PAGE_SIZE - ret, - "%u\t%u\n", i, ustats->thread_stack_max[i]); - /* Not checking ret < PAGE_SIZE is intended. */ - } - mutex_unlock(&ustats->usage_stats_lock); - return ret; -} +static const struct gcip_usage_stats_ops ustats_ops = { + .update_usage_kci = update_usage_kci, + .get_default_dvfs_freqs_num = get_default_dvfs_freqs_num, + .get_default_dvfs_freq = get_default_dvfs_freq, +}; -static ssize_t fw_thread_stats_store( - struct device *dev, struct device_attribute *attr, - const char *buf, size_t count) +void edgetpu_usage_stats_process_buffer(struct edgetpu_dev *etdev, void *buf) { - struct edgetpu_dev *etdev = dev_get_drvdata(dev); - struct edgetpu_usage_stats *ustats = etdev->usage_stats; - int i; - - mutex_lock(&ustats->usage_stats_lock); - for (i = 0; i < EDGETPU_FW_THREAD_COUNT; i++) - ustats->thread_stack_max[i] = 0; - mutex_unlock(&ustats->usage_stats_lock); - return count; + if (!etdev->usage_stats) + return; + gcip_usage_stats_process_buffer(&etdev->usage_stats->ustats, buf); } -static DEVICE_ATTR(fw_thread_stats, 0664, fw_thread_stats_show, - fw_thread_stats_store); - -static struct attribute *usage_stats_dev_attrs[] = { - &dev_attr_tpu_usage.attr, - &dev_attr_device_utilization.attr, - &dev_attr_tpu_utilization.attr, - &dev_attr_tpu_active_cycle_count.attr, - &dev_attr_tpu_throttle_stall_count.attr, - &dev_attr_inference_count.attr, - &dev_attr_tpu_op_count.attr, - &dev_attr_param_cache_hit_count.attr, - &dev_attr_param_cache_miss_count.attr, - &dev_attr_context_preempt_count.attr, - &dev_attr_hardware_preempt_count.attr, - &dev_attr_hardware_ctx_save_time.attr, - &dev_attr_scalar_fence_wait_time.attr, - &dev_attr_long_suspend_count.attr, -#if EDGETPU_TPU_CLUSTER_COUNT > 1 - &dev_attr_reconfigurations.attr, - &dev_attr_preempt_reconfigurations.attr, -#endif - &dev_attr_outstanding_commands_max.attr, - &dev_attr_preempt_depth_max.attr, - &dev_attr_hardware_ctx_save_time_max.attr, - &dev_attr_scalar_fence_wait_time_max.attr, - &dev_attr_suspend_time_max.attr, - &dev_attr_fw_thread_stats.attr, - NULL, -}; - -static const struct attribute_group usage_stats_attr_group = { - .attrs = usage_stats_dev_attrs, -}; void edgetpu_usage_stats_init(struct edgetpu_dev *etdev) { struct edgetpu_usage_stats *ustats; + struct gcip_usage_stats_args args; int ret; - ustats = devm_kzalloc(etdev->dev, sizeof(*etdev->usage_stats), - GFP_KERNEL); + ustats = devm_kzalloc(etdev->dev, sizeof(*etdev->usage_stats), GFP_KERNEL); if (!ustats) { - etdev_warn(etdev, - "failed to allocate memory for usage stats\n"); + etdev_warn(etdev, "failed to allocate memory for usage stats\n"); return; } - hash_init(ustats->uid_hash_table); - mutex_init(&ustats->usage_stats_lock); - etdev->usage_stats = ustats; + args.version = EDGETPU_USAGE_METRIC_VERSION; + args.dev = etdev->dev; + args.ops = &ustats_ops; + args.attrs = attrs; + args.num_attrs = ARRAY_SIZE(attrs); + args.subcomponents = EDGETPU_TPU_CLUSTER_COUNT; + args.data = ustats; + ustats->etdev = etdev; + + ret = gcip_usage_stats_init(&ustats->ustats, &args); + if (ret) { + etdev_warn(etdev, "failed to create the usage_stats attrs: %d", ret); + devm_kfree(etdev->dev, ustats); + return; + } - ret = device_add_group(etdev->dev, &usage_stats_attr_group); - if (ret) - etdev_warn(etdev, "failed to create the usage_stats attrs\n"); + etdev->usage_stats = ustats; etdev_dbg(etdev, "%s init\n", __func__); } @@ -981,16 +267,10 @@ void edgetpu_usage_stats_exit(struct edgetpu_dev *etdev) struct edgetpu_usage_stats *ustats = etdev->usage_stats; if (ustats) { - usage_stats_remove_uids(ustats); - device_remove_group(etdev->dev, &usage_stats_attr_group); - /* free the frequency table if allocated */ - mutex_lock(&etdev->freq_lock); - if (etdev->freq_table) - kvfree(etdev->freq_table); - etdev->freq_table = NULL; - etdev->freq_count = 0; - mutex_unlock(&etdev->freq_lock); + gcip_usage_stats_exit(&ustats->ustats); + devm_kfree(etdev->dev, ustats); } + etdev->usage_stats = NULL; etdev_dbg(etdev, "%s exit\n", __func__); } diff --git a/drivers/edgetpu/edgetpu-usage-stats.h b/drivers/edgetpu/edgetpu-usage-stats.h index 2d97043..afc9ff0 100644 --- a/drivers/edgetpu/edgetpu-usage-stats.h +++ b/drivers/edgetpu/edgetpu-usage-stats.h @@ -2,250 +2,24 @@ /* * EdgeTPU usage stats header * - * Copyright (C) 2020 Google, Inc. + * Copyright (C) 2020-2023 Google, Inc. */ + #ifndef __EDGETPU_USAGE_STATS_H__ #define __EDGETPU_USAGE_STATS_H__ -#include -#include - -/* The highest version of usage metrics handled by this driver. */ -#define EDGETPU_USAGE_METRIC_VERSION 2 - -/* Max # of TPU clusters accounted for in the highest supported metrics version. */ -#define EDGETPU_USAGE_CLUSTERS_MAX 3 - -/* - * Size in bytes of usage metric v1. - * If fewer bytes than this are received then discard the invalid buffer. - * This size also identifies the fw response as v1; subsequent versions will add another field - * with the version number. - */ -#define EDGETPU_USAGE_METRIC_SIZE_V1 20 - -/* v1 metric header struct. */ -struct edgetpu_usage_header_v1 { - uint32_t num_metrics; /* Number of metrics being reported */ - uint32_t metric_size; /* Size of each metric struct */ -}; - -/* Header struct in the metric buffer. */ -/* Must be kept in sync with firmware struct UsageTrackerHeader */ -struct edgetpu_usage_header { - uint16_t header_bytes; /* Number of bytes in this header */ - uint16_t version; /* Metrics version */ - uint32_t num_metrics; /* Number of metrics being reported */ - uint32_t metric_size; /* Size of each metric struct */ -}; - -/* - * Encapsulate TPU core usage information of a specific application for a - * specific power state. - * Must be kept in sync with firmware struct CoreUsage. - */ -struct tpu_usage { - /* Unique identifier of the application. */ - int32_t uid; - /* The power state of the device (values are chip dependent) */ - /* Now called operating_point in FW. */ - uint32_t power_state; - /* Duration of usage in microseconds. */ - uint32_t duration_us; - - /* Following fields are added in metrics v2 */ - - /* Compute Core: TPU cluster ID. */ - /* Called core_id in FW. */ - /* Note: as of metrics v2 the cluster_id is always zero and is ignored. */ - uint8_t cluster_id; - /* Reserved. Filling out the next 32-bit boundary. */ - uint8_t reserved[3]; -}; - -/* - * An enum to represent the different components we can track metrics for. - * Must be kept in sync with firmware struct Component. - */ -enum edgetpu_usage_component { - /* The device as a whole */ - EDGETPU_USAGE_COMPONENT_DEVICE = 0, - /* Just the TPU core (scalar core and tiles) */ - EDGETPU_USAGE_COMPONENT_TPU = 1, - /* Control core (ARM Cortex-R52 CPU) */ - /* Note: this component is not reported as of metrics v2. */ - EDGETPU_USAGE_COMPONENT_CONTROLCORE = 2, - - EDGETPU_USAGE_COMPONENT_COUNT = 3, /* number of components above */ -}; - -/* - * Encapsulates information about activity of a component. - * Must be kept in sync with firmware struct ComponentActivity. - */ -struct edgetpu_component_activity { - enum edgetpu_usage_component component; - /* Utilization as a percentage since the last read. */ - int32_t utilization; -}; - -/* - * Defines different counter types we track. - * Must be kept in sync with firmware enum class CounterType. - */ -enum edgetpu_usage_counter_type { - /* TPU active cycles. */ - EDGETPU_COUNTER_TPU_ACTIVE_CYCLES = 0, - /* Number of stalls caused by throttling. */ - EDGETPU_COUNTER_TPU_THROTTLE_STALLS = 1, - /* Number of graph invocations. (Now called kWorkload in FW.) */ - EDGETPU_COUNTER_INFERENCES = 2, - /* Number of TPU offload op invocations. */ - EDGETPU_COUNTER_TPU_OPS = 3, - /* Number of times a TPU op invocation used its cached parameters. */ - EDGETPU_COUNTER_PARAM_CACHE_HITS = 4, - /* Number of times a TPU op invocation had to cache its parameters. */ - EDGETPU_COUNTER_PARAM_CACHE_MISSES = 5, - /* Number of times a context got preempted by another. */ - EDGETPU_COUNTER_CONTEXT_PREEMPTS = 6, - /* Number of times a hardware preemption occurred. */ - EDGETPU_COUNTER_HARDWARE_PREEMPTS = 7, - /* Total time(us) spent in saving hw ctx during hw preemption */ - EDGETPU_COUNTER_HARDWARE_CTX_SAVE_TIME_US = 8, - /* Total time(us) spent in waiting to hit scalar fence during hw preemption */ - EDGETPU_COUNTER_SCALAR_FENCE_WAIT_TIME_US = 9, - /* Number of times (firmware)suspend function takes longer than SLA time. */ - EDGETPU_COUNTER_LONG_SUSPEND = 10, - - /* The following counters are added in metrics v2. */ - - /* Counter 11 not used on TPU. */ - EDGETPU_COUNTER_CONTEXT_SWITCHES = 11, +#include - /* Number of TPU Cluster Reconfigurations. */ - EDGETPU_COUNTER_RECONFIGURATIONS = 12, - - /* Number of TPU Cluster Reconfigurations motivated exclusively by a preemption. */ - EDGETPU_COUNTER_PREEMPT_RECONFIGURATIONS = 13, - - EDGETPU_COUNTER_COUNT = 14, /* number of counters above */ -}; - -/* Generic counter. Only reported if it has a value larger than 0. */ -struct __packed edgetpu_usage_counter { - /* What it counts. */ - enum edgetpu_usage_counter_type type; - - /* Accumulated value since last initialization. */ - uint64_t value; - - /* Following fields are added in metrics v2 */ - - /* Reporting component. */ - uint8_t component_id; -}; - -/* Defines different max watermarks we track. */ -/* Must be kept in sync with firmware MaxWatermarkType */ -enum edgetpu_usage_max_watermark_type { - /* Number of outstanding commands in VII trackers of all contexts. */ - EDGETPU_MAX_WATERMARK_OUT_CMDS = 0, - /* Number of preempted contexts at any given time. */ - EDGETPU_MAX_WATERMARK_PREEMPT_DEPTH = 1, - /* Max time(us) spent in saving hw ctx during hw preemption */ - EDGETPU_MAX_WATERMARK_HARDWARE_CTX_SAVE_TIME_US = 2, - /* Max time(us) spent in waiting to hit scalar fence during hw preemption */ - EDGETPU_MAX_WATERMARK_SCALAR_FENCE_WAIT_TIME_US = 3, - /* Max time(us) spent during (firmware)suspend function. */ - EDGETPU_MAX_WATERMARK_SUSPEND_TIME_US = 4, - - /* Number of watermark types above */ - EDGETPU_MAX_WATERMARK_TYPE_COUNT = 5, -}; - -/* Max watermark. Only reported if it has a value larger than 0. */ -struct __packed edgetpu_usage_max_watermark { - /* What it counts. */ - enum edgetpu_usage_max_watermark_type type; - - /* - * Maximum value since last initialization (virtual device join in - * non-mobile, firmware boot on mobile). - */ - uint64_t value; - - /* Following fields are added in metrics v2 */ - - /* Reporting component. */ - uint8_t component_id; -}; - -/* An enum to identify the tracked firmware threads. */ -/* Must be kept in sync with firmware enum class UsageTrackerThreadId. */ -enum edgetpu_usage_threadid { - /* Individual thread IDs do not have identifiers assigned. */ - - /* Thread ID 14 is not used for TPU */ - - /* Number of task identifiers. */ - EDGETPU_FW_THREAD_COUNT = 17, -}; - -/* Statistics related to a single thread in firmware. */ -/* Must be kept in sync with firmware struct ThreadStats. */ -struct edgetpu_thread_stats { - /* The thread in question. */ - enum edgetpu_usage_threadid thread_id; - - /* Maximum stack usage (in bytes) since last firmware boot. */ - uint32_t max_stack_usage_bytes; -}; - -/* Must be kept in sync with firmware enum class UsageTrackerMetric::Type */ -enum edgetpu_usage_metric_type { - EDGETPU_METRIC_TYPE_RESERVED = 0, - EDGETPU_METRIC_TYPE_TPU_USAGE = 1, - EDGETPU_METRIC_TYPE_COMPONENT_ACTIVITY = 2, - EDGETPU_METRIC_TYPE_COUNTER = 3, - EDGETPU_METRIC_TYPE_THREAD_STATS = 4, - EDGETPU_METRIC_TYPE_MAX_WATERMARK = 5, - EDGETPU_METRIC_TYPE_DVFS_FREQUENCY_INFO = 6, -}; - -/* - * Encapsulates a single metric reported to the kernel. - * Must be kept in sync with firmware struct UsageTrackerMetric. - */ -struct edgetpu_usage_metric { - uint32_t type; - uint8_t reserved[4]; - union { - struct tpu_usage tpu_usage; - struct edgetpu_component_activity component_activity; - struct edgetpu_usage_counter counter; - struct edgetpu_thread_stats thread_stats; - struct edgetpu_usage_max_watermark max_watermark; - uint32_t dvfs_frequency_info; - }; -}; +#include "edgetpu-internal.h" -#define UID_HASH_BITS 3 +/* The version of metric TPU KD is using currently. */ +#define EDGETPU_USAGE_METRIC_VERSION GCIP_USAGE_STATS_V2 struct edgetpu_usage_stats { - /* if true the current firmware only implements metrics V1 */ - bool use_metrics_v1; - DECLARE_HASHTABLE(uid_hash_table, UID_HASH_BITS); - /* component utilization values reported by firmware */ - int32_t component_utilization[EDGETPU_USAGE_COMPONENT_COUNT]; - int64_t counter[EDGETPU_COUNTER_COUNT][EDGETPU_USAGE_CLUSTERS_MAX]; - int64_t max_watermark[EDGETPU_MAX_WATERMARK_TYPE_COUNT][EDGETPU_USAGE_CLUSTERS_MAX]; - int32_t thread_stack_max[EDGETPU_FW_THREAD_COUNT]; - struct mutex usage_stats_lock; + struct edgetpu_dev *etdev; + struct gcip_usage_stats ustats; }; -int edgetpu_usage_add(struct edgetpu_dev *etdev, struct tpu_usage *tpu_usage); -int edgetpu_usage_get_utilization(struct edgetpu_dev *etdev, - enum edgetpu_usage_component component); void edgetpu_usage_stats_process_buffer(struct edgetpu_dev *etdev, void *buf); void edgetpu_usage_stats_init(struct edgetpu_dev *etdev); void edgetpu_usage_stats_exit(struct edgetpu_dev *etdev); diff --git a/drivers/edgetpu/edgetpu.h b/drivers/edgetpu/edgetpu.h index f5e3dec..d8824ba 100644 --- a/drivers/edgetpu/edgetpu.h +++ b/drivers/edgetpu/edgetpu.h @@ -182,6 +182,7 @@ struct edgetpu_mailbox_attr { __u32 cmdq_tail_doorbell: 1; /* auto doorbell on cmd queue tail move */ /* Type of memory partitions to be used for this group, exact meaning is chip-dependent. */ __u32 partition_type : 1; + __u32 client_priv : 1; /* client privilege level */ }; /* diff --git a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c index ab0ef51..75509cd 100644 --- a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c +++ b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-iommu.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -22,13 +21,9 @@ #include #include -/* - * TODO(b/277649169) Best fit IOVA allocator was removed in 6.1 GKI - * The API needs to either be upstreamed, integrated into this driver, or disabled for 6.1 - * compatibility. For now, disable best-fit on all non-Android kernels and any GKI > 5.15. - */ -#define HAS_IOVAD_BEST_FIT_ALGO (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) && \ - (IS_ENABLED(CONFIG_GCIP_TEST) || IS_ENABLED(CONFIG_ANDROID))) +#if HAS_IOVAD_BEST_FIT_ALGO +#include +#endif /* Macros for manipulating @gcip_map_flags parameter. */ #define GCIP_MAP_FLAGS_GET_VALUE(ATTR, flags) \ diff --git a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-kci.c b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-kci.c index 33f8021..417b078 100644 --- a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-kci.c +++ b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-kci.c @@ -136,20 +136,6 @@ static void gcip_kci_set_resp_elem_seq(struct gcip_mailbox *mailbox, void *resp, elem->seq = seq; } -static u16 gcip_kci_get_resp_elem_status(struct gcip_mailbox *mailbox, void *resp) -{ - struct gcip_kci_response_element *elem = resp; - - return elem->status; -} - -static void gcip_kci_set_resp_elem_status(struct gcip_mailbox *mailbox, void *resp, u16 status) -{ - struct gcip_kci_response_element *elem = resp; - - elem->status = status; -} - static void gcip_kci_acquire_wait_list_lock(struct gcip_mailbox *mailbox, bool irqsave, unsigned long *flags) { @@ -273,8 +259,6 @@ static const struct gcip_mailbox_ops gcip_mailbox_ops = { .release_resp_queue_lock = gcip_kci_release_resp_queue_lock, .get_resp_elem_seq = gcip_kci_get_resp_elem_seq, .set_resp_elem_seq = gcip_kci_set_resp_elem_seq, - .get_resp_elem_status = gcip_kci_get_resp_elem_status, - .set_resp_elem_status = gcip_kci_set_resp_elem_status, .acquire_wait_list_lock = gcip_kci_acquire_wait_list_lock, .release_wait_list_lock = gcip_kci_release_wait_list_lock, .wait_for_cmd_queue_not_full = gcip_kci_wait_for_cmd_queue_not_full, diff --git a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c index c7aa921..4571aa9 100644 --- a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c +++ b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c @@ -42,8 +42,6 @@ #define GET_RESP_ELEM_SEQ(resp) mailbox->ops->get_resp_elem_seq(mailbox, resp) #define SET_RESP_ELEM_SEQ(resp, seq) mailbox->ops->set_resp_elem_seq(mailbox, resp, seq) -#define GET_RESP_ELEM_STATUS(resp) mailbox->ops->get_resp_elem_status(mailbox, resp) -#define SET_RESP_ELEM_STATUS(resp, status) mailbox->ops->set_resp_elem_status(mailbox, resp, status) #define ACQUIRE_WAIT_LIST_LOCK(irqsave, flags) \ mailbox->ops->acquire_wait_list_lock(mailbox, irqsave, flags) @@ -52,7 +50,7 @@ struct gcip_mailbox_wait_list_elem { struct list_head list; - void *resp; + struct gcip_mailbox_async_resp *async_resp; struct gcip_mailbox_resp_awaiter *awaiter; }; @@ -74,16 +72,17 @@ static void gcip_mailbox_awaiter_dec_refs(struct gcip_mailbox_resp_awaiter *awai * * This is used when the kernel gives up waiting for the response. */ -static void gcip_mailbox_del_wait_resp(struct gcip_mailbox *mailbox, void *resp) +static void gcip_mailbox_del_wait_resp(struct gcip_mailbox *mailbox, + struct gcip_mailbox_async_resp *async_resp) { struct gcip_mailbox_wait_list_elem *cur; unsigned long flags; - u64 cur_seq, seq = GET_RESP_ELEM_SEQ(resp); + u64 cur_seq, seq = GET_RESP_ELEM_SEQ(async_resp->resp); ACQUIRE_WAIT_LIST_LOCK(true, &flags); list_for_each_entry (cur, &mailbox->wait_list, list) { - cur_seq = GET_RESP_ELEM_SEQ(cur->resp); + cur_seq = GET_RESP_ELEM_SEQ(cur->async_resp->resp); if (cur_seq > seq) break; if (cur_seq == seq) { @@ -108,7 +107,8 @@ static void gcip_mailbox_del_wait_resp(struct gcip_mailbox *mailbox, void *resp) * * Returns 0 on success, or -ENOMEM if failed on allocation. */ -static int gcip_mailbox_push_wait_resp(struct gcip_mailbox *mailbox, void *resp, +static int gcip_mailbox_push_wait_resp(struct gcip_mailbox *mailbox, + struct gcip_mailbox_async_resp *async_resp, struct gcip_mailbox_resp_awaiter *awaiter, bool atomic) { struct gcip_mailbox_wait_list_elem *entry; @@ -120,7 +120,7 @@ static int gcip_mailbox_push_wait_resp(struct gcip_mailbox *mailbox, void *resp, return -ENOMEM; if (mailbox->ops->before_enqueue_wait_list) { - ret = mailbox->ops->before_enqueue_wait_list(mailbox, resp, awaiter); + ret = mailbox->ops->before_enqueue_wait_list(mailbox, async_resp->resp, awaiter); if (ret) { kfree(entry); return ret; @@ -131,7 +131,7 @@ static int gcip_mailbox_push_wait_resp(struct gcip_mailbox *mailbox, void *resp, if (awaiter) refcount_inc(&awaiter->refs); - entry->resp = resp; + entry->async_resp = async_resp; entry->awaiter = awaiter; ACQUIRE_WAIT_LIST_LOCK(true, &flags); list_add_tail(&entry->list, &mailbox->wait_list); @@ -146,7 +146,8 @@ static int gcip_mailbox_push_wait_resp(struct gcip_mailbox *mailbox, void *resp, * synchronous, the @cmd will be put into the queue, but the caller may not wait the response and * ignore it. If the request is async, @awaiter should be passed too. */ -static int gcip_mailbox_enqueue_cmd(struct gcip_mailbox *mailbox, void *cmd, void *resp, +static int gcip_mailbox_enqueue_cmd(struct gcip_mailbox *mailbox, void *cmd, + struct gcip_mailbox_async_resp *async_resp, struct gcip_mailbox_resp_awaiter *awaiter) { int ret = 0; @@ -177,11 +178,11 @@ static int gcip_mailbox_enqueue_cmd(struct gcip_mailbox *mailbox, void *cmd, voi goto out; } - if (resp) { + if (async_resp->resp) { /* Adds @resp to the wait_list only if the cmd can be pushed successfully. */ - SET_RESP_ELEM_SEQ(resp, GET_CMD_ELEM_SEQ(cmd)); - SET_RESP_ELEM_STATUS(resp, GCIP_MAILBOX_STATUS_WAITING_RESPONSE); - ret = gcip_mailbox_push_wait_resp(mailbox, resp, awaiter, atomic); + SET_RESP_ELEM_SEQ(async_resp->resp, GET_CMD_ELEM_SEQ(cmd)); + async_resp->status = GCIP_MAILBOX_STATUS_WAITING_RESPONSE; + ret = gcip_mailbox_push_wait_resp(mailbox, async_resp, awaiter, atomic); if (ret) goto out; } @@ -245,11 +246,10 @@ static void gcip_mailbox_handle_response(struct gcip_mailbox *mailbox, void *res if (mailbox->ops->before_handle_resp && !mailbox->ops->before_handle_resp(mailbox, resp)) return; - SET_RESP_ELEM_STATUS(resp, GCIP_MAILBOX_STATUS_OK); ACQUIRE_WAIT_LIST_LOCK(true, &flags); list_for_each_entry_safe (cur, nxt, &mailbox->wait_list, list) { - cur_seq = GET_RESP_ELEM_SEQ(cur->resp); + cur_seq = GET_RESP_ELEM_SEQ(cur->async_resp->resp); if (cur_seq > seq) { /* * This response has already timed out and been removed @@ -259,7 +259,8 @@ static void gcip_mailbox_handle_response(struct gcip_mailbox *mailbox, void *res break; } if (cur_seq == seq) { - memcpy(cur->resp, resp, mailbox->resp_elem_size); + cur->async_resp->status = GCIP_MAILBOX_STATUS_OK; + memcpy(cur->async_resp->resp, resp, mailbox->resp_elem_size); list_del(&cur->list); if (cur->awaiter) { awaiter = cur->awaiter; @@ -293,7 +294,7 @@ static void gcip_mailbox_handle_response(struct gcip_mailbox *mailbox, void *res break; } if (!mailbox->ignore_seq_order && cur_seq < seq) { - SET_RESP_ELEM_STATUS(cur->resp, GCIP_MAILBOX_STATUS_NO_RESPONSE); + cur->async_resp->status = GCIP_MAILBOX_STATUS_NO_RESPONSE; list_del(&cur->list); if (cur->awaiter) { /* Remove the reference of the arrived handler. */ @@ -435,7 +436,7 @@ static void gcip_mailbox_async_cmd_timeout_work(struct work_struct *work) * Once this function has the wait_list_lock, no future response * processing will begin until this response has been removed. */ - gcip_mailbox_del_wait_resp(mailbox, awaiter->resp); + gcip_mailbox_del_wait_resp(mailbox, &awaiter->async_resp); /* * Handle timed out awaiter. If `handle_awaiter_timedout` is defined, @awaiter @@ -521,8 +522,7 @@ static int gcip_mailbox_set_ops(struct gcip_mailbox *mailbox, const struct gcip_ if (!ops->get_resp_queue_size || !ops->get_resp_queue_head || !ops->get_resp_queue_tail || !ops->inc_resp_queue_head || !ops->acquire_resp_queue_lock || - !ops->release_resp_queue_lock || !ops->get_resp_elem_seq || !ops->set_resp_elem_seq || - !ops->get_resp_elem_status || !ops->set_resp_elem_status) { + !ops->release_resp_queue_lock || !ops->get_resp_elem_seq || !ops->set_resp_elem_seq) { dev_err(mailbox->dev, "Incomplete mailbox RESP queue ops.\n"); return -EINVAL; } @@ -605,26 +605,33 @@ void gcip_mailbox_consume_responses_work(struct gcip_mailbox *mailbox) int gcip_mailbox_send_cmd(struct gcip_mailbox *mailbox, void *cmd, void *resp) { + struct gcip_mailbox_async_resp async_resp = { + .resp = resp, + }; int ret; - ret = gcip_mailbox_enqueue_cmd(mailbox, cmd, resp, NULL); + ret = gcip_mailbox_enqueue_cmd(mailbox, cmd, &async_resp, NULL); if (ret) return ret; + /* + * If @resp is NULL, it will not enqueue the response into the waiting list. Therefore, it + * is fine to release @async_resp. + */ if (!resp) return 0; ret = wait_event_timeout(mailbox->wait_list_waitq, - GET_RESP_ELEM_STATUS(resp) != GCIP_MAILBOX_STATUS_WAITING_RESPONSE, + async_resp.status != GCIP_MAILBOX_STATUS_WAITING_RESPONSE, msecs_to_jiffies(mailbox->timeout)); if (!ret) { dev_dbg(mailbox->dev, "event wait timeout"); - gcip_mailbox_del_wait_resp(mailbox, resp); + gcip_mailbox_del_wait_resp(mailbox, &async_resp); return -ETIMEDOUT; } - if (GET_RESP_ELEM_STATUS(resp) != GCIP_MAILBOX_STATUS_OK) { + if (async_resp.status != GCIP_MAILBOX_STATUS_OK) { dev_err(mailbox->dev, "Mailbox cmd %u response status %u", GET_CMD_ELEM_CODE(cmd), - GET_RESP_ELEM_STATUS(resp)); + async_resp.status); return -ENOMSG; } @@ -641,7 +648,7 @@ struct gcip_mailbox_resp_awaiter *gcip_mailbox_put_cmd(struct gcip_mailbox *mail if (!awaiter) return ERR_PTR(-ENOMEM); - awaiter->resp = resp; + awaiter->async_resp.resp = resp; awaiter->mailbox = mailbox; awaiter->data = data; awaiter->release_data = mailbox->ops->release_awaiter_data; @@ -651,7 +658,7 @@ struct gcip_mailbox_resp_awaiter *gcip_mailbox_put_cmd(struct gcip_mailbox *mail INIT_DELAYED_WORK(&awaiter->timeout_work, gcip_mailbox_async_cmd_timeout_work); schedule_delayed_work(&awaiter->timeout_work, msecs_to_jiffies(mailbox->timeout)); - ret = gcip_mailbox_enqueue_cmd(mailbox, cmd, awaiter->resp, awaiter); + ret = gcip_mailbox_enqueue_cmd(mailbox, cmd, &awaiter->async_resp, awaiter); if (ret) goto err_free_resp; @@ -665,7 +672,7 @@ err_free_resp: void gcip_mailbox_cancel_awaiter(struct gcip_mailbox_resp_awaiter *awaiter) { - gcip_mailbox_del_wait_resp(awaiter->mailbox, awaiter->resp); + gcip_mailbox_del_wait_resp(awaiter->mailbox, &awaiter->async_resp); gcip_mailbox_cancel_awaiter_timeout(awaiter); } diff --git a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c index 88efbd9..5c50705 100644 --- a/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c +++ b/drivers/edgetpu/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c @@ -84,7 +84,8 @@ gcip_usage_stats_find_core_usage_entry_locked(int32_t uid, uint8_t core_id, static unsigned int gcip_usage_stats_find_dvfs_freq_index(struct gcip_usage_stats *ustats, uint32_t dvfs_freq) { - int i, nums, idx = 0; + int i, nums, closest_freq_idx, idx = 0; + uint32_t cur_freq, closest_freq = 0; mutex_lock(&ustats->dvfs_freqs_lock); @@ -94,8 +95,10 @@ static unsigned int gcip_usage_stats_find_dvfs_freq_index(struct gcip_usage_stat */ if (ustats->dvfs_freqs_num) { for (i = ustats->dvfs_freqs_num - 1; i >= 0; i--) { - if (dvfs_freq == ustats->dvfs_freqs[i]) + if (dvfs_freq == ustats->dvfs_freqs[i]) { idx = i; + break; + } } if (i < 0) @@ -117,10 +120,20 @@ static unsigned int gcip_usage_stats_find_dvfs_freq_index(struct gcip_usage_stat } for (i = nums - 1; i >= 0; i--) { - if (dvfs_freq >= ustats->ops->get_default_dvfs_freq(i, ustats->data)) + cur_freq = ustats->ops->get_default_dvfs_freq(i, ustats->data); + + if (dvfs_freq == cur_freq) return i; + + if (dvfs_freq > cur_freq && closest_freq < cur_freq) { + closest_freq = cur_freq; + closest_freq_idx = i; + } } + if (closest_freq) + return closest_freq_idx; + dev_warn(ustats->dev, "Failed to find the freq from the default ones of the kernel driver, freq=%u", dvfs_freq); @@ -328,7 +341,6 @@ static ssize_t gcip_usage_stats_component_utilization_show(struct device *dev, container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); struct gcip_usage_stats *ustats = attr->ustats; int32_t val; - ssize_t written; ustats->ops->update_usage_kci(ustats->data); @@ -339,11 +351,7 @@ static ssize_t gcip_usage_stats_component_utilization_show(struct device *dev, mutex_unlock(&ustats->usage_stats_lock); - written = scnprintf(buf, PAGE_SIZE, "%d\n", val); - if (written < 0) - return written; - - return written; + return scnprintf(buf, PAGE_SIZE, "%d\n", val); } /* Following functions are related to `COUNTER` metrics. */ @@ -396,11 +404,17 @@ static ssize_t gcip_usage_stats_counter_show(struct device *dev, struct device_a container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); struct gcip_usage_stats *ustats = attr->ustats; ssize_t written = 0; - int subcomponent = ustats->version >= GCIP_USAGE_STATS_V2 ? attr->subcomponent : 0; - int i; + int subcomponent, i; ustats->ops->update_usage_kci(ustats->data); + /* + * We need to decide @subcomponent after calling `update_usage_kci` because IP kernel + * drivers may want to change the version of @ustats to lower one if the firmware doesn't + * support a higher version. + */ + subcomponent = ustats->version >= GCIP_USAGE_STATS_V2 ? attr->subcomponent : 0; + mutex_lock(&ustats->usage_stats_lock); if (subcomponent == GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS) { @@ -579,11 +593,17 @@ static ssize_t gcip_usage_stats_max_watermark_show(struct device *dev, container_of(dev_attr, struct gcip_usage_stats_attr, dev_attr); struct gcip_usage_stats *ustats = attr->ustats; ssize_t written = 0; - int subcomponent = ustats->version >= GCIP_USAGE_STATS_V2 ? attr->subcomponent : 0; - int i; + int subcomponent, i; ustats->ops->update_usage_kci(ustats->data); + /* + * We need to decide @subcomponent after calling `update_usage_kci` because IP kernel + * drivers may want to change the version of @ustats to lower one if the firmware doesn't + * support a higher version. + */ + subcomponent = ustats->version >= GCIP_USAGE_STATS_V2 ? attr->subcomponent : 0; + mutex_lock(&ustats->usage_stats_lock); if (subcomponent == GCIP_USAGE_STATS_ATTR_ALL_SUBCOMPONENTS) { @@ -711,6 +731,8 @@ static ssize_t gcip_usage_stats_dvfs_freqs_show(struct device *dev, mutex_unlock(&ustats->dvfs_freqs_lock); } + written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); + return written; } diff --git a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-iommu.h b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-iommu.h index 4e04b7e..1797f94 100644 --- a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-iommu.h +++ b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-iommu.h @@ -23,10 +23,20 @@ #include #include #include +#include #include #include +/* + * TODO(b/277649169) Best fit IOVA allocator was removed in 6.1 GKI + * The API needs to either be upstreamed, integrated into this driver, or disabled for 6.1 + * compatibility. For now, disable best-fit on all non-Android kernels and any GKI > 5.15. + */ +#define HAS_IOVAD_BEST_FIT_ALGO \ + (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) && \ + (IS_ENABLED(CONFIG_GCIP_TEST) || IS_ENABLED(CONFIG_ANDROID))) + /* * Helpers for manipulating @gcip_map_flags parameter of the `gcip_iommu_domain_{map,unmap}_sg` * functions. diff --git a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-kci.h b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-kci.h index 51dd803..1cfc82e 100644 --- a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-kci.h +++ b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-kci.h @@ -16,22 +16,6 @@ #include -/* - * The status field in a firmware response is set to this by us when the response is fetched from - * the queue. - */ -#define GCIP_KCI_STATUS_OK GCIP_MAILBOX_STATUS_OK -/* - * gcip_kci#mailbox.wait_list uses this value to record the status of responses that haven't been - * received yet. - */ -#define GCIP_KCI_STATUS_WAITING_RESPONSE GCIP_MAILBOX_STATUS_WAITING_RESPONSE -/* - * Used when an expected response is not received, see the documentation of - * gcip_mailbox_handle_response() for details. - */ -#define GCIP_KCI_STATUS_NO_RESPONSE GCIP_MAILBOX_STATUS_NO_RESPONSE - /* * Command/response sequence numbers capped at half the range of the 64-bit value range. The second * half is reserved for incoming requests from firmware. @@ -62,9 +46,9 @@ struct gcip_kci_response_element { u64 seq; u16 code; /* - * Reserved for host use - firmware can't touch this. - * If a value is written here it will be discarded and overwritten during response - * processing. However, when repurposed as an RKCI command, the FW can set this field. + * Firmware can set some data according to the type of the response. + * TODO(b/279386960): as we don't manage the status of responses using this field anymore, + * rename this field to more reasonable name. */ u16 status; /* diff --git a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-mailbox.h b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-mailbox.h index 835503f..af48ba6 100644 --- a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-mailbox.h +++ b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-mailbox.h @@ -88,10 +88,21 @@ static inline bool gcip_valid_circ_queue_size(u32 size, u32 wrap_bit) struct gcip_mailbox; +/* + * A struct wraps the IP-defined response to manage additional information such as status needed by + * the logic of GCIP. + */ +struct gcip_mailbox_async_resp { + /* Status code. Must be one of GCIP_MAILBOX_STATUS_*. */ + uint16_t status; + /* IP-defined response. */ + void *resp; +}; + /* Wrapper struct for responses consumed by a thread other than the one which sent the command. */ struct gcip_mailbox_resp_awaiter { /* Response. */ - void *resp; + struct gcip_mailbox_async_resp async_resp; /* The work which will be executed when the timeout occurs. */ struct delayed_work timeout_work; /* @@ -217,16 +228,6 @@ struct gcip_mailbox_ops { * Context: normal and in_interrupt(). */ void (*set_resp_elem_seq)(struct gcip_mailbox *mailbox, void *resp, u64 seq); - /* - * Gets the status of @resp queue element. - * Context: normal and in_interrupt(). - */ - u16 (*get_resp_elem_status)(struct gcip_mailbox *mailbox, void *resp); - /* - * Sets the status of @resp queue element. - * Context: normal and in_interrupt(). - */ - void (*set_resp_elem_status)(struct gcip_mailbox *mailbox, void *resp, u16 status); /* * Acquires the lock of wait_list. If @irqsave is true, "_irqsave" functions can be used to diff --git a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-usage-stats.h b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-usage-stats.h index a20fe33..a4e0cb4 100644 --- a/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-usage-stats.h +++ b/drivers/edgetpu/gcip-kernel-driver/include/gcip/gcip-usage-stats.h @@ -152,6 +152,8 @@ enum gcip_usage_stats_component_utilization_type { GCIP_USAGE_STATS_COMPONENT_UTILIZATION_IP, /* A compute core. */ GCIP_USAGE_STATS_COMPONENT_UTILIZATION_CORES, + /* The DSP or TPU Control Core (R52). */ + GCIP_USAGE_STATS_COMPONENT_UTILIZATION_CONTROL, /* The number of total types. Must be located at the end of this enum. */ GCIP_USAGE_STATS_COMPONENT_UTILIZATION_NUM_TYPES, @@ -302,6 +304,10 @@ enum gcip_usage_stats_thread_stats_thread_id { * DSP cores. */ GCIP_USAGE_STATS_THREAD_DSP_CORE_MANAGER, + /* The driving thread for intercore message handling. */ + GCIP_USAGE_STATS_THREAD_INTERCORE_SUBORDINATE, + /* The thread that executes callback when the timer expires. */ + GCIP_USAGE_STATS_THREAD_TIMER_SERVICE, /* The number of total threads. Must be located at the end of this enum. */ GCIP_USAGE_STATS_THREAD_NUM_TYPES, -- cgit v1.2.3