diff options
| author | Aurora pro automerger <aurora-pro-automerger@google.com> | 2023-03-14 02:07:34 +0000 |
|---|---|---|
| committer | Copybara-Service <copybara-worker@google.com> | 2023-03-28 22:19:52 -0700 |
| commit | ec2e64f0bb54d942059d75ccc7c4ab6daa775f78 (patch) | |
| tree | ce23e15e88a8c51e74a637916f0404087ac5459b | |
| parent | 0ffef44f5b07977a5608985c66e7eeaadc29b426 (diff) | |
| download | gs201-ec2e64f0bb54d942059d75ccc7c4ab6daa775f78.tar.gz | |
gxp: [Copybara Auto Merge] Merge branch 'gs201-u' into 'android13-gs-pixel-5.10-udc'android-u-beta-4_r0.7android-u-beta-4_r0.5android-u-beta-4_r0.4android-u-beta-4_r0.3android-u-beta-4_r0.2android-u-beta-4_r0.1android-u-beta-3_r0.3android-u-beta-3_r0.2android-u-beta-2.1_r0.4android-u-beta-2.1_r0.3android-u-beta-2.1_r0.2android-gs-tangorpro-5.10-u-beta4android-gs-raviole-5.10-u-beta4android-gs-raviole-5.10-u-beta3android-gs-raviole-5.10-u-beta2android-gs-pantah-5.10-u-beta4android-gs-pantah-5.10-u-beta2android-gs-lynx-5.10-u-beta4android-gs-felix-5.10-u-beta4android-gs-bluejay-5.10-u-beta4android-gs-bluejay-5.10-u-beta3android-gs-bluejay-5.10-u-beta2
gxp: fix compiler warning in gxp-thermal
gxp: move fw_rw_section into if-guard
gxp: fix memory leak with invalid telemetry type
Bug: 273254318
gxp: remove unneeded checks in vd.c
gxp: skip configuring when missing VD config
gxp: remove unused host-dev structs
Bug: 265748027
gxp: remove unused range-alloc module
gxp: entirely remove app_metadata
Bug: 265748027 (repeat)
gxp: remove legacy firmware data init
Bug: 265748027 (repeat)
gxp: remove legacy telemetry descriptor
Bug: 265748027 (repeat)
gxp: move system cfg population to device probe
Bug: 265748027 (repeat)
gxp: add gxp_fw_data_system_cfg
Bug: 265748027 (repeat)
gxp: add gxp_fw_data_resource
Bug: 265748027 (repeat)
gxp: bump version 1.11
Bugs:
gxp: fix panic on buffer flushing
Bug: 268389591
gxp: Do not put core in reset if already in ACG
Bug: 272664140
gxp: add cached core telemetry desc
Bug: 265748027 (repeat)
gxp: Adpot GCIP thermal
Bug: 264729080
Bug: 213272324
gxp: remove core_boot parameter
Bug: 251612313
gxp: use GPL-2.0-only license
Bug: 271797962
gxp: authenticates firmware through GSA
Bug: 260533620
gxp: Enable debug dump for imageconfig 2.
Bug: 271371895
gxp: Add size check while loading image to memory
Bug: 265105909
gxp: refactor MCU firmware life cycle
Bug: 233159020
gxp: Create a function for gxp_core_boot
Bug: 271716712
gxp: Increase UCI command timeout to 2 seconds
Bug: 271622596
gxp: remove checks of data_mgr
gxp: skip mapping core resources in MCU mode
gxp: add gxp_firmware_loader module
gxp: log a warning on failing to map CMU reg
gxp: disable out-dated auth mechanism in MCU mode
Bug: 260533620 (repeat)
gxp: refactor core firmware loading
gxp: stop mapping core->TPU queues
gxp: get core_count by counting bits
Bug: 270097855
gxp: set SSMT to bypass in both mode
Bug: 269855604
gxp: Skip gxp_vd_block_unready if gxp_vd_block_ready was not executed
Bug: 268427254
gcip: Make gcip_pm_{get_count,is_powered} lockless
Bug: 271756295
gcip: generalize mem_pool
gcip: utilize mock files on unittests
Bug: 272216562
gcip: use GPL-2.0-only license
Bug: 271797962 (repeat)
gcip: Remove gcip_pm_put_async
Bug: 271756295 (repeat)
gcip: Only call .power_up if needed
gcip: Print GCIP_FW_LOG_LEVEL_FATAL as error message
Bug: 271596603
gcip: Postfix gcip_firmware_tracing_restore
gcip: fix undefined variable on !THERMAL
gcip: always return NULL on domain alloc error
gcip: Add gcip_thermal_destroy
Bug; 264729080 (repeat)
gcip: Add thermal votes
Bug: 271194361
Bug: 264729080 (repeat)
gcip: Cleanup abandoned domains on domain-pool destroy
gcip: Prefix MAX_NUM_THERMAL_STATES
Bug: 264729080 (repeat)
gcip: Add const to thermal_cooling_device_ops
Bug: 264729080 (repeat)
gcip: Add thermal support
Bug: 264729080 (repeat)
gcip: remove redundant else in pm.c
GCIP_MAIN_REV_ID: a5b6843ab58f30d6ce086016214cbf56a46610a8
gcip: Add gcip_pm_lock
Bug: 264729080 (repeat)
gcip: Make gcip_pm_{get_count,is_powered} lockless
Bug: 271756295 (repeat)
gcip: generalize mem_pool
gcip: use GPL-2.0-only license
Bug: 271797962 (repeat)
gcip: Remove gcip_pm_put_async
Bug: 271756295 (repeat)
gcip: Add gcip_pm_trylock
Bug: 271756295 (repeat)
gcip: Add level -3 (FATAL) for firmware log
Bug: 271596603 (repeat)
gcip: Update outdated comments
gcip: Postfix gcip_firmware_tracing_restore
gcip: Add gcip_thermal_destroy
Bug: 264729080 (repeat)
gcip: Add thermal votes
Bug: 271194361 (repeat)
Bug: 264729080 (repeat)
gcip: Add missing includes to gcip-domain-pool.h
gcip: Add list of dynamic domains to domain-pool
gcip: Prefix MAX_NUM_THERMAL_STATES
Bug: 264729080 (repeat)
gcip: add watchdog timeout crash type
Bug:255416846
gcip: Add thermal header
Bug: 264729080 (repeat)
GCIP_HEADERS_REV_ID: 126a2bd1e5ac72231c88425fbddb4b9fe2fd85b1
GitOrigin-RevId: 03d9970af576cb6267f25715de67d4ffd3e2934c
Change-Id: Ia0e7a0d8113d209e0bb57e160bc3f2d89b1d4c5e
66 files changed, 1769 insertions, 2050 deletions
@@ -22,6 +22,7 @@ gxp-objs += \ gxp-doorbell.o \ gxp-eventfd.o \ gxp-firmware-data.o \ + gxp-firmware-loader.o \ gxp-firmware.o \ gxp-lpm.o \ gxp-mailbox-manager.o \ @@ -29,7 +30,6 @@ gxp-objs += \ gxp-mapping.o \ gxp-mb-notification.o \ gxp-pm.o \ - gxp-range-alloc.o \ gxp-ssmt.o \ gxp-thermal.o \ gxp-vd.o diff --git a/amalthea/config-pwr-state.h b/amalthea/config-pwr-state.h index 3c8a4be..2712797 100644 --- a/amalthea/config-pwr-state.h +++ b/amalthea/config-pwr-state.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Chip-dependent power configuration and states. * diff --git a/amalthea/config.h b/amalthea/config.h index bc81e42..34f658b 100644 --- a/amalthea/config.h +++ b/amalthea/config.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Include all configuration files for Amalthea. * diff --git a/amalthea/context.h b/amalthea/context.h index 3f88930..8afc99c 100644 --- a/amalthea/context.h +++ b/amalthea/context.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Amalthea context related macros. * diff --git a/amalthea/csrs.h b/amalthea/csrs.h index 5b9dac4..8fee289 100644 --- a/amalthea/csrs.h +++ b/amalthea/csrs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Amalthea CSR definitions. * diff --git a/amalthea/iova.h b/amalthea/iova.h index b1a07ae..505e895 100644 --- a/amalthea/iova.h +++ b/amalthea/iova.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GXP IOVAs. The list of addresses for fixed device-side IOVAs * diff --git a/amalthea/lpm.h b/amalthea/lpm.h index fa36976..1d86d69 100644 --- a/amalthea/lpm.h +++ b/amalthea/lpm.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Amalthea LPM chip-dependent settings. * diff --git a/amalthea/mailbox-regs.h b/amalthea/mailbox-regs.h index 050398e..0a2fb27 100644 --- a/amalthea/mailbox-regs.h +++ b/amalthea/mailbox-regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GXP mailbox registers. * diff --git a/gcip-kernel-driver/drivers/gcip/Makefile b/gcip-kernel-driver/drivers/gcip/Makefile index bc370e5..7de0874 100644 --- a/gcip-kernel-driver/drivers/gcip/Makefile +++ b/gcip-kernel-driver/drivers/gcip/Makefile @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0 +# SPDX-License-Identifier: GPL-2.0-only # # Makefile for GCIP framework. # @@ -15,7 +15,8 @@ gcip-objs := gcip-alloc-helper.o \ gcip-mailbox.o \ gcip-mem-pool.o \ gcip-pm.o \ - gcip-telemetry.o + gcip-telemetry.o \ + gcip-thermal.o CURRENT_DIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) @@ -23,6 +24,8 @@ ccflags-y += -I$(CURRENT_DIR)/../../include ifdef CONFIG_GCIP_TEST obj-y += unittests/ +include $(srctree)/drivers/gcip/unittests/Makefile.include +$(call include_test_path, $(gcip-objs)) endif modules modules_install clean: diff --git a/gcip-kernel-driver/drivers/gcip/gcip-alloc-helper.c b/gcip-kernel-driver/drivers/gcip/gcip-alloc-helper.c index 85af8e5..4008dff 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-alloc-helper.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-alloc-helper.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GCIP helpers for allocating memories. * diff --git a/gcip-kernel-driver/drivers/gcip/gcip-dma-fence.c b/gcip-kernel-driver/drivers/gcip/gcip-dma-fence.c index 4f83670..ca49526 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-dma-fence.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-dma-fence.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GCIP support of DMA fences. * diff --git a/gcip-kernel-driver/drivers/gcip/gcip-domain-pool.c b/gcip-kernel-driver/drivers/gcip/gcip-domain-pool.c index 2341b52..882aa80 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-domain-pool.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-domain-pool.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GCIP IOMMU domain allocator. * @@ -12,6 +12,11 @@ #include <gcip/gcip-domain-pool.h> +struct dynamic_domain { + struct list_head list_entry; + struct iommu_domain *domain; +}; + int gcip_domain_pool_init(struct device *dev, struct gcip_domain_pool *pool, unsigned int size) { unsigned int i; @@ -19,6 +24,8 @@ int gcip_domain_pool_init(struct device *dev, struct gcip_domain_pool *pool, uns pool->size = size; pool->dev = dev; + INIT_LIST_HEAD(&pool->dynamic_domains); + mutex_init(&pool->lock); if (!size) return 0; @@ -48,9 +55,23 @@ int gcip_domain_pool_init(struct device *dev, struct gcip_domain_pool *pool, uns struct iommu_domain *gcip_domain_pool_alloc(struct gcip_domain_pool *pool) { int id; + struct dynamic_domain *ddomain; - if (!pool->size) - return iommu_domain_alloc(pool->dev->bus); + if (!pool->size) { + ddomain = vzalloc(sizeof(*ddomain)); + if (!ddomain) + return NULL; + + ddomain->domain = iommu_domain_alloc(pool->dev->bus); + if (!ddomain->domain) { + vfree(ddomain); + return NULL; + } + mutex_lock(&pool->lock); + list_add_tail(&ddomain->list_entry, &pool->dynamic_domains); + mutex_unlock(&pool->lock); + return ddomain->domain; + } id = ida_alloc_max(&pool->idp, pool->size - 1, GFP_KERNEL); @@ -67,11 +88,25 @@ struct iommu_domain *gcip_domain_pool_alloc(struct gcip_domain_pool *pool) void gcip_domain_pool_free(struct gcip_domain_pool *pool, struct iommu_domain *domain) { int id; + struct dynamic_domain *ddomain; + struct list_head *cur, *nxt; if (!pool->size) { - iommu_domain_free(domain); + mutex_lock(&pool->lock); + list_for_each_safe(cur, nxt, &pool->dynamic_domains) { + ddomain = container_of(cur, struct dynamic_domain, list_entry); + if (ddomain->domain == domain) { + list_del(&ddomain->list_entry); + mutex_unlock(&pool->lock); + iommu_domain_free(domain); + vfree(ddomain); + return; + } + } + mutex_unlock(&pool->lock); return; } + for (id = 0; id < pool->size; id++) { if (pool->array[id] == domain) { dev_dbg(pool->dev, "Released domain from pool with id = %d\n", id); @@ -85,9 +120,20 @@ void gcip_domain_pool_free(struct gcip_domain_pool *pool, struct iommu_domain *d void gcip_domain_pool_destroy(struct gcip_domain_pool *pool) { int i; + struct dynamic_domain *ddomain; + struct list_head *cur, *nxt; - if (!pool->size) + if (!pool->size) { + mutex_lock(&pool->lock); + list_for_each_safe(cur, nxt, &pool->dynamic_domains) { + ddomain = container_of(cur, struct dynamic_domain, list_entry); + list_del(&ddomain->list_entry); + iommu_domain_free(ddomain->domain); + vfree(ddomain); + } + mutex_unlock(&pool->lock); return; + } dev_dbg(pool->dev, "Destroying domain pool with %u domains\n", pool->size); diff --git a/gcip-kernel-driver/drivers/gcip/gcip-firmware.c b/gcip-kernel-driver/drivers/gcip/gcip-firmware.c index 1d9392c..52c3940 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-firmware.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-firmware.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GCIP firmware interface. * @@ -136,7 +136,7 @@ void gcip_firmware_tracing_destroy(struct gcip_fw_tracing *fw_tracing) kfree(fw_tracing); } -int gcip_firmware_tracing_restore(struct gcip_fw_tracing *fw_tracing) +int gcip_firmware_tracing_restore_on_powering(struct gcip_fw_tracing *fw_tracing) { int ret = 0; diff --git a/gcip-kernel-driver/drivers/gcip/gcip-image-config.c b/gcip-kernel-driver/drivers/gcip/gcip-image-config.c index 62acd0b..98a3546 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-image-config.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-image-config.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * Framework for parsing the firmware image configuration. * diff --git a/gcip-kernel-driver/drivers/gcip/gcip-kci.c b/gcip-kernel-driver/drivers/gcip/gcip-kci.c index 15b2c53..c3da416 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-kci.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-kci.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * Kernel Control Interface, implements the protocol between AP kernel and GCIP firmware. * diff --git a/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c b/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c index 334a51d..6d20771 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GCIP Mailbox Interface. * diff --git a/gcip-kernel-driver/drivers/gcip/gcip-mem-pool.c b/gcip-kernel-driver/drivers/gcip/gcip-mem-pool.c index 3e18051..564991b 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-mem-pool.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-mem-pool.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * A simple memory allocator to help allocating reserved memory pools. * @@ -12,21 +12,21 @@ #include <gcip/gcip-mem-pool.h> -int gcip_mem_pool_init(struct gcip_mem_pool *pool, struct device *dev, phys_addr_t base_paddr, +int gcip_mem_pool_init(struct gcip_mem_pool *pool, struct device *dev, unsigned long base_addr, size_t size, size_t granule) { int ret; - if (!base_paddr || granule == 0) + if (!base_addr || granule == 0) return -EINVAL; - if (base_paddr % granule || size % granule) + if (base_addr % granule || size % granule) return -EINVAL; pool->gen_pool = gen_pool_create(ilog2(granule), -1); if (!pool->gen_pool) { dev_err(dev, "gcip memory pool allocate gen_pool failed"); return -ENOMEM; } - ret = gen_pool_add(pool->gen_pool, base_paddr, size, -1); + ret = gen_pool_add(pool->gen_pool, base_addr, size, -1); if (ret) { gen_pool_destroy(pool->gen_pool); pool->gen_pool = NULL; @@ -35,7 +35,7 @@ int gcip_mem_pool_init(struct gcip_mem_pool *pool, struct device *dev, phys_addr } pool->dev = dev; pool->granule = granule; - pool->base_paddr = base_paddr; + pool->base_addr = base_addr; return 0; } @@ -47,23 +47,20 @@ void gcip_mem_pool_exit(struct gcip_mem_pool *pool) pool->gen_pool = NULL; } -phys_addr_t gcip_mem_pool_alloc(struct gcip_mem_pool *pool, size_t size) +unsigned long gcip_mem_pool_alloc(struct gcip_mem_pool *pool, size_t size) { unsigned long addr; - size_t aligned_size = ALIGN(size, pool->granule); - addr = gen_pool_alloc(pool->gen_pool, aligned_size); + addr = gen_pool_alloc(pool->gen_pool, size); if (!addr) return 0; - dev_dbg(pool->dev, "%s @ size = %#zx paddr=%#lx", __func__, size, addr); - return (phys_addr_t)addr; + dev_dbg(pool->dev, "%s @ size = %#zx addr=%#lx", __func__, size, addr); + return addr; } -void gcip_mem_pool_free(struct gcip_mem_pool *pool, phys_addr_t paddr, size_t size) +void gcip_mem_pool_free(struct gcip_mem_pool *pool, unsigned long addr, size_t size) { - unsigned long addr = paddr; - - dev_dbg(pool->dev, "%s @ size = %#zx paddr=%#lx", __func__, size, addr); + dev_dbg(pool->dev, "%s @ size = %#zx addr=%#lx", __func__, size, addr); size = ALIGN(size, pool->granule); gen_pool_free(pool->gen_pool, addr, size); } diff --git a/gcip-kernel-driver/drivers/gcip/gcip-pm.c b/gcip-kernel-driver/drivers/gcip/gcip-pm.c index 43d9654..b9907a1 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-pm.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-pm.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * Power management interface for GCIP devices. * @@ -113,8 +113,10 @@ static int gcip_pm_get_locked(struct gcip_pm *pm) gcip_pm_lockdep_assert_held(pm); if (!pm->count) { - pm->power_down_pending = false; - ret = pm->power_up(pm->data); + if (pm->power_down_pending) + pm->power_down_pending = false; + else + ret = pm->power_up(pm->data); } if (!ret) @@ -163,7 +165,7 @@ int gcip_pm_get(struct gcip_pm *pm) return ret; } -static void __gcip_pm_put(struct gcip_pm *pm, bool async) +void gcip_pm_put(struct gcip_pm *pm) { if (!pm) return; @@ -175,10 +177,7 @@ static void __gcip_pm_put(struct gcip_pm *pm, bool async) if (!--pm->count) { pm->power_down_pending = true; - if (async) - schedule_delayed_work(&pm->power_down_work, 0); - else - gcip_pm_try_power_down(pm); + gcip_pm_try_power_down(pm); } dev_dbg(pm->dev, "%s: %d\n", __func__, pm->count); @@ -187,29 +186,12 @@ unlock: mutex_unlock(&pm->lock); } -void gcip_pm_put(struct gcip_pm *pm) -{ - __gcip_pm_put(pm, false); -} - -void gcip_pm_put_async(struct gcip_pm *pm) -{ - __gcip_pm_put(pm, true); -} - int gcip_pm_get_count(struct gcip_pm *pm) { - int count = -EAGAIN; - if (!pm) return 0; - if (mutex_trylock(&pm->lock)) { - count = pm->count; - mutex_unlock(&pm->lock); - } - - return count; + return pm->count; } bool gcip_pm_is_powered(struct gcip_pm *pm) @@ -228,8 +210,7 @@ void gcip_pm_shutdown(struct gcip_pm *pm, bool force) if (pm->count) { if (!force) goto unlock; - else - dev_warn(pm->dev, "Force shutdown with power up count: %d", pm->count); + dev_warn(pm->dev, "Force shutdown with power up count: %d", pm->count); } gcip_pm_try_power_down(pm); diff --git a/gcip-kernel-driver/drivers/gcip/gcip-telemetry.c b/gcip-kernel-driver/drivers/gcip/gcip-telemetry.c index f557c24..1599889 100644 --- a/gcip-kernel-driver/drivers/gcip/gcip-telemetry.c +++ b/gcip-kernel-driver/drivers/gcip/gcip-telemetry.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GCIP telemetry: logging and tracing. * @@ -126,6 +126,7 @@ void gcip_telemetry_fw_log(struct gcip_telemetry *log) case GCIP_FW_LOG_LEVEL_WARN: dev_warn(dev, "%s", buffer); break; + case GCIP_FW_LOG_LEVEL_FATAL: case GCIP_FW_LOG_LEVEL_ERROR: dev_err(dev, "%s", buffer); break; diff --git a/gcip-kernel-driver/drivers/gcip/gcip-thermal.c b/gcip-kernel-driver/drivers/gcip/gcip-thermal.c new file mode 100644 index 0000000..bc06cd5 --- /dev/null +++ b/gcip-kernel-driver/drivers/gcip/gcip-thermal.c @@ -0,0 +1,517 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Thermal management support for GCIP devices. + * + * Copyright (C) 2023 Google LLC + */ + +#include <linux/debugfs.h> +#include <linux/device.h> +#include <linux/minmax.h> +#include <linux/mutex.h> +#include <linux/notifier.h> +#include <linux/thermal.h> +#include <linux/version.h> + +#include <gcip/gcip-pm.h> +#include <gcip/gcip-thermal.h> + +#define OF_DATA_NUM_MAX (GCIP_THERMAL_MAX_NUM_STATES * 2) + +#define to_cdev(dev) container_of(dev, struct thermal_cooling_device, device) +#define to_gcip_thermal(dev) ((struct gcip_thermal *)to_cdev(dev)->devdata) + +/* Struct for state to rate and state to power mappings. */ +struct gcip_rate_pwr { + unsigned long rate; + u32 power; +}; + +static struct gcip_rate_pwr state_map[GCIP_THERMAL_MAX_NUM_STATES] = { 0 }; + +static int gcip_thermal_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state) +{ + struct gcip_thermal *thermal = cdev->devdata; + + if (!thermal->num_states) + return -ENODEV; + + *state = thermal->num_states - 1; + + return 0; +} + +static int gcip_thermal_get_cur_state(struct thermal_cooling_device *cdev, unsigned long *state) +{ + struct gcip_thermal *thermal = cdev->devdata; + + mutex_lock(&thermal->lock); + *state = thermal->state; + mutex_unlock(&thermal->lock); + + return 0; +} + +static int gcip_thermal_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state) +{ + struct gcip_thermal *thermal = cdev->devdata; + int i, ret = 0; + + if (state >= thermal->num_states) { + dev_err(thermal->dev, "Invalid thermal cooling state %lu\n", state); + return -EINVAL; + } + + mutex_lock(&thermal->lock); + + thermal->vote[GCIP_THERMAL_COOLING_DEVICE] = state; + for (i = 0; i < GCIP_THERMAL_MAX_NUM_VOTERS; i++) + state = max(state, thermal->vote[i]); + + if (state == thermal->state) + goto out; + + if (!gcip_pm_get_if_powered(thermal->pm, false)) { + ret = thermal->set_rate(thermal->data, state_map[state].rate); + gcip_pm_put(thermal->pm); + } + + if (ret) + dev_err(thermal->dev, "Failed to set thermal cooling state: %d\n", ret); + else + thermal->state = state; +out: + mutex_unlock(&thermal->lock); + + return ret; +} + +static int gcip_thermal_rate2power_internal(struct gcip_thermal *thermal, unsigned long rate, + u32 *power) +{ + int i; + + for (i = 0; i < thermal->num_states; i++) { + if (rate == state_map[i].rate) { + *power = state_map[i].power; + return 0; + } + } + + dev_err(thermal->dev, "Unknown rate for: %lu\n", rate); + *power = 0; + + return -EINVAL; +} + +static int gcip_thermal_get_requested_power(struct thermal_cooling_device *cdev, u32 *power) +{ + struct gcip_thermal *thermal = cdev->devdata; + unsigned long rate; + int ret; + + if (gcip_pm_get_if_powered(thermal->pm, false)) { + *power = 0; + return 0; + } + + mutex_lock(&thermal->lock); + + ret = thermal->get_rate(thermal->data, &rate); + + mutex_unlock(&thermal->lock); + gcip_pm_put(thermal->pm); + + if (ret) + return ret; + + return gcip_thermal_rate2power_internal(thermal, rate, power); +} + +static int gcip_thermal_state2power(struct thermal_cooling_device *cdev, unsigned long state, + u32 *power) +{ + struct gcip_thermal *thermal = cdev->devdata; + + if (state >= thermal->num_states) { + dev_err(thermal->dev, "Invalid state: %lu\n", state); + return -EINVAL; + } + + return gcip_thermal_rate2power_internal(thermal, state_map[state].rate, power); +} + +static int gcip_thermal_power2state(struct thermal_cooling_device *cdev, u32 power, + unsigned long *state) +{ + struct gcip_thermal *thermal = cdev->devdata; + + if (!thermal->num_states) + return -ENODEV; + + /* + * Argument "power" is the maximum allowed power consumption in mW as defined by the PID + * control loop. Checks for the first state that is less than or equal to the current + * allowed power. state_map is descending, so lowest power consumption is last value in the + * array. Returns lowest state even if it consumes more power than allowed as not all + * platforms can handle throttling below an active state. + */ + for (*state = 0; *state < thermal->num_states; (*state)++) + if (power >= state_map[*state].power) + return 0; + + *state = thermal->num_states - 1; + + return 0; +} + +static const struct thermal_cooling_device_ops gcip_thermal_ops = { + .get_max_state = gcip_thermal_get_max_state, + .get_cur_state = gcip_thermal_get_cur_state, + .set_cur_state = gcip_thermal_set_cur_state, + .get_requested_power = gcip_thermal_get_requested_power, + .state2power = gcip_thermal_state2power, + .power2state = gcip_thermal_power2state, +}; + +/* This API was removed, but Android still uses it to update thermal request. */ +#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0) && IS_ENABLED(CONFIG_ANDROID) +void thermal_cdev_update(struct thermal_cooling_device *cdev); +#endif + +static void gcip_thermal_update(struct gcip_thermal *thermal) +{ + struct thermal_cooling_device *cdev = thermal->cdev; + + cdev->updated = false; + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 12, 0) || IS_ENABLED(CONFIG_ANDROID) + thermal_cdev_update(cdev); +#elif IS_ENABLED(CONFIG_THERMAL) + dev_err_once(thermal->dev, "Thermal update not implemented"); +#endif +} + +static ssize_t user_vote_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct gcip_thermal *thermal = to_gcip_thermal(dev); + ssize_t ret; + + if (!thermal) + return -ENODEV; + + mutex_lock(&thermal->lock); + ret = sysfs_emit(buf, "%lu\n", thermal->vote[GCIP_THERMAL_SYSFS]); + mutex_unlock(&thermal->lock); + + return ret; +} + +static ssize_t user_vote_store(struct device *dev, struct device_attribute *attr, const char *buf, + size_t count) +{ + struct gcip_thermal *thermal = to_gcip_thermal(dev); + unsigned long state; + int ret; + + if (!thermal) + return -ENODEV; + + ret = kstrtoul(buf, 0, &state); + if (ret) + return ret; + + if (state >= thermal->num_states) + return -EINVAL; + + mutex_lock(&thermal->lock); + thermal->vote[GCIP_THERMAL_SYSFS] = state; + mutex_unlock(&thermal->lock); + + gcip_thermal_update(thermal); + + return count; +} + +static DEVICE_ATTR_RW(user_vote); + +static int gcip_thermal_rate2state(struct gcip_thermal *thermal, unsigned long rate) +{ + int i; + + for (i = 0; i < thermal->num_states; i++) { + if (state_map[i].rate <= rate) + return i; + } + + /* Returns lowest state on an invalid input. */ + return thermal->num_states - 1; +} + +static int gcip_thermal_notifier(struct notifier_block *nb, unsigned long rate, void *nb_data) +{ + struct gcip_thermal *thermal = container_of(nb, struct gcip_thermal, nb); + unsigned long state = gcip_thermal_rate2state(thermal, rate); + + dev_dbg(thermal->dev, "Thermal notifier req original: %lu, state: %lu\n", rate, state); + + mutex_lock(&thermal->lock); + thermal->vote[GCIP_THERMAL_NOTIFIER_BLOCK] = state; + mutex_unlock(&thermal->lock); + + gcip_thermal_update(thermal); + + return NOTIFY_OK; +} + +struct notifier_block *gcip_thermal_get_notifier_block(struct gcip_thermal *thermal) +{ + if (IS_ERR_OR_NULL(thermal)) + return NULL; + + return &thermal->nb; +} + +void gcip_thermal_destroy(struct gcip_thermal *thermal) +{ + if (IS_ERR_OR_NULL(thermal)) + return; + + debugfs_remove_recursive(thermal->dentry); + thermal_cooling_device_unregister(thermal->cdev); + devm_kfree(thermal->dev, thermal); +} + +static int gcip_thermal_enable_get(void *data, u64 *val) +{ + struct gcip_thermal *thermal = (struct gcip_thermal *)data; + + mutex_lock(&thermal->lock); + *val = thermal->enabled; + mutex_unlock(&thermal->lock); + + return 0; +} + +static int gcip_thermal_enable_set(void *data, u64 val) +{ + struct gcip_thermal *thermal = (struct gcip_thermal *)data; + int ret = 0; + + mutex_lock(&thermal->lock); + + if (thermal->enabled != (bool)val) { + /* + * If the device is not powered, the value will be restored by + * gcip_thermal_restore_on_powering in next fw boot. + */ + if (!gcip_pm_get_if_powered(thermal->pm, false)) { + ret = thermal->control(thermal->data, val); + gcip_pm_put(thermal->pm); + } + + if (!ret) { + thermal->enabled = val; + dev_info_ratelimited(thermal->dev, "%s thermal control", + thermal->enabled ? "Enable" : "Disable"); + } else { + dev_err(thermal->dev, "Failed to %s thermal control: %d ", + val ? "enable" : "disable", ret); + } + } + + mutex_unlock(&thermal->lock); + + return ret; +} + +DEFINE_DEBUGFS_ATTRIBUTE(fops_gcip_thermal_enable, gcip_thermal_enable_get, gcip_thermal_enable_set, + "%llu\n"); + +static int gcip_thermal_parse_dvfs_table(struct gcip_thermal *thermal) +{ + int row_size, col_size, tbl_size, i; + int of_data_int_array[OF_DATA_NUM_MAX]; + + if (of_property_read_u32_array(thermal->dev->of_node, GCIP_THERMAL_TABLE_SIZE_NAME, + of_data_int_array, 2)) + goto error; + + row_size = of_data_int_array[0]; + col_size = of_data_int_array[1]; + tbl_size = row_size * col_size; + if (row_size > GCIP_THERMAL_MAX_NUM_STATES) { + dev_err(thermal->dev, "Too many states\n"); + goto error; + } + + if (tbl_size > OF_DATA_NUM_MAX) + goto error; + + if (of_property_read_u32_array(thermal->dev->of_node, GCIP_THERMAL_TABLE_NAME, + of_data_int_array, tbl_size)) + goto error; + + thermal->num_states = row_size; + for (i = 0; i < row_size; ++i) { + int idx = col_size * i; + + state_map[i].rate = of_data_int_array[idx]; + state_map[i].power = of_data_int_array[idx + 1]; + } + + return 0; + +error: + dev_err(thermal->dev, "Failed to parse DVFS table\n"); + + return -EINVAL; +} + +static int gcip_thermal_cooling_register(struct gcip_thermal *thermal, const char *type, + const char *node_name) +{ + struct device_node *node = NULL; + int ret; + + ret = gcip_thermal_parse_dvfs_table(thermal); + if (ret) + return ret; + + if (node_name) + node = of_find_node_by_name(NULL, node_name); + if (!node) + dev_warn(thermal->dev, "Failed to find thermal cooling node\n"); + + thermal->cdev = thermal_of_cooling_device_register(node, type, thermal, &gcip_thermal_ops); + if (IS_ERR(thermal->cdev)) + return PTR_ERR(thermal->cdev); + + ret = device_create_file(&thermal->cdev->device, &dev_attr_user_vote); + if (ret) + thermal_cooling_device_unregister(thermal->cdev); + + return ret; +} + +struct gcip_thermal *gcip_thermal_create(const struct gcip_thermal_args *args) +{ + struct gcip_thermal *thermal; + int ret; + + if (!args->dev || !args->get_rate || !args->set_rate || !args->control) + return ERR_PTR(-EINVAL); + + thermal = devm_kzalloc(args->dev, sizeof(*thermal), GFP_KERNEL); + if (!thermal) + return ERR_PTR(-ENOMEM); + + thermal->dev = args->dev; + thermal->nb.notifier_call = gcip_thermal_notifier; + thermal->pm = args->pm; + thermal->enabled = true; + thermal->data = args->data; + thermal->get_rate = args->get_rate; + thermal->set_rate = args->set_rate; + thermal->control = args->control; + + mutex_init(&thermal->lock); + + ret = gcip_thermal_cooling_register(thermal, args->type, args->node_name); + if (ret) { + dev_err(args->dev, "Failed to initialize external thermal cooling\n"); + devm_kfree(args->dev, thermal); + return ERR_PTR(ret); + } + + thermal->dentry = debugfs_create_dir("cooling", args->dentry); + /* Don't let debugfs creation failure abort the init procedure. */ + if (IS_ERR_OR_NULL(thermal->dentry)) + dev_warn(args->dev, "Failed to create debugfs for thermal cooling"); + else + debugfs_create_file("enable", 0660, thermal->dentry, thermal, + &fops_gcip_thermal_enable); + + return thermal; +} + +int gcip_thermal_suspend_device(struct gcip_thermal *thermal) +{ + int ret = 0; + + if (IS_ERR_OR_NULL(thermal)) + return 0; + + mutex_lock(&thermal->lock); + + /* + * Always sets as suspended even when the request cannot be handled for unknown reasons + * because we still want to prevent the client from using device. + */ + thermal->device_suspended = true; + if (!gcip_pm_get_if_powered(thermal->pm, false)) { + ret = thermal->set_rate(thermal->data, 0); + gcip_pm_put(thermal->pm); + } + + mutex_unlock(&thermal->lock); + + return ret; +} + +int gcip_thermal_resume_device(struct gcip_thermal *thermal) +{ + int ret = 0; + + if (IS_ERR_OR_NULL(thermal)) + return 0; + + mutex_lock(&thermal->lock); + + if (!gcip_pm_get_if_powered(thermal->pm, false)) { + ret = thermal->set_rate(thermal->data, state_map[thermal->state].rate); + gcip_pm_put(thermal->pm); + } + + /* + * Unlike gcip_thermal_suspend_device(), only sets the device as resumed if the request is + * fulfilled. + */ + if (!ret) + thermal->device_suspended = false; + + mutex_unlock(&thermal->lock); + + return ret; +} + +bool gcip_thermal_is_device_suspended(struct gcip_thermal *thermal) +{ + if (IS_ERR_OR_NULL(thermal)) + return false; + + return thermal->device_suspended; +} + +int gcip_thermal_restore_on_powering(struct gcip_thermal *thermal) +{ + int ret = 0; + + if (IS_ERR_OR_NULL(thermal)) + return 0; + + gcip_pm_lockdep_assert_held(thermal->pm); + mutex_lock(&thermal->lock); + + if (!thermal->enabled) + ret = thermal->control(thermal->data, thermal->enabled); + else if (thermal->device_suspended) + ret = thermal->set_rate(thermal->data, 0); + else if (thermal->state) + /* Skips state 0 since it's the default thermal state. */ + ret = thermal->set_rate(thermal->data, state_map[thermal->state].rate); + + mutex_unlock(&thermal->lock); + + return ret; +} diff --git a/gcip-kernel-driver/include/gcip/gcip-alloc-helper.h b/gcip-kernel-driver/include/gcip/gcip-alloc-helper.h index 3d2c110..17208bf 100644 --- a/gcip-kernel-driver/include/gcip/gcip-alloc-helper.h +++ b/gcip-kernel-driver/include/gcip/gcip-alloc-helper.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GCIP helpers for allocating memories. * diff --git a/gcip-kernel-driver/include/gcip/gcip-common-image-header.h b/gcip-kernel-driver/include/gcip/gcip-common-image-header.h index d986fbc..b86b430 100644 --- a/gcip-kernel-driver/include/gcip/gcip-common-image-header.h +++ b/gcip-kernel-driver/include/gcip/gcip-common-image-header.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Common authenticated image format for Google SoCs * diff --git a/gcip-kernel-driver/include/gcip/gcip-dma-fence.h b/gcip-kernel-driver/include/gcip/gcip-dma-fence.h index 1d4030a..ad765d2 100644 --- a/gcip-kernel-driver/include/gcip/gcip-dma-fence.h +++ b/gcip-kernel-driver/include/gcip/gcip-dma-fence.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GCIP support of DMA fences. * diff --git a/gcip-kernel-driver/include/gcip/gcip-domain-pool.h b/gcip-kernel-driver/include/gcip/gcip-domain-pool.h index b740bf9..3a6ae4b 100644 --- a/gcip-kernel-driver/include/gcip/gcip-domain-pool.h +++ b/gcip-kernel-driver/include/gcip/gcip-domain-pool.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GCIP IOMMU domain allocator. * @@ -8,8 +8,11 @@ #ifndef __GCIP_DOMAIN_POOL_H__ #define __GCIP_DOMAIN_POOL_H__ +#include <linux/device.h> #include <linux/idr.h> #include <linux/iommu.h> +#include <linux/mutex.h> +#include <linux/types.h> struct gcip_domain_pool { struct ida idp; /* ID allocator to keep track of used domains. */ @@ -20,6 +23,8 @@ struct gcip_domain_pool { unsigned int size; struct iommu_domain **array; /* Array holding the pointers to pre-allocated domains. */ struct device *dev; /* The device used for logging warnings/errors. */ + struct list_head dynamic_domains; /* Tracks dynamically allocated domains for cleanup. */ + struct mutex lock; /* Protects dynamic_domains. */ }; /* diff --git a/gcip-kernel-driver/include/gcip/gcip-firmware.h b/gcip-kernel-driver/include/gcip/gcip-firmware.h index 8cf4353..52f5d11 100644 --- a/gcip-kernel-driver/include/gcip/gcip-firmware.h +++ b/gcip-kernel-driver/include/gcip/gcip-firmware.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GCIP firmware interface. * @@ -44,9 +44,10 @@ enum gcip_fw_flavor { GCIP_FW_FLAVOR_CUSTOM = 4, }; -/* Type of firmware crash which will be sent by GCIP_RKCI_FIRMWARE_CRASH RKCI command. */ +/* Type of firmware crash. */ enum gcip_fw_crash_type { - /* Assert happened. */ + /* Type which will be sent by GCIP_RKCI_FIRMWARE_CRASH reverse KCI. */ + /*Assert happened. */ GCIP_FW_CRASH_ASSERT_FAIL = 0, /* Data abort exception. */ GCIP_FW_CRASH_DATA_ABORT = 1, @@ -58,6 +59,9 @@ enum gcip_fw_crash_type { GCIP_FW_CRASH_UNRECOVERABLE_FAULT = 4, /* Used in debug dump. */ GCIP_FW_CRASH_DUMMY_CRASH_TYPE = 0xFF, + + /* HW watchdog timeout. */ + GCIP_FW_CRASH_HW_WDG_TIMEOUT = 0x100, }; /* Firmware info filled out via KCI FIRMWARE_INFO command. */ @@ -80,7 +84,9 @@ struct gcip_fw_tracing { * Lock to protect the struct members listed below. * * Note that since the request of tracing level adjusting might happen during power state - * transitions, this lock must be acquired after holding the pm lock to avoid deadlock. + * transitions (i.e., another thread calling gcip_firmware_tracing_restore_on_powering() + * with pm lock held), one must either use the non-blocking gcip_pm_get_if_powered() or make + * sure there won't be any new power transition after holding this lock to prevent deadlock. */ struct mutex lock; /* Actual firmware tracing level. */ @@ -123,6 +129,6 @@ void gcip_firmware_tracing_destroy(struct gcip_fw_tracing *fw_tracing); * This function is designed to restore the firmware tracing level during power management calls and * thus it assumes the caller holds the pm lock. */ -int gcip_firmware_tracing_restore(struct gcip_fw_tracing *fw_tracing); +int gcip_firmware_tracing_restore_on_powering(struct gcip_fw_tracing *fw_tracing); #endif /* __GCIP_FIRMWARE_H__ */ diff --git a/gcip-kernel-driver/include/gcip/gcip-image-config.h b/gcip-kernel-driver/include/gcip/gcip-image-config.h index bcc506f..df09d39 100644 --- a/gcip-kernel-driver/include/gcip/gcip-image-config.h +++ b/gcip-kernel-driver/include/gcip/gcip-image-config.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Framework for parsing the firmware image configuration. * diff --git a/gcip-kernel-driver/include/gcip/gcip-kci.h b/gcip-kernel-driver/include/gcip/gcip-kci.h index 03cc078..eb83550 100644 --- a/gcip-kernel-driver/include/gcip/gcip-kci.h +++ b/gcip-kernel-driver/include/gcip/gcip-kci.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Kernel Control Interface, implements the protocol between AP kernel and GCIP firmware. * diff --git a/gcip-kernel-driver/include/gcip/gcip-mailbox.h b/gcip-kernel-driver/include/gcip/gcip-mailbox.h index 649b574..c88d2d7 100644 --- a/gcip-kernel-driver/include/gcip/gcip-mailbox.h +++ b/gcip-kernel-driver/include/gcip/gcip-mailbox.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GCIP Mailbox Interface. * diff --git a/gcip-kernel-driver/include/gcip/gcip-mem-pool.h b/gcip-kernel-driver/include/gcip/gcip-mem-pool.h index c770300..44ea5f5 100644 --- a/gcip-kernel-driver/include/gcip/gcip-mem-pool.h +++ b/gcip-kernel-driver/include/gcip/gcip-mem-pool.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * A simple memory allocator to help allocating reserved memory pools. * @@ -15,7 +15,7 @@ struct gcip_mem_pool { struct device *dev; struct gen_pool *gen_pool; - phys_addr_t base_paddr; + unsigned long base_addr; size_t granule; }; @@ -24,8 +24,7 @@ struct gcip_mem_pool { * * @pool: The memory pool object to be initialized. * @dev: Used for logging only. - * @base_paddr: The base physical address of the pool. Must be greater than 0 and a multiple of - * @granule. + * @base_addr: The base address of the pool. Must be greater than 0 and a multiple of @granule. * @size: The size of the pool. @size should be a multiple of @granule. * @granule: The granule when invoking the allocator. Should be a power of 2. * @@ -33,7 +32,7 @@ struct gcip_mem_pool { * * Call gcip_mem_pool_exit() to release the resources of @pool. */ -int gcip_mem_pool_init(struct gcip_mem_pool *pool, struct device *dev, phys_addr_t base_paddr, +int gcip_mem_pool_init(struct gcip_mem_pool *pool, struct device *dev, unsigned long base_addr, size_t size, size_t granule); /* * Releases resources of @pool. @@ -44,28 +43,28 @@ int gcip_mem_pool_init(struct gcip_mem_pool *pool, struct device *dev, phys_addr void gcip_mem_pool_exit(struct gcip_mem_pool *pool); /* - * Allocates and returns the allocated physical address. + * Allocates and returns the allocated address. * * @size: Size to be allocated. * * Returns the allocated address. Returns 0 on allocation failure. */ -phys_addr_t gcip_mem_pool_alloc(struct gcip_mem_pool *pool, size_t size); +unsigned long gcip_mem_pool_alloc(struct gcip_mem_pool *pool, size_t size); /* * Returns the address previously allocated by gcip_mem_pool_alloc(). * * The size and address must match what previously passed to / returned by gcip_mem_pool_alloc(). */ -void gcip_mem_pool_free(struct gcip_mem_pool *pool, phys_addr_t paddr, size_t size); +void gcip_mem_pool_free(struct gcip_mem_pool *pool, unsigned long addr, size_t size); /* - * Returns the offset between @paddr and @base_paddr passed to gcip_mem_pool_init(). + * Returns the offset between @addr and @base_addr passed to gcip_mem_pool_init(). * - * @paddr must be a value returned by gcip_mem_pool_alloc(). + * @addr must be a value returned by gcip_mem_pool_alloc(). */ -static inline size_t gcip_mem_pool_offset(struct gcip_mem_pool *pool, phys_addr_t paddr) +static inline size_t gcip_mem_pool_offset(struct gcip_mem_pool *pool, unsigned long addr) { - return paddr - pool->base_paddr; + return addr - pool->base_addr; } #endif /* __GCIP_MEM_POOL_H__ */ diff --git a/gcip-kernel-driver/include/gcip/gcip-pm.h b/gcip-kernel-driver/include/gcip/gcip-pm.h index c7673d8..4842598 100644 --- a/gcip-kernel-driver/include/gcip/gcip-pm.h +++ b/gcip-kernel-driver/include/gcip/gcip-pm.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Power management support for GCIP devices. * @@ -97,17 +97,10 @@ int gcip_pm_get(struct gcip_pm *pm); */ void gcip_pm_put(struct gcip_pm *pm); -/* - * Same as gcip_pm_put, but the power off will be scheduled later. - * Caller should use this async gcip_pm_put if they're on the power off path to prevent deadlock, - * e.g., a workqueue that will be canceled during power off. - */ -void gcip_pm_put_async(struct gcip_pm *pm); - -/* Gets the power up counter. Retures -EAGAIN if device is in power state transition. */ +/* Gets the power up counter. Note that this is checked without PM lock. */ int gcip_pm_get_count(struct gcip_pm *pm); -/* Checks if device is already on. Retures false if device is off or in power state transition. */ +/* Checks if device is already on. Note that this is checked without PM lock. */ bool gcip_pm_is_powered(struct gcip_pm *pm); /* Shuts down the device if @pm->count equals to 0 or @force is true. */ @@ -122,4 +115,41 @@ static inline void gcip_pm_lockdep_assert_held(struct gcip_pm *pm) lockdep_assert_held(&pm->lock); } +/* + * Lock the PM lock. + * Since all the PM requests will be blocked until gcip_pm_unlock is called, one should use the + * gcip_pm_{get,get_if_powered,put} if possible and uses this only if a power state transition can + * not be triggered, e.g., in a workqueue that will be canceled during power off or crash handler. + */ +static inline void gcip_pm_lock(struct gcip_pm *pm) +{ + if (!pm) + return; + + mutex_lock(&pm->lock); +} + +/* + * Lock the PM lock. + * Same as gcip_pm_lock, but returns 1 if the lock has been acquired successfully, and 0 on + * contention. + */ +static inline int gcip_pm_trylock(struct gcip_pm *pm) +{ + if (!pm) + return 1; + + return mutex_trylock(&pm->lock); +} + +/* Unlock the PM lock. */ +static inline void gcip_pm_unlock(struct gcip_pm *pm) +{ + if (!pm) + return; + + lockdep_assert_held(&pm->lock); + mutex_unlock(&pm->lock); +} + #endif /* __GCIP_PM_H__ */ diff --git a/gcip-kernel-driver/include/gcip/gcip-telemetry.h b/gcip-kernel-driver/include/gcip/gcip-telemetry.h index 4556291..ad26ee9 100644 --- a/gcip-kernel-driver/include/gcip/gcip-telemetry.h +++ b/gcip-kernel-driver/include/gcip/gcip-telemetry.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GCIP telemetry: logging and tracing. * @@ -22,6 +22,7 @@ #define GCIP_FW_LOG_LEVEL_INFO (0) #define GCIP_FW_LOG_LEVEL_WARN (-1) #define GCIP_FW_LOG_LEVEL_ERROR (-2) +#define GCIP_FW_LOG_LEVEL_FATAL (-3) #define GCIP_FW_DMESG_LOG_LEVEL (GCIP_FW_LOG_LEVEL_WARN) diff --git a/gcip-kernel-driver/include/gcip/gcip-thermal.h b/gcip-kernel-driver/include/gcip/gcip-thermal.h new file mode 100644 index 0000000..7c9ebc4 --- /dev/null +++ b/gcip-kernel-driver/include/gcip/gcip-thermal.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Thermal management support for GCIP devices. + * + * Copyright (C) 2023 Google LLC + */ + +#ifndef __GCIP_THERMAL_H__ +#define __GCIP_THERMAL_H__ + +#include <linux/device.h> +#include <linux/mutex.h> +#include <linux/notifier.h> +#include <linux/thermal.h> + +#define GCIP_THERMAL_TABLE_SIZE_NAME "gcip-dvfs-table-size" +#define GCIP_THERMAL_TABLE_NAME "gcip-dvfs-table" +#define GCIP_THERMAL_MAX_NUM_STATES 10 + +enum gcip_thermal_voter { + GCIP_THERMAL_COOLING_DEVICE, + GCIP_THERMAL_SYSFS, + GCIP_THERMAL_NOTIFIER_BLOCK, + + /* Keeps as the last entry for the total number of voters. */ + GCIP_THERMAL_MAX_NUM_VOTERS, +}; + +struct gcip_thermal { + struct device *dev; + struct thermal_cooling_device *cdev; + struct notifier_block nb; + struct dentry *dentry; + struct gcip_pm *pm; + + /* + * Lock to protect the struct members listed below. + * + * Note that since the request of thermal state adjusting might happen during power state + * transitions (i.e., another thread calling gcip_thermal_restore_on_powering() with pm lock + * held), one must either use the non-blocking gcip_pm_get_if_powered() or make sure there + * won't be any new power transition after holding this thermal lock to prevent deadlock. + */ + struct mutex lock; + unsigned long num_states; + unsigned long state; + unsigned long vote[GCIP_THERMAL_MAX_NUM_VOTERS]; + bool device_suspended; + bool enabled; + + /* Private data. See struct gcip_thermal_args.*/ + void *data; + + /* Callbacks. See struct gcip_thermal_args. */ + int (*get_rate)(void *data, unsigned long *rate); + int (*set_rate)(void *data, unsigned long rate); + int (*control)(void *data, bool enable); +}; + +/* Arguments for devm_gcip_thermal_create. */ +struct gcip_thermal_args { + /* Device struct of GCIP device. */ + struct device *dev; + /* GCIP power management. */ + struct gcip_pm *pm; + /* Top-level debugfs directory for the device. */ + struct dentry *dentry; + /* Name of the thermal cooling-device node in device tree. */ + const char *node_name; + /* Thermal cooling device type for thermal_of_cooling_device_register() . */ + const char *type; + /* Private data for callbacks listed below. */ + void *data; + /* + * Callbacks listed below are called only if the device is powered and with the guarantee + * that there won't be any new power transition during the call (i.e., after + * gcip_pm_get_if_powered() succeeds or during the power up triggered by gcip_pm_get()) + * to prevent deadlock since they are called with thermal lock held. See the note about + * thermal lock in struct gcip_thermal. + */ + /* Callback to get the device clock rate. */ + int (*get_rate)(void *data, unsigned long *rate); + /* + * Callback to set the device clock rate. + * Might be called with pm lock held in gcip_thermal_restore_on_powering(). + */ + int (*set_rate)(void *data, unsigned long rate); + /* + * Callback to enable/disable the thermal control. + * Might be called with pm lock held in gcip_thermal_restore_on_powering(). + */ + int (*control)(void *data, bool enable); +}; + +/* Gets the notifier_block struct for thermal throttling requests. */ +struct notifier_block *gcip_thermal_get_notifier_block(struct gcip_thermal *thermal); +/* Allocates and initializes GCIP thermal struct. */ +struct gcip_thermal *gcip_thermal_create(const struct gcip_thermal_args *args); +/* Destroys and frees GCIP thermal struct. */ +void gcip_thermal_destroy(struct gcip_thermal *thermal); +/* Suspends the device due to thermal request. */ +int gcip_thermal_suspend_device(struct gcip_thermal *thermal); +/* Resumes the device and restores previous thermal state. */ +int gcip_thermal_resume_device(struct gcip_thermal *thermal); +/* + * Checks whether the device is suspended by thermal. + * Note that it's checked without thermal lock and state might change subsequently. + */ +bool gcip_thermal_is_device_suspended(struct gcip_thermal *thermal); +/* + * Restores the previous thermal state. + * + * This function is designed to restore the thermal state during power management calls and thus it + * assumes the caller holds the pm lock. + */ +int gcip_thermal_restore_on_powering(struct gcip_thermal *thermal); + +#endif /* __GCIP_THERMAL_H__ */ diff --git a/gsx01-mailbox-driver.c b/gsx01-mailbox-driver.c index f0090f4..9876998 100644 --- a/gsx01-mailbox-driver.c +++ b/gsx01-mailbox-driver.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GXP hardware-based mailbox csr driver implementation for GSX01. * diff --git a/gxp-client.c b/gxp-client.c index 7c96e9a..a776542 100644 --- a/gxp-client.c +++ b/gxp-client.c @@ -65,10 +65,7 @@ void gxp_client_destroy(struct gxp_client *client) if (client->vd) { if (gxp->before_unmap_tpu_mbx_queue) gxp->before_unmap_tpu_mbx_queue(gxp, client); - /* - * TODO(b/237624453): remove '|| 1' once the MCU supports DSP->TPU interop - */ - if (gxp_is_direct_mode(gxp) || 1) + if (gxp_is_direct_mode(gxp)) gxp_dma_unmap_tpu_buffer(gxp, client->vd->domain, client->mbx_desc); diff --git a/gxp-common-platform.c b/gxp-common-platform.c index de3417c..7514b11 100644 --- a/gxp-common-platform.c +++ b/gxp-common-platform.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GXP platform driver utilities. * @@ -9,6 +9,7 @@ #include <linux/platform_data/sscoredump.h> #endif +#include <linux/bitops.h> #include <linux/device.h> #include <linux/dma-mapping.h> #include <linux/file.h> @@ -36,6 +37,7 @@ #include "gxp-domain-pool.h" #include "gxp-firmware.h" #include "gxp-firmware-data.h" +#include "gxp-firmware-loader.h" #include "gxp-internal.h" #include "gxp-lpm.h" #include "gxp-mailbox.h" @@ -144,13 +146,10 @@ static int gxp_open(struct inode *inode, struct file *file) misc_dev); int ret = 0; - /* If this is the first call to open(), request the firmware files */ - ret = gxp_firmware_request_if_needed(gxp); - if (ret) { - dev_err(gxp->dev, - "Failed to request dsp firmware files (ret=%d)\n", ret); + /* If this is the first call to open(), load the firmware files */ + ret = gxp_firmware_loader_load_if_needed(gxp); + if (ret) return ret; - } client = gxp_client_create(gxp); if (IS_ERR(client)) @@ -893,8 +892,8 @@ static int map_tpu_mbx_queue(struct gxp_client *client, down_read(&gxp->vd_semaphore); - core_count = client->vd->num_cores; phys_core_list = client->vd->core_list; + core_count = hweight_long(phys_core_list); mbx_info = kmalloc( sizeof(struct edgetpu_ext_mailbox_info) + @@ -977,8 +976,7 @@ static int gxp_map_tpu_mbx_queue(struct gxp_client *client, int ret = 0; if (!gxp->tpu_dev.mbx_paddr) { - dev_err(gxp->dev, "%s: TPU is not available for interop\n", - __func__); + dev_err(gxp->dev, "TPU is not available for interop\n"); return -EINVAL; } @@ -1014,8 +1012,7 @@ static int gxp_map_tpu_mbx_queue(struct gxp_client *client, goto out_unlock_client_semaphore; } - /* TODO(b/237624453): remove '|| 1' once the MCU supports DSP->TPU interop */ - if (gxp_is_direct_mode(gxp) || 1) { + if (gxp_is_direct_mode(gxp)) { ret = map_tpu_mbx_queue(client, &ibuf); if (ret) goto err_fput_tpu_file; @@ -1030,7 +1027,8 @@ static int gxp_map_tpu_mbx_queue(struct gxp_client *client, goto out_unlock_client_semaphore; err_unmap_tpu_mbx_queue: - unmap_tpu_mbx_queue(client, &ibuf); + if (gxp_is_direct_mode(gxp)) + unmap_tpu_mbx_queue(client, &ibuf); err_fput_tpu_file: fput(client->tpu_file); client->tpu_file = NULL; @@ -1068,8 +1066,7 @@ static int gxp_unmap_tpu_mbx_queue(struct gxp_client *client, if (gxp->before_unmap_tpu_mbx_queue) gxp->before_unmap_tpu_mbx_queue(gxp, client); - /* TODO(b/237624453): remove '|| 1' once the MCU supports DSP->TPU interop */ - if (gxp_is_direct_mode(gxp) || 1) + if (gxp_is_direct_mode(gxp)) unmap_tpu_mbx_queue(client, &ibuf); fput(client->tpu_file); @@ -1837,6 +1834,8 @@ static int gxp_set_reg_resources(struct platform_device *pdev, struct gxp_dev *g gxp->cmu.paddr = r->start; gxp->cmu.size = resource_size(r); gxp->cmu.vaddr = devm_ioremap_resource(dev, r); + if (IS_ERR_OR_NULL(gxp->cmu.vaddr)) + dev_warn(dev, "Failed to map CMU registers\n"); } /* * TODO (b/224685748): Remove this block after CMU CSR is supported @@ -2062,6 +2061,12 @@ static int gxp_common_platform_probe(struct platform_device *pdev, struct gxp_de goto err_domain_pool_destroy; } + ret = gxp_firmware_loader_init(gxp); + if (ret) { + dev_err(dev, "Failed to initialize firmware loader (ret=%d)\n", + ret); + goto err_fw_destroy; + } gxp_dma_init_default_resources(gxp); gxp_vd_init(gxp); @@ -2085,11 +2090,10 @@ static int gxp_common_platform_probe(struct platform_device *pdev, struct gxp_de dev_err(dev, "Failed to initialize core telemetry (ret=%d)", ret); goto err_fw_data_destroy; } - gxp->thermal_mgr = gxp_thermal_init(gxp); - if (IS_ERR(gxp->thermal_mgr)) { - ret = PTR_ERR(gxp->thermal_mgr); + + ret = gxp_thermal_init(gxp); + if (ret) dev_warn(dev, "Failed to init thermal driver: %d\n", ret); - } gxp->gfence_mgr = gcip_dma_fence_manager_create(gxp->dev); if (IS_ERR(gxp->gfence_mgr)) { @@ -2105,6 +2109,11 @@ static int gxp_common_platform_probe(struct platform_device *pdev, struct gxp_de if (ret) goto err_dma_fence_destroy; } + /* + * We only know where the system config region is after after_probe is + * done so this can't be called earlier. + */ + gxp_fw_data_populate_system_config(gxp); gxp->misc_dev.minor = MISC_DYNAMIC_MINOR; gxp->misc_dev.name = GXP_NAME; @@ -2127,12 +2136,14 @@ err_before_remove: err_dma_fence_destroy: /* DMA fence manager creation doesn't need revert */ err_thermal_destroy: - /* thermal init doesn't need revert */ + gxp_thermal_exit(gxp); gxp_core_telemetry_exit(gxp); err_fw_data_destroy: gxp_fw_data_destroy(gxp); err_vd_destroy: gxp_vd_destroy(gxp); + gxp_firmware_loader_destroy(gxp); +err_fw_destroy: gxp_fw_destroy(gxp); err_domain_pool_destroy: gxp_domain_pool_destroy(gxp->domain_pool); @@ -2156,6 +2167,11 @@ static int gxp_common_platform_remove(struct platform_device *pdev) { struct gxp_dev *gxp = platform_get_drvdata(pdev); + /* + * Call gxp_thermal_exit before gxp_remove_debugdir since it will + * remove its own debugfs. + */ + gxp_thermal_exit(gxp); gxp_remove_debugdir(gxp); misc_deregister(&gxp->misc_dev); if (gxp->before_remove) @@ -2163,6 +2179,7 @@ static int gxp_common_platform_remove(struct platform_device *pdev) gxp_core_telemetry_exit(gxp); gxp_fw_data_destroy(gxp); gxp_vd_destroy(gxp); + gxp_firmware_loader_destroy(gxp); gxp_fw_destroy(gxp); gxp_domain_pool_destroy(gxp->domain_pool); kfree(gxp->domain_pool); diff --git a/gxp-config.h b/gxp-config.h index dbc2d24..ebc1c78 100644 --- a/gxp-config.h +++ b/gxp-config.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Include all configuration files for GXP. * diff --git a/gxp-core-telemetry.c b/gxp-core-telemetry.c index bce27c6..bfa9264 100644 --- a/gxp-core-telemetry.c +++ b/gxp-core-telemetry.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GXP core telemetry support * @@ -839,6 +839,7 @@ int gxp_core_telemetry_register_eventfd(struct gxp_dev *gxp, u8 type, int fd) break; default: ret = -EINVAL; + eventfd_ctx_put(new_ctx); goto out; } diff --git a/gxp-core-telemetry.h b/gxp-core-telemetry.h index 9a89c0e..0ceeb60 100644 --- a/gxp-core-telemetry.h +++ b/gxp-core-telemetry.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GXP core telemetry support * diff --git a/gxp-debug-dump.c b/gxp-debug-dump.c index 94ae78b..4df7add 100644 --- a/gxp-debug-dump.c +++ b/gxp-debug-dump.c @@ -14,17 +14,16 @@ #include <linux/string.h> #include <linux/workqueue.h> -#if IS_ENABLED(CONFIG_GXP_TEST) || IS_ENABLED(CONFIG_SUBSYSTEM_COREDUMP) -#include <linux/platform_data/sscoredump.h> -#endif - #include <gcip/gcip-pm.h> +#include <gcip/gcip-alloc-helper.h> #include "gxp-client.h" #include "gxp-debug-dump.h" #include "gxp-dma.h" #include "gxp-doorbell.h" #include "gxp-firmware.h" +#include "gxp-firmware-data.h" +#include "gxp-firmware-loader.h" #include "gxp-host-device-structs.h" #include "gxp-internal.h" #include "gxp-lpm.h" @@ -32,6 +31,10 @@ #include "gxp-pm.h" #include "gxp-vd.h" +#if HAS_COREDUMP +#include <linux/platform_data/sscoredump.h> +#endif + #define SSCD_MSG_LENGTH 64 #define SYNC_BARRIER_BLOCK 0x00100000 @@ -39,6 +42,13 @@ #define DEBUG_DUMP_MEMORY_SIZE 0x400000 /* size in bytes */ +/* + * CORE_FIRMWARE_RW_STRIDE & CORE_FIRMWARE_RW_ADDR must match with their + * values defind in core firmware image config. + */ +#define CORE_FIRMWARE_RW_STRIDE 0x200000 /* 2 MB */ +#define CORE_FIRMWARE_RW_ADDR(x) (0xFA400000 + CORE_FIRMWARE_RW_STRIDE * x) + /* Enum indicating the debug dump request reason. */ enum gxp_debug_dump_init_type { DEBUG_DUMP_FW_INIT, DEBUG_DUMP_KERNEL_INIT }; @@ -301,7 +311,7 @@ static int gxp_get_common_dump(struct gxp_dev *gxp) return ret; } -#if IS_ENABLED(CONFIG_GXP_TEST) || IS_ENABLED(CONFIG_SUBSYSTEM_COREDUMP) +#if HAS_COREDUMP static void gxp_send_to_sscd(struct gxp_dev *gxp, void *segs, int seg_cnt, const char *info) { @@ -454,7 +464,62 @@ static int gxp_user_buffers_vmap(struct gxp_dev *gxp, out: return cnt; } -#endif + +/** + * gxp_map_fw_rw_section() - Maps the fw rw section address and size to be + * sent to sscd module for taking the dump. + * @gxp: The GXP device. + * @vd: vd of the crashed client. + * @core_id: physical core_id of crashed core. + * @seg_idx: Pointer to a index that is keeping track of + * gxp->debug_dump_mgr->segs[] array. + * + * This function parses the ns_regions of the given vd to find + * fw_rw_section details. + * + * Return: + * * 0 - Successfully mapped fw_rw_section data. + * * -EOPNOTSUPP - Operation not supported for invalid image config. + * * -ENXIO - No IOVA found for the fw_rw_section. + */ +static int gxp_map_fw_rw_section(struct gxp_dev *gxp, + struct gxp_virtual_device *vd, + uint32_t core_id, int *seg_idx) +{ + size_t idx; + struct sg_table *sgt; + struct gxp_debug_dump_manager *mgr = gxp->debug_dump_mgr; + dma_addr_t fw_rw_section_daddr = CORE_FIRMWARE_RW_ADDR(core_id); + const size_t n_reg = ARRAY_SIZE(vd->ns_regions); + + if (!gxp_fw_data_use_per_vd_config(vd)) { + dev_err(gxp->dev, "Unsupported Image config version = %d.", + gxp->fw_loader_mgr->core_img_cfg.config_version); + return -EOPNOTSUPP; + } + + for (idx = 0; idx < n_reg; idx++) { + sgt = vd->ns_regions[idx].sgt; + if (!sgt) + break; + + if (fw_rw_section_daddr != vd->ns_regions[idx].daddr) + continue; + + mgr->segs[core_id][*seg_idx].addr = + gcip_noncontiguous_sgt_to_mem(sgt); + mgr->segs[core_id][*seg_idx].size = gcip_ns_config_to_size( + gxp->fw_loader_mgr->core_img_cfg.ns_iommu_mappings[idx]); + *seg_idx += 1; + return 0; + } + dev_err(gxp->dev, + "fw_rw_section mapping for core %u at iova 0x%llx does not exist", + core_id, fw_rw_section_daddr); + return -ENXIO; +} + +#endif /* HAS_COREDUMP */ void gxp_debug_dump_invalidate_segments(struct gxp_dev *gxp, uint32_t core_id) { @@ -505,7 +570,7 @@ static int gxp_handle_debug_dump(struct gxp_dev *gxp, &core_dump->core_dump_header[core_id]; struct gxp_core_header *core_header = &core_dump_header->core_header; int ret = 0; -#if IS_ENABLED(CONFIG_GXP_TEST) || IS_ENABLED(CONFIG_SUBSYSTEM_COREDUMP) +#if HAS_COREDUMP struct gxp_common_dump *common_dump = mgr->common_dump; int i; int seg_idx = 0; @@ -513,7 +578,7 @@ static int gxp_handle_debug_dump(struct gxp_dev *gxp, char sscd_msg[SSCD_MSG_LENGTH]; void *user_buf_vaddrs[GXP_NUM_BUFFER_MAPPINGS]; int user_buf_cnt; -#endif +#endif /* HAS_COREDUMP */ /* Core */ if (!core_header->dump_available) { @@ -522,7 +587,7 @@ static int gxp_handle_debug_dump(struct gxp_dev *gxp, goto out; } -#if IS_ENABLED(CONFIG_GXP_TEST) || IS_ENABLED(CONFIG_SUBSYSTEM_COREDUMP) +#if HAS_COREDUMP /* Common */ data_addr = &common_dump->common_dump_data.common_regs; for (i = 0; i < GXP_NUM_COMMON_SEGMENTS; i++) { @@ -571,14 +636,16 @@ static int gxp_handle_debug_dump(struct gxp_dev *gxp, ret = -EFAULT; goto out_efault; } - /* - * TODO(b/265105909): Implement the logic for collecting fw rw section - * separately for mcu mode. - */ + /* fw ro section */ mgr->segs[core_id][seg_idx].addr = gxp->fwbufs[core_id].vaddr; - mgr->segs[core_id][seg_idx].size = gxp->fwbufs[core_id].size; + mgr->segs[core_id][seg_idx].size = vd->fw_ro_size; seg_idx++; + /* fw rw section */ + ret = gxp_map_fw_rw_section(gxp, vd, core_id, &seg_idx); + if (ret) + goto out; + /* User Buffers */ user_buf_cnt = gxp_user_buffers_vmap(gxp, vd, core_header, user_buf_vaddrs); @@ -605,7 +672,7 @@ out_efault: gxp_user_buffers_vunmap(gxp, vd, core_header); } -#endif +#endif /* HAS_COREDUMP */ out: gxp_debug_dump_invalidate_segments(gxp, core_id); @@ -615,7 +682,7 @@ out: static int gxp_init_segments(struct gxp_dev *gxp) { -#if !(IS_ENABLED(CONFIG_GXP_TEST) || IS_ENABLED(CONFIG_SUBSYSTEM_COREDUMP)) +#if !HAS_COREDUMP return 0; #else struct gxp_debug_dump_manager *mgr = gxp->debug_dump_mgr; @@ -625,7 +692,7 @@ static int gxp_init_segments(struct gxp_dev *gxp) return -ENOMEM; return 0; -#endif +#endif /* HAS_COREDUMP */ } /* @@ -663,23 +730,12 @@ out: static void gxp_generate_debug_dump(struct gxp_dev *gxp, uint core_id, struct gxp_virtual_device *vd) { - u32 boot_mode; - bool gxp_generate_coredump_called = false; - + bool gxp_generate_coredump_called = true; mutex_lock(&gxp->debug_dump_mgr->debug_dump_lock); - /* - * TODO(b/265105909): Checks below to be verified after implementation for - * firmware loading for mcu mode are completed. - */ - boot_mode = gxp_firmware_get_boot_mode(gxp, vd, core_id); - - if (gxp_is_fw_running(gxp, core_id) && - (boot_mode == GXP_BOOT_MODE_STATUS_COLD_BOOT_COMPLETED || - boot_mode == GXP_BOOT_MODE_STATUS_RESUME_COMPLETED)) { - gxp_generate_coredump_called = true; - if (gxp_generate_coredump(gxp, vd, core_id)) - dev_err(gxp->dev, "Failed to generate coredump\n"); + if (gxp_generate_coredump(gxp, vd, core_id)) { + gxp_generate_coredump_called = false; + dev_err(gxp->dev, "Failed to generate the coredump.\n"); } /* Invalidate segments to prepare for the next debug dump trigger */ @@ -706,8 +762,14 @@ static void gxp_debug_dump_process_dump_direct_mode(struct work_struct *work) struct gxp_virtual_device *vd = NULL; down_read(&gxp->vd_semaphore); - if (gxp->core_to_vd[core_id]) + if (gxp->core_to_vd[core_id]) { vd = gxp_vd_get(gxp->core_to_vd[core_id]); + } else { + dev_err(gxp->dev, "debug dump failed for null vd on core %d.", + core_id); + up_read(&gxp->vd_semaphore); + return; + } up_read(&gxp->vd_semaphore); /* @@ -715,15 +777,12 @@ static void gxp_debug_dump_process_dump_direct_mode(struct work_struct *work) * of @vd while generating a debug dump. This will help not to block other virtual devices * proceeding their jobs. */ - if (vd) - mutex_lock(&vd->debug_dump_lock); + mutex_lock(&vd->debug_dump_lock); gxp_generate_debug_dump(gxp, core_id, vd); - if (vd) { - mutex_unlock(&vd->debug_dump_lock); - gxp_vd_put(vd); - } + mutex_unlock(&vd->debug_dump_lock); + gxp_vd_put(vd); } int gxp_debug_dump_process_dump_mcu_mode(struct gxp_dev *gxp, uint core_list, diff --git a/gxp-debug-dump.h b/gxp-debug-dump.h index 1cf9219..aeb8229 100644 --- a/gxp-debug-dump.h +++ b/gxp-debug-dump.h @@ -12,13 +12,16 @@ #include <linux/types.h> #include <linux/workqueue.h> -#if IS_ENABLED(CONFIG_GXP_TEST) || IS_ENABLED(CONFIG_SUBSYSTEM_COREDUMP) -#include <linux/platform_data/sscoredump.h> -#endif - #include "gxp-dma.h" #include "gxp-internal.h" +#define HAS_COREDUMP \ + (IS_ENABLED(CONFIG_GXP_TEST) || IS_ENABLED(CONFIG_SUBSYSTEM_COREDUMP)) + +#if HAS_COREDUMP +#include <linux/platform_data/sscoredump.h> +#endif + #define GXP_NUM_COMMON_SEGMENTS 2 #define GXP_NUM_CORE_SEGMENTS 8 #define GXP_NUM_BUFFER_MAPPINGS 32 @@ -188,7 +191,7 @@ struct gxp_debug_dump_manager { * time */ struct mutex debug_dump_lock; -#if IS_ENABLED(CONFIG_GXP_TEST) || IS_ENABLED(CONFIG_SUBSYSTEM_COREDUMP) +#if HAS_COREDUMP struct sscd_segment segs[GXP_NUM_CORES][GXP_NUM_SEGMENTS_PER_CORE]; #endif }; diff --git a/gxp-debugfs.c b/gxp-debugfs.c index a9a2e14..6dacde9 100644 --- a/gxp-debugfs.c +++ b/gxp-debugfs.c @@ -15,6 +15,7 @@ #include "gxp-debugfs.h" #include "gxp-dma.h" #include "gxp-firmware-data.h" +#include "gxp-firmware-loader.h" #include "gxp-firmware.h" #include "gxp-internal.h" #include "gxp-notification.h" @@ -136,9 +137,9 @@ static int gxp_firmware_run_set(void *data, u64 val) uint core; bool acquired_block_wakelock; - ret = gxp_firmware_request_if_needed(gxp); + ret = gxp_firmware_loader_load_if_needed(gxp); if (ret) { - dev_err(gxp->dev, "Unable to request dsp firmware files\n"); + dev_err(gxp->dev, "Unable to load firmware files\n"); return ret; } diff --git a/gxp-dma-fence.c b/gxp-dma-fence.c index 7bcc11b..4733081 100644 --- a/gxp-dma-fence.c +++ b/gxp-dma-fence.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GXP support for DMA fence. * diff --git a/gxp-dma-fence.h b/gxp-dma-fence.h index c7ad95e..38f8cf2 100644 --- a/gxp-dma-fence.h +++ b/gxp-dma-fence.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GXP support for DMA fence. * diff --git a/gxp-dma-iommu.c b/gxp-dma-iommu.c index 94a78b3..ad1111b 100644 --- a/gxp-dma-iommu.c +++ b/gxp-dma-iommu.c @@ -81,7 +81,7 @@ static int gxp_dma_ssmt_program(struct gxp_dev *gxp, uint core; /* Program VID only when cores are managed by us. */ - if (gxp_is_direct_mode(gxp) || gxp_core_boot) { + if (gxp_is_direct_mode(gxp) || gxp_core_boot(gxp)) { pasid = iommu_aux_get_pasid(domain, gxp->dev); for (core = 0; core < GXP_NUM_CORES; core++) if (BIT(core) & core_list) { @@ -90,8 +90,7 @@ static int gxp_dma_ssmt_program(struct gxp_dev *gxp, gxp_ssmt_set_core_vid(&mgr->ssmt, core, pasid); } } else { - for (core = 0; core < GXP_NUM_CORES; core++) - gxp_ssmt_set_core_bypass(&mgr->ssmt, core); + gxp_ssmt_set_bypass(&mgr->ssmt); } return 0; } @@ -323,6 +322,9 @@ int gxp_dma_map_core_resources(struct gxp_dev *gxp, uint i; struct iommu_domain *domain = gdomain->domain; + if (!gxp_is_direct_mode(gxp)) + return 0; + ret = gxp_map_csrs(gxp, domain, &gxp->regs); if (ret) goto err; @@ -377,6 +379,9 @@ void gxp_dma_unmap_core_resources(struct gxp_dev *gxp, uint i; struct iommu_domain *domain = gdomain->domain; + if (!gxp_is_direct_mode(gxp)) + return; + /* Only unmap the TPU mailboxes if they were found on probe */ if (gxp->tpu_dev.mbx_paddr) { for (i = 0; i < GXP_NUM_CORES; i++) { diff --git a/gxp-firmware-data.c b/gxp-firmware-data.c index 4a35ccc..841e80e 100644 --- a/gxp-firmware-data.c +++ b/gxp-firmware-data.c @@ -5,93 +5,30 @@ * Copyright (C) 2021 Google LLC */ -#include <linux/bitops.h> -#include <linux/dma-mapping.h> -#include <linux/genalloc.h> +#include <linux/slab.h> +#include "gxp-config.h" #include "gxp-debug-dump.h" #include "gxp-firmware-data.h" #include "gxp-firmware.h" /* gxp_core_boot */ #include "gxp-host-device-structs.h" #include "gxp-internal.h" -#include "gxp-range-alloc.h" #include "gxp-vd.h" #include "gxp.h" -/* - * The minimum alignment order (power of 2) of allocations in the firmware data - * region. - */ -#define FW_DATA_STORAGE_ORDER 3 - /* A byte pattern to pre-populate the FW region with */ #define FW_DATA_DEBUG_PATTERN 0x66 -/* IDs for dedicated doorbells used by some system components */ -#define DOORBELL_ID_CORE_WAKEUP(__core__) (0 + __core__) - -/* IDs for dedicated sync barriers used by some system components */ -#define SYNC_BARRIER_ID_UART 1 - /* Default application parameters */ #define DEFAULT_APP_ID 1 -#define DEFAULT_APP_USER_MEM_SIZE (120 * 1024) -#define DEFAULT_APP_USER_MEM_ALIGNMENT 8 -#define DEFAULT_APP_THREAD_COUNT 2 -#define DEFAULT_APP_TCM_PER_BANK (100 * 1024) -#define DEFAULT_APP_USER_DOORBELL_COUNT 2 -#define DEFAULT_APP_USER_BARRIER_COUNT 2 - -/* Core-to-core mailbox communication constants */ -#define CORE_TO_CORE_MBX_CMD_COUNT 10 -#define CORE_TO_CORE_MBX_RSP_COUNT 10 - -/* A block allocator managing and partitioning a memory region for device use */ -struct fw_memory_allocator { - struct gen_pool *pool; - struct gxp_dev *gxp; - void *base_host_addr; - uint32_t base_device_addr; -}; - -/* A memory region allocated for device use */ -struct fw_memory { - void *host_addr; - uint32_t device_addr; - size_t sz; -}; /* * Holds information about system-wide HW and memory resources given to the FWs * of GXP devices. */ struct gxp_fw_data_manager { - /* Host-side pointers for book keeping */ - void *fw_data_virt; - struct gxp_system_descriptor *system_desc; - - /* Doorbells allocator and reserved doorbell IDs */ - struct range_alloc *doorbell_allocator; - int core_wakeup_doorbells[GXP_NUM_WAKEUP_DOORBELLS]; - int semaphore_doorbells[GXP_NUM_CORES]; - - /* Sync barriers allocator and reserved sync barrier IDs */ - struct range_alloc *sync_barrier_allocator; - int uart_sync_barrier; - int timer_regions_barrier; - int watchdog_region_barrier; - int uart_region_barrier; - int doorbell_regions_barrier; - int sync_barrier_regions_barrier; - int semaphores_regions_barrier; - - /* System-wide device memory resources */ - struct fw_memory_allocator *allocator; - struct fw_memory sys_desc_mem; - struct fw_memory wdog_mem; - struct fw_memory core_telemetry_mem; - struct fw_memory debug_dump_mem; - + /* Cached core telemetry descriptors. */ + struct gxp_core_telemetry_descriptor core_telemetry_desc; /* * A host-view of the System configuration descriptor. This same desc * is provided to all VDs and all cores. This is the R/O section. @@ -104,477 +41,6 @@ struct gxp_fw_data_manager { struct gxp_system_descriptor_rw *sys_desc_rw; }; -/* A container holding information for a single GXP application. */ -struct app_metadata { - struct gxp_fw_data_manager *mgr; - struct gxp_virtual_device *vd; - uint application_id; - uint core_count; - uint core_list; /* bitmap of cores allocated to this app */ - - /* Per-app doorbell IDs */ - int user_doorbells_count; - int *user_doorbells; - - /* Per-app sync barrier IDs */ - int user_barriers_count; - int *user_barriers; - - /* Per-app memory regions */ - struct fw_memory user_mem; - struct fw_memory doorbells_mem; - struct fw_memory sync_barriers_mem; - struct fw_memory semaphores_mem; - struct fw_memory cores_mem; - struct fw_memory core_cmd_queues_mem[GXP_NUM_CORES]; - struct fw_memory core_rsp_queues_mem[GXP_NUM_CORES]; - struct fw_memory app_mem; -}; - -static struct fw_memory_allocator *mem_alloc_create(struct gxp_dev *gxp, - void *host_base, - uint32_t device_base, - size_t size) -{ - struct fw_memory_allocator *allocator; - int ret = 0; - - allocator = kzalloc(sizeof(*allocator), GFP_KERNEL); - if (!allocator) - return ERR_PTR(-ENOMEM); - - /* - * Use a genpool to allocate and free chunks of the virtual address - * space reserved for FW data. The genpool doesn't use the passed - * addresses internally to access any data, thus it is safe to use it to - * manage memory that the host may not be able to access directly. - * The allocator also records the host-side address so that the code - * here can access and populate data in this region. - */ - allocator->gxp = gxp; - allocator->pool = gen_pool_create(FW_DATA_STORAGE_ORDER, /*nid=*/-1); - if (!allocator->pool) { - dev_err(gxp->dev, "Failed to create memory pool\n"); - kfree(allocator); - return ERR_PTR(-ENOMEM); - } - - ret = gen_pool_add(allocator->pool, device_base, size, /*nid=*/-1); - if (ret) { - dev_err(gxp->dev, "Failed to add memory to pool (ret = %d)\n", - ret); - gen_pool_destroy(allocator->pool); - kfree(allocator); - return ERR_PTR(ret); - } - allocator->base_host_addr = host_base; - allocator->base_device_addr = device_base; - - return allocator; -} - -static int mem_alloc_allocate(struct fw_memory_allocator *allocator, - struct fw_memory *mem, size_t size, - uint8_t alignment) -{ - struct genpool_data_align data = { .align = alignment }; - uint32_t dev_addr; - - dev_addr = gen_pool_alloc_algo(allocator->pool, size, - gen_pool_first_fit_align, &data); - if (!dev_addr) - return -ENOMEM; - - mem->host_addr = allocator->base_host_addr + - (dev_addr - allocator->base_device_addr); - mem->device_addr = dev_addr; - mem->sz = size; - - return 0; -} - -static void mem_alloc_free(struct fw_memory_allocator *allocator, - struct fw_memory *mem) -{ - gen_pool_free(allocator->pool, mem->device_addr, mem->sz); -} - -static void mem_alloc_destroy(struct fw_memory_allocator *allocator) -{ - WARN_ON(gen_pool_avail(allocator->pool) != - gen_pool_size(allocator->pool)); - gen_pool_destroy(allocator->pool); - kfree(allocator); -} - -static struct fw_memory init_doorbells(struct app_metadata *app) -{ - struct gxp_doorbells_descriptor *db_region; - struct fw_memory mem; - uint32_t mem_size; - uint32_t doorbell_count; - int i; - - doorbell_count = app->user_doorbells_count; - mem_size = sizeof(*db_region) + - doorbell_count * sizeof(db_region->doorbells[0]); - - mem_alloc_allocate(app->mgr->allocator, &mem, mem_size, - __alignof__(struct gxp_doorbells_descriptor)); - - db_region = mem.host_addr; - db_region->application_id = app->application_id; - db_region->protection_barrier = app->mgr->doorbell_regions_barrier; - db_region->num_items = doorbell_count; - for (i = 0; i < doorbell_count; i++) { - db_region->doorbells[i].users_count = 0; - db_region->doorbells[i].hw_doorbell_idx = - app->user_doorbells[i]; - } - - return mem; -} - -static struct fw_memory init_sync_barriers(struct app_metadata *app) -{ - struct gxp_sync_barriers_descriptor *sb_region; - struct fw_memory mem; - uint32_t mem_size; - uint32_t barrier_count; - int i; - - barrier_count = app->user_barriers_count; - mem_size = sizeof(*sb_region) + - barrier_count * sizeof(sb_region->barriers[0]); - - mem_alloc_allocate(app->mgr->allocator, &mem, mem_size, - __alignof__(struct gxp_sync_barriers_descriptor)); - - sb_region = mem.host_addr; - sb_region->application_id = app->application_id; - sb_region->protection_barrier = app->mgr->sync_barrier_regions_barrier; - sb_region->num_items = barrier_count; - for (i = 0; i < barrier_count; i++) { - sb_region->barriers[i].users_count = 0; - sb_region->barriers[i].hw_barrier_idx = app->user_barriers[i]; - } - - return mem; -} - -static struct fw_memory init_watchdog(struct gxp_fw_data_manager *mgr) -{ - struct gxp_watchdog_descriptor *wd_region; - struct fw_memory mem; - - mem_alloc_allocate(mgr->allocator, &mem, sizeof(*wd_region), - __alignof__(struct gxp_watchdog_descriptor)); - - wd_region = mem.host_addr; - wd_region->protection_barrier = mgr->watchdog_region_barrier; - wd_region->target_value = 0; - wd_region->participating_cores = 0; - wd_region->responded_cores = 0; - wd_region->tripped = 0; - - return mem; -} - -static struct fw_memory init_core_telemetry(struct gxp_fw_data_manager *mgr) -{ - struct gxp_core_telemetry_descriptor *tel_region; - struct fw_memory mem; - - mem_alloc_allocate(mgr->allocator, &mem, sizeof(*tel_region), - __alignof__(struct gxp_core_telemetry_descriptor)); - - tel_region = mem.host_addr; - - /* - * Core telemetry is disabled for now. - * Subsuequent calls to the FW data module can be used to populate or - * depopulate the descriptor pointers on demand. - */ - memset(tel_region, 0x00, sizeof(*tel_region)); - - return mem; -} - -static struct fw_memory init_debug_dump(struct gxp_dev *gxp) -{ - struct fw_memory mem; - - if (gxp->debug_dump_mgr) { - mem.host_addr = gxp->debug_dump_mgr->buf.vaddr; - mem.device_addr = gxp->debug_dump_mgr->buf.dsp_addr; - mem.sz = gxp->debug_dump_mgr->buf.size; - } else { - mem.host_addr = 0; - mem.device_addr = 0; - mem.sz = 0; - } - - return mem; -} - -static struct fw_memory init_app_user_memory(struct app_metadata *app, - int memory_size) -{ - struct fw_memory mem; - - mem_alloc_allocate(app->mgr->allocator, &mem, memory_size, - DEFAULT_APP_USER_MEM_ALIGNMENT); - - return mem; -} - -static struct fw_memory init_app_semaphores(struct app_metadata *app) -{ - struct gxp_semaphores_descriptor *sm_region; - struct fw_memory mem; - uint32_t mem_size; - uint32_t semaphore_count; - int core; - int i; - - semaphore_count = NUM_SYSTEM_SEMAPHORES; - mem_size = sizeof(*sm_region) + - semaphore_count * sizeof(sm_region->semaphores[0]); - - mem_alloc_allocate(app->mgr->allocator, &mem, mem_size, - __alignof__(struct gxp_semaphores_descriptor)); - - sm_region = mem.host_addr; - sm_region->application_id = app->application_id; - sm_region->protection_barrier = app->mgr->semaphores_regions_barrier; - - core = 0; - for (i = 0; i < GXP_NUM_CORES; i++) { - if (app->core_list & BIT(i)) - sm_region->wakeup_doorbells[core++] = - app->mgr->semaphore_doorbells[i]; - sm_region->woken_pending_semaphores[i] = 0; - } - - sm_region->num_items = semaphore_count; - for (i = 0; i < semaphore_count; i++) { - sm_region->semaphores[i].users_count = 0; - sm_region->semaphores[i].count = 0; - sm_region->semaphores[i].waiters = 0; - } - - return mem; -} - -static struct fw_memory init_app_cores(struct app_metadata *app) -{ - struct gxp_cores_descriptor *cd_region; - struct gxp_queue_info *q_info; - struct fw_memory mem; - uint32_t mem_size; - int semaphore_id; - int core_count; - int i; - const int cmd_queue_items = CORE_TO_CORE_MBX_CMD_COUNT; - const int resp_queue_items = CORE_TO_CORE_MBX_RSP_COUNT; - - /* Core info structures. */ - core_count = app->core_count; - mem_size = - sizeof(*cd_region) + core_count * sizeof(cd_region->cores[0]); - - mem_alloc_allocate(app->mgr->allocator, &mem, mem_size, - __alignof__(struct gxp_cores_descriptor)); - - cd_region = mem.host_addr; - cd_region->num_items = core_count; - - /* Command and response queues. */ - semaphore_id = 0; - for (i = 0; i < core_count; i++) { - /* Allocate per-core command queue storage. */ - mem_size = cmd_queue_items * - sizeof(struct gxp_core_to_core_command); - mem_alloc_allocate( - app->mgr->allocator, &app->core_cmd_queues_mem[i], - mem_size, __alignof__(struct gxp_core_to_core_command)); - - /* Update per-core command queue info. */ - q_info = &cd_region->cores[i].incoming_commands_queue; - q_info->header.storage = - app->core_cmd_queues_mem[i].device_addr; - q_info->header.head_idx = 0; - q_info->header.tail_idx = 0; - q_info->header.element_size = - sizeof(struct gxp_core_to_core_command); - q_info->header.elements_count = cmd_queue_items; - q_info->access_sem_id = semaphore_id++; - q_info->posted_slots_sem_id = semaphore_id++; - q_info->free_slots_sem_id = semaphore_id++; - - /* Allocate per-core response queue storage. */ - mem_size = resp_queue_items * - sizeof(struct gxp_core_to_core_response); - mem_alloc_allocate( - app->mgr->allocator, &app->core_rsp_queues_mem[i], - mem_size, - __alignof__(struct gxp_core_to_core_response)); - - /* Update per-core response queue info. */ - q_info = &cd_region->cores[i].incoming_responses_queue; - q_info->header.storage = - app->core_rsp_queues_mem[i].device_addr; - q_info->header.head_idx = 0; - q_info->header.tail_idx = 0; - q_info->header.element_size = - sizeof(struct gxp_core_to_core_response); - q_info->header.elements_count = resp_queue_items; - q_info->access_sem_id = semaphore_id++; - q_info->posted_slots_sem_id = semaphore_id++; - q_info->free_slots_sem_id = semaphore_id++; - } - - return mem; -} - -static struct fw_memory init_application(struct app_metadata *app) -{ - struct gxp_application_descriptor *app_region; - struct fw_memory mem; - const int user_mem_size = DEFAULT_APP_USER_MEM_SIZE; - - /* App's system memory. */ - app->user_mem = init_app_user_memory(app, user_mem_size); - - /* App's doorbells region. */ - app->doorbells_mem = init_doorbells(app); - - /* App's sync barriers region. */ - app->sync_barriers_mem = init_sync_barriers(app); - - /* App's semaphores region. */ - app->semaphores_mem = init_app_semaphores(app); - - /* App's cores info and core-to-core queues. */ - app->cores_mem = init_app_cores(app); - - /* App's descriptor. */ - mem_alloc_allocate(app->mgr->allocator, &mem, sizeof(*app_region), - __alignof__(struct gxp_application_descriptor)); - app_region = mem.host_addr; - app_region->application_id = app->application_id; - app_region->core_count = app->core_count; - app_region->cores_mask = app->core_list; - app_region->threads_count = DEFAULT_APP_THREAD_COUNT; - app_region->tcm_memory_per_bank = DEFAULT_APP_TCM_PER_BANK; - app_region->system_memory_size = user_mem_size; - app_region->system_memory_addr = app->user_mem.device_addr; - app_region->doorbells_dev_addr = app->doorbells_mem.device_addr; - app_region->sync_barriers_dev_addr = app->sync_barriers_mem.device_addr; - app_region->semaphores_dev_addr = app->semaphores_mem.device_addr; - app_region->cores_info_dev_addr = app->cores_mem.device_addr; - - return mem; -} - -static struct app_metadata *gxp_fw_data_create_app_legacy(struct gxp_dev *gxp, - uint core_list) -{ - struct gxp_fw_data_manager *mgr = gxp->data_mgr; - struct app_metadata *app; - void *err; - int i; - - app = kzalloc(sizeof(*app), GFP_KERNEL); - if (!app) - return ERR_PTR(-ENOMEM); - - /* Create resource and memory allocations for new app */ - app->mgr = mgr; - app->application_id = DEFAULT_APP_ID; - app->core_count = hweight_long(core_list); - app->core_list = core_list; - - /* User doorbells */ - app->user_doorbells_count = DEFAULT_APP_USER_DOORBELL_COUNT; - app->user_doorbells = - kcalloc(app->user_doorbells_count, sizeof(int), GFP_KERNEL); - if (!app->user_doorbells) { - err = ERR_PTR(-ENOMEM); - goto err_user_doorbells; - } - - for (i = 0; i < app->user_doorbells_count; i++) { - range_alloc_get_any(mgr->doorbell_allocator, - &app->user_doorbells[i]); - } - - /* User sync barrier */ - app->user_barriers_count = DEFAULT_APP_USER_BARRIER_COUNT; - app->user_barriers = - kcalloc(app->user_barriers_count, sizeof(int), GFP_KERNEL); - if (!app->user_barriers) { - err = ERR_PTR(-ENOMEM); - goto err_user_barriers; - } - - for (i = 0; i < app->user_barriers_count; i++) { - range_alloc_get_any(mgr->sync_barrier_allocator, - &app->user_barriers[i]); - } - - /* Application region. */ - app->app_mem = init_application(app); - for (i = 0; i < GXP_NUM_CORES; i++) { - if (core_list & BIT(i)) { - mgr->system_desc->app_descriptor_dev_addr[i] = - app->app_mem.device_addr; - } - } - - return app; - -err_user_barriers: - for (i = 0; i < app->user_doorbells_count; i++) - range_alloc_put(mgr->doorbell_allocator, - app->user_doorbells[i]); - kfree(app->user_doorbells); -err_user_doorbells: - kfree(app); - - return err; -} - -static void gxp_fw_data_destroy_app_legacy(struct gxp_dev *gxp, - struct app_metadata *app) -{ - struct gxp_fw_data_manager *mgr = gxp->data_mgr; - int i; - - for (i = 0; i < app->user_doorbells_count; i++) - range_alloc_put(mgr->doorbell_allocator, - app->user_doorbells[i]); - kfree(app->user_doorbells); - - for (i = 0; i < app->user_barriers_count; i++) - range_alloc_put(mgr->sync_barrier_allocator, - app->user_barriers[i]); - kfree(app->user_barriers); - - mem_alloc_free(mgr->allocator, &app->user_mem); - mem_alloc_free(mgr->allocator, &app->doorbells_mem); - mem_alloc_free(mgr->allocator, &app->sync_barriers_mem); - mem_alloc_free(mgr->allocator, &app->semaphores_mem); - mem_alloc_free(mgr->allocator, &app->cores_mem); - for (i = 0; i < app->core_count; i++) { - mem_alloc_free(mgr->allocator, &app->core_cmd_queues_mem[i]); - mem_alloc_free(mgr->allocator, &app->core_rsp_queues_mem[i]); - } - mem_alloc_free(mgr->allocator, &app->app_mem); - - kfree(app); -} - /* * Here assumes sys_cfg contains gxp_system_descriptor_ro in the first page and * gxp_system_descriptor_rw in the second page. @@ -584,7 +50,7 @@ static void set_system_cfg_region(struct gxp_dev *gxp, void *sys_cfg) struct gxp_system_descriptor_ro *des_ro = sys_cfg; struct gxp_system_descriptor_rw *des_rw = sys_cfg + PAGE_SIZE; struct gxp_core_telemetry_descriptor *descriptor = - gxp->data_mgr->core_telemetry_mem.host_addr; + &gxp->data_mgr->core_telemetry_desc; struct telemetry_descriptor_ro *tel_ro; struct telemetry_descriptor_rw *tel_rw; struct core_telemetry_descriptor *tel_des; @@ -620,31 +86,23 @@ static void set_system_cfg_region(struct gxp_dev *gxp, void *sys_cfg) gxp->data_mgr->sys_desc_rw = des_rw; } -static struct app_metadata * -_gxp_fw_data_create_app(struct gxp_dev *gxp, struct gxp_virtual_device *vd) +static void _gxp_fw_data_populate_vd_cfg(struct gxp_dev *gxp, + struct gxp_virtual_device *vd) { - struct app_metadata *app; struct gxp_host_control_region *core_cfg; struct gxp_job_descriptor job; struct gxp_vd_descriptor *vd_desc; int i; - /* - * If we are able to know where sys_cfg's virt is on init() then we - * don't need this here, but to keep compatibility with - * !use_per_vd_config, we keep gxp_fw_data_init() doing the - * initialization of legacy mode, and have here copy the values to the - * config region. - */ - if (vd->vdid == 1) - set_system_cfg_region(gxp, vd->sys_cfg.vaddr); - app = kzalloc(sizeof(*app), GFP_KERNEL); - if (!app) - return ERR_PTR(-ENOMEM); - - if (!gxp_core_boot) { + if (!gxp_core_boot(gxp)) { dev_info(gxp->dev, "Skip setting VD and core CFG"); - return app; + return; + } + if (!vd->vd_cfg.vaddr || !vd->core_cfg.vaddr) { + dev_warn( + gxp->dev, + "Missing VD and core CFG in image config, firmware is not bootable\n"); + return; } /* Set up VD config region. */ vd_desc = vd->vd_cfg.vaddr; @@ -673,213 +131,61 @@ _gxp_fw_data_create_app(struct gxp_dev *gxp, struct gxp_virtual_device *vd) vd->core_cfg.size / GXP_NUM_CORES * i; core_cfg->job_descriptor = job; } - - return app; } -static void _gxp_fw_data_destroy_app(struct gxp_dev *gxp, - struct app_metadata *app) +static struct core_telemetry_descriptor * +gxp_fw_data_get_core_telemetry_descriptor(struct gxp_dev *gxp, u8 type) { - kfree(app); + struct gxp_core_telemetry_descriptor *descriptor = + &gxp->data_mgr->core_telemetry_desc; + + if (type == GXP_TELEMETRY_TYPE_LOGGING) + return descriptor->per_core_loggers; + else if (type == GXP_TELEMETRY_TYPE_TRACING) + return descriptor->per_core_tracers; + else + return ERR_PTR(-EINVAL); } int gxp_fw_data_init(struct gxp_dev *gxp) { struct gxp_fw_data_manager *mgr; - int res; - int i; + void *virt; mgr = devm_kzalloc(gxp->dev, sizeof(*mgr), GFP_KERNEL); if (!mgr) return -ENOMEM; - gxp->data_mgr = mgr; - /* - * TODO (b/200169232) Using memremap until devm_memremap is added to - * the GKI ABI - */ - mgr->fw_data_virt = memremap(gxp->fwdatabuf.paddr, gxp->fwdatabuf.size, - MEMREMAP_WC); + virt = memremap(gxp->fwdatabuf.paddr, gxp->fwdatabuf.size, MEMREMAP_WC); - if (IS_ERR_OR_NULL(mgr->fw_data_virt)) { + if (IS_ERR_OR_NULL(virt)) { dev_err(gxp->dev, "Failed to map fw data region\n"); - res = -ENODEV; - goto err; - } - gxp->fwdatabuf.vaddr = mgr->fw_data_virt; - - /* Instantiate the doorbells allocator with all doorbells */ - mgr->doorbell_allocator = - range_alloc_create(/*start=*/0, DOORBELL_COUNT); - if (IS_ERR(mgr->doorbell_allocator)) { - dev_err(gxp->dev, "Failed to create doorbells allocator\n"); - res = PTR_ERR(mgr->doorbell_allocator); - mgr->doorbell_allocator = NULL; - goto err; - } - - /* Instantiate the sync barriers allocator with all sync barriers */ - mgr->sync_barrier_allocator = - range_alloc_create(/*start=*/0, SYNC_BARRIER_COUNT); - if (IS_ERR(mgr->sync_barrier_allocator)) { - dev_err(gxp->dev, "Failed to create sync barriers allocator\n"); - res = PTR_ERR(mgr->sync_barrier_allocator); - mgr->sync_barrier_allocator = NULL; - goto err; - } - - /* Allocate doorbells */ - - /* Pinned: Cores wakeup doorbell */ - for (i = 0; i < GXP_NUM_WAKEUP_DOORBELLS; i++) { - mgr->core_wakeup_doorbells[i] = DOORBELL_ID_CORE_WAKEUP(i); - res = range_alloc_get(mgr->doorbell_allocator, - mgr->core_wakeup_doorbells[i]); - if (res) - goto err; - } - - /* Semaphores operation doorbells */ - for (i = 0; i < GXP_NUM_CORES; i++) { - range_alloc_get_any(mgr->doorbell_allocator, - &mgr->semaphore_doorbells[i]); - } - - /* Allocate sync barriers */ - - /* Pinned: UART sync barrier */ - mgr->uart_sync_barrier = SYNC_BARRIER_ID_UART; - mgr->uart_region_barrier = SYNC_BARRIER_ID_UART; - res = range_alloc_get(mgr->sync_barrier_allocator, - mgr->uart_sync_barrier); - if (res) - goto err; - - /* Doorbell regions for all apps */ - res = range_alloc_get_any(mgr->sync_barrier_allocator, - &mgr->doorbell_regions_barrier); - if (res) - goto err; - - /* Sync barrier regions for all apps */ - res = range_alloc_get_any(mgr->sync_barrier_allocator, - &mgr->sync_barrier_regions_barrier); - if (res) - goto err; - - /* Timer regions for all apps */ - res = range_alloc_get_any(mgr->sync_barrier_allocator, - &mgr->timer_regions_barrier); - if (res) - goto err; - - /* Watchdog regions for all apps */ - res = range_alloc_get_any(mgr->sync_barrier_allocator, - &mgr->watchdog_region_barrier); - if (res) - goto err; - - /* Semaphore regions for all apps */ - res = range_alloc_get_any(mgr->sync_barrier_allocator, - &mgr->semaphores_regions_barrier); - if (res) - goto err; - - /* Shared firmware data memory region */ - mgr->allocator = - mem_alloc_create(gxp, mgr->fw_data_virt, gxp->fwdatabuf.daddr, - gxp->fwdatabuf.size); - if (IS_ERR(mgr->allocator)) { - dev_err(gxp->dev, - "Failed to create the FW data memory allocator\n"); - res = PTR_ERR(mgr->allocator); - mgr->allocator = NULL; - goto err; + return -ENODEV; } + gxp->fwdatabuf.vaddr = virt; /* Populate the region with a pre-defined pattern. */ - memset(mgr->fw_data_virt, FW_DATA_DEBUG_PATTERN, gxp->fwdatabuf.size); - - /* Allocate the root system descriptor from the region */ - mem_alloc_allocate(mgr->allocator, &mgr->sys_desc_mem, - sizeof(struct gxp_system_descriptor), - __alignof__(struct gxp_system_descriptor)); - mgr->system_desc = mgr->sys_desc_mem.host_addr; - - /* Allocate the watchdog descriptor from the region */ - mgr->wdog_mem = init_watchdog(mgr); - mgr->system_desc->watchdog_dev_addr = mgr->wdog_mem.device_addr; - - /* Allocate the descriptor for device-side core telemetry */ - mgr->core_telemetry_mem = init_core_telemetry(mgr); - mgr->system_desc->core_telemetry_dev_addr = - mgr->core_telemetry_mem.device_addr; - - /* Set the debug dump region parameters if available */ - mgr->debug_dump_mem = init_debug_dump(gxp); - mgr->system_desc->debug_dump_dev_addr = mgr->debug_dump_mem.device_addr; - - return res; - -err: - range_alloc_destroy(mgr->sync_barrier_allocator); - range_alloc_destroy(mgr->doorbell_allocator); - devm_kfree(gxp->dev, mgr); - return res; -} - -void *gxp_fw_data_create_app(struct gxp_dev *gxp, struct gxp_virtual_device *vd) -{ - struct app_metadata *app; - - if (gxp_fw_data_use_per_vd_config(vd)) - app = _gxp_fw_data_create_app(gxp, vd); - else - app = gxp_fw_data_create_app_legacy(gxp, vd->core_list); - - if (IS_ERR(app)) - return app; - app->vd = vd; - - return app; -} - -void gxp_fw_data_destroy_app(struct gxp_dev *gxp, void *application) -{ - struct app_metadata *app = application; + memset(virt, FW_DATA_DEBUG_PATTERN, gxp->fwdatabuf.size); + gxp->data_mgr = mgr; - if (!app) - return; - if (gxp_fw_data_use_per_vd_config(app->vd)) - return _gxp_fw_data_destroy_app(gxp, app); - return gxp_fw_data_destroy_app_legacy(gxp, app); + return 0; } void gxp_fw_data_destroy(struct gxp_dev *gxp) { struct gxp_fw_data_manager *mgr = gxp->data_mgr; - if (!mgr) - return; - - mem_alloc_free(mgr->allocator, &mgr->core_telemetry_mem); - mem_alloc_free(mgr->allocator, &mgr->wdog_mem); - mem_alloc_free(mgr->allocator, &mgr->sys_desc_mem); - mem_alloc_destroy(mgr->allocator); - - range_alloc_destroy(mgr->sync_barrier_allocator); - range_alloc_destroy(mgr->doorbell_allocator); + if (gxp->fwdatabuf.vaddr) + memunmap(gxp->fwdatabuf.vaddr); - /* TODO (b/200169232) Remove this once we're using devm_memremap */ - if (mgr->fw_data_virt) { - memunmap(mgr->fw_data_virt); - mgr->fw_data_virt = NULL; - } + devm_kfree(gxp->dev, mgr); + gxp->data_mgr = NULL; +} - if (gxp->data_mgr) { - devm_kfree(gxp->dev, gxp->data_mgr); - gxp->data_mgr = NULL; - } +void gxp_fw_data_populate_vd_cfg(struct gxp_dev *gxp, struct gxp_virtual_device *vd) +{ + if (gxp_fw_data_use_per_vd_config(vd)) + _gxp_fw_data_populate_vd_cfg(gxp, vd); } int gxp_fw_data_set_core_telemetry_descriptors(struct gxp_dev *gxp, u8 type, @@ -887,18 +193,13 @@ int gxp_fw_data_set_core_telemetry_descriptors(struct gxp_dev *gxp, u8 type, struct gxp_coherent_buf *buffers, u32 per_buffer_size) { - struct gxp_core_telemetry_descriptor *descriptor = - gxp->data_mgr->core_telemetry_mem.host_addr; struct core_telemetry_descriptor *core_descriptors; uint core; bool enable; - if (type == GXP_TELEMETRY_TYPE_LOGGING) - core_descriptors = descriptor->per_core_loggers; - else if (type == GXP_TELEMETRY_TYPE_TRACING) - core_descriptors = descriptor->per_core_tracers; - else - return -EINVAL; + core_descriptors = gxp_fw_data_get_core_telemetry_descriptor(gxp, type); + if (IS_ERR(core_descriptors)) + return PTR_ERR(core_descriptors); enable = (host_status & GXP_CORE_TELEMETRY_HOST_STATUS_ENABLED); @@ -927,27 +228,13 @@ int gxp_fw_data_set_core_telemetry_descriptors(struct gxp_dev *gxp, u8 type, return 0; } -static u32 -gxp_fw_data_get_core_telemetry_device_status_legacy(struct gxp_dev *gxp, - uint core, u8 type) +u32 gxp_fw_data_get_core_telemetry_device_status(struct gxp_dev *gxp, uint core, + u8 type) { - struct gxp_core_telemetry_descriptor *descriptor = - gxp->data_mgr->core_telemetry_mem.host_addr; + struct gxp_system_descriptor_rw *des_rw = gxp->data_mgr->sys_desc_rw; - switch (type) { - case GXP_TELEMETRY_TYPE_LOGGING: - return descriptor->per_core_loggers[core].device_status; - case GXP_TELEMETRY_TYPE_TRACING: - return descriptor->per_core_tracers[core].device_status; - default: + if (core >= GXP_NUM_CORES) return 0; - } -} - -static u32 _gxp_fw_data_get_core_telemetry_device_status(struct gxp_dev *gxp, - uint core, u8 type) -{ - struct gxp_system_descriptor_rw *des_rw = gxp->data_mgr->sys_desc_rw; switch (type) { case GXP_TELEMETRY_TYPE_LOGGING: @@ -961,18 +248,32 @@ static u32 _gxp_fw_data_get_core_telemetry_device_status(struct gxp_dev *gxp, } } -u32 gxp_fw_data_get_core_telemetry_device_status(struct gxp_dev *gxp, uint core, - u8 type) +struct gxp_mapped_resource gxp_fw_data_resource(struct gxp_dev *gxp) { - if (core >= GXP_NUM_CORES) - return 0; + /* + * For direct mode, the config regions are programmed by host (us); for + * MCU mode, the config regions are programmed by MCU. + */ + if (gxp_is_direct_mode(gxp)) { + struct gxp_mapped_resource tmp = gxp->fwdatabuf; - if (gxp->firmware_mgr->img_cfg.config_version >= - FW_DATA_PROTOCOL_PER_VD_CONFIG) { - return _gxp_fw_data_get_core_telemetry_device_status(gxp, core, - type); + /* Leave the first piece be used for gxp_fw_data_init() */ + tmp.vaddr += tmp.size / 2; + tmp.paddr += tmp.size / 2; + return tmp; } else { - return gxp_fw_data_get_core_telemetry_device_status_legacy( - gxp, core, type); + return gxp->shared_buf; } } + +void *gxp_fw_data_system_cfg(struct gxp_dev *gxp) +{ + /* Use the end of the shared region for system cfg. */ + return gxp_fw_data_resource(gxp).vaddr + GXP_SHARED_BUFFER_SIZE - + GXP_FW_DATA_SYSCFG_SIZE; +} + +void gxp_fw_data_populate_system_config(struct gxp_dev *gxp) +{ + set_system_cfg_region(gxp, gxp_fw_data_system_cfg(gxp)); +} diff --git a/gxp-firmware-data.h b/gxp-firmware-data.h index e2296bd..89bf9e4 100644 --- a/gxp-firmware-data.h +++ b/gxp-firmware-data.h @@ -9,10 +9,14 @@ #ifndef __GXP_FIRMWARE_DATA_H__ #define __GXP_FIRMWARE_DATA_H__ +#include <linux/sizes.h> + #include "gxp-dma.h" #include "gxp-internal.h" #include "gxp-vd.h" +#define GXP_FW_DATA_SYSCFG_SIZE SZ_8K + enum gxp_fw_data_protocol { /* Use the per-VD configuration region. */ FW_DATA_PROTOCOL_PER_VD_CONFIG = 2, @@ -31,35 +35,20 @@ enum gxp_fw_data_protocol { int gxp_fw_data_init(struct gxp_dev *gxp); /** - * gxp_fw_data_create_app() - Allocates HW and memory resources needed to create - * a GXP device application (1:1 with a GXP driver - * virtual device) used by the specified physical - * cores. - * @gxp: The parent GXP device - * @vd: The virtual device this app is being created for - * - * Return: - * ptr - A pointer of the newly created application handle, an error pointer - * (PTR_ERR) otherwise. - * -ENOMEM - Insufficient memory to create the application - */ -void *gxp_fw_data_create_app(struct gxp_dev *gxp, - struct gxp_virtual_device *vd); - -/** - * gxp_fw_data_destroy_app() - Deallocates the HW and memory resources used by - * the specified application. + * gxp_fw_data_destroy() - Destroys the FW data manager submodule and free all + * its resources. * @gxp: The parent GXP device - * @application: The handle to the application to deallocate */ -void gxp_fw_data_destroy_app(struct gxp_dev *gxp, void *application); +void gxp_fw_data_destroy(struct gxp_dev *gxp); /** - * gxp_fw_data_destroy() - Destroys the FW data manager submodule and free all - * its resources. + * gxp_fw_data_populate_vd_cfg() - Sets up the resources to VD's per-core config + * regions and per-VD config regions. * @gxp: The parent GXP device + * @vd: The virtual device to be populated for */ -void gxp_fw_data_destroy(struct gxp_dev *gxp); +void gxp_fw_data_populate_vd_cfg(struct gxp_dev *gxp, + struct gxp_virtual_device *vd); /** * gxp_fw_data_set_core_telemetry_descriptors() - Set new logging or tracing @@ -106,4 +95,37 @@ static inline bool gxp_fw_data_use_per_vd_config(struct gxp_virtual_device *vd) return vd->config_version >= FW_DATA_PROTOCOL_PER_VD_CONFIG; } +/** + * gxp_fw_data_resource() - Returns the resource of data region for host<->core + * communication. + * @gxp: The GXP device + * + * This function requires either @gxp->fwdatabuf or @gxp->shared_buf be + * initialized, so it couldn't be called during device probe time. + * + * Return: The resource. + */ +struct gxp_mapped_resource gxp_fw_data_resource(struct gxp_dev *gxp); + +/** + * gxp_fw_data_system_cfg() - Returns the pointer to the system config region. + * @gxp: The GXP device + * + * This function requires either @gxp->fwdatabuf or @gxp->shared_buf be + * initialized, so it couldn't be called during device probe time. + * + * Return: The pointer. This function never fails. + */ +void *gxp_fw_data_system_cfg(struct gxp_dev *gxp); + +/** + * gxp_fw_data_populate_system_config() - Populate settings onto firmware system + * config region. + * @gxp: The GXP device + * + * This function is expected to be called after "after_probe" in the probe + * procedure since it uses gxp_fw_data_system_cfg(). + */ +void gxp_fw_data_populate_system_config(struct gxp_dev *gxp); + #endif /* __GXP_FIRMWARE_DATA_H__ */ diff --git a/gxp-firmware-loader.c b/gxp-firmware-loader.c new file mode 100644 index 0000000..5f64bd4 --- /dev/null +++ b/gxp-firmware-loader.c @@ -0,0 +1,301 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * GXP firmware loading management. + * + * Copyright (C) 2023 Google LLC + */ + +#include <linux/dma-mapping.h> +#include <linux/slab.h> + +#include <gcip/gcip-common-image-header.h> +#include <gcip/gcip-image-config.h> + +#include "gxp-config.h" +#include "gxp-firmware-loader.h" +#include "gxp-firmware.h" +#include "gxp-internal.h" + +#if GXP_HAS_MCU +#include <linux/gsa/gsa_dsp.h> + +#include "gxp-mcu-firmware.h" +#endif + +#if GXP_HAS_MCU +static int gxp_firmware_loader_gsa_auth(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + int ret; + uint core; + dma_addr_t headers_dma_addr; + void *header_vaddr; + const u8 *data; + struct gxp_mcu_firmware *mcu_fw = gxp_mcu_firmware_of(gxp); + + if (!mcu_fw->is_secure) { + dev_warn( + gxp->dev, + "No need to do firmware authentication with non-secure privilege\n"); + return 0; + } + if (!gxp->gsa_dev) { + dev_warn( + gxp->dev, + "No GSA device available, skipping firmware authentication\n"); + return 0; + } + /* Authenticate MCU firmware */ + header_vaddr = dma_alloc_coherent(gxp->gsa_dev, GCIP_FW_HEADER_SIZE, + &headers_dma_addr, GFP_KERNEL); + if (!header_vaddr) { + dev_err(gxp->dev, + "Failed to allocate coherent memory for header\n"); + return -ENOMEM; + } + memcpy(header_vaddr, mgr->mcu_firmware->data, GCIP_FW_HEADER_SIZE); + ret = gsa_load_dsp_fw_image(gxp->gsa_dev, headers_dma_addr, + mcu_fw->image_buf.paddr); + if (ret) { + dev_err(gxp->dev, "MCU fw GSA authentication fails"); + goto err_load_mcu_fw; + } + + for (core = 0; core < GXP_NUM_CORES; core++) { + data = mgr->core_firmware[core]->data; + /* Authenticate core firmware */ + memcpy(header_vaddr, data, GCIP_FW_HEADER_SIZE); + ret = gsa_load_dsp_fw_image(gxp->gsa_dev, headers_dma_addr, + gxp->fwbufs[core].paddr); + if (ret) { + dev_err(gxp->dev, + "Core %u firmware authentication fails", core); + goto err_load_core_fw; + } + } + dma_free_coherent(gxp->gsa_dev, GCIP_FW_HEADER_SIZE, header_vaddr, + headers_dma_addr); + return 0; +err_load_core_fw: + gsa_unload_dsp_fw_image(gxp->gsa_dev); +err_load_mcu_fw: + dma_free_coherent(gxp->gsa_dev, GCIP_FW_HEADER_SIZE, header_vaddr, + headers_dma_addr); + return ret; +} + +static void gxp_firmware_loader_gsa_unload(struct gxp_dev *gxp) +{ + struct gxp_mcu_firmware *mcu_fw = gxp_mcu_firmware_of(gxp); + + if (mcu_fw->is_secure) + gsa_unload_dsp_fw_image(gxp->gsa_dev); +} +#endif /* GXP_HAS_MCU */ + +int gxp_firmware_loader_init(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr; + + mgr = devm_kzalloc(gxp->dev, sizeof(*mgr), GFP_KERNEL); + if (!mgr) + return -ENOMEM; + gxp->fw_loader_mgr = mgr; + mutex_init(&mgr->lock); + return 0; +} + +void gxp_firmware_loader_destroy(struct gxp_dev *gxp) +{ + gxp_firmware_loader_unload(gxp); +} + +void gxp_firmware_loader_set_core_fw_name(struct gxp_dev *gxp, + const char *fw_name) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + + mutex_lock(&mgr->lock); + mgr->core_firmware_name = kstrdup(fw_name, GFP_KERNEL); + mutex_unlock(&mgr->lock); +} + +char *gxp_firmware_loader_get_core_fw_name(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + char *name; + + mutex_lock(&mgr->lock); + if (mgr->core_firmware_name) + name = kstrdup(mgr->core_firmware_name, GFP_KERNEL); + else + name = kstrdup(DSP_FIRMWARE_DEFAULT_PREFIX, GFP_KERNEL); + mutex_unlock(&mgr->lock); + return name; +} + +/* + * Fetches and records image config of the first core firmware. + */ +static void gxp_firmware_loader_get_core_image_config(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + struct gcip_common_image_header *hdr = + (struct gcip_common_image_header *)mgr->core_firmware[0]->data; + struct gcip_image_config *cfg; + + if (unlikely(mgr->core_firmware[0]->size < GCIP_FW_HEADER_SIZE)) + return; + cfg = get_image_config_from_hdr(hdr); + if (cfg) + mgr->core_img_cfg = *cfg; + else + dev_warn(gxp->dev, + "Core 0 Firmware doesn't have a valid image config"); +} + +/* + * Call this function when mgr->core_firmware have been populated. + * This function sets is_loaded to true. + * + */ +static void gxp_firmware_loader_has_loaded(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + + lockdep_assert_held(&mgr->lock); + gxp_firmware_loader_get_core_image_config(gxp); + mgr->is_loaded = true; +} + +static void gxp_firmware_loader_unload_core_firmware(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + uint core; + + lockdep_assert_held(&mgr->lock); + for (core = 0; core < GXP_NUM_CORES; core++) { + if (mgr->core_firmware[core]) { + release_firmware(mgr->core_firmware[core]); + mgr->core_firmware[core] = NULL; + } + } + kfree(mgr->core_firmware_name); + mgr->core_firmware_name = NULL; +} + +#if GXP_HAS_MCU +static void gxp_firmware_loader_unload_mcu_firmware(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + + lockdep_assert_held(&mgr->lock); + if (!gxp_is_direct_mode(gxp)) { + if (mgr->mcu_firmware) { + gxp_mcu_firmware_unload(gxp, mgr->mcu_firmware); + release_firmware(mgr->mcu_firmware); + mgr->mcu_firmware = NULL; + } + kfree(mgr->mcu_firmware_name); + mgr->mcu_firmware_name = NULL; + } +} +#endif /* GXP_HAS_MCU */ + +static int gxp_firmware_loader_load_locked(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + int ret; + + lockdep_assert_held(&mgr->lock); + ret = gxp_firmware_load_core_firmware(gxp, mgr->core_firmware_name, + mgr->core_firmware); + if (ret) + return ret; + +#if GXP_HAS_MCU + if (!gxp_is_direct_mode(gxp)) { + ret = gxp_mcu_firmware_load(gxp, mgr->mcu_firmware_name, + &mgr->mcu_firmware); + if (ret) + goto err_unload_core; + + ret = gxp_firmware_loader_gsa_auth(gxp); + if (ret) + goto err_unload_mcu; + } +#endif + ret = gxp_firmware_rearrange_elf(gxp, mgr->core_firmware); + if (ret) + goto err_unload; + gxp_firmware_loader_has_loaded(gxp); + return 0; + +err_unload: +#if GXP_HAS_MCU + if (!gxp_is_direct_mode(gxp)) + gxp_firmware_loader_gsa_unload(gxp); +err_unload_mcu: + if (!gxp_is_direct_mode(gxp)) + gxp_firmware_loader_unload_mcu_firmware(gxp); +err_unload_core: +#endif + gxp_firmware_loader_unload_core_firmware(gxp); + return ret; +} + +int gxp_firmware_loader_load_if_needed(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + int ret = 0; + + mutex_lock(&mgr->lock); + if (mgr->is_loaded) + goto out; + ret = gxp_firmware_loader_load_locked(gxp); +out: + mutex_unlock(&mgr->lock); + return ret; +} + +void gxp_firmware_loader_unload(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + + mutex_lock(&mgr->lock); + if (mgr->is_loaded) { +#if GXP_HAS_MCU + gxp_firmware_loader_gsa_unload(gxp); + gxp_firmware_loader_unload_mcu_firmware(gxp); +#endif + gxp_firmware_loader_unload_core_firmware(gxp); + } + mgr->is_loaded = false; + mutex_unlock(&mgr->lock); +} + +#if GXP_HAS_MCU +void gxp_firmware_loader_set_mcu_fw_name(struct gxp_dev *gxp, + const char *fw_name) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + + mutex_lock(&mgr->lock); + mgr->mcu_firmware_name = kstrdup(fw_name, GFP_KERNEL); + mutex_unlock(&mgr->lock); +} + +char *gxp_firmware_loader_get_mcu_fw_name(struct gxp_dev *gxp) +{ + struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr; + char *name; + + mutex_lock(&mgr->lock); + if (mgr->mcu_firmware_name) + name = kstrdup(mgr->mcu_firmware_name, GFP_KERNEL); + else + name = kstrdup(GXP_DEFAULT_MCU_FIRMWARE, GFP_KERNEL); + mutex_unlock(&mgr->lock); + return name; +} +#endif /* GXP_HAS_MCU */ diff --git a/gxp-firmware-loader.h b/gxp-firmware-loader.h new file mode 100644 index 0000000..d081af2 --- /dev/null +++ b/gxp-firmware-loader.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * GXP firmware loading management. + * + * Copyright (C) 2023 Google LLC + */ + +#ifndef __GXP_FIRMWARE_LOADER_H_ +#define __GXP_FIRMWARE_LOADER_H_ + +#include <gcip/gcip-image-config.h> + +#include "gxp-config.h" +#include "gxp-internal.h" + +struct gxp_firmware_loader_manager { + const struct firmware *core_firmware[GXP_NUM_CORES]; + char *core_firmware_name; + /* + * Cached core 0 firmware image config, for easier fetching config entries. + * Not a pointer to the firmware buffer because we want to forcely change the + * privilege level to NS. + * Only valid on the firmware loaded. + */ + struct gcip_image_config core_img_cfg; +#if GXP_HAS_MCU + const struct firmware *mcu_firmware; + char *mcu_firmware_name; +#endif + bool is_loaded; + /* Protects above fields */ + struct mutex lock; +}; + +/* + * Initializes the firmware loader subsystem. + */ +int gxp_firmware_loader_init(struct gxp_dev *gxp); + +/* + * Tears down the firmware loader subsystem. + */ +void gxp_firmware_loader_destroy(struct gxp_dev *gxp); + +/* + * Requests and loads all firmware only if firmware is not loaded. + * + * Returns 0 on success, a negative errno on failure. + */ +int gxp_firmware_loader_load_if_needed(struct gxp_dev *gxp); + +/* + * Unloads firmware. + */ +void gxp_firmware_loader_unload(struct gxp_dev *gxp); + +/* + * Returns a copied core firmware name prefix, the caller needs to release it by + * kfree. + */ +char *gxp_firmware_loader_get_core_fw_name(struct gxp_dev *gxp); + +/* + * Set the core firmware name prefix to be requested in + * `gxp_firmware_loader_load_if_needed()`. + * It's safe for caller to release @fw_name after calling this function. + */ +void gxp_firmware_loader_set_core_fw_name(struct gxp_dev *gxp, + const char *fw_name); +/* + * + * Returns a copied MCU firmware name, the caller needs to release it by + * kfree. + */ +char *gxp_firmware_loader_get_mcu_fw_name(struct gxp_dev *gxp); + +/* + * Set the MCU firmware name to be requested in + * `gxp_firmware_loader_load_if_needed()`. + * It's safe for caller to release @fw_name after calling this function. + */ +void gxp_firmware_loader_set_mcu_fw_name(struct gxp_dev *gxp, + const char *fw_name); + +#endif /* __GXP_FIRMWARE_LOADER_H_ */ diff --git a/gxp-firmware.c b/gxp-firmware.c index b0453d5..d532fdf 100644 --- a/gxp-firmware.c +++ b/gxp-firmware.c @@ -26,6 +26,7 @@ #include "gxp-debug-dump.h" #include "gxp-doorbell.h" #include "gxp-firmware-data.h" +#include "gxp-firmware-loader.h" #include "gxp-firmware.h" #include "gxp-host-device-structs.h" #include "gxp-internal.h" @@ -44,42 +45,6 @@ static int gxp_dsp_fw_auth_disable; module_param_named(dsp_fw_auth_disable, gxp_dsp_fw_auth_disable, int, 0660); -bool gxp_core_boot = true; -module_param_named(core_boot, gxp_core_boot, bool, 0660); - -/* - * Fetches and records image config of the first firmware. - */ -static void gxp_firmware_get_image_config(struct gxp_dev *gxp, - struct gxp_firmware_manager *mgr) -{ - struct gcip_common_image_header *hdr = - (struct gcip_common_image_header *)mgr->firmwares[0]->data; - struct gcip_image_config *cfg; - - if (unlikely(mgr->firmwares[0]->size < FW_HEADER_SIZE)) - return; - cfg = get_image_config_from_hdr(hdr); - if (cfg) - mgr->img_cfg = *cfg; - else - dev_warn(gxp->dev, - "Firmware doesn't have a valid image config"); -} - -/* - * Call this function when mgr->firmwares have been populated. - * This function sets is_firmware_requested to true. - * - * Caller holds mgr->dsp_firmware_lock. - */ -static void gxp_firmware_has_requested(struct gxp_dev *gxp, - struct gxp_firmware_manager *mgr) -{ - gxp_firmware_get_image_config(gxp, mgr); - mgr->is_firmware_requested = true; -} - static int request_dsp_firmware(struct gxp_dev *gxp, char *name_prefix, const struct firmware *out_firmwares[GXP_NUM_CORES]) @@ -117,8 +82,10 @@ request_dsp_firmware(struct gxp_dev *gxp, char *name_prefix, return ret; err: - for (core -= 1; core >= 0; core--) + for (core -= 1; core >= 0; core--) { release_firmware(out_firmwares[core]); + out_firmwares[core] = NULL; + } kfree(name_buf); return ret; } @@ -238,6 +205,9 @@ gxp_firmware_authenticate(struct gxp_dev *gxp, return 0; } + if (!gxp_is_direct_mode(gxp)) + return 0; + for (core = 0; core < GXP_NUM_CORES; core++) { data = firmwares[core]->data; size = firmwares[core]->size; @@ -302,9 +272,6 @@ error: return ret; } -/* Forward declaration for usage inside gxp_firmware_load(..). */ -static void gxp_firmware_unload(struct gxp_dev *gxp, uint core); - static void gxp_program_reset_vector(struct gxp_dev *gxp, uint core, uint phys_core, bool verbose) { @@ -383,40 +350,6 @@ static void reset_core_config_region(struct gxp_dev *gxp, } } -static int gxp_firmware_load(struct gxp_dev *gxp, struct gxp_virtual_device *vd, - uint core) -{ - struct gxp_firmware_manager *mgr = gxp->firmware_mgr; - int ret; - - if (!mgr->firmwares[core]) - return -ENODEV; - if (mgr->loaded[core]) - return 0; - - /* Load firmware to System RAM */ - ret = elf_load_segments(gxp, - mgr->firmwares[core]->data + FW_HEADER_SIZE, - mgr->firmwares[core]->size - FW_HEADER_SIZE, - &gxp->fwbufs[core]); - if (ret) { - dev_err(gxp->dev, "Unable to load elf file\n"); - goto out_firmware_unload; - } - - /* TODO(b/188970444): Cleanup logging of addresses */ - dev_notice(gxp->dev, - "ELF loaded at virtual: %pK and physical: %#llx\n", - gxp->fwbufs[core].vaddr, gxp->fwbufs[core].paddr); - mgr->loaded[core] = true; - - return 0; - -out_firmware_unload: - gxp_firmware_unload(gxp, core); - return ret; -} - static int gxp_firmware_handshake(struct gxp_dev *gxp, struct gxp_virtual_device *vd, uint core, uint phys_core) @@ -515,9 +448,64 @@ static int gxp_firmware_handshake(struct gxp_dev *gxp, return 0; } -static void gxp_firmware_unload(struct gxp_dev *gxp, uint core) +static int +gxp_firmware_load_into_memories(struct gxp_dev *gxp, + const struct firmware *firmwares[GXP_NUM_CORES]) +{ + int core; + int ret; + + for (core = 0; core < GXP_NUM_CORES; core++) { + /* Load firmware to System RAM */ + if (FW_HEADER_SIZE > firmwares[core]->size) { + dev_err(gxp->dev, + "Invalid Core %u firmware Image size (%d > %zu)\n", + core, FW_HEADER_SIZE, firmwares[core]->size); + ret = -EINVAL; + goto error; + } + + if ((firmwares[core]->size - FW_HEADER_SIZE) > + gxp->fwbufs[core].size) { + dev_err(gxp->dev, + "Core %u firmware image does not fit (%zu > %llu)\n", + core, firmwares[core]->size - FW_HEADER_SIZE, + gxp->fwbufs[core].size); + ret = -EINVAL; + goto error; + } + memcpy_toio(gxp->fwbufs[core].vaddr, + firmwares[core]->data + FW_HEADER_SIZE, + firmwares[core]->size - FW_HEADER_SIZE); + } + return 0; +error: + /* Zero out firmware buffers if we got invalid size on any core. */ + for (core -= 1; core >= 0; core--) + memset_io(gxp->fwbufs[core].vaddr, 0, gxp->fwbufs[core].size); + return ret; +} + +int gxp_firmware_rearrange_elf(struct gxp_dev *gxp, + const struct firmware *firmwares[GXP_NUM_CORES]) { - /* NO-OP for now. */ + int ret = 0; + uint core; + + for (core = 0; core < GXP_NUM_CORES; core++) { + /* Re-arrange ELF firmware in System RAM */ + ret = elf_load_segments(gxp, + firmwares[core]->data + FW_HEADER_SIZE, + firmwares[core]->size - FW_HEADER_SIZE, + &gxp->fwbufs[core]); + if (ret) { + dev_err(gxp->dev, + "Failed to parse ELF firmware on core %u\n", + core); + return ret; + } + } + return ret; } /* Helper function to parse name written to sysfs "load_dsp_firmware" node */ @@ -544,17 +532,11 @@ static ssize_t load_dsp_firmware_show(struct device *dev, struct device_attribute *attr, char *buf) { struct gxp_dev *gxp = dev_get_drvdata(dev); - struct gxp_firmware_manager *mgr = gxp->firmware_mgr; ssize_t ret; + char *firmware_name = gxp_firmware_loader_get_core_fw_name(gxp); - mutex_lock(&mgr->dsp_firmware_lock); - - ret = scnprintf(buf, PAGE_SIZE, "%s\n", - mgr->firmware_name ? mgr->firmware_name : - DSP_FIRMWARE_DEFAULT_PREFIX); - - mutex_unlock(&mgr->dsp_firmware_lock); - + ret = scnprintf(buf, PAGE_SIZE, "%s\n", firmware_name); + kfree(firmware_name); return ret; } @@ -564,10 +546,8 @@ static ssize_t load_dsp_firmware_store(struct device *dev, { struct gxp_dev *gxp = dev_get_drvdata(dev); struct gxp_firmware_manager *mgr = gxp->firmware_mgr; - const struct firmware *firmwares[GXP_NUM_CORES]; char *name_buf = NULL; int ret; - int core; /* * Lock the VD semaphore to ensure no core is executing the firmware @@ -589,43 +569,28 @@ static ssize_t load_dsp_firmware_store(struct device *dev, goto err_out; } - mutex_lock(&mgr->dsp_firmware_lock); - dev_notice(gxp->dev, "Requesting firmware be reloaded: %s\n", name_buf); - ret = request_dsp_firmware(gxp, name_buf, firmwares); + /* + * It's possible a race condition bug here that someone opens a gxp + * device and loads the firmware between below unload/load functions in + * another thread, but this interface is only for developer debugging. + * We don't insist on preventing the race condition bug. + */ + gxp_firmware_loader_unload(gxp); + gxp_firmware_loader_set_core_fw_name(gxp, name_buf); + ret = gxp_firmware_loader_load_if_needed(gxp); if (ret) { - dev_err(gxp->dev, - "Failed to request firmwares with names \"%sX\" (ret=%d)\n", - name_buf, ret); - goto err_request_firmware; + dev_err(gxp->dev, "Failed to load core firmware: %s\n", name_buf); + goto err_firmware_load; } - ret = gxp_firmware_authenticate(gxp, firmwares); - if (ret) - goto err_authenticate_firmware; - - for (core = 0; core < GXP_NUM_CORES; core++) { - if (mgr->firmwares[core]) - release_firmware(mgr->firmwares[core]); - mgr->firmwares[core] = firmwares[core]; - mgr->loaded[core] = false; - } - - kfree(mgr->firmware_name); - mgr->firmware_name = name_buf; - gxp_firmware_has_requested(gxp, mgr); - - mutex_unlock(&mgr->dsp_firmware_lock); + kfree(name_buf); up_read(&gxp->vd_semaphore); return count; -err_authenticate_firmware: - for (core = 0; core < GXP_NUM_CORES; core++) - release_firmware(firmwares[core]); -err_request_firmware: +err_firmware_load: kfree(name_buf); - mutex_unlock(&mgr->dsp_firmware_lock); err_out: up_read(&gxp->vd_semaphore); return ret; @@ -654,7 +619,6 @@ int gxp_fw_init(struct gxp_dev *gxp) if (!mgr) return -ENOMEM; gxp->firmware_mgr = mgr; - mutex_init(&mgr->dsp_firmware_lock); /* Power on BLK_AUR to read the revision and processor ID registers */ gxp_pm_blk_on(gxp); @@ -750,53 +714,34 @@ void gxp_fw_destroy(struct gxp_dev *gxp) memunmap(gxp->fwbufs[core].vaddr); gxp->fwbufs[core].vaddr = NULL; } - - if (mgr->firmwares[core]) { - release_firmware(mgr->firmwares[core]); - mgr->firmwares[core] = NULL; - } } - - kfree(mgr->firmware_name); } -int gxp_firmware_request_if_needed(struct gxp_dev *gxp) +int gxp_firmware_load_core_firmware( + struct gxp_dev *gxp, char *name_prefix, + const struct firmware *core_firmware[GXP_NUM_CORES]) { - int ret = 0; uint core; - struct gxp_firmware_manager *mgr = gxp->firmware_mgr; - char *name = NULL; - - mutex_lock(&mgr->dsp_firmware_lock); - - if (mgr->is_firmware_requested) - goto out; - - if (mgr->firmware_name == NULL) - name = DSP_FIRMWARE_DEFAULT_PREFIX; - else - name = mgr->firmware_name; + int ret; - ret = request_dsp_firmware(gxp, name, mgr->firmwares); + if (name_prefix == NULL) + name_prefix = DSP_FIRMWARE_DEFAULT_PREFIX; + ret = request_dsp_firmware(gxp, name_prefix, core_firmware); if (ret) - goto out; - - ret = gxp_firmware_authenticate(gxp, mgr->firmwares); + return ret; + ret = gxp_firmware_load_into_memories(gxp, core_firmware); if (ret) - goto err_authenticate_firmware; - - gxp_firmware_has_requested(gxp, mgr); - -out: - mutex_unlock(&mgr->dsp_firmware_lock); - return ret; + goto error; + ret = gxp_firmware_authenticate(gxp, core_firmware); + if (ret) + goto error; -err_authenticate_firmware: + return 0; +error: for (core = 0; core < GXP_NUM_CORES; core++) { - release_firmware(mgr->firmwares[core]); - mgr->firmwares[core] = NULL; + release_firmware(core_firmware[core]); + core_firmware[core] = NULL; } - mutex_unlock(&mgr->dsp_firmware_lock); return ret; } @@ -811,7 +756,7 @@ static void enable_core_interrupts(struct gxp_dev *gxp, uint core) gxp_write_32(gxp, GXP_CORE_REG_DEDICATED_INT_MASK(core), 0xffffffff); } -static void disable_core_interrupts(struct gxp_dev *gxp, uint core) +void gxp_firmware_disable_ext_interrupts(struct gxp_dev *gxp, uint core) { gxp_write_32(gxp, GXP_CORE_REG_COMMON_INT_MASK_0(core), 0); gxp_write_32(gxp, GXP_CORE_REG_COMMON_INT_MASK_1(core), 0); @@ -831,31 +776,24 @@ static int gxp_firmware_setup(struct gxp_dev *gxp, int ret = 0; struct gxp_firmware_manager *mgr = gxp->firmware_mgr; - if (gxp_core_boot && mgr->firmware_running & BIT(phys_core)) { + if (gxp_core_boot(gxp) && mgr->firmware_running & BIT(phys_core)) { dev_err(gxp->dev, "Firmware is already running on core %u\n", phys_core); return -EBUSY; } - ret = gxp_firmware_load(gxp, vd, core); - if (ret) { - dev_err(gxp->dev, "Failed to load firmware on core %u\n", - phys_core); - return ret; - } /* Configure bus performance monitors */ gxp_bpm_configure(gxp, phys_core, INST_BPM_OFFSET, BPM_EVENT_READ_XFER); gxp_bpm_configure(gxp, phys_core, DATA_BPM_OFFSET, BPM_EVENT_WRITE_XFER); /* Mark this as a cold boot */ - if (gxp_core_boot) { + if (gxp_core_boot(gxp)) { reset_core_config_region(gxp, vd, core); ret = gxp_firmware_setup_hw_after_block_off(gxp, core, phys_core, /*verbose=*/true); if (ret) { dev_err(gxp->dev, "Failed to power up core %u\n", core); - gxp_firmware_unload(gxp, core); return ret; } enable_core_interrupts(gxp, phys_core); @@ -889,7 +827,7 @@ static int gxp_firmware_finish_startup(struct gxp_dev *gxp, int ret = 0; uint core = select_core(vd, virt_core, phys_core); - if (gxp_core_boot) { + if (gxp_core_boot(gxp)) { ret = gxp_firmware_handshake(gxp, vd, core, phys_core); if (ret) { dev_err(gxp->dev, @@ -931,9 +869,8 @@ static int gxp_firmware_finish_startup(struct gxp_dev *gxp, return ret; err_firmware_off: - if (gxp_core_boot) + if (gxp_core_boot(gxp)) gxp_pm_core_off(gxp, phys_core); - gxp_firmware_unload(gxp, core); return ret; } @@ -943,7 +880,7 @@ static void gxp_firmware_stop_core(struct gxp_dev *gxp, { struct gxp_firmware_manager *mgr = gxp->firmware_mgr; - if (gxp_core_boot && !(mgr->firmware_running & BIT(phys_core))) + if (gxp_core_boot(gxp) && !(mgr->firmware_running & BIT(phys_core))) dev_err(gxp->dev, "Firmware is not running on core %u\n", phys_core); @@ -954,7 +891,7 @@ static void gxp_firmware_stop_core(struct gxp_dev *gxp, gxp_notification_unregister_handler(gxp, phys_core, HOST_NOTIF_CORE_TELEMETRY_STATUS); - if (gxp_core_boot) { + if (gxp_core_boot(gxp)) { if (gxp->mailbox_mgr->release_mailbox) { gxp->mailbox_mgr->release_mailbox( gxp->mailbox_mgr, vd, virt_core, @@ -968,12 +905,10 @@ static void gxp_firmware_stop_core(struct gxp_dev *gxp, * Disable interrupts to prevent cores from being woken up * unexpectedly. */ - disable_core_interrupts(gxp, phys_core); + gxp_firmware_disable_ext_interrupts(gxp, phys_core); gxp_pm_core_off(gxp, phys_core); } } - - gxp_firmware_unload(gxp, select_core(vd, virt_core, phys_core)); } int gxp_firmware_run(struct gxp_dev *gxp, struct gxp_virtual_device *vd, @@ -1007,14 +942,11 @@ int gxp_firmware_run(struct gxp_dev *gxp, struct gxp_virtual_device *vd, */ virt_core = 0; for (phys_core = 0; phys_core < GXP_NUM_CORES; phys_core++) { - uint core = select_core(vd, virt_core, phys_core); - if (!(core_list & BIT(phys_core))) continue; if (!(failed_cores & BIT(phys_core))) { - if (gxp_core_boot) + if (gxp_core_boot(gxp)) gxp_pm_core_off(gxp, phys_core); - gxp_firmware_unload(gxp, core); } virt_core++; } @@ -1034,7 +966,7 @@ int gxp_firmware_run(struct gxp_dev *gxp, struct gxp_virtual_device *vd, } #endif /* Switch clock mux to the normal state to guarantee LPM works */ - if (gxp_core_boot) { + if (gxp_core_boot(gxp)) { gxp_pm_force_clkmux_normal(gxp); gxp_firmware_wakeup_cores(gxp, core_list); } @@ -1065,7 +997,7 @@ int gxp_firmware_run(struct gxp_dev *gxp, struct gxp_virtual_device *vd, } } /* Check if we need to set clock mux to low state as requested */ - if (gxp_core_boot) + if (gxp_core_boot(gxp)) gxp_pm_resume_clkmux(gxp); return ret; @@ -1138,3 +1070,8 @@ u32 gxp_firmware_get_boot_status(struct gxp_dev *gxp, core_cfg = get_scratchpad_base(gxp, vd, core); return core_cfg->boot_status; } + +bool gxp_core_boot(struct gxp_dev *gxp) +{ + return gxp_is_direct_mode(gxp); +} diff --git a/gxp-firmware.h b/gxp-firmware.h index fdaff99..e1f44ce 100644 --- a/gxp-firmware.h +++ b/gxp-firmware.h @@ -43,8 +43,6 @@ #define PRIVATE_FW_DATA_SIZE SZ_2M #define SHARED_FW_DATA_SIZE SZ_1M -extern bool gxp_core_boot; - /* Indexes same as image_config.IommuMappingIdx in the firmware side. */ enum gxp_imgcfg_idx { CORE_CFG_REGION_IDX, @@ -53,22 +51,8 @@ enum gxp_imgcfg_idx { }; struct gxp_firmware_manager { - const struct firmware *firmwares[GXP_NUM_CORES]; - char *firmware_name; - bool is_firmware_requested; - /* Protects `firmwares` and `firmware_name` */ - struct mutex dsp_firmware_lock; - /* FW is readonly, we only need to load it once per image. */ - bool loaded[GXP_NUM_CORES]; /* Firmware status bitmap. Accessors must hold `vd_semaphore`. */ u32 firmware_running; - /* - * Cached image config, for easier fetching config entries. - * Not a pointer to the firmware buffer because we want to forcely change the - * privilege level to NS. - * Only valid on firmware requested. - */ - struct gcip_image_config img_cfg; }; enum aurora_msg { @@ -85,7 +69,7 @@ static inline bool gxp_is_fw_running(struct gxp_dev *gxp, uint core) } /* - * Initializes the firmware loading/unloading subsystem. This includes + * Initializes the core firmware loading/unloading subsystem. This includes * initializing the LPM and obtaining the memory regions needed to load the FW. * The function needs to be called once after a block power up event. */ @@ -98,14 +82,22 @@ int gxp_fw_init(struct gxp_dev *gxp); void gxp_fw_destroy(struct gxp_dev *gxp); /* - * Check if the DSP firmware files have been requested yet, and if not, request - * them. + * Requests and loads core firmware into memories. + * If the loaded firmware is ELF, rearranges it. + * + * Returns 0 on success, a negative errno on failure. + */ +int gxp_firmware_load_core_firmware( + struct gxp_dev *gxp, char *name_prefix, + const struct firmware *core_firmwares[GXP_NUM_CORES]); + +/* + * Rearranges firmware data if the firmware is ELF. * - * Returns 0 if the files have already been requested or were successfully - * requested by this call; Returns an errno if this call attempted to request - * the files and it failed. + * Returns 0 on success, a negative errno on failure. */ -int gxp_firmware_request_if_needed(struct gxp_dev *gxp); +int gxp_firmware_rearrange_elf(struct gxp_dev *gxp, + const struct firmware *firmwares[GXP_NUM_CORES]); /* * Re-program the reset vector and power on the core's LPM if the block had @@ -159,4 +151,12 @@ void gxp_firmware_set_boot_status(struct gxp_dev *gxp, u32 gxp_firmware_get_boot_status(struct gxp_dev *gxp, struct gxp_virtual_device *vd, uint core); +/* Returns whether the core firmware running states are managed by us. */ +bool gxp_core_boot(struct gxp_dev *gxp); + +/* + * Disable external interrupts to core. + */ +void gxp_firmware_disable_ext_interrupts(struct gxp_dev *gxp, uint core); + #endif /* __GXP_FIRMWARE_H__ */ diff --git a/gxp-host-device-structs.h b/gxp-host-device-structs.h index 4597a28..efb39a9 100644 --- a/gxp-host-device-structs.h +++ b/gxp-host-device-structs.h @@ -11,11 +11,11 @@ * headers or data structures. * */ + #ifndef __GXP_HOST_DEVICE_STRUCTURES_H__ #define __GXP_HOST_DEVICE_STRUCTURES_H__ #define MAX_NUM_CORES 4 -#define NUM_SYSTEM_SEMAPHORES 64 /* The number of physical doorbells and sync barriers allocated to each VD */ #define GXP_NUM_DOORBELLS_PER_VD 7 @@ -148,66 +148,6 @@ /* Invalid boot mode request code */ #define GXP_BOOT_MODE_STATUS_INVALID_MODE 10 -/* A structure describing the state of the doorbells on the system. */ -struct gxp_doorbells_descriptor { - /* The app this descriptor belongs to. */ - uint32_t application_id; - /* The physical ID of the sync barrier protecting this region. */ - uint32_t protection_barrier; - /* The number of doorbells described in this region. */ - uint32_t num_items; - /* The list of doorbells available for usage. */ - struct dooorbell_metadata_t { - /* - * The number of users using this doorbell. 0 when it's - * available. - */ - uint32_t users_count; - /* The 0-based index of the doorbell described by this entry. */ - uint32_t hw_doorbell_idx; - } doorbells[]; -}; - -/* A structure describing the state of the sync barriers on the system. */ -struct gxp_sync_barriers_descriptor { - /* The app this descriptor belongs to. */ - uint32_t application_id; - /* The physical ID of the sync barrier protecting this region. */ - uint32_t protection_barrier; - /* The number of sync barriers described in this region. */ - uint32_t num_items; - /* The list of sync barriers available for usage. */ - struct sync_barrier_metadata_t { - /* - * The number of users using this barrier. 0 when it's - * available. - */ - uint32_t users_count; - /* - * The 0-based index of the sync barrier described by this - * entry. - */ - uint32_t hw_barrier_idx; - } barriers[]; -}; - -/* A structure describing the state of the watchdog on the system. */ -struct gxp_watchdog_descriptor { - /* The physical ID of the sync barrier protecting this region. */ - uint32_t protection_barrier; - /* - * The number of timer ticks before the watchdog expires. - * This is in units of 244.14 ns. - */ - uint32_t target_value; - /* A bit mask of the cores expected to tickle the watchdog. */ - uint32_t participating_cores; - /* A bit mask of the cores that have tickled the watchdog. */ - uint32_t responded_cores; - /* A flag indicating whether or not the watchdog has tripped. */ - uint32_t tripped; -}; - /* * A structure describing the core telemetry (logging and tracing) parameters * and buffers. @@ -239,171 +179,6 @@ struct gxp_core_telemetry_descriptor { }; /* - * A structure describing the state and allocations of the SW-based semaphores - * on the system. - */ -struct gxp_semaphores_descriptor { - /* The app this descriptor belongs to. */ - uint32_t application_id; - /* The physical ID of the sync barrier protecting this region. */ - uint32_t protection_barrier; - /* - * An array where each element is dedicated to a core. The element is a - * bit map describing of all the semaphores in the list below that have - * been unlocked but haven't been processed yet by the receiptient core. - */ - uint64_t woken_pending_semaphores[MAX_NUM_CORES]; - /* - * A mapping of which doorbells to use as a wakeup signal source per - * core. - */ - uint32_t wakeup_doorbells[MAX_NUM_CORES]; - /* The number of items described in this region. */ - uint32_t num_items; - /* The list of semaphores available for usage. */ - struct semaphore_metadata { - /* - * The number of users using this semaphore. 0 when it's for - * creation. - * Note: this is not the count value of the semaphore, but just - * an indication if this slot is available. - */ - uint32_t users_count; - /* - * This is the semaphore count. Cores will block when they call - * 'Wait()' while this count is 0. - */ - uint32_t count; - /* - * A bit map of 'NUM_DSP_CORES' bits indicating which cores are - * currently waiting on this semaphore to become available. - */ - uint32_t waiters; - } semaphores[NUM_SYSTEM_SEMAPHORES]; -}; - -/* A basic unidirectional queue. */ -struct gxp_queue_info { - /* A header describing the queue and its state. */ - struct queue_header { - /* A device-side pointer of the storage managed by this queue */ - uint32_t storage; - /* The index to the head of the queue. */ - uint32_t head_idx; - /* The index to the tail of the queue. */ - uint32_t tail_idx; - /* The size of an element stored this queue. */ - uint32_t element_size; - /* The number of elements that can be stored in this queue. */ - uint32_t elements_count; - } header; - /* The semaphore ID controlling exclusive access to this core. */ - uint32_t access_sem_id; - /* - * The ID for the semaphore containing the number of unprocessed items - * pushed to this queue. - */ - uint32_t posted_slots_sem_id; - /* - * The ID for the semaphore containing the number of free slots - * available to store data in this queue. - */ - uint32_t free_slots_sem_id; -}; - -/* A struct describing a single core's set of incoming queues. */ -struct gxp_core_info { - /* - * The metadata for the queue holding incoming commands from other - * cores. - */ - struct gxp_queue_info incoming_commands_queue; - /* - * The metadata for the queue holding incoming responses from other - * cores. - */ - struct gxp_queue_info incoming_responses_queue; -}; - -/* A structure describing all the cores' per-core metadata. */ -struct gxp_cores_descriptor { - /* The number of cores described in this descriptor. */ - uint32_t num_items; - /* The descriptors for each core. */ - struct gxp_core_info cores[]; -}; - -/* - * The top level descriptor describing memory regions used to access system-wide - * structures and resources. - */ -struct gxp_system_descriptor { - /* A device address for the application data descriptor. */ - uint32_t app_descriptor_dev_addr[MAX_NUM_CORES]; - /* A device address for the watchdog descriptor. */ - uint32_t watchdog_dev_addr; - /* A device address for the core telemetry descriptor */ - uint32_t core_telemetry_dev_addr; - /* A device address for the common debug dump region */ - uint32_t debug_dump_dev_addr; -}; - -/* A structure describing the metadata belonging to a specific application. */ -struct gxp_application_descriptor { - /* The ID for this GXP application. */ - uint32_t application_id; - /* The number of cores this application has. */ - uint16_t core_count; - /* - * The cores mask; a bit at index `n` indicates that core `n` is part of - * this app. - */ - uint16_t cores_mask; - /* The number of threads allocated for each core. */ - uint16_t threads_count; - /* The size of system memory given to this app. */ - uint32_t system_memory_size; - /* The device-address of the system memory given to this app. */ - uint32_t system_memory_addr; - /* The size of TCM memory allocated per bank for this app. */ - uint32_t tcm_memory_per_bank; /* in units of 4 kB */ - /* A device address for the doorbells descriptor. */ - uint32_t doorbells_dev_addr; - /* A device address for the sync barriers descriptor. */ - uint32_t sync_barriers_dev_addr; - /* A device address for the semaphores descriptor. */ - uint32_t semaphores_dev_addr; - /* A device address for the cores cmd/rsp queues descriptor. */ - uint32_t cores_info_dev_addr; -}; - -/* The structure describing a core-to-core command. */ -struct gxp_core_to_core_command { - /* The source of port number (the core's virtual ID) of the command. */ - uint32_t source; - /* The command's sequence number. */ - uint64_t sequence_number; - /* The command payload device address. */ - uint64_t device_address; - /* The size of the payload in bytes. */ - uint32_t size; - /* The generic command flags. */ - uint32_t flags; -}; - -/* The structure describing a core-to-core response. */ -struct gxp_core_to_core_response { - /* The source of port number (the core's virtual ID) of the response. */ - uint32_t source; - /* The response's sequence number. */ - uint64_t sequence_number; - /* The response error code (if any). */ - uint16_t error_code; - /* The response return value (filled-in by the user). */ - int32_t cmd_retval; -}; - -/* * A structure for describing the state of the job this worker core is part of. * This struct is expected to change per dispatch/context switch/preepmtion as * it describes the HW resources, FW IDs, and other parameters that may change diff --git a/gxp-internal.h b/gxp-internal.h index cc6cffe..e00401b 100644 --- a/gxp-internal.h +++ b/gxp-internal.h @@ -23,6 +23,8 @@ #include <linux/rwsem.h> #include <linux/spinlock.h> +#include <gcip/gcip-thermal.h> + #include "gxp-config.h" #define IS_GXP_TEST IS_ENABLED(CONFIG_GXP_TEST) @@ -83,6 +85,7 @@ struct gxp_dev { struct gxp_mailbox_manager *mailbox_mgr; struct gxp_power_manager *power_mgr; struct gxp_debug_dump_manager *debug_dump_mgr; + struct gxp_firmware_loader_manager *fw_loader_mgr; struct gxp_firmware_manager *firmware_mgr; /* * Lock to ensure only one thread at a time is ever calling @@ -104,12 +107,12 @@ struct gxp_dev { struct gxp_client *debugfs_client; struct mutex debugfs_client_lock; bool debugfs_wakelock_held; - struct gxp_thermal_manager *thermal_mgr; struct gxp_dma_manager *dma_mgr; struct gxp_fw_data_manager *data_mgr; struct gxp_tpu_dev tpu_dev; struct gxp_core_telemetry_manager *core_telemetry_mgr; struct gxp_iommu_domain *default_domain; + struct gcip_thermal *thermal; /* * Pointer to GSA device for firmware authentication. * May be NULL if the chip does not support firmware authentication diff --git a/gxp-mailbox-impl.c b/gxp-mailbox-impl.c index 4044620..6d84dbf 100644 --- a/gxp-mailbox-impl.c +++ b/gxp-mailbox-impl.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * Legacy implementation of the GXP mailbox interface. * This file must be used only when the kernel driver has to compile the implementation of the diff --git a/gxp-mailbox-impl.h b/gxp-mailbox-impl.h index 2f4b5d8..9a78e65 100644 --- a/gxp-mailbox-impl.h +++ b/gxp-mailbox-impl.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Legacy implementation of the GXP mailbox interface. * This file must be used only when the kernel driver has to compile the implementation of the diff --git a/gxp-mailbox-regs.h b/gxp-mailbox-regs.h index 05fb414..5c518ed 100644 --- a/gxp-mailbox-regs.h +++ b/gxp-mailbox-regs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GXP mailbox registers. * diff --git a/gxp-mailbox.c b/gxp-mailbox.c index 0e0b365..758b707 100644 --- a/gxp-mailbox.c +++ b/gxp-mailbox.c @@ -33,7 +33,7 @@ #endif /* Timeout of 1s by default */ -int gxp_mbx_timeout = 1000; +int gxp_mbx_timeout = 2000; module_param_named(mbx_timeout, gxp_mbx_timeout, int, 0660); /* diff --git a/gxp-mapping.c b/gxp-mapping.c index 398ff05..1a89b1c 100644 --- a/gxp-mapping.c +++ b/gxp-mapping.c @@ -272,6 +272,13 @@ int gxp_mapping_sync(struct gxp_mapping *mapping, u32 offset, u32 size, } /* + * Since the scatter-gather list of the mapping is modified while it is + * being synced, only one sync for a given mapping can occur at a time. + * Rather than maintain a mutex for every mapping, lock the mapping list + * mutex, making all syncs mutually exclusive. + */ + mutex_lock(&mapping->sync_lock); + /* * Mappings are created at a PAGE_SIZE granularity, however other data * which is not part of the mapped buffer may be present in the first * and last pages of the buffer's scattergather list. @@ -302,17 +309,9 @@ int gxp_mapping_sync(struct gxp_mapping *mapping, u32 offset, u32 size, /* Make sure a valid starting scatterlist was found for the start */ if (!start_sg) { ret = -EINVAL; - goto out; + goto out_unlock; } - /* - * Since the scatter-gather list of the mapping is modified while it is - * being synced, only one sync for a given mapping can occur at a time. - * Rather than maintain a mutex for every mapping, lock the mapping list - * mutex, making all syncs mutually exclusive. - */ - mutex_lock(&mapping->sync_lock); - start_sg->offset += start_diff; start_sg->dma_address += start_diff; start_sg->length -= start_diff; @@ -336,8 +335,8 @@ int gxp_mapping_sync(struct gxp_mapping *mapping, u32 offset, u32 size, start_sg->length += start_diff; start_sg->dma_length += start_diff; +out_unlock: mutex_unlock(&mapping->sync_lock); - out: gxp_mapping_put(mapping); diff --git a/gxp-mba-driver.c b/gxp-mba-driver.c index cb91092..14a8057 100644 --- a/gxp-mba-driver.c +++ b/gxp-mba-driver.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GXP mailbox array driver implementation. * diff --git a/gxp-range-alloc.c b/gxp-range-alloc.c deleted file mode 100644 index 73aa6af..0000000 --- a/gxp-range-alloc.c +++ /dev/null @@ -1,118 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * GXP ranged resource allocator. - * - * Copyright (C) 2021 Google LLC - */ - -#include "gxp-range-alloc.h" - -struct range_alloc *range_alloc_create(int start, int end) -{ - struct range_alloc *ra; - int count; - int size; - - count = end - start; - if (count <= 0) - return ERR_PTR(-EINVAL); - - size = sizeof(struct range_alloc) + count * sizeof(int); - ra = kzalloc(size, GFP_KERNEL); - if (!ra) - return ERR_PTR(-ENOMEM); - - ra->total_count = count; - ra->free_count = count; - ra->start_index = start; - mutex_init(&ra->lock); - - return ra; -} - -int range_alloc_get(struct range_alloc *r, int element) -{ - int index = element - r->start_index; - - mutex_lock(&r->lock); - if (index < 0 || index >= r->total_count) { - mutex_unlock(&r->lock); - return -EINVAL; - } - - if (r->elements[index]) { - mutex_unlock(&r->lock); - return -EBUSY; - } - - r->elements[index] = 1; - r->free_count--; - - mutex_unlock(&r->lock); - return 0; -} - -int range_alloc_get_any(struct range_alloc *r, int *element) -{ - int i; - - mutex_lock(&r->lock); - if (!r->free_count) { - mutex_unlock(&r->lock); - return -ENOMEM; - } - - for (i = 0; i < r->total_count; i++) { - if (r->elements[i] == 0) { - r->elements[i] = 1; - r->free_count--; - *element = i + r->start_index; - mutex_unlock(&r->lock); - return 0; - } - } - mutex_unlock(&r->lock); - return -ENOMEM; -} - -int range_alloc_put(struct range_alloc *r, int element) -{ - int index = element - r->start_index; - - mutex_lock(&r->lock); - if (index < 0 || index >= r->total_count) { - mutex_unlock(&r->lock); - return -EINVAL; - } - - if (r->elements[index] == 0) { - mutex_unlock(&r->lock); - return -EBUSY; - } - - r->elements[index] = 0; - r->free_count++; - - mutex_unlock(&r->lock); - return 0; -} - -int range_alloc_num_free(struct range_alloc *r) -{ - int free_count; - - mutex_lock(&r->lock); - free_count = r->free_count; - mutex_unlock(&r->lock); - - return free_count; -} - -int range_alloc_destroy(struct range_alloc *r) -{ - if (!r) - return -EFAULT; - kfree(r); - - return 0; -} diff --git a/gxp-range-alloc.h b/gxp-range-alloc.h deleted file mode 100644 index ed8c2f0..0000000 --- a/gxp-range-alloc.h +++ /dev/null @@ -1,94 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * GXP ranged resource allocator. - * - * Copyright (C) 2021 Google LLC - */ -#ifndef __GXP_RANGE_ALLOC_H__ -#define __GXP_RANGE_ALLOC_H__ - -#include <linux/mutex.h> -#include <linux/slab.h> - -struct range_alloc { - int total_count; - int free_count; - int start_index; - struct mutex lock; - int elements[]; -}; - -/** - * range_alloc_create() - Creates a range allocator starting at the specified - * start (inclusive) and ends at the specified end - * (exclusive). - * @start: The start of the range (inclusive). - * @end: The end of the range (exclusive) - * - * Return: - * ptr - A pointer of the newly created allocator handle on success, an - * error pointer (PTR_ERR) otherwise. - * -EINVAL - Invalid start/end combination - * -ENOMEM - Insufficient memory to create the allocator - */ -struct range_alloc *range_alloc_create(int start, int end); - -/** - * range_alloc_get() - Gets the specified element from the range. - * @r: The range allocator - * @element: The element to acquire from the range - * - * The @element argument should be within the allocator's range and has not been - * allocated before. - * - * Return: - * 0 - Successfully reserved @element - * -EINVAL - Invalid element index (negative or outside allocator range) - * -EBUSY - Element is already allocated - */ -int range_alloc_get(struct range_alloc *r, int element); - -/** - * range_alloc_get_any() - Gets any free element in the range. - * @r: The range allocator - * @element: A pointer to use to store the allocated element - * - * Return: - * 0 - Successful reservation - * -ENOMEM - No elements left in the range to allocate - */ -int range_alloc_get_any(struct range_alloc *r, int *element); - -/** - * range_alloc_put() - Puts an element back into the range. - * @r: The range allocator - * @element: The element to put back into the range - * - * Return: - * 0 - Successful placement back into the range - * -EINVAL - Invalid element index (negative or outside allocator range) - * -EBUSY - The element is still present in the range - */ -int range_alloc_put(struct range_alloc *r, int element); - -/** - * range_alloc_num_free() - Returns the number of free elements in the range. - * @r: The range allocator - * - * Return: the number of free elements in the range - */ -int range_alloc_num_free(struct range_alloc *r); - -/** - * range_alloc_destroy() - Destroys the range allocator - * @r: The range allocator to destroy - * - * The destruction does not validate that the range is empty. - * - * Return: - * 0 - Successfully destroyed range allocator - * -EFAULT - Invalid allocator address - */ -int range_alloc_destroy(struct range_alloc *r); - -#endif /* __GXP_RANGE_ALLOC_H__ */ @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only /* * GXP SSMT driver. * @@ -11,12 +11,12 @@ #include "gxp-internal.h" #include "gxp-ssmt.h" -static inline void ssmt_set_vid_for_sid(void __iomem *ssmt, uint vid, uint sid) +static inline void ssmt_set_vid_for_idx(void __iomem *ssmt, uint vid, uint idx) { /* NS_READ_STREAM_VID_<sid> */ - writel(vid, ssmt + 0x1000u + 0x4u * sid); + writel(vid, ssmt + 0x1000u + 0x4u * idx); /* NS_WRITE_STREAM_VID_<sid> */ - writel(vid, ssmt + 0x1200u + 0x4u * sid); + writel(vid, ssmt + 0x1200u + 0x4u * idx); } int gxp_ssmt_init(struct gxp_dev *gxp, struct gxp_ssmt *ssmt) @@ -69,7 +69,25 @@ void gxp_ssmt_set_core_vid(struct gxp_ssmt *ssmt, uint core, uint vid) int i; for (i = 0; i < ARRAY_SIZE(sids); i++) { - ssmt_set_vid_for_sid(ssmt->idma_ssmt_base, vid, sids[i]); - ssmt_set_vid_for_sid(ssmt->inst_data_ssmt_base, vid, sids[i]); + ssmt_set_vid_for_idx(ssmt->idma_ssmt_base, vid, sids[i]); + ssmt_set_vid_for_idx(ssmt->inst_data_ssmt_base, vid, sids[i]); + } +} + +void gxp_ssmt_set_bypass(struct gxp_ssmt *ssmt) +{ + u32 mode; + uint core, i; + + mode = readl(ssmt->idma_ssmt_base + SSMT_CFG_OFFSET); + if (mode == SSMT_MODE_CLIENT) { + for (i = 0; i < MAX_NUM_CONTEXTS; i++) { + ssmt_set_vid_for_idx(ssmt->idma_ssmt_base, i, i); + ssmt_set_vid_for_idx(ssmt->inst_data_ssmt_base, i, i); + } + } else { + for (core = 0; core < GXP_NUM_CORES; core++) + gxp_ssmt_set_core_vid(ssmt, core, + SSMT_CLAMP_MODE_BYPASS); } } @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0 */ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * GXP SSMT driver. * @@ -10,7 +10,12 @@ #include "gxp-internal.h" +#define SSMT_CFG_OFFSET (0x0004) +#define SSMT_MODE_CLAMPED (0x0u) +#define SSMT_MODE_CLIENT (0x1u) + #define SSMT_CLAMP_MODE_BYPASS (1u << 31) +#define MAX_NUM_CONTEXTS 8 struct gxp_ssmt { struct gxp_dev *gxp; @@ -33,9 +38,10 @@ int gxp_ssmt_init(struct gxp_dev *gxp, struct gxp_ssmt *ssmt); */ void gxp_ssmt_set_core_vid(struct gxp_ssmt *ssmt, uint core, uint vid); -static inline void gxp_ssmt_set_core_bypass(struct gxp_ssmt *ssmt, uint core) -{ - gxp_ssmt_set_core_vid(ssmt, core, SSMT_CLAMP_MODE_BYPASS); -} +/* + * Programs SSMT to always use SCIDs as VIDs. + * Supports both client-driven and clamp mode. + */ +void gxp_ssmt_set_bypass(struct gxp_ssmt *ssmt); #endif /* __GXP_SSMT_H__ */ diff --git a/gxp-thermal.c b/gxp-thermal.c index 8a9c24b..671d140 100644 --- a/gxp-thermal.c +++ b/gxp-thermal.c @@ -2,358 +2,93 @@ /* * Platform thermal driver for GXP. * - * Copyright (C) 2021 Google LLC + * Copyright (C) 2021-2023 Google LLC */ #include <linux/acpm_dvfs.h> -#include <linux/debugfs.h> #include <linux/device.h> -#include <linux/gfp.h> -#include <linux/kernel.h> -#include <linux/mutex.h> -#include <linux/of.h> -#include <linux/platform_device.h> -#include <linux/pm_runtime.h> -#include <linux/slab.h> -#include <linux/thermal.h> -#include <linux/version.h> +#include <linux/minmax.h> #include <gcip/gcip-pm.h> +#include <gcip/gcip-thermal.h> -/* - * thermal_cdev_update is moved to drivers/thermal/thermal_core.h in kernel - * 5.12. The symbol is still exported, manually declare the function prototype - * to get rid of the implicit declaration compilation error. - */ -#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 12, 0) -void thermal_cdev_update(struct thermal_cooling_device *cdev); -#endif - +#include "gxp-config.h" #include "gxp-internal.h" #include "gxp-pm.h" #include "gxp-thermal.h" -#include "gxp-lpm.h" - #if GXP_HAS_MCU #include "gxp-kci.h" #include "gxp-mcu.h" #endif /* GXP_HAS_MCU */ -/* - * Value comes from internal measurement - * b/229623553 - */ -static struct gxp_state_pwr state_pwr_map[] = { - { AUR_NOM_RATE, 78 }, - { AUR_UD_PLUS_RATE, 58 }, - { AUR_UD_RATE, 40 }, - { AUR_SUD_PLUS_RATE, 27 }, - { AUR_SUD_RATE, 20 }, - { AUR_UUD_PLUS_RATE, 16 }, - { AUR_UUD_RATE, 13 }, -}; - -static int gxp_get_max_state(struct thermal_cooling_device *cdev, - unsigned long *state) +static int gxp_thermal_get_rate(void *data, unsigned long *rate) { - struct gxp_thermal_manager *thermal = cdev->devdata; + *rate = exynos_acpm_get_rate(AUR_DVFS_DOMAIN, 0); - if (!thermal->gxp_num_states) - return -EIO; - - *state = thermal->gxp_num_states - 1; return 0; } -/* - * Set cooling state. - */ -static int gxp_set_cur_state(struct thermal_cooling_device *cdev, - unsigned long cooling_state) +static int gxp_thermal_set_rate(void *data, unsigned long rate) { + struct gxp_dev *gxp = data; int ret = 0; - struct gxp_thermal_manager *thermal = cdev->devdata; - struct gxp_dev *gxp = thermal->gxp; - struct device *dev = gxp->dev; - unsigned long pwr_state; - - if (cooling_state >= thermal->gxp_num_states) { - dev_err(dev, "%s: invalid cooling state %lu\n", __func__, - cooling_state); - return -EINVAL; - } - mutex_lock(&thermal->lock); - cooling_state = max(thermal->sysfs_req, cooling_state); - if (cooling_state >= ARRAY_SIZE(state_pwr_map)) { - dev_err(dev, "Unsupported cooling state: %lu\n", cooling_state); - ret = -EINVAL; - goto out; - } - pwr_state = state_pwr_map[cooling_state].state; - dev_dbg(dev, "setting state %ld\n", pwr_state); - if (cooling_state != thermal->cooling_state) { - if (!gxp_is_direct_mode(gxp)) { + if (!gxp_is_direct_mode(gxp)) { #if GXP_HAS_MCU - struct gxp_mcu *mcu = gxp_mcu_of(gxp); - - ret = gcip_pm_get_if_powered(mcu->gxp->power_mgr->pm, - false); - if (ret) { - dev_err(dev, - "Can't acquire wakelock when powered down: %d\n", - ret); - goto out; - } + struct gxp_mcu *mcu = gxp_mcu_of(gxp); - ret = gxp_kci_notify_throttling(&mcu->kci, pwr_state); - gcip_pm_put(gxp->power_mgr->pm); + ret = gxp_kci_notify_throttling(&mcu->kci, rate); #endif /* GXP_HAS_MCU */ - } else { - ret = gxp_pm_blk_set_rate_acpm( - gxp, - max(pwr_state, - (unsigned long) - aur_power_state2rate[AUR_UUD])); - } - - if (ret) { - dev_err(dev, "error setting gxp cooling state: %d\n", - ret); - ret = -ENODEV; - goto out; - } - thermal->cooling_state = cooling_state; - gxp_pm_set_thermal_limit(thermal->gxp, pwr_state); } else { - ret = -EALREADY; - } - -out: - mutex_unlock(&thermal->lock); - return ret; -} - -static int gxp_get_cur_state(struct thermal_cooling_device *cdev, - unsigned long *state) -{ - int ret = 0; - struct gxp_thermal_manager *thermal = cdev->devdata; - - mutex_lock(&thermal->lock); - *state = thermal->cooling_state; - if (*state >= thermal->gxp_num_states) { - dev_err(thermal->gxp->dev, - "Unknown cooling state: %lu, resetting\n", *state); - ret = -EINVAL; - goto out; - } -out: - mutex_unlock(&thermal->lock); - return ret; -} - -static int gxp_state2power_internal(unsigned long state, u32 *power, - struct gxp_thermal_manager *thermal) -{ - int i; - - for (i = 0; i < thermal->gxp_num_states; i++) { - if (state == state_pwr_map[i].state) { - *power = state_pwr_map[i].power; - return 0; - } + rate = max_t(unsigned long, rate, + aur_power_state2rate[AUR_UUD]); + ret = gxp_pm_blk_set_rate_acpm(gxp, rate); } - dev_err(thermal->gxp->dev, "Unknown state req for: %lu\n", state); - *power = 0; - return -EINVAL; -} - -static int gxp_get_requested_power(struct thermal_cooling_device *cdev, - u32 *power) -{ - unsigned long power_state; - struct gxp_thermal_manager *cooling = cdev->devdata; - - power_state = exynos_acpm_get_rate(AUR_DVFS_DOMAIN, 0); - return gxp_state2power_internal(power_state, power, cooling); -} -/* TODO(b/213272324): Move state2power table to dts */ -static int gxp_state2power(struct thermal_cooling_device *cdev, - unsigned long state, u32 *power) -{ - struct gxp_thermal_manager *thermal = cdev->devdata; - - if (state >= thermal->gxp_num_states) { - dev_err(thermal->gxp->dev, "%s: invalid state: %lu\n", __func__, - state); - return -EINVAL; + if (ret) { + dev_err(gxp->dev, "error setting gxp cooling state: %d\n", ret); + return ret; } - return gxp_state2power_internal(state_pwr_map[state].state, power, - thermal); -} - -static int gxp_power2state(struct thermal_cooling_device *cdev, - u32 power, unsigned long *state) -{ - int i, penultimate_throttle_state; - struct gxp_thermal_manager *thermal = cdev->devdata; - - *state = 0; - /* Less than 2 state means we cannot really throttle */ - if (thermal->gxp_num_states < 2) - return thermal->gxp_num_states == 1 ? 0 : -EIO; + gxp_pm_set_thermal_limit(gxp, rate); - penultimate_throttle_state = thermal->gxp_num_states - 2; - /* - * argument "power" is the maximum allowed power consumption in mW as - * defined by the PID control loop. Check for the first state that is - * less than or equal to the current allowed power. state_pwr_map is - * descending, so lowest power consumption is last value in the array - * return lowest state even if it consumes more power than allowed as - * not all platforms can handle throttling below an active state - */ - for (i = penultimate_throttle_state; i >= 0; --i) { - if (power < state_pwr_map[i].power) { - *state = i + 1; - break; - } - } return 0; } -static struct thermal_cooling_device_ops gxp_cooling_ops = { - .get_max_state = gxp_get_max_state, - .get_cur_state = gxp_get_cur_state, - .set_cur_state = gxp_set_cur_state, - .get_requested_power = gxp_get_requested_power, - .state2power = gxp_state2power, - .power2state = gxp_power2state, -}; - -static void gxp_thermal_exit(struct gxp_thermal_manager *thermal) -{ - if (!IS_ERR_OR_NULL(thermal->cdev)) - thermal_cooling_device_unregister(thermal->cdev); -} - -static void devm_gxp_thermal_release(struct device *dev, void *res) -{ - struct gxp_thermal_manager *thermal = res; - - gxp_thermal_exit(thermal); -} - -static ssize_t -user_vote_show(struct device *dev, struct device_attribute *attr, char *buf) -{ - struct thermal_cooling_device *cdev = - container_of(dev, struct thermal_cooling_device, - device); - struct gxp_thermal_manager *cooling = cdev->devdata; - - if (!cooling) - return -ENODEV; - - return sysfs_emit(buf, "%lu\n", cooling->sysfs_req); -} - -static ssize_t user_vote_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t count) -{ - struct thermal_cooling_device *cdev = - container_of(dev, struct thermal_cooling_device, - device); - struct gxp_thermal_manager *cooling = cdev->devdata; - int ret; - unsigned long state; - - if (!cooling) - return -ENODEV; - - ret = kstrtoul(buf, 0, &state); - if (ret) - return ret; - - if (state >= cooling->gxp_num_states) - return -EINVAL; - - mutex_lock(&cdev->lock); - cooling->sysfs_req = state; - cdev->updated = false; - mutex_unlock(&cdev->lock); - thermal_cdev_update(cdev); - return count; -} - -static DEVICE_ATTR_RW(user_vote); - -static int -gxp_thermal_cooling_register(struct gxp_thermal_manager *thermal, char *type) +static int gxp_thermal_control(void *data, bool enable) { - struct device_node *cooling_node = NULL; - - thermal->op_data = NULL; - thermal->gxp_num_states = ARRAY_SIZE(state_pwr_map); - - mutex_init(&thermal->lock); - cooling_node = of_find_node_by_name(NULL, GXP_COOLING_NAME); - - /* TODO: Change this to fatal error once dts change is merged */ - if (!cooling_node) - dev_warn(thermal->gxp->dev, "failed to find cooling node\n"); - /* Initialize the cooling state as 0, means "no cooling" */ - thermal->cooling_state = 0; - thermal->cdev = thermal_of_cooling_device_register( - cooling_node, type, thermal, &gxp_cooling_ops); - if (IS_ERR(thermal->cdev)) - return PTR_ERR(thermal->cdev); - - return device_create_file(&thermal->cdev->device, &dev_attr_user_vote); + return -EOPNOTSUPP; } -static int cooling_init(struct gxp_thermal_manager *thermal, struct device *dev) +int gxp_thermal_init(struct gxp_dev *gxp) { - int err; - struct dentry *d; + const struct gcip_thermal_args args = { + .dev = gxp->dev, + .pm = gxp->power_mgr->pm, + .dentry = gxp->d_entry, + .node_name = GXP_COOLING_NAME, + .type = GXP_COOLING_NAME, + .data = gxp, + .get_rate = gxp_thermal_get_rate, + .set_rate = gxp_thermal_set_rate, + .control = gxp_thermal_control, + }; + struct gcip_thermal *thermal; + + if (gxp->thermal) + return -EEXIST; + + thermal = gcip_thermal_create(&args); + if (IS_ERR(thermal)) + return PTR_ERR(thermal); + + gxp->thermal = thermal; - d = debugfs_create_dir("cooling", thermal->gxp->d_entry); - /* don't let debugfs creation failure abort the init procedure */ - if (IS_ERR_OR_NULL(d)) - dev_warn(dev, "failed to create debug fs for cooling"); - thermal->cooling_root = d; - - err = gxp_thermal_cooling_register(thermal, GXP_COOLING_NAME); - if (err) { - dev_err(dev, "failed to initialize external cooling\n"); - gxp_thermal_exit(thermal); - return err; - } return 0; } -struct gxp_thermal_manager -*gxp_thermal_init(struct gxp_dev *gxp) +void gxp_thermal_exit(struct gxp_dev *gxp) { - struct device *dev = gxp->dev; - struct gxp_thermal_manager *thermal; - int err; - - thermal = devres_alloc(devm_gxp_thermal_release, sizeof(*thermal), - GFP_KERNEL); - if (!thermal) - return ERR_PTR(-ENOMEM); - - thermal->gxp = gxp; - err = cooling_init(thermal, dev); - if (err) { - devres_free(thermal); - return ERR_PTR(err); - } - - devres_add(dev, thermal); - return thermal; + gcip_thermal_destroy(gxp->thermal); + gxp->thermal = NULL; } diff --git a/gxp-thermal.h b/gxp-thermal.h index c1939ef..aa4fe3a 100644 --- a/gxp-thermal.h +++ b/gxp-thermal.h @@ -2,43 +2,18 @@ /* * Platform thermal driver for GXP. * - * Copyright (C) 2021 Google LLC + * Copyright (C) 2021-2023 Google LLC */ #ifndef __GXP_THERMAL_H__ #define __GXP_THERMAL_H__ -#include <linux/debugfs.h> -#include <linux/device.h> -#include <linux/mutex.h> -#include <linux/thermal.h> +#include <gcip/gcip-thermal.h> #include "gxp-internal.h" -#include "gxp-pm.h" #define GXP_COOLING_NAME "gxp-cooling" -struct gxp_thermal_manager { - struct dentry *cooling_root; - struct thermal_cooling_device *cdev; - struct mutex lock; - void *op_data; - unsigned long cooling_state; - unsigned long sysfs_req; - unsigned int gxp_num_states; - struct gxp_dev *gxp; - bool thermal_suspended; /* GXP thermal suspended state */ -}; - -/* - * Internal structure to do the state/pwr mapping - * state: kHz that AUR is running - * power: mW that the state consume - */ -struct gxp_state_pwr { - unsigned long state; - u32 power; -}; - -struct gxp_thermal_manager *gxp_thermal_init(struct gxp_dev *gxp); +int gxp_thermal_init(struct gxp_dev *gxp); +void gxp_thermal_exit(struct gxp_dev *gxp); #endif /* __GXP_THERMAL_H__ */ @@ -23,8 +23,9 @@ #include "gxp-domain-pool.h" #include "gxp-doorbell.h" #include "gxp-eventfd.h" -#include "gxp-firmware.h" #include "gxp-firmware-data.h" +#include "gxp-firmware-loader.h" +#include "gxp-firmware.h" #include "gxp-host-device-structs.h" #include "gxp-internal.h" #include "gxp-lpm.h" @@ -200,32 +201,19 @@ static int map_cfg_regions(struct gxp_virtual_device *vd, struct gcip_image_config *img_cfg) { struct gxp_dev *gxp = vd->gxp; - struct gxp_mapped_resource *pool; - struct gxp_mapped_resource res, tmp; + struct gxp_mapped_resource pool; + struct gxp_mapped_resource res; size_t offset; int ret; - if (img_cfg->num_iommu_mappings < 2) + if (img_cfg->num_iommu_mappings < 3) return map_core_shared_buffer(vd); - - /* - * For direct mode, the config regions are programmed by host (us); for - * MCU mode, the config regions are programmed by MCU. - */ - if (gxp_is_direct_mode(gxp)) { - tmp = gxp->fwdatabuf; - /* Leave the first piece be used for gxp_fw_data_init() */ - tmp.vaddr += tmp.size / 2; - tmp.paddr += tmp.size / 2; - pool = &tmp; - } else { - pool = &gxp->shared_buf; - } + pool = gxp_fw_data_resource(gxp); assign_resource(&res, img_cfg, CORE_CFG_REGION_IDX); offset = vd->slice_index * GXP_SHARED_SLICE_SIZE; - res.vaddr = pool->vaddr + offset; - res.paddr = pool->paddr + offset; + res.vaddr = pool.vaddr + offset; + res.paddr = pool.paddr + offset; ret = map_resource(vd, &res); if (ret) { dev_err(gxp->dev, "map core config %pad -> offset %#zx failed", @@ -236,8 +224,8 @@ static int map_cfg_regions(struct gxp_virtual_device *vd, assign_resource(&res, img_cfg, VD_CFG_REGION_IDX); offset += vd->core_cfg.size; - res.vaddr = pool->vaddr + offset; - res.paddr = pool->paddr + offset; + res.vaddr = pool.vaddr + offset; + res.paddr = pool.paddr + offset; ret = map_resource(vd, &res); if (ret) { dev_err(gxp->dev, "map VD config %pad -> offset %#zx failed", @@ -254,15 +242,15 @@ static int map_cfg_regions(struct gxp_virtual_device *vd, ret = -ENOSPC; goto err_unmap_vd; } - /* - * It's okay when mappings[sys_cfg_region_idx] is not set, in which case - * map_resource does nothing. - */ assign_resource(&res, img_cfg, SYS_CFG_REGION_IDX); - /* Use the end of the shared region for system cfg. */ - offset = GXP_SHARED_BUFFER_SIZE - res.size; - res.vaddr = pool->vaddr + offset; - res.paddr = pool->paddr + offset; + if (res.size != GXP_FW_DATA_SYSCFG_SIZE) { + dev_err(gxp->dev, "invalid system cfg size: %#llx", res.size); + ret = -EINVAL; + goto err_unmap_vd; + } + res.vaddr = gxp_fw_data_system_cfg(gxp); + offset = res.vaddr - pool.vaddr; + res.paddr = pool.paddr + offset; ret = map_resource(vd, &res); if (ret) { dev_err(gxp->dev, "map sys config %pad -> offset %#zx failed", @@ -314,9 +302,9 @@ static void gxp_vd_imgcfg_unmap(void *data, dma_addr_t daddr, size_t size, unmap_ns_region(vd, daddr); } -static int map_fw_image_config(struct gxp_dev *gxp, - struct gxp_virtual_device *vd, - struct gxp_firmware_manager *fw_mgr) +static int +map_fw_image_config(struct gxp_dev *gxp, struct gxp_virtual_device *vd, + struct gxp_firmware_loader_manager *fw_loader_mgr) { int ret; struct gcip_image_config *cfg; @@ -325,12 +313,7 @@ static int map_fw_image_config(struct gxp_dev *gxp, .unmap = gxp_vd_imgcfg_unmap, }; - /* - * Allow to skip for test suites need VD but doesn't need the FW module. - */ - if (IS_ENABLED(CONFIG_GXP_TEST) && !fw_mgr) - return 0; - cfg = &fw_mgr->img_cfg; + cfg = &fw_loader_mgr->core_img_cfg; ret = gcip_image_config_parser_init(&vd->cfg_parser, &gxp_vd_imgcfg_ops, gxp->dev, vd); /* parser_init() never fails unless we pass invalid OPs. */ @@ -568,7 +551,7 @@ static int assign_cores(struct gxp_virtual_device *vd) uint core; uint available_cores = 0; - if (!gxp_core_boot) { + if (!gxp_core_boot(gxp)) { /* We don't do core assignment when cores are managed by MCU. */ vd->core_list = BIT(GXP_NUM_CORES) - 1; return 0; @@ -598,7 +581,7 @@ static void unassign_cores(struct gxp_virtual_device *vd) struct gxp_dev *gxp = vd->gxp; uint core; - if (!gxp_core_boot) + if (!gxp_core_boot(gxp)) return; for (core = 0; core < GXP_NUM_CORES; core++) { if (gxp->core_to_vd[core] == vd) @@ -645,8 +628,7 @@ static void vd_restore_doorbells(struct gxp_virtual_device *vd) static void set_config_version(struct gxp_dev *gxp, struct gxp_virtual_device *vd) { - if (gxp->firmware_mgr && vd->sys_cfg.daddr) - vd->config_version = gxp->firmware_mgr->img_cfg.config_version; + vd->config_version = gxp->fw_loader_mgr->core_img_cfg.config_version; /* * Let gxp_dma_map_core_resources() map this region only when using the * legacy protocol. @@ -744,24 +726,17 @@ struct gxp_virtual_device *gxp_vd_allocate(struct gxp_dev *gxp, * Here assumes firmware is requested before allocating a VD, which is * true because we request firmware on first GXP device open. */ - err = map_fw_image_config(gxp, vd, gxp->firmware_mgr); + err = map_fw_image_config(gxp, vd, gxp->fw_loader_mgr); if (err) goto error_unassign_cores; set_config_version(gxp, vd); - if (gxp->data_mgr) { - /* After map_fw_image_config because it needs vd->sys_cfg. */ - vd->fw_app = gxp_fw_data_create_app(gxp, vd); - if (IS_ERR(vd->fw_app)) { - err = PTR_ERR(vd->fw_app); - vd->fw_app = NULL; - goto error_unmap_imgcfg; - } - } + /* After map_fw_image_config because it needs vd->vd/core_cfg. */ + gxp_fw_data_populate_vd_cfg(gxp, vd); err = gxp_dma_map_core_resources(gxp, vd->domain, vd->core_list, vd->slice_index); if (err) - goto error_destroy_fw_data; + goto error_unmap_imgcfg; err = alloc_and_map_fw_image(gxp, vd); if (err) goto error_unmap_core_resources; @@ -780,8 +755,6 @@ error_unmap_fw_data: unmap_and_free_fw_image(gxp, vd); error_unmap_core_resources: gxp_dma_unmap_core_resources(gxp, vd->domain, vd->core_list); -error_destroy_fw_data: - gxp_fw_data_destroy_app(gxp, vd->fw_app); error_unmap_imgcfg: unmap_fw_image_config(gxp, vd); error_unassign_cores: @@ -819,7 +792,6 @@ void gxp_vd_release(struct gxp_virtual_device *vd) unmap_core_telemetry_buffers(gxp, vd, core_list); unmap_and_free_fw_image(gxp, vd); gxp_dma_unmap_core_resources(gxp, vd->domain, core_list); - gxp_fw_data_destroy_app(gxp, vd->fw_app); unmap_fw_image_config(gxp, vd); unassign_cores(vd); @@ -894,11 +866,12 @@ int gxp_vd_run(struct gxp_virtual_device *vd) { struct gxp_dev *gxp = vd->gxp; int ret; + enum gxp_virtual_device_state orig_state = vd->state; lockdep_assert_held_write(&gxp->vd_semaphore); - if (vd->state != GXP_VD_READY && vd->state != GXP_VD_OFF) + if (orig_state != GXP_VD_READY && orig_state != GXP_VD_OFF) return -EINVAL; - if (vd->state == GXP_VD_OFF) { + if (orig_state == GXP_VD_OFF) { ret = gxp_vd_block_ready(vd); /* * The failure of `gxp_vd_block_ready` function means following two things: @@ -933,7 +906,9 @@ int gxp_vd_run(struct gxp_virtual_device *vd) err_vd_block_unready: debug_dump_unlock(vd); - gxp_vd_block_unready(vd); + /* Run this only when gxp_vd_block_ready was executed. */ + if (orig_state == GXP_VD_OFF) + gxp_vd_block_unready(vd); err_vd_unavailable: vd->state = GXP_VD_UNAVAILABLE; return ret; @@ -968,19 +943,31 @@ void gxp_vd_stop(struct gxp_virtual_device *vd) lockdep_assert_held_write(&gxp->vd_semaphore); debug_dump_lock(gxp, vd); - if (gxp_core_boot && + if (gxp_core_boot(gxp) && (vd->state == GXP_VD_OFF || vd->state == GXP_VD_READY || vd->state == GXP_VD_RUNNING) && gxp_pm_get_blk_state(gxp) != AUR_OFF) { - /* - * Put all cores in the VD into reset so they can not wake each other up - */ + for (phys_core = 0; phys_core < GXP_NUM_CORES; phys_core++) { if (core_list & BIT(phys_core)) { - lpm_state = gxp_lpm_get_state( - gxp, CORE_TO_PSM(phys_core)); - if (lpm_state != LPM_PG_STATE) + + lpm_state = gxp_lpm_get_state(gxp, CORE_TO_PSM(phys_core)); + + if (lpm_state == LPM_ACTIVE_STATE) { + /* + * If the core is in PS0 (not idle), it should + * be held in reset before attempting SW PG. + */ hold_core_in_reset(gxp, phys_core); + } else { + /* + * If the core is idle and has already transtioned to PS1, + * we can attempt HW PG. In this case, we should ensure + * that the core doesn't get awakened by an external + * interrupt source before we attempt to HW PG the core. + */ + gxp_firmware_disable_ext_interrupts(gxp, phys_core); + } } } } @@ -1040,7 +1027,7 @@ void gxp_vd_suspend(struct gxp_virtual_device *vd) u32 boot_state; uint failed_cores = 0; - if (!gxp_is_direct_mode(gxp) && gxp_core_boot) + if (!gxp_is_direct_mode(gxp) && gxp_core_boot(gxp)) return gxp_vd_stop(vd); lockdep_assert_held_write(&gxp->vd_semaphore); debug_dump_lock(gxp, vd); @@ -1052,7 +1039,7 @@ void gxp_vd_suspend(struct gxp_virtual_device *vd) "Attempt to suspend a virtual device twice\n"); goto out; } - if (!gxp_core_boot) { + if (!gxp_core_boot(gxp)) { vd->state = GXP_VD_SUSPENDED; goto out; } @@ -1162,7 +1149,7 @@ int gxp_vd_resume(struct gxp_virtual_device *vd) ret = -EBUSY; goto out; } - if (!gxp_core_boot) { + if (!gxp_core_boot(gxp)) { vd->state = GXP_VD_RUNNING; goto out; } @@ -13,7 +13,7 @@ /* Interface Version */ #define GXP_INTERFACE_VERSION_MAJOR 1 -#define GXP_INTERFACE_VERSION_MINOR 10 +#define GXP_INTERFACE_VERSION_MINOR 11 #define GXP_INTERFACE_VERSION_BUILD 0 /* |