diff options
Diffstat (limited to 'dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c')
| -rw-r--r-- | dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c | 611 |
1 files changed, 611 insertions, 0 deletions
diff --git a/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c new file mode 100644 index 0000000..8ecf235 --- /dev/null +++ b/dvalin/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c @@ -0,0 +1,611 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2019-2021 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#include <tl/mali_kbase_tracepoints.h> + +#include "mali_kbase_csf_tiler_heap.h" +#include "mali_kbase_csf_tiler_heap_def.h" +#include "mali_kbase_csf_heap_context_alloc.h" + +/** + * encode_chunk_ptr - Encode the address and size of a chunk as an integer. + * + * The size and address of the next chunk in a list are packed into a single + * 64-bit value for storage in a chunk's header. This function returns that + * value. + * + * @chunk_size: Size of a tiler heap chunk, in bytes. + * @chunk_addr: GPU virtual address of the same tiler heap chunk. + * + * Return: Next chunk pointer suitable for writing into a chunk header. + */ +static u64 encode_chunk_ptr(u32 const chunk_size, u64 const chunk_addr) +{ + u64 encoded_size, encoded_addr; + + WARN_ON(chunk_size & ~CHUNK_SIZE_MASK); + WARN_ON(chunk_addr & ~CHUNK_ADDR_MASK); + + encoded_size = + (u64)(chunk_size >> CHUNK_HDR_NEXT_SIZE_ENCODE_SHIFT) << + CHUNK_HDR_NEXT_SIZE_POS; + + encoded_addr = + (chunk_addr >> CHUNK_HDR_NEXT_ADDR_ENCODE_SHIFT) << + CHUNK_HDR_NEXT_ADDR_POS; + + return (encoded_size & CHUNK_HDR_NEXT_SIZE_MASK) | + (encoded_addr & CHUNK_HDR_NEXT_ADDR_MASK); +} + +/** + * get_last_chunk - Get the last chunk of a tiler heap + * + * @heap: Pointer to the tiler heap. + * + * Return: The address of the most recently-linked chunk, or NULL if none. + */ +static struct kbase_csf_tiler_heap_chunk *get_last_chunk( + struct kbase_csf_tiler_heap *const heap) +{ + lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); + + if (list_empty(&heap->chunks_list)) + return NULL; + + return list_last_entry(&heap->chunks_list, + struct kbase_csf_tiler_heap_chunk, link); +} + +/** + * link_chunk - Link a chunk into a tiler heap + * + * Unless the @chunk is the first in the kernel's list of chunks belonging to + * a given tiler heap, this function stores the size and address of the @chunk + * in the header of the preceding chunk. This requires the GPU memory region + * containing the header to be be mapped temporarily, which can fail. + * + * @heap: Pointer to the tiler heap. + * @chunk: Pointer to the heap chunk to be linked. + * + * Return: 0 if successful or a negative error code on failure. + */ +static int link_chunk(struct kbase_csf_tiler_heap *const heap, + struct kbase_csf_tiler_heap_chunk *const chunk) +{ + struct kbase_csf_tiler_heap_chunk *const prev = get_last_chunk(heap); + + if (prev) { + struct kbase_context *const kctx = heap->kctx; + struct kbase_vmap_struct map; + u64 *const prev_hdr = kbase_vmap_prot(kctx, prev->gpu_va, + sizeof(*prev_hdr), KBASE_REG_CPU_WR, &map); + + if (unlikely(!prev_hdr)) { + dev_err(kctx->kbdev->dev, + "Failed to map tiler heap chunk 0x%llX\n", + prev->gpu_va); + return -ENOMEM; + } + + *prev_hdr = encode_chunk_ptr(heap->chunk_size, chunk->gpu_va); + kbase_vunmap(kctx, &map); + + dev_dbg(kctx->kbdev->dev, + "Linked tiler heap chunks, 0x%llX -> 0x%llX\n", + prev->gpu_va, chunk->gpu_va); + } + + return 0; +} + +/** + * init_chunk - Initialize and link a tiler heap chunk + * + * Zero-initialize a new chunk's header (including its pointer to the next + * chunk, which doesn't exist yet) and then update the previous chunk's + * header to link the new chunk into the chunk list. + * + * @heap: Pointer to the tiler heap. + * @chunk: Pointer to the heap chunk to be initialized and linked. + * @link_with_prev: Flag to indicate if the new chunk needs to be linked with + * the previously allocated chunk. + * + * Return: 0 if successful or a negative error code on failure. + */ +static int init_chunk(struct kbase_csf_tiler_heap *const heap, + struct kbase_csf_tiler_heap_chunk *const chunk, bool link_with_prev) +{ + struct kbase_vmap_struct map; + struct u64 *chunk_hdr = NULL; + struct kbase_context *const kctx = heap->kctx; + + if (unlikely(chunk->gpu_va & ~CHUNK_ADDR_MASK)) { + dev_err(kctx->kbdev->dev, + "Tiler heap chunk address is unusable\n"); + return -EINVAL; + } + + chunk_hdr = kbase_vmap_prot(kctx, + chunk->gpu_va, CHUNK_HDR_SIZE, KBASE_REG_CPU_WR, &map); + + if (unlikely(!chunk_hdr)) { + dev_err(kctx->kbdev->dev, + "Failed to map a tiler heap chunk header\n"); + return -ENOMEM; + } + + memset(chunk_hdr, 0, CHUNK_HDR_SIZE); + kbase_vunmap(kctx, &map); + + if (link_with_prev) + return link_chunk(heap, chunk); + else + return 0; +} + +/** + * create_chunk - Create a tiler heap chunk + * + * This function allocates a chunk of memory for a tiler heap and adds it to + * the end of the list of chunks associated with that heap. The size of the + * chunk is not a parameter because it is configured per-heap not per-chunk. + * + * @heap: Pointer to the tiler heap for which to allocate memory. + * @link_with_prev: Flag to indicate if the chunk to be allocated needs to be + * linked with the previously allocated chunk. + * + * Return: 0 if successful or a negative error code on failure. + */ +static int create_chunk(struct kbase_csf_tiler_heap *const heap, + bool link_with_prev) +{ + int err = 0; + struct kbase_context *const kctx = heap->kctx; + u64 nr_pages = PFN_UP(heap->chunk_size); + u64 flags = BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | + BASE_MEM_PROT_CPU_WR | BASEP_MEM_NO_USER_FREE | + BASE_MEM_COHERENT_LOCAL; + struct kbase_csf_tiler_heap_chunk *chunk = NULL; + + flags |= base_mem_group_id_set(kctx->jit_group_id); + +#if defined(CONFIG_MALI_DEBUG) || defined(CONFIG_MALI_VECTOR_DUMP) + flags |= BASE_MEM_PROT_CPU_RD; +#endif + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + chunk = kzalloc(sizeof(*chunk), GFP_KERNEL); + if (unlikely(!chunk)) { + dev_err(kctx->kbdev->dev, + "No kernel memory for a new tiler heap chunk\n"); + return -ENOMEM; + } + + /* Allocate GPU memory for the new chunk. */ + INIT_LIST_HEAD(&chunk->link); + chunk->region = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, + &flags, &chunk->gpu_va); + + if (unlikely(!chunk->region)) { + dev_err(kctx->kbdev->dev, + "Failed to allocate a tiler heap chunk\n"); + err = -ENOMEM; + } else { + err = init_chunk(heap, chunk, link_with_prev); + if (unlikely(err)) { + kbase_gpu_vm_lock(kctx); + chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; + kbase_mem_free_region(kctx, chunk->region); + kbase_gpu_vm_unlock(kctx); + } + } + + if (unlikely(err)) { + kfree(chunk); + } else { + list_add_tail(&chunk->link, &heap->chunks_list); + heap->chunk_count++; + + dev_dbg(kctx->kbdev->dev, "Created tiler heap chunk 0x%llX\n", + chunk->gpu_va); + } + + return err; +} + +/** + * delete_chunk - Delete a tiler heap chunk + * + * This function frees a tiler heap chunk previously allocated by @create_chunk + * and removes it from the list of chunks associated with the heap. + * + * WARNING: The deleted chunk is not unlinked from the list of chunks used by + * the GPU, therefore it is only safe to use this function when + * deleting a heap. + * + * @heap: Pointer to the tiler heap for which @chunk was allocated. + * @chunk: Pointer to a chunk to be deleted. + */ +static void delete_chunk(struct kbase_csf_tiler_heap *const heap, + struct kbase_csf_tiler_heap_chunk *const chunk) +{ + struct kbase_context *const kctx = heap->kctx; + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + kbase_gpu_vm_lock(kctx); + chunk->region->flags &= ~KBASE_REG_NO_USER_FREE; + kbase_mem_free_region(kctx, chunk->region); + kbase_gpu_vm_unlock(kctx); + list_del(&chunk->link); + heap->chunk_count--; + kfree(chunk); +} + +/** + * delete_all_chunks - Delete all chunks belonging to a tiler heap + * + * This function empties the list of chunks associated with a tiler heap by + * freeing all chunks previously allocated by @create_chunk. + * + * @heap: Pointer to a tiler heap. + */ +static void delete_all_chunks(struct kbase_csf_tiler_heap *heap) +{ + struct list_head *entry = NULL, *tmp = NULL; + struct kbase_context *const kctx = heap->kctx; + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + list_for_each_safe(entry, tmp, &heap->chunks_list) { + struct kbase_csf_tiler_heap_chunk *chunk = list_entry( + entry, struct kbase_csf_tiler_heap_chunk, link); + + delete_chunk(heap, chunk); + } +} + +/** + * create_initial_chunks - Create the initial list of chunks for a tiler heap + * + * This function allocates a given number of chunks for a tiler heap and + * adds them to the list of chunks associated with that heap. + * + * @heap: Pointer to the tiler heap for which to allocate memory. + * @nchunks: Number of chunks to create. + * + * Return: 0 if successful or a negative error code on failure. + */ +static int create_initial_chunks(struct kbase_csf_tiler_heap *const heap, + u32 const nchunks) +{ + int err = 0; + u32 i; + + for (i = 0; (i < nchunks) && likely(!err); i++) + err = create_chunk(heap, true); + + if (unlikely(err)) + delete_all_chunks(heap); + + return err; +} + +/** + * delete_heap - Delete a tiler heap + * + * This function frees any chunks allocated for a tiler heap previously + * initialized by @kbase_csf_tiler_heap_init and removes it from the list of + * heaps associated with the kbase context. The heap context structure used by + * the firmware is also freed. + * + * @heap: Pointer to a tiler heap to be deleted. + */ +static void delete_heap(struct kbase_csf_tiler_heap *heap) +{ + struct kbase_context *const kctx = heap->kctx; + + dev_dbg(kctx->kbdev->dev, "Deleting tiler heap 0x%llX\n", heap->gpu_va); + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + delete_all_chunks(heap); + + /* We could optimize context destruction by not freeing leaked heap + * contexts but it doesn't seem worth the extra complexity. + */ + kbase_csf_heap_context_allocator_free(&kctx->csf.tiler_heaps.ctx_alloc, + heap->gpu_va); + + list_del(&heap->link); + + WARN_ON(heap->chunk_count); + KBASE_TLSTREAM_AUX_TILER_HEAP_STATS(kctx->kbdev, kctx->id, + heap->heap_id, 0, 0, heap->max_chunks, heap->chunk_size, 0, + heap->target_in_flight, 0); + + kfree(heap); +} + +/** + * find_tiler_heap - Find a tiler heap from the address of its heap context + * + * Each tiler heap managed by the kernel has an associated heap context + * structure used by the firmware. This function finds a tiler heap object from + * the GPU virtual address of its associated heap context. The heap context + * should have been allocated by @kbase_csf_heap_context_allocator_alloc in the + * same @kctx. + * + * @kctx: Pointer to the kbase context to search for a tiler heap. + * @heap_gpu_va: GPU virtual address of a heap context structure. + * + * Return: pointer to the tiler heap object, or NULL if not found. + */ +static struct kbase_csf_tiler_heap *find_tiler_heap( + struct kbase_context *const kctx, u64 const heap_gpu_va) +{ + struct kbase_csf_tiler_heap *heap = NULL; + + lockdep_assert_held(&kctx->csf.tiler_heaps.lock); + + list_for_each_entry(heap, &kctx->csf.tiler_heaps.list, link) { + if (heap_gpu_va == heap->gpu_va) + return heap; + } + + dev_dbg(kctx->kbdev->dev, "Tiler heap 0x%llX was not found\n", + heap_gpu_va); + + return NULL; +} + +int kbase_csf_tiler_heap_context_init(struct kbase_context *const kctx) +{ + int err = kbase_csf_heap_context_allocator_init( + &kctx->csf.tiler_heaps.ctx_alloc, kctx); + + if (unlikely(err)) + return err; + + INIT_LIST_HEAD(&kctx->csf.tiler_heaps.list); + mutex_init(&kctx->csf.tiler_heaps.lock); + + dev_dbg(kctx->kbdev->dev, "Initialized a context for tiler heaps\n"); + + return 0; +} + +void kbase_csf_tiler_heap_context_term(struct kbase_context *const kctx) +{ + struct list_head *entry = NULL, *tmp = NULL; + + dev_dbg(kctx->kbdev->dev, "Terminating a context for tiler heaps\n"); + + mutex_lock(&kctx->csf.tiler_heaps.lock); + + list_for_each_safe(entry, tmp, &kctx->csf.tiler_heaps.list) { + struct kbase_csf_tiler_heap *heap = list_entry( + entry, struct kbase_csf_tiler_heap, link); + delete_heap(heap); + } + + mutex_unlock(&kctx->csf.tiler_heaps.lock); + mutex_destroy(&kctx->csf.tiler_heaps.lock); + + kbase_csf_heap_context_allocator_term(&kctx->csf.tiler_heaps.ctx_alloc); +} + +int kbase_csf_tiler_heap_init(struct kbase_context *const kctx, + u32 const chunk_size, u32 const initial_chunks, u32 const max_chunks, + u16 const target_in_flight, u64 *const heap_gpu_va, + u64 *const first_chunk_va) +{ + int err = 0; + struct kbase_csf_tiler_heap *heap = NULL; + struct kbase_csf_heap_context_allocator *const ctx_alloc = + &kctx->csf.tiler_heaps.ctx_alloc; + + dev_dbg(kctx->kbdev->dev, + "Creating a tiler heap with %u chunks (limit: %u) of size %u\n", + initial_chunks, max_chunks, chunk_size); + + if (chunk_size == 0) + return -EINVAL; + + if (chunk_size & ~CHUNK_SIZE_MASK) + return -EINVAL; + + if (initial_chunks == 0) + return -EINVAL; + + if (initial_chunks > max_chunks) + return -EINVAL; + + if (target_in_flight == 0) + return -EINVAL; + + heap = kzalloc(sizeof(*heap), GFP_KERNEL); + if (unlikely(!heap)) { + dev_err(kctx->kbdev->dev, + "No kernel memory for a new tiler heap\n"); + return -ENOMEM; + } + + heap->kctx = kctx; + heap->chunk_size = chunk_size; + heap->max_chunks = max_chunks; + heap->target_in_flight = target_in_flight; + INIT_LIST_HEAD(&heap->chunks_list); + + heap->gpu_va = kbase_csf_heap_context_allocator_alloc(ctx_alloc); + + mutex_lock(&kctx->csf.tiler_heaps.lock); + + if (unlikely(!heap->gpu_va)) { + dev_err(kctx->kbdev->dev, + "Failed to allocate a tiler heap context\n"); + err = -ENOMEM; + } else { + err = create_initial_chunks(heap, initial_chunks); + if (unlikely(err)) { + kbase_csf_heap_context_allocator_free(ctx_alloc, + heap->gpu_va); + } + } + + if (unlikely(err)) { + kfree(heap); + } else { + struct kbase_csf_tiler_heap_chunk const *first_chunk = + list_first_entry(&heap->chunks_list, + struct kbase_csf_tiler_heap_chunk, link); + + kctx->csf.tiler_heaps.nr_of_heaps++; + heap->heap_id = kctx->csf.tiler_heaps.nr_of_heaps; + list_add(&heap->link, &kctx->csf.tiler_heaps.list); + + *heap_gpu_va = heap->gpu_va; + *first_chunk_va = first_chunk->gpu_va; + + KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( + kctx->kbdev, kctx->id, heap->heap_id, + PFN_UP(heap->chunk_size * heap->max_chunks), + PFN_UP(heap->chunk_size * heap->chunk_count), + heap->max_chunks, heap->chunk_size, heap->chunk_count, + heap->target_in_flight, 0); + + dev_dbg(kctx->kbdev->dev, "Created tiler heap 0x%llX\n", + heap->gpu_va); + } + + mutex_unlock(&kctx->csf.tiler_heaps.lock); + + return err; +} + +int kbase_csf_tiler_heap_term(struct kbase_context *const kctx, + u64 const heap_gpu_va) +{ + int err = 0; + struct kbase_csf_tiler_heap *heap = NULL; + + mutex_lock(&kctx->csf.tiler_heaps.lock); + + heap = find_tiler_heap(kctx, heap_gpu_va); + if (likely(heap)) + delete_heap(heap); + else + err = -EINVAL; + + mutex_unlock(&kctx->csf.tiler_heaps.lock); + + return err; +} + +/** + * alloc_new_chunk - Allocate a new chunk for the tiler heap. + * + * This function will allocate a new chunk for the chunked tiler heap depending + * on the settings provided by userspace when the heap was created and the + * heap's statistics (like number of render passes in-flight). + * + * @heap: Pointer to the tiler heap. + * @nr_in_flight: Number of render passes that are in-flight, must not be zero. + * @pending_frag_count: Number of render passes in-flight with completed vertex/tiler stage. + * The minimum value is zero but it must be less or equal to + * the total number of render passes in flight + * @new_chunk_ptr: Where to store the GPU virtual address & size of the new + * chunk allocated for the heap. + * + * Return: 0 if a new chunk was allocated otherwise an appropriate negative + * error code. + */ +static int alloc_new_chunk(struct kbase_csf_tiler_heap *heap, + u32 nr_in_flight, u32 pending_frag_count, u64 *new_chunk_ptr) +{ + int err = -ENOMEM; + + lockdep_assert_held(&heap->kctx->csf.tiler_heaps.lock); + + if (WARN_ON(!nr_in_flight) || + WARN_ON(pending_frag_count > nr_in_flight)) + return -EINVAL; + + if (nr_in_flight <= heap->target_in_flight) { + if (heap->chunk_count < heap->max_chunks) { + /* Not exceeded the target number of render passes yet so be + * generous with memory. + */ + err = create_chunk(heap, false); + + if (likely(!err)) { + struct kbase_csf_tiler_heap_chunk *new_chunk = + get_last_chunk(heap); + if (!WARN_ON(!new_chunk)) { + *new_chunk_ptr = + encode_chunk_ptr(heap->chunk_size, + new_chunk->gpu_va); + return 0; + } + } + } else if (pending_frag_count > 0) { + err = -EBUSY; + } else { + err = -ENOMEM; + } + } else { + /* Reached target number of render passes in flight. + * Wait for some of them to finish + */ + err = -EBUSY; + } + + return err; +} + +int kbase_csf_tiler_heap_alloc_new_chunk(struct kbase_context *kctx, + u64 gpu_heap_va, u32 nr_in_flight, u32 pending_frag_count, u64 *new_chunk_ptr) +{ + struct kbase_csf_tiler_heap *heap; + int err = -EINVAL; + + mutex_lock(&kctx->csf.tiler_heaps.lock); + + heap = find_tiler_heap(kctx, gpu_heap_va); + + if (likely(heap)) { + err = alloc_new_chunk(heap, nr_in_flight, pending_frag_count, + new_chunk_ptr); + + KBASE_TLSTREAM_AUX_TILER_HEAP_STATS( + kctx->kbdev, kctx->id, heap->heap_id, + PFN_UP(heap->chunk_size * heap->max_chunks), + PFN_UP(heap->chunk_size * heap->chunk_count), + heap->max_chunks, heap->chunk_size, heap->chunk_count, + heap->target_in_flight, nr_in_flight); + } + + mutex_unlock(&kctx->csf.tiler_heaps.lock); + + return err; +} |