this is the flle which have the wrong part. Can you fix it for me please ? it’s not easy for me because the string you told me to remove don’t match perfectly.
I found it here : /mnt/fisso/jetson_nano_kvm/linux-tegra-4.9-32.6/drivers/base/dma-coherent.c
/*
* Coherent per-device memory handling.
* Borrowed from i386
*/
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/dma-mapping.h>
#include <linux/dma-contiguous.h>
#include <linux/debugfs.h>
#include <linux/highmem.h>
#include <linux/kthread.h>
#ifdef pr_fmt
#undef pr_fmt
#endif
#define pr_fmt(fmt) "%s:%d: " fmt, __func__, __LINE__
#define RESIZE_MAGIC 0xC11A900d
struct heap_info {
int magic;
char *name;
/* number of chunks memory to manage in */
unsigned int num_chunks;
/* dev to manage cma/coherent memory allocs, if resize allowed */
struct device dev;
/* device to allocate memory from cma */
struct device *cma_dev;
/* lock to synchronise heap resizing */
struct mutex resize_lock;
/* CMA chunk size if resize supported */
size_t cma_chunk_size;
/* heap current base */
phys_addr_t curr_base;
/* heap current allocated memory in bytes */
size_t curr_used;
/* heap current length */
size_t curr_len;
/* heap lowest base */
phys_addr_t cma_base;
/* heap max length */
size_t cma_len;
size_t rem_chunk_size;
struct dentry *dma_debug_root;
int (*update_resize_cfg)(phys_addr_t , size_t);
/* The timer used to wakeup the shrink thread */
struct timer_list shrink_timer;
/* Pointer to the current shrink thread for this resizable heap */
struct task_struct *task;
unsigned long shrink_interval;
size_t floor_size;
};
#ifdef CONFIG_ARM_DMA_IOMMU_ALIGNMENT
#define DMA_BUF_ALIGNMENT CONFIG_ARM_DMA_IOMMU_ALIGNMENT
#else
#define DMA_BUF_ALIGNMENT 8
#endif
struct dma_coherent_mem {
void *virt_base;
dma_addr_t device_base;
unsigned long pfn_base;
size_t size;
int flags;
unsigned long alloc_shift;
unsigned long *bitmap;
spinlock_t spinlock;
};
static int shrink_thread(void *arg);
static void shrink_resizable_heap(struct heap_info *h);
static int heap_resize_locked(struct heap_info *h, bool skip_vpr_config);
static void release_from_contiguous_heap(struct heap_info *h, phys_addr_t base,
size_t len);
static int update_vpr_config(struct heap_info *h);
#define RESIZE_DEFAULT_SHRINK_AGE 3
bool dma_is_coherent_dev(struct device *dev)
{
struct heap_info *h;
if (!dev)
return false;
h = dev_get_drvdata(dev);
if (!h)
return false;
if (h->magic != RESIZE_MAGIC)
return false;
return true;
}
EXPORT_SYMBOL(dma_is_coherent_dev);
static void dma_debugfs_init(struct device *dev, struct heap_info *heap)
{
if (!heap->dma_debug_root) {
heap->dma_debug_root = debugfs_create_dir(dev_name(dev), NULL);
if (IS_ERR_OR_NULL(heap->dma_debug_root)) {
dev_err(dev, "couldn't create debug files\n");
return;
}
}
if (sizeof(phys_addr_t) == sizeof(u64)) {
debugfs_create_x64("curr_base", S_IRUGO,
heap->dma_debug_root, (u64 *)&heap->curr_base);
debugfs_create_x64("curr_size", S_IRUGO,
heap->dma_debug_root, (u64 *)&heap->curr_len);
debugfs_create_x64("cma_base", S_IRUGO,
heap->dma_debug_root, (u64 *)&heap->cma_base);
debugfs_create_x64("cma_size", S_IRUGO,
heap->dma_debug_root, (u64 *)&heap->cma_len);
debugfs_create_x64("cma_chunk_size", S_IRUGO,
heap->dma_debug_root, (u64 *)&heap->cma_chunk_size);
debugfs_create_x64("floor_size", S_IRUGO,
heap->dma_debug_root, (u64 *)&heap->floor_size);
} else {
debugfs_create_x32("curr_base", S_IRUGO,
heap->dma_debug_root, (u32 *)&heap->curr_base);
debugfs_create_x32("curr_size", S_IRUGO,
heap->dma_debug_root, (u32 *)&heap->curr_len);
debugfs_create_x32("cma_base", S_IRUGO,
heap->dma_debug_root, (u32 *)&heap->cma_base);
debugfs_create_x32("cma_size", S_IRUGO,
heap->dma_debug_root, (u32 *)&heap->cma_len);
debugfs_create_x32("cma_chunk_size", S_IRUGO,
heap->dma_debug_root, (u32 *)&heap->cma_chunk_size);
debugfs_create_x32("floor_size", S_IRUGO,
heap->dma_debug_root, (u32 *)&heap->floor_size);
}
debugfs_create_x32("num_cma_chunks", S_IRUGO,
heap->dma_debug_root, (u32 *)&heap->num_chunks);
}
int dma_set_resizable_heap_floor_size(struct device *dev, size_t floor_size)
{
int ret = 0;
struct heap_info *h = NULL;
phys_addr_t orig_base, prev_base, left_chunks_base, right_chunks_base;
size_t orig_len, prev_len, left_chunks_len, right_chunks_len;
if (!dma_is_coherent_dev(dev))
return -ENODEV;
h = dev_get_drvdata(dev);
if (!h)
return -ENOENT;
mutex_lock(&h->resize_lock);
orig_base = h->curr_base;
orig_len = h->curr_len;
right_chunks_base = h->curr_base + h->curr_len;
left_chunks_len = right_chunks_len = 0;
h->floor_size = floor_size > h->cma_len ? h->cma_len : floor_size;
while (h->curr_len < h->floor_size) {
prev_base = h->curr_base;
prev_len = h->curr_len;
ret = heap_resize_locked(h, true);
if (ret)
goto fail_set_floor;
if (h->curr_base < prev_base) {
left_chunks_base = h->curr_base;
left_chunks_len += (h->curr_len - prev_len);
} else {
right_chunks_len += (h->curr_len - prev_len);
}
}
ret = update_vpr_config(h);
if (!ret) {
dev_dbg(&h->dev,
"grow heap base from=%pa to=%pa,"
" len from=0x%zx to=0x%zx\n",
&orig_base, &h->curr_base, orig_len, h->curr_len);
goto success_set_floor;
}
fail_set_floor:
if (left_chunks_len != 0)
release_from_contiguous_heap(h, left_chunks_base,
left_chunks_len);
if (right_chunks_len != 0)
release_from_contiguous_heap(h, right_chunks_base,
right_chunks_len);
h->curr_base = orig_base;
h->curr_len = orig_len;
success_set_floor:
if (h->task)
mod_timer(&h->shrink_timer, jiffies + h->shrink_interval);
mutex_unlock(&h->resize_lock);
if (!h->task)
shrink_resizable_heap(h);
return ret;
}
EXPORT_SYMBOL(dma_set_resizable_heap_floor_size);
static bool dma_init_coherent_memory(
phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags,
unsigned long alloc_shift, struct dma_coherent_mem **mem)
{
struct dma_coherent_mem *dma_mem = NULL;
void __iomem *mem_base = NULL;
int bits = size >> alloc_shift;
int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long);
if ((flags &
(DMA_MEMORY_MAP | DMA_MEMORY_IO | DMA_MEMORY_NOMAP)) == 0)
goto out;
if (!size)
goto out;
if (flags & DMA_MEMORY_NOMAP)
goto skip_mapping;
if (flags & DMA_MEMORY_MAP)
mem_base = memremap(phys_addr, size, MEMREMAP_WC);
else
mem_base = ioremap(phys_addr, size);
if (!mem_base)
goto out;
skip_mapping:
dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
if (!dma_mem)
goto out;
dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
if (!dma_mem->bitmap)
goto out;
dma_mem->virt_base = mem_base;
dma_mem->device_base = device_addr;
dma_mem->pfn_base = PFN_DOWN(phys_addr);
dma_mem->size = bits;
dma_mem->flags = flags;
dma_mem->alloc_shift = alloc_shift;
spin_lock_init(&dma_mem->spinlock);
*mem = dma_mem;
return true;
out:
kfree(dma_mem);
if (mem_base) {
if (flags & DMA_MEMORY_MAP)
memunmap(mem_base);
else
iounmap(mem_base);
}
return false;
}
static void dma_release_coherent_memory(struct dma_coherent_mem *mem)
{
if (!mem)
return;
if (!(mem->flags & DMA_MEMORY_NOMAP))
goto skip_unmapping;
if (mem->flags & DMA_MEMORY_MAP)
memunmap(mem->virt_base);
else
iounmap(mem->virt_base);
skip_unmapping:
kfree(mem->bitmap);
kfree(mem);
}
static int declare_coherent_heap(struct device *dev, phys_addr_t base,
size_t size, int map)
{
int err;
int flags = map ? DMA_MEMORY_MAP : DMA_MEMORY_NOMAP;
BUG_ON(dev->dma_mem);
dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
err = dma_declare_coherent_memory(dev, 0,
base, size, flags);
if (err & flags) {
dev_dbg(dev, "dma coherent mem base (%pa) size (0x%zx) %x\n",
&base, size, flags);
return 0;
}
dev_err(dev, "declare dma coherent_mem fail %pa 0x%zx %x\n",
&base, size, flags);
return -ENOMEM;
}
int dma_declare_coherent_resizable_cma_memory(struct device *dev,
struct dma_declare_info *dma_info)
{
#ifdef CONFIG_DMA_CMA
int err = 0;
struct heap_info *heap_info = NULL;
struct dma_contiguous_stats stats;
if (!dev || !dma_info || !dma_info->name || !dma_info->cma_dev)
return -EINVAL;
heap_info = kzalloc(sizeof(*heap_info), GFP_KERNEL);
if (!heap_info)
return -ENOMEM;
heap_info->magic = RESIZE_MAGIC;
heap_info->name = kmalloc(strlen(dma_info->name) + 1, GFP_KERNEL);
if (!heap_info->name) {
kfree(heap_info);
return -ENOMEM;
}
dma_get_contiguous_stats(dma_info->cma_dev, &stats);
pr_info("resizable heap=%s, base=%pa, size=0x%zx\n",
dma_info->name, &stats.base, stats.size);
strcpy(heap_info->name, dma_info->name);
dev_set_name(dev, "dma-%s", heap_info->name);
heap_info->cma_dev = dma_info->cma_dev;
heap_info->cma_chunk_size = dma_info->size ? : stats.size;
heap_info->cma_base = stats.base;
heap_info->cma_len = stats.size;
heap_info->curr_base = stats.base;
dev_set_name(heap_info->cma_dev, "cma-%s-heap", heap_info->name);
mutex_init(&heap_info->resize_lock);
if (heap_info->cma_len < heap_info->cma_chunk_size) {
dev_err(dev, "error cma_len(0x%zx) < cma_chunk_size(0x%zx)\n",
heap_info->cma_len, heap_info->cma_chunk_size);
err = -EINVAL;
goto fail;
}
heap_info->num_chunks = div64_u64_rem(heap_info->cma_len,
(u64)heap_info->cma_chunk_size, (u64 *)&heap_info->rem_chunk_size);
if (heap_info->rem_chunk_size) {
heap_info->num_chunks++;
dev_info(dev, "heap size is not multiple of cma_chunk_size "
"heap_info->num_chunks (%d) rem_chunk_size(0x%zx)\n",
heap_info->num_chunks, heap_info->rem_chunk_size);
} else
heap_info->rem_chunk_size = heap_info->cma_chunk_size;
dev_set_name(&heap_info->dev, "%s-heap", heap_info->name);
if (dma_info->notifier.ops)
heap_info->update_resize_cfg =
dma_info->notifier.ops->resize;
dev_set_drvdata(dev, heap_info);
dma_debugfs_init(dev, heap_info);
if (declare_coherent_heap(&heap_info->dev,
heap_info->cma_base, heap_info->cma_len,
(dma_info->notifier.ops &&
dma_info->notifier.ops->resize) ? 0 : 1))
goto declare_fail;
heap_info->dev.dma_mem->size = 0;
heap_info->shrink_interval = HZ * RESIZE_DEFAULT_SHRINK_AGE;
kthread_run(shrink_thread, heap_info, "%s-shrink_thread",
heap_info->name);
if (dma_info->notifier.ops && dma_info->notifier.ops->resize)
dma_contiguous_enable_replace_pages(dma_info->cma_dev);
pr_info("resizable cma heap=%s create successful", heap_info->name);
return 0;
declare_fail:
kfree(heap_info->name);
fail:
kfree(heap_info);
return err;
#else
return -EINVAL;
#endif
}
EXPORT_SYMBOL(dma_declare_coherent_resizable_cma_memory);
static int dma_assign_coherent_memory(struct device *dev,
struct dma_coherent_mem *mem)
{
if (dev->dma_mem)
return -EBUSY;
dev->dma_mem = mem;
/* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
return 0;
}
int _dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
dma_addr_t device_addr, size_t size,
unsigned long alloc_shift, int flags)
{
struct dma_coherent_mem *mem;
if (!dma_init_coherent_memory(phys_addr, device_addr, size, flags,
alloc_shift, &mem))
return 0;
if (dma_assign_coherent_memory(dev, mem) == 0)
return flags & DMA_MEMORY_NOMAP ? DMA_MEMORY_NOMAP :
flags & DMA_MEMORY_MAP ? DMA_MEMORY_MAP : DMA_MEMORY_IO;
dma_release_coherent_memory(mem);
return 0;
}
EXPORT_SYMBOL(_dma_declare_coherent_memory);
unsigned long dma_get_coherent_memory_alloc_shift(struct device *dev)
{
if (dev->dma_mem)
return dev->dma_mem->alloc_shift;
return -EINVAL;
}
EXPORT_SYMBOL(dma_get_coherent_memory_alloc_shift);
int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
dma_addr_t device_addr, size_t size, int flags)
{
return _dma_declare_coherent_memory(dev, phys_addr, device_addr, size,
PAGE_SHIFT, flags);
}
EXPORT_SYMBOL(dma_declare_coherent_memory);
static phys_addr_t alloc_from_contiguous_heap(
struct heap_info *h,
phys_addr_t base, size_t len)
{
size_t count;
struct page *page;
unsigned long order;
dev_dbg(h->cma_dev, "req at base (%pa) size (0x%zx)\n",
&base, len);
order = get_order(len);
count = PAGE_ALIGN(len) >> PAGE_SHIFT;
page = dma_alloc_at_from_contiguous(h->cma_dev, count,
order, base, true);
if (!page) {
dev_err(h->cma_dev, "dma_alloc_at_from_contiguous failed\n");
goto dma_alloc_err;
}
base = page_to_phys(page);
dev_dbg(h->cma_dev, "allocated at base (%pa) size (0x%zx)\n",
&base, len);
BUG_ON(base < h->cma_base ||
base - h->cma_base + len > h->cma_len);
return base;
dma_alloc_err:
return DMA_ERROR_CODE;
}
static void release_from_contiguous_heap(
struct heap_info *h,
phys_addr_t base, size_t len)
{
struct page *page = phys_to_page(base);
size_t count = PAGE_ALIGN(len) >> PAGE_SHIFT;
dma_release_from_contiguous(h->cma_dev, page, count);
dev_dbg(h->cma_dev, "released at base (%pa) size (0x%zx)\n",
&base, len);
}
static void get_first_and_last_idx(struct heap_info *h,
int *first_alloc_idx, int *last_alloc_idx)
{
if (!h->curr_len) {
*first_alloc_idx = -1;
*last_alloc_idx = h->num_chunks;
} else {
*first_alloc_idx = div_u64(h->curr_base - h->cma_base,
h->cma_chunk_size);
*last_alloc_idx = div_u64(h->curr_base - h->cma_base +
h->curr_len + h->cma_chunk_size -
h->rem_chunk_size,
h->cma_chunk_size) - 1;
}
}
static void update_alloc_range(struct heap_info *h)
{
if (!h->curr_len)
h->dev.dma_mem->size = 0;
else
h->dev.dma_mem->size = (h->curr_base - h->cma_base +
h->curr_len) >> PAGE_SHIFT;
}
static int heap_resize_locked(struct heap_info *h, bool skip_vpr_config)
{
phys_addr_t base = -1;
size_t len = h->cma_chunk_size;
phys_addr_t prev_base = h->curr_base;
size_t prev_len = h->curr_len;
int alloc_at_idx = 0;
int first_alloc_idx;
int last_alloc_idx;
phys_addr_t start_addr = h->cma_base;
get_first_and_last_idx(h, &first_alloc_idx, &last_alloc_idx);
pr_debug("req resize, fi=%d,li=%d\n", first_alloc_idx, last_alloc_idx);
/* All chunks are in use. Can't grow it. */
if (first_alloc_idx == 0 && last_alloc_idx == h->num_chunks - 1)
return -ENOMEM;
/* All chunks are free. Attempt to allocate the first chunk. */
if (first_alloc_idx == -1) {
base = alloc_from_contiguous_heap(h, start_addr, len);
if (base == start_addr)
goto alloc_success;
BUG_ON(!dma_mapping_error(h->cma_dev, base));
}
/* Free chunk before previously allocated chunk. Attempt
* to allocate only immediate previous chunk.
*/
if (first_alloc_idx > 0) {
alloc_at_idx = first_alloc_idx - 1;
start_addr = alloc_at_idx * h->cma_chunk_size + h->cma_base;
base = alloc_from_contiguous_heap(h, start_addr, len);
if (base == start_addr)
goto alloc_success;
BUG_ON(!dma_mapping_error(h->cma_dev, base));
}
/* Free chunk after previously allocated chunk. */
if (last_alloc_idx < h->num_chunks - 1) {
alloc_at_idx = last_alloc_idx + 1;
len = (alloc_at_idx == h->num_chunks - 1) ?
h->rem_chunk_size : h->cma_chunk_size;
start_addr = alloc_at_idx * h->cma_chunk_size + h->cma_base;
base = alloc_from_contiguous_heap(h, start_addr, len);
if (base == start_addr)
goto alloc_success;
BUG_ON(!dma_mapping_error(h->cma_dev, base));
}
if (dma_mapping_error(h->cma_dev, base))
dev_err(&h->dev,
"Failed to allocate contiguous memory on heap grow req\n");
return -ENOMEM;
alloc_success:
if (!h->curr_len || h->curr_base > base)
h->curr_base = base;
h->curr_len += len;
if (!skip_vpr_config && update_vpr_config(h))
goto fail_update;
dev_dbg(&h->dev,
"grow heap base from=%pa to=%pa,"
" len from=0x%zx to=0x%zx\n",
&prev_base, &h->curr_base, prev_len, h->curr_len);
return 0;
fail_update:
release_from_contiguous_heap(h, base, len);
h->curr_base = prev_base;
h->curr_len = prev_len;
return -ENOMEM;
}
static int update_vpr_config(struct heap_info *h)
{
/* Handle VPR configuration updates*/
if (h->update_resize_cfg) {
int err = h->update_resize_cfg(h->curr_base, h->curr_len);
if (err) {
dev_err(&h->dev, "Failed to update heap resize\n");
return err;
}
dev_dbg(&h->dev, "update vpr base to %pa, size=%zx\n",
&h->curr_base, h->curr_len);
}
update_alloc_range(h);
return 0;
}
static inline struct page **kvzalloc_pages(u32 count)
{
if (count * sizeof(struct page *) <= PAGE_SIZE)
return kzalloc(count * sizeof(struct page *), GFP_KERNEL);
else
return vzalloc(count * sizeof(struct page *));
}
/* retval: !0 on success, 0 on failure */
static int dma_alloc_from_coherent_dev_at(struct device *dev, ssize_t size,
dma_addr_t *dma_handle, void **ret,
unsigned long attrs, ulong start)
{
struct dma_coherent_mem *mem;
int order;
unsigned long flags;
int pageno, i = 0;
int dma_memory_map = 0;
unsigned int count;
unsigned int alloc_size;
unsigned long align;
struct page **pages = NULL;
if (!dev)
return 0;
mem = dev->dma_mem;
if (!mem)
return 0;
order = get_order(size << PAGE_SHIFT >> mem->alloc_shift);
if (dma_get_attr(DMA_ATTR_ALLOC_EXACT_SIZE, attrs))
count = ALIGN(size, 1 << mem->alloc_shift) >> mem->alloc_shift;
else
count = 1 << order;
if (!count)
return 0;
if ((mem->flags & DMA_MEMORY_NOMAP) &&
dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
alloc_size = 1;
pages = kvzalloc_pages(count);
if (!pages)
return 0;
} else {
alloc_size = count;
}
*dma_handle = DMA_ERROR_CODE;
*ret = NULL;
spin_lock_irqsave(&mem->spinlock, flags);
if (unlikely(size > (mem->size << mem->alloc_shift)))
goto err;
if ((mem->flags & DMA_MEMORY_NOMAP) &&
dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
align = 0;
} else {
if (order > DMA_BUF_ALIGNMENT)
align = (1 << DMA_BUF_ALIGNMENT) - 1;
else
align = (1 << order) - 1;
}
while (count) {
pageno = bitmap_find_next_zero_area(mem->bitmap, mem->size,
start, alloc_size, align);
if (pageno >= mem->size)
goto err;
count -= alloc_size;
if (pages)
pages[i++] = pfn_to_page(mem->pfn_base + pageno);
bitmap_set(mem->bitmap, pageno, alloc_size);
}
/*
* Memory was found in the per-device area.
*/
*dma_handle = mem->device_base + (pageno << mem->alloc_shift);
if (!(mem->flags & DMA_MEMORY_NOMAP)) {
*ret = mem->virt_base + (pageno << mem->alloc_shift);
dma_memory_map = (mem->flags & DMA_MEMORY_MAP);
} else if (dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
*ret = pages;
}
spin_unlock_irqrestore(&mem->spinlock, flags);
if (mem->flags & DMA_MEMORY_NOMAP)
/* Do nothing */;
else if (dma_memory_map)
memset(*ret, 0, size);
else if (*ret)
memset_io(*ret, 0, size);
return 1;
err:
while (i--)
bitmap_clear(mem->bitmap, page_to_pfn(pages[i]) -
mem->pfn_base, alloc_size);
spin_unlock_irqrestore(&mem->spinlock, flags);
kvfree(pages);
/*
* In the case where the allocation can not be satisfied from the
* per-device area, try to fall back to generic memory if the
* constraints allow it.
*/
return mem->flags & DMA_MEMORY_EXCLUSIVE;
}
/**
* dma_release_from_coherent_dev() - try to free the memory allocated from
* per-device coherent memory pool
* @dev: device from which the memory was allocated
* @size: size of the memory area to free
* @vaddr: virtual address of allocated pages
* @attrs: DMA Attribute
*
* This checks whether the memory was allocated from the per-device
* coherent memory pool and if so, releases that memory.
*
* Returns 1 if we correctly released the memory, or 0 if
* dma_release_coherent_attr() should proceed with releasing memory from
* generic pools.
*/
int dma_release_from_coherent_dev(struct device *dev, size_t size, void *vaddr,
unsigned long attrs)
{
struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
void *mem_addr;
unsigned int count;
unsigned int pageno;
unsigned long flags;
if (!mem)
return 0;
if ((mem->flags & DMA_MEMORY_NOMAP) &&
dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
struct page **pages = vaddr;
int i;
spin_lock_irqsave(&mem->spinlock, flags);
for (i = 0; i < (size >> PAGE_SHIFT); i++) {
pageno = page_to_pfn(pages[i]) - mem->pfn_base;
if (WARN_ONCE(pageno > mem->size,
"invalid pageno:%d\n", pageno))
continue;
bitmap_clear(mem->bitmap, pageno, 1);
}
spin_unlock_irqrestore(&mem->spinlock, flags);
kvfree(pages);
return 1;
}
if (mem->flags & DMA_MEMORY_NOMAP)
mem_addr = (void *)(uintptr_t)mem->device_base;
else
mem_addr = mem->virt_base;
if (mem && vaddr >= mem_addr &&
vaddr - mem_addr < mem->size << PAGE_SHIFT) {
unsigned long flags;
pageno = (vaddr - mem_addr) >> mem->alloc_shift;
if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs)
count = PAGE_ALIGN(size) >> PAGE_SHIFT;
else
count = 1 << get_order(size);
spin_lock_irqsave(&mem->spinlock, flags);
bitmap_clear(mem->bitmap, pageno, count);
spin_unlock_irqrestore(&mem->spinlock, flags);
return 1;
}
return 0;
}
static int dma_alloc_from_coherent_dev(struct device *dev, ssize_t size,
dma_addr_t *dma_handle, void **ret,
unsigned long attrs)
{
return dma_alloc_from_coherent_dev_at(dev, size, dma_handle,
ret, attrs, 0);
}
/* retval: !0 on success, 0 on failure */
static int dma_alloc_from_coherent_heap_dev(struct device *dev, size_t len,
dma_addr_t *dma_handle, void **ret,
unsigned long attrs)
{
struct heap_info *h = NULL;
if (!dma_is_coherent_dev(dev))
return 0;
*dma_handle = DMA_ERROR_CODE;
h = dev_get_drvdata(dev);
BUG_ON(!h);
if (!h)
return DMA_MEMORY_EXCLUSIVE;
attrs |= DMA_ATTR_ALLOC_EXACT_SIZE;
mutex_lock(&h->resize_lock);
retry_alloc:
/* Try allocation from already existing CMA chunks */
if (dma_alloc_from_coherent_dev_at(
&h->dev, len, dma_handle, ret, attrs,
(h->curr_base - h->cma_base) >> PAGE_SHIFT)) {
if (*dma_handle != DMA_ERROR_CODE) {
dev_dbg(&h->dev, "allocated addr %pa len 0x%zx\n",
dma_handle, len);
h->curr_used += len;
}
goto out;
}
if (!heap_resize_locked(h, false))
goto retry_alloc;
out:
mutex_unlock(&h->resize_lock);
return DMA_MEMORY_EXCLUSIVE;
}
/* retval: !0 on success, 0 on failure */
static int dma_release_from_coherent_heap_dev(struct device *dev, size_t len,
void *base, unsigned long attrs)
{
int idx = 0;
int err = 0;
struct heap_info *h = NULL;
if (!dma_is_coherent_dev(dev))
return 0;
h = dev_get_drvdata(dev);
BUG_ON(!h);
if (!h)
return 1;
mutex_lock(&h->resize_lock);
if (!dma_get_attr(DMA_ATTR_ALLOC_SINGLE_PAGES, attrs)) {
if ((uintptr_t)base < h->curr_base || len > h->curr_len ||
(uintptr_t)base - h->curr_base > h->curr_len - len) {
BUG();
mutex_unlock(&h->resize_lock);
return 1;
}
idx = div_u64((uintptr_t)base - h->cma_base, h->cma_chunk_size);
dev_dbg(&h->dev, "req free addr (%p) size (0x%zx) idx (%d)\n",
base, len, idx);
}
attrs |= DMA_ATTR_ALLOC_EXACT_SIZE;
err = dma_release_from_coherent_dev(&h->dev, len, base, attrs);
/* err = 0 on failure, !0 on successful release */
if (err && h->task)
mod_timer(&h->shrink_timer, jiffies + h->shrink_interval);
if (err)
h->curr_used -= len;
mutex_unlock(&h->resize_lock);
if (err && !h->task)
shrink_resizable_heap(h);
return err;
}
static bool shrink_chunk_locked(struct heap_info *h, int idx)
{
size_t chunk_size;
int resize_err;
void *ret = NULL;
dma_addr_t dev_base;
unsigned long attrs = DMA_ATTR_ALLOC_EXACT_SIZE;
/* check if entire chunk is free */
chunk_size = (idx == h->num_chunks - 1) ? h->rem_chunk_size :
h->cma_chunk_size;
resize_err = dma_alloc_from_coherent_dev_at(&h->dev,
chunk_size, &dev_base, &ret, attrs,
idx * h->cma_chunk_size >> PAGE_SHIFT);
if (!resize_err) {
goto out;
} else if (dev_base != h->cma_base + idx * h->cma_chunk_size) {
resize_err = dma_release_from_coherent_dev(
&h->dev, chunk_size,
(void *)(uintptr_t)dev_base, attrs);
BUG_ON(!resize_err);
goto out;
} else {
dev_dbg(&h->dev,
"prep to remove chunk b=%pa, s=0x%zx\n",
&dev_base, chunk_size);
resize_err = dma_release_from_coherent_dev(
&h->dev, chunk_size,
(void *)(uintptr_t)dev_base, attrs);
BUG_ON(!resize_err);
if (!resize_err) {
dev_err(&h->dev, "failed to rel mem\n");
goto out;
}
/* Handle VPR configuration updates */
if (h->update_resize_cfg) {
phys_addr_t new_base = h->curr_base;
size_t new_len = h->curr_len - chunk_size;
if (h->curr_base == dev_base)
new_base += chunk_size;
dev_dbg(&h->dev, "update vpr base to %pa, size=%zx\n",
&new_base, new_len);
resize_err =
h->update_resize_cfg(new_base, new_len);
if (resize_err) {
dev_err(&h->dev,
"update resize failed\n");
goto out;
}
}
if (h->curr_base == dev_base)
h->curr_base += chunk_size;
h->curr_len -= chunk_size;
update_alloc_range(h);
release_from_contiguous_heap(h, dev_base, chunk_size);
dev_dbg(&h->dev, "removed chunk b=%pa, s=0x%zx"
" new heap b=%pa, s=0x%zx\n", &dev_base,
chunk_size, &h->curr_base, h->curr_len);
return true;
}
out:
return false;
}
static void shrink_resizable_heap(struct heap_info *h)
{
bool unlock = false;
int first_alloc_idx, last_alloc_idx;
check_next_chunk:
if (unlock) {
mutex_unlock(&h->resize_lock);
cond_resched();
}
mutex_lock(&h->resize_lock);
unlock = true;
if (h->curr_len <= h->floor_size)
goto out_unlock;
get_first_and_last_idx(h, &first_alloc_idx, &last_alloc_idx);
/* All chunks are free. Exit. */
if (first_alloc_idx == -1)
goto out_unlock;
if (shrink_chunk_locked(h, first_alloc_idx))
goto check_next_chunk;
/* Only one chunk is in use. */
if (first_alloc_idx == last_alloc_idx)
goto out_unlock;
if (shrink_chunk_locked(h, last_alloc_idx))
goto check_next_chunk;
out_unlock:
mutex_unlock(&h->resize_lock);
}
/*
* Helper function used to manage resizable heap shrink timeouts
*/
static void shrink_timeout(unsigned long __data)
{
struct task_struct *p = (struct task_struct *) __data;
wake_up_process(p);
}
static int shrink_thread(void *arg)
{
struct heap_info *h = arg;
/*
* Set up an interval timer which can be used to trigger a commit wakeup
* after the commit interval expires
*/
setup_timer(&h->shrink_timer, shrink_timeout,
(unsigned long)current);
h->task = current;
while (1) {
if (kthread_should_stop())
break;
shrink_resizable_heap(h);
/* resize done. goto sleep */
set_current_state(TASK_INTERRUPTIBLE);
schedule();
}
return 0;
}
void dma_release_declared_memory(struct device *dev)
{
struct dma_coherent_mem *mem = dev->dma_mem;
if (!mem)
return;
dev->dma_mem = NULL;
if (!(mem->flags & DMA_MEMORY_NOMAP))
iounmap(mem->virt_base);
kfree(mem->bitmap);
kfree(mem);
}
EXPORT_SYMBOL(dma_release_declared_memory);
void *dma_mark_declared_memory_occupied(struct device *dev,
dma_addr_t device_addr, size_t size,
unsigned long attrs)
{
struct dma_coherent_mem *mem = dev->dma_mem;
int order = get_order(size);
int pos, freepage;
unsigned int count;
unsigned long align = 0;
size += device_addr & ~PAGE_MASK;
if (!mem)
return ERR_PTR(-EINVAL);
if (!(DMA_ATTR_ALLOC_EXACT_SIZE & attrs)) {
if (order > DMA_BUF_ALIGNMENT)
align = (1 << DMA_BUF_ALIGNMENT) - 1;
else
align = (1 << order) - 1;
}
if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs)
count = PAGE_ALIGN(size) >> PAGE_SHIFT;
else
count = 1 << order;
pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
freepage = bitmap_find_next_zero_area(mem->bitmap, mem->size,
pos, count, align);
if (pos >= mem->size || freepage != pos)
return ERR_PTR(-EBUSY);
bitmap_set(mem->bitmap, pos, count);
return mem->virt_base + (pos << PAGE_SHIFT);
}
EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
void dma_mark_declared_memory_unoccupied(struct device *dev,
dma_addr_t device_addr, size_t size,
unsigned long attrs)
{
struct dma_coherent_mem *mem = dev->dma_mem;
int pos;
int count;
if (!mem)
return;
size += device_addr & ~PAGE_MASK;
if (DMA_ATTR_ALLOC_EXACT_SIZE & attrs)
count = PAGE_ALIGN(size) >> PAGE_SHIFT;
else
count = 1 << get_order(size);
pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
bitmap_clear(mem->bitmap, pos, count);
}
EXPORT_SYMBOL(dma_mark_declared_memory_unoccupied);
/**
* dma_alloc_from_coherent_attr() - try to allocate memory from the per-device
* coherent area
*
* @dev: device from which we allocate memory
* @size: size of requested memory area
* @dma_handle: This will be filled with the correct dma handle
* @ret: This pointer will be filled with the virtual address
* to allocated area.
* @attrs: DMA Attribute
* This function should be only called from per-arch dma_alloc_coherent()
* to support allocation from per-device coherent memory pools.
*
* Returns 0 if dma_alloc_coherent_attr should continue with allocating from
* generic memory areas, or !0 if dma_alloc_coherent should return @ret.
*/
int dma_alloc_from_coherent_attr(struct device *dev, ssize_t size,
dma_addr_t *dma_handle, void **ret,
unsigned long attrs)
{
if (!dev)
return 0;
if (dev->dma_mem)
return dma_alloc_from_coherent_dev(dev, size, dma_handle, ret,
attrs);
else
return dma_alloc_from_coherent_heap_dev(dev, size, dma_handle,
ret, attrs);
}
EXPORT_SYMBOL(dma_alloc_from_coherent_attr);
/**
* dma_release_from_coherent_attr() - try to free the memory allocated from
* per-device coherent memory pool
* @dev: device from which the memory was allocated
* @size: size of the memory area to free
* @vaddr: virtual address of allocated pages
* @attrs: DMA Attribute
*
* This checks whether the memory was allocated from the per-device
* coherent memory pool and if so, releases that memory.
*
* Returns 1 if we correctly released the memory, or 0 if
* dma_release_coherent_attr() should proceed with releasing memory from
* generic pools.
*/
int dma_release_from_coherent_attr(struct device *dev, size_t size, void *vaddr,
unsigned long attrs, dma_addr_t dma_handle)
{
if (!dev)
return 0;
if (!vaddr)
/*
* The only possible valid case where vaddr is NULL is when
* dma_alloc_attrs() is called on coherent dev which was
* initialized with DMA_MEMORY_NOMAP.
*/
vaddr = (void *)dma_handle;
if (dev->dma_mem)
return dma_release_from_coherent_dev(dev, size, vaddr, attrs);
else
return dma_release_from_coherent_heap_dev(dev, size, vaddr,
attrs);
}
EXPORT_SYMBOL(dma_release_from_coherent_attr);
/**
* dma_mmap_from_coherent() - try to mmap the memory allocated from
* per-device coherent memory pool to userspace
* @dev: device from which the memory was allocated
* @vma: vm_area for the userspace memory
* @vaddr: cpu address returned by dma_alloc_from_coherent
* @size: size of the memory buffer allocated by dma_alloc_from_coherent
* @ret: result from remap_pfn_range()
*
* This checks whether the memory was allocated from the per-device
* coherent memory pool and if so, maps that memory to the provided vma.
*
* Returns 1 if we correctly mapped the memory, or 0 if the caller should
* proceed with mapping memory from generic pools.
*/
int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma,
void *vaddr, size_t size, int *ret)
{
struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
void *mem_addr;
if (!mem)
return 0;
if (mem->flags & DMA_MEMORY_NOMAP)
mem_addr = (void *)(uintptr_t)mem->device_base;
else
mem_addr = mem->virt_base;
if (mem && vaddr >= mem_addr && vaddr + size <=
(mem_addr + (mem->size << PAGE_SHIFT))) {
unsigned long off = vma->vm_pgoff;
int start = (vaddr - mem_addr) >> PAGE_SHIFT;
int user_count = vma_pages(vma);
int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
*ret = -ENXIO;
if (off < count && user_count <= count - off) {
unsigned long pfn = mem->pfn_base + start + off;
*ret = remap_pfn_range(vma, vma->vm_start, pfn,
user_count << PAGE_SHIFT,
vma->vm_page_prot);
}
return 1;
}
return 0;
}
EXPORT_SYMBOL(dma_mmap_from_coherent);
/*
* Support for reserved memory regions defined in device tree
*/
#ifdef CONFIG_OF_RESERVED_MEM
#include <linux/of.h>
#include <linux/of_fdt.h>
#include <linux/of_reserved_mem.h>
static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev)
{
struct dma_coherent_mem *mem = rmem->priv;
if (!mem &&
!dma_init_coherent_memory(rmem->base, rmem->base, rmem->size,
DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE,
PAGE_SHIFT, &mem)) {
pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",
&rmem->base, (unsigned long)rmem->size / SZ_1M);
return -ENODEV;
}
rmem->priv = mem;
dma_assign_coherent_memory(dev, mem);
return 0;
}
static void rmem_dma_device_release(struct reserved_mem *rmem,
struct device *dev)
{
dev->dma_mem = NULL;
}
static const struct reserved_mem_ops rmem_dma_ops = {
.device_init = rmem_dma_device_init,
.device_release = rmem_dma_device_release,
};
static int __init rmem_dma_setup(struct reserved_mem *rmem)
{
unsigned long node = rmem->fdt_node;
if (of_get_flat_dt_prop(node, "reusable", NULL))
return -EINVAL;
#ifdef CONFIG_ARM
if (!of_get_flat_dt_prop(node, "no-map", NULL)) {
pr_err("Reserved memory: regions without no-map are not yet supported\n");
return -EINVAL;
}
#endif
rmem->ops = &rmem_dma_ops;
pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n",
&rmem->base, (unsigned long)rmem->size / SZ_1M);
return 0;
}
RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);
#endif
int dma_get_coherent_stats(struct device *dev,
struct dma_coherent_stats *stats)
{
struct heap_info *h = NULL;
struct dma_coherent_mem *mem = dev->dma_mem;
if ((!dev) || !stats)
return -EINVAL;
h = dev_get_drvdata(dev);
if (h && (h->magic == RESIZE_MAGIC)) {
stats->size = h->curr_len;
stats->base = h->curr_base;
stats->used = h->curr_used;
stats->max = h->cma_len;
goto out;
}
if (!mem)
return -EINVAL;
stats->size = mem->size << PAGE_SHIFT;
stats->base = mem->device_base;
out:
return 0;
}
EXPORT_SYMBOL(dma_get_coherent_stats);