/* * CARMA DATA-FPGA Access Driver * * Copyright (c) 2009-2011 Ira W. Snyder <iws@ovro.caltech.edu> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. */ /* * FPGA Memory Dump Format * * FPGA #0 control registers (32 x 32-bit words) * FPGA #1 control registers (32 x 32-bit words) * FPGA #2 control registers (32 x 32-bit words) * FPGA #3 control registers (32 x 32-bit words) * SYSFPGA control registers (32 x 32-bit words) * FPGA #0 correlation array (NUM_CORL0 correlation blocks) * FPGA #1 correlation array (NUM_CORL1 correlation blocks) * FPGA #2 correlation array (NUM_CORL2 correlation blocks) * FPGA #3 correlation array (NUM_CORL3 correlation blocks) * * Each correlation array consists of: * * Correlation Data (2 x NUM_LAGSn x 32-bit words) * Pipeline Metadata (2 x NUM_METAn x 32-bit words) * Quantization Counters (2 x NUM_QCNTn x 32-bit words) * * The NUM_CORLn, NUM_LAGSn, NUM_METAn, and NUM_QCNTn values come from * the FPGA configuration registers. They do not change once the FPGA's * have been programmed, they only change on re-programming. */ /* * Basic Description: * * This driver is used to capture correlation spectra off of the four data * processing FPGAs. The FPGAs are often reprogrammed at runtime, therefore * this driver supports dynamic enable/disable of capture while the device * remains open. * * The nominal capture rate is 64Hz (every 15.625ms). To facilitate this fast * capture rate, all buffers are pre-allocated to avoid any potentially long * running memory allocations while capturing. * * There are two lists and one pointer which are used to keep track of the * different states of data buffers. * * 1) free list * This list holds all empty data buffers which are ready to receive data. * * 2) inflight pointer * This pointer holds the currently inflight data buffer. This buffer is having * data copied into it by the DMA engine. * * 3) used list * This list holds data buffers which have been filled, and are waiting to be * read by userspace. * * All buffers start life on the free list, then move successively to the * inflight pointer, and then to the used list. After they have been read by * userspace, they are moved back to the free list. The cycle repeats as long * as necessary. * * It should be noted that all buffers are mapped and ready for DMA when they * are on any of the three lists. They are only unmapped when they are in the * process of being read by userspace. */ /* * Notes on the IRQ masking scheme: * * The IRQ masking scheme here is different than most other hardware. The only * way for the DATA-FPGAs to detect if the kernel has taken too long to copy * the data is if the status registers are not cleared before the next * correlation data dump is ready. * * The interrupt line is connected to the status registers, such that when they * are cleared, the interrupt is de-asserted. Therein lies our problem. We need * to schedule a long-running DMA operation and return from the interrupt * handler quickly, but we cannot clear the status registers. * * To handle this, the system controller FPGA has the capability to connect the * interrupt line to a user-controlled GPIO pin. This pin is driven high * (unasserted) and left that way. To mask the interrupt, we change the * interrupt source to the GPIO pin. Tada, we hid the interrupt. :) */ #include <linux/of_platform.h> #include <linux/dma-mapping.h> #include <linux/miscdevice.h> #include <linux/interrupt.h> #include <linux/dmaengine.h> #include <linux/seq_file.h> #include <linux/highmem.h> #include <linux/debugfs.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/poll.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/kref.h> #include <linux/io.h> #include <media/videobuf-dma-sg.h> /* system controller registers */ #define SYS_IRQ_SOURCE_CTL 0x24 #define SYS_IRQ_OUTPUT_EN 0x28 #define SYS_IRQ_OUTPUT_DATA 0x2C #define SYS_IRQ_INPUT_DATA 0x30 #define SYS_FPGA_CONFIG_STATUS 0x44 /* GPIO IRQ line assignment */ #define IRQ_CORL_DONE 0x10 /* FPGA registers */ #define MMAP_REG_VERSION 0x00 #define MMAP_REG_CORL_CONF1 0x08 #define MMAP_REG_CORL_CONF2 0x0C #define MMAP_REG_STATUS 0x48 #define SYS_FPGA_BLOCK 0xF0000000 #define DATA_FPGA_START 0x400000 #define DATA_FPGA_SIZE 0x80000 static const char drv_name[] = "carma-fpga"; #define NUM_FPGA 4 #define MIN_DATA_BUFS 8 #define MAX_DATA_BUFS 64 struct fpga_info { unsigned int num_lag_ram; unsigned int blk_size; }; struct data_buf { struct list_head entry; struct videobuf_dmabuf vb; size_t size; }; struct fpga_device { /* character device */ struct miscdevice miscdev; struct device *dev; struct mutex mutex; /* reference count */ struct kref ref; /* FPGA registers and information */ struct fpga_info info[NUM_FPGA]; void __iomem *regs; int irq; /* FPGA Physical Address/Size Information */ resource_size_t phys_addr; size_t phys_size; /* DMA structures */ struct sg_table corl_table; unsigned int corl_nents; struct dma_chan *chan; /* Protection for all members below */ spinlock_t lock; /* Device enable/disable flag */ bool enabled; /* Correlation data buffers */ wait_queue_head_t wait; struct list_head free; struct list_head used; struct data_buf *inflight; /* Information about data buffers */ unsigned int num_dropped; unsigned int num_buffers; size_t bufsize; struct dentry *dbg_entry; }; struct fpga_reader { struct fpga_device *priv; struct data_buf *buf; off_t buf_start; }; static void fpga_device_release(struct kref *ref) { struct fpga_device *priv = container_of(ref, struct fpga_device, ref); /* the last reader has exited, cleanup the last bits */ mutex_destroy(&priv->mutex); kfree(priv); } /* * Data Buffer Allocation Helpers */ /** * data_free_buffer() - free a single data buffer and all allocated memory * @buf: the buffer to free * * This will free all of the pages allocated to the given data buffer, and * then free the structure itself */ static void data_free_buffer(struct data_buf *buf) { /* It is ok to free a NULL buffer */ if (!buf) return; /* free all memory */ videobuf_dma_free(&buf->vb); kfree(buf); } /** * data_alloc_buffer() - allocate and fill a data buffer with pages * @bytes: the number of bytes required * * This allocates all space needed for a data buffer. It must be mapped before * use in a DMA transaction using videobuf_dma_map(). * * Returns NULL on failure */ static struct data_buf *data_alloc_buffer(const size_t bytes) { unsigned int nr_pages; struct data_buf *buf; int ret; /* calculate the number of pages necessary */ nr_pages = DIV_ROUND_UP(bytes, PAGE_SIZE); /* allocate the buffer structure */ buf = kzalloc(sizeof(*buf), GFP_KERNEL); if (!buf) goto out_return; /* initialize internal fields */ INIT_LIST_HEAD(&buf->entry); buf->size = bytes; /* allocate the videobuf */ videobuf_dma_init(&buf->vb); ret = videobuf_dma_init_kernel(&buf->vb, DMA_FROM_DEVICE, nr_pages); if (ret) goto out_free_buf; return buf; out_free_buf: kfree(buf); out_return: return NULL; } /** * data_free_buffers() - free all allocated buffers * @priv: the driver's private data structure * * Free all buffers allocated by the driver (except those currently in the * process of being read by userspace). * * LOCKING: must hold dev->mutex * CONTEXT: user */ static void data_free_buffers(struct fpga_device *priv) { struct data_buf *buf, *tmp; /* the device should be stopped, no DMA in progress */ BUG_ON(priv->inflight != NULL); list_for_each_entry_safe(buf, tmp, &priv->free, entry) { list_del_init(&buf->entry); videobuf_dma_unmap(priv->dev, &buf->vb); data_free_buffer(buf); } list_for_each_entry_safe(buf, tmp, &priv->used, entry) { list_del_init(&buf->entry); videobuf_dma_unmap(priv->dev, &buf->vb); data_free_buffer(buf); } priv->num_buffers = 0; priv->bufsize = 0; } /** * data_alloc_buffers() - allocate 1 seconds worth of data buffers * @priv: the driver's private data structure * * Allocate enough buffers for a whole second worth of data * * This routine will attempt to degrade nicely by succeeding even if a full * second worth of data buffers could not be allocated, as long as a minimum * number were allocated. In this case, it will print a message to the kernel * log. * * The device must not be modifying any lists when this is called. * * CONTEXT: user * LOCKING: must hold dev->mutex * * Returns 0 on success, -ERRNO otherwise */ static int data_alloc_buffers(struct fpga_device *priv) { struct data_buf *buf; int i, ret; for (i = 0; i < MAX_DATA_BUFS; i++) { /* allocate a buffer */ buf = data_alloc_buffer(priv->bufsize); if (!buf) break; /* map it for DMA */ ret = videobuf_dma_map(priv->dev, &buf->vb); if (ret) { data_free_buffer(buf); break; } /* add it to the list of free buffers */ list_add_tail(&buf->entry, &priv->free); priv->num_buffers++; } /* Make sure we allocated the minimum required number of buffers */ if (priv->num_buffers < MIN_DATA_BUFS) { dev_err(priv->dev, "Unable to allocate enough data buffers\n"); data_free_buffers(priv); return -ENOMEM; } /* Warn if we are running in a degraded state, but do not fail */ if (priv->num_buffers < MAX_DATA_BUFS) { dev_warn(priv->dev, "Unable to allocate %d buffers, using %d buffers instead\n", MAX_DATA_BUFS, i); } return 0; } /* * DMA Operations Helpers */ /** * fpga_start_addr() - get the physical address a DATA-FPGA * @priv: the driver's private data structure * @fpga: the DATA-FPGA number (zero based) */ static dma_addr_t fpga_start_addr(struct fpga_device *priv, unsigned int fpga) { return priv->phys_addr + 0x400000 + (0x80000 * fpga); } /** * fpga_block_addr() - get the physical address of a correlation data block * @priv: the driver's private data structure * @fpga: the DATA-FPGA number (zero based) * @blknum: the correlation block number (zero based) */ static dma_addr_t fpga_block_addr(struct fpga_device *priv, unsigned int fpga, unsigned int blknum) { return fpga_start_addr(priv, fpga) + (0x10000 * (1 + blknum)); } #define REG_BLOCK_SIZE (32 * 4) /** * data_setup_corl_table() - create the scatterlist for correlation dumps * @priv: the driver's private data structure * * Create the scatterlist for transferring a correlation dump from the * DATA FPGAs. This structure will be reused for each buffer than needs * to be filled with correlation data. * * Returns 0 on success, -ERRNO otherwise */ static int data_setup_corl_table(struct fpga_device *priv) { struct sg_table *table = &priv->corl_table; struct scatterlist *sg; struct fpga_info *info; int i, j, ret; /* Calculate the number of entries needed */ priv->corl_nents = (1 + NUM_FPGA) * REG_BLOCK_SIZE; for (i = 0; i < NUM_FPGA; i++) priv->corl_nents += priv->info[i].num_lag_ram; /* Allocate the scatterlist table */ ret = sg_alloc_table(table, priv->corl_nents, GFP_KERNEL); if (ret) { dev_err(priv->dev, "unable to allocate DMA table\n"); return ret; } /* Add the DATA FPGA registers to the scatterlist */ sg = table->sgl; for (i = 0; i < NUM_FPGA; i++) { sg_dma_address(sg) = fpga_start_addr(priv, i); sg_dma_len(sg) = REG_BLOCK_SIZE; sg = sg_next(sg); } /* Add the SYS-FPGA registers to the scatterlist */ sg_dma_address(sg) = SYS_FPGA_BLOCK; sg_dma_len(sg) = REG_BLOCK_SIZE; sg = sg_next(sg); /* Add the FPGA correlation data blocks to the scatterlist */ for (i = 0; i < NUM_FPGA; i++) { info = &priv->info[i]; for (j = 0; j < info->num_lag_ram; j++) { sg_dma_address(sg) = fpga_block_addr(priv, i, j); sg_dma_len(sg) = info->blk_size; sg = sg_next(sg); } } /* * All physical addresses and lengths are present in the structure * now. It can be reused for every FPGA DATA interrupt */ return 0; } /* * FPGA Register Access Helpers */ static void fpga_write_reg(struct fpga_device *priv, unsigned int fpga, unsigned int reg, u32 val) { const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE); iowrite32be(val, priv->regs + fpga_start + reg); } static u32 fpga_read_reg(struct fpga_device *priv, unsigned int fpga, unsigned int reg) { const int fpga_start = DATA_FPGA_START + (fpga * DATA_FPGA_SIZE); return ioread32be(priv->regs + fpga_start + reg); } /** * data_calculate_bufsize() - calculate the data buffer size required * @priv: the driver's private data structure * * Calculate the total buffer size needed to hold a single block * of correlation data * * CONTEXT: user * * Returns 0 on success, -ERRNO otherwise */ static int data_calculate_bufsize(struct fpga_device *priv) { u32 num_corl, num_lags, num_meta, num_qcnt, num_pack; u32 conf1, conf2, version; u32 num_lag_ram, blk_size; int i; /* Each buffer starts with the 5 FPGA register areas */ priv->bufsize = (1 + NUM_FPGA) * REG_BLOCK_SIZE; /* Read and store the configuration data for each FPGA */ for (i = 0; i < NUM_FPGA; i++) { version = fpga_read_reg(priv, i, MMAP_REG_VERSION); conf1 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF1); conf2 = fpga_read_reg(priv, i, MMAP_REG_CORL_CONF2); /* minor version 2 and later */ if ((version & 0x000000FF) >= 2) { num_corl = (conf1 & 0x000000F0) >> 4; num_pack = (conf1 & 0x00000F00) >> 8; num_lags = (conf1 & 0x00FFF000) >> 12; num_meta = (conf1 & 0x7F000000) >> 24; num_qcnt = (conf2 & 0x00000FFF) >> 0; } else { num_corl = (conf1 & 0x000000F0) >> 4; num_pack = 1; /* implied */ num_lags = (conf1 & 0x000FFF00) >> 8; num_meta = (conf1 & 0x7FF00000) >> 20; num_qcnt = (conf2 & 0x00000FFF) >> 0; } num_lag_ram = (num_corl + num_pack - 1) / num_pack; blk_size = ((num_pack * num_lags) + num_meta + num_qcnt) * 8; priv->info[i].num_lag_ram = num_lag_ram; priv->info[i].blk_size = blk_size; priv->bufsize += num_lag_ram * blk_size; dev_dbg(priv->dev, "FPGA %d NUM_CORL: %d\n", i, num_corl); dev_dbg(priv->dev, "FPGA %d NUM_PACK: %d\n", i, num_pack); dev_dbg(priv->dev, "FPGA %d NUM_LAGS: %d\n", i, num_lags); dev_dbg(priv->dev, "FPGA %d NUM_META: %d\n", i, num_meta); dev_dbg(priv->dev, "FPGA %d NUM_QCNT: %d\n", i, num_qcnt); dev_dbg(priv->dev, "FPGA %d BLK_SIZE: %d\n", i, blk_size); } dev_dbg(priv->dev, "TOTAL BUFFER SIZE: %zu bytes\n", priv->bufsize); return 0; } /* * Interrupt Handling */ /** * data_disable_interrupts() - stop the device from generating interrupts * @priv: the driver's private data structure * * Hide interrupts by switching to GPIO interrupt source * * LOCKING: must hold dev->lock */ static void data_disable_interrupts(struct fpga_device *priv) { /* hide the interrupt by switching the IRQ driver to GPIO */ iowrite32be(0x2F, priv->regs + SYS_IRQ_SOURCE_CTL); } /** * data_enable_interrupts() - allow the device to generate interrupts * @priv: the driver's private data structure * * Unhide interrupts by switching to the FPGA interrupt source. At the * same time, clear the DATA-FPGA status registers. * * LOCKING: must hold dev->lock */ static void data_enable_interrupts(struct fpga_device *priv) { /* clear the actual FPGA corl_done interrupt */ fpga_write_reg(priv, 0, MMAP_REG_STATUS, 0x0); fpga_write_reg(priv, 1, MMAP_REG_STATUS, 0x0); fpga_write_reg(priv, 2, MMAP_REG_STATUS, 0x0); fpga_write_reg(priv, 3, MMAP_REG_STATUS, 0x0); /* flush the writes */ fpga_read_reg(priv, 0, MMAP_REG_STATUS); /* switch back to the external interrupt source */ iowrite32be(0x3F, priv->regs + SYS_IRQ_SOURCE_CTL); } /** * data_dma_cb() - DMAEngine callback for DMA completion * @data: the driver's private data structure * * Complete a DMA transfer from the DATA-FPGA's * * This is called via the DMA callback mechanism, and will handle moving the * completed DMA transaction to the used list, and then wake any processes * waiting for new data * * CONTEXT: any, softirq expected */ static void data_dma_cb(void *data) { struct fpga_device *priv = data; unsigned long flags; spin_lock_irqsave(&priv->lock, flags); /* If there is no inflight buffer, we've got a bug */ BUG_ON(priv->inflight == NULL); /* Move the inflight buffer onto the used list */ list_move_tail(&priv->inflight->entry, &priv->used); priv->inflight = NULL; /* clear the FPGA status and re-enable interrupts */ data_enable_interrupts(priv); spin_unlock_irqrestore(&priv->lock, flags); /* * We've changed both the inflight and used lists, so we need * to wake up any processes that are blocking for those events */ wake_up(&priv->wait); } /** * data_submit_dma() - prepare and submit the required DMA to fill a buffer * @priv: the driver's private data structure * @buf: the data buffer * * Prepare and submit the necessary DMA transactions to fill a correlation * data buffer. * * LOCKING: must hold dev->lock * CONTEXT: hardirq only * * Returns 0 on success, -ERRNO otherwise */ static int data_submit_dma(struct fpga_device *priv, struct data_buf *buf) { struct scatterlist *dst_sg, *src_sg; unsigned int dst_nents, src_nents; struct dma_chan *chan = priv->chan; struct dma_async_tx_descriptor *tx; dma_cookie_t cookie; dma_addr_t dst, src; dst_sg = buf->vb.sglist; dst_nents = buf->vb.sglen; src_sg = priv->corl_table.sgl; src_nents = priv->corl_nents; /* * All buffers passed to this function should be ready and mapped * for DMA already. Therefore, we don't need to do anything except * submit it to the Freescale DMA Engine for processing */ /* setup the scatterlist to scatterlist transfer */ tx = chan->device->device_prep_dma_sg(chan, dst_sg, dst_nents, src_sg, src_nents, 0); if (!tx) { dev_err(priv->dev, "unable to prep scatterlist DMA\n"); return -ENOMEM; } /* submit the transaction to the DMA controller */ cookie = tx->tx_submit(tx); if (dma_submit_error(cookie)) { dev_err(priv->dev, "unable to submit scatterlist DMA\n"); return -ENOMEM; } /* Prepare the re-read of the SYS-FPGA block */ dst = sg_dma_address(dst_sg) + (NUM_FPGA * REG_BLOCK_SIZE); src = SYS_FPGA_BLOCK; tx = chan->device->device_prep_dma_memcpy(chan, dst, src, REG_BLOCK_SIZE, DMA_PREP_INTERRUPT); if (!tx) { dev_err(priv->dev, "unable to prep SYS-FPGA DMA\n"); return -ENOMEM; } /* Setup the callback */ tx->callback = data_dma_cb; tx->callback_param = priv; /* submit the transaction to the DMA controller */ cookie = tx->tx_submit(tx); if (dma_submit_error(cookie)) { dev_err(priv->dev, "unable to submit SYS-FPGA DMA\n"); return -ENOMEM; } return 0; } #define CORL_DONE 0x1 #define CORL_ERR 0x2 static irqreturn_t data_irq(int irq, void *dev_id) { struct fpga_device *priv = dev_id; bool submitted = false; struct data_buf *buf; u32 status; int i; /* detect spurious interrupts via FPGA status */ for (i = 0; i < 4; i++) { status = fpga_read_reg(priv, i, MMAP_REG_STATUS); if (!(status & (CORL_DONE | CORL_ERR))) { dev_err(priv->dev, "spurious irq detected (FPGA)\n"); return IRQ_NONE; } } /* detect spurious interrupts via raw IRQ pin readback */ status = ioread32be(priv->regs + SYS_IRQ_INPUT_DATA); if (status & IRQ_CORL_DONE) { dev_err(priv->dev, "spurious irq detected (IRQ)\n"); return IRQ_NONE; } spin_lock(&priv->lock); /* hide the interrupt by switching the IRQ driver to GPIO */ data_disable_interrupts(priv); /* If there are no free buffers, drop this data */ if (list_empty(&priv->free)) { priv->num_dropped++; goto out; } buf = list_first_entry(&priv->free, struct data_buf, entry); list_del_init(&buf->entry); BUG_ON(buf->size != priv->bufsize); /* Submit a DMA transfer to get the correlation data */ if (data_submit_dma(priv, buf)) { dev_err(priv->dev, "Unable to setup DMA transfer\n"); list_move_tail(&buf->entry, &priv->free); goto out; } /* Save the buffer for the DMA callback */ priv->inflight = buf; submitted = true; /* Start the DMA Engine */ dma_async_memcpy_issue_pending(priv->chan); out: /* If no DMA was submitted, re-enable interrupts */ if (!submitted) data_enable_interrupts(priv); spin_unlock(&priv->lock); return IRQ_HANDLED; } /* * Realtime Device Enable Helpers */ /** * data_device_enable() - enable the device for buffered dumping * @priv: the driver's private data structure * * Enable the device for buffered dumping. Allocates buffers and hooks up * the interrupt handler. When this finishes, data will come pouring in. * * LOCKING: must hold dev->mutex * CONTEXT: user context only * * Returns 0 on success, -ERRNO otherwise */ static int data_device_enable(struct fpga_device *priv) { u32 val; int ret; /* multiple enables are safe: they do nothing */ if (priv->enabled) return 0; /* check that the FPGAs are programmed */ val = ioread32be(priv->regs + SYS_FPGA_CONFIG_STATUS); if (!(val & (1 << 18))) { dev_err(priv->dev, "DATA-FPGAs are not enabled\n"); return -ENODATA; } /* read the FPGAs to calculate the buffer size */ ret = data_calculate_bufsize(priv); if (ret) { dev_err(priv->dev, "unable to calculate buffer size\n"); goto out_error; } /* allocate the correlation data buffers */ ret = data_alloc_buffers(priv); if (ret) { dev_err(priv->dev, "unable to allocate buffers\n"); goto out_error; } /* setup the source scatterlist for dumping correlation data */ ret = data_setup_corl_table(priv); if (ret) { dev_err(priv->dev, "unable to setup correlation DMA table\n"); goto out_error; } /* hookup the irq handler */ ret = request_irq(priv->irq, data_irq, IRQF_SHARED, drv_name, priv); if (ret) { dev_err(priv->dev, "unable to request IRQ handler\n"); goto out_error; } /* switch to the external FPGA IRQ line */ data_enable_interrupts(priv); /* success, we're enabled */ priv->enabled = true; return 0; out_error: sg_free_table(&priv->corl_table); priv->corl_nents = 0; data_free_buffers(priv); return ret; } /** * data_device_disable() - disable the device for buffered dumping * @priv: the driver's private data structure * * Disable the device for buffered dumping. Stops new DMA transactions from * being generated, waits for all outstanding DMA to complete, and then frees * all buffers. * * LOCKING: must hold dev->mutex * CONTEXT: user only * * Returns 0 on success, -ERRNO otherwise */ static int data_device_disable(struct fpga_device *priv) { int ret; /* allow multiple disable */ if (!priv->enabled) return 0; /* switch to the internal GPIO IRQ line */ data_disable_interrupts(priv); /* unhook the irq handler */ free_irq(priv->irq, priv); /* * wait for all outstanding DMA to complete * * Device interrupts are disabled, therefore another buffer cannot * be marked inflight. */ ret = wait_event_interruptible(priv->wait, priv->inflight == NULL); if (ret) return ret; /* free the correlation table */ sg_free_table(&priv->corl_table); priv->corl_nents = 0; /* * We are taking the spinlock not to protect priv->enabled, but instead * to make sure that there are no readers in the process of altering * the free or used lists while we are setting this flag. */ spin_lock_irq(&priv->lock); priv->enabled = false; spin_unlock_irq(&priv->lock); /* free all buffers: the free and used lists are not being changed */ data_free_buffers(priv); return 0; } /* * DEBUGFS Interface */ #ifdef CONFIG_DEBUG_FS /* * Count the number of entries in the given list */ static unsigned int list_num_entries(struct list_head *list) { struct list_head *entry; unsigned int ret = 0; list_for_each(entry, list) ret++; return ret; } static int data_debug_show(struct seq_file *f, void *offset) { struct fpga_device *priv = f->private; int ret; /* * Lock the mutex first, so that we get an accurate value for enable * Lock the spinlock next, to get accurate list counts */ ret = mutex_lock_interruptible(&priv->mutex); if (ret) return ret; spin_lock_irq(&priv->lock); seq_printf(f, "enabled: %d\n", priv->enabled); seq_printf(f, "bufsize: %d\n", priv->bufsize); seq_printf(f, "num_buffers: %d\n", priv->num_buffers); seq_printf(f, "num_free: %d\n", list_num_entries(&priv->free)); seq_printf(f, "inflight: %d\n", priv->inflight != NULL); seq_printf(f, "num_used: %d\n", list_num_entries(&priv->used)); seq_printf(f, "num_dropped: %d\n", priv->num_dropped); spin_unlock_irq(&priv->lock); mutex_unlock(&priv->mutex); return 0; } static int data_debug_open(struct inode *inode, struct file *file) { return single_open(file, data_debug_show, inode->i_private); } static const struct file_operations data_debug_fops = { .owner = THIS_MODULE, .open = data_debug_open, .read = seq_read, .llseek = seq_lseek, .release = single_release, }; static int data_debugfs_init(struct fpga_device *priv) { priv->dbg_entry = debugfs_create_file(drv_name, S_IRUGO, NULL, priv, &data_debug_fops); if (IS_ERR(priv->dbg_entry)) return PTR_ERR(priv->dbg_entry); return 0; } static void data_debugfs_exit(struct fpga_device *priv) { debugfs_remove(priv->dbg_entry); } #else static inline int data_debugfs_init(struct fpga_device *priv) { return 0; } static inline void data_debugfs_exit(struct fpga_device *priv) { } #endif /* CONFIG_DEBUG_FS */ /* * SYSFS Attributes */ static ssize_t data_en_show(struct device *dev, struct device_attribute *attr, char *buf) { struct fpga_device *priv = dev_get_drvdata(dev); return snprintf(buf, PAGE_SIZE, "%u\n", priv->enabled); } static ssize_t data_en_set(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct fpga_device *priv = dev_get_drvdata(dev); unsigned long enable; int ret; ret = strict_strtoul(buf, 0, &enable); if (ret) { dev_err(priv->dev, "unable to parse enable input\n"); return -EINVAL; } ret = mutex_lock_interruptible(&priv->mutex); if (ret) return ret; if (enable) ret = data_device_enable(priv); else ret = data_device_disable(priv); if (ret) { dev_err(priv->dev, "device %s failed\n", enable ? "enable" : "disable"); count = ret; goto out_unlock; } out_unlock: mutex_unlock(&priv->mutex); return count; } static DEVICE_ATTR(enable, S_IWUSR | S_IRUGO, data_en_show, data_en_set); static struct attribute *data_sysfs_attrs[] = { &dev_attr_enable.attr, NULL, }; static const struct attribute_group rt_sysfs_attr_group = { .attrs = data_sysfs_attrs, }; /* * FPGA Realtime Data Character Device */ static int data_open(struct inode *inode, struct file *filp) { /* * The miscdevice layer puts our struct miscdevice into the * filp->private_data field. We use this to find our private * data and then overwrite it with our own private structure. */ struct fpga_device *priv = container_of(filp->private_data, struct fpga_device, miscdev); struct fpga_reader *reader; int ret; /* allocate private data */ reader = kzalloc(sizeof(*reader), GFP_KERNEL); if (!reader) return -ENOMEM; reader->priv = priv; reader->buf = NULL; filp->private_data = reader; ret = nonseekable_open(inode, filp); if (ret) { dev_err(priv->dev, "nonseekable-open failed\n"); kfree(reader); return ret; } /* * success, increase the reference count of the private data structure * so that it doesn't disappear if the device is unbound */ kref_get(&priv->ref); return 0; } static int data_release(struct inode *inode, struct file *filp) { struct fpga_reader *reader = filp->private_data; struct fpga_device *priv = reader->priv; /* free the per-reader structure */ data_free_buffer(reader->buf); kfree(reader); filp->private_data = NULL; /* decrement our reference count to the private data */ kref_put(&priv->ref, fpga_device_release); return 0; } static ssize_t data_read(struct file *filp, char __user *ubuf, size_t count, loff_t *f_pos) { struct fpga_reader *reader = filp->private_data; struct fpga_device *priv = reader->priv; struct list_head *used = &priv->used; struct data_buf *dbuf; size_t avail; void *data; int ret; /* check if we already have a partial buffer */ if (reader->buf) { dbuf = reader->buf; goto have_buffer; } spin_lock_irq(&priv->lock); /* Block until there is at least one buffer on the used list */ while (list_empty(used)) { spin_unlock_irq(&priv->lock); if (filp->f_flags & O_NONBLOCK) return -EAGAIN; ret = wait_event_interruptible(priv->wait, !list_empty(used)); if (ret) return ret; spin_lock_irq(&priv->lock); } /* Grab the first buffer off of the used list */ dbuf = list_first_entry(used, struct data_buf, entry); list_del_init(&dbuf->entry); spin_unlock_irq(&priv->lock); /* Buffers are always mapped: unmap it */ videobuf_dma_unmap(priv->dev, &dbuf->vb); /* save the buffer for later */ reader->buf = dbuf; reader->buf_start = 0; have_buffer: /* Get the number of bytes available */ avail = dbuf->size - reader->buf_start; data = dbuf->vb.vaddr + reader->buf_start; /* Get the number of bytes we can transfer */ count = min(count, avail); /* Copy the data to the userspace buffer */ if (copy_to_user(ubuf, data, count)) return -EFAULT; /* Update the amount of available space */ avail -= count; /* * If there is still some data available, save the buffer for the * next userspace call to read() and return */ if (avail > 0) { reader->buf_start += count; reader->buf = dbuf; return count; } /* * Get the buffer ready to be reused for DMA * * If it fails, we pretend that the read never happed and return * -EFAULT to userspace. The read will be retried. */ ret = videobuf_dma_map(priv->dev, &dbuf->vb); if (ret) { dev_err(priv->dev, "unable to remap buffer for DMA\n"); return -EFAULT; } /* Lock against concurrent enable/disable */ spin_lock_irq(&priv->lock); /* the reader is finished with this buffer */ reader->buf = NULL; /* * One of two things has happened, the device is disabled, or the * device has been reconfigured underneath us. In either case, we * should just throw away the buffer. */ if (!priv->enabled || dbuf->size != priv->bufsize) { videobuf_dma_unmap(priv->dev, &dbuf->vb); data_free_buffer(dbuf); goto out_unlock; } /* The buffer is safe to reuse, so add it back to the free list */ list_add_tail(&dbuf->entry, &priv->free); out_unlock: spin_unlock_irq(&priv->lock); return count; } static unsigned int data_poll(struct file *filp, struct poll_table_struct *tbl) { struct fpga_reader *reader = filp->private_data; struct fpga_device *priv = reader->priv; unsigned int mask = 0; poll_wait(filp, &priv->wait, tbl); if (!list_empty(&priv->used)) mask |= POLLIN | POLLRDNORM; return mask; } static int data_mmap(struct file *filp, struct vm_area_struct *vma) { struct fpga_reader *reader = filp->private_data; struct fpga_device *priv = reader->priv; unsigned long offset, vsize, psize, addr; /* VMA properties */ offset = vma->vm_pgoff << PAGE_SHIFT; vsize = vma->vm_end - vma->vm_start; psize = priv->phys_size - offset; addr = (priv->phys_addr + offset) >> PAGE_SHIFT; /* Check against the FPGA region's physical memory size */ if (vsize > psize) { dev_err(priv->dev, "requested mmap mapping too large\n"); return -EINVAL; } /* IO memory (stop cacheing) */ vma->vm_flags |= VM_IO | VM_RESERVED; vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); return io_remap_pfn_range(vma, vma->vm_start, addr, vsize, vma->vm_page_prot); } static const struct file_operations data_fops = { .owner = THIS_MODULE, .open = data_open, .release = data_release, .read = data_read, .poll = data_poll, .mmap = data_mmap, .llseek = no_llseek, }; /* * OpenFirmware Device Subsystem */ static bool dma_filter(struct dma_chan *chan, void *data) { /* * DMA Channel #0 is used for the FPGA Programmer, so ignore it * * This probably won't survive an unload/load cycle of the Freescale * DMAEngine driver, but that won't be a problem */ if (chan->chan_id == 0 && chan->device->dev_id == 0) return false; return true; } static int data_of_probe(struct platform_device *op) { struct device_node *of_node = op->dev.of_node; struct device *this_device; struct fpga_device *priv; struct resource res; dma_cap_mask_t mask; int ret; /* Allocate private data */ priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) { dev_err(&op->dev, "Unable to allocate device private data\n"); ret = -ENOMEM; goto out_return; } dev_set_drvdata(&op->dev, priv); priv->dev = &op->dev; kref_init(&priv->ref); mutex_init(&priv->mutex); dev_set_drvdata(priv->dev, priv); spin_lock_init(&priv->lock); INIT_LIST_HEAD(&priv->free); INIT_LIST_HEAD(&priv->used); init_waitqueue_head(&priv->wait); /* Setup the misc device */ priv->miscdev.minor = MISC_DYNAMIC_MINOR; priv->miscdev.name = drv_name; priv->miscdev.fops = &data_fops; /* Get the physical address of the FPGA registers */ ret = of_address_to_resource(of_node, 0, &res); if (ret) { dev_err(&op->dev, "Unable to find FPGA physical address\n"); ret = -ENODEV; goto out_free_priv; } priv->phys_addr = res.start; priv->phys_size = resource_size(&res); /* ioremap the registers for use */ priv->regs = of_iomap(of_node, 0); if (!priv->regs) { dev_err(&op->dev, "Unable to ioremap registers\n"); ret = -ENOMEM; goto out_free_priv; } dma_cap_zero(mask); dma_cap_set(DMA_MEMCPY, mask); dma_cap_set(DMA_INTERRUPT, mask); dma_cap_set(DMA_SLAVE, mask); dma_cap_set(DMA_SG, mask); /* Request a DMA channel */ priv->chan = dma_request_channel(mask, dma_filter, NULL); if (!priv->chan) { dev_err(&op->dev, "Unable to request DMA channel\n"); ret = -ENODEV; goto out_unmap_regs; } /* Find the correct IRQ number */ priv->irq = irq_of_parse_and_map(of_node, 0); if (priv->irq == NO_IRQ) { dev_err(&op->dev, "Unable to find IRQ line\n"); ret = -ENODEV; goto out_release_dma; } /* Drive the GPIO for FPGA IRQ high (no interrupt) */ iowrite32be(IRQ_CORL_DONE, priv->regs + SYS_IRQ_OUTPUT_DATA); /* Register the miscdevice */ ret = misc_register(&priv->miscdev); if (ret) { dev_err(&op->dev, "Unable to register miscdevice\n"); goto out_irq_dispose_mapping; } /* Create the debugfs files */ ret = data_debugfs_init(priv); if (ret) { dev_err(&op->dev, "Unable to create debugfs files\n"); goto out_misc_deregister; } /* Create the sysfs files */ this_device = priv->miscdev.this_device; dev_set_drvdata(this_device, priv); ret = sysfs_create_group(&this_device->kobj, &rt_sysfs_attr_group); if (ret) { dev_err(&op->dev, "Unable to create sysfs files\n"); goto out_data_debugfs_exit; } dev_info(&op->dev, "CARMA FPGA Realtime Data Driver Loaded\n"); return 0; out_data_debugfs_exit: data_debugfs_exit(priv); out_misc_deregister: misc_deregister(&priv->miscdev); out_irq_dispose_mapping: irq_dispose_mapping(priv->irq); out_release_dma: dma_release_channel(priv->chan); out_unmap_regs: iounmap(priv->regs); out_free_priv: kref_put(&priv->ref, fpga_device_release); out_return: return ret; } static int data_of_remove(struct platform_device *op) { struct fpga_device *priv = dev_get_drvdata(&op->dev); struct device *this_device = priv->miscdev.this_device; /* remove all sysfs files, now the device cannot be re-enabled */ sysfs_remove_group(&this_device->kobj, &rt_sysfs_attr_group); /* remove all debugfs files */ data_debugfs_exit(priv); /* disable the device from generating data */ data_device_disable(priv); /* remove the character device to stop new readers from appearing */ misc_deregister(&priv->miscdev); /* cleanup everything not needed by readers */ irq_dispose_mapping(priv->irq); dma_release_channel(priv->chan); iounmap(priv->regs); /* release our reference */ kref_put(&priv->ref, fpga_device_release); return 0; } static struct of_device_id data_of_match[] = { { .compatible = "carma,carma-fpga", }, {}, }; static struct platform_driver data_of_driver = { .probe = data_of_probe, .remove = data_of_remove, .driver = { .name = drv_name, .of_match_table = data_of_match, .owner = THIS_MODULE, }, }; /* * Module Init / Exit */ static int __init data_init(void) { return platform_driver_register(&data_of_driver); } static void __exit data_exit(void) { platform_driver_unregister(&data_of_driver); } MODULE_AUTHOR("Ira W. Snyder <iws@ovro.caltech.edu>"); MODULE_DESCRIPTION("CARMA DATA-FPGA Access Driver"); MODULE_LICENSE("GPL"); module_init(data_init); module_exit(data_exit);