Hardware
/
litex
mirror of https://github.com/enjoy-digital/litex.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
litex/misoclib/com/litepcie/software/linux/kernel/main.c

640 lines
18 KiB
C
Raw Blame History

/*
* LitePCIe driver
*
*/
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/types.h>
#include <linux/ioctl.h>
#include <linux/init.h>
#include <linux/errno.h>
#include <linux/mm.h>
#include <linux/fs.h>
#include <linux/mmtimer.h>
#include <linux/miscdevice.h>
#include <linux/posix-timers.h>
#include <linux/interrupt.h>
#include <linux/time.h>
#include <linux/math64.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/pci.h>
#include <linux/pci_regs.h>
#include <linux/delay.h>
#include <linux/wait.h>
#include "litepcie.h"
#include "config.h"
#include "csr.h"
#include "flags.h"
#define LITEPCIE_NAME "litepcie"
#define LITEPCIE_MINOR_COUNT 4
#define DMA_BUFFER_SIZE PAGE_ALIGN(32768)
#define DMA_BUFFER_MAP_SIZE (DMA_BUFFER_SIZE * DMA_BUFFER_COUNT)
#define IRQ_MASK_DMA_READER (1 << DMA_READER_INTERRUPT)
#define IRQ_MASK_DMA_WRITER (1 << DMA_WRITER_INTERRUPT)
typedef struct {
int minor;
struct pci_dev *dev;
phys_addr_t bar0_phys_addr;
uint8_t *bar0_addr; /* virtual address of BAR0 */
uint8_t *dma_tx_bufs[DMA_BUFFER_COUNT];
unsigned long dma_tx_bufs_addr[DMA_BUFFER_COUNT];
uint8_t *dma_rx_bufs[DMA_BUFFER_COUNT];
unsigned long dma_rx_bufs_addr[DMA_BUFFER_COUNT];
uint8_t tx_dma_started;
uint8_t rx_dma_started;
wait_queue_head_t dma_waitqueue;
} LitePCIeState;
static dev_t litepcie_cdev;
static struct cdev litepcie_cdev_struct;
static LitePCIeState *litepcie_minor_table[LITEPCIE_MINOR_COUNT];
static void litepcie_end(struct pci_dev *dev, LitePCIeState *s);
static int litepcie_dma_stop(LitePCIeState *s);
static inline uint32_t litepcie_readl(LitePCIeState *s, uint32_t addr)
{
return readl(s->bar0_addr + addr);
}
static inline void litepcie_writel(LitePCIeState *s, uint32_t addr, uint32_t val)
{
return writel(val, s->bar0_addr + addr);
}
static void litepcie_enable_interrupt(LitePCIeState *s, int irq_num)
{
uint32_t v;
v = litepcie_readl(s, CSR_IRQ_CONTROLLER_ENABLE_ADDR);
v |= (1 << irq_num);
litepcie_writel(s, CSR_IRQ_CONTROLLER_ENABLE_ADDR, v);
}
static void litepcie_disable_interrupt(LitePCIeState *s, int irq_num)
{
uint32_t v;
v = litepcie_readl(s, CSR_IRQ_CONTROLLER_ENABLE_ADDR);
v &= ~(1 << irq_num);
litepcie_writel(s, CSR_IRQ_CONTROLLER_ENABLE_ADDR, v);
}
static int litepcie_open(struct inode *inode, struct file *file)
{
LitePCIeState *s;
int minor;
/* find PCI device */
minor = iminor(inode);
if (minor < 0 || minor >= LITEPCIE_MINOR_COUNT)
return -ENODEV;
s = litepcie_minor_table[minor];
if (!s)
return -ENODEV;
file->private_data = s;
return 0;
}
/* mmap the DMA buffers and registers to user space */
static int litepcie_mmap(struct file *file, struct vm_area_struct *vma)
{
LitePCIeState *s = file->private_data;
unsigned long pfn;
int is_tx, i;
if (vma->vm_pgoff == 0) {
if (vma->vm_end - vma->vm_start != DMA_BUFFER_MAP_SIZE)
return -EINVAL;
is_tx = 1;
goto remap_ram;
} else if (vma->vm_pgoff == (DMA_BUFFER_MAP_SIZE >> PAGE_SHIFT)) {
if (vma->vm_end - vma->vm_start != DMA_BUFFER_MAP_SIZE)
return -EINVAL;
is_tx = 0;
remap_ram:
for(i = 0; i < DMA_BUFFER_COUNT; i++) {
if (is_tx)
pfn = __pa(s->dma_tx_bufs[i]) >> PAGE_SHIFT;
else
pfn = __pa(s->dma_rx_bufs[i]) >> PAGE_SHIFT;
/* Note: the memory is cached, so the user must explicitly
flush the CPU caches on architectures which require it. */
if (remap_pfn_range(vma, vma->vm_start + i * DMA_BUFFER_SIZE, pfn,
DMA_BUFFER_SIZE, vma->vm_page_prot)) {
printk(KERN_ERR LITEPCIE_NAME " remap_pfn_range failed\n");
return -EAGAIN;
}
}
} else if (vma->vm_pgoff == ((2 * DMA_BUFFER_MAP_SIZE) >> PAGE_SHIFT)) {
if (vma->vm_end - vma->vm_start != PCI_FPGA_BAR0_SIZE)
return -EINVAL;
pfn = s->bar0_phys_addr >> PAGE_SHIFT;
/* not cached */
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
vma->vm_flags |= VM_IO;
if (io_remap_pfn_range(vma, vma->vm_start, pfn,
vma->vm_end - vma->vm_start,
vma->vm_page_prot)) {
printk(KERN_ERR LITEPCIE_NAME " io_remap_pfn_range failed\n");
return -EAGAIN;
}
} else {
return -EINVAL;
}
return 0;
}
static int litepcie_release(struct inode *inode, struct file *file)
{
LitePCIeState *s = file->private_data;
litepcie_dma_stop(s); /* just in case: stop the DMA */
return 0;
}
static irqreturn_t litepcie_interrupt(int irq, void *data)
{
LitePCIeState *s = data;
uint32_t clear_mask, irq_vector;
irq_vector = litepcie_readl(s, CSR_IRQ_CONTROLLER_VECTOR_ADDR);
clear_mask = 0;
if (irq_vector & (IRQ_MASK_DMA_READER | IRQ_MASK_DMA_WRITER)) {
/* wake up processes waiting on dma_wait() */
wake_up_interruptible(&s->dma_waitqueue);
clear_mask |= (IRQ_MASK_DMA_READER | IRQ_MASK_DMA_WRITER);
}
litepcie_writel(s, CSR_IRQ_CONTROLLER_CLEAR_ADDR, clear_mask);
return IRQ_HANDLED;
}
static int litepcie_dma_start(LitePCIeState *s, struct litepcie_ioctl_dma_start *m)
{
int i, val;
if (s->tx_dma_started || s->rx_dma_started)
return -EIO;
if (m->tx_buf_size == 0 && m->rx_buf_size == 0)
return -EINVAL;
/* check alignment (XXX: what is the exact constraint ?) */
if ((m->tx_buf_size & 7) != 0 ||
(m->rx_buf_size & 7) != 0 ||
m->tx_buf_size > DMA_BUFFER_SIZE ||
m->rx_buf_size > DMA_BUFFER_SIZE)
return -EINVAL;
/* check buffer count */
if (m->tx_buf_count > DMA_BUFFER_COUNT)
return -EINVAL;
if (m->rx_buf_count > DMA_BUFFER_COUNT)
return -EINVAL;
val = ((m->dma_flags & DMA_LOOPBACK_ENABLE) != 0);
litepcie_writel(s, CSR_DMA_LOOPBACK_ENABLE_ADDR, val);
/* init DMA write */
if (m->rx_buf_size != 0) {
litepcie_writel(s, CSR_DMA_WRITER_ENABLE_ADDR, 0);
litepcie_writel(s, CSR_DMA_WRITER_TABLE_FLUSH_ADDR, 1);
litepcie_writel(s, CSR_DMA_WRITER_TABLE_LOOP_PROG_N_ADDR, 0);
for(i = 0; i < m->rx_buf_count; i++) {
litepcie_writel(s, CSR_DMA_WRITER_TABLE_VALUE_ADDR, m->rx_buf_size);
litepcie_writel(s, CSR_DMA_WRITER_TABLE_VALUE_ADDR + 4,
s->dma_rx_bufs_addr[i]);
litepcie_writel(s, CSR_DMA_WRITER_TABLE_WE_ADDR, 1);
}
litepcie_writel(s, CSR_DMA_WRITER_TABLE_LOOP_PROG_N_ADDR, 1);
}
/* init DMA read */
if (m->tx_buf_size != 0) {
litepcie_writel(s, CSR_DMA_READER_ENABLE_ADDR, 0);
litepcie_writel(s, CSR_DMA_READER_TABLE_FLUSH_ADDR, 1);
litepcie_writel(s, CSR_DMA_READER_TABLE_LOOP_PROG_N_ADDR, 0);
for(i = 0; i < m->tx_buf_count; i++) {
litepcie_writel(s, CSR_DMA_READER_TABLE_VALUE_ADDR, m->tx_buf_size);
litepcie_writel(s, CSR_DMA_READER_TABLE_VALUE_ADDR + 4,
s->dma_tx_bufs_addr[i]);
litepcie_writel(s, CSR_DMA_READER_TABLE_WE_ADDR, 1);
}
litepcie_writel(s, CSR_DMA_READER_TABLE_LOOP_PROG_N_ADDR, 1);
}
/* start DMA */
if (m->rx_buf_size != 0) {
litepcie_writel(s, CSR_DMA_WRITER_ENABLE_ADDR, 1);
s->rx_dma_started = 1;
}
if (m->tx_buf_size != 0) {
litepcie_writel(s, CSR_DMA_READER_ENABLE_ADDR, 1);
s->tx_dma_started = 1;
}
return 0;
}
static int litepcie_dma_wait(LitePCIeState *s, struct litepcie_ioctl_dma_wait *m)
{
unsigned long timeout;
int ret, last_buf_num;
DECLARE_WAITQUEUE(wait, current);
if (m->tx_wait) {
if (!s->tx_dma_started)
return -EIO;
last_buf_num = m->tx_buf_num;
litepcie_enable_interrupt(s, DMA_READER_INTERRUPT);
} else {
if (!s->rx_dma_started)
return -EIO;
last_buf_num = m->rx_buf_num;
litepcie_enable_interrupt(s, DMA_WRITER_INTERRUPT);
}
add_wait_queue(&s->dma_waitqueue, &wait);
timeout = jiffies + msecs_to_jiffies(m->timeout);
for (;;) {
/* set current buffer */
if (s->tx_dma_started) {
m->tx_buf_num = (litepcie_readl(s, CSR_DMA_READER_TABLE_LOOP_STATUS_ADDR) & 0xffff);
} else {
m->tx_buf_num = 0;
}
if (s->rx_dma_started) {
m->rx_buf_num = (litepcie_readl(s, CSR_DMA_WRITER_TABLE_LOOP_STATUS_ADDR) & 0xfffff);
} else {
m->rx_buf_num = 0;
}
if (m->tx_wait) {
if (m->tx_buf_num != last_buf_num)
break;
} else {
if (m->rx_buf_num != last_buf_num)
break;
}
if ((long)(jiffies - timeout) > 0) {
ret = -EAGAIN;
goto done;
}
set_current_state(TASK_INTERRUPTIBLE);
if (signal_pending(current)) {
ret = -EINTR;
goto done;
}
schedule();
}
ret = 0;
done:
if (m->tx_wait) {
litepcie_disable_interrupt(s, DMA_READER_INTERRUPT);
} else {
litepcie_disable_interrupt(s, DMA_WRITER_INTERRUPT);
}
__set_current_state(TASK_RUNNING);
remove_wait_queue(&s->dma_waitqueue, &wait);
return ret;
}
static int litepcie_dma_stop(LitePCIeState *s)
{
/* just to be sure, we disable the interrupts */
litepcie_disable_interrupt(s, DMA_READER_INTERRUPT);
litepcie_disable_interrupt(s, DMA_WRITER_INTERRUPT);
s->tx_dma_started = 0;
litepcie_writel(s, CSR_DMA_READER_TABLE_LOOP_PROG_N_ADDR, 0);
litepcie_writel(s, CSR_DMA_READER_TABLE_FLUSH_ADDR, 1);
udelay(100);
litepcie_writel(s, CSR_DMA_READER_ENABLE_ADDR, 0);
s->rx_dma_started = 0;
litepcie_writel(s, CSR_DMA_WRITER_TABLE_LOOP_PROG_N_ADDR, 0);
litepcie_writel(s, CSR_DMA_WRITER_TABLE_FLUSH_ADDR, 1);
udelay(100);
litepcie_writel(s, CSR_DMA_WRITER_ENABLE_ADDR, 0);
return 0;
}
static long litepcie_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
LitePCIeState *s = file->private_data;
long ret;
switch(cmd) {
case LITEPCIE_IOCTL_GET_MMAP_INFO:
{
struct litepcie_ioctl_mmap_info m;
m.dma_tx_buf_offset = 0;
m.dma_tx_buf_size = DMA_BUFFER_SIZE;
m.dma_tx_buf_count = DMA_BUFFER_COUNT;
m.dma_rx_buf_offset = DMA_BUFFER_MAP_SIZE;
m.dma_rx_buf_size = DMA_BUFFER_SIZE;
m.dma_rx_buf_count = DMA_BUFFER_COUNT;
m.reg_offset = 2 * DMA_BUFFER_MAP_SIZE;
m.reg_size = PCI_FPGA_BAR0_SIZE;
if (copy_to_user((void *)arg, &m, sizeof(m))) {
ret = -EFAULT;
break;
}
ret = 0;
}
break;
case LITEPCIE_IOCTL_DMA_START:
{
struct litepcie_ioctl_dma_start m;
if (copy_from_user(&m, (void *)arg, sizeof(m))) {
ret = -EFAULT;
break;
}
ret = litepcie_dma_start(s, &m);
}
break;
case LITEPCIE_IOCTL_DMA_STOP:
{
ret = litepcie_dma_stop(s);
}
break;
case LITEPCIE_IOCTL_DMA_WAIT:
{
struct litepcie_ioctl_dma_wait m;
if (copy_from_user(&m, (void *)arg, sizeof(m))) {
ret = -EFAULT;
break;
}
ret = litepcie_dma_wait(s, &m);
if (ret == 0) {
if (copy_to_user((void *)arg, &m, sizeof(m))) {
ret = -EFAULT;
break;
}
}
}
break;
default:
ret = -ENOIOCTLCMD;
break;
}
return ret;
}
static const struct file_operations litepcie_fops = {
.owner = THIS_MODULE,
.unlocked_ioctl = litepcie_ioctl,
.open = litepcie_open,
.release = litepcie_release,
.mmap = litepcie_mmap,
.llseek = no_llseek,
};
static int litepcie_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
{
LitePCIeState *s = NULL;
uint8_t rev_id;
int ret, minor, i;
printk(KERN_INFO LITEPCIE_NAME " Probing device\n");
/* find available minor */
for(minor = 0; minor < LITEPCIE_MINOR_COUNT; minor++) {
if (!litepcie_minor_table[minor])
break;
}
if (minor == LITEPCIE_MINOR_COUNT) {
printk(KERN_ERR LITEPCIE_NAME " Cannot allocate a minor\n");
ret = -ENODEV;
goto fail1;
}
s = kzalloc(sizeof(LitePCIeState), GFP_KERNEL);
if (!s) {
printk(KERN_ERR LITEPCIE_NAME " Cannot allocate memory\n");
ret = -ENOMEM;
goto fail1;
}
s->minor = minor;
s->dev = dev;
pci_set_drvdata(dev, s);
ret = pci_enable_device(dev);
if (ret != 0) {
printk(KERN_ERR LITEPCIE_NAME " Cannot enable device\n");
goto fail1;
}
/* check device version */
pci_read_config_byte(dev, PCI_REVISION_ID, &rev_id);
if (rev_id != 1) {
printk(KERN_ERR LITEPCIE_NAME " Unsupported device version %d\n", rev_id);
goto fail2;
}
if (pci_request_regions(dev, LITEPCIE_NAME) < 0) {
printk(KERN_ERR LITEPCIE_NAME " Could not request regions\n");
goto fail2;
}
/* check BAR0 config */
if (!(pci_resource_flags(dev, 0) & IORESOURCE_MEM)) {
printk(KERN_ERR LITEPCIE_NAME " Invalid BAR0 config\n");
goto fail3;
}
s->bar0_phys_addr = pci_resource_start(dev, 0);
s->bar0_addr = pci_ioremap_bar(dev, 0);
if (!s->bar0_addr) {
printk(KERN_ERR LITEPCIE_NAME " Could not map BAR0\n");
goto fail3;
}
pci_set_master(dev);
ret = pci_set_dma_mask(dev, DMA_BIT_MASK(32));
if (ret) {
printk(KERN_ERR LITEPCIE_NAME " Failed to set DMA mask\n");
goto fail4;
};
ret = pci_enable_msi(dev);
if (ret) {
printk(KERN_ERR LITEPCIE_NAME " Failed to enable MSI\n");
goto fail4;
}
if (request_irq(dev->irq, litepcie_interrupt, IRQF_SHARED, LITEPCIE_NAME, s) < 0) {
printk(KERN_ERR LITEPCIE_NAME " Failed to allocate irq %d\n", dev->irq);
goto fail5;
}
/* soft reset */
litepcie_writel(s, CSR_CRG_SOFT_RST_ADDR, 1);
udelay(5);
/* allocate DMA buffers */
for(i = 0; i < DMA_BUFFER_COUNT; i++) {
s->dma_tx_bufs[i] = kzalloc(DMA_BUFFER_SIZE, GFP_KERNEL | GFP_DMA32);
if (!s->dma_tx_bufs[i]) {
printk(KERN_ERR LITEPCIE_NAME " Failed to allocate dma_tx_buf\n");
goto fail6;
}
s->dma_tx_bufs_addr[i] = pci_map_single(dev, s->dma_tx_bufs[i],
DMA_BUFFER_SIZE,
DMA_TO_DEVICE);
if (!s->dma_tx_bufs_addr[i]) {
ret = -ENOMEM;
goto fail6;
}
}
for(i = 0; i < DMA_BUFFER_COUNT; i++) {
s->dma_rx_bufs[i] = kzalloc(DMA_BUFFER_SIZE, GFP_KERNEL | GFP_DMA32);
if (!s->dma_rx_bufs[i]) {
printk(KERN_ERR LITEPCIE_NAME " Failed to allocate dma_rx_buf\n");
goto fail6;
}
s->dma_rx_bufs_addr[i] = pci_map_single(dev, s->dma_rx_bufs[i],
DMA_BUFFER_SIZE,
DMA_FROM_DEVICE);
if (!s->dma_rx_bufs_addr[i]) {
ret = -ENOMEM;
goto fail6;
}
}
init_waitqueue_head(&s->dma_waitqueue);
litepcie_minor_table[minor] = s;
printk(KERN_INFO LITEPCIE_NAME " Assigned to minor %d\n", minor);
return 0;
fail6:
litepcie_end(dev, s);
free_irq(dev->irq, s);
fail5:
pci_disable_msi(dev);
fail4:
pci_iounmap(dev, s->bar0_addr);
fail3:
pci_release_regions(dev);
fail2:
pci_disable_device(dev);
ret = -EIO;
fail1:
kfree(s);
printk(KERN_ERR LITEPCIE_NAME " Error while probing device\n");
return ret;
}
static void litepcie_end(struct pci_dev *dev, LitePCIeState *s)
{
int i;
for(i = 0; i < DMA_BUFFER_COUNT; i++) {
if (s->dma_tx_bufs_addr[i]) {
dma_unmap_single(&dev->dev, s->dma_tx_bufs_addr[i],
DMA_BUFFER_SIZE, DMA_TO_DEVICE);
}
kfree(s->dma_tx_bufs[i]);
}
for(i = 0; i < DMA_BUFFER_COUNT; i++) {
if (s->dma_rx_bufs_addr[i]) {
dma_unmap_single(&dev->dev, s->dma_rx_bufs_addr[i],
DMA_BUFFER_SIZE, DMA_FROM_DEVICE);
}
kfree(s->dma_rx_bufs[i]);
}
}
static void litepcie_pci_remove(struct pci_dev *dev)
{
LitePCIeState *s = pci_get_drvdata(dev);
printk(KERN_INFO LITEPCIE_NAME " Removing device\n");
litepcie_minor_table[s->minor] = NULL;
litepcie_end(dev, s);
free_irq(dev->irq, s);
pci_disable_msi(dev);
pci_iounmap(dev, s->bar0_addr);
pci_disable_device(dev);
pci_release_regions(dev);
kfree(s);
};
static const struct pci_device_id litepcie_pci_ids[] = {
{ PCI_DEVICE(PCI_FPGA_VENDOR_ID, PCI_FPGA_DEVICE_ID), },
{ 0, }
};
MODULE_DEVICE_TABLE(pci, litepcie_pci_ids);
static struct pci_driver litepcie_pci_driver = {
.name = LITEPCIE_NAME,
.id_table = litepcie_pci_ids,
.probe = litepcie_pci_probe,
.remove = litepcie_pci_remove,
};
static int __init litepcie_module_init(void)
{
int ret;
ret = pci_register_driver(&litepcie_pci_driver);
if (ret < 0) {
printk(KERN_ERR LITEPCIE_NAME " Error while registering PCI driver\n");
goto fail1;
}
ret = alloc_chrdev_region(&litepcie_cdev, 0, LITEPCIE_MINOR_COUNT, LITEPCIE_NAME);
if (ret < 0) {
printk(KERN_ERR LITEPCIE_NAME " Could not allocate char device\n");
goto fail2;
}
cdev_init(&litepcie_cdev_struct, &litepcie_fops);
ret = cdev_add(&litepcie_cdev_struct, litepcie_cdev, LITEPCIE_MINOR_COUNT);
if (ret < 0) {
printk(KERN_ERR LITEPCIE_NAME " Could not register char device\n");
goto fail3;
}
return 0;
fail3:
unregister_chrdev_region(litepcie_cdev, LITEPCIE_MINOR_COUNT);
fail2:
pci_unregister_driver(&litepcie_pci_driver);
fail1:
return ret;
}
static void __exit litepcie_module_exit(void)
{
cdev_del(&litepcie_cdev_struct);
unregister_chrdev_region(litepcie_cdev, LITEPCIE_MINOR_COUNT);
pci_unregister_driver(&litepcie_pci_driver);
}
module_init(litepcie_module_init);
module_exit(litepcie_module_exit);
MODULE_LICENSE("GPL");
Reference in New Issue View Git Blame Copy Permalink