#ifndef __KERNEL__
#define __KERNEL__
#endif

#ifndef MODULE
#define MODULE
#endif

#define PCI17X4_DEBUG	0
#define PCI17X4_MEMMAP	0

#include <linux/module.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/kdev_t.h>
#include <linux/cdev.h>
#include <linux/string.h>
#include <linux/sysfs.h>
#include <linux/major.h>
#include <linux/delay.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/poll.h>
#include <linux/mm.h>
#include <linux/pci.h>
#include <asm/uaccess.h>
#include <asm/io.h>

#include "pci17x4.h"
#include "Reg9056.h"
#include "Reg8311.h"

/* ==== Module interface ==== */

struct pci17x4_private_data;

static int __init pci17x4_init(void);
static void __exit pci17x4_exit(void);
static int __devinit pci17x4_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
static void __devexit pci17x4_remove_one(struct pci_dev *pdev);
static int pci17x4_open(struct inode *ino, struct file *fp);
static int pci17x4_close(struct inode *ino, struct file *fp);
static ssize_t pci17x4_read(struct file *fp, char *buf, size_t count, loff_t *off);
static ssize_t pci17x4_write(struct file *fp, const char *buf, size_t count, loff_t *off);
static long pci17x4_ioctl(struct file *fp, unsigned int cmd, unsigned long arg);
static int pci17x4_mmap(struct file *fp, struct vm_area_struct *vma);
static int pci17x4_allocdmabuffers(struct pci17x4_private_data *adc, unsigned blkshift, unsigned numblocks);
static void pci17x4_freedmabuffers(struct pci17x4_private_data *adc);
static void pci17x4_dmastop(struct pci17x4_private_data *adc);
static void pci17x4_doio(struct pci17x4_private_data *adc);
static int pci17x4_dmastart(struct pci17x4_private_data *adc);
static irqreturn_t pci17x4_interrupt(int irq, void *devid);

module_init(pci17x4_init);
module_exit(pci17x4_exit);

static struct pci_device_id pci17x4_id_tbl[] __devinitdata = {
  { 0x13fe, 0x1744, 0x1001, PCI1744_TYPE },
  { 0 }
};

static struct pci_driver pci17x4_driver = {
  .name 	  = "pci17x4",
  .id_table       = pci17x4_id_tbl,
  .probe          = pci17x4_init_one,
  .remove         = pci17x4_remove_one,
};

static struct file_operations pci17x4_fops = {
  .owner   	  = THIS_MODULE,
  .read    	  = pci17x4_read,
  .write   	  = pci17x4_write,
  .unlocked_ioctl = pci17x4_ioctl,
  .compat_ioctl   = pci17x4_ioctl,
  .open    	  = pci17x4_open,
  .release 	  = pci17x4_close,
  .mmap    	  = pci17x4_mmap
};

MODULE_AUTHOR("Csaba Biegl");
MODULE_DESCRIPTION("Driver for the Advantech PCI-1714, PCI-1714UL and PCIe-1744 DAQ cards");
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, pci17x4_id_tbl);

/* ==== pci17x4 private device structure, module private data, and some accessor macros ==== */

enum {
#if PCI17X4_MEMMAP
  PLX_PCI_BAR = 0,						/* PCI BAR index for the PLX 9054 run-time registers */
  ADC_PCI_BAR = 3,						/* PCI BAR index for the PCI 1714 A/D control registers */
#else
  PLX_PCI_BAR = 1,						/* PCI BAR index for the PLX 9054 run-time registers */
  ADC_PCI_BAR = 2,						/* PCI BAR index for the PCI 1714 A/D control registers */
#endif
  PLX_REGS = 0,							/* PCI BAR index for the PLX 9054 run-time registers */
  ADC_REGS = 1							/* PCI BAR index for the PCI 1714 A/D control registers */
};

struct pci17x4_private_data {
  char name[32];						/* device name (same as generated by udev rules in "90-advpci-daq.rules") */
  union {
    uint64_t waitcount;						/* if non-0 user space waits until we read this xfer count */
    struct pci17x4_register ioregister;				/* used to peform pending register I/O when running */
  };
  struct pci17x4_dmastatus dma;					/* part of our structure that is visible to the user via ioctl()-s */
  struct pci17x4_private_data *next;				/* next item in driver's linked list of all attached devices */
  struct pci_dev *pdev;						/* kernel's PCI device desriptor */
  struct device *dev;						/* kernel's Linux device descriptor */
  void **dmabuffers;						/* our DMA buffers */
  wait_queue_head_t waitqueue;					/* usual stuff: user space goes to sleep here */
  spinlock_t lock;						/* used only while running to protect the DMA status record */
  dma_addr_t dma0prev;						/* device mapped address of the previous DMA buffer */
  dma_addr_t dma0addr;						/* device mapped address of the currently transferred DMA buffer */
  dma_addr_t dma0next;						/* device mapped address of the next DMA buffer */
#if PCI17X4_MEMMAP
  char __iomem *raddr[2];					/* PCI BAR start addresses (remapped) */
#else
  uint16_t raddr[2];						/* PCI BAR start addresses (copied from pdev) */
#endif
  uint16_t rsize[2];						/* PCI BAR lengths (copied from pdev) */
  uint8_t minor;						/* this device's minor number */
  uint8_t busy;							/* indicates device is opened */
  uint8_t subtype;						/* 0: PCI1714U, 1:PCI171UL, 2:PCIE1744 */
  uint8_t doregister;						/* If set user space waits for register I/O */
  uint8_t doblockwait;						/* If set user space waits for the given transfer count */
};

#define pci17x4_printk						if(PCI17X4_DEBUG) printk
#define pci17x4_timeout(adc)					(max(5, (int)msecs_to_jiffies(adc->dma.setup.timeout)))
#define pci17x4_validport(adc,bar,idx,size)			({ size_t _ix = (idx), _sz = (size) - 1; (((_ix + _sz) < (adc)->rsize[(bar)]) && !(_ix & _sz)); })
#define pci17x4_reg8to16(rval)					((rval) | ((rval) << 8))
#define pci17x4_reg16to32(rval)					((rval) | ((rval) << 16))
#define pci17x4_reg8to32(rval)					(pci17x4_reg16to32(pci17x4_reg8to16(rval)))
#if     PCI17X4_MEMMAP
#define pci17x4_inportb(adc,bar,idx)				((uint32_t)(readb((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_inportw(adc,bar,idx)				((uint32_t)(readw((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_inportl(adc,bar,idx)				((uint32_t)(readl((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_outportb(adc,bar,idx,data)			(writeb((uint8_t)(data), ((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_outportw(adc,bar,idx,data)			(writew((uint16_t)(data), ((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_outportl(adc,bar,idx,data)			(writel((uint32_t)(data), ((adc)->raddr[(bar)] + (uint16_t)(idx))))
#else
#define pci17x4_inportb(adc,bar,idx)				((uint32_t)(inb((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_inportw(adc,bar,idx)				((uint32_t)(inw((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_inportl(adc,bar,idx)				((uint32_t)(inl((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_outportb(adc,bar,idx,data)			(outb((uint8_t)(data), ((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_outportw(adc,bar,idx,data)			(outw((uint16_t)(data), ((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci17x4_outportl(adc,bar,idx,data)			(outl((uint32_t)(data), ((adc)->raddr[(bar)] + (uint16_t)(idx))))
#endif

static const char pci17x4_name[] = { "pci17x4" };		/* our module name and the root of all generated device names */
static struct pci17x4_private_data *pci17x4_boards;		/* the list of devices */
static struct class *pci17x4_class;				/* Linux stuff for registering our device class */
static struct cdev pci17x4_cdev;				/* Linux stuff for registering our device class */
static spinlock_t pci17x4_lock;					/* used to protect the linked list of devices */
static int pci17x4_numboards[3];				/* number of board found (separate counts for PCIe and PIC 30MHz and 10MHz (UL) boards */
static int pci17x4_major;					/* our dynamically assugned device major */

/* ==== module init / exit ==== */

static int __init
pci17x4_init(void)
{
  int err;
  dev_t dev;
  printk(KERN_INFO "%s: Driver loading.\n", pci17x4_name);
  printk(KERN_INFO "%s: Copyright (C) 2007 EDAS, Inc.\n", pci17x4_name);
  /* Register as a device with kernel.	*/
  if((err = alloc_chrdev_region(&dev, 0, 0xff, pci17x4_name)) < 0) {
    printk(KERN_INFO "%s: Could not allocate device major number, error = %d.\n", pci17x4_name, err);
    return(err);
  }
  pci17x4_major = MAJOR(dev);
  /* Register our file operations with the kernel */
  cdev_init(&pci17x4_cdev, &pci17x4_fops);
  pci17x4_cdev.owner = THIS_MODULE;
  pci17x4_cdev.ops = &pci17x4_fops;
  err = cdev_add(&pci17x4_cdev, dev, 0xff);
  if(err) {
    printk(KERN_INFO "%s: Error %d in registering file operations.\n", pci17x4_name, err);
    unregister_chrdev_region(MKDEV(pci17x4_major, 0), 0xff);
    return(err);
  }
  /* Create the device class */
  pci17x4_class = class_create(THIS_MODULE, pci17x4_name);
  if(IS_ERR(pci17x4_class)) {
    cdev_del(&pci17x4_cdev);
    unregister_chrdev_region(MKDEV(pci17x4_major, 0), 0xff);
    return (PTR_ERR(pci17x4_class));
  }
  spin_lock_init(&pci17x4_lock);
  pci17x4_boards = NULL;
  pci17x4_numboards[0] = 0;
  pci17x4_numboards[1] = 0;
  pci17x4_numboards[2] = 0;
  return (pci_register_driver(&pci17x4_driver));
}

void __exit
pci17x4_exit(void)
{
  pci_unregister_driver(&pci17x4_driver);
  class_destroy(pci17x4_class);
  cdev_del(&pci17x4_cdev);
  unregister_chrdev_region(MKDEV(pci17x4_major, 0), 0xff);
  printk(KERN_INFO "%s: module removed.\n", pci17x4_name);
}

/* ==== per device init / remove ==== */

static int __devinit
pci17x4_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
{
  struct pci17x4_private_data *adc = NULL;
  int err;
  printk(KERN_INFO "%s: intializing device at bus=%d, slot=%d.\n", pci17x4_name, pdev->bus->number, PCI_SLOT(pdev->devfn));
  if((err = pci_enable_device(pdev)) != 0) {
    printk(KERN_WARNING "%s: error enabling PCI device.\n", pci17x4_name);
    return(err);
  }
  /* enable bus-master */
  pci_set_master(pdev);
  /* alloc & initialize the private data structure */
  if((adc = kmalloc(sizeof(*adc), GFP_KERNEL)) == NULL) {
    printk(KERN_WARNING "%s: error allocating memory for device data.\n", pci17x4_name);
    return(-ENOMEM);
  }
  /* initialize the private data in the device */
  memset(adc, 0, sizeof(*adc));
  switch(pdev->subsystem_device) {
  case PCI1744_TYPE:
    adc->subtype = 2;
    sprintf(adc->name, "pci1744e%d", pci17x4_numboards[2]);
    break;
  case PCI1714_TYPE_:
  case PCI1714_TYPE_U:
    sprintf(adc->name, "pci1714u%d", pci17x4_numboards[0]);
    break;
  default:
    sprintf(adc->name, "pci1714ul%d", pci17x4_numboards[1]);
    adc->subtype = 1;
    adc->dma.ulboard = 1;
    break;
  }
  adc->pdev = pdev;
  adc->minor = (adc->subtype << 6) | pci17x4_numboards[adc->subtype];
  adc->rsize[PLX_REGS] = pci_resource_len(pdev, PLX_PCI_BAR);
  adc->rsize[ADC_REGS] = pci_resource_len(pdev, ADC_PCI_BAR);
  if(((err = 0), pci_request_region(pdev, PLX_PCI_BAR, adc->name)) ||
     ((err = 1), pci_request_region(pdev, ADC_PCI_BAR, adc->name))) {
    printk(KERN_WARNING "%s: error reserving %s I/O space.\n", adc->name, (err ? "local register" : "PCI configuration"));
    err ? (void)pci_release_region(pdev, PLX_PCI_BAR) : (void)0;
    kfree(adc);
    return (-EADDRNOTAVAIL);
  }
#if PCI17X4_MEMMAP
  /* remap memory mapped I/O regions */
  if(!((adc->raddr[PLX_REGS] = pci_ioremap_bar(pdev, PLX_PCI_BAR))) ||
     !((adc->raddr[ADC_REGS] = pci_ioremap_bar(pdev, ADC_PCI_BAR)))) {
    printk(KERN_WARNING "%s: error remapping %s I/O space.\n", adc->name, (adc->raddr[PLX_REGS] ? "local register" : "PCI configuration"));
    adc->raddr[PLX_REGS] ? (void)pcim_iounmap(pdev, adc->raddr[PLX_REGS]) : (void)0;
    pci_release_region(pdev, PLX_PCI_BAR);
    pci_release_region(pdev, ADC_PCI_BAR);
    kfree(adc);
    return (-EADDRNOTAVAIL);
  }
#else
  /* store I/O region start addresses */
  adc->raddr[PLX_REGS] = pci_resource_start(pdev, PLX_PCI_BAR);
  adc->raddr[ADC_REGS] = pci_resource_start(pdev, ADC_PCI_BAR);
#endif
  /* initialize working data */
  init_waitqueue_head(&adc->waitqueue);
  spin_lock_init(&adc->lock);
  /* register device with kernel */
  pci_set_drvdata(pdev, adc);
  adc->dev = device_create(pci17x4_class, NULL, MKDEV(pci17x4_major, adc->minor), adc, adc->name);
  /* add device to our linked list */
  spin_lock(&pci17x4_lock);
  adc->next = pci17x4_boards;
  pci17x4_boards = adc;
  pci17x4_numboards[adc->subtype]++;
  spin_unlock(&pci17x4_lock);
  return(0);
}

static void __devexit
pci17x4_remove_one(struct pci_dev *pdev)
{
  struct pci17x4_private_data **padc, *adc = NULL;
  spin_lock(&pci17x4_lock);
  for(padc = &pci17x4_boards; padc[0]; padc = &padc[0]->next) {
    if(padc[0]->pdev == pdev) {
      adc = padc[0];
      padc[0] = adc->next;
      pci17x4_numboards[adc->subtype]--;
      break;
    }
  }
  spin_unlock(&pci17x4_lock);
  if(!adc) {
    printk(KERN_WARNING "%s: error attempt to remove unknown device 0x%x:0x%x\n", pci17x4_name, pdev->bus->number, PCI_SLOT(pdev->devfn));
    return;
  }
  printk(KERN_INFO "%s: removing device.\n", adc->name);
#if PCI17X4_MEMMAP
  pcim_iounmap(pdev, adc->raddr[PLX_REGS]);
  pcim_iounmap(pdev, adc->raddr[ADC_REGS]);
#endif
  pci_release_region(adc->pdev, PLX_PCI_BAR);
  pci_release_region(adc->pdev, ADC_PCI_BAR);
  device_destroy(pci17x4_class, MKDEV(pci17x4_major, adc->minor));
  kfree(adc);
}

/* ==== device open / close ==== */

static int
pci17x4_open(struct inode *ino, struct file *fp)
{
  struct pci17x4_private_data *adc;
  uint32_t arbit;
  spin_lock(&pci17x4_lock);
  for(adc = pci17x4_boards; adc; adc = adc->next) {
    if(adc->minor == MINOR(ino->i_rdev)) {
      break;
    }
  }
  spin_unlock(&pci17x4_lock);
  if(!adc) {
    return(-ENODEV);
  }
  if(adc->busy) {
    return(-EBUSY);
  }
  //if(pci_reset_function(adc->pdev) < 0) {
  //  printk(KERN_WARNING "%s: reset not supported??\n", adc->name);
  //} else {
  //  printk(KERN_INFO "%s: device reset successful\n", adc->name);
  //}
  if(request_irq(adc->pdev->irq, pci17x4_interrupt, IRQF_SHARED, adc->name, adc) < 0) {
    printk(KERN_WARNING "%s: can not get interrupt %d\n", adc->name, adc->pdev->irq);
    return(-EBUSY);
  }
  if(pci17x4_allocdmabuffers(adc, PAGE_SHIFT, 16) != 0) {
    free_irq(adc->pdev->irq, adc);
    return(-ENOMEM);
  }
  adc->busy = 1;
  fp->private_data = adc;
  adc->dma.setup.timeout = 1000;
  adc->dma.error = 0;
  /* PLX9056: enable pci addresses */
  pci17x4_outportl(adc, PLX_REGS, PCI9056_SPACE0_REMAP, 0x0001);
  /* PLX9056: enable LOCAL bus BREQi (bit 18), enable bit 21 for C&J Modes */
  arbit = pci17x4_inportl(adc, PLX_REGS, PCI9056_LOCAL_DMA_ARBIT);
  pci17x4_printk(KERN_INFO "%s: local DMA arbit = 0x%08x.\n", adc->name, arbit);
  pci17x4_outportl(adc, PLX_REGS, PCI9056_LOCAL_DMA_ARBIT, (arbit | 0x00240000));
  /* PLX9056: disable all interrupts */
  pci17x4_outportl(adc, PLX_REGS, PCI9056_INT_CTRL_STAT, 0);
  /* PLX9056: 32 bits local bus data width */
  pci17x4_outportl(adc, PLX_REGS, PCI9056_SPACE0_ROM_DESC, 0x0007);
  /* PLX9056:  DMA threshold */
  pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA_THRESHOLD, 0x0000);
  printk(KERN_INFO "%s: opened.\n", adc->name);
  return(0);
}

static int
pci17x4_close(struct inode *ino, struct file *fp)
{
  struct pci17x4_private_data *adc = (struct pci17x4_private_data *)fp->private_data;
  if(adc->dma.running) {
    pci17x4_dmastop(adc);
  }
  pci17x4_freedmabuffers(adc);
  free_irq(adc->pdev->irq, adc);
  adc->busy = 0;
  printk(KERN_INFO "%s: closed.\n", adc->name);
  return(0);
}

/* ==== file operations on opened devices ==== */

static long
pci17x4_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
{
  struct pci17x4_private_data *adc = (struct pci17x4_private_data *)fp->private_data;
  struct pci17x4_register *rp = &adc->ioregister;
  unsigned long flags;
  int err;
  switch(cmd) {
  case IOCTL_PCI17X4_REGISTER_IO:
    if(!access_ok(VERIFY_READ, (void *)arg, sizeof(*rp)) || (copy_from_user(rp, (void *)arg, sizeof(*rp)) != 0)) {
      return(-EFAULT);
    }
    if((rp->size != 8) && (rp->size != 16) && (rp->size != 32)) {
      return(-EIO);
    }
    if(!pci17x4_validport(adc, ((rp->flags & PCI17X4_IOCTL_REGISTER_PLX) ? PLX_REGS : ADC_REGS), rp->index, (rp->size >> 3))) {
      return(-EIO);
    }
    if(!(rp->flags & (PCI17X4_IOCTL_REGISTER_READ | PCI17X4_IOCTL_REGISTER_WRITE))) {
      return(-EIO);
    }
    if((rp->flags & PCI17X4_IOCTL_REGISTER_WRITE) && !access_ok(VERIFY_WRITE, (void *)arg, sizeof(*rp))) {
      return(-EFAULT);
    }
    if(adc->dma.running && !(rp->flags & PCI17X4_IOCTL_REGISTER_IMMEDIATE)) {
      adc->doregister = 1;
      err = wait_event_interruptible_timeout(adc->waitqueue, (adc->doregister == 0), pci17x4_timeout(adc));
      if(err == (-ERESTARTSYS)) {
        return (err);
      }
      if(adc->doregister) {
        printk(KERN_WARNING "%s: register I/O request timed out\n", adc->name);
        return((adc->doregister = 0), (adc->dma.errtout = 1), (-EIO));
      }
    } else {
      pci17x4_doio(adc);
    }
    if((rp->flags & PCI17X4_IOCTL_REGISTER_READ) && (copy_to_user((void *)arg, rp, sizeof(*rp)) != 0)) {
      return(-EFAULT);
    }
    return(0);
  case IOCTL_PCI17X4_DMA_SETUP:
    if(!adc->dma.running) {
      struct pci17x4_dmasetup dmasetup;
      if(!access_ok(VERIFY_READ, (void *)arg, sizeof(dmasetup)) || (copy_from_user(&dmasetup, (void *)arg, sizeof(dmasetup)) != 0)) {
        return(-EFAULT);
      }
      if((err = pci17x4_allocdmabuffers(adc, dmasetup.blockshift, dmasetup.numblocks)) != 0) {
        return(err);
      }
      adc->dma.setup = dmasetup;
      return(0);
    }
    return(-EBUSY);
  case IOCTL_PCI17X4_DMA_WAIT:
    if(adc->dma.running) {
      struct pci17x4_dmastatus dmastatus;
      if(!access_ok(VERIFY_READ, (void *)arg, sizeof(dmastatus)) || (copy_from_user(&dmastatus, (void *)arg, sizeof(dmastatus)) != 0)) {
        return (-EFAULT);
      }
      spin_lock_irqsave(&adc->lock, flags);
      if(adc->dma.readcount < dmastatus.readcount) {
        adc->waitcount = dmastatus.readcount;
        adc->doblockwait = 1;
        spin_unlock_irqrestore(&adc->lock, flags);
        err = wait_event_interruptible_timeout(adc->waitqueue, (adc->doblockwait == 0), pci17x4_timeout(adc));
        if(err == (-ERESTARTSYS)) {
          return (err);
        }
        if(adc->doblockwait) {
          printk(KERN_WARNING "%s: buffer wait timed out\n", adc->name);
          return((adc->doblockwait = 0), (adc->dma.errtout = 1), (-EIO));
        }
      } else {
        spin_unlock_irqrestore(&adc->lock, flags);
      }
    }
  case IOCTL_PCI17X4_DMA_STATUS:
    if(access_ok(VERIFY_WRITE, (void *)arg, sizeof(struct pci17x4_dmastatus))) {
      struct pci17x4_dmastatus dmastatus;
      if(adc->dma.running) {
        spin_lock_irqsave(&adc->lock, flags);
        dmastatus = adc->dma;
        spin_unlock_irqrestore(&adc->lock, flags);
      } else {
        dmastatus = adc->dma;
      }
      if(copy_to_user((void *)arg, &dmastatus, sizeof(dmastatus)) == 0) {
        return(0);
      }
    }
    return (-EFAULT);
  case IOCTL_PCI17X4_DMA_START:
    if(adc->dma.running) {
      return(-EINVAL);
    }
    if((err = pci17x4_dmastart(adc)) != 0) {
      return(err);
    }
    return(0);
  case IOCTL_PCI17X4_DMA_STOP:
    if(!adc->dma.running) {
      return(-EINVAL);
    }
    pci17x4_dmastop(adc);
    return(0);
  default:
    printk(KERN_WARNING "%s: invalid ioctl() code (%u).\n", adc->name, cmd);
    return(-EINVAL);
  }
}

static int
pci17x4_mmap(struct file *fp, struct vm_area_struct *vma)
{
  struct pci17x4_private_data *adc = (struct pci17x4_private_data *)fp->private_data;
  size_t mapsize = (size_t)(vma->vm_end - vma->vm_start);
  size_t blksize = (size_t)(1) << adc->dma.setup.blockshift;
  if(((size_t)vma->vm_start % PAGE_SIZE) || (mapsize != (adc->dma.setup.numblocks * blksize)) || vma->vm_pgoff) {
    return(-EINVAL);
  }
  printk(KERN_INFO "%s: mapping start=0x%lx, size=%ld\n", adc->name, vma->vm_start, (long)mapsize);
  while(mapsize > 0) {
    unsigned long addr = vma->vm_start + (mapsize -= blksize);
    unsigned long pfn = virt_to_phys(adc->dmabuffers[mapsize >> adc->dma.setup.blockshift]) >> PAGE_SHIFT;
    int err = remap_pfn_range(vma, addr, pfn, blksize, vma->vm_page_prot);
    if(err < 0) {
      printk(KERN_WARNING "%s: remap block #%d failed, addr=0x%lx, pfn=0x%lx, size=%d.\n", adc->name, (int)mapsize, addr, pfn, (int)blksize);
      return(err);
    }
  }
  return(0);
}

static ssize_t
pci17x4_read(struct file *fp, char *buf, size_t count, loff_t *off)
{
  return (-EIO);
}

static ssize_t
pci17x4_write(struct file *fp, const char *buf, size_t count, loff_t *off)
{
  return (-EPERM);
}

/* ==== helper functions ==== */

static void
pci17x4_doio(struct pci17x4_private_data *adc)
{
  struct pci17x4_register *regs = &adc->ioregister;
  int pcibar = (regs->flags & PCI17X4_IOCTL_REGISTER_PLX) ? PLX_REGS : ADC_REGS;
  switch((regs->flags & PCI17X4_IOCTL_REGISTER_WRITE) ? regs->size : 0) {
  case 8:
    pci17x4_outportb(adc, pcibar, regs->index, regs->value);
    break;
  case 16:
    pci17x4_outportw(adc, pcibar, regs->index, regs->value);
    break;
  case 32:
    pci17x4_outportl(adc, pcibar, regs->index, regs->value);
    break;
  }
  switch((regs->flags & PCI17X4_IOCTL_REGISTER_READ) ? regs->size : 0) {
  case 8:
    regs->value = pci17x4_inportb(adc, pcibar, regs->index);
    break;
  case 16:
    regs->value = pci17x4_inportw(adc, pcibar, regs->index);
    break;
  case 32:
    regs->value = pci17x4_inportl(adc, pcibar, regs->index);
    break;
  }
}

static int
pci17x4_allocdmabuffers(struct pci17x4_private_data *adc, unsigned blkshift, unsigned numblocks)
{
  unsigned idx = 0;
  if((blkshift < PAGE_SHIFT) || (numblocks < PCI17X4_IOCTL_MIN_NUMBLOCKS) || (numblocks > PCI17X4_IOCTL_MAX_NUMBLOCKS)) {
    return(-EINVAL);
  }
  pci17x4_freedmabuffers(adc);
  adc->dma.setup.blockshift = blkshift;
  adc->dma.setup.numblocks = numblocks;
  adc->dma.pageshift = PAGE_SHIFT;
  if((adc->dmabuffers = kzalloc((sizeof(adc->dmabuffers[0]) * numblocks), GFP_KERNEL))) {
    unsigned buforder = blkshift - PAGE_SHIFT;
    unsigned bufpages = 1 << buforder;
    for(idx = 0; idx < numblocks; idx++) {
      if((adc->dmabuffers[idx] = (void *)__get_free_pages((GFP_KERNEL | GFP_DMA32), buforder))) {
        struct page *pgend, *page = virt_to_page(adc->dmabuffers[idx]);
        for(pgend = page + bufpages; page < pgend; SetPageReserved(page++));
      } else {
        pci17x4_freedmabuffers(adc);
        break;
      }
    }
    printk(KERN_INFO "%s: Allocated %d buffers of %d bytes each.\n", adc->name, numblocks, (1 << blkshift));
    return(0);
  }
  printk(KERN_WARNING "%s: Buffer allocation error (%d:%d:%d) .\n", adc->name, (1 << blkshift), numblocks, idx);
  return(-ENOMEM);
}

static void
pci17x4_freedmabuffers(struct pci17x4_private_data *adc)
{
  if(adc->dmabuffers) {
    unsigned idx;
    unsigned buforder = adc->dma.setup.blockshift - PAGE_SHIFT;
    unsigned bufpages = 1 << buforder;
    for(idx = 0; idx < adc->dma.setup.numblocks; idx++) {
      if(adc->dmabuffers[idx]) {
        struct page *pgend, *page = virt_to_page(adc->dmabuffers[idx]);
        for(pgend = page + bufpages; page < pgend; ClearPageReserved(page++));
        free_pages((unsigned long)(adc->dmabuffers[idx]), buforder);
        adc->dmabuffers[idx] = NULL;
      }
    }
    kfree(adc->dmabuffers);
    adc->dmabuffers = NULL;
  }
}

static int
pci17x4_dmastart(struct pci17x4_private_data *adc)
{
  const int dma0mode = (1 << 2) | (1 << 8) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 17);
  unsigned long flags;
  uint32_t trigger;
  /* PLX9056: DMA 0 mode descriptor register (no need to reload them per transfer) */
  pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_DESC_PTR, ((1 << 2) | (1 << 3)));
  /* Channel count specific DMA settings (no need to reload them per transfer) */
  switch(pci17x4_inportw(adc, ADC_REGS, PCI17X4_ADCENABLE_REG16) & 0x0f) {
  case 0x01:
    /* PLX9056: 16 bit local bus transfers from channel 0 */
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_MODE, (1 | dma0mode));
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_LOCAL_ADDR, PCI17X4_DATA_0_REG16);
    /* PCI17X4: use Channel 0 FIFO as the local bus request source */
    pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMAREQUEST_REG16, PCI17X4_DMAREQUEST_FIFO0);
    break;
  case 0x04:
    /* PLX9056: 16 bit local bus transfers from channel 2 */
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_MODE, (1 | dma0mode));
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_LOCAL_ADDR, PCI17X4_DATA_2_REG16);
    /* PCI17X4: use Channel 2 FIFO as the local bus request source */
    pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMAREQUEST_REG16, PCI17X4_DMAREQUEST_FIFO2);
    break;
  case 0x03:
    /* PLX9056: 32 bit local bus transfers from channels 0,1 */
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_MODE, (2 | dma0mode));
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_LOCAL_ADDR, PCI17X4_DATA_01_REG32);
    /* PCI17X4: use Channel 0 FIFO as the local bus request source */
    pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMAREQUEST_REG16, PCI17X4_DMAREQUEST_FIFO0);
    break;
  case 0x0c:
    /* PLX9056: 32 bit local bus transfers from channels 2,3 */
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_MODE, (2 | dma0mode));
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_LOCAL_ADDR, PCI17X4_DATA_23_REG32);
    /* PCI17X4: use Channel 0 FIFO as the local bus request source */
    pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMAREQUEST_REG16, PCI17X4_DMAREQUEST_FIFO0);
    break;
  case 0x0f:
    /* PLX9056: 32 bit local bus transfers from all channels using the alternating data register */
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_MODE, (2 | dma0mode));
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_LOCAL_ADDR, PCI17X4_DMADATA_REG32);
    /* PCI17X4: use Channel 2 FIFO as the local bus request source */
    pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMAREQUEST_REG16, PCI17X4_DMAREQUEST_FIFO2);
    /* PCI17X4: reset DMA start channel register */
    pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMACHRESET_REG16, 0xffff);
    break;
  default:
    printk(KERN_WARNING "%s: attempt to start DMA with invalid channel mask setting.\n", adc->name);
    return(-EINVAL);
  }
  printk(KERN_INFO "%s: starting DMA transfers (timeout is %d jiffies).\n", adc->name, pci17x4_timeout(adc));
  /* PCI17X4: clear & reset fifos */
  pci17x4_outportl(adc, ADC_REGS, PCI17X4_FIFOCTR_REG32, pci17x4_reg8to32(PCI17X4_FIFOCTR_CLEAR | PCI17X4_FIFOCTR_RESET));
  /* PCI17X4: "safe" values for trigger and post trigger DMA counter registers for "hiccup-free" streaming */
  trigger = pci17x4_inportw(adc, ADC_REGS, PCI17X4_TRIGGER_REG16);
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_TRIGGER_REG16, (trigger | PCI17X4_TRIGGER_FLAG | PCI17X4_TRIGGER_DMATC));
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMACOUNTER_REG16, 0xfff0);
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMACRESET_REG16, 0xffff);
  /* PCI17X4: enable FIFO half full and almost full interrupts */
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQ_REG16, 0x80ff);
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQCLEAR_REG16, 0xffff);
  /* PLX9056: set DMA0 thresholds */
  pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA_THRESHOLD, 0x3333);
  /* PLX9056: enable DMA0 done and local interrupts */
  spin_lock_irqsave(&adc->lock, flags);
  pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x08);
  pci17x4_outportl(adc, PLX_REGS, PCI9056_INT_CTRL_STAT, ((1 << 8) | (1 << 11) | (1 << 18)));
  /* administer running DMA */
  adc->dma0next = dma_map_single(&adc->pdev->dev, adc->dmabuffers[0], ((size_t)(1) << adc->dma.setup.blockshift), DMA_FROM_DEVICE);
  adc->dma0prev = adc->dma0addr = 0;
  adc->dma.readcount = 0;
  adc->dma.errsys = 0;
  adc->dma.running = 1;
  spin_unlock_irqrestore(&adc->lock, flags);
  return(0);
}

static void
pci17x4_dmastop(struct pci17x4_private_data *adc)
{
  int i;
  unsigned long flags;
  printk(KERN_INFO "%s: stopping DMA transfers.\n", adc->name);
  spin_lock_irqsave(&adc->lock, flags);
  adc->dma.running = 0;
  /* PLX9056: abort DMA 0 */
  pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x0c);
  /* PLX9056: disable all interrupts */
  pci17x4_outportl(adc, PLX_REGS, PCI9056_INT_CTRL_STAT, 0);
  /* PCI17X4: disable all interrupts */
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQ_REG16, 0);
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQCLEAR_REG16, 0xffff);
  /* PLX9056: actually wait until DMA terminates */
  for(i = 0; i < 100; i++) {
    if(pci17x4_inportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT) & 0x10) {
      break;
    }
    spin_unlock_irqrestore(&adc->lock, flags);
    udelay(1);
    spin_lock_irqsave(&adc->lock, flags);
  }
  pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x08);
  if(adc->dma0addr) {
    dma_unmap_single(&adc->pdev->dev, adc->dma0addr, ((size_t)(1) << adc->dma.setup.blockshift), DMA_FROM_DEVICE);
    adc->dma0addr = 0;
  }
  if(adc->dma0prev) {
    dma_unmap_single(&adc->pdev->dev, adc->dma0prev, ((size_t)(1) << adc->dma.setup.blockshift), DMA_FROM_DEVICE);
    adc->dma0prev = 0;
  }
  if(adc->dma0next) {
    dma_unmap_single(&adc->pdev->dev, adc->dma0next, ((size_t)(1) << adc->dma.setup.blockshift), DMA_FROM_DEVICE);
    adc->dma0next = 0;
  }
  spin_unlock_irqrestore(&adc->lock, flags);
}

/*
 * Interrupt handler.
 * Triggered by DMA0 done interrupt of the PLX9056 or by the FIFO status (HF, AF) interrupts of the A/D.
 * Optimized for fast turnaround time between the end of the previous DMA and the start or the next one.
 * It checks a few registers in the "gap" between DMA-s, but postpones most of the "administration" to
 * the point after the next DMA has been started.
 */
static irqreturn_t
pci17x4_interrupt(int irq, void *devptr)
{
  struct pci17x4_private_data *adc = (struct pci17x4_private_data *)devptr;
  uint32_t irqbits = pci17x4_inportl(adc, PLX_REGS, PCI9056_INT_CTRL_STAT);
  uint32_t irqdma = (1 << 8) | (1 << 18) | (1 << 21);
  uint32_t irqfifo = (1 << 8) | (1 << 11) | (1 << 15);
  irqreturn_t result = IRQ_NONE;
  pci17x4_printk(KERN_INFO "%s: Interrupt: irq=0x%08x masked=0x%08x adcirq=0x%04x\n", adc->name, irqbits, (irqbits & (irqfifo | irqdma)), pci17x4_inportw(adc, ADC_REGS, PCI17X4_IRQ_REG16));
  /* Check for either of the accepted interrupts from device */
  irqfifo = ((irqbits & irqfifo) == irqfifo);
  irqdma = ((irqbits & irqdma) == irqdma);
  if(irqfifo || irqdma)  {
    result = IRQ_HANDLED;
    /* But handle them only if the device is still "officially" running */
    if(adc->dma.running) {
      uint32_t adcfifo = 0, adcirq = 0;
      pci17x4_printk(KERN_INFO "%s: Interrupt: fifo=%d dma=%d\n", adc->name, irqfifo, irqdma);
      spin_lock(&adc->lock);
      /* Administer completed DMA buffer */
      if(irqdma) {
        /* PLX9056: disable DMA0 and clear the DMA0 done interrupt */
        pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x08);
        pci17x4_printk(KERN_INFO "%s: Interrupt: DMA done\n", adc->name);
        if((adc->dma0addr != 0) && (adc->dma0prev == 0)) {
          adc->dma0prev = adc->dma0addr;
          adc->dma0addr = 0;
          adc->dma.readcount++;
        } else {
          printk(KERN_WARNING "%s: WTF at %s %d\n", adc->name, __FILE__, __LINE__);
        }
      }
      /* Check if periodic trigger needs to be re-armed */
      if(adc->dma.setup.armtrigger) {
        uint32_t trigger = pci17x4_inportw(adc, ADC_REGS, PCI17X4_TRIGGER_REG16);
        if(trigger & PCI17X4_TRIGGER_FLAG) {
          //pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMACOUNTER_REG16, 0xfff0);
          pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMACRESET_REG16, 0xffff);
          pci17x4_outportw(adc, ADC_REGS, PCI17X4_TRIGGER_REG16, (trigger | PCI17X4_TRIGGER_FLAG | PCI17X4_TRIGGER_DMATC));
        }
      }
      /* Check if there is a pending register I/O ioctl() */
      if(adc->doregister) {
        pci17x4_doio(adc);
      }
      /* Get ADC FIFO and Interrupt registers and administer ADC FIFO status interrupt */
      if(irqfifo) {
        adcirq = pci17x4_inportw(adc, ADC_REGS, PCI17X4_IRQ_REG16);
        // adcfifo = pci17x4_inportl(adc, ADC_REGS, PCI17X4_FIFOSTAT_REG32);
        pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQ_REG16, ((adcirq & 0xaa) ? 0 : 0x80aa));
        pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQCLEAR_REG16, 0xffff);
        pci17x4_printk(KERN_INFO "%s: Interrupt: A/D fifo=0x%08x irq=0x%04x\n", adc->name, adcfifo, adcirq);
      }
      /* Check if we can start next DMA */
      if((adc->dma0addr == 0) && (adc->dma0next != 0)) {
        pci17x4_printk(KERN_INFO "%s: Interrupt: Starting DMA to 0x%08x\n", adc->name, (unsigned)adc->dma0next);
        adc->dma0addr = adc->dma0next;
        adc->dma0next = 0;
        /* PLX9056: set up destination PCI address and count for a block DMA0 transfer */
        pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_PCI_ADDR, (uint32_t)adc->dma0addr);
        pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_COUNT, (1 << adc->dma.setup.blockshift));
        /* PLX9056: enable and start DMA on channel 0 */
        pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x0b);
      }
      /* Unmap just completed DMA buffer */
      if(adc->dma0prev != 0) {
        dma_unmap_single(&adc->pdev->dev, adc->dma0prev, ((size_t)(1) << adc->dma.setup.blockshift), DMA_FROM_DEVICE);
        adc->dma0prev = 0;
      }
      /* Map next DMA buffer. (We do it now with the DMA already restarted to reduce turnaround time) */
      if(adc->dma0next == 0) {
        int which = (int)((adc->dma.readcount + 1) % adc->dma.setup.numblocks);
        adc->dma0next = dma_map_single(&adc->pdev->dev, adc->dmabuffers[which], ((size_t)(1) << adc->dma.setup.blockshift), DMA_FROM_DEVICE);
      }
      spin_unlock(&adc->lock);
      /* Check ADC FIFO overflow */
      if((adcfifo & 0x24242424) || (adcirq & 0xaa)) {
        printk(KERN_WARNING "%s: Fifo error 0x%08x:0x%04x at block %llu (irq fifo=%d dma=%d)\n", adc->name, (unsigned)adcfifo, (unsigned)adcirq, (unsigned long long)adc->dma.readcount, irqfifo, irqdma);
        adc->dma.errfifo = 1;
      }
      /* Wake up user space if needed */
      if(adc->dma.error || adc->doregister || (adc->doblockwait && (adc->dma.readcount >= adc->waitcount))) {
        adc->doregister = adc->doblockwait = 0;
        wake_up_interruptible(&adc->waitqueue);
      }
    }
  }
  return(result);
}
