#ifndef __KERNEL__
#define __KERNEL__
#endif

#ifndef MODULE
#define MODULE
#endif

#include <linux/module.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/kdev_t.h>
#include <linux/cdev.h>
#include <linux/string.h>
#include <linux/sysfs.h>
#include <linux/major.h>
#include <linux/delay.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/poll.h>
#include <linux/mm.h>
#include <linux/pci.h>
#include <asm/uaccess.h>
#include <asm/io.h>

#include "pci1714.h"
#include "Reg9056.h"

/* ==== Module interface ==== */

struct pci1714_private_data;

static int __init pci1714_init(void);
static void __exit pci1714_exit(void);
static int __devinit pci1714_init_one(struct pci_dev *pdev, const struct pci_device_id *ent);
static void __devexit pci1714_remove_one(struct pci_dev *pdev);
static int pci1714_open(struct inode *ino, struct file *fp);
static int pci1714_close(struct inode *ino, struct file *fp);
static ssize_t pci1714_read(struct file *fp, char *buf, size_t count, loff_t *off);
static ssize_t pci1714_write(struct file *fp, const char *buf, size_t count, loff_t *off);
static int pci1714_ioctl(struct inode *ino, struct file *fp, unsigned int cmd, unsigned long arg);
static int pci1714_mmap(struct file *fp, struct vm_area_struct *vma);
static int pci1714_allocdmabuffers(struct pci1714_private_data *adc, size_t size, size_t nbuffers);
static void pci1714_freedmabuffers(struct pci1714_private_data *adc);
static int pci1714_dmastart(struct pci1714_private_data *adc);
static void pci1714_dmastop(struct pci1714_private_data *adc);
static int pci1714_dmaupdate(struct pci1714_private_data *adc);
static void pci1714_timer(unsigned long arg);

module_init(pci1714_init);
module_exit(pci1714_exit);

static struct pci_device_id pci1714_id_tbl[] __devinitdata = {
  { 0x13fe, 0x1714, 0x13fe, PCI1714_TYPE_     },
  { 0x13fe, 0x1714, 0x13fe, PCI1714_TYPE_U    },
  { 0x13fe, 0x1714, 0x13fe, PCI1714_TYPE_UL   },
  { 0x13fe, 0x1714, 0x13fe, PCI1714_TYPE_UL_A },
  { 0x13fe, 0x1714, 0x13fe, PCI1714_TYPE_UL_B },
  { 0 }
};

static struct pci_driver pci1714_driver = {
  .name = "pci1714",
  .id_table = pci1714_id_tbl,
  .probe = pci1714_init_one,
  .remove = pci1714_remove_one,
};

static struct file_operations pci1714_fops = {
  .owner   = THIS_MODULE,
  .read    = pci1714_read,
  .write   = pci1714_write,
  .ioctl   = pci1714_ioctl,
  .open    = pci1714_open,
  .release = pci1714_close,
  .mmap    = pci1714_mmap
};

MODULE_AUTHOR("Csaba Biegl");
MODULE_DESCRIPTION("Driver for the Advantech PCI-1714 and PCI-1714UL DAQ cards");
MODULE_LICENSE("GPL");
MODULE_DEVICE_TABLE(pci, pci1714_id_tbl);

/* ==== pci1714 private device structure, module private data, and some accessor macros ==== */

enum {
  MAX_DMABUFFER_ORDER = 3,					/* maximum allocation order for an individual contiguous DMA buffer */
  MAX_DMABUFFER_SIZE = (PAGE_SIZE << MAX_DMABUFFER_ORDER),	/* maximum size of an individual DMA buffer */
  DMA_BPTABLE_SHIFT = 10,					/* lower bits of the two level DMA buffer table indexing */
  DMA_BPTABLE_ENTRIES = (1 << DMA_BPTABLE_SHIFT),		/* size of the second level tables */
  DMA_BPTABLE_MASK = (DMA_BPTABLE_ENTRIES - 1),			/* index mask for second level tables */
  NUM_DMA_BPTABLES = 60,					/* number of first level page table slots (in the device descriptor) */
  MAX_DMA_BUFFERS = (NUM_DMA_BPTABLES * DMA_BPTABLE_ENTRIES),	/* maximum possible number of DMA buffers */
  MAX_DMA_MEMORY = (MAX_DMA_BUFFERS * MAX_DMABUFFER_SIZE),	/* maximum possible memory used for DMA (over 3G from the 4G of 32bit memory!) */
  PLX_REGS = 1,							/* PCI BAR index for the PLX 9054 run-time registers */
  ADC_REGS = 2							/* PCI BAR index for the PCI 1714 A/D control registers */
};

struct pci1714_private_data {
  char name[32];						/* device name (same as generated by udev rules in "90-advpci-daq.rules") */
  wait_queue_head_t waitqueue;					/* usual stuff: go to sleep here */
  struct pci_dev *pdev;						/* kernel's PCI device desriptor */
  struct device *dev;						/* kernel's Linux device descriptor */
  struct pci1714_private_data *next;				/* next item in driver's linked list of all attached devices */
  struct pci1714_dmastatus dma;					/* part of our structure that is visible to the user via ioctl()-s */
  struct timer_list timer;    	     				/* used to implement DMA check without using irq */	   
  spinlock_t lock;						/* used only while running to protect the DMA status record */
  struct {
    uint32_t pciaddr;						/* PCI destination address (physical address, must be in the 1st 4GB) */
    uint32_t ioaddr;						/* ADC I/O port to transfer data from */
    uint32_t count;						/* number of bytes to transfer */
    uint32_t chain;						/* pointer to next DMA descriptor */
  } *dmadescr[NUM_DMA_BPTABLES];				/* PLX DMA descriptors (with physical addresses) of our circular buffers (two level indexing) */
  struct {
    void *address;
  } *dmabuffer[NUM_DMA_BPTABLES];				/* virtual addresses of our circular DMA buffers (two level indexing) */
  char *readptr;						/* read() uses it to copy out data from the entry at the head of circular DMA buffer list */
  int readcnt;							/* read() uses it to copy out data from the entry at the head of circular DMA buffer list */
  uint16_t raddr[4];						/* PCI BAR start addresses (copied from pdev) */
  uint16_t rsize[4];						/* PCI BAR lengths (copied from pdev) */
  uint8_t minor;						/* this device's minor number */
  uint8_t busy;							/* indicates device is opened */
};

#define pci1714_dmadescr(adc,idx)				({ size_t _ix = (idx); &(adc)->dmadescr[_ix >> DMA_BPTABLE_SHIFT][_ix & DMA_BPTABLE_MASK]; })
#define pci1714_dmabuffer(adc,idx)				({ size_t _ix = (idx); &(adc)->dmabuffer[_ix >> DMA_BPTABLE_SHIFT][_ix & DMA_BPTABLE_MASK]; })
#define pci1714_validport(adc,bar,idx,size)			({ size_t _ix = (idx), _sz = (size) - 1; (((_ix + _sz) < (adc)->rsize[(bar)]) && !(_ix & _sz)); })
#define pci1714_inportb(adc,bar,idx)				((uint32_t)(inb((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci1714_inportw(adc,bar,idx)				((uint32_t)(inw((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci1714_inportl(adc,bar,idx)				((uint32_t)(inl((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci1714_outportb(adc,bar,idx,data)			(outb((uint8_t)(data), ((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci1714_outportw(adc,bar,idx,data)			(outw((uint16_t)(data), ((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci1714_outportl(adc,bar,idx,data)			(outl((uint32_t)(data), ((adc)->raddr[(bar)] + (uint16_t)(idx))))
#define pci1714_reg16to32(rval)					((rval) | ((rval) << 16))
#define pci1714_reg8to16(rval)					((rval) | ((rval) << 8))
#define pci1714_reg8to32(rval)					(pci1714_reg16to32(pci1714_reg8to16(rval)))

static const char pci1714_name[] = { "pci1714" };		/* our module name and the root of all generated device names */
static spinlock_t pci1714_lock;					/* used to protect the linked list of devices */
static struct pci1714_private_data *pci1714_boards;		/* the list of devices */
static struct class *pci1714_class;				/* Linux stuff for registering our device class */
static struct cdev pci1714_cdev;				/* Linux stuff for registering our device class */
static int pci1714_numboards[2];				/* number of board found (separate counts for 30MHz and 10MHz (UL) boards */
static int pci1714_major;					/* our dynamically assugned device major */

/* ==== module init / exit ==== */

static int __init
pci1714_init(void)
{
  int err;
  dev_t dev;
  printk(KERN_INFO "%s: Driver loading.\n", pci1714_name);
  printk(KERN_INFO "%s: Copyright (C) 2007 EDAS, Inc.\n", pci1714_name);
  /* Register as a device with kernel.	*/
  if ((err = alloc_chrdev_region(&dev, 0, 0xff, pci1714_name)) < 0) {
    printk(KERN_INFO "%s: Could not allocate device major number, error = %d.\n", pci1714_name, err);
    return(err);
  }
  pci1714_major = MAJOR(dev);
  /* Register our file operations with the kernel */
  cdev_init(&pci1714_cdev, &pci1714_fops);
  pci1714_cdev.owner = THIS_MODULE;
  pci1714_cdev.ops = &pci1714_fops;
  err = cdev_add(&pci1714_cdev, dev, 0xff);
  if (err) {
    printk(KERN_INFO "%s: Error %d in registering file operations.\n", pci1714_name, err);
    return(err);
  }
  /* create the class for the das4020 */
  pci1714_class = class_create(THIS_MODULE, pci1714_name);
  if (IS_ERR(pci1714_class)) {
    return PTR_ERR(pci1714_class);
  }
  spin_lock_init(&pci1714_lock);
  pci1714_boards = NULL;
  pci1714_numboards[0] = 0;
  pci1714_numboards[1] = 0;
  return (pci_register_driver(&pci1714_driver));
}

void __exit
pci1714_exit(void)
{
  dev_t dev = MKDEV(pci1714_major, 0);
  class_destroy(pci1714_class);
  cdev_del(&pci1714_cdev);
  unregister_chrdev_region(dev, 0xff);
  printk(KERN_INFO "%s: module removed.\n", pci1714_name);
  pci_unregister_driver(&pci1714_driver);
}

/* ==== per device init / remove ==== */

static int __devinit
pci1714_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
{
  struct pci1714_private_data *adc = NULL;
  int err;
  printk(KERN_INFO "%s: intializing device at bus=%d, slot=%d.\n", pci1714_name, pdev->bus->number, PCI_SLOT(pdev->devfn));
  if ((err = pci_enable_device(pdev)) != 0) {
    printk(KERN_WARNING "%s: error enabling PCI device.\n", pci1714_name);
    return(err);
  }
  /* enable bus-master */
  pci_set_master(pdev);
  /* alloc & initialize the private data structure */
  if ((adc = kmalloc(sizeof(*adc), GFP_KERNEL)) == NULL) {
    printk(KERN_WARNING "%s: error allocating memory for device data.\n", pci1714_name);
    return(-ENOMEM);
  }
  /* initialize the private data in the device */
  memset(adc, 0, sizeof(*adc));
  switch (pdev->subsystem_device) {
  case PCI1714_TYPE_:
  case PCI1714_TYPE_U:
    sprintf(adc->name, "%su%d", pci1714_name, pci1714_numboards[0]);
    break;
  default:
    sprintf(adc->name, "%sul%d", pci1714_name, pci1714_numboards[1]);
    adc->dma.ulboard = 1;
    break;
  }
  adc->minor = (adc->dma.ulboard << 6) | pci1714_numboards[adc->dma.ulboard];
  adc->raddr[PLX_REGS] = pci_resource_start(pdev, PLX_REGS);
  adc->raddr[ADC_REGS] = pci_resource_start(pdev, ADC_REGS);
  adc->rsize[PLX_REGS] = pci_resource_len(pdev, PLX_REGS);
  adc->rsize[ADC_REGS] = pci_resource_len(pdev, ADC_REGS);
  adc->pdev = pdev;
  /* request I/O regions */
  if (request_region(adc->raddr[PLX_REGS], adc->rsize[PLX_REGS], adc->name) == NULL) {
    printk(KERN_WARNING "%s: error reserving io range: 0x%x:0x%x.\n", adc->name, (int)adc->raddr[PLX_REGS], (int)adc->rsize[PLX_REGS]);
    kfree(adc);
    return (-EADDRNOTAVAIL);
  }
  if (request_region(adc->raddr[ADC_REGS], adc->rsize[ADC_REGS], adc->name) == NULL) {
    printk(KERN_WARNING "%s: error reserving io range: 0x%x:0x%x.\n", adc->name, (int)adc->raddr[ADC_REGS], (int)adc->rsize[ADC_REGS]);
    release_region(adc->raddr[PLX_REGS], adc->rsize[PLX_REGS]);
    kfree(adc);
    return (-EADDRNOTAVAIL);
  }
  /* initialize working data */
  init_waitqueue_head(&adc->waitqueue);
  spin_lock_init(&adc->lock);
  init_timer(&adc->timer);
  adc->timer.function = pci1714_timer;
  adc->timer.data = (unsigned long)adc;
  /* register device with kernel */
  pci_set_drvdata(pdev, adc);
  adc->dev = device_create(pci1714_class, NULL, MKDEV(pci1714_major, adc->minor), adc, &adc->name[strlen(pci1714_name) + 1]);
  /* add device to our linked list */
  spin_lock(&pci1714_lock);
  adc->next = pci1714_boards;
  pci1714_boards = adc;
  pci1714_numboards[adc->dma.ulboard]++;
  spin_unlock(&pci1714_lock);
  return(0);
}

static void __devexit
pci1714_remove_one(struct pci_dev *pdev)
{
  struct pci1714_private_data **padc, *adc = NULL;
  spin_lock(&pci1714_lock);
  for (padc = &pci1714_boards; padc[0]; padc = &padc[0]->next) {
    if (padc[0]->pdev == pdev) {
      adc = padc[0];
      padc[0] = adc->next;
      pci1714_numboards[adc->dma.ulboard]--;
      break;
    }
  }
  spin_unlock(&pci1714_lock);
  if (!adc) {
    printk(KERN_WARNING "%s: error attempt to remove unknown device 0x%x:0x%x\n", pci1714_name, pdev->bus->number, PCI_SLOT(pdev->devfn));
    return;
  }
  printk(KERN_INFO "%s: removing device.\n", adc->name);
  release_region(adc->raddr[PLX_REGS], adc->rsize[PLX_REGS]);
  release_region(adc->raddr[ADC_REGS], adc->rsize[ADC_REGS]);
  device_destroy(pci1714_class, MKDEV(pci1714_major, adc->minor));
  kfree(adc);
}

/* ==== device open / close ==== */

static int
pci1714_open(struct inode *ino, struct file *fp)
{
  struct pci1714_private_data *adc;
  spin_lock(&pci1714_lock);
  for (adc = pci1714_boards; adc; adc = adc->next) {
    if (adc->minor == MINOR(ino->i_rdev)) {
      break;
    }
  }
  spin_unlock(&pci1714_lock);
  if (!adc) {
    return(-ENODEV);
  }
  if (adc->busy) {
    return(-EBUSY);
  }
  printk(KERN_INFO "%s: opened.\n", adc->name);
  if (!pci1714_allocdmabuffers(adc, 16384, 4)) {
    return(-ENOMEM);
  }
  adc->busy = 1;
  fp->private_data = adc;
  if (0) {
    /* PLX9056: enable pci addresses */
    pci1714_outportl(adc, PLX_REGS, PCI9056_SPACE0_REMAP, 0x0001);
    /* PLX9056: enable LOCAL bus BREQi (bit 18), enable bit 21 for C&J Modes */
    pci1714_outportl(adc, PLX_REGS, PCI9056_LOCAL_DMA_ARBIT, 0x00240000);
    /* PLX9056: disable all interrupts */
    pci1714_outportl(adc, PLX_REGS, PCI9056_INT_CTRL_STAT, 0);
    /* PLX9056: 32 bits local bus data width */
    pci1714_outportl(adc, PLX_REGS, PCI9056_SPACE0_ROM_DESC, 0x0007);
    /* PLX9056:  DMA threshold */
    pci1714_outportl(adc, PLX_REGS, PCI9056_DMA_THRESHOLD, 0x0000);
  }
  return(0);
}

static int
pci1714_close(struct inode *ino, struct file *fp)
{
  struct pci1714_private_data *adc = (struct pci1714_private_data *)fp->private_data;
  if (adc->dma.running) {
    pci1714_dmastop(adc);
  }
  del_timer_sync(&adc->timer);
  pci1714_freedmabuffers(adc);
  adc->busy = 0;
  printk(KERN_INFO "%s: closed.\n", adc->name);
  return(0);
}

/* ==== file operations on opened devices ==== */

static int
pci1714_ioctl(struct inode *ino, struct file *fp, unsigned int cmd, unsigned long arg)
{
  struct pci1714_private_data *adc = (struct pci1714_private_data *)fp->private_data;
  struct pci1714_dmastatus dmastatus;
  struct pci1714_register regs;
  int err = 1, size = 1;
  switch (cmd) {
  case IOCTL_PCI1714_READ_REGISTER32:
    size <<= 1;
  case IOCTL_PCI1714_READ_REGISTER16:
    size <<= 1;
  case IOCTL_PCI1714_READ_REGISTER8:
    if (access_ok(VERIFY_WRITE, (void *)arg, sizeof(regs))) {
      if ((err = copy_from_user(&regs, (void *)arg, sizeof(regs))) == 0) {
        if (!pci1714_validport(adc, ADC_REGS, regs.index, size)) {
          return(-EIO);
        }
        switch (size) {
        case 1:
          regs.value = pci1714_inportb(adc, ADC_REGS, regs.index);
          break;
        case 2:
          regs.value = pci1714_inportw(adc, ADC_REGS, regs.index);
          break;
        default:
          regs.value = pci1714_inportl(adc, ADC_REGS, regs.index);
          break;
        }
        err = copy_to_user((void *)arg, &regs, sizeof(regs));
      }
    }
    return (err ? (-EFAULT) : 0);
  case IOCTL_PCI1714_WRITE_REGISTER32:
    size <<= 1;
  case IOCTL_PCI1714_WRITE_REGISTER16:
    size <<= 1;
  case IOCTL_PCI1714_WRITE_REGISTER8:
    if (access_ok(VERIFY_READ, (void *)arg, sizeof(regs))) {
      if ((err = copy_from_user(&regs, (void *)arg, sizeof(regs))) == 0) {
        if (!pci1714_validport(adc, ADC_REGS, regs.index, size)) {
          return(-EIO);
        }
        switch (size) {
        case 1:
          pci1714_outportb(adc, ADC_REGS, regs.index, regs.value);
          break;
        case 2:
          pci1714_outportw(adc, ADC_REGS, regs.index, regs.value);
          break;
        default:
          pci1714_outportl(adc, ADC_REGS, regs.index, regs.value);
          break;
        }
      }
    }
    return (err ? (-EFAULT) : 0);
  case IOCTL_PCI1714_DMA_SETUP:
    if (adc->dma.running) {
      return(-EBUSY);
    }
    if (access_ok(VERIFY_READ, (void *)arg, sizeof(struct pci1714_dmasetup))) {
      struct pci1714_dmasetup dmasetup;
      if ((err = copy_from_user(&dmasetup, (void *)arg, sizeof(dmasetup))) == 0) {
        if (!pci1714_allocdmabuffers(adc, dmasetup.blocksize, dmasetup.numblocks)) {
          return(-ENOMEM);
        }
      }
    }
    return (err ? (-EFAULT) : 0);
  case IOCTL_PCI1714_DMA_WAIT:
    if (adc->dma.running) {
      int tout = max(2, (int)msecs_to_jiffies(adc->dma.readwait));
      interruptible_sleep_on_timeout(&adc->waitqueue, tout);
    }
  case IOCTL_PCI1714_DMA_STATUS:
    if (adc->dma.running) {
      unsigned long flags;
      spin_lock_irqsave(&adc->lock, flags);
      pci1714_dmaupdate(adc);
      dmastatus = adc->dma;
      spin_unlock_irqrestore(&adc->lock, flags);
    }
    else {
      dmastatus = adc->dma;
    }
    if (access_ok(VERIFY_WRITE, (void *)arg, sizeof(struct pci1714_dmastatus))) {
      err = copy_to_user((void *)arg, &dmastatus, sizeof(dmastatus));
    }
    return (err ? (-EFAULT) : 0);
  case IOCTL_PCI1714_DMA_START:
    if (adc->dma.running) {
      return(-EINVAL);
    }
    if ((err = pci1714_dmastart(adc)) != 0) {
      return(err);
    }
    adc->timer.expires = jiffies + 1;
    add_timer(&adc->timer);
    return(0);
  case IOCTL_PCI1714_DMA_STOP:
    if (!adc->dma.running) {
      return(-EINVAL);
    }
    pci1714_dmastop(adc);
    del_timer_sync(&adc->timer);
    return(0);
  default:
    printk(KERN_WARNING "%s: invalid ioctl() code (%u).\n", adc->name, cmd);
    return(-EINVAL);
  }
}

static int
pci1714_mmap(struct file *fp, struct vm_area_struct *vma)
{
  struct pci1714_private_data *adc = (struct pci1714_private_data *)fp->private_data;
  size_t mapsize = (size_t)(vma->vm_end - vma->vm_start);
  size_t mapoffs = (size_t)vma->vm_pgoff * PAGE_SIZE;
  if (((size_t)vma->vm_start % PAGE_SIZE) || (mapsize % adc->dma.setup.blocksize) || (mapoffs % adc->dma.setup.blocksize)) {
    return(-EINVAL);
  }
  printk(KERN_INFO "%s: mapping start=0x%lx, size=%d, offset=%d\n", adc->name, vma->vm_start, (int)mapsize, (int)mapoffs);
  mapsize /= adc->dma.setup.blocksize;
  mapoffs /= adc->dma.setup.blocksize;
  while (mapsize > 0) {
    unsigned long addr = vma->vm_start + (--mapsize * adc->dma.setup.blocksize);
    unsigned long pfn = pci1714_dmadescr(adc, ((mapsize + mapoffs) % adc->dma.setup.numblocks))->pciaddr >> PAGE_SHIFT;
    int err = remap_pfn_range(vma, addr, pfn, adc->dma.setup.blocksize, vma->vm_page_prot);
    if (err < 0) {
      printk(KERN_WARNING "%s: remap block #%d failed, addr=0x%lx, pfn=0x%lx, size=%d.\n", adc->name, (int)mapsize, addr, pfn, (int)adc->dma.setup.blocksize);
      return(err);
    }
  }
  return(0);
}

static ssize_t
pci1714_read(struct file *fp, char *buf, size_t count, loff_t *off)
{
  return (-EIO);
}

static ssize_t
pci1714_write(struct file *fp, const char *buf, size_t count, loff_t *off)
{
  return (-EPERM);
}

/* ==== DMA buffer pointer table and DMA buffer allocation / deallocation ==== */

#define pci1714_allocbptables(_table,_count,_flags) ({					\
  /* buffer pointer table allocator: macro because called with different data types */	\
  size_t _todo = (_count), _ndone = 0, _at = 0;						\
  while (_ndone < _todo) {								\
    size_t _nalloc = min((_todo - _ndone), (size_t)DMA_BPTABLE_ENTRIES);		\
    size_t _nbytes = sizeof(_table[0][0]) * _nalloc;					\
    if (!((_table[_at] = (void *)__get_free_pages(_flags, get_order(_nbytes))))) {	\
      break;										\
    }											\
    _ndone += _nalloc;									\
    _at++;										\
  }											\
  _ndone;										\
})

#define pci1714_freebptables(_table,_count) ({						\
  /* buffer pointer table free: macro because called with different data types */	\
  size_t _todo = (_count), _at = 0;							\
  while(_todo > 0) {									\
    size_t _nfree = min(_todo, (size_t)DMA_BPTABLE_ENTRIES);				\
    size_t _nbytes = sizeof(_table[0][0]) * _nfree;					\
    free_pages((unsigned long)_table[_at], get_order(_nbytes));				\
    _todo -= _nfree;									\
    _at++;										\
  }											\
})

static void
pci1714_freedmabuffers_aux(struct pci1714_private_data *adc, size_t nb, size_t border, size_t ndt, size_t nbt)
{
  while (nb > 0) {
    void *bufadr = pci1714_dmabuffer(adc, (--nb))->address;
    struct page *pgend, *page = virt_to_page(bufadr);
    for (pgend = page + (1 << border); page < pgend; ClearPageReserved(page++));
    free_pages((unsigned long)bufadr, border);
  }
  pci1714_freebptables(adc->dmadescr, ndt);
  pci1714_freebptables(adc->dmabuffer, nbt);
  adc->dma.setup.numblocks = 0;
  adc->dma.setup.blocksize = 0;
  adc->dma.totalsize = 0;
}

static int
pci1714_allocdmabuffers(struct pci1714_private_data *adc, size_t blocksize, size_t numblocks)
{
  size_t nbufs = (PAGE_SIZE + (blocksize * numblocks) - 1) >> PAGE_SHIFT;
  size_t bsize = PAGE_SIZE, border = 0, nptdone = 0, nvtdone = 0, nb = 0;
  /* adjust user requested sizes to reasonable values */
  while ((bsize < (size_t)MAX_DMABUFFER_SIZE) && ((bsize < blocksize) || (nbufs > (size_t)MAX_DMA_BUFFERS))) {
    nbufs = (nbufs + 1) >> 1;
    bsize <<= 1;
    border++;
  }
  nbufs = clamp(nbufs, (size_t)4, (size_t)MAX_DMA_BUFFERS);
  /* free previous buffers (if any) */
  pci1714_freedmabuffers(adc);
  /* allocate (or set) buffer pointer tables first, then the actual buffers */
  if ((nptdone = pci1714_allocbptables(adc->dmadescr, nbufs, (GFP_KERNEL | GFP_DMA32))) == nbufs) {
    if ((nvtdone = pci1714_allocbptables(adc->dmabuffer, nbufs, GFP_KERNEL)) == nbufs) {
      while (nb < nbufs) {
        void *bufp = pci1714_dmabuffer(adc, nb)->address = (void *)__get_free_pages((GFP_KERNEL | GFP_DMA32), border);
        if (bufp) {
          uint32_t *datap = (uint32_t *)bufp;
          struct page *pgend, *page = virt_to_page(bufp);
          for (pgend = page + (1 << border); page < pgend; SetPageReserved(page++));
          for (datap += (bsize / sizeof(datap[0])); datap > (uint32_t *)bufp; *(--datap) = (uint32_t)nb);
          pci1714_dmadescr(adc, nb)->pciaddr = (uint32_t)virt_to_phys(bufp);
          pci1714_dmadescr(adc, nb)->ioaddr = PCI1714_DMADATA_REG32;
          pci1714_dmadescr(adc, nb)->chain = (uint32_t)virt_to_phys(pci1714_dmadescr(adc, ((nb + 1) % nbufs))) | 0x9;
          pci1714_dmadescr(adc, nb++)->count = bsize;
          continue;
        }
        break;
      }
    }
  }
  if (nb != nbufs) {
    printk(KERN_WARNING "%s: error allocating DMA buffers.\n", adc->name);
    printk(KERN_WARNING "%s: requested %d physical pointer table entries, got %d.\n", adc->name, (int)nbufs, (int)nptdone);
    printk(KERN_WARNING "%s: requested %d virtual pointer table entries, got %d.\n", adc->name, (int)nbufs, (int)nvtdone);
    printk(KERN_WARNING "%s: requested %d buffers of %dk each, got %d.\n", adc->name, (int)nbufs, (int)(bsize >> 10), (int)nb);
    pci1714_freedmabuffers_aux(adc, nb, border, nptdone, nvtdone);
    return (0);
  }
  printk(KERN_INFO "%s: allocated %d buffers of %d bytes each.\n", adc->name, (int)nbufs, (int)bsize);
  adc->dma.totalsize = nbufs * bsize;
  adc->dma.setup.numblocks = nbufs;
  adc->dma.setup.blocksize = bsize;
  return (1);
}

static void
pci1714_freedmabuffers(struct pci1714_private_data *adc)
{
  size_t nb = adc->dma.setup.numblocks;
  if (nb) {
    pci1714_freedmabuffers_aux(adc, nb, get_order(adc->dma.setup.blocksize), nb, nb);
    printk(KERN_INFO "%s: freed DMA buffers.\n", adc->name);
  }
}

/* ==== DMA helper functions ==== */

static int
pci1714_dmastart(struct pci1714_private_data *adc)
{
  uint32_t i, dma0mode, dma0source, dmarate;
  printk(KERN_INFO "%s: starting DMA transfers.\n", adc->name);
  if ((dmarate = pci1714_inportw(adc, ADC_REGS, PCI1714_CLOCK_REG16) & PCI1714_CLOCK_DIVMASK) == 0) {
    printk(KERN_WARNING "%s: attempt to start DMA with invalid clock setting.\n", adc->name);
    return(-EINVAL);
  }
  dmarate = (adc->dma.ulboard ? PCI1714_CLOCKRATE_UL : PCI1714_CLOCKRATE) / (dmarate + PCI1714_CLOCK_DIVBIAS);
  /* determine channel count specific settings */
  switch (pci1714_inportw(adc, ADC_REGS, PCI1714_ADCENABLE_REG16) & 0x0f) {
  case 0x01:
    /* PLX9056: 16 bit local bus transfers from channel 0 */
    dma0mode = 1 | (1 << 2) | (1 << 8) | (1 << 9) | (1 << 11) | (1 << 12);
    dma0source = PCI1714_DATA_0_REG16;
    dmarate *= sizeof(int16_t);
    break;
  case 0x04:
    /* PLX9056: 16 bit local bus transfers from channel 2 */
    dma0mode = 1 | (1 << 2) | (1 << 8) | (1 << 9) | (1 << 11) | (1 << 12);
    dma0source = PCI1714_DATA_2_REG16;
    dmarate *= sizeof(int16_t);
    break;
  case 0x03:
    /* PLX9056: 32 bit local bus transfers from channels 0,1 */
    dma0mode = 2 | (1 << 2) | (1 << 8) | (1 << 9) | (1 << 11) | (1 << 12);
    dma0source = PCI1714_DATA_01_REG32;
    dmarate *= sizeof(int16_t) * 2;
    break;
  case 0x0c:
    /* PLX9056: 32 bit local bus transfers from channels 2,3 */
    dma0mode = 2 | (1 << 2) | (1 << 8) | (1 << 9) | (1 << 11) | (1 << 12);
    dma0source = PCI1714_DATA_23_REG32;
    dmarate *= sizeof(int16_t) * 2;
    break;
  case 0x0f:
    /* PLX9056: 32 bit local bus transfers from all channels */
    dma0mode = 2 | (1 << 2) | (1 << 8) | (1 << 9) | (1 << 11) | (1 << 12);
    dma0source = PCI1714_DMADATA_REG32;
    dmarate *= sizeof(int16_t) * 4;
    break;
  default:
    printk(KERN_WARNING "%s: attempt to start DMA with invalid channel mask setting.\n", adc->name);
    return(-EINVAL);
  }
  if (!adc->dma.setup.numblocks) {
    return(-ENOMEM);
  }
  for (i = 0; i < adc->dma.setup.numblocks; i++) {
    pci1714_dmadescr(adc, i)->ioaddr = dma0source;
  }
  /* PCI1714: clear & reset fifos */
  pci1714_outportl(adc, ADC_REGS, PCI1714_FIFOCTR_REG32, pci1714_reg8to32(PCI1714_FIFOCTR_CLEAR | PCI1714_FIFOCTR_RESET));
  /* PCI1714: reset DMA start channel register */
  pci1714_outportw(adc, ADC_REGS, PCI1714_DMACHRESET_REG16, 0xffff);
  /* PLX9056: set up registers for a chain DMA transfer with descriptors in PCI memory*/
  pci1714_outportl(adc, PLX_REGS, PCI9056_DMA0_MODE, dma0mode);
  pci1714_outportl(adc, PLX_REGS, PCI9056_DMA0_DESC_PTR, pci1714_dmadescr(adc, (adc->dma.setup.numblocks - 1))->chain);
  /* PLX9056: enable and start DMA on channel 0 */
  pci1714_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x09);
  pci1714_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x0b);
  /* administer running DMA */
  adc->dma.readwait = (1500 * adc->dma.setup.blocksize) / dmarate;
  adc->dma.dmapos = 0;
  adc->dma.dmanumfull = 0;
  adc->readptr = NULL;
  adc->readcnt = 0;
  adc->dma.error = 0;
  adc->dma.running = 1;
  return(0);
}

static void
pci1714_dmastop(struct pci1714_private_data *adc)
{
  int i;
  printk(KERN_INFO "%s: stopping DMA transfers.\n", adc->name);
  pci1714_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x04);
  for (i = 0; i < 100; i++) {
    udelay(1);
    if (pci1714_inportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT) & 0x10) {
      break;
    }
  }
  pci1714_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0);
  adc->dma.running = 0;
}

static int
pci1714_dmaupdate(struct pci1714_private_data *adc)
{
  uint32_t i, dmapos = pci1714_inportl(adc, PLX_REGS, PCI9056_DMA0_PCI_ADDR);
  for (i = adc->dma.dmapos; i < adc->dma.setup.numblocks; i++) {
    uint32_t phybuf = pci1714_dmadescr(adc, i)->pciaddr;
    if ((dmapos >= phybuf) && (dmapos < (phybuf + adc->dma.setup.blocksize))) {
      int nadvance = i - adc->dma.dmapos;
      adc->dma.dmapos = i;
      // if (adc->dma.dmanumfull == 0) {
      //   printk(KERN_INFO "%s: update DMA, transfer at 0x%08x, write at %d.\n", adc->name, (unsigned)dmapos, (int)i);
      // }
      return (nadvance);
    }
  }
  for (i = 0; i < adc->dma.dmapos; i++) {
    uint32_t phybuf = pci1714_dmadescr(adc, i)->pciaddr;
    if ((dmapos >= phybuf) && (dmapos < (phybuf + adc->dma.setup.blocksize))) {
      int nadvance = i + adc->dma.setup.numblocks - adc->dma.dmapos;
      adc->dma.dmanumfull++;
      adc->dma.dmapos = i;
      return (nadvance);
    }
  }
  printk(KERN_WARNING "%s: bad DMA address 0x%08lx.\n", adc->name, (unsigned long)dmapos);
  return (0);
}

static void
pci1714_timer(unsigned long arg)
{
  struct pci1714_private_data *adc = (struct pci1714_private_data *)arg;
  if (adc->dma.running) {
    unsigned long flags;
    int newdata, trigger;
    adc->timer.expires = jiffies + 1;
    add_timer(&adc->timer);
    spin_lock_irqsave(&adc->lock, flags);
    newdata = pci1714_dmaupdate(adc);
    if ((trigger = pci1714_inportw(adc, ADC_REGS, PCI1714_TRIGGER_REG16)) & PCI1714_TRIGGER_FLAG) {
      pci1714_outportw(adc, ADC_REGS, PCI1714_TRIGGER_REG16, (trigger | PCI1714_TRIGGER_FLAG | PCI1714_TRIGGER_DMATC));
    }
    if (pci1714_inportl(adc, ADC_REGS, PCI1714_FIFOSTAT_REG32) & pci1714_reg8to32(PCI1714_FIFOSTAT_FULL | PCI1714_FIFOSTAT_AFULL)) {
      adc->dma.errfifo = 1;
    }
    spin_unlock_irqrestore(&adc->lock, flags);
    if (newdata > 0) {
      wake_up_interruptible(&adc->waitqueue);
    }
  }
}

