/*
 * Interrupt handler.
 * Triggered by DMA0 done interrupt of the PLX9056 or by the FIFO status (AF) interrupts of the A/D.
 * The DMA interrupt is used to signal user space about available data.
 * If we ever get a FIFO interrupt then it signals a FIFO overflow condition.
 */
static irqreturn_t
pci17x4_interrupt(int irq, void *devptr)
{
  struct pci17x4_private_data *adc = (struct pci17x4_private_data *)devptr;
  uint32_t irqbits = pci17x4_inportl(adc, PLX_REGS, PCI9056_INT_CTRL_STAT);
  uint32_t irqdma = (1 << 8) | (1 << 18) | (1 << 21);
  uint32_t irqfifo = (1 << 8) | (1 << 11) | (1 << 15);
  irqreturn_t result = IRQ_NONE;
  pci17x4_printk(KERN_INFO "%s: Interrupt: irq=0x%08x masked=0x%08x", adc->name, irqbits, (irqbits & (irqfifo | irqdma)));
  /* Check for either of the accepted interrupts from device */
  irqfifo = ((irqbits & irqfifo) == irqfifo);
  irqdma = ((irqbits & irqdma) == irqdma);
  if(irqfifo || irqdma)  {
    result = IRQ_HANDLED;
    if(irqfifo) {
      // This is a FIFO error
      printk(KERN_INFO "%s: Interrupt: ADC FIFO error\n", adc->name);
      adc->dma.errfifo = 1;
      /* PCI17X4: disable interrupts */
      pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQ_REG16, 0);
      pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQCLEAR_REG16, 0xffff);
    }
    if(irqdma) {
      // Count number of completed DMA blocks
      uint32_t dmapos = pci17x4_inportl(adc, PLX_REGS, PCI9056_DMA0_PCI_ADDR);
      unsigned nnew, pos = (unsigned)(adc->dma.readcount % adc->dma.setup.numblocks);
      // Acknowledge DMA Irq
      pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x09);
      // Reset trigger if needed
      pci17x4_checktrigger(adc);
      // Count advance
      for(nnew = 0; nnew < adc->dma.setup.numblocks; nnew++, pos = (pos + 1) % adc->dma.setup.numblocks) {
        uint32_t phybuf = pci17x4_dmadesc(adc, pos).pciaddr;
        if((dmapos >= phybuf) && (dmapos < (phybuf + (1 << adc->dma.setup.blockshift)))) {
          break;
        }
      }
      if(nnew == 0) {
        pci17x4_printk(KERN_INFO "%s: Interrupt: spurious irq (0x%08x@%llu)\n", adc->name, (unsigned)dmapos, (unsigned long long)adc->dma.readcount);
      } else if(nnew > 1) {
        pci17x4_printk(KERN_INFO "%s: Interrupt: missed irq (0x%08x:%u@%llu)\n", adc->name, (unsigned)dmapos, nnew, (unsigned long long)adc->dma.readcount);
      }
      pci17x4_printk(KERN_INFO "%s: Interrupt: %u new DMA blocks\n", adc->name, nnew);
      adc->dma.readcount += nnew;
    }
    /* Wake up user space if needed */
    if(adc->dma.error || (adc->doblockwait && (adc->dma.readcount >= adc->waitcount))) {
      adc->doblockwait = 0;
      wake_up_interruptible(&adc->waitqueue);
    }
  }
  return(result);
}

static void
pci17x4_freedmabuffers(struct pci17x4_private_data *adc)
{
  unsigned idx;
  if(adc->dmabuffers) {
    unsigned buforder = adc->dma.setup.blockshift - PAGE_SHIFT;
    unsigned bufpages = 1 << buforder;
    for(idx = 0; idx < adc->dma.setup.numblocks; idx++) {
      if(adc->dmabuffers[idx]) {
        struct page *pgend, *page = virt_to_page(adc->dmabuffers[idx]);
        for(pgend = page + bufpages; page < pgend; ClearPageReserved(page++));
        free_pages((unsigned long)(adc->dmabuffers[idx]), buforder);
        adc->dmabuffers[idx] = NULL;
      }
    }
    vfree(adc->dmabuffers);
    adc->dmabuffers = NULL;
  }
  if(adc->dmadescs) {
    unsigned descpages = (adc->dma.setup.numblocks + DMA_DESCS_PER_PAGE - 1) / DMA_DESCS_PER_PAGE;
    for(idx = 0; idx < descpages; idx++) {
      if(adc->dmadescs[idx]) {
        ClearPageReserved(virt_to_page(adc->dmadescs[idx]));
        free_pages((unsigned long)(adc->dmadescs[idx]), 0);
        adc->dmadescs[idx] = NULL;
      }
    }
    vfree(adc->dmadescs);
    adc->dmadescs = NULL;
  }
}

static int
pci17x4_allocdmabuffers(struct pci17x4_private_data *adc, struct pci17x4_dmasetup *s)
{
  unsigned idx, descpages = (s->numblocks + DMA_DESCS_PER_PAGE - 1) / DMA_DESCS_PER_PAGE;
  if((s->blockshift < PAGE_SHIFT) || (s->numblocks < PCI17X4_IOCTL_MIN_NUMBLOCKS) || (s->numblocks > PCI17X4_IOCTL_MAX_NUMBLOCKS) || (s->numblocks & 1)) {
    return(-EINVAL);
  }
  pci17x4_freedmabuffers(adc);
  adc->dma.setup = (*s);
  adc->dma.pageshift = PAGE_SHIFT;
  if((adc->dmabuffers = vmalloc(sizeof(adc->dmabuffers[0]) * s->numblocks))) {
    unsigned buforder = s->blockshift - PAGE_SHIFT;
    unsigned bufpages = 1 << buforder;
    memset(adc->dmabuffers, 0, (sizeof(adc->dmabuffers[0]) * s->numblocks));
    for(idx = 0; idx < s->numblocks; idx++) {
      if((adc->dmabuffers[idx] = (void *)__get_free_pages((GFP_KERNEL | GFP_DMA32), buforder))) {
        struct page *pgend, *page = virt_to_page(adc->dmabuffers[idx]);
        for(pgend = page + bufpages; page < pgend; SetPageReserved(page++));
      } else {
        pci17x4_freedmabuffers(adc);
        break;
      }
    }
  }
  if(adc->dmabuffers && ((adc->dmadescs = vmalloc(sizeof(adc->dmadescs[0]) * descpages)))) {
    memset(adc->dmadescs, 0, (sizeof(adc->dmadescs[0]) * descpages));
    for(idx = 0; idx < descpages; idx++) {
      if((adc->dmadescs[idx] = (void *)__get_free_pages((GFP_KERNEL | GFP_DMA32), 0))) {
        memset(adc->dmadescs[idx], 0, PAGE_SIZE);
        SetPageReserved(virt_to_page(adc->dmadescs[idx]));
      } else {
        pci17x4_freedmabuffers(adc);
        break;
      }
    }
  }
  if(adc->dmabuffers && adc->dmadescs) {
    for(idx = 0; idx < s->numblocks; idx++) {
      pci17x4_dmadesc(adc, idx).pciaddr = (uint32_t)virt_to_phys(adc->dmabuffers[idx]);
      pci17x4_dmadesc(adc, idx).ioaddr = PCI17X4_DMADATA_REG32;
      pci17x4_dmadesc(adc, idx).chain = (uint32_t)virt_to_phys(&pci17x4_dmadesc(adc, ((idx + 1) % s->numblocks))) | (1 << 3) | (1 << 2) | 1;
      pci17x4_dmadesc(adc, idx).count = (1 << s->blockshift);
    }
    printk(KERN_INFO "%s: Allocated %d buffers of %d bytes each.\n", adc->name, s->numblocks, (1 << s->blockshift));
    return(0);
  }
  printk(KERN_WARNING "%s: Buffer allocation error (%d:%d:%d) .\n", adc->name, (1 << s->blockshift), s->numblocks, idx);
  return(-ENOMEM);
}

/* ==== DMA helper functions ==== */
static int
pci17x4_dmastart(struct pci17x4_private_data *adc)
{
  uint32_t dma0mode = (1 << 2) | (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12) | (1 << 17);
  uint32_t i, dma0source;
  unsigned long flags;
  if(!adc->dma.setup.numblocks || !adc->dmabuffers || !adc->dmadescs) {
    return(-ENOMEM);
  }
  /* determine channel count specific settings */
  switch(pci17x4_inportw(adc, ADC_REGS, PCI17X4_ADCENABLE_REG16) & 0x0f) {
  case 0x01:
    /* PLX9056: 16 bit local bus transfers from channel 0 */
    dma0mode |= 1;
    dma0source = PCI17X4_DATA_0_REG16;
    break;
  case 0x04:
    /* PLX9056: 16 bit local bus transfers from channel 2 */
    dma0mode |= 1;
    dma0source = PCI17X4_DATA_2_REG16;
    break;
  case 0x03:
    /* PLX9056: 32 bit local bus transfers from channels 0,1 */
    dma0mode |= 2;
    dma0source = PCI17X4_DATA_01_REG32;
    break;
  case 0x0c:
    /* PLX9056: 32 bit local bus transfers from channels 2,3 */
    dma0mode |= 2;
    dma0source = PCI17X4_DATA_23_REG32;
    break;
  case 0x0f:
    /* PLX9056: 32 bit local bus transfers from all channels */
    dma0mode |= 2;
    dma0source = PCI17X4_DMADATA_REG32;
    break;
  default:
    printk(KERN_WARNING "%s: attempt to start DMA with invalid channel mask setting.\n", adc->name);
    return(-EINVAL);
  }
  printk(KERN_INFO "%s: starting DMA transfers.\n", adc->name);
  for(i = 0; i < adc->dma.setup.numblocks; i++) {
    pci17x4_dmadesc(adc, i).ioaddr = dma0source;
  }
  /* PCI17X4: clear & reset fifos */
  pci17x4_outportl(adc, ADC_REGS, PCI17X4_FIFOCTR_REG32, pci17x4_reg8to32(PCI17X4_FIFOCTR_CLEAR | PCI17X4_FIFOCTR_RESET));
  /* PCI17X4: enable FIFO almost full interrupts */
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQ_REG16, 0x80aa);
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQCLEAR_REG16, 0xffff);
  /* PCI17X4: reset DMA start channel register */
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_DMACHRESET_REG16, 0xffff);
  /* PCI17X4: set up trigger registers */
  pci17x4_setuptrigger(adc);
  /* PLX9056: set up registers for a chain DMA transfer with descriptors in PCI memory*/
  pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_MODE, dma0mode);
  pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA0_DESC_PTR, pci17x4_dmadesc(adc, (adc->dma.setup.numblocks - 1)).chain);
  /* PLX9056: enable DMA0 done and local interrupts */
  spin_lock_irqsave(&adc->lock, flags);
  pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x08);
  pci17x4_outportl(adc, PLX_REGS, PCI9056_INT_CTRL_STAT, ((1 << 7) | (1 << 8) | (1 << 11) | (1 << 18)));
  /* PLX9056: enable and start DMA on channel 0 */
  pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x09);
  pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x0b);
  /* administer running DMA */
  adc->dma.readcount = 0;
  adc->dma.error = 0;
  adc->dma.running = 1;
  spin_unlock_irqrestore(&adc->lock, flags);
  return(0);
}

static void
pci17x4_dmastop(struct pci17x4_private_data *adc)
{
  int i;
  unsigned long flags;
  printk(KERN_INFO "%s: stopping DMA transfers.\n", adc->name);
  spin_lock_irqsave(&adc->lock, flags);
  adc->dma.running = 0;
  /* PLX9056: abort DMA 0 */
  pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x0c);
  /* PLX9056: disable all interrupts */
  pci17x4_outportl(adc, PLX_REGS, PCI9056_INT_CTRL_STAT, 0);
  /* PCI17X4: disable all interrupts */
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQ_REG16, 0);
  pci17x4_outportw(adc, ADC_REGS, PCI17X4_IRQCLEAR_REG16, 0xffff);
  /* PLX9056: actually wait until DMA terminates */
  for(i = 0; i < 100; i++) {
    if(pci17x4_inportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT) & 0x10) {
      break;
    }
    spin_unlock_irqrestore(&adc->lock, flags);
    udelay(1);
    spin_lock_irqsave(&adc->lock, flags);
  }
  pci17x4_outportb(adc, PLX_REGS, PCI9056_DMA_COMMAND_STAT, 0x08);
  spin_unlock_irqrestore(&adc->lock, flags);
}

static void
pci17x4_doio(struct pci17x4_private_data *adc, struct pci17x4_register *regs)
{
  int pcibar = (regs->flags & PCI17X4_IOCTL_REGISTER_PLX) ? PLX_REGS : ADC_REGS;
  switch((regs->flags & PCI17X4_IOCTL_REGISTER_WRITE) ? regs->size : 0) {
  case 8:
    pci17x4_outportb(adc, pcibar, regs->index, regs->value);
    break;
  case 16:
    pci17x4_outportw(adc, pcibar, regs->index, regs->value);
    break;
  case 32:
    pci17x4_outportl(adc, pcibar, regs->index, regs->value);
    break;
  }
  switch((regs->flags & PCI17X4_IOCTL_REGISTER_READ) ? regs->size : 0) {
  case 8:
    regs->value = pci17x4_inportb(adc, pcibar, regs->index);
    break;
  case 16:
    regs->value = pci17x4_inportw(adc, pcibar, regs->index);
    break;
  case 32:
    regs->value = pci17x4_inportl(adc, pcibar, regs->index);
    break;
  }
}

/* ==== device open / close ==== */
static int
pci17x4_open(struct inode *ino, struct file *fp)
{
  struct pci17x4_private_data *adc;
  uint32_t arbit;
  spin_lock(&pci17x4_lock);
  for(adc = pci17x4_boards; adc; adc = adc->next) {
    if(adc->minor == MINOR(ino->i_rdev)) {
      break;
    }
  }
  spin_unlock(&pci17x4_lock);
  if(!adc) {
    return(-ENODEV);
  }
  if(adc->busy) {
    return(-EBUSY);
  }
  if(request_irq(adc->pdev->irq, pci17x4_interrupt, IRQF_SHARED, adc->name, adc) < 0) {
    printk(KERN_WARNING "%s: can not get interrupt %d\n", adc->name, adc->pdev->irq);
    return(-EBUSY);
  }
  adc->busy = 1;
  adc->dma.error = 0;
  fp->private_data = adc;
  if(0) {
    pci17x4_outportl(adc, PLX_REGS, PCI9056_SPACE0_REMAP, 0x0001);
    /* PLX9056: enable LOCAL bus BREQi (bit 18), enable bit 21 for C&J Modes */
    arbit = pci17x4_inportl(adc, PLX_REGS, PCI9056_LOCAL_DMA_ARBIT);
    pci17x4_printk(KERN_INFO "%s: local DMA arbit = 0x%08x.\n", adc->name, arbit);
    pci17x4_outportl(adc, PLX_REGS, PCI9056_LOCAL_DMA_ARBIT, (arbit | 0x00240000));
    /* PLX9056: disable all interrupts */
    pci17x4_outportl(adc, PLX_REGS, PCI9056_INT_CTRL_STAT, 0);
    /* PLX9056: 32 bits local bus data width */
    pci17x4_outportl(adc, PLX_REGS, PCI9056_SPACE0_ROM_DESC, 0x0007);
    /* PLX9056:  DMA threshold */
    pci17x4_outportl(adc, PLX_REGS, PCI9056_DMA_THRESHOLD, 0x0000);
  }
  printk(KERN_INFO "%s: opened.\n", adc->name);
  return(0);
}

static int
pci17x4_close(struct inode *ino, struct file *fp)
{
  struct pci17x4_private_data *adc = (struct pci17x4_private_data *)fp->private_data;
  if(adc->dma.running) {
    pci17x4_dmastop(adc);
  }
  pci17x4_freedmabuffers(adc);
  free_irq(adc->pdev->irq, adc);
  adc->busy = 0;
  printk(KERN_INFO "%s: closed.\n", adc->name);
  return(0);
}

/* ==== file operations on opened devices ==== */
static long
pci17x4_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
{
  struct pci17x4_private_data *adc = (struct pci17x4_private_data *)fp->private_data;
  struct pci17x4_dmastatus dmastatus;
  struct pci17x4_register ioreg;
  unsigned long flags;
  int err = 0;
  switch(cmd) {
  case IOCTL_PCI17X4_REGISTER_IO:
    if(!access_ok(VERIFY_READ, (void *)arg, sizeof(ioreg)) || (copy_from_user(&ioreg, (void *)arg, sizeof(ioreg)) != 0)) {
      return(-EFAULT);
    }
    if((ioreg.size != 8) && (ioreg.size != 16) && (ioreg.size != 32)) {
      return(-EIO);
    }
    if(!pci17x4_validport(adc, ((ioreg.flags & PCI17X4_IOCTL_REGISTER_PLX) ? PLX_REGS : ADC_REGS), ioreg.index, (ioreg.size >> 3))) {
      return(-EIO);
    }
    if(!(ioreg.flags & (PCI17X4_IOCTL_REGISTER_READ | PCI17X4_IOCTL_REGISTER_WRITE))) {
      return(-EIO);
    }
    if((ioreg.flags & PCI17X4_IOCTL_REGISTER_WRITE) && !access_ok(VERIFY_WRITE, (void *)arg, sizeof(ioreg))) {
      return(-EFAULT);
    }
    pci17x4_doio(adc, &ioreg);
    if((ioreg.flags & PCI17X4_IOCTL_REGISTER_READ) && (copy_to_user((void *)arg, &ioreg, sizeof(ioreg)) != 0)) {
      return(-EFAULT);
    }
    return(0);
  case IOCTL_PCI17X4_DMA_SETUP:
    if(adc->dma.running) {
      return(-EBUSY);
    }
    if(access_ok(VERIFY_READ, (void *)arg, sizeof(struct pci17x4_dmasetup))) {
      struct pci17x4_dmasetup dmasetup;
      if((err = copy_from_user(&dmasetup, (void *)arg, sizeof(dmasetup))) == 0) {
        return(pci17x4_allocdmabuffers(adc, &dmasetup));
      }
    }
    return (-EFAULT);
  case IOCTL_PCI17X4_DMA_WAIT:
    if(adc->dma.running) {
      if(!access_ok(VERIFY_READ, (void *)arg, sizeof(dmastatus)) || (copy_from_user(&dmastatus, (void *)arg, sizeof(dmastatus)) != 0)) {
        return (-EFAULT);
      }
      spin_lock_irqsave(&adc->lock, flags);
      if(adc->dma.readcount < dmastatus.readcount) {
        adc->waitcount = dmastatus.readcount;
        adc->doblockwait = 1;
        spin_unlock_irqrestore(&adc->lock, flags);
        err = wait_event_interruptible_timeout(adc->waitqueue, (adc->doblockwait == 0), pci17x4_timeout(adc));
        if(err == (-ERESTARTSYS)) {
          return (err);
        }
        if(adc->doblockwait) {
          printk(KERN_WARNING "%s: buffer wait timed out\n", adc->name);
          return((adc->doblockwait = 0), (adc->dma.errtout = 1), (-EIO));
        }
      } else {
        spin_unlock_irqrestore(&adc->lock, flags);
      }
    }
  case IOCTL_PCI17X4_DMA_STATUS:
    if(access_ok(VERIFY_WRITE, (void *)arg, sizeof(struct pci17x4_dmastatus))) {
      if(adc->dma.running) {
        spin_lock_irqsave(&adc->lock, flags);
        dmastatus = adc->dma;
        spin_unlock_irqrestore(&adc->lock, flags);
      } else {
        dmastatus = adc->dma;
      }
      if(copy_to_user((void *)arg, &dmastatus, sizeof(dmastatus)) == 0) {
        return(0);
      }
    }
    return (-EFAULT);
  case IOCTL_PCI17X4_DMA_START:
    if(adc->dma.running) {
      return(-EINVAL);
    }
    if(!(adc->dmabuffers && adc->dmadescs)) {
      return(-ENOMEM);
    }
    if((err = pci17x4_dmastart(adc)) != 0) {
      return(err);
    }
    return(0);
  case IOCTL_PCI17X4_DMA_STOP:
    if(!adc->dma.running) {
      return(-EINVAL);
    }
    pci17x4_dmastop(adc);
    return(0);
  default:
    printk(KERN_WARNING "%s: invalid ioctl() code (%u).\n", adc->name, cmd);
    return(-EINVAL);
  }
}

static int
pci17x4_mmap(struct file *fp, struct vm_area_struct *vma)
{
  struct pci17x4_private_data *adc = (struct pci17x4_private_data *)fp->private_data;
  size_t mapsize = (size_t)(vma->vm_end - vma->vm_start);
  size_t blksize = (size_t)(1) << adc->dma.setup.blockshift;
  if(((size_t)vma->vm_start % PAGE_SIZE) || (mapsize != (adc->dma.setup.numblocks * blksize)) || vma->vm_pgoff) {
    return(-EINVAL);
  }
  printk(KERN_INFO "%s: mapping start=0x%lx, size=%ld\n", adc->name, vma->vm_start, (long)mapsize);
  while(mapsize > 0) {
    unsigned long addr = vma->vm_start + (mapsize -= blksize);
    unsigned long pfn = virt_to_phys(adc->dmabuffers[mapsize >> adc->dma.setup.blockshift]) >> PAGE_SHIFT;
    int err = remap_pfn_range(vma, addr, pfn, blksize, vma->vm_page_prot);
    if(err < 0) {
      printk(KERN_WARNING "%s: remap block #%d failed, addr=0x%lx, pfn=0x%lx, size=%d.\n", adc->name, (int)mapsize, addr, pfn, (int)blksize);
      return(err);
    }
  }
  return(0);
}

static ssize_t
pci17x4_read(struct file *fp, char *buf, size_t count, loff_t *off)
{
  return (-EIO);
}

static ssize_t
pci17x4_write(struct file *fp, const char *buf, size_t count, loff_t *off)
{
  return (-EPERM);
}
