lkml.org 
[lkml]   [2006]   [Mar]   [15]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    /
    Date
    From
    SubjectRe: libata/sata_nv latency on NVIDIA CK804 [was Re: AMD64 X2 lost ticks on PM timer]

    * Jeff Garzik <jeff@garzik.org> wrote:

    > It won't work at all...

    ok.

    > You have to stop talking to PCI IDE registers completely (consumes 5
    > PCI BARs), and talk exclusively to the MMIO 6th PCI BAR, at
    > non-standard offsets and a using a proprietary DMA descriptor format
    > [all public now in that link I just sent].

    just to make it easier to test: i've attached the new sata_nv.c file,
    which, to test it, should be copied over the existing
    drivers/scsi/sata_nv.c file, correct?

    Ingo
    /*
    * sata_nv.c - NVIDIA nForce SATA
    *
    * Copyright 2004 NVIDIA Corp. All rights reserved.
    * Copyright 2004 Andrew Chew
    *
    *
    * This program is free software; you can redistribute it and/or modify
    * it under the terms of the GNU General Public License as published by
    * the Free Software Foundation; either version 2, or (at your option)
    * any later version.
    *
    * This program is distributed in the hope that it will be useful,
    * but WITHOUT ANY WARRANTY; without even the implied warranty of
    * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    * GNU General Public License for more details.
    *
    * You should have received a copy of the GNU General Public License
    * along with this program; see the file COPYING. If not, write to
    * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
    *
    *
    * libata documentation is available via 'make {ps|pdf}docs',
    * as Documentation/DocBook/libata.*
    *
    * No hardware documentation available outside of NVIDIA.
    * This driver programs the NVIDIA SATA controller in a similar
    * fashion as with other PCI IDE BMDMA controllers, with a few
    * NV-specific details such as register offsets, SATA phy location,
    * hotplug info, etc.
    *
    *
    * 0.08
    * - Added support for MCP51 and MCP55.
    *
    * 0.07
    * - Added support for RAID class code.
    *
    * 0.06
    * - Added generic SATA support by using a pci_device_id that filters on
    * the IDE storage class code.
    *
    * 0.03
    * - Fixed a bug where the hotplug handlers for non-CK804/MCP04 were using
    * mmio_base, which is only set for the CK804/MCP04 case.
    *
    * 0.02
    * - Added support for CK804 SATA controller.
    *
    * 0.01
    * - Initial revision.
    */

    #include <linux/config.h>
    #include <linux/kernel.h>
    #include <linux/module.h>
    #include <linux/pci.h>
    #include <linux/init.h>
    #include <linux/blkdev.h>
    #include <linux/delay.h>
    #include <linux/interrupt.h>
    #include "scsi.h"
    #include <scsi/scsi_host.h>
    #include <linux/libata.h>

    //#define DEBUG

    #define DRV_NAME "sata_nv"
    #define DRV_VERSION "0.8"

    #define NV_PORTS 2
    #define NV_PIO_MASK 0x1f
    #define NV_MWDMA_MASK 0x07
    #define NV_UDMA_MASK 0x7f
    #define NV_PORT0_SCR_REG_OFFSET 0x00
    #define NV_PORT1_SCR_REG_OFFSET 0x40

    #define NV_INT_STATUS 0x10
    #define NV_INT_STATUS_CK804 0x440
    #define NV_INT_STATUS_PDEV_INT 0x01
    #define NV_INT_STATUS_PDEV_PM 0x02
    #define NV_INT_STATUS_PDEV_ADDED 0x04
    #define NV_INT_STATUS_PDEV_REMOVED 0x08
    #define NV_INT_STATUS_SDEV_INT 0x10
    #define NV_INT_STATUS_SDEV_PM 0x20
    #define NV_INT_STATUS_SDEV_ADDED 0x40
    #define NV_INT_STATUS_SDEV_REMOVED 0x80
    #define NV_INT_STATUS_PDEV_HOTPLUG (NV_INT_STATUS_PDEV_ADDED | \
    NV_INT_STATUS_PDEV_REMOVED)
    #define NV_INT_STATUS_SDEV_HOTPLUG (NV_INT_STATUS_SDEV_ADDED | \
    NV_INT_STATUS_SDEV_REMOVED)
    #define NV_INT_STATUS_HOTPLUG (NV_INT_STATUS_PDEV_HOTPLUG | \
    NV_INT_STATUS_SDEV_HOTPLUG)

    #define NV_INT_ENABLE 0x11
    #define NV_INT_ENABLE_CK804 0x441
    #define NV_INT_ENABLE_PDEV_MASK 0x01
    #define NV_INT_ENABLE_PDEV_PM 0x02
    #define NV_INT_ENABLE_PDEV_ADDED 0x04
    #define NV_INT_ENABLE_PDEV_REMOVED 0x08
    #define NV_INT_ENABLE_SDEV_MASK 0x10
    #define NV_INT_ENABLE_SDEV_PM 0x20
    #define NV_INT_ENABLE_SDEV_ADDED 0x40
    #define NV_INT_ENABLE_SDEV_REMOVED 0x80
    #define NV_INT_ENABLE_PDEV_HOTPLUG (NV_INT_ENABLE_PDEV_ADDED | \
    NV_INT_ENABLE_PDEV_REMOVED)
    #define NV_INT_ENABLE_SDEV_HOTPLUG (NV_INT_ENABLE_SDEV_ADDED | \
    NV_INT_ENABLE_SDEV_REMOVED)
    #define NV_INT_ENABLE_HOTPLUG (NV_INT_ENABLE_PDEV_HOTPLUG | \
    NV_INT_ENABLE_SDEV_HOTPLUG)

    #define NV_INT_CONFIG 0x12
    #define NV_INT_CONFIG_METHD 0x01 // 0 = INT, 1 = SMI

    // For PCI config register 20
    #define NV_MCP_SATA_CFG_20 0x50
    #define NV_MCP_SATA_CFG_20_SATA_SPACE_EN 0x04
    #define NV_MCP_SATA_CFG_20_PORT0_EN (1 << 17)
    #define NV_MCP_SATA_CFG_20_PORT1_EN (1 << 16)
    #define NV_MCP_SATA_CFG_20_PORT0_PWB_EN (1 << 14)
    #define NV_MCP_SATA_CFG_20_PORT1_PWB_EN (1 << 12)

    //#define NV_ADMA_NCQ

    #ifdef NV_ADMA_NCQ
    #define NV_ADMA_CAN_QUEUE ATA_MAX_QUEUE
    #else
    #define NV_ADMA_CAN_QUEUE ATA_DEF_QUEUE
    #endif

    #define NV_ADMA_CPB_SZ 128
    #define NV_ADMA_APRD_SZ 16
    #define NV_ADMA_SGTBL_LEN (1024 - NV_ADMA_CPB_SZ) / NV_ADMA_APRD_SZ
    #define NV_ADMA_SGTBL_SZ NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ
    #define NV_ADMA_PORT_PRIV_DMA_SZ NV_ADMA_CAN_QUEUE * (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ)
    //#define NV_ADMA_MAX_CPBS 32

    // BAR5 offset to ADMA general registers
    #define NV_ADMA_GEN 0x400
    #define NV_ADMA_GEN_CTL 0x00
    #define NV_ADMA_NOTIFIER_CLEAR 0x30

    #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & ( 1 << (19 + (12 * (PORT)))))

    // BAR5 offset to ADMA ports
    #define NV_ADMA_PORT 0x480

    // size of ADMA port register space
    #define NV_ADMA_PORT_SIZE 0x100

    // ADMA port registers
    #define NV_ADMA_CTL 0x40
    #define NV_ADMA_CPB_COUNT 0x42
    #define NV_ADMA_NEXT_CPB_IDX 0x43
    #define NV_ADMA_STAT 0x44
    #define NV_ADMA_CPB_BASE_LOW 0x48
    #define NV_ADMA_CPB_BASE_HIGH 0x4C
    #define NV_ADMA_APPEND 0x50
    #define NV_ADMA_NOTIFIER 0x68
    #define NV_ADMA_NOTIFIER_ERROR 0x6C

    // NV_ADMA_CTL register bits
    #define NV_ADMA_CTL_HOTPLUG_IEN (1 << 0)
    #define NV_ADMA_CTL_CHANNEL_RESET (1 << 5)
    #define NV_ADMA_CTL_GO (1 << 7)
    #define NV_ADMA_CTL_AIEN (1 << 8)
    #define NV_ADMA_CTL_READ_NON_COHERENT (1 << 11)
    #define NV_ADMA_CTL_WRITE_NON_COHERENT (1 << 12)

    // CPB response flag bits
    #define NV_CPB_RESP_DONE (1 << 0)
    #define NV_CPB_RESP_ATA_ERR (1 << 3)
    #define NV_CPB_RESP_CMD_ERR (1 << 4)
    #define NV_CPB_RESP_CPB_ERR (1 << 7)

    // CPB control flag bits
    #define NV_CPB_CTL_CPB_VALID (1 << 0)
    #define NV_CPB_CTL_QUEUE (1 << 1)
    #define NV_CPB_CTL_APRD_VALID (1 << 2)
    #define NV_CPB_CTL_IEN (1 << 3)
    #define NV_CPB_CTL_FPDMA (1 << 4)

    // APRD flags
    #define NV_APRD_WRITE (1 << 1)
    #define NV_APRD_END (1 << 2)
    #define NV_APRD_CONT (1 << 3)

    // NV_ADMA_STAT flags
    #define NV_ADMA_STAT_TIMEOUT (1 << 0)
    #define NV_ADMA_STAT_HOTUNPLUG (1 << 1)
    #define NV_ADMA_STAT_HOTPLUG (1 << 2)
    #define NV_ADMA_STAT_CPBERR (1 << 4)
    #define NV_ADMA_STAT_SERROR (1 << 5)
    #define NV_ADMA_STAT_CMD_COMPLETE (1 << 6)
    #define NV_ADMA_STAT_IDLE (1 << 8)
    #define NV_ADMA_STAT_LEGACY (1 << 9)
    #define NV_ADMA_STAT_STOPPED (1 << 10)
    #define NV_ADMA_STAT_DONE (1 << 12)
    #define NV_ADMA_STAT_ERR (NV_ADMA_STAT_CPBERR | NV_ADMA_STAT_TIMEOUT)

    // port flags
    #define NV_ADMA_PORT_REGISTER_MODE (1 << 0)

    #ifndef min
    #define min(x,y) ((x) < (y) ? x : y)
    #endif

    struct nv_adma_prd {
    u64 addr;
    u32 len;
    u8 flags;
    u8 packet_len;
    u16 reserved;
    };

    enum nv_adma_regbits {
    CMDEND = (1 << 15), /* end of command list */
    WNB = (1 << 14), /* wait-not-BSY */
    IGN = (1 << 13), /* ignore this entry */
    CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */
    DA2 = (1 << (2 + 8)),
    DA1 = (1 << (1 + 8)),
    DA0 = (1 << (0 + 8)),
    };

    struct nv_adma_cpb {
    u8 resp_flags; //0
    u8 reserved1; //1
    u8 ctl_flags; //2
    // len is length of taskfile in 64 bit words
    u8 len; //3
    u8 tag; //4
    u8 next_cpb_idx; //5
    u16 reserved2; //6-7
    u16 tf[12]; //8-31
    struct nv_adma_prd aprd[5]; //32-111
    u64 next_aprd; //112-119
    u64 reserved3; //120-127
    };


    struct nv_adma_port_priv {
    struct nv_adma_cpb *cpb;
    // u8 cpb_idx;
    u8 flags;
    u32 notifier;
    u32 notifier_error;
    dma_addr_t cpb_dma;
    struct nv_adma_prd *aprd;
    dma_addr_t aprd_dma;
    };

    static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
    static irqreturn_t nv_interrupt (int irq, void *dev_instance,
    struct pt_regs *regs);
    static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg);
    static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val);
    static void nv_host_stop (struct ata_host_set *host_set);
    static int nv_port_start(struct ata_port *ap);
    static void nv_port_stop(struct ata_port *ap);
    static int nv_adma_port_start(struct ata_port *ap);
    static void nv_adma_port_stop(struct ata_port *ap);
    static void nv_irq_clear(struct ata_port *ap);
    static void nv_adma_irq_clear(struct ata_port *ap);
    static void nv_enable_hotplug(struct ata_probe_ent *probe_ent);
    static void nv_disable_hotplug(struct ata_host_set *host_set);
    static void nv_check_hotplug(struct ata_host_set *host_set);
    static void nv_enable_hotplug_ck804(struct ata_probe_ent *probe_ent);
    static void nv_disable_hotplug_ck804(struct ata_host_set *host_set);
    static void nv_check_hotplug_ck804(struct ata_host_set *host_set);
    static void nv_enable_hotplug_adma(struct ata_probe_ent *probe_ent);
    static void nv_disable_hotplug_adma(struct ata_host_set *host_set);
    static void nv_check_hotplug_adma(struct ata_host_set *host_set);
    static void nv_qc_prep(struct ata_queued_cmd *qc);
    static int nv_qc_issue(struct ata_queued_cmd *qc);
    static int nv_adma_qc_issue(struct ata_queued_cmd *qc);
    static void nv_adma_qc_prep(struct ata_queued_cmd *qc);
    static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, u16 *cpb);
    static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb);
    static void nv_adma_fill_aprd(struct ata_queued_cmd *qc, int idx, struct nv_adma_prd *aprd);
    static void nv_adma_register_mode(struct ata_port *ap);
    static void nv_adma_mode(struct ata_port *ap);
    static u8 nv_bmdma_status(struct ata_port *ap);
    static u8 nv_adma_bmdma_status(struct ata_port *ap);
    static void nv_bmdma_stop(struct ata_queued_cmd *qc);
    static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc);
    static void nv_eng_timeout(struct ata_port *ap);
    static void nv_adma_eng_timeout(struct ata_port *ap);
    #ifdef DEBUG
    static void nv_adma_dump_cpb(struct nv_adma_cpb *cpb);
    static void nv_adma_dump_aprd(struct nv_adma_prd *aprd);
    static void nv_adma_dump_cpb_tf(u16 tf);
    static void nv_adma_dump_port(struct ata_port *ap);
    static void nv_adma_dump_iomem(void __iomem *m, int len);
    #endif

    enum nv_host_type
    {
    GENERIC,
    NFORCE2,
    NFORCE3,
    CK804,
    MCP51,
    MCP55,
    ADMA
    };

    static struct pci_device_id nv_pci_tbl[] = {
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, NFORCE2 },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, NFORCE3 },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, NFORCE3 },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, ADMA },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, ADMA },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, ADMA },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, ADMA },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, MCP51 },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, MCP51 },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, MCP55 },
    { PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2,
    PCI_ANY_ID, PCI_ANY_ID, 0, 0, MCP55 },
    { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
    PCI_ANY_ID, PCI_ANY_ID,
    PCI_CLASS_STORAGE_IDE<<8, 0xffff00, GENERIC },
    { PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
    PCI_ANY_ID, PCI_ANY_ID,
    PCI_CLASS_STORAGE_RAID<<8, 0xffff00, GENERIC },
    { 0, } /* terminate list */
    };

    #define NV_HOST_FLAGS_SCR_MMIO 0x00000001

    struct nv_host_desc
    {
    enum nv_host_type host_type;
    void (*enable_hotplug)(struct ata_probe_ent *probe_ent);
    void (*disable_hotplug)(struct ata_host_set *host_set);
    void (*check_hotplug)(struct ata_host_set *host_set);

    };
    static struct nv_host_desc nv_device_tbl[] = {
    {
    .host_type = GENERIC,
    .enable_hotplug = NULL,
    .disable_hotplug= NULL,
    .check_hotplug = NULL,
    },
    {
    .host_type = NFORCE2,
    .enable_hotplug = nv_enable_hotplug,
    .disable_hotplug= nv_disable_hotplug,
    .check_hotplug = nv_check_hotplug,
    },
    {
    .host_type = NFORCE3,
    .enable_hotplug = nv_enable_hotplug,
    .disable_hotplug= nv_disable_hotplug,
    .check_hotplug = nv_check_hotplug,
    },
    { .host_type = CK804,
    .enable_hotplug = nv_enable_hotplug_ck804,
    .disable_hotplug= nv_disable_hotplug_ck804,
    .check_hotplug = nv_check_hotplug_ck804,
    },
    { .host_type = MCP51,
    .enable_hotplug = nv_enable_hotplug,
    .disable_hotplug= nv_disable_hotplug,
    .check_hotplug = nv_check_hotplug,
    },
    { .host_type = MCP55,
    .enable_hotplug = nv_enable_hotplug,
    .disable_hotplug= nv_disable_hotplug,
    .check_hotplug = nv_check_hotplug,
    },
    { .host_type = ADMA,
    .enable_hotplug = nv_enable_hotplug_adma,
    .disable_hotplug= nv_disable_hotplug_adma,
    .check_hotplug = nv_check_hotplug_adma,
    },
    };

    struct nv_host
    {
    struct nv_host_desc *host_desc;
    unsigned long host_flags;
    };

    static struct pci_driver nv_pci_driver = {
    .name = DRV_NAME,
    .id_table = nv_pci_tbl,
    .probe = nv_init_one,
    .remove = ata_pci_remove_one,
    };

    static struct scsi_host_template nv_sht = {
    .module = THIS_MODULE,
    .name = DRV_NAME,
    .ioctl = ata_scsi_ioctl,
    .queuecommand = ata_scsi_queuecmd,
    .eh_strategy_handler = ata_scsi_error,
    .can_queue = ATA_DEF_QUEUE,
    .this_id = ATA_SHT_THIS_ID,
    .sg_tablesize = LIBATA_MAX_PRD,
    .max_sectors = ATA_MAX_SECTORS,
    .cmd_per_lun = ATA_SHT_CMD_PER_LUN,
    .emulated = ATA_SHT_EMULATED,
    .use_clustering = ATA_SHT_USE_CLUSTERING,
    .proc_name = DRV_NAME,
    .dma_boundary = ATA_DMA_BOUNDARY,
    .slave_configure = ata_scsi_slave_config,
    .bios_param = ata_std_bios_param,
    .ordered_flush = 1,
    };

    static struct ata_port_operations nv_ops = {
    .port_disable = ata_port_disable,
    .tf_load = ata_tf_load,
    .tf_read = ata_tf_read,
    .exec_command = ata_exec_command,
    .check_status = ata_check_status,
    .dev_select = ata_std_dev_select,
    .phy_reset = sata_phy_reset,
    .bmdma_setup = ata_bmdma_setup,
    .bmdma_start = ata_bmdma_start,
    .bmdma_stop = nv_bmdma_stop,
    .bmdma_status = nv_bmdma_status,
    .qc_prep = nv_qc_prep,
    .qc_issue = nv_qc_issue,
    .eng_timeout = nv_eng_timeout,
    .irq_handler = nv_interrupt,
    .irq_clear = nv_irq_clear,
    .scr_read = nv_scr_read,
    .scr_write = nv_scr_write,
    .port_start = nv_port_start,
    .port_stop = nv_port_stop,
    .host_stop = nv_host_stop,
    };

    static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, u16 *cpb)
    {
    unsigned int idx = 0;

    cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device | WNB);

    if ((tf->flags & ATA_TFLAG_LBA48) == 0) {
    cpb[idx++] = cpu_to_le16(IGN);
    cpb[idx++] = cpu_to_le16(IGN);
    cpb[idx++] = cpu_to_le16(IGN);
    cpb[idx++] = cpu_to_le16(IGN);
    cpb[idx++] = cpu_to_le16(IGN);
    }
    else {
    cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature);
    cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect);
    cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal);
    cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam);
    cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah);
    }
    cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature);
    cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect);
    cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal);
    cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam);
    cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah);

    cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND);

    return idx;
    }

    static inline void __iomem *__nv_adma_ctl_block(void __iomem *mmio,
    unsigned int port_no)
    {
    mmio += NV_ADMA_PORT + port_no * NV_ADMA_PORT_SIZE;
    return mmio;
    }

    static inline void __iomem *nv_adma_ctl_block(struct ata_port *ap)
    {
    return __nv_adma_ctl_block(ap->host_set->mmio_base, ap->port_no);
    }

    static inline void __iomem *nv_adma_gen_block(struct ata_port *ap)
    {
    return (ap->host_set->mmio_base + NV_ADMA_GEN);
    }

    static inline void __iomem *nv_adma_notifier_clear_block(struct ata_port *ap)
    {
    return (nv_adma_gen_block(ap) + NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no));
    }

    static inline void nv_adma_reset_channel(struct ata_port *ap)
    {
    void __iomem *mmio = nv_adma_ctl_block(ap);
    u16 tmp;

    // clear CPB fetch count
    writew(0, mmio + NV_ADMA_CPB_COUNT);

    // clear GO
    tmp = readw(mmio + NV_ADMA_CTL);
    writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);

    tmp = readw(mmio + NV_ADMA_CTL);
    writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
    udelay(1);
    writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL);
    }

    static inline int nv_adma_host_intr(struct ata_port *ap, struct ata_queued_cmd *qc)
    {
    void __iomem *mmio = nv_adma_ctl_block(ap);
    struct nv_adma_port_priv *pp = ap->private_data;
    struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];
    u16 status;
    u32 gen_ctl;
    u16 flags;
    int have_err = 0;
    int handled = 0;

    status = readw(mmio + NV_ADMA_STAT);

    // if in ATA register mode, use standard ata interrupt handler
    if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
    VPRINTK("in ATA register mode\n");
    return ata_host_intr(ap, qc);
    }

    gen_ctl = readl(nv_adma_gen_block(ap) + NV_ADMA_GEN_CTL);
    if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no)) {
    return 0;
    }

    if (!pp->notifier && !pp->notifier_error) {
    if (status) {
    VPRINTK("XXX no notifier, but status 0x%x\n", status);
    #ifdef DEBUG
    nv_adma_dump_port(ap);
    nv_adma_dump_cpb(cpb);
    #endif
    } else {
    return 0;
    }
    }
    if (pp->notifier_error) {
    have_err = 1;
    handled = 1;
    }

    if (status & NV_ADMA_STAT_TIMEOUT) {
    VPRINTK("timeout, stat = 0x%x\n", status);
    have_err = 1;
    handled = 1;
    }
    if (status & NV_ADMA_STAT_CPBERR) {
    VPRINTK("CPB error, stat = 0x%x\n", status);
    have_err = 1;
    handled = 1;
    }
    if (status & NV_ADMA_STAT_STOPPED) {
    VPRINTK("ADMA stopped, stat = 0x%x, resp_flags = 0x%x\n", status, cpb->resp_flags);
    if (!(status & NV_ADMA_STAT_DONE)) {
    have_err = 1;
    handled = 1;
    }
    }
    if (status & NV_ADMA_STAT_CMD_COMPLETE) {
    VPRINTK("ADMA command complete, stat = 0x%x\n", status);
    }
    if (status & NV_ADMA_STAT_DONE) {
    flags = cpb->resp_flags;
    VPRINTK("CPB done, stat = 0x%x, flags = 0x%x\n", status, flags);
    handled = 1;
    if (!(status & NV_ADMA_STAT_IDLE)) {
    VPRINTK("XXX CPB done, but not idle\n");
    }
    if (flags & NV_CPB_RESP_DONE) {
    VPRINTK("CPB flags done, flags = 0x%x\n", flags);
    }
    if (flags & NV_CPB_RESP_ATA_ERR) {
    VPRINTK("CPB flags ATA err, flags = 0x%x\n", flags);
    have_err = 1;
    }
    if (flags & NV_CPB_RESP_CMD_ERR) {
    VPRINTK("CPB flags CMD err, flags = 0x%x\n", flags);
    have_err = 1;
    }
    if (flags & NV_CPB_RESP_CPB_ERR) {
    VPRINTK("CPB flags CPB err, flags = 0x%x\n", flags);
    have_err = 1;
    }
    }

    // clear status
    writew(status, mmio + NV_ADMA_STAT);

    if (handled) {
    u8 ata_status = readb(mmio + (ATA_REG_STATUS * 4));
    ata_qc_complete(qc, have_err ? (ata_status | ATA_ERR) : ata_status);
    }

    return handled; /* irq handled */
    }

    /* FIXME: The hardware provides the necessary SATA PHY controls
    * to support ATA_FLAG_SATA_RESET. However, it is currently
    * necessary to disable that flag, to solve misdetection problems.
    * See http://bugme.osdl.org/show_bug.cgi?id=3352 for more info.
    *
    * This problem really needs to be investigated further. But in the
    * meantime, we avoid ATA_FLAG_SATA_RESET to get people working.
    */

    static struct ata_port_info nv_port_info = {
    .sht = &nv_sht,
    .host_flags = ATA_FLAG_SATA |
    /* ATA_FLAG_SATA_RESET | */
    ATA_FLAG_SRST |
    ATA_FLAG_NO_LEGACY,
    .pio_mask = NV_PIO_MASK,
    .mwdma_mask = NV_MWDMA_MASK,
    .udma_mask = NV_UDMA_MASK,
    .port_ops = &nv_ops,
    };

    MODULE_AUTHOR("NVIDIA");
    MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller");
    MODULE_LICENSE("GPL");
    MODULE_DEVICE_TABLE(pci, nv_pci_tbl);
    MODULE_VERSION(DRV_VERSION);

    static inline void nv_enable_adma_space (struct pci_dev *pdev)
    {
    u8 regval;

    VPRINTK("ENTER\n");

    pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
    regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
    pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
    }

    static inline void nv_disable_adma_space (struct pci_dev *pdev)
    {
    u8 regval;

    VPRINTK("ENTER\n");

    pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, &regval);
    regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN;
    pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval);
    }

    static void nv_irq_clear(struct ata_port *ap)
    {
    struct ata_host_set *host_set = ap->host_set;
    struct nv_host *host = host_set->private_data;

    if (host->host_desc->host_type == ADMA) {
    nv_adma_irq_clear(ap);
    } else {
    ata_bmdma_irq_clear(ap);
    }
    }

    static void nv_adma_irq_clear(struct ata_port *ap)
    {
    /* TODO */
    }

    static u8 nv_bmdma_status(struct ata_port *ap)
    {
    struct ata_host_set *host_set = ap->host_set;
    struct nv_host *host = host_set->private_data;

    if (host->host_desc->host_type == ADMA) {
    return nv_adma_bmdma_status(ap);
    } else {
    return ata_bmdma_status(ap);
    }
    }

    static u8 nv_adma_bmdma_status(struct ata_port *ap)
    {
    return inb(ap->ioaddr.bmdma_addr + ATA_DMA_STATUS);
    }

    static void nv_bmdma_stop(struct ata_queued_cmd *qc)
    {
    struct ata_host_set *host_set = qc->ap->host_set;
    struct nv_host *host = host_set->private_data;

    if (host->host_desc->host_type == ADMA) {
    nv_adma_bmdma_stop(qc);
    } else {
    ata_bmdma_stop(qc);
    }
    }

    static void nv_adma_bmdma_stop(struct ata_queued_cmd *qc)
    {
    /* TODO */
    }

    static irqreturn_t nv_interrupt (int irq, void *dev_instance,
    struct pt_regs *regs)
    {
    struct ata_host_set *host_set = dev_instance;
    struct nv_host *host = host_set->private_data;
    unsigned int i;
    unsigned int handled = 0;
    unsigned long flags;

    spin_lock_irqsave(&host_set->lock, flags);

    for (i = 0; i < host_set->n_ports; i++) {
    struct ata_port *ap = host_set->ports[i];
    struct nv_adma_port_priv *pp = ap->private_data;

    if (ap &&
    !(ap->flags & (ATA_FLAG_PORT_DISABLED | ATA_FLAG_NOINTR))) {
    void __iomem *mmio = nv_adma_ctl_block(ap);
    struct ata_queued_cmd *qc;

    // read notifiers
    pp->notifier = readl(mmio + NV_ADMA_NOTIFIER);
    pp->notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR);

    qc = ata_qc_from_tag(ap, ap->active_tag);
    if (qc && (!(qc->tf.ctl & ATA_NIEN))) {
    if (host->host_desc->host_type == ADMA) {
    handled += nv_adma_host_intr(ap, qc);
    } else {
    handled += ata_host_intr(ap, qc);
    }
    }

    }

    }

    if (host->host_desc->check_hotplug)
    host->host_desc->check_hotplug(host_set);

    // clear notifier
    if (handled) {
    for (i = 0; i < host_set->n_ports; i++) {
    struct ata_port *ap = host_set->ports[i];
    struct nv_adma_port_priv *pp = ap->private_data;
    writel(pp->notifier | pp->notifier_error,
    nv_adma_notifier_clear_block(ap));
    }
    }

    spin_unlock_irqrestore(&host_set->lock, flags);

    return IRQ_RETVAL(handled);
    }

    static u32 nv_scr_read (struct ata_port *ap, unsigned int sc_reg)
    {
    struct ata_host_set *host_set = ap->host_set;
    struct nv_host *host = host_set->private_data;
    u32 val = 0;

    VPRINTK("ENTER\n");

    VPRINTK("reading SCR reg %d, got 0x%08x\n", sc_reg, val);

    if (sc_reg > SCR_CONTROL)
    return 0xffffffffU;

    if (host->host_flags & NV_HOST_FLAGS_SCR_MMIO)
    val = readl((void*)ap->ioaddr.scr_addr + (sc_reg * 4));
    else
    val = inl(ap->ioaddr.scr_addr + (sc_reg * 4));

    VPRINTK("reading SCR reg %d, got 0x%08x\n", sc_reg, val);
    return val;
    }

    static void nv_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val)
    {
    struct ata_host_set *host_set = ap->host_set;
    struct nv_host *host = host_set->private_data;

    VPRINTK("ENTER\n");

    VPRINTK("writing SCR reg %d with 0x%08x\n", sc_reg, val);
    if (sc_reg > SCR_CONTROL)
    return;

    if (host->host_flags & NV_HOST_FLAGS_SCR_MMIO)
    writel(val, (void*)ap->ioaddr.scr_addr + (sc_reg * 4));
    else
    outl(val, ap->ioaddr.scr_addr + (sc_reg * 4));
    }

    static void nv_host_stop (struct ata_host_set *host_set)
    {
    struct nv_host *host = host_set->private_data;
    struct pci_dev *pdev = to_pci_dev(host_set->dev);

    VPRINTK("ENTER\n");

    // Disable hotplug event interrupts.
    if (host->host_desc->disable_hotplug)
    host->host_desc->disable_hotplug(host_set);

    kfree(host);

    if (host_set->mmio_base)
    pci_iounmap(pdev, host_set->mmio_base);
    }

    static int nv_port_start(struct ata_port *ap)
    {
    struct ata_host_set *host_set = ap->host_set;
    struct nv_host *host = host_set->private_data;

    if (host->host_desc->host_type == ADMA) {
    return nv_adma_port_start(ap);
    } else {
    return ata_port_start(ap);
    }
    }

    static void nv_port_stop(struct ata_port *ap)
    {
    struct ata_host_set *host_set = ap->host_set;
    struct nv_host *host = host_set->private_data;

    if (host->host_desc->host_type == ADMA) {
    nv_adma_port_stop(ap);
    } else {
    ata_port_stop(ap);
    }
    }

    static int nv_adma_port_start(struct ata_port *ap)
    {
    struct device *dev = ap->host_set->dev;
    struct nv_adma_port_priv *pp;
    int rc;
    void *mem;
    dma_addr_t mem_dma;
    void __iomem *mmio = nv_adma_ctl_block(ap);

    VPRINTK("ENTER\n");

    nv_adma_reset_channel(ap);

    #ifdef DEBUG
    VPRINTK("after reset:\n");
    nv_adma_dump_port(ap);
    #endif

    rc = ata_port_start(ap);
    if (rc)
    return rc;

    pp = kmalloc(sizeof(*pp), GFP_KERNEL);
    if (!pp) {
    rc = -ENOMEM;
    goto err_out;
    }
    memset(pp, 0, sizeof(*pp));

    mem = dma_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ,
    &mem_dma, GFP_KERNEL);

    VPRINTK("dma memory: vaddr = 0x%08x, paddr = 0x%08x\n", (u32)mem, (u32)mem_dma);

    if (!mem) {
    rc = -ENOMEM;
    goto err_out_kfree;
    }
    memset(mem, 0, NV_ADMA_PORT_PRIV_DMA_SZ);

    /*
    * First item in chunk of DMA memory:
    * 128-byte command parameter block (CPB)
    * one for each command tag
    */
    pp->cpb = mem;
    pp->cpb_dma = mem_dma;

    VPRINTK("cpb = 0x%08x, cpb_dma = 0x%08x\n", (u32)pp->cpb, (u32)pp->cpb_dma);

    writel(mem_dma, mmio + NV_ADMA_CPB_BASE_LOW);
    writel(0, mmio + NV_ADMA_CPB_BASE_HIGH);

    mem += NV_ADMA_CAN_QUEUE * NV_ADMA_CPB_SZ;
    mem_dma += NV_ADMA_CAN_QUEUE * NV_ADMA_CPB_SZ;

    /*
    * Second item: block of ADMA_SGTBL_LEN s/g entries
    */
    pp->aprd = mem;
    pp->aprd_dma = mem_dma;

    VPRINTK("aprd = 0x%08x, aprd_dma = 0x%08x\n", (u32)pp->aprd, (u32)pp->aprd_dma);

    ap->private_data = pp;

    // clear any outstanding interrupt conditions
    writew(0xffff, mmio + NV_ADMA_STAT);

    // initialize port variables
    // pp->cpb_idx = 0;
    pp->flags = NV_ADMA_PORT_REGISTER_MODE;

    // make sure controller is in ATA register mode
    nv_adma_register_mode(ap);

    return 0;

    err_out_kfree:
    kfree(pp);
    err_out:
    ata_port_stop(ap);
    return rc;
    }

    static void nv_adma_port_stop(struct ata_port *ap)
    {
    struct device *dev = ap->host_set->dev;
    struct nv_adma_port_priv *pp = ap->private_data;
    void __iomem *mmio = nv_adma_ctl_block(ap);

    VPRINTK("ENTER\n");

    writew(0, mmio + NV_ADMA_CTL);

    ap->private_data = NULL;
    dma_free_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ, pp->cpb, pp->cpb_dma);
    kfree(pp);
    ata_port_stop(ap);
    }


    static void nv_adma_setup_port(struct ata_probe_ent *probe_ent, unsigned int port)
    {
    void __iomem *mmio = probe_ent->mmio_base;
    struct ata_ioports *ioport = &probe_ent->port[port];

    VPRINTK("ENTER\n");

    mmio += NV_ADMA_PORT + port * NV_ADMA_PORT_SIZE;

    ioport->cmd_addr = (unsigned long) mmio;
    ioport->data_addr = (unsigned long) mmio + (ATA_REG_DATA * 4);
    ioport->error_addr =
    ioport->feature_addr = (unsigned long) mmio + (ATA_REG_ERR * 4);
    ioport->nsect_addr = (unsigned long) mmio + (ATA_REG_NSECT * 4);
    ioport->lbal_addr = (unsigned long) mmio + (ATA_REG_LBAL * 4);
    ioport->lbam_addr = (unsigned long) mmio + (ATA_REG_LBAM * 4);
    ioport->lbah_addr = (unsigned long) mmio + (ATA_REG_LBAH * 4);
    ioport->device_addr = (unsigned long) mmio + (ATA_REG_DEVICE * 4);
    ioport->status_addr =
    ioport->command_addr = (unsigned long) mmio + (ATA_REG_STATUS * 4);
    ioport->altstatus_addr =
    ioport->ctl_addr = (unsigned long) mmio + 0x20;
    }

    static int nv_adma_host_init(struct ata_probe_ent *probe_ent)
    {
    struct pci_dev *pdev = to_pci_dev(probe_ent->dev);
    unsigned int i;
    u32 tmp32;

    VPRINTK("ENTER\n");

    probe_ent->n_ports = NV_PORTS;

    nv_enable_adma_space(pdev);

    // enable ADMA on the ports
    pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32);
    tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN |
    NV_MCP_SATA_CFG_20_PORT0_PWB_EN |
    NV_MCP_SATA_CFG_20_PORT1_EN |
    NV_MCP_SATA_CFG_20_PORT1_PWB_EN;

    pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32);

    for (i = 0; i < probe_ent->n_ports; i++)
    nv_adma_setup_port(probe_ent, i);

    for (i = 0; i < probe_ent->n_ports; i++) {
    void __iomem *mmio = __nv_adma_ctl_block(probe_ent->mmio_base, i);
    u16 tmp;

    /* enable interrupt, clear reset if not already clear */
    tmp = readw(mmio + NV_ADMA_CTL);
    writew(tmp | NV_ADMA_CTL_AIEN, mmio + NV_ADMA_CTL);
    }

    pci_set_master(pdev);

    return 0;
    }

    static int nv_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
    {
    static int printed_version = 0;
    struct nv_host *host;
    struct ata_port_info *ppi;
    struct ata_probe_ent *probe_ent;
    struct nv_host_desc *host_desc;
    int pci_dev_busy = 0;
    int rc;
    u32 bar;

    VPRINTK("ENTER\n");

    // Make sure this is a SATA controller by counting the number of bars
    // (NVIDIA SATA controllers will always have six bars). Otherwise,
    // it's an IDE controller and we ignore it.
    for (bar=0; bar<6; bar++)
    if (pci_resource_start(pdev, bar) == 0)
    return -ENODEV;

    if (!printed_version++)
    printk(KERN_DEBUG DRV_NAME " version " DRV_VERSION "\n");

    rc = pci_enable_device(pdev);
    if (rc)
    goto err_out;

    rc = pci_request_regions(pdev, DRV_NAME);
    if (rc) {
    pci_dev_busy = 1;
    goto err_out_disable;
    }

    rc = pci_set_dma_mask(pdev, ATA_DMA_MASK);
    if (rc)
    goto err_out_regions;
    rc = pci_set_consistent_dma_mask(pdev, ATA_DMA_MASK);
    if (rc)
    goto err_out_regions;

    rc = -ENOMEM;

    ppi = &nv_port_info;

    host_desc = &nv_device_tbl[ent->driver_data];
    if (host_desc->host_type == ADMA) {
    // ADMA overrides
    ppi->host_flags |= ATA_FLAG_MMIO | ATA_FLAG_SATA_RESET;
    #ifdef NV_ADMA_NCQ
    ppi->host_flags |= ATA_FLAG_NCQ;
    #endif
    ppi->sht->can_queue = NV_ADMA_CAN_QUEUE;
    ppi->sht->sg_tablesize = NV_ADMA_SGTBL_LEN;
    // ppi->port_ops->irq_handler = nv_adma_interrupt;
    }

    probe_ent = ata_pci_init_native_mode(pdev, &ppi);
    if (!probe_ent)
    goto err_out_regions;

    host = kmalloc(sizeof(struct nv_host), GFP_KERNEL);
    if (!host)
    goto err_out_free_ent;

    memset(host, 0, sizeof(struct nv_host));
    host->host_desc = host_desc;

    probe_ent->private_data = host;

    if (pci_resource_flags(pdev, 5) & IORESOURCE_MEM)
    host->host_flags |= NV_HOST_FLAGS_SCR_MMIO;

    if (host->host_flags & NV_HOST_FLAGS_SCR_MMIO) {
    unsigned long base;

    probe_ent->mmio_base = pci_iomap(pdev, 5, 0);
    if (probe_ent->mmio_base == NULL) {
    rc = -EIO;
    goto err_out_free_host;
    }

    base = (unsigned long)probe_ent->mmio_base;
    VPRINTK("BAR5 base is at 0x%x\n", (u32)base);

    probe_ent->port[0].scr_addr =
    base + NV_PORT0_SCR_REG_OFFSET;
    probe_ent->port[1].scr_addr =
    base + NV_PORT1_SCR_REG_OFFSET;
    } else {

    probe_ent->port[0].scr_addr =
    pci_resource_start(pdev, 5) | NV_PORT0_SCR_REG_OFFSET;
    probe_ent->port[1].scr_addr =
    pci_resource_start(pdev, 5) | NV_PORT1_SCR_REG_OFFSET;
    }

    pci_set_master(pdev);

    if (ent->driver_data == ADMA) {
    rc = nv_adma_host_init(probe_ent);
    if (rc)
    goto err_out_iounmap;
    }

    rc = ata_device_add(probe_ent);
    if (rc != NV_PORTS)
    goto err_out_iounmap;

    // Enable hotplug event interrupts.
    if (host->host_desc->enable_hotplug)
    host->host_desc->enable_hotplug(probe_ent);

    kfree(probe_ent);

    return 0;

    err_out_iounmap:
    if (host->host_flags & NV_HOST_FLAGS_SCR_MMIO)
    pci_iounmap(pdev, probe_ent->mmio_base);
    err_out_free_host:
    kfree(host);
    err_out_free_ent:
    kfree(probe_ent);
    err_out_regions:
    pci_release_regions(pdev);
    err_out_disable:
    if (!pci_dev_busy)
    pci_disable_device(pdev);
    err_out:
    return rc;
    }

    static void nv_eng_timeout(struct ata_port *ap)
    {
    struct ata_host_set *host_set = ap->host_set;
    struct nv_host *host = host_set->private_data;

    if (host->host_desc->host_type == ADMA) {
    nv_adma_eng_timeout(ap);
    } else {
    return ata_eng_timeout(ap);
    }
    }

    static void nv_adma_eng_timeout(struct ata_port *ap)
    {
    struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->active_tag);
    struct nv_adma_port_priv *pp = ap->private_data;
    u8 drv_stat;

    VPRINTK("ENTER\n");

    if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) {
    ata_eng_timeout(ap);
    goto out;
    }


    if (!qc) {
    printk(KERN_ERR "ata%u: BUG: timeout without command\n",
    ap->id);
    goto out;
    }


    // spin_lock_irqsave(&host_set->lock, flags);

    qc->scsidone = scsi_finish_command;

    drv_stat = ata_chk_status(ap);

    printk(KERN_ERR "ata%u: command 0x%x timeout, stat 0x%x\n",
    ap->id, qc->tf.command, drv_stat);

    // reset channel
    nv_adma_reset_channel(ap);

    /* complete taskfile transaction */
    ata_qc_complete(qc, drv_stat);

    // spin_unlock_irqrestore(&host_set->lock, flags);

    out:
    DPRINTK("EXIT\n");
    }

    static void nv_qc_prep(struct ata_queued_cmd *qc)
    {
    struct ata_host_set *host_set = qc->ap->host_set;
    struct nv_host *host = host_set->private_data;

    if (host->host_desc->host_type == ADMA) {
    nv_adma_qc_prep(qc);
    } else {
    ata_qc_prep(qc);
    }
    }

    static void nv_adma_qc_prep(struct ata_queued_cmd *qc)
    {
    struct nv_adma_port_priv *pp = qc->ap->private_data;
    struct nv_adma_cpb *cpb = &pp->cpb[qc->tag];

    VPRINTK("ENTER\n");

    VPRINTK("qc->flags = 0x%x\n", (u32)qc->flags);

    if (!(qc->flags & ATA_QCFLAG_DMAMAP)) {
    ata_qc_prep(qc);
    return;
    }

    memset(cpb, 0, sizeof(struct nv_adma_cpb));

    cpb->ctl_flags = NV_CPB_CTL_CPB_VALID |
    NV_CPB_CTL_APRD_VALID |
    NV_CPB_CTL_IEN;
    cpb->len = 3;
    cpb->tag = qc->tag;
    cpb->next_cpb_idx = 0;

    #ifdef NV_ADMA_NCQ
    // turn on NCQ flags for NCQ commands
    if (qc->flags & ATA_QCFLAG_NCQ)
    cpb->ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA;
    #endif

    nv_adma_tf_to_cpb(&qc->tf, cpb->tf);

    nv_adma_fill_sg(qc, cpb);
    }

    static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)
    {
    struct nv_adma_port_priv *pp = qc->ap->private_data;
    unsigned int idx;
    struct nv_adma_prd *aprd;

    VPRINTK("ENTER\n");

    idx = 0;

    for (idx = 0; idx < qc->n_elem; idx++) {
    if (idx < 5) {
    aprd = &cpb->aprd[idx];
    } else {
    aprd = &pp->aprd[idx-5];
    }
    nv_adma_fill_aprd(qc, idx, aprd);
    }
    if (idx > 5) {
    cpb->next_aprd = (u64)(pp->aprd_dma + NV_ADMA_APRD_SZ * qc->tag);
    }
    }

    static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,
    int idx,
    struct nv_adma_prd *aprd)
    {
    u32 sg_len, addr, flags;

    memset(aprd, 0, sizeof(struct nv_adma_prd));

    addr = sg_dma_address(&qc->sg[idx]);
    sg_len = sg_dma_len(&qc->sg[idx]);

    flags = 0;
    if (qc->tf.flags & ATA_TFLAG_WRITE)
    flags |= NV_APRD_WRITE;
    if (idx == qc->n_elem - 1) {
    flags |= NV_APRD_END;
    } else if (idx != 4) {
    flags |= NV_APRD_CONT;
    }

    aprd->addr = cpu_to_le32(addr);
    aprd->len = cpu_to_le32(sg_len); /* len in bytes */
    aprd->flags = cpu_to_le32(flags);
    }

    static void nv_adma_register_mode(struct ata_port *ap)
    {
    void __iomem *mmio = nv_adma_ctl_block(ap);
    struct nv_adma_port_priv *pp = ap->private_data;
    u16 tmp;

    tmp = readw(mmio + NV_ADMA_CTL);
    writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);

    pp->flags |= NV_ADMA_PORT_REGISTER_MODE;
    }

    static void nv_adma_mode(struct ata_port *ap)
    {
    void __iomem *mmio = nv_adma_ctl_block(ap);
    struct nv_adma_port_priv *pp = ap->private_data;
    u16 tmp;

    if(!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) {
    return;
    }

    #if 0
    nv_adma_reset_channel(ap);
    #endif

    tmp = readw(mmio + NV_ADMA_CTL);
    writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL);

    pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE;
    }

    static int nv_qc_issue(struct ata_queued_cmd *qc)
    {
    struct ata_host_set *host_set = qc->ap->host_set;
    struct nv_host *host = host_set->private_data;

    if (host->host_desc->host_type == ADMA) {
    return nv_adma_qc_issue(qc);
    } else {
    return ata_qc_issue_prot(qc);
    }
    }

    static int nv_adma_qc_issue(struct ata_queued_cmd *qc)
    {
    #if 0
    struct nv_adma_port_priv *pp = qc->ap->private_data;
    #endif
    void __iomem *mmio = nv_adma_ctl_block(qc->ap);

    VPRINTK("ENTER\n");

    if (!(qc->flags & ATA_QCFLAG_DMAMAP)) {
    VPRINTK("no dmamap, using ATA register mode: 0x%x\n", (u32)qc->flags);
    // use ATA register mode
    nv_adma_register_mode(qc->ap);
    return ata_qc_issue_prot(qc);
    } else {
    nv_adma_mode(qc->ap);
    }

    #if 0
    nv_adma_dump_port(qc->ap);
    nv_adma_dump_cpb(&pp->cpb[qc->tag]);
    if (qc->n_elem > 5) {
    int i;
    for (i = 0; i < qc->n_elem - 5; i++) {
    nv_adma_dump_aprd(&pp->aprd[i]);
    }
    }
    #endif

    //
    // write append register, command tag in lower 8 bits
    // and (number of cpbs to append -1) in top 8 bits
    //
    mb();
    writew(qc->tag, mmio + NV_ADMA_APPEND);

    VPRINTK("EXIT\n");

    return 0;
    }

    static void nv_enable_hotplug(struct ata_probe_ent *probe_ent)
    {
    u8 intr_mask;

    outb(NV_INT_STATUS_HOTPLUG,
    probe_ent->port[0].scr_addr + NV_INT_STATUS);

    intr_mask = inb(probe_ent->port[0].scr_addr + NV_INT_ENABLE);
    intr_mask |= NV_INT_ENABLE_HOTPLUG;

    outb(intr_mask, probe_ent->port[0].scr_addr + NV_INT_ENABLE);
    }

    static void nv_disable_hotplug(struct ata_host_set *host_set)
    {
    u8 intr_mask;

    intr_mask = inb(host_set->ports[0]->ioaddr.scr_addr + NV_INT_ENABLE);

    intr_mask &= ~(NV_INT_ENABLE_HOTPLUG);

    outb(intr_mask, host_set->ports[0]->ioaddr.scr_addr + NV_INT_ENABLE);
    }

    static void nv_check_hotplug(struct ata_host_set *host_set)
    {
    u8 intr_status;

    intr_status = inb(host_set->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);

    // Clear interrupt status.
    outb(0xff, host_set->ports[0]->ioaddr.scr_addr + NV_INT_STATUS);

    if (intr_status & NV_INT_STATUS_HOTPLUG) {
    if (intr_status & NV_INT_STATUS_PDEV_ADDED)
    printk(KERN_WARNING "nv_sata: "
    "Primary device added\n");

    if (intr_status & NV_INT_STATUS_PDEV_REMOVED)
    printk(KERN_WARNING "nv_sata: "
    "Primary device removed\n");

    if (intr_status & NV_INT_STATUS_SDEV_ADDED)
    printk(KERN_WARNING "nv_sata: "
    "Secondary device added\n");

    if (intr_status & NV_INT_STATUS_SDEV_REMOVED)
    printk(KERN_WARNING "nv_sata: "
    "Secondary device removed\n");
    }
    }

    static void nv_enable_hotplug_ck804(struct ata_probe_ent *probe_ent)
    {
    struct pci_dev *pdev = to_pci_dev(probe_ent->dev);
    u8 intr_mask;

    nv_enable_adma_space(pdev);

    writeb(NV_INT_STATUS_HOTPLUG, probe_ent->mmio_base + NV_INT_STATUS_CK804);

    intr_mask = readb(probe_ent->mmio_base + NV_INT_ENABLE_CK804);
    intr_mask |= NV_INT_ENABLE_HOTPLUG;

    writeb(intr_mask, probe_ent->mmio_base + NV_INT_ENABLE_CK804);
    }

    static void nv_disable_hotplug_ck804(struct ata_host_set *host_set)
    {
    struct pci_dev *pdev = to_pci_dev(host_set->dev);
    u8 intr_mask;

    intr_mask = readb(host_set->mmio_base + NV_INT_ENABLE_CK804);

    intr_mask &= ~(NV_INT_ENABLE_HOTPLUG);

    writeb(intr_mask, host_set->mmio_base + NV_INT_ENABLE_CK804);

    nv_disable_adma_space(pdev);
    }

    static void nv_check_hotplug_ck804(struct ata_host_set *host_set)
    {
    u8 intr_status;

    intr_status = readb(host_set->mmio_base + NV_INT_STATUS_CK804);

    // Clear interrupt status.
    writeb(0xff, host_set->mmio_base + NV_INT_STATUS_CK804);

    if (intr_status & NV_INT_STATUS_HOTPLUG) {
    if (intr_status & NV_INT_STATUS_PDEV_ADDED)
    printk(KERN_WARNING "nv_sata: "
    "Primary device added\n");

    if (intr_status & NV_INT_STATUS_PDEV_REMOVED)
    printk(KERN_WARNING "nv_sata: "
    "Primary device removed\n");

    if (intr_status & NV_INT_STATUS_SDEV_ADDED)
    printk(KERN_WARNING "nv_sata: "
    "Secondary device added\n");

    if (intr_status & NV_INT_STATUS_SDEV_REMOVED)
    printk(KERN_WARNING "nv_sata: "
    "Secondary device removed\n");
    }
    }

    static void nv_enable_hotplug_adma(struct ata_probe_ent *probe_ent)
    {
    struct pci_dev *pdev = to_pci_dev(probe_ent->dev);
    unsigned int i;
    u16 tmp;

    nv_enable_adma_space(pdev);

    for (i = 0; i < probe_ent->n_ports; i++) {
    void __iomem *mmio = __nv_adma_ctl_block(probe_ent->mmio_base, i);
    writew(NV_ADMA_STAT_HOTPLUG | NV_ADMA_STAT_HOTUNPLUG,
    mmio + NV_ADMA_STAT);

    tmp = readw(mmio + NV_ADMA_CTL);
    writew(tmp | NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);

    }
    }

    static void nv_disable_hotplug_adma(struct ata_host_set *host_set)
    {
    unsigned int i;
    u16 tmp;

    for (i = 0; i < host_set->n_ports; i++) {
    void __iomem *mmio = __nv_adma_ctl_block(host_set->mmio_base, i);

    tmp = readw(mmio + NV_ADMA_CTL);
    writew(tmp & ~NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL);

    }
    }

    static void nv_check_hotplug_adma(struct ata_host_set *host_set)
    {
    unsigned int i;
    u16 adma_status;

    for (i = 0; i < host_set->n_ports; i++) {
    void __iomem *mmio = __nv_adma_ctl_block(host_set->mmio_base, i);
    adma_status = readw(mmio + NV_ADMA_STAT);
    if (adma_status & NV_ADMA_STAT_HOTPLUG) {
    printk(KERN_WARNING "nv_sata: "
    "port %d device added\n", i);
    writew(NV_ADMA_STAT_HOTPLUG, mmio + NV_ADMA_STAT);
    }
    if (adma_status & NV_ADMA_STAT_HOTUNPLUG) {
    printk(KERN_WARNING "nv_sata: "
    "port %d device removed\n", i);
    writew(NV_ADMA_STAT_HOTUNPLUG, mmio + NV_ADMA_STAT);
    }
    }
    }

    static int __init nv_init(void)
    {
    return pci_module_init(&nv_pci_driver);
    }

    static void __exit nv_exit(void)
    {
    pci_unregister_driver(&nv_pci_driver);
    }

    module_init(nv_init);
    module_exit(nv_exit);

    #ifdef DEBUG
    static void nv_adma_dump_aprd(struct nv_adma_prd *aprd)
    {
    printk("%016llx %08x %02x %s %s %s\n",
    aprd->addr,
    aprd->len,
    aprd->flags,
    (aprd->flags & NV_APRD_WRITE) ? "WRITE" : " ",
    (aprd->flags & NV_APRD_END) ? "END" : " ",
    (aprd->flags & NV_APRD_CONT) ? "CONT" : " ");
    }
    static void nv_adma_dump_iomem(void __iomem *m, int len)
    {
    int i, j;

    for (i = 0; i < len/16; i++) {
    printk(KERN_WARNING "%02x: ", 16*i);
    for (j = 0; j < 16; j++) {
    printk("%02x%s", (u32)readb(m + 16*i + j),
    (j == 7) ? "-" : " ");
    }
    printk("\n");
    }
    }

    static void nv_adma_dump_cpb_tf(u16 tf)
    {
    printk("0x%04x %s %s %s 0x%02x 0x%02x\n",
    tf,
    (tf & CMDEND) ? "END" : " ",
    (tf & WNB) ? "WNB" : " ",
    (tf & IGN) ? "IGN" : " ",
    ((tf >> 8) & 0x1f),
    (tf & 0xff));
    }

    static void nv_adma_dump_port(struct ata_port *ap)
    {
    void __iomem *mmio = nv_adma_ctl_block(ap);
    nv_adma_dump_iomem(mmio, NV_ADMA_PORT_SIZE);
    }

    static void nv_adma_dump_cpb(struct nv_adma_cpb *cpb)
    {
    int i;

    printk("resp_flags: 0x%02x\n", cpb->resp_flags);
    printk("ctl_flags: 0x%02x\n", cpb->ctl_flags);
    printk("len: 0x%02x\n", cpb->len);
    printk("tag: 0x%02x\n", cpb->tag);
    printk("next_cpb_idx: 0x%02x\n", cpb->next_cpb_idx);
    printk("tf:\n");
    for (i=0; i<12; i++) {
    nv_adma_dump_cpb_tf(cpb->tf[i]);
    }
    printk("aprd:\n");
    for (i=0; i<5; i++) {
    nv_adma_dump_aprd(&cpb->aprd[i]);
    }
    printk("next_aprd: 0x%016llx\n", cpb->next_aprd);
    }

    #endif

    \
     
     \ /
      Last update: 2006-03-15 23:49    [W:4.932 / U:0.032 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site