Address mapping of PCI-memory in Kernel space

2019-05-06 12:11发布

站内文章 / C
64 0 0

问题:

I'm trying to read and write to and PCI-device from a loadable kernel module.

Therefore I follow this post:

pci_enable_device(dev);
pci_request_regions(dev, "expdev");
bar1 = pci_iomap(dev, 1, 0);
// void iowrite32(u32 val, void __iomem *addr)
iowrite32( 0xaaaaaaaa, bar1 + 0x060000);  /* offset from device spec */

But at the end the device doesn't do his work as expected. Then I look to the address behind bar1 and found a very big value ffffbaaaaa004500.

At this point I don't really understand what was happen there and what was right. Can I interpret bar1 as an address inside my kernel address space which points directly to the base address which is 0x60000 offset to the PCI Chip Select Address?

And how can it be that the value I write to bar1 + offset copied to the device? How does the mechanism work in behind of iowrite32 and pci_iomap?

Thanks and regards

Alex

PS: I tested a read back from the same address successfully.

Register description of the PCI device:

  • PCIBAR0 PCI Base Address 0; used for Memory-Mapped Configuration Registers
  • PCIBAR1 PCI Base Address 1; used for I/O-Mapped Configuration Registers
  • PCIBAR2 PCI Base Address 2; used for Local Address Space 0
  • PCIBAR3 PCI Base Address 3; used for Local Address Space 1
  • PCIBAR4 Unused Base Address
  • PCIBAR5 Unused Base Address

Hello again.

In the last time I tried several approaches to communicate with the BAR2 register but without success. Here my actual code:

#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/sched.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Alex");
MODULE_DESCRIPTION("test module.");
MODULE_VERSION("0.1");

#define DEV_PCI_VENDORID 0x10B5
#define DEV_PCI_DEVICEID 0x1860

static struct pci_device_id pci_drvIdTable[] =
{
  {
    .vendor      = DEV_PCI_VENDORID,      // vendor ID
    .device      = DEV_PCI_DEVICEID,      // device ID
    .subvendor   = PCI_ANY_ID,            // no subsystem available
    .subdevice   = PCI_ANY_ID,            // no subsystem available
    .class       = PCI_CLASS_NOT_DEFINED, // no device class
    .class_mask  = 0,                     // no device class
    .driver_data = 0                      // no private data to the driver
  },
  { 0, }                                  // end of table
};

struct pci_data
{
  /// the IO mapping for the PCI config space
  uint32_t *pciConfigAddr;
  uint32_t *pciB2Addr;
//  void __iomem *pciConfigAddr;

  wait_queue_head_t waitq;
  uint8_t flag;
} *data;

static irqreturn_t
_expdev_irq (int irq, void *pdata)
{
  struct pci_data *data = pdata;

  printk(KERN_INFO "Interrupt talks...\n");
  data->flag = 1;
  wake_up_interruptible( &data->waitq );

  return IRQ_HANDLED;
}


static int
_pci_probe ( struct pci_dev *pdev,
             const struct pci_device_id *ent )
{
  int ret = 0;
  int i;
  u16 reg_16;
  unsigned long bas2addr;

  data = kzalloc( sizeof(*data) , GFP_KERNEL );
  if( !data )
  {
    printk(KERN_ERR "Failed to allocate memory.\n");
    return -ENOMEM;
  }

  pci_set_drvdata(pdev, data);

  // enabling the device
  ret = pci_enable_device(pdev);
  if( ret )
  {
    printk(KERN_ERR "Failed to enable PCI device.\n");
    goto no_enable;
  }

  pci_read_config_word(pdev,0,&reg_16);
  printk(KERN_INFO "VendorID. %x\n",reg_16);

  // checking if PCI-device reachable by checking that BAR0 is defined and
  // memory mapped
  if( !(pci_resource_flags(pdev,0) & IORESOURCE_MEM) )
  {
    printk(KERN_ERR "Incorrect BAR configuration.\n");
    ret = -ENODEV;
    goto bad_bar;
  }

  // taking ownership of a memory region
  ret = pcim_iomap_regions(pdev, 0b0100, "expdev");
//  ret = pci_request_regions(pdev,"expdev");
  if( ret )
  {
    printk(KERN_ERR "Failed to request regions.\n");
    goto failed_request_regions;
  }

  bas2addr = pci_resource_start(pdev, 2);

  reg_16 = 0xAA;
  i = 0x060000;
  iowrite16( reg_16 , (unsigned long *)(bas2addr+i) );
  printk( KERN_INFO "BAR2 Register[0x%x] = 0x%x\n",
          i, ioread32( (unsigned long *)(bas2addr+i) ) );

  printk(KERN_INFO "Module successfully initialised.\n");

  return 0;

  // Error handling - backward disabling the device
failed_request_regions:
bad_bar:
  pci_disable_device(pdev);
no_enable:
  pci_set_drvdata(pdev, data);

  return ret;
}


static void
_pci_remove( struct pci_dev *pdev )
{
  free_irq(pdev->irq, data);
  pci_disable_msi(pdev);
  pci_clear_master(pdev);
  pci_iounmap(pdev,data->pciConfigAddr);
  pci_release_regions(pdev);
  pci_disable_device(pdev);

  printk(KERN_INFO "PCI-device removed.\n");
}


static struct pci_driver pci_drv =
{
  .name     = "expdev",
  .id_table = pci_drvIdTable,
  .probe    = _pci_probe,
  .remove   = _pci_remove,
};

// module related functions ///////////////////////////////////////////////////

static int __init _module_init(void){
   printk(KERN_INFO "Hello!\n");

   pci_register_driver(&pci_drv);

   return 0;
}

static void __exit _module_exit(void){

   pci_unregister_driver(&pci_drv);

   printk(KERN_INFO "Goodbye!\n");
}

module_init(_module_init);
module_exit(_module_exit);

This is the responding output with tail -f /var/log/kern.log

kernel: [  493.719999] Hello!
kernel: [  493.720071] expdev 0000:05:02.0: enabling device (0000 -> 0003)
kernel: [  493.720845] VendorID. 10b5
kernel: [  493.722375] BUG: unable to handle kernel paging request at 00000000eb060000
kernel: [  493.722381] IP: [<ffffffff8137aca8>] iowrite16+0x38/0x40
kernel: [  493.722388] PGD 0 
kernel: [  493.722390] Oops: 0002 [#1] SMP 
kernel: [  493.722394] Modules linked in: expdev(OX+) rfcomm bnep bluetooth nvidia(POX) snd_hda_codec_hdmi joydev snd_hda_codec_idt snd_hda_intel snd_hda_codec snd_hwdep snd_pcm coretemp snd_page_alloc snd_seq_midi snd_seq_midi_event snd_rawmidi gpio_ich kvm snd_seq snd_seq_device drm dcdbas snd_timer lpc_ich snd soundcore shpchp serio_raw ppdev i82975x_edac lp parport_pc edac_core parport mac_hid hid_generic usbhid hid psmouse ahci tg3 libahci ptp pps_core pata_acpi
kernel: [  493.722429] CPU: 0 PID: 3542 Comm: insmod Tainted: P           OX 3.13.0-79-generic #123-Ubuntu
kernel: [  493.722431] Hardware name: Dell Inc.                 Precision WorkStation 390    /0DN075, BIOS 2.3.0  05/01/2007
kernel: [  493.722434] task: ffff8800549c3000 ti: ffff8800555e6000 task.ti: ffff8800555e6000
kernel: [  493.722436] RIP: 0010:[<ffffffff8137aca8>]  [<ffffffff8137aca8>] iowrite16+0x38/0x40
kernel: [  493.722440] RSP: 0018:ffff8800555e7b88  EFLAGS: 00010212
kernel: [  493.722442] RAX: 00000000eb000000 RBX: ffff88007c2b4000 RCX: 0000000000000000
kernel: [  493.722444] RDX: 00000000eb060000 RSI: 00000000eb060000 RDI: 00000000000000aa
kernel: [  493.722446] RBP: ffff8800555e7bb0 R08: 00000000ebffffff R09: 00000000ffffffec
kernel: [  493.722448] R10: 0000000000003692 R11: 0000000000000000 R12: 00000000eb060000
kernel: [  493.722450] R13: ffff88007c2b4098 R14: ffff88007c2b4098 R15: ffffffffa022b140
kernel: [  493.722452] FS:  00007fa8053ef740(0000) GS:ffff88007f800000(0000) knlGS:0000000000000000
kernel: [  493.722454] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
kernel: [  493.722456] CR2: 00000000eb060000 CR3: 000000005a74b000 CR4: 00000000000007f0
kernel: [  493.722458] Stack:
kernel: [  493.722460]  ffffffffa02291c4 00aa88007c2b4000 ffff88007c2b4000 0000000000000000
kernel: [  493.722464]  ffffffffa022b000 ffff8800555e7be8 ffffffff813ac8a5 ffffffff813adb45
kernel: [  493.722467]  ffff88007c2b4098 ffffffffffffffff ffff88007c2b4000 0000000000000017
kernel: [  493.722471] Call Trace:
kernel: [  493.722477]  [<ffffffffa02291c4>] ? _pci_probe+0x114/0x215 [expdev]
kernel: [  493.722481]  [<ffffffff813ac8a5>] local_pci_probe+0x45/0xa0
kernel: [  493.722484]  [<ffffffff813adb45>] ? pci_match_device+0xc5/0xd0
kernel: [  493.722487]  [<ffffffff813adc69>] pci_device_probe+0xd9/0x130
kernel: [  493.722492]  [<ffffffff8149a4bd>] driver_probe_device+0x12d/0x3e0
kernel: [  493.722495]  [<ffffffff8149a843>] __driver_attach+0x93/0xa0
kernel: [  493.722498]  [<ffffffff8149a7b0>] ? __device_attach+0x40/0x40
kernel: [  493.722501]  [<ffffffff81498403>] bus_for_each_dev+0x63/0xa0
kernel: [  493.722504]  [<ffffffff81499e6e>] driver_attach+0x1e/0x20
kernel: [  493.722507]  [<ffffffff81499a50>] bus_add_driver+0x180/0x250
kernel: [  493.722510]  [<ffffffffa0005000>] ? 0xffffffffa0004fff
kernel: [  493.722514]  [<ffffffff8149aec4>] driver_register+0x64/0xf0
kernel: [  493.722517]  [<ffffffffa0005000>] ? 0xffffffffa0004fff
kernel: [  493.722520]  [<ffffffff813ac23c>] __pci_register_driver+0x4c/0x50
kernel: [  493.722523]  [<ffffffffa000502c>] _module_init+0x2c/0x1000 [expdev]
kernel: [  493.722528]  [<ffffffff8100214a>] do_one_initcall+0xfa/0x1b0
kernel: [  493.722532]  [<ffffffff810598f3>] ? set_memory_nx+0x43/0x50
kernel: [  493.722536]  [<ffffffff810e2b7d>] load_module+0x12ed/0x1b50
kernel: [  493.722540]  [<ffffffff810de5f0>] ? store_uevent+0x40/0x40
kernel: [  493.722544]  [<ffffffff810e3556>] SyS_finit_module+0x86/0xb0
kernel: [  493.722548]  [<ffffffff817363dd>] system_call_fastpath+0x1a/0x1f
kernel: [  493.722550] Code: 81 fe 00 00 01 00 76 0b 0f b7 d6 89 f8 66 ef c3 0f 1f 00 55 48 c7 c6 b0 10 a9 81 48 89 d7 48 89 e5 e8 5d fe ff ff 5d c3 0f 1f 00 <66> 89 3e c3 0f 1f 40 00 48 81 fe ff ff 03 00 48 89 f2 77 2c 48 
kernel: [  493.722583] RIP  [<ffffffff8137aca8>] iowrite16+0x38/0x40
kernel: [  493.722586]  RSP <ffff8800555e7b88>
kernel: [  493.722588] CR2: 00000000eb060000
kernel: [  493.722591] ---[ end trace 2d3dfa92998d58a7 ]---

According to Ian Abbott now I tried this approach successfully. I don't really understand the mechanism behind but it works for now. So BAR2 is an Memory Register type. This approach uses ioremap and not memory-mapping. How to get access to BAR2 via memory-mapping?

#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Alex");
MODULE_DESCRIPTION("test module.");
MODULE_VERSION("0.1");

#define DEV_PCI_VENDORID 0x10B5
#define DEV_PCI_DEVICEID 0x1860

static struct pci_device_id pci_drvIdTable[] =
{
  {
    .vendor      = DEV_PCI_VENDORID,      // vendor ID
    .device      = DEV_PCI_DEVICEID,      // device ID
    .subvendor   = PCI_ANY_ID,            // no subsystem available
    .subdevice   = PCI_ANY_ID,            // no subsystem available
    .class       = PCI_CLASS_NOT_DEFINED, // no device class
    .class_mask  = 0,                     // no device class
    .driver_data = 0                      // no private data to the driver
  },
  { 0, }                                  // end of table
};

struct pci_data
{
//  struct pci_dev *pci_dev;

  /// the IO mapping for the PCI config space
  uint32_t *pciConfigAddr;
  uint32_t *pciB2Addr;

  wait_queue_head_t waitq;
  uint8_t flag;
} *data;

static int
_pci_probe ( struct pci_dev *pdev,
             const struct pci_device_id *ent )
{
  int ret = 0;
  int i;
  u16 reg_16;
  unsigned long *pbas2addr;

  data = kzalloc( sizeof(*data) , GFP_KERNEL );
  if( !data )
  {
    printk(KERN_ERR "Failed to allocate memory.\n");
    return -ENOMEM;
  }

  pci_set_drvdata(pdev, data);

  // enabling the device
  ret = pci_enable_device(pdev);
  if( ret )
  {
    printk(KERN_ERR "Failed to enable PCI device.\n");
    goto no_enable;
  }

  pci_read_config_word(pdev,0,&reg_16);
  printk(KERN_INFO "VendorID. %x\n",reg_16);

  // checking if PCI-device reachable by checking that BAR0 is defined and
  // memory mapped
  if( !(pci_resource_flags(pdev,0) & IORESOURCE_MEM) )
  {
    printk(KERN_ERR "Incorrect BAR configuration.\n");
    ret = -ENODEV;
    goto bad_bar;
  }

  // taking ownership of a memory region
  pbas2addr = pci_ioremap_bar(pdev, 2);

  // void iowrite8(u8 val, void __iomem *addr)
    for ( i = 0x060000; i<0x070000; i++ )
    {
      iowrite8( 0x11 , pbas2addr+i );
    }

  // further read/write function in the kernel:
  // inp,  readl,  readw,  readb,  ioread8,  ioread16,  ioread32
  // outp, writel, writew, writeb, iowrite8, iowrite16, iowrite32

bad_bar:
  pci_disable_device(pdev);
no_enable:
  pci_set_drvdata(pdev, data);

  return ret;
}

static void
_pci_remove( struct pci_dev *pdev )
{
  pci_disable_device(pdev);
  printk(KERN_INFO "PCI-device removed.\n");
}

static struct pci_driver pci_drv =
{
  .name     = "expdev",
  .id_table = pci_drvIdTable,
  .probe    = _pci_probe,
  .remove   = _pci_remove,
};

// module related functions ///////////////////////////////////////////////////

static int __init _module_init(void){
   printk(KERN_INFO "Hello!\n");
   pci_register_driver(&pci_drv);
   return 0;
}

static void __exit _module_exit(void){
   pci_unregister_driver(&pci_drv);
   printk(KERN_INFO "Goodbye!\n");
}

module_init(_module_init);
module_exit(_module_exit);

I'm really at the end. I think I have fully follow the docs but it doesn't work as expected.

Here the code:

#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/pci.h>
#include <linux/interrupt.h>
#include <linux/sched.h>

#include <linux/delay.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Alex");
MODULE_DESCRIPTION("test module.");
MODULE_VERSION("0.1");

#define DEV_PCI_VENDORID 0x10B5
#define DEV_PCI_DEVICEID 0x1860

static struct pci_device_id pci_drvIdTable[] =
{
  {
    .vendor      = DEV_PCI_VENDORID,      // vendor ID
    .device      = DEV_PCI_DEVICEID,        // device ID
    .subvendor   = PCI_ANY_ID,            // no subsystem available
    .subdevice   = PCI_ANY_ID,            // no subsystem available
    .class       = PCI_CLASS_NOT_DEFINED, // no device class
    .class_mask  = 0,                       // no device class
    .driver_data = 0                        // no private data to the driver
  },
  { 0, }
};

static int
_pci_probe ( struct pci_dev *pdev,
             const struct pci_device_id *ent )
{
  int ret = 0;
  int i;
  unsigned long *pbas2addr;

  // enabling the device
  ret = pci_enable_device(pdev);
  if( ret )
  {
    printk(KERN_ERR "Failed to enable PCI device.\n");
    goto no_enable;
  }

  pci_request_regions(pdev, "expdev");

  // checking if PCI-device reachable by checking that BAR0 is defined and
  // memory mapped
  if( !(pci_resource_flags(pdev,0) & IORESOURCE_MEM) )
  {
    printk(KERN_ERR "Incorrect BAR configuration.\n");
    ret = -ENODEV;
    goto bad_bar;
  }

  // taking ownership of a memory region
  pbas2addr = pci_ioremap_bar(pdev, 2);

  printk(KERN_INFO "BAR2: %p\n",pbas2addr);

  for ( i = 0x060000; i<0x070000; i++ )
  {
    iowrite8( 0x00 , pbas2addr+i );
  }

  // the next write operations cause crashing the the module
  // load control word to set ICD in set-up-mode
  iowrite32(0b01111000000101, pbas2addr+0x200014);
  // load the bit-stuffed set up word
  iowrite32(0b10110001001001101110000, pbas2addr+0x200018); // 39.5 MHz
  // load control word to set ICD in set-up-mode
  iowrite32(0b01111000000100, pbas2addr+0x200014);
  msleep(10);
  // load control word to set ICD in set-up-mode
  iowrite32(0b01111000000000, pbas2addr+0x200014);

  return 0;

bad_bar:
  pci_disable_device(pdev);

  return ret;
}


static void
_pci_remove( struct pci_dev *pdev )
{
  pci_release_regions(pdev);
  pci_disable_device(pdev);

  printk(KERN_INFO "PCI-device removed.\n");
}

static struct pci_driver pci_drv =
{
  .name     = "expdev",
  .id_table = pci_drvIdTable,
  .probe    = _pci_probe,
  .remove   = _pci_remove,
};

// module related functions     
static int __init _module_init(void){
   printk(KERN_INFO "Hello!\n");
   pci_register_driver(&pci_drv);
   return 0;
}

static void __exit _module_exit(void){
   pci_unregister_driver(&pci_drv);
   printk(KERN_INFO "Goodbye!\n");
}

module_init(_module_init);
module_exit(_module_exit);

With only the for-loop:

kernel: [  467.545079] Hello!
kernel: [  467.545136] expdev 0000:05:02.0: enabling device (0000 -> 0003)
kernel: [  467.546807] BAR2: ffffc90006c00000
kernel: [  467.562146] PCI-device removed.
kernel: [  467.562489] Goodbye!

and I can see some outputs on the device GPIO.

If I write to the higher addresses as needed regarding to the device manual the LKM crashes:

kernel: [ 1324.591578] Hello!
kernel: [ 1324.593300] BAR2: ffffc90007c80000
kernel: [ 1324.605162] BUG: unable to handle kernel paging request at ffffc90008c800a0
kernel: [ 1324.605168] IP: [<ffffffff8137ace8>] iowrite32+0x38/0x40
kernel: [ 1324.605175] PGD 7d00d067 PUD 7d00e067 PMD 611e7067 PTE 0
kernel: [ 1324.605179] Oops: 0002 [#1] SMP 
kernel: [ 1324.605183] Modules linked in: expdev(OX+) rfcomm bnep bluetooth nvidia(POX) snd_hda_codec_hdmi joydev snd_hda_codec_idt snd_hda_intel snd_hda_codec gpio_ich coretemp drm snd_seq_midi kvm snd_seq_midi_event dcdbas snd_rawmidi snd_hwdep lpc_ich snd_seq snd_pcm snd_seq_device snd_page_alloc shpchp ppdev serio_raw snd_timer lp snd soundcore mac_hid i82975x_edac edac_core parport_pc parport hid_generic usbhid hid psmouse ahci libahci pata_acpi tg3 ptp pps_core [last unloaded: expdev]
kernel: [ 1324.605219] CPU: 0 PID: 3155 Comm: insmod Tainted: P           OX 3.13.0-79-generic #123-Ubuntu
kernel: [ 1324.605221] Hardware name: Dell Inc.                 Precision WorkStation 390    /0DN075, BIOS 2.3.0  05/01/2007
kernel: [ 1324.605224] task: ffff88007c048000 ti: ffff880061122000 task.ti: ffff880061122000
kernel: [ 1324.605226] RIP: 0010:[<ffffffff8137ace8>]  [<ffffffff8137ace8>] iowrite32+0x38/0x40
kernel: [ 1324.605229] RSP: 0018:ffff880061123b90  EFLAGS: 00010292
kernel: [ 1324.605231] RAX: 0000000000000016 RBX: ffffc90008c800a0 RCX: 0000000000000000
kernel: [ 1324.605233] RDX: ffffc90008c800a0 RSI: ffffc90008c800a0 RDI: 0000000000001e05
kernel: [ 1324.605235] RBP: ffff880061123bb0 R08: 0000000000000096 R09: 0000000000000306
kernel: [ 1324.605237] R10: 0000000000000000 R11: ffff8800611238c6 R12: ffffc90007f80000
kernel: [ 1324.605239] R13: ffffc90007c80000 R14: ffff88007c2b4098 R15: ffffffffa01fc140
kernel: [ 1324.605242] FS:  00007fc6802cb740(0000) GS:ffff88007f800000(0000) knlGS:0000000000000000
kernel: [ 1324.605244] CS:  0010 DS: 0000 ES: 0000 CR0: 0000000080050033
kernel: [ 1324.605246] CR2: ffffc90008c800a0 CR3: 0000000062f96000 CR4: 00000000000007f0
kernel: [ 1324.605248] Stack:
kernel: [ 1324.605249]  ffffffffa01fa0ec ffff88007c2b4000 0000000000000000 ffffffffa01fc000
kernel: [ 1324.605253]  ffff880061123be8 ffffffff813ac8a5 ffffffff813adb45 ffff88007c2b4098
kernel: [ 1324.605257]  ffffffffffffffff ffff88007c2b4000 0000000000000018 ffff880061123c30
kernel: [ 1324.605260] Call Trace:
kernel: [ 1324.605267]  [<ffffffffa01fa0ec>] ? _pci_probe+0xbc/0x110 [expdev]
kernel: [ 1324.605271]  [<ffffffff813ac8a5>] local_pci_probe+0x45/0xa0
kernel: [ 1324.605274]  [<ffffffff813adb45>] ? pci_match_device+0xc5/0xd0
kernel: [ 1324.605277]  [<ffffffff813adc69>] pci_device_probe+0xd9/0x130
kernel: [ 1324.605281]  [<ffffffff8149a4bd>] driver_probe_device+0x12d/0x3e0
kernel: [ 1324.605285]  [<ffffffff8149a843>] __driver_attach+0x93/0xa0
kernel: [ 1324.605288]  [<ffffffff8149a7b0>] ? __device_attach+0x40/0x40
kernel: [ 1324.605290]  [<ffffffff81498403>] bus_for_each_dev+0x63/0xa0
kernel: [ 1324.605293]  [<ffffffff81499e6e>] driver_attach+0x1e/0x20
kernel: [ 1324.605296]  [<ffffffff81499a50>] bus_add_driver+0x180/0x250
kernel: [ 1324.605300]  [<ffffffffa0006000>] ? 0xffffffffa0005fff
kernel: [ 1324.605303]  [<ffffffff8149aec4>] driver_register+0x64/0xf0
kernel: [ 1324.605306]  [<ffffffffa0006000>] ? 0xffffffffa0005fff
kernel: [ 1324.605309]  [<ffffffff813ac23c>] __pci_register_driver+0x4c/0x50
kernel: [ 1324.605313]  [<ffffffffa000602c>] _module_init+0x2c/0x1000 [expdev]
kernel: [ 1324.605317]  [<ffffffff8100214a>] do_one_initcall+0xfa/0x1b0
kernel: [ 1324.605321]  [<ffffffff810598f3>] ? set_memory_nx+0x43/0x50
kernel: [ 1324.605326]  [<ffffffff810e2b7d>] load_module+0x12ed/0x1b50
kernel: [ 1324.605330]  [<ffffffff810de5f0>] ? store_uevent+0x40/0x40
kernel: [ 1324.605334]  [<ffffffff810e3556>] SyS_finit_module+0x86/0xb0
kernel: [ 1324.605338]  [<ffffffff817363dd>] system_call_fastpath+0x1a/0x1f
kernel: [ 1324.605340] Code: 81 fe 00 00 01 00 76 0b 0f b7 d6 89 f8 ef c3 0f 1f 40 00 55 48 c7 c6 bf 10 a9 81 48 89 d7 48 89 e5 e8 1d fe ff ff 5d c3 0f 1f 00 <89> 3e c3 0f 1f 44 00 00 48 81 ff ff ff 03 00 77 37 48 81 ff 00 
kernel: [ 1324.605373] RIP  [<ffffffff8137ace8>] iowrite32+0x38/0x40
kernel: [ 1324.605376]  RSP <ffff880061123b90>
kernel: [ 1324.605378] CR2: ffffc90008c800a0
kernel: [ 1324.605381] ---[ end trace 9b1029fd3f919791 ]---

RIP - but why. The offset is within the limits of 16 MB.

回答1:

I think your code is corrent but you are supposed to check your resource files for the PCI host driver. When you call the funtion "pci_ioremap_bar(pdev, 2)", the function needs some resource data to remap the memory space.

For example,

[2] = {
    .name   = "ep_mem2",
    .start  = PCIE_BASE + 0x1000,
    .end    = PCIE_BASE + 0x2000 - 1,
    .flags  = IORESOURCE_MEM,
}


回答2:

Originally PCI buses were like a tree, with "PCI host controller" at the root, bridges at branches, and devices at the leaves. When the CPU writes to the physical address space corresponding to a device's MMIO area, the memory controller has to forward the access to the right PCI host controller (and not RAM or a different PCI host controller and not a different NUMA node), the PCI host controller forwards it to the first PCI bus, then all the bridges on that PCI bus decide to accept or ignore the access and one of them will forward the access to its "secondary bus" (the bus on the other side of the bridge), and so on until it ends up on the bus that the device is on and that device accepts the access (and any other devices on that bus ignore it). Of course you can't have 2 or more devices using the same area of the physical address space for different MMIO areas because that'd be cause conflicts.

For more modern systems there's extra layers of shenanigans (e.g. IOMMUs); and for PCI-express it changed from "tree" to "direct links" (but this doesn't mean you can't and won't see "PCI-E to PCI conventional" bridges with "tree" on the other side of the bridge).

The important thing to understand here is that the firmware configures most of this (and a few other things) to ensure that accesses actually get to the device that they're supposed to (and the kernel may also configure some - e.g. IOMMUs), but a device driver should not randomly decide to trash everything on a whim.

A device driver should not modify PCI BARs, at all, ever. No excuses.

Note: Ideally (for buses that provide device enumeration and resource auto-configuration capabilities - e.g. EISA, MCA, PCI, ..) the kernel would figure everything out, then start the driver and tell the driver which resources (MMIO areas, IRQs, ...) its device uses; and the device driver wouldn't know or care which type of bus the device uses and wouldn't touch PCI configuration space (which might not exist as soon as a "yet another different bus" is invented and the same device is slapped onto a different type of bus).



标签： c linux-kernel kernel kernel-module pci
举报

收藏的人(0)

Ta的文章 更多文章
登录 后发表评论
0条评论
还没有人评论过~