|
|
b718a98 |
From f9b32cd97783f2be14386f1347439e86109050b9 Mon Sep 17 00:00:00 2001
|
|
|
b718a98 |
From: Jeroen Van den Keybus <jeroen.vandenkeybus@gmail.com>
|
|
|
b718a98 |
Date: Mon, 30 Jan 2012 22:37:28 +0100
|
|
Dave Jones |
766e8e8 |
Subject: [PATCH] Unhandled IRQs on AMD E-450: temporarily switch to
|
|
Dave Jones |
766e8e8 |
low-performance polling IRQ mode
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
It seems that some motherboard designs using the ASM1083 PCI/PCIe
|
|
Dave Jones |
766e8e8 |
bridge (PCI device ID 1b21:1080, Rev. 01) suffer from stuck IRQ lines
|
|
Dave Jones |
766e8e8 |
on the PCI bus (causing the kernel to emit 'IRQxx: nobody cared' and
|
|
Dave Jones |
766e8e8 |
disable the IRQ). The following patch is an attempt to mitigate the
|
|
Dave Jones |
766e8e8 |
serious impact of permanently disabling an IRQ in that case and
|
|
Dave Jones |
766e8e8 |
actually make PCI devices better usable on this platform.
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
It seems that the bridge fails to issue a IRQ deassertion message on
|
|
Dave Jones |
766e8e8 |
the PCIe bus, when the relevant driver causes the interrupting PCI
|
|
Dave Jones |
766e8e8 |
device to deassert its IRQ line. To solve this issue, it was tried to
|
|
Dave Jones |
766e8e8 |
re-issue an IRQ on a PCI device being able to do so (e1000 in this
|
|
Dave Jones |
766e8e8 |
case), but we suspect that the attempt to re-assert/deassert may have
|
|
Dave Jones |
766e8e8 |
occurred too soon after the initial IRQ for the ASM1083. Anyway, it
|
|
Dave Jones |
766e8e8 |
didn't work but if, after some delay, a new IRQ occurred, the related
|
|
Dave Jones |
766e8e8 |
IRQ deassertion message eventually did clear the IOAPIC IRQ. It would
|
|
Dave Jones |
766e8e8 |
be useful to re-enable the IRQ here.
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
Therefore the patch below to poll_spurious_irqs() in spurious.c is
|
|
Dave Jones |
766e8e8 |
proposed, It does the following:
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
1. lets the kernel decide that an IRQ is unhandled after only 10
|
|
Dave Jones |
766e8e8 |
positives (instead of 100,000);
|
|
Dave Jones |
766e8e8 |
2. briefly (a few seconds or so, currently 1 s) switches to polling
|
|
Dave Jones |
766e8e8 |
IRQ at a higher rate than usual (100..1,000Hz instead of 10Hz,
|
|
Dave Jones |
766e8e8 |
currently 100Hz), but not too high to avoid excessive CPU load. Any
|
|
Dave Jones |
766e8e8 |
device drivers 'see' their interrupts handled with a higher latency
|
|
Dave Jones |
766e8e8 |
than usual, but they will still operate properly;
|
|
Dave Jones |
766e8e8 |
3. afterwards, simply reenable the IRQ.
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
If proper operation of the PCIe legacy IRQ line emulation is restored
|
|
Dave Jones |
766e8e8 |
after 3, the system operates again at normal performance. If the IRQ
|
|
Dave Jones |
766e8e8 |
is still stuck after this procedure, the sequence repeats.
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
If a genuinely stuck IRQ is used with this solution, the system would
|
|
Dave Jones |
766e8e8 |
simply sustain short bursts of 10 unhandled IRQs per second, and use
|
|
Dave Jones |
766e8e8 |
polling mode indefinitely at a moderate 100Hz rate. It seemed a good
|
|
Dave Jones |
766e8e8 |
alternative to the default irqpoll behaviour to me, which is why I
|
|
Dave Jones |
766e8e8 |
left it in poll_spurious_irqs() (instead of creating a new kernel
|
|
Dave Jones |
766e8e8 |
option). Additionally, if any device happens to share an IRQ with a
|
|
Dave Jones |
766e8e8 |
faulty one, that device is no longer banned forever.
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
Debugging output is still present and may be removed. Bad IRQ
|
|
Dave Jones |
766e8e8 |
reporting is also commented out now.
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
I have now tried it for about 2 months and I can conclude the following:
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
1. The patch works and, judging from my Firewire card interrupt on
|
|
Dave Jones |
766e8e8 |
IRQ16, which repeats every 64 secs, I can confirm that the IRQ usually
|
|
Dave Jones |
766e8e8 |
gets reset when a new IRQ arrives (polling mode runs for 64 seconds
|
|
Dave Jones |
766e8e8 |
every time).
|
|
Dave Jones |
766e8e8 |
2. When testing a SiL-3114 SATA PCI card behind the ASM1083, I could
|
|
Dave Jones |
766e8e8 |
keep this running at fairly high speeds (50..70MB/s) for an hour or
|
|
Dave Jones |
766e8e8 |
so, but eventually the SiL driver crashed. In such conditions the PCI
|
|
Dave Jones |
766e8e8 |
system had to deal with a few hundred IRQs per second / polling mode
|
|
Dave Jones |
766e8e8 |
kicking in every 5..10 seconds).
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
I would like to thank Clemens Ladisch for his invaluable help in
|
|
Dave Jones |
766e8e8 |
finding a solution (and providing a patch to avoid my SATA going down
|
|
Dave Jones |
766e8e8 |
every time during debugging).
|
|
Dave Jones |
766e8e8 |
|
|
Dave Jones |
766e8e8 |
Signed-off-by: Jeroen Van den Keybus <jeroen.vandenkeybus@gmail.com>
|
|
|
25313f2 |
|
|
|
c4069fe |
Make it less chatty. Only kick it in if we detect an ASM1083 PCI bridge.
|
|
|
b718a98 |
Fix logic error due to lack of braces
|
|
|
c4069fe |
|
|
|
c4069fe |
Josh Boyer <jwboyer@redhat.com>
|
|
Dave Jones |
766e8e8 |
======
|
|
|
b718a98 |
---
|
|
|
b718a98 |
drivers/pci/quirks.c | 16 +++++++++++
|
|
|
b718a98 |
kernel/irq/spurious.c | 73 +++++++++++++++++++++++++++++++++++++++---------
|
|
|
b718a98 |
2 files changed, 75 insertions(+), 14 deletions(-)
|
|
Dave Jones |
766e8e8 |
|
|
|
b718a98 |
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
|
|
|
b718a98 |
index 78fda9c..6ba5dbf 100644
|
|
|
b718a98 |
--- a/drivers/pci/quirks.c
|
|
|
b718a98 |
+++ b/drivers/pci/quirks.c
|
|
|
b718a98 |
@@ -1677,6 +1677,22 @@ DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x2609, quirk_intel_pcie_pm);
|
|
|
b718a98 |
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x260a, quirk_intel_pcie_pm);
|
|
|
b718a98 |
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_INTEL, 0x260b, quirk_intel_pcie_pm);
|
|
|
b718a98 |
|
|
|
b718a98 |
+/* ASM108x transparent PCI bridges apparently have broken IRQ deassert
|
|
|
b718a98 |
+ * handling. This causes interrupts to get "stuck" and eventually disabled.
|
|
|
b718a98 |
+ * However, the interrupts are often shared and disabling them is fairly bad.
|
|
|
b718a98 |
+ * It's been somewhat successful to switch to polling mode and retry after
|
|
|
b718a98 |
+ * a bit, so let's do that.
|
|
|
b718a98 |
+ */
|
|
|
b718a98 |
+extern int irq_poll_and_retry;
|
|
|
b718a98 |
+static void quirk_asm108x_poll_interrupts(struct pci_dev *dev)
|
|
|
b718a98 |
+{
|
|
|
b718a98 |
+ dev_info(&dev->dev, "Buggy bridge found [%04x:%04x]\n",
|
|
|
b718a98 |
+ dev->vendor, dev->device);
|
|
|
b718a98 |
+ dev_info(&dev->dev, "Stuck interrupts will be polled and retried\n");
|
|
|
b718a98 |
+ irq_poll_and_retry = 1;
|
|
|
b718a98 |
+}
|
|
|
b718a98 |
+DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_ASMEDIA, 0x1080, quirk_asm108x_poll_interrupts);
|
|
|
b718a98 |
+
|
|
|
b718a98 |
#ifdef CONFIG_X86_IO_APIC
|
|
|
b718a98 |
/*
|
|
|
b718a98 |
* Boot interrupts on some chipsets cannot be turned off. For these chipsets,
|
|
|
b718a98 |
diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c
|
|
|
b718a98 |
index 611cd60..f722eb6 100644
|
|
|
b718a98 |
--- a/kernel/irq/spurious.c
|
|
|
b718a98 |
+++ b/kernel/irq/spurious.c
|
|
|
c4069fe |
@@ -18,6 +18,8 @@
|
|
|
25313f2 |
|
|
Dave Jones |
766e8e8 |
static int irqfixup __read_mostly;
|
|
|
25313f2 |
|
|
|
c4069fe |
+int irq_poll_and_retry = 0;
|
|
|
c4069fe |
+
|
|
|
c4069fe |
#define POLL_SPURIOUS_IRQ_INTERVAL (HZ/10)
|
|
Dave Jones |
766e8e8 |
static void poll_spurious_irqs(unsigned long dummy);
|
|
Dave Jones |
766e8e8 |
static DEFINE_TIMER(poll_spurious_irq_timer, poll_spurious_irqs, 0, 0);
|
|
|
c4069fe |
@@ -141,12 +143,13 @@ out:
|
|
Dave Jones |
766e8e8 |
static void poll_spurious_irqs(unsigned long dummy)
|
|
Dave Jones |
766e8e8 |
{
|
|
Dave Jones |
766e8e8 |
struct irq_desc *desc;
|
|
Dave Jones |
766e8e8 |
- int i;
|
|
Dave Jones |
766e8e8 |
+ int i, poll_again;
|
|
|
25313f2 |
|
|
Dave Jones |
766e8e8 |
if (atomic_inc_return(&irq_poll_active) != 1)
|
|
Dave Jones |
766e8e8 |
goto out;
|
|
Dave Jones |
766e8e8 |
irq_poll_cpu = smp_processor_id();
|
|
|
25313f2 |
|
|
Dave Jones |
766e8e8 |
+ poll_again = 0; /* Will stay false as long as no polling candidate is found */
|
|
Dave Jones |
766e8e8 |
for_each_irq_desc(i, desc) {
|
|
|
c4069fe |
unsigned int state;
|
|
|
25313f2 |
|
|
|
b718a98 |
@@ -159,14 +162,33 @@ static void poll_spurious_irqs(unsigned long dummy)
|
|
Dave Jones |
766e8e8 |
if (!(state & IRQS_SPURIOUS_DISABLED))
|
|
Dave Jones |
766e8e8 |
continue;
|
|
|
25313f2 |
|
|
Dave Jones |
766e8e8 |
- local_irq_disable();
|
|
Dave Jones |
766e8e8 |
- try_one_irq(i, desc, true);
|
|
Dave Jones |
766e8e8 |
- local_irq_enable();
|
|
Dave Jones |
766e8e8 |
+ /* We end up here with a disabled spurious interrupt.
|
|
Dave Jones |
766e8e8 |
+ desc->irqs_unhandled now tracks the number of times
|
|
Dave Jones |
766e8e8 |
+ the interrupt has been polled */
|
|
|
c4069fe |
+ if (irq_poll_and_retry) {
|
|
|
c4069fe |
+ if (desc->irqs_unhandled < 100) { /* 1 second delay with poll frequency 100 Hz */
|
|
|
c4069fe |
+ local_irq_disable();
|
|
|
c4069fe |
+ try_one_irq(i, desc, true);
|
|
|
c4069fe |
+ local_irq_enable();
|
|
|
c4069fe |
+ desc->irqs_unhandled++;
|
|
|
c4069fe |
+ poll_again = 1;
|
|
|
c4069fe |
+ } else {
|
|
|
c4069fe |
+ irq_enable(desc); /* Reenable the interrupt line */
|
|
|
c4069fe |
+ desc->depth--;
|
|
|
c4069fe |
+ desc->istate &= (~IRQS_SPURIOUS_DISABLED);
|
|
|
c4069fe |
+ desc->irqs_unhandled = 0;
|
|
|
c4069fe |
+ }
|
|
|
c4069fe |
+ } else {
|
|
Dave Jones |
766e8e8 |
+ local_irq_disable();
|
|
Dave Jones |
766e8e8 |
+ try_one_irq(i, desc, true);
|
|
Dave Jones |
766e8e8 |
+ local_irq_enable();
|
|
Dave Jones |
766e8e8 |
+ }
|
|
Dave Jones |
766e8e8 |
}
|
|
Dave Jones |
766e8e8 |
+ if (poll_again)
|
|
Dave Jones |
766e8e8 |
+ mod_timer(&poll_spurious_irq_timer,
|
|
Dave Jones |
766e8e8 |
+ jiffies + POLL_SPURIOUS_IRQ_INTERVAL);
|
|
Dave Jones |
766e8e8 |
out:
|
|
Dave Jones |
766e8e8 |
atomic_dec(&irq_poll_active);
|
|
Dave Jones |
766e8e8 |
- mod_timer(&poll_spurious_irq_timer,
|
|
Dave Jones |
766e8e8 |
- jiffies + POLL_SPURIOUS_IRQ_INTERVAL);
|
|
Dave Jones |
766e8e8 |
}
|
|
|
25313f2 |
|
|
Dave Jones |
766e8e8 |
static inline int bad_action_ret(irqreturn_t action_ret)
|
|
|
b718a98 |
@@ -177,11 +199,19 @@ static inline int bad_action_ret(irqreturn_t action_ret)
|
|
Dave Jones |
766e8e8 |
}
|
|
|
25313f2 |
|
|
Dave Jones |
766e8e8 |
/*
|
|
Dave Jones |
766e8e8 |
- * If 99,900 of the previous 100,000 interrupts have not been handled
|
|
Dave Jones |
766e8e8 |
+ * If 9 of the previous 10 interrupts have not been handled
|
|
Dave Jones |
766e8e8 |
* then assume that the IRQ is stuck in some manner. Drop a diagnostic
|
|
Dave Jones |
766e8e8 |
* and try to turn the IRQ off.
|
|
Dave Jones |
766e8e8 |
*
|
|
Dave Jones |
766e8e8 |
- * (The other 100-of-100,000 interrupts may have been a correctly
|
|
Dave Jones |
766e8e8 |
+ * Although this may cause early deactivation of a sporadically
|
|
Dave Jones |
766e8e8 |
+ * malfunctioning IRQ line, the poll system will:
|
|
Dave Jones |
766e8e8 |
+ * a) Poll it for 100 cycles at a 100 Hz rate
|
|
Dave Jones |
766e8e8 |
+ * b) Reenable it afterwards
|
|
Dave Jones |
766e8e8 |
+ *
|
|
Dave Jones |
766e8e8 |
+ * In worst case, with current settings, this will cause short bursts
|
|
Dave Jones |
766e8e8 |
+ * of 10 interrupts every second.
|
|
Dave Jones |
766e8e8 |
+ *
|
|
Dave Jones |
766e8e8 |
+ * (The other single interrupt may have been a correctly
|
|
Dave Jones |
766e8e8 |
* functioning device sharing an IRQ with the failing one)
|
|
Dave Jones |
766e8e8 |
*/
|
|
Dave Jones |
766e8e8 |
static void
|
|
|
b718a98 |
@@ -269,6 +299,8 @@ try_misrouted_irq(unsigned int irq, struct irq_desc *desc,
|
|
|
6df15dd |
void note_interrupt(unsigned int irq, struct irq_desc *desc,
|
|
|
6df15dd |
irqreturn_t action_ret)
|
|
|
6df15dd |
{
|
|
|
6df15dd |
+ int unhandled_thresh = 999000;
|
|
|
6df15dd |
+
|
|
|
6df15dd |
if (desc->istate & IRQS_POLL_INPROGRESS)
|
|
|
6df15dd |
return;
|
|
|
6df15dd |
|
|
|
b718a98 |
@@ -302,19 +334,32 @@ void note_interrupt(unsigned int irq, struct irq_desc *desc,
|
|
Dave Jones |
766e8e8 |
}
|
|
|
25313f2 |
|
|
Dave Jones |
766e8e8 |
desc->irq_count++;
|
|
Dave Jones |
766e8e8 |
- if (likely(desc->irq_count < 100000))
|
|
|
6df15dd |
- return;
|
|
|
b718a98 |
+ if (!irq_poll_and_retry) {
|
|
|
6df15dd |
+ if (likely(desc->irq_count < 100000))
|
|
|
6df15dd |
+ return;
|
|
|
b718a98 |
+ } else {
|
|
|
6df15dd |
+ if (likely(desc->irq_count < 10))
|
|
|
6df15dd |
+ return;
|
|
|
b718a98 |
+ }
|
|
|
25313f2 |
|
|
Dave Jones |
766e8e8 |
desc->irq_count = 0;
|
|
Dave Jones |
766e8e8 |
- if (unlikely(desc->irqs_unhandled > 99900)) {
|
|
|
6df15dd |
+ if (irq_poll_and_retry)
|
|
|
6df15dd |
+ unhandled_thresh = 9;
|
|
|
6df15dd |
+
|
|
|
6df15dd |
+ if (unlikely(desc->irqs_unhandled >= unhandled_thresh)) {
|
|
Dave Jones |
766e8e8 |
/*
|
|
|
c4069fe |
- * The interrupt is stuck
|
|
|
c4069fe |
+ * The interrupt might be stuck
|
|
Dave Jones |
766e8e8 |
*/
|
|
Dave Jones |
766e8e8 |
- __report_bad_irq(irq, desc, action_ret);
|
|
|
c4069fe |
+ if (!irq_poll_and_retry) {
|
|
|
c4069fe |
+ __report_bad_irq(irq, desc, action_ret);
|
|
|
c4069fe |
+ printk(KERN_EMERG "Disabling IRQ %d\n", irq);
|
|
|
c4069fe |
+ } else {
|
|
|
c4069fe |
+ printk(KERN_INFO "IRQ %d might be stuck. Polling\n",
|
|
|
c4069fe |
+ irq);
|
|
|
c4069fe |
+ }
|
|
Dave Jones |
766e8e8 |
/*
|
|
Dave Jones |
766e8e8 |
* Now kill the IRQ
|
|
Dave Jones |
766e8e8 |
*/
|
|
Dave Jones |
766e8e8 |
- printk(KERN_EMERG "Disabling IRQ #%d\n", irq);
|
|
Dave Jones |
766e8e8 |
desc->istate |= IRQS_SPURIOUS_DISABLED;
|
|
Dave Jones |
766e8e8 |
desc->depth++;
|
|
Dave Jones |
766e8e8 |
irq_disable(desc);
|
|
|
b718a98 |
--
|
|
|
b718a98 |
1.7.7.6
|
|
|
b718a98 |
|