|
 |
9bf39b3 |
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
|
|
|
9bf39b3 |
index 15763af..f6978b0 100644
|
|
|
9bf39b3 |
--- a/arch/x86/kernel/process.c
|
|
|
9bf39b3 |
+++ b/arch/x86/kernel/process.c
|
|
|
9bf39b3 |
@@ -386,17 +386,21 @@ void default_idle(void)
|
|
|
9bf39b3 |
*/
|
|
|
9bf39b3 |
smp_mb();
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
+ rcu_idle_enter();
|
|
|
9bf39b3 |
if (!need_resched())
|
|
|
9bf39b3 |
safe_halt(); /* enables interrupts racelessly */
|
|
|
9bf39b3 |
else
|
|
|
9bf39b3 |
local_irq_enable();
|
|
|
9bf39b3 |
+ rcu_idle_exit();
|
|
|
9bf39b3 |
current_thread_info()->status |= TS_POLLING;
|
|
|
9bf39b3 |
trace_power_end(smp_processor_id());
|
|
|
9bf39b3 |
trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
|
|
|
9bf39b3 |
} else {
|
|
|
9bf39b3 |
local_irq_enable();
|
|
|
9bf39b3 |
/* loop is done by the caller */
|
|
|
9bf39b3 |
+ rcu_idle_enter();
|
|
|
9bf39b3 |
cpu_relax();
|
|
|
9bf39b3 |
+ rcu_idle_exit();
|
|
|
9bf39b3 |
}
|
|
|
9bf39b3 |
}
|
|
|
9bf39b3 |
#ifdef CONFIG_APM_MODULE
|
|
|
9bf39b3 |
@@ -457,14 +461,19 @@ static void mwait_idle(void)
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
__monitor((void *)¤t_thread_info()->flags, 0, 0);
|
|
|
9bf39b3 |
smp_mb();
|
|
|
9bf39b3 |
+ rcu_idle_enter();
|
|
|
9bf39b3 |
if (!need_resched())
|
|
|
9bf39b3 |
__sti_mwait(0, 0);
|
|
|
9bf39b3 |
else
|
|
|
9bf39b3 |
local_irq_enable();
|
|
|
9bf39b3 |
+ rcu_idle_exit();
|
|
|
9bf39b3 |
trace_power_end(smp_processor_id());
|
|
|
9bf39b3 |
trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
|
|
|
9bf39b3 |
- } else
|
|
|
9bf39b3 |
+ } else {
|
|
|
9bf39b3 |
local_irq_enable();
|
|
|
9bf39b3 |
+ rcu_idle_enter();
|
|
|
9bf39b3 |
+ rcu_idle_exit();
|
|
|
9bf39b3 |
+ }
|
|
|
9bf39b3 |
}
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
/*
|
|
|
9bf39b3 |
@@ -477,8 +486,10 @@ static void poll_idle(void)
|
|
|
9bf39b3 |
trace_power_start(POWER_CSTATE, 0, smp_processor_id());
|
|
|
9bf39b3 |
trace_cpu_idle(0, smp_processor_id());
|
|
|
9bf39b3 |
local_irq_enable();
|
|
|
9bf39b3 |
+ rcu_idle_enter();
|
|
|
9bf39b3 |
while (!need_resched())
|
|
|
9bf39b3 |
cpu_relax();
|
|
|
9bf39b3 |
+ rcu_idle_exit();
|
|
|
9bf39b3 |
trace_power_end(smp_processor_id());
|
|
|
9bf39b3 |
trace_cpu_idle(PWR_EVENT_EXIT, smp_processor_id());
|
|
|
9bf39b3 |
}
|
|
|
9bf39b3 |
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
|
|
|
9bf39b3 |
index 485204f..6d9d4d5 100644
|
|
|
9bf39b3 |
--- a/arch/x86/kernel/process_32.c
|
|
|
9bf39b3 |
+++ b/arch/x86/kernel/process_32.c
|
|
|
9bf39b3 |
@@ -100,7 +100,6 @@ void cpu_idle(void)
|
|
|
9bf39b3 |
/* endless idle loop with no priority at all */
|
|
|
9bf39b3 |
while (1) {
|
|
|
9bf39b3 |
tick_nohz_idle_enter();
|
|
|
9bf39b3 |
- rcu_idle_enter();
|
|
|
9bf39b3 |
while (!need_resched()) {
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
check_pgt_cache();
|
|
|
9bf39b3 |
@@ -117,7 +116,6 @@ void cpu_idle(void)
|
|
|
9bf39b3 |
pm_idle();
|
|
|
9bf39b3 |
start_critical_timings();
|
|
|
9bf39b3 |
}
|
|
|
9bf39b3 |
- rcu_idle_exit();
|
|
|
9bf39b3 |
tick_nohz_idle_exit();
|
|
|
9bf39b3 |
preempt_enable_no_resched();
|
|
|
9bf39b3 |
schedule();
|
|
|
9bf39b3 |
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
|
|
|
9bf39b3 |
index 9b9fe4a..55a1a35 100644
|
|
|
9bf39b3 |
--- a/arch/x86/kernel/process_64.c
|
|
|
9bf39b3 |
+++ b/arch/x86/kernel/process_64.c
|
|
|
9bf39b3 |
@@ -140,13 +140,9 @@ void cpu_idle(void)
|
|
|
9bf39b3 |
/* Don't trace irqs off for idle */
|
|
|
9bf39b3 |
stop_critical_timings();
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
- /* enter_idle() needs rcu for notifiers */
|
|
|
9bf39b3 |
- rcu_idle_enter();
|
|
|
9bf39b3 |
-
|
|
|
9bf39b3 |
if (cpuidle_idle_call())
|
|
|
9bf39b3 |
pm_idle();
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
- rcu_idle_exit();
|
|
|
9bf39b3 |
start_critical_timings();
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
/* In many cases the interrupt that ended idle
|
|
|
9bf39b3 |
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
|
|
|
9bf39b3 |
index 20bce51..a9ddab8 100644
|
|
|
9bf39b3 |
--- a/drivers/idle/intel_idle.c
|
|
|
9bf39b3 |
+++ b/drivers/idle/intel_idle.c
|
|
|
9bf39b3 |
@@ -261,6 +261,7 @@ static int intel_idle(struct cpuidle_device *dev,
|
|
|
9bf39b3 |
kt_before = ktime_get_real();
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
stop_critical_timings();
|
|
|
9bf39b3 |
+ rcu_idle_enter();
|
|
|
9bf39b3 |
if (!need_resched()) {
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
__monitor((void *)¤t_thread_info()->flags, 0, 0);
|
|
|
9bf39b3 |
@@ -268,6 +269,7 @@ static int intel_idle(struct cpuidle_device *dev,
|
|
|
9bf39b3 |
if (!need_resched())
|
|
|
9bf39b3 |
__mwait(eax, ecx);
|
|
|
9bf39b3 |
}
|
|
|
9bf39b3 |
+ rcu_idle_exit();
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
start_critical_timings();
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
|
|
|
9bf39b3 |
|