diff mbox series

[v1,5/6] core/cpu: make cpu idle states simpler

Message ID 20210811054851.861482-6-npiggin@gmail.com
State Superseded
Headers show
Series idle synchronisation improvements | expand

Commit Message

Nicholas Piggin Aug. 11, 2021, 5:48 a.m. UTC 
in_idle is true for any kind of idle. This is not used anywhere except
for state assertions, but it could be used to remove the idle_lock and
global counter if that becomes a significant cost.

in_sleep is true for sleep that requires an IPI to wake up.

in_job_sleep is true for sleep that wants an IPI sent after a job is
queued, and implies in_sleep.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
 core/cpu.c    | 26 +++++++++++++++++++-------
 include/cpu.h |  3 ++-
 2 files changed, 21 insertions(+), 8 deletions(-)
diff mbox series

Patch

diff --git a/core/cpu.c b/core/cpu.c
index 0b05d28c6..2cafa4301 100644
--- a/core/cpu.c
+++ b/core/cpu.c
@@ -104,7 +104,7 @@  static void cpu_wake(struct cpu_thread *cpu)
 {
 	/* Is it idle ? If not, no need to wake */
 	sync();
-	if (cpu->in_idle)
+	if (cpu->in_job_sleep)
 		cpu_send_ipi(cpu);
 }
 
@@ -391,7 +391,8 @@  static unsigned int cpu_idle_p8(enum cpu_wake_cause wake_on)
 	/* Synchronize with wakers */
 	if (wake_on == cpu_wake_on_job) {
 		/* Mark ourselves in idle so other CPUs know to send an IPI */
-		cpu->in_idle = true;
+		cpu->in_sleep = true;
+		cpu->in_job_sleep = true;
 		sync();
 
 		/* Check for jobs again */
@@ -425,8 +426,8 @@  static unsigned int cpu_idle_p8(enum cpu_wake_cause wake_on)
 skip_sleep:
 	/* Restore */
 	sync();
-	cpu->in_idle = false;
 	cpu->in_sleep = false;
+	cpu->in_job_sleep = false;
 	reset_cpu_icp();
 
 	return vec;
@@ -447,7 +448,8 @@  static unsigned int cpu_idle_p9(enum cpu_wake_cause wake_on)
 	/* Synchronize with wakers */
 	if (wake_on == cpu_wake_on_job) {
 		/* Mark ourselves in idle so other CPUs know to send an IPI */
-		cpu->in_idle = true;
+		cpu->in_sleep = true;
+		cpu->in_job_sleep = true;
 		sync();
 
 		/* Check for jobs again */
@@ -493,8 +495,8 @@  static unsigned int cpu_idle_p9(enum cpu_wake_cause wake_on)
  skip_sleep:
 	/* Restore */
 	sync();
-	cpu->in_idle = false;
 	cpu->in_sleep = false;
+	cpu->in_job_sleep = false;
 
 	return vec;
 }
@@ -543,10 +545,15 @@  static int nr_cpus_idle = 0;
 
 static void enter_idle(void)
 {
+	struct cpu_thread *cpu = this_cpu();
+
+	assert(!cpu->in_idle);
+
 	for (;;) {
 		lock(&idle_lock);
 		if (!reconfigure_idle) {
 			nr_cpus_idle++;
+			cpu->in_idle = true;
 			break;
 		}
 		unlock(&idle_lock);
@@ -563,9 +570,14 @@  static void enter_idle(void)
 
 static void exit_idle(void)
 {
+	struct cpu_thread *cpu = this_cpu();
+
+	assert(cpu->in_idle);
+
 	lock(&idle_lock);
 	assert(nr_cpus_idle > 0);
 	nr_cpus_idle--;
+	cpu->in_idle = false;
 	unlock(&idle_lock);
 }
 
@@ -596,12 +608,12 @@  static void reconfigure_idle_start(void)
 
 	/*
 	 * Order earlier store to reconfigure_idle=true vs load from
-	 * cpu->in_sleep and cpu->in_idle.
+	 * cpu->in_sleep.
 	 */
 	sync();
 
 	for_each_available_cpu(cpu) {
-		if (cpu->in_sleep || cpu->in_idle)
+		if (cpu->in_sleep)
 			cpu_send_ipi(cpu);
 	}
 
diff --git a/include/cpu.h b/include/cpu.h
index b0c78ce62..1be5cb0d4 100644
--- a/include/cpu.h
+++ b/include/cpu.h
@@ -60,8 +60,9 @@  struct cpu_thread {
 	bool				in_poller;
 	bool				in_reinit;
 	bool				in_fast_sleep;
-	bool				in_sleep;
 	bool				in_idle;
+	bool				in_sleep;
+	bool				in_job_sleep;
 	uint32_t			hbrt_spec_wakeup; /* primary only */
 	uint64_t			save_l2_fir_action1;
 	uint64_t			current_token;