-----Original Message-----
From: acrn-dev@... <acrn-dev@...> On
Behalf Of Minggui Cao
Sent: Tuesday, August 23, 2022 6:08 AM
To: acrn-dev@...
Cc: Cao, Minggui <minggui.cao@...>
Subject: [acrn-dev] [RFC][PATCH 1/2] hv: unify kicking off vCPU by INIT signal

INIT signal has been used to kick off the partitioned vCPU, like RTVM, whose
LAPIC is pass-through. It also can be used to kick off sharing vCPU instead of IPI.
This commit is to unify the usage.

Signed-off-by: Minggui Cao <minggui.cao@...>
---
hypervisor/arch/x86/cpu.c | 2 +-
hypervisor/arch/x86/guest/vcpu.c | 9 ++-------
hypervisor/arch/x86/guest/vm.c | 2 +-
hypervisor/arch/x86/guest/vmcs.c | 6 ------
hypervisor/common/sched_bvt.c | 4 ++--
hypervisor/common/sched_iorr.c | 2 +-
hypervisor/common/schedule.c | 9 ++-------
hypervisor/include/common/schedule.h | 2 --
8 files changed, 9 insertions(+), 27 deletions(-)

diff --git a/hypervisor/arch/x86/cpu.c b/hypervisor/arch/x86/cpu.c index
0b51fb9c8..927c50218 100644
--- a/hypervisor/arch/x86/cpu.c
+++ b/hypervisor/arch/x86/cpu.c
@@ -415,7 +415,7 @@ void make_pcpu_offline(uint16_t pcpu_id) {
bitmap_set_lock(NEED_OFFLINE, &per_cpu(pcpu_flag, pcpu_id));
if (get_pcpu_id() != pcpu_id) {
- send_single_ipi(pcpu_id, NOTIFY_VCPU_VECTOR);
+ send_single_init(pcpu_id);
}
}

diff --git a/hypervisor/arch/x86/guest/vcpu.c
b/hypervisor/arch/x86/guest/vcpu.c
index e07f0dafb..23785a97f 100755
--- a/hypervisor/arch/x86/guest/vcpu.c
+++ b/hypervisor/arch/x86/guest/vcpu.c
@@ -256,7 +256,7 @@ static void vcpu_reset_internal(struct acrn_vcpu *vcpu,
enum reset_mode mode)
}

/* TODO: we may need to add one scheduler->reset_data to reset the
thread_obj */
- vcpu->thread_obj.notify_mode = SCHED_NOTIFY_IPI;
+ vcpu->thread_obj.notify_mode = SCHED_NOTIFY_INIT;

vlapic = vcpu_vlapic(vcpu);
vlapic_reset(vlapic, apicv_ops, mode); @@ -792,12 +792,7 @@ void
kick_vcpu(struct acrn_vcpu *vcpu)

if ((get_pcpu_id() != pcpu_id) &&
(per_cpu(vmcs_run, pcpu_id) == vcpu->arch.vmcs)) {
- if (is_lapic_pt_enabled(vcpu)) {
- /* For lapic-pt vCPUs */
- send_single_init(pcpu_id);
- } else {
- send_single_ipi(pcpu_id, NOTIFY_VCPU_VECTOR);
- }
+ send_single_init(pcpu_id);
}
}

diff --git a/hypervisor/arch/x86/guest/vm.c b/hypervisor/arch/x86/guest/vm.c
index bdd31d3fa..300152fe3 100644
--- a/hypervisor/arch/x86/guest/vm.c
+++ b/hypervisor/arch/x86/guest/vm.c
@@ -1245,7 +1245,7 @@ void make_shutdown_vm_request(uint16_t
pcpu_id) {
bitmap_set_lock(NEED_SHUTDOWN_VM, &per_cpu(pcpu_flag,
pcpu_id));
if (get_pcpu_id() != pcpu_id) {
- send_single_ipi(pcpu_id, NOTIFY_VCPU_VECTOR);
+ send_single_init(pcpu_id);
}
}

diff --git a/hypervisor/arch/x86/guest/vmcs.c
b/hypervisor/arch/x86/guest/vmcs.c
index 684507d7d..cc1ad7297 100644
--- a/hypervisor/arch/x86/guest/vmcs.c
+++ b/hypervisor/arch/x86/guest/vmcs.c
@@ -629,12 +629,6 @@ void switch_apicv_mode_x2apic(struct acrn_vcpu
*vcpu)

update_msr_bitmap_x2apic_passthru(vcpu);

- /*
- * After passthroughing lapic to guest, we should use INIT
signal to
- * notify vcpu thread instead of IPI. Because the IPI will be
delivered
- * the guest directly without vmexit.
- */
- vcpu->thread_obj.notify_mode = SCHED_NOTIFY_INIT;
} else {
value32 =
exec_vmread32(VMX_PROC_VM_EXEC_CONTROLS2);
value32 &= ~VMX_PROCBASED_CTLS2_VAPIC; diff --git
a/hypervisor/common/sched_bvt.c b/hypervisor/common/sched_bvt.c index
56470953a..704c9292d 100644
--- a/hypervisor/common/sched_bvt.c
+++ b/hypervisor/common/sched_bvt.c
@@ -139,11 +139,11 @@ static void sched_tick_handler(void *param)
if (!is_idle_thread(current)) {
data->run_countdown -= 1U;
if (data->run_countdown == 0U) {
- make_reschedule_request(pcpu_id,
DEL_MODE_IPI);
+ make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
}
} else {
if (!list_empty(&bvt_ctl->runqueue)) {
- make_reschedule_request(pcpu_id,
DEL_MODE_IPI);
+ make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
}
}
}
diff --git a/hypervisor/common/sched_iorr.c
b/hypervisor/common/sched_iorr.c index 538b9ffec..f0e2ca29a 100644
--- a/hypervisor/common/sched_iorr.c
+++ b/hypervisor/common/sched_iorr.c
@@ -94,7 +94,7 @@ static void sched_tick_handler(void *param)
}
/* make reschedule request if current ran out of its
cycles */
if (is_idle_thread(current) || data->left_cycles <= 0) {
- make_reschedule_request(pcpu_id,
DEL_MODE_IPI);
+ make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
}
}
}
diff --git a/hypervisor/common/schedule.c b/hypervisor/common/schedule.c
index e0a123247..ac11a5355 100644
--- a/hypervisor/common/schedule.c
+++ b/hypervisor/common/schedule.c
@@ -134,9 +134,6 @@ void make_reschedule_request(uint16_t pcpu_id,
uint16_t delmode)
bitmap_set_lock(NEED_RESCHEDULE, &ctl->flags);
if (get_pcpu_id() != pcpu_id) {
switch (delmode) {
- case DEL_MODE_IPI:
- send_single_ipi(pcpu_id, NOTIFY_VCPU_VECTOR);
- break;
case DEL_MODE_INIT:
send_single_init(pcpu_id);
break;
@@ -204,8 +201,6 @@ void sleep_thread(struct thread_object *obj)
if (is_running(obj)) {
if (obj->notify_mode == SCHED_NOTIFY_INIT) {
make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
- } else {
- make_reschedule_request(pcpu_id, DEL_MODE_IPI);
}
obj->be_blocking = true;
} else {
@@ -236,7 +231,7 @@ void wake_thread(struct thread_object *obj)
}
if (is_blocked(obj)) {
set_thread_status(obj, THREAD_STS_RUNNABLE);
- make_reschedule_request(pcpu_id, DEL_MODE_IPI);
+ make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
}
obj->be_blocking = false;
}
@@ -245,7 +240,7 @@ void wake_thread(struct thread_object *obj)

void yield_current(void)
{
- make_reschedule_request(get_pcpu_id(), DEL_MODE_IPI);
+ make_reschedule_request(get_pcpu_id(), DEL_MODE_INIT);
}

void run_thread(struct thread_object *obj) diff --git
a/hypervisor/include/common/schedule.h
b/hypervisor/include/common/schedule.h
index abac7b8bb..1d7436969 100644
--- a/hypervisor/include/common/schedule.h
+++ b/hypervisor/include/common/schedule.h
@@ -13,7 +13,6 @@
#define NEED_RESCHEDULE (1U)

#define DEL_MODE_INIT (1U)
-#define DEL_MODE_IPI (2U)

#define THREAD_DATA_SIZE (256U)

@@ -25,7 +24,6 @@ enum thread_object_state {

enum sched_notify_mode {
SCHED_NOTIFY_INIT,
- SCHED_NOTIFY_IPI
};

/* Tools can configure a VM to use PRIO_LOW or PRIO_HIGH */
--
2.25.1

-----Original Message-----
From: Dong, Eddie <eddie.dong@...>
Sent: Thursday, August 25, 2022 7:35 AM
To: acrn-dev@...
Cc: Cao, Minggui <minggui.cao@...>
Subject: RE: [acrn-dev] [RFC][PATCH 1/2] hv: unify kicking off vCPU by INIT
signal

Hi Minggui:
Did u see any latency difference with INIT signal?

BTW, do we test the guest issued normal INIT/SIPI usage? Make
sure it work well, though not directly related to this patch.

Thx Eddie

-----Original Message-----
From: acrn-dev@... <acrn-dev@...>
On Behalf Of Minggui Cao
Sent: Tuesday, August 23, 2022 6:08 AM
To: acrn-dev@...
Cc: Cao, Minggui <minggui.cao@...>
Subject: [acrn-dev] [RFC][PATCH 1/2] hv: unify kicking off vCPU by
INIT signal

INIT signal has been used to kick off the partitioned vCPU, like
RTVM, whose LAPIC is pass-through. It also can be used to kick off sharing

vCPU instead of IPI.

This commit is to unify the usage.

Signed-off-by: Minggui Cao <minggui.cao@...>
---
hypervisor/arch/x86/cpu.c | 2 +-
hypervisor/arch/x86/guest/vcpu.c | 9 ++-------
hypervisor/arch/x86/guest/vm.c | 2 +-
hypervisor/arch/x86/guest/vmcs.c | 6 ------
hypervisor/common/sched_bvt.c | 4 ++--
hypervisor/common/sched_iorr.c | 2 +-
hypervisor/common/schedule.c | 9 ++-------
hypervisor/include/common/schedule.h | 2 --
8 files changed, 9 insertions(+), 27 deletions(-)

diff --git a/hypervisor/arch/x86/cpu.c b/hypervisor/arch/x86/cpu.c
index
0b51fb9c8..927c50218 100644
--- a/hypervisor/arch/x86/cpu.c
+++ b/hypervisor/arch/x86/cpu.c
@@ -415,7 +415,7 @@ void make_pcpu_offline(uint16_t pcpu_id) {
bitmap_set_lock(NEED_OFFLINE, &per_cpu(pcpu_flag, pcpu_id));
if (get_pcpu_id() != pcpu_id) {
- send_single_ipi(pcpu_id, NOTIFY_VCPU_VECTOR);
+ send_single_init(pcpu_id);
}
}

diff --git a/hypervisor/arch/x86/guest/vcpu.c
b/hypervisor/arch/x86/guest/vcpu.c
index e07f0dafb..23785a97f 100755
--- a/hypervisor/arch/x86/guest/vcpu.c
+++ b/hypervisor/arch/x86/guest/vcpu.c
@@ -256,7 +256,7 @@ static void vcpu_reset_internal(struct acrn_vcpu
*vcpu, enum reset_mode mode)
}

/* TODO: we may need to add one scheduler->reset_data to reset

the

thread_obj */
- vcpu->thread_obj.notify_mode = SCHED_NOTIFY_IPI;
+ vcpu->thread_obj.notify_mode = SCHED_NOTIFY_INIT;

vlapic = vcpu_vlapic(vcpu);
vlapic_reset(vlapic, apicv_ops, mode); @@ -792,12 +792,7 @@ void
kick_vcpu(struct acrn_vcpu *vcpu)

if ((get_pcpu_id() != pcpu_id) &&
(per_cpu(vmcs_run, pcpu_id) == vcpu->arch.vmcs)) {
- if (is_lapic_pt_enabled(vcpu)) {
- /* For lapic-pt vCPUs */
- send_single_init(pcpu_id);
- } else {
- send_single_ipi(pcpu_id, NOTIFY_VCPU_VECTOR);
- }
+ send_single_init(pcpu_id);
}
}

diff --git a/hypervisor/arch/x86/guest/vm.c
b/hypervisor/arch/x86/guest/vm.c index bdd31d3fa..300152fe3 100644
--- a/hypervisor/arch/x86/guest/vm.c
+++ b/hypervisor/arch/x86/guest/vm.c
@@ -1245,7 +1245,7 @@ void make_shutdown_vm_request(uint16_t
pcpu_id) {
bitmap_set_lock(NEED_SHUTDOWN_VM, &per_cpu(pcpu_flag,

pcpu_id));

if (get_pcpu_id() != pcpu_id) {
- send_single_ipi(pcpu_id, NOTIFY_VCPU_VECTOR);
+ send_single_init(pcpu_id);
}
}

diff --git a/hypervisor/arch/x86/guest/vmcs.c
b/hypervisor/arch/x86/guest/vmcs.c
index 684507d7d..cc1ad7297 100644
--- a/hypervisor/arch/x86/guest/vmcs.c
+++ b/hypervisor/arch/x86/guest/vmcs.c
@@ -629,12 +629,6 @@ void switch_apicv_mode_x2apic(struct acrn_vcpu
*vcpu)

update_msr_bitmap_x2apic_passthru(vcpu);

- /*
- * After passthroughing lapic to guest, we should use INIT
signal to
- * notify vcpu thread instead of IPI. Because the IPI will be
delivered
- * the guest directly without vmexit.
- */
- vcpu->thread_obj.notify_mode = SCHED_NOTIFY_INIT;
} else {
value32 =
exec_vmread32(VMX_PROC_VM_EXEC_CONTROLS2);
value32 &= ~VMX_PROCBASED_CTLS2_VAPIC; diff --git
a/hypervisor/common/sched_bvt.c b/hypervisor/common/sched_bvt.c

index

56470953a..704c9292d 100644
--- a/hypervisor/common/sched_bvt.c
+++ b/hypervisor/common/sched_bvt.c
@@ -139,11 +139,11 @@ static void sched_tick_handler(void *param)
if (!is_idle_thread(current)) {
data->run_countdown -= 1U;
if (data->run_countdown == 0U) {
- make_reschedule_request(pcpu_id,
DEL_MODE_IPI);
+ make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
}
} else {
if (!list_empty(&bvt_ctl->runqueue)) {
- make_reschedule_request(pcpu_id,
DEL_MODE_IPI);
+ make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
}
}
}
diff --git a/hypervisor/common/sched_iorr.c
b/hypervisor/common/sched_iorr.c index 538b9ffec..f0e2ca29a 100644
--- a/hypervisor/common/sched_iorr.c
+++ b/hypervisor/common/sched_iorr.c
@@ -94,7 +94,7 @@ static void sched_tick_handler(void *param)
}
/* make reschedule request if current ran out of its

cycles */

if (is_idle_thread(current) || data->left_cycles <= 0) {
- make_reschedule_request(pcpu_id,
DEL_MODE_IPI);
+ make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
}
}
}
diff --git a/hypervisor/common/schedule.c
b/hypervisor/common/schedule.c index e0a123247..ac11a5355 100644
--- a/hypervisor/common/schedule.c
+++ b/hypervisor/common/schedule.c
@@ -134,9 +134,6 @@ void make_reschedule_request(uint16_t pcpu_id,
uint16_t delmode)
bitmap_set_lock(NEED_RESCHEDULE, &ctl->flags);
if (get_pcpu_id() != pcpu_id) {
switch (delmode) {
- case DEL_MODE_IPI:
- send_single_ipi(pcpu_id, NOTIFY_VCPU_VECTOR);
- break;
case DEL_MODE_INIT:
send_single_init(pcpu_id);
break;
@@ -204,8 +201,6 @@ void sleep_thread(struct thread_object *obj)
if (is_running(obj)) {
if (obj->notify_mode == SCHED_NOTIFY_INIT) {
make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
- } else {
- make_reschedule_request(pcpu_id, DEL_MODE_IPI);
}
obj->be_blocking = true;
} else {
@@ -236,7 +231,7 @@ void wake_thread(struct thread_object *obj)
}
if (is_blocked(obj)) {
set_thread_status(obj, THREAD_STS_RUNNABLE);
- make_reschedule_request(pcpu_id, DEL_MODE_IPI);
+ make_reschedule_request(pcpu_id,
DEL_MODE_INIT);
}
obj->be_blocking = false;
}
@@ -245,7 +240,7 @@ void wake_thread(struct thread_object *obj)

void yield_current(void)
{
- make_reschedule_request(get_pcpu_id(), DEL_MODE_IPI);
+ make_reschedule_request(get_pcpu_id(), DEL_MODE_INIT);
}

void run_thread(struct thread_object *obj) diff --git
a/hypervisor/include/common/schedule.h
b/hypervisor/include/common/schedule.h
index abac7b8bb..1d7436969 100644
--- a/hypervisor/include/common/schedule.h
+++ b/hypervisor/include/common/schedule.h
@@ -13,7 +13,6 @@
#define NEED_RESCHEDULE (1U)

#define DEL_MODE_INIT (1U)
-#define DEL_MODE_IPI (2U)

#define THREAD_DATA_SIZE (256U)

@@ -25,7 +24,6 @@ enum thread_object_state {

enum sched_notify_mode {
SCHED_NOTIFY_INIT,
- SCHED_NOTIFY_IPI
};

/* Tools can configure a VM to use PRIO_LOW or PRIO_HIGH */
--
2.25.1

-----Original Message-----
From: Dong, Eddie <eddie.dong@...>
Sent: Friday, August 26, 2022 12:15 AM
To: Cao, Minggui <minggui.cao@...>
Cc: Mao, Junjie <junjie.mao@...>; acrn-dev@...
Subject: RE: [acrn-dev] [RFC][PATCH 1/2] hv: unify kicking off vCPU by INIT
signal

-----Original Message-----
From: Dong, Eddie <eddie.dong@...>
Sent: Thursday, August 25, 2022 7:35 AM
To: acrn-dev@...
Cc: Cao, Minggui <minggui.cao@...>
Subject: RE: [acrn-dev] [RFC][PATCH 1/2] hv: unify kicking off vCPU
by INIT signal

Hi Minggui:
Did u see any latency difference with INIT signal?

[Cao, Minggui] No. the latency of INIT or IPI to kick off vCPU could
be similar (both are from LAPIC). If needed, I'll think how to calculate the

latency.

Because both INIT and IPI are sent to another core, the
beginning TSC is from one core, and end TSC are from target core.

This is fine. As if the delta between them using INIT vs. that of using IPI is
close to be same, it is fine.

BTW, do we test the guest issued normal INIT/SIPI usage? Make

sure

it work well, though not directly related to this patch.

[Cao, Minggui] They shall work well. In ACRN HV, the guest VM
INIT/SIPI goes the path of VLAPIC (APIC-write or x2APIC MSR). They work

for multi SMP cores.

I mean under the situation of LAPIC passthru, the VM can still issue INIT/SIPI
signal to reboot. Can you have a test?

-----Original Message-----
From: Dong, Eddie <eddie.dong@...>
Sent: Friday, August 26, 2022 12:15 AM
To: Cao, Minggui <minggui.cao@...>
Cc: Mao, Junjie <junjie.mao@...>; acrn-dev@...
Subject: RE: [acrn-dev] [RFC][PATCH 1/2] hv: unify kicking off vCPU by
INIT signal

-----Original Message-----
From: Dong, Eddie <eddie.dong@...>
Sent: Thursday, August 25, 2022 7:35 AM
To: acrn-dev@...
Cc: Cao, Minggui <minggui.cao@...>
Subject: RE: [acrn-dev] [RFC][PATCH 1/2] hv: unify kicking off
vCPU by INIT signal

Hi Minggui:
Did u see any latency difference with INIT signal?

[Cao, Minggui] No. the latency of INIT or IPI to kick off vCPU could
be similar (both are from LAPIC). If needed, I'll think how to
calculate the

latency.

Because both INIT and IPI are sent to another core,
the beginning TSC is from one core, and end TSC are from target core.

This is fine. As if the delta between them using INIT vs. that of
using IPI is close to be same, it is fine.

BTW, do we test the guest issued normal INIT/SIPI usage? Make

sure

it work well, though not directly related to this patch.

[Cao, Minggui] They shall work well. In ACRN HV, the guest VM
INIT/SIPI goes the path of VLAPIC (APIC-write or x2APIC MSR). They
work

for multi SMP cores.

I mean under the situation of LAPIC passthru, the VM can still issue
INIT/SIPI signal to reboot. Can you have a test?