Re: [PATCH 1/2] hv: allocate cpu dynamically


Xu, Anthony
 

-----Original Message-----
From: acrn-dev@... <acrn-dev@...> On Behalf Of Zhao, Yuanyuan
Sent: Wednesday, February 23, 2022 7:14 PM
To: acrn-dev@...
Cc: Wang, Yu1 <yu1.wang@...>; yuanyuan.zhao@...
Subject: [acrn-dev] [PATCH 1/2] hv: allocate cpu dynamically

Allocate vCPU dynamically for post launch stardard vm.

The CPUs which belong to service OS and will not asign to rtvm
are within the selection. The CPU with the fewest user will
be selected.

Signed-off-by: Yuanyuan Zhao <yuanyuan.zhao@...>
---
hypervisor/arch/x86/configs/vm_config.c | 19 +++++
hypervisor/arch/x86/guest/vm.c | 4 +
hypervisor/arch/x86/guest/vmcall.c | 79 ++++++++++++++++++
hypervisor/common/hypercall.c | 91 +++++++++++++--------
hypervisor/include/arch/x86/asm/guest/vm.h | 4 +-
hypervisor/include/arch/x86/asm/vm_config.h | 2 +
6 files changed, 162 insertions(+), 37 deletions(-)

diff --git a/hypervisor/arch/x86/configs/vm_config.c b/hypervisor/arch/x86/configs/vm_config.c
index be947309c..f4a5ec992 100644
--- a/hypervisor/arch/x86/configs/vm_config.c
+++ b/hypervisor/arch/x86/configs/vm_config.c
@@ -5,9 +5,28 @@
*/

#include <asm/vm_config.h>
+#include <asm/lib/bits.h>
#include <util.h>
#include <rtl.h>

+static uint64_t rtvm_cpu_bitmap = 0UL;
+
+void update_rtvm_cpu_bitmap()
+{
+ uint32_t i;
+
+ for (i = 0; i < CONFIG_MAX_VM_NUM; i++) {
+ if (vm_configs[i].severity == SEVERITY_RTVM) {
+ rtvm_cpu_bitmap |= vm_configs[i].cpu_affinity;
+ }
+ }
+}
+
+bool is_rtvm_cpu(uint16_t cpu)
+{
+ return bitmap_test(cpu, &rtvm_cpu_bitmap);
+}
+
/*
* @pre vm_id < CONFIG_MAX_VM_NUM
* @post return != NULL
diff --git a/hypervisor/arch/x86/guest/vm.c b/hypervisor/arch/x86/guest/vm.c
index b59f7e510..16a08ef42 100644
--- a/hypervisor/arch/x86/guest/vm.c
+++ b/hypervisor/arch/x86/guest/vm.c
@@ -753,6 +753,10 @@ int32_t create_vm(uint16_t vm_id, uint64_t pcpu_bitmap, struct acrn_vm_config *v
}
}

+ if (status == 0 && is_service_vm(vm)) {
+ update_rtvm_cpu_bitmap();
+ }
+
if ((status != 0) && (vm->arch_vm.nworld_eptp != NULL)) {
(void)memset(vm->arch_vm.nworld_eptp, 0U, PAGE_SIZE);
}
diff --git a/hypervisor/arch/x86/guest/vmcall.c b/hypervisor/arch/x86/guest/vmcall.c
index 04cb7d368..37b53ea38 100644
--- a/hypervisor/arch/x86/guest/vmcall.c
+++ b/hypervisor/arch/x86/guest/vmcall.c
@@ -111,11 +111,90 @@ static const struct hc_dispatch hc_dispatch_table[] = {
.permission_flags = (GUEST_FLAG_TEE | GUEST_FLAG_REE)},
};

+uint64_t get_allocatable_cpu()
+{
+ uint64_t bitmap = 0;
+ uint16_t i, service_vm_id;
+
+ service_vm_id = (get_service_vm())->vm_id;
+
+ for (i = 0; i < MAX_PCPU_NUM; i++) {
+ if (is_rtvm_cpu(i) || per_cpu(vcpu_array, i)[service_vm_id] == NULL) {
+ continue;
+ }
+ bitmap_set_nolock(i, &bitmap);
+ }
+
+ return bitmap;
+}
+
+static bool allocate_dynamical_cpus(uint16_t vcpu_num, uint64_t *cpu_affinity)
+{
+ uint8_t pcpu_occupancy[MAX_PCPU_NUM] = {0xff};
+ uint8_t count;
+ uint16_t i, j, cpu;
+ uint64_t affinity = 0, allocatable_cpu;
+
+ allocatable_cpu = get_allocatable_cpu();
+
+ for (i = 0; i < MAX_PCPU_NUM; i++) {
+ if (!bitmap_test(i, &allocatable_cpu)) {
+ pcpu_occupancy[i] = 0xff;
+ continue;
+ }
+
+ count = 0;
+ for (j = 0; j < CONFIG_MAX_VM_NUM; j++) {
+ if(per_cpu(vcpu_array, i)[j] == NULL) {
+ continue;
+ }
+ count++;
+ }
+
+ pcpu_occupancy[i] = count;
+ }
Make pcpu_occupancy global variable, update pcpu_occupancy when start/stop VM,
then we don't need to calculate pcpu_occupancy from scratch every time.



+
+ /* Find out the cpu which have the lest user by bubble algorithm */
+ for (i = 0; i < vcpu_num; i++) {
+ cpu = 0xff;
+ for(j = 0; j < MAX_PCPU_NUM; j++) {
+ if (pcpu_occupancy[j] == 0xff) {
+ continue;
+ }
+
+ if (cpu == 0xff || pcpu_occupancy[cpu] > pcpu_occupancy[j]) {
+ cpu = j;
+ }
+ }
+
+ if (cpu != 0xff) {
+ bitmap_set_nolock(cpu, &affinity);
+ pcpu_occupancy[cpu] = 0xff;
+ } else {
+ pr_err("no enough cpu\n");
+ return false;
+ }
+ }
+
+ *cpu_affinity = affinity;
+ return true;
+}
+
uint16_t allocate_dynamical_vmid(struct acrn_vm_creation *cv)
{
uint16_t vm_id;
struct acrn_vm_config *vm_config;

+ if ((cv->vm_flag & GUEST_FLAG_RT) != 0) {
+ pr_err("RTVM (%s) cann't create dynamically.", cv->name);
+ return ACRN_INVALID_VMID;
+ }
+ if(cv->cpu_affinity == 0) {
+ if (!allocate_dynamical_cpus(cv->vcpu_num, &cv->cpu_affinity)) {
We may not want to call allocate_dynamical_cpus in allocate_dynamical_vmid.
It is hard to read.

We may want to call allocate_dynamical_cpus in vm_start not vm_create.



Anthony

+ return ACRN_INVALID_VMID;
+ }
+
+ }
spinlock_obtain(&vm_id_lock);
vm_id = get_unused_vmid();
if (vm_id != ACRN_INVALID_VMID) {
diff --git a/hypervisor/common/hypercall.c b/hypervisor/common/hypercall.c
index 4c9062c18..1a5d40275 100644
--- a/hypervisor/common/hypercall.c
+++ b/hypervisor/common/hypercall.c
@@ -205,51 +205,70 @@ int32_t hcall_create_vm(struct acrn_vcpu *vcpu, struct acrn_vm *target_vm, uint6
struct acrn_vm *tgt_vm = NULL;
struct acrn_vm_creation cv;
struct acrn_vm_config *vm_config = get_vm_config(vmid);
+
+ ret = copy_from_gpa(vm, &cv, param1, sizeof(cv));
+ if (ret == 0) {
+ uint64_t pcpu_bitmap = vm_config->cpu_affinity;
+ if (!is_poweroff_vm(get_vm_from_vmid(vmid))) {
+ ret = -1;
+ goto exit;
+ }

- if (copy_from_gpa(vm, &cv, param1, sizeof(cv)) == 0) {
- if (is_poweroff_vm(get_vm_from_vmid(vmid))) {
-
- /* Filter out the bits should not set by DM and then assign it to guest_flags */
- vm_config->guest_flags &= ~DM_OWNED_GUEST_FLAG_MASK;
- vm_config->guest_flags |= (cv.vm_flag & DM_OWNED_GUEST_FLAG_MASK);
-
- /* post-launched VM is allowed to choose pCPUs from vm_config->cpu_affinity only */
- if ((cv.cpu_affinity & ~(vm_config->cpu_affinity)) == 0UL) {
- /* By default launch VM with all the configured pCPUs */
- uint64_t pcpu_bitmap = vm_config->cpu_affinity;
-
- if (cv.cpu_affinity != 0UL) {
- /* overwrite the statically configured CPU affinity */
- pcpu_bitmap = cv.cpu_affinity;
- }
+ /* Filter out the bits should not set by DM and then assign it to guest_flags */
+ vm_config->guest_flags &= ~DM_OWNED_GUEST_FLAG_MASK;
+ vm_config->guest_flags |= (cv.vm_flag & DM_OWNED_GUEST_FLAG_MASK);
+
+ /*
+ * GUEST_FLAG_RT must be set if we have GUEST_FLAG_LAPIC_PASSTHROUGH
+ * set in guest_flags
+ */
+ if (((vm_config->guest_flags & GUEST_FLAG_LAPIC_PASSTHROUGH) != 0UL)
+ && ((vm_config->guest_flags & GUEST_FLAG_RT) == 0UL)) {
+ pr_err("Wrong guest flags 0x%lx\n", vm_config->guest_flags);
+ ret = -1;
+ goto exit;
+ }

- /*
- * GUEST_FLAG_RT must be set if we have GUEST_FLAG_LAPIC_PASSTHROUGH
- * set in guest_flags
+ if(cv.cpu_affinity == 0) {
+ if(bitmap_weight(pcpu_bitmap) == cv.vcpu_num) {
+ /* Set cpu affinity of static config to acrn_vm_creation
+ * to feedback config info to user.
*/
- if (((vm_config->guest_flags & GUEST_FLAG_LAPIC_PASSTHROUGH) != 0UL)
- && ((vm_config->guest_flags & GUEST_FLAG_RT) == 0UL)) {
- pr_err("Wrong guest flags 0x%lx\n", vm_config->guest_flags);
- } else {
- if (create_vm(vmid, pcpu_bitmap, vm_config, &tgt_vm) == 0) {
- /* return a relative vm_id from Service VM view */
- cv.vmid = vmid_2_rel_vmid(vm->vm_id, vmid);
- cv.vcpu_num = tgt_vm->hw.created_vcpus;
- } else {
- dev_dbg(DBG_LEVEL_HYCALL, "HCALL: Create VM failed");
- cv.vmid = ACRN_INVALID_VMID;
- }
-
- ret = copy_to_gpa(vm, &cv, param1, sizeof(cv));
- }
+ cv.cpu_affinity = pcpu_bitmap;
} else {
- pr_err("Post-launched VM%u chooses invalid pCPUs(0x%llx).",
- vmid, cv.cpu_affinity);
+ pr_err("vCPU num(%d) does not match CPU affinity 0x%llx.",
+ cv.vcpu_num, pcpu_bitmap);
+ ret = -1;
+ goto exit;
}
+ } else if (cv.cpu_affinity != pcpu_bitmap) {
+ pr_err("DM CPU affinity(0x%llx) does not match static CPU affinity 0x%llx.",
+ cv.cpu_affinity, pcpu_bitmap);
+ ret = -1;
+ goto exit;
+ }
+
+ if ((vm_config->guest_flags & GUEST_FLAG_RT) == 0UL
+ && (pcpu_bitmap & ~(get_allocatable_cpu())) != 0UL) {
+ pr_err("Post-launched VM%u chooses invalid pCPUs(0x%llx).",
+ vmid, cv.cpu_affinity);
+ ret = -1;
+ goto exit;
+ }
+
+ if (create_vm(vmid, pcpu_bitmap, vm_config, &tgt_vm) == 0) {
+ /* return a relative vm_id from Service VM view */
+ cv.vmid = vmid_2_rel_vmid(vm->vm_id, vmid);
+ cv.vcpu_num = tgt_vm->hw.created_vcpus;
+ } else {
+ dev_dbg(DBG_LEVEL_HYCALL, "HCALL: Create VM failed");
+ cv.vmid = ACRN_INVALID_VMID;
}

+ ret = copy_to_gpa(vm, &cv, param1, sizeof(cv));
}

+exit:
if (((ret != 0) || (cv.vmid == ACRN_INVALID_VMID)) && (!is_static_configured_vm(target_vm))) {
memset(vm_config->name, 0U, MAX_VM_NAME_LEN);
}
diff --git a/hypervisor/include/arch/x86/asm/guest/vm.h b/hypervisor/include/arch/x86/asm/guest/vm.h
index aa22fc783..42b542536 100644
--- a/hypervisor/include/arch/x86/asm/guest/vm.h
+++ b/hypervisor/include/arch/x86/asm/guest/vm.h
@@ -82,7 +82,7 @@ struct vm_pm_info {

/* Enumerated type for VM states */
enum vm_state {
- VM_POWERED_OFF = 0, /* MUST set 0 because vm_state's initialization depends on clear BSS section */
+ VM_POWERED_OFF = 0, /* Must set 0 because vm_state's initialization depends on clear BSS section */
VM_CREATED, /* VM created / awaiting start (boot) */
VM_RUNNING, /* VM running */
VM_READY_TO_POWEROFF, /* RTVM only, it is trying to poweroff by itself */
@@ -254,6 +254,8 @@ int32_t prepare_os_image(struct acrn_vm *vm);

void vrtc_init(struct acrn_vm *vm);

+uint64_t get_allocatable_cpu(void);
+
bool is_lapic_pt_configured(const struct acrn_vm *vm);
bool is_rt_vm(const struct acrn_vm *vm);
bool is_stateful_vm(const struct acrn_vm *vm);
diff --git a/hypervisor/include/arch/x86/asm/vm_config.h b/hypervisor/include/arch/x86/asm/vm_config.h
index 102e6ac60..26b7e6861 100644
--- a/hypervisor/include/arch/x86/asm/vm_config.h
+++ b/hypervisor/include/arch/x86/asm/vm_config.h
@@ -210,6 +210,8 @@ struct acrn_vm_config {
struct acrn_vm_config *get_vm_config(uint16_t vm_id);
uint8_t get_vm_severity(uint16_t vm_id);
bool vm_has_matched_name(uint16_t vmid, const char *name);
+void update_rtvm_cpu_bitmap();
+bool is_rtvm_cpu(uint16_t cpu);

extern struct acrn_vm_config vm_configs[CONFIG_MAX_VM_NUM];

--
2.25.1




Join acrn-dev@lists.projectacrn.org to automatically receive all group messages.