Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1441705 > unrolled thread

[PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic broadcast quirk

Back to article view | Back to linux.kernel

This discussion starts older than the indexed window; earlier articles aren't shown. The article labeled Started by below is the oldest one visible, not the original post.

Contents

  [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic broadcast quirk Radim Krčmář <rkrcmar@redhat.com> - 2016-07-12 22:20 +0200
    Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic  broadcast quirk Paolo Bonzini <pbonzini@redhat.com> - 2016-07-13 10:40 +0200
      Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic  broadcast quirk Radim Krčmář <rkrcmar@redhat.com> - 2016-07-13 17:20 +0200
        Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic  broadcast quirk Paolo Bonzini <pbonzini@redhat.com> - 2016-07-13 17:40 +0200
    Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic  broadcast quirk Paolo Bonzini <pbonzini@redhat.com> - 2016-07-13 12:20 +0200
      Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic  broadcast quirk Radim Krčmář <rkrcmar@redhat.com> - 2016-07-13 17:00 +0200

#1441705 — [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic broadcast quirk

Add KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK as a feature flag to
KVM_CAP_X2APIC_API.

The quirk made KVM interpret 0xff as a broadcast even in x2APIC mode.
The enableable capability is needed in order to support standard x2APIC and
remain backward compatible.

Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
 v3: new

 Documentation/virtual/kvm/api.txt |  5 +++++
 arch/x86/include/asm/kvm_host.h   |  1 +
 arch/x86/kvm/lapic.c              | 40 +++++++++++++++++++++++++++++----------
 arch/x86/kvm/x86.c                |  5 ++++-
 include/uapi/linux/kvm.h          |  1 +
 5 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index 10e2bf903e57..c2753ff4a499 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -3843,12 +3843,17 @@ Returns: 0 on success, -EINVAL when args[0] contains invalid features
 Valid feature flags in args[0] are
 
 #define KVM_X2APIC_API_USE_32BIT_FORMAT         (1ULL << 0)
+#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK  (1ULL << 1)
 
 Enabling KVM_X2APIC_API_USE_32BIT_FORMAT changes the behavior of
 KVM_SET_GSI_ROUTING, KVM_SIGNAL_MSI, KVM_SET_LAPIC, and KVM_GET_LAPIC, allowing
 the use of 32-bit APIC IDs.  See KVM_CAP_X2APIC_API in their respective
 sections.
 
+KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK must be enabled for x2APIC to work in
+logical mode or with more than 255 VCPUs.  A KVM quirk treats 0xff as a
+broadcast even in x2APIC mode (a consequence of allowing physical x2APIC
+without interrupt remapping).
 
 
 8. Other capabilities.
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index f13522f85a1d..25bf9d627c15 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -784,6 +784,7 @@ struct kvm_arch {
 	struct page *avic_physical_id_table_page;
 
 	bool x2apic_format;
+	bool x2apic_broadcast_quirk_disabled;
 };
 
 struct kvm_vm_stat {
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index d27a7829a4ce..46f1d1c93420 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -619,14 +619,17 @@ static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
 /* KVM APIC implementation has two quirks
  *  - dest always begins at 0 while xAPIC MDA has offset 24,
  *  - IOxAPIC messages have to be delivered (directly) to x2APIC.
+ *
+ * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.
  */
-static u32 kvm_apic_mda(unsigned int dest_id, struct kvm_lapic *source,
-                                              struct kvm_lapic *target)
+static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
+		struct kvm_lapic *source, struct kvm_lapic *target)
 {
 	bool ipi = source != NULL;
 	bool x2apic_mda = apic_x2apic_mode(ipi ? source : target);
 
-	if (!ipi && dest_id == APIC_BROADCAST && x2apic_mda)
+	if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
+	    !ipi && dest_id == APIC_BROADCAST && x2apic_mda)
 		return X2APIC_BROADCAST;
 
 	return x2apic_mda ? dest_id : SET_APIC_DEST_FIELD(dest_id);
@@ -636,7 +639,7 @@ bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 			   int short_hand, unsigned int dest, int dest_mode)
 {
 	struct kvm_lapic *target = vcpu->arch.apic;
-	u32 mda = kvm_apic_mda(dest, source, target);
+	u32 mda = kvm_apic_mda(vcpu, dest, source, target);
 
 	apic_debug("target %p, source %p, dest 0x%x, "
 		   "dest_mode 0x%x, short_hand 0x%x\n",
@@ -688,6 +691,28 @@ static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
 	}
 }
 
+static bool kvm_apic_map_need_slowpath(struct kvm *kvm, struct kvm_lapic **src,
+		struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
+{
+	if (!map)
+		return true;
+
+	if (kvm->arch.x2apic_broadcast_quirk_disabled) {
+		if ((irq->dest_id == APIC_BROADCAST &&
+				map->mode != KVM_APIC_MODE_X2APIC))
+			return true;
+		if (irq->dest_id == X2APIC_BROADCAST)
+			return true;
+	} else {
+		bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
+		if (irq->dest_id == (x2apic_ipi ?
+		                     X2APIC_BROADCAST : APIC_BROADCAST))
+			return true;
+	}
+
+	return false;
+}
+
 /* Return true if the interrupt can be handled by using *bitmap as index mask
  * for valid destinations in *dst array.
  * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
@@ -701,7 +726,6 @@ static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
 		unsigned long *bitmap)
 {
 	int i, lowest;
-	bool x2apic_ipi;
 
 	if (irq->shorthand == APIC_DEST_SELF && src) {
 		*dst = src;
@@ -710,11 +734,7 @@ static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
 	} else if (irq->shorthand)
 		return false;
 
-	x2apic_ipi = src && *src && apic_x2apic_mode(*src);
-	if (irq->dest_id == (x2apic_ipi ? X2APIC_BROADCAST : APIC_BROADCAST))
-		return false;
-
-	if (!map)
+	if (kvm_apic_map_need_slowpath(kvm, src, irq, map))
 		return false;
 
 	if (irq->dest_mode == APIC_DEST_PHYSICAL) {
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 330276315daa..4bab85f82a96 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -90,7 +90,8 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE);
 #define VM_STAT(x) offsetof(struct kvm, stat.x), KVM_STAT_VM
 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
 
-#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_FORMAT)
+#define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_FORMAT | \
+                                    KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
 
 static void update_cr8_intercept(struct kvm_vcpu *vcpu);
 static void process_nmi(struct kvm_vcpu *vcpu);
@@ -3811,6 +3812,8 @@ split_irqchip_unlock:
 
 		if (cap->args[0] & KVM_X2APIC_API_USE_32BIT_FORMAT)
 			kvm->arch.x2apic_format = true;
+		if (cap->args[0] & KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK)
+			kvm->arch.x2apic_broadcast_quirk_disabled = true;
 
 		r = 0;
 		break;
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index b6de3febabaa..13b30dfd93b2 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -1315,5 +1315,6 @@ struct kvm_assigned_msix_entry {
 };
 
 #define KVM_X2APIC_API_USE_32BIT_FORMAT         (1ULL << 0)
+#define KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK  (1ULL << 1)
 
 #endif /* __LINUX_KVM_H */
-- 
2.9.0

[toc] | [next] | [standalone]

#1442177 — Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic broadcast quirk

On 12/07/2016 22:09, Radim Krčmář wrote:
> @@ -619,14 +619,17 @@ static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
>  /* KVM APIC implementation has two quirks
>   *  - dest always begins at 0 while xAPIC MDA has offset 24,
>   *  - IOxAPIC messages have to be delivered (directly) to x2APIC.
> + *
> + * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.

--verbose version:

 /* KVM APIC implementation has two quirks
- *  - dest always begins at 0 while xAPIC MDA has offset 24,
- *  - IOxAPIC messages have to be delivered (directly) to x2APIC.
+ *  - dest always begins at 0 while xAPIC MDA has offset 24.  This is
+ *    just a quirk in the API and is not problematic.
  *
- * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.
+ *  - in-kernel IOAPIC messages have to be delivered directly to
+ *    x2APIC, because the kernel does not support interrupt remapping.
+ *    In order to support broadcast without interrupt remapping, x2APIC
+ *    rewrites the destination of non-IPI messages (and also of IPIs sent
+ *    from xAPIC-mode LAPICs) from APIC_BROADCAST to X2APIC_BROADCAST.
+ *
+ * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
+ * important when userspace wants to use x2APIC-format MSIs, because
+ * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
  */

Sounds good?

Paolo

[toc] | [prev] | [next] | [standalone]

#1442541 — Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic broadcast quirk

2016-07-13 10:38+0200, Paolo Bonzini:
> On 12/07/2016 22:09, Radim Krčmář wrote:
>> @@ -619,14 +619,17 @@ static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
>>  /* KVM APIC implementation has two quirks
>>   *  - dest always begins at 0 while xAPIC MDA has offset 24,
>>   *  - IOxAPIC messages have to be delivered (directly) to x2APIC.
>> + *
>> + * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.
> 
> --verbose version:
> 
>  /* KVM APIC implementation has two quirks
> - *  - dest always begins at 0 while xAPIC MDA has offset 24,
> - *  - IOxAPIC messages have to be delivered (directly) to x2APIC.
> + *  - dest always begins at 0 while xAPIC MDA has offset 24.  This is
> + *    just a quirk in the API and is not problematic.
>   *
> - * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.
> + *  - in-kernel IOAPIC messages have to be delivered directly to
> + *    x2APIC, because the kernel does not support interrupt remapping.
> + *    In order to support broadcast without interrupt remapping, x2APIC
> + *    rewrites the destination of non-IPI messages (and also of IPIs sent
> + *    from xAPIC-mode LAPICs) from APIC_BROADCAST to X2APIC_BROADCAST.

KVM doesn't do the operation in parentheses since 394457a928e0 ("KVM:
x86: some apic broadcast modes does not work").

Following patches added only !ipi case (I already regret posting them).

> + *
> + * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
> + * important when userspace wants to use x2APIC-format MSIs, because
> + * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
>   */
> 
> Sounds good?

Except that one bit.  (Code alone really can't explain the what happens
here.)

[toc] | [prev] | [next] | [standalone]

#1442555 — Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic broadcast quirk

On 13/07/2016 17:14, Radim Krčmář wrote:
> 2016-07-13 10:38+0200, Paolo Bonzini:
>> On 12/07/2016 22:09, Radim Krčmář wrote:
>>> @@ -619,14 +619,17 @@ static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
>>>  /* KVM APIC implementation has two quirks
>>>   *  - dest always begins at 0 while xAPIC MDA has offset 24,
>>>   *  - IOxAPIC messages have to be delivered (directly) to x2APIC.
>>> + *
>>> + * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.
>>
>> --verbose version:
>>
>>  /* KVM APIC implementation has two quirks
>> - *  - dest always begins at 0 while xAPIC MDA has offset 24,
>> - *  - IOxAPIC messages have to be delivered (directly) to x2APIC.
>> + *  - dest always begins at 0 while xAPIC MDA has offset 24.  This is
>> + *    just a quirk in the API and is not problematic.
>>   *
>> - * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.
>> + *  - in-kernel IOAPIC messages have to be delivered directly to
>> + *    x2APIC, because the kernel does not support interrupt remapping.
>> + *    In order to support broadcast without interrupt remapping, x2APIC
>> + *    rewrites the destination of non-IPI messages (and also of IPIs sent
>> + *    from xAPIC-mode LAPICs) from APIC_BROADCAST to X2APIC_BROADCAST.
> 
> KVM doesn't do the operation in parentheses since 394457a928e0 ("KVM:
> x86: some apic broadcast modes does not work").
> 
> Following patches added only !ipi case (I already regret posting them).
> 
>> + *
>> + * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
>> + * important when userspace wants to use x2APIC-format MSIs, because
>> + * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
>>   */
>>
>> Sounds good?
> 
> Except that one bit.  (Code alone really can't explain the what happens
> here.)
> 

Ok, will fix both bits tomorrow and repush to kvm/queue.  Then test,
push to kvm/next, and pass the tree to you.

Thanks!

Paolo

[toc] | [prev] | [next] | [standalone]

#1442300 — Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic broadcast quirk

On 12/07/2016 22:09, Radim Krčmář wrote:
> +static bool kvm_apic_map_need_slowpath(struct kvm *kvm, struct kvm_lapic **src,
> +		struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
> +{
> +	if (!map)
> +		return true;
> +
> +	if (kvm->arch.x2apic_broadcast_quirk_disabled) {
> +		if ((irq->dest_id == APIC_BROADCAST &&
> +				map->mode != KVM_APIC_MODE_X2APIC))
> +			return true;
> +		if (irq->dest_id == X2APIC_BROADCAST)
> +			return true;
> +	} else {
> +		bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
> +		if (irq->dest_id == (x2apic_ipi ?
> +		                     X2APIC_BROADCAST : APIC_BROADCAST))
> +			return true;
> +	}
> +
> +	return false;
> +}

This isn't the only case where you go through the slowpath.  What about
renaming to kvm_apic_dest_is_broadcast?

Paolo

[toc] | [prev] | [next] | [standalone]

#1442522 — Re: [PATCH v3 12/14] KVM: x86: add a flag to disable KVM x2apic broadcast quirk

2016-07-13 12:15+0200, Paolo Bonzini:
> On 12/07/2016 22:09, Radim Krčmář wrote:
>> +static bool kvm_apic_map_need_slowpath(struct kvm *kvm, struct kvm_lapic **src,
>> +		struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
>> +{
>> +	if (!map)
>> +		return true;
>> +
>> +	if (kvm->arch.x2apic_broadcast_quirk_disabled) {
>> +		if ((irq->dest_id == APIC_BROADCAST &&
>> +				map->mode != KVM_APIC_MODE_X2APIC))
>> +			return true;
>> +		if (irq->dest_id == X2APIC_BROADCAST)
>> +			return true;
>> +	} else {
>> +		bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
>> +		if (irq->dest_id == (x2apic_ipi ?
>> +		                     X2APIC_BROADCAST : APIC_BROADCAST))
>> +			return true;
>> +	}
>> +
>> +	return false;
>> +}
> 
> This isn't the only case where you go through the slowpath.  What about
> renaming to kvm_apic_dest_is_broadcast?

Yeah, sorry.  I have a patch that adds a check for 0xff in flat lapic
mode (can be handled in the fast path) and used the same name even
before the check ...

[toc] | [prev] | [standalone]

Back to top | Article view | linux.kernel

csiph-web