Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1405230 > unrolled thread

[PATCH v5 2/2] skb_array: ring test

Back to article view | Back to linux.kernel

This discussion starts older than the indexed window; earlier articles aren't shown. The article labeled Started by below is the oldest one visible, not the original post.

Contents

  [PATCH v5 2/2] skb_array: ring test "Michael S. Tsirkin" <mst@redhat.com> - 2016-05-23 12:50 +0200
    Re: [PATCH v5 2/2] skb_array: ring test Jesper Dangaard Brouer <brouer@redhat.com> - 2016-05-23 15:10 +0200
      Re: [PATCH v5 2/2] skb_array: ring test "Michael S. Tsirkin" <mst@redhat.com> - 2016-05-23 23:00 +0200
        Re: [PATCH v5 2/2] skb_array: ring test Jesper Dangaard Brouer <brouer@redhat.com> - 2016-05-24 12:30 +0200
          Re: [PATCH v5 2/2] skb_array: ring test "Michael S. Tsirkin" <mst@redhat.com> - 2016-05-24 12:40 +0200
          Re: [PATCH v5 2/2] skb_array: ring test "Michael S. Tsirkin" <mst@redhat.com> - 2016-05-24 14:00 +0200
          Re: [PATCH v5 2/2] skb_array: ring test "Michael S. Tsirkin" <mst@redhat.com> - 2016-05-24 14:20 +0200
          Re: [PATCH v5 2/2] skb_array: ring test Jesper Dangaard Brouer <brouer@redhat.com> - 2016-05-24 19:10 +0200
            Re: [PATCH v5 2/2] skb_array: ring test "Michael S. Tsirkin" <mst@redhat.com> - 2016-05-24 22:40 +0200

#1405230 — [PATCH v5 2/2] skb_array: ring test

Add ringtest based unit test for skb array.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
---
 tools/virtio/ringtest/skb_array.c | 167 ++++++++++++++++++++++++++++++++++++++
 tools/virtio/ringtest/Makefile    |   4 +-
 2 files changed, 170 insertions(+), 1 deletion(-)
 create mode 100644 tools/virtio/ringtest/skb_array.c

diff --git a/tools/virtio/ringtest/skb_array.c b/tools/virtio/ringtest/skb_array.c
new file mode 100644
index 0000000..4ab7e31
--- /dev/null
+++ b/tools/virtio/ringtest/skb_array.c
@@ -0,0 +1,167 @@
+#define _GNU_SOURCE
+#include "main.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <pthread.h>
+#include <malloc.h>
+#include <assert.h>
+#include <errno.h>
+#include <limits.h>
+
+struct sk_buff;
+#define SMP_CACHE_BYTES 64
+#define cache_line_size() SMP_CACHE_BYTES
+#define ____cacheline_aligned_in_smp __attribute__ ((aligned (SMP_CACHE_BYTES)))
+#define unlikely(x)    (__builtin_expect(!!(x), 0))
+#define ALIGN(x, a) (((x) + (a) - 1) / (a) * (a))
+typedef pthread_spinlock_t  spinlock_t;
+
+typedef int gfp_t;
+static void *kzalloc(unsigned size, gfp_t gfp)
+{
+	void *p = memalign(64, size);
+	if (!p)
+		return p;
+	memset(p, 0, size);
+
+	return p;
+}
+
+static void kfree(void *p)
+{
+	if (p)
+		free(p);
+}
+
+static void spin_lock_init(spinlock_t *lock)
+{
+	int r = pthread_spin_init(lock, 0);
+	assert(!r);
+}
+
+static void spin_lock_bh(spinlock_t *lock)
+{
+	int ret = pthread_spin_lock(lock);
+	assert(!ret);
+}
+
+static void spin_unlock_bh(spinlock_t *lock)
+{
+	int ret = pthread_spin_unlock(lock);
+	assert(!ret);
+}
+
+#include "../../../include/linux/skb_array.h"
+
+static unsigned long long headcnt, tailcnt;
+static struct skb_array array ____cacheline_aligned_in_smp;
+
+/* implemented by ring */
+void alloc_ring(void)
+{
+	int ret = skb_array_init(&array, ring_size, 0);
+	assert(!ret);
+}
+
+/* guest side */
+int add_inbuf(unsigned len, void *buf, void *datap)
+{
+	int ret;
+	
+	assert(headcnt - tailcnt <= ring_size);
+	ret = __skb_array_produce(&array, buf);
+	if (ret >= 0) {
+		ret = 0;
+		headcnt++;
+	}
+
+	return ret;
+}
+
+/*
+ * skb_array API provides no way for producer to find out whether a given
+ * buffer was consumed.  Our tests merely require that a successful get_buf
+ * implies that add_inbuf succeed in the past, and that add_inbuf will succeed,
+ * fake it accordingly.
+ */
+void *get_buf(unsigned *lenp, void **bufp)
+{
+	void *datap;
+
+	if (tailcnt == headcnt || __skb_array_full(&array))
+		datap = NULL;
+	else {
+		datap = "Buffer\n";
+		++tailcnt;
+	}
+
+	return datap;
+}
+
+void poll_used(void)
+{
+	void *b;
+
+	do {
+		if (tailcnt == headcnt || __skb_array_full(&array)) {
+			b = NULL;
+			barrier();
+		} else {
+			b = "Buffer\n";
+		}
+	} while (!b);
+}
+
+void disable_call()
+{
+	assert(0);
+}
+
+bool enable_call()
+{
+	assert(0);
+}
+
+void kick_available(void)
+{
+	assert(0);
+}
+
+/* host side */
+void disable_kick()
+{
+	assert(0);
+}
+
+bool enable_kick()
+{
+	assert(0);
+}
+
+void poll_avail(void)
+{
+	void *b;
+
+	do {
+		b = __skb_array_peek(&array);
+		barrier();
+	} while (!b);
+}
+
+bool use_buf(unsigned *lenp, void **bufp)
+{
+	struct sk_buff *skb;
+
+	skb = __skb_array_peek(&array);
+	if (skb) {
+		__skb_array_consume(&array);
+	}
+
+	return skb;
+}
+
+void call_used(void)
+{
+	assert(0);
+}
diff --git a/tools/virtio/ringtest/Makefile b/tools/virtio/ringtest/Makefile
index 6ba7455..87e58cf 100644
--- a/tools/virtio/ringtest/Makefile
+++ b/tools/virtio/ringtest/Makefile
@@ -1,6 +1,6 @@
 all:
 
-all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder
+all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder skb_array
 
 CFLAGS += -Wall
 CFLAGS += -pthread -O2 -ggdb
@@ -8,6 +8,7 @@ LDFLAGS += -pthread -O2 -ggdb
 
 main.o: main.c main.h
 ring.o: ring.c main.h
+skb_array.o: skb_array.c main.h ../../../include/linux/skb_array.h
 virtio_ring_0_9.o: virtio_ring_0_9.c main.h
 virtio_ring_poll.o: virtio_ring_poll.c virtio_ring_0_9.c main.h
 virtio_ring_inorder.o: virtio_ring_inorder.c virtio_ring_0_9.c main.h
@@ -15,6 +16,7 @@ ring: ring.o main.o
 virtio_ring_0_9: virtio_ring_0_9.o main.o
 virtio_ring_poll: virtio_ring_poll.o main.o
 virtio_ring_inorder: virtio_ring_inorder.o main.o
+skb_array: skb_array.o main.o
 clean:
 	-rm main.o
 	-rm ring.o ring
-- 
MST

[toc] | [next] | [standalone]

#1405354

On Mon, 23 May 2016 13:43:46 +0300
"Michael S. Tsirkin" <mst@redhat.com> wrote:

> Add ringtest based unit test for skb array.
>
> Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> ---
>  tools/virtio/ringtest/skb_array.c | 167 ++++++++++++++++++++++++++++++++++++++
>  tools/virtio/ringtest/Makefile    |   4 +-

Patch didn't apply cleanly to Makefile, as you also seems to have
"virtio_ring_inorder", I manually applied it.

I chdir to tools/virtio/ringtest/ and I could compile "skb_array",
BUT how do I use it??? (the README is not helpful)

What is the "output", are there any performance measurement results?


> diff --git a/tools/virtio/ringtest/Makefile b/tools/virtio/ringtest/Makefile
> index 6ba7455..87e58cf 100644
> --- a/tools/virtio/ringtest/Makefile
> +++ b/tools/virtio/ringtest/Makefile
> @@ -1,6 +1,6 @@
>  all:
>  
> -all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder
> +all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder skb_array
                                              ^^^^^^^^^^^^^^^^^^^
>  
>  CFLAGS += -Wall
>  CFLAGS += -pthread -O2 -ggdb
> @@ -8,6 +8,7 @@ LDFLAGS += -pthread -O2 -ggdb
>  
>  main.o: main.c main.h
>  ring.o: ring.c main.h
> +skb_array.o: skb_array.c main.h ../../../include/linux/skb_array.h
>  virtio_ring_0_9.o: virtio_ring_0_9.c main.h
>  virtio_ring_poll.o: virtio_ring_poll.c virtio_ring_0_9.c main.h
>  virtio_ring_inorder.o: virtio_ring_inorder.c virtio_ring_0_9.c main.h
> @@ -15,6 +16,7 @@ ring: ring.o main.o
>  virtio_ring_0_9: virtio_ring_0_9.o main.o
>  virtio_ring_poll: virtio_ring_poll.o main.o
>  virtio_ring_inorder: virtio_ring_inorder.o main.o
   ^^^^^^^^^^^^^^^^^^^  ^^^^^^^^^^^^^^^^^^^^^
> +skb_array: skb_array.o main.o
>  clean:
>  	-rm main.o
>  	-rm ring.o ring


-- 
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  Author of http://www.iptv-analyzer.org
  LinkedIn: http://www.linkedin.com/in/brouer

[toc] | [prev] | [next] | [standalone]

#1405628

On Mon, May 23, 2016 at 03:09:18PM +0200, Jesper Dangaard Brouer wrote:
> On Mon, 23 May 2016 13:43:46 +0300
> "Michael S. Tsirkin" <mst@redhat.com> wrote:
> 
> > Add ringtest based unit test for skb array.
> >
> > Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> > ---
> >  tools/virtio/ringtest/skb_array.c | 167 ++++++++++++++++++++++++++++++++++++++
> >  tools/virtio/ringtest/Makefile    |   4 +-
> 
> Patch didn't apply cleanly to Makefile, as you also seems to have
> "virtio_ring_inorder", I manually applied it.
> 
> I chdir to tools/virtio/ringtest/ and I could compile "skb_array",
> BUT how do I use it??? (the README is not helpful)
> 
> What is the "output", are there any performance measurement results?

First, if it completes successfully this means it completed
a ton of cycles without errors. It caches any missing barriers
which aren't nops on your system.

Second - use perf.

E.g. simple perf stat will measure how long does it take to execute.
there's a script that runs it on different CPUs,
so I normally do:

sh run-on-all.sh perf stat -r 5 ./skb_array


> > diff --git a/tools/virtio/ringtest/Makefile b/tools/virtio/ringtest/Makefile
> > index 6ba7455..87e58cf 100644
> > --- a/tools/virtio/ringtest/Makefile
> > +++ b/tools/virtio/ringtest/Makefile
> > @@ -1,6 +1,6 @@
> >  all:
> >  
> > -all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder
> > +all: ring virtio_ring_0_9 virtio_ring_poll virtio_ring_inorder skb_array
>                                               ^^^^^^^^^^^^^^^^^^^
> >  
> >  CFLAGS += -Wall
> >  CFLAGS += -pthread -O2 -ggdb
> > @@ -8,6 +8,7 @@ LDFLAGS += -pthread -O2 -ggdb
> >  
> >  main.o: main.c main.h
> >  ring.o: ring.c main.h
> > +skb_array.o: skb_array.c main.h ../../../include/linux/skb_array.h
> >  virtio_ring_0_9.o: virtio_ring_0_9.c main.h
> >  virtio_ring_poll.o: virtio_ring_poll.c virtio_ring_0_9.c main.h
> >  virtio_ring_inorder.o: virtio_ring_inorder.c virtio_ring_0_9.c main.h
> > @@ -15,6 +16,7 @@ ring: ring.o main.o
> >  virtio_ring_0_9: virtio_ring_0_9.o main.o
> >  virtio_ring_poll: virtio_ring_poll.o main.o
> >  virtio_ring_inorder: virtio_ring_inorder.o main.o
>    ^^^^^^^^^^^^^^^^^^^  ^^^^^^^^^^^^^^^^^^^^^
> > +skb_array: skb_array.o main.o
> >  clean:
> >  	-rm main.o
> >  	-rm ring.o ring
> 
> 
> -- 
> Best regards,
>   Jesper Dangaard Brouer
>   MSc.CS, Principal Kernel Engineer at Red Hat
>   Author of http://www.iptv-analyzer.org
>   LinkedIn: http://www.linkedin.com/in/brouer

[toc] | [prev] | [next] | [standalone]

#1406042

On Mon, 23 May 2016 23:52:47 +0300
"Michael S. Tsirkin" <mst@redhat.com> wrote:

> On Mon, May 23, 2016 at 03:09:18PM +0200, Jesper Dangaard Brouer wrote:
> > On Mon, 23 May 2016 13:43:46 +0300
> > "Michael S. Tsirkin" <mst@redhat.com> wrote:
> >   
> > > Add ringtest based unit test for skb array.
> > >
> > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> > > ---
> > >  tools/virtio/ringtest/skb_array.c | 167 ++++++++++++++++++++++++++++++++++++++
> > >  tools/virtio/ringtest/Makefile    |   4 +-  
> > 
> > Patch didn't apply cleanly to Makefile, as you also seems to have
> > "virtio_ring_inorder", I manually applied it.
> > 
> > I chdir to tools/virtio/ringtest/ and I could compile "skb_array",
> > BUT how do I use it??? (the README is not helpful)
> > 
> > What is the "output", are there any performance measurement results?  
> 
> First, if it completes successfully this means it completed
> a ton of cycles without errors. It caches any missing barriers
> which aren't nops on your system.

I applied these patches on net-next (at commit 07b75260e) and the
skb_array test program never terminates.   Strangely if I use your git
tree[1] (on branch vhost) the program does terminate... I didn't spot
the difference.
 
> Second - use perf.

I do like perf, but it does not answer my questions about the
performance of this queue. I will code something up in my own
framework[2] to answer my own performance questions.

Like what is be minimum overhead (in cycles) achievable with this type
of queue, in the most optimal situation (e.g. same CPU enq+deq cache hot)
for fastpath usage.

Then I also want to know how this performs when two CPUs are involved.
As this is also a primary use-case, for you when sending packets into a
guest.


> E.g. simple perf stat will measure how long does it take to execute.
> there's a script that runs it on different CPUs,
> so I normally do:
>
> sh run-on-all.sh perf stat -r 5 ./skb_array

I recommend documenting this in the README file in the same dir ;-)

[1] https://git.kernel.org/cgit/linux/kernel/git/mst/vhost.git/log/?h=vhost
[2] https://github.com/netoptimizer/prototype-kernel
-- 
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  Author of http://www.iptv-analyzer.org
  LinkedIn: http://www.linkedin.com/in/brouer

[toc] | [prev] | [next] | [standalone]

#1406050

On Tue, May 24, 2016 at 12:28:09PM +0200, Jesper Dangaard Brouer wrote:
> On Mon, 23 May 2016 23:52:47 +0300
> "Michael S. Tsirkin" <mst@redhat.com> wrote:
> 
> > On Mon, May 23, 2016 at 03:09:18PM +0200, Jesper Dangaard Brouer wrote:
> > > On Mon, 23 May 2016 13:43:46 +0300
> > > "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > >   
> > > > Add ringtest based unit test for skb array.
> > > >
> > > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> > > > ---
> > > >  tools/virtio/ringtest/skb_array.c | 167 ++++++++++++++++++++++++++++++++++++++
> > > >  tools/virtio/ringtest/Makefile    |   4 +-  
> > > 
> > > Patch didn't apply cleanly to Makefile, as you also seems to have
> > > "virtio_ring_inorder", I manually applied it.
> > > 
> > > I chdir to tools/virtio/ringtest/ and I could compile "skb_array",
> > > BUT how do I use it??? (the README is not helpful)
> > > 
> > > What is the "output", are there any performance measurement results?  
> > 
> > First, if it completes successfully this means it completed
> > a ton of cycles without errors. It caches any missing barriers
> > which aren't nops on your system.
> 
> I applied these patches on net-next (at commit 07b75260e) and the
> skb_array test program never terminates.   Strangely if I use your git
> tree[1] (on branch vhost) the program does terminate... I didn't spot
> the difference.

Disassemble the binaries and compare? Should be identical.
Or attach gdb and look at array.producer and array.consumer.

> > Second - use perf.
> 
> I do like perf, but it does not answer my questions about the
> performance of this queue. I will code something up in my own
> framework[2] to answer my own performance questions.

Sounds good.

> Like what is be minimum overhead (in cycles) achievable with this type
> of queue, in the most optimal situation (e.g. same CPU enq+deq cache hot)
> for fastpath usage.

Interesting.

> Then I also want to know how this performs when two CPUs are involved.

This has flags to pin threads to different CPUs.


> As this is also a primary use-case, for you when sending packets into a
> guest.
> 

That's absolutely the primary usecase.
Was designed with this in mind.

> 
> > E.g. simple perf stat will measure how long does it take to execute.
> > there's a script that runs it on different CPUs,
> > so I normally do:
> >
> > sh run-on-all.sh perf stat -r 5 ./skb_array
> 
> I recommend documenting this in the README file in the same dir ;-)

Good idea.  Will do.

> [1] https://git.kernel.org/cgit/linux/kernel/git/mst/vhost.git/log/?h=vhost
> [2] https://github.com/netoptimizer/prototype-kernel
> -- 
> Best regards,
>   Jesper Dangaard Brouer
>   MSc.CS, Principal Kernel Engineer at Red Hat
>   Author of http://www.iptv-analyzer.org
>   LinkedIn: http://www.linkedin.com/in/brouer

[toc] | [prev] | [next] | [standalone]

#1406099

On Tue, May 24, 2016 at 12:28:09PM +0200, Jesper Dangaard Brouer wrote:
> On Mon, 23 May 2016 23:52:47 +0300
> "Michael S. Tsirkin" <mst@redhat.com> wrote:
> 
> > On Mon, May 23, 2016 at 03:09:18PM +0200, Jesper Dangaard Brouer wrote:
> > > On Mon, 23 May 2016 13:43:46 +0300
> > > "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > >   
> > > > Add ringtest based unit test for skb array.
> > > >
> > > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> > > > ---
> > > >  tools/virtio/ringtest/skb_array.c | 167 ++++++++++++++++++++++++++++++++++++++
> > > >  tools/virtio/ringtest/Makefile    |   4 +-  
> > > 
> > > Patch didn't apply cleanly to Makefile, as you also seems to have
> > > "virtio_ring_inorder", I manually applied it.
> > > 
> > > I chdir to tools/virtio/ringtest/ and I could compile "skb_array",
> > > BUT how do I use it??? (the README is not helpful)
> > > 
> > > What is the "output", are there any performance measurement results?  
> > 
> > First, if it completes successfully this means it completed
> > a ton of cycles without errors. It caches any missing barriers
> > which aren't nops on your system.
> 
> I applied these patches on net-next (at commit 07b75260e) and the
> skb_array test program never terminates.   Strangely if I use your git
> tree[1] (on branch vhost) the program does terminate... I didn't spot
> the difference.

Oh, that's my bad. You need

    ringtest: pass buf != NULL
    
    just a stub pointer for now.
    
    Signed-off-by: Michael S. Tsirkin <mst@redhat.com>

from my tree.

-- 
MST

[toc] | [prev] | [next] | [standalone]

#1406109

On Tue, May 24, 2016 at 12:28:09PM +0200, Jesper Dangaard Brouer wrote:
> On Mon, 23 May 2016 23:52:47 +0300
> "Michael S. Tsirkin" <mst@redhat.com> wrote:
> 
> > On Mon, May 23, 2016 at 03:09:18PM +0200, Jesper Dangaard Brouer wrote:
> > > On Mon, 23 May 2016 13:43:46 +0300
> > > "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > >   
> > > > Add ringtest based unit test for skb array.
> > > >
> > > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> > > > ---
> > > >  tools/virtio/ringtest/skb_array.c | 167 ++++++++++++++++++++++++++++++++++++++
> > > >  tools/virtio/ringtest/Makefile    |   4 +-  
> > > 
> > > Patch didn't apply cleanly to Makefile, as you also seems to have
> > > "virtio_ring_inorder", I manually applied it.
> > > 
> > > I chdir to tools/virtio/ringtest/ and I could compile "skb_array",
> > > BUT how do I use it??? (the README is not helpful)
> > > 
> > > What is the "output", are there any performance measurement results?  
> > 
> > First, if it completes successfully this means it completed
> > a ton of cycles without errors. It caches any missing barriers
> > which aren't nops on your system.
> 
> I applied these patches on net-next (at commit 07b75260e) and the
> skb_array test program never terminates.   Strangely if I use your git
> tree[1] (on branch vhost) the program does terminate... I didn't spot
> the difference.
>  
> > Second - use perf.
> 
> I do like perf, but it does not answer my questions about the
> performance of this queue. I will code something up in my own
> framework[2] to answer my own performance questions.
> 
> Like what is be minimum overhead (in cycles) achievable with this type
> of queue, in the most optimal situation (e.g. same CPU enq+deq cache hot)
> for fastpath usage.

Actually there is, kind of, a way to find out with my tool
if you have an HT CPU.  When you do run-on-all.sh
it will pin consumer to the last CPU, then run producer
on all of them. Look for the number for the HT pair -
this shares cache between producer and consumer.

This is not the same as doing produce + consume on
the same CPU but it's close enough I think.

To measure overhead I guess I should build a NOP tool
that does not actually produce or consume anything.
Will do.

> Then I also want to know how this performs when two CPUs are involved.
> As this is also a primary use-case, for you when sending packets into a
> guest.
> 
> 
> > E.g. simple perf stat will measure how long does it take to execute.
> > there's a script that runs it on different CPUs,
> > so I normally do:
> >
> > sh run-on-all.sh perf stat -r 5 ./skb_array
> 
> I recommend documenting this in the README file in the same dir ;-)
> 
> [1] https://git.kernel.org/cgit/linux/kernel/git/mst/vhost.git/log/?h=vhost
> [2] https://github.com/netoptimizer/prototype-kernel
> -- 
> Best regards,
>   Jesper Dangaard Brouer
>   MSc.CS, Principal Kernel Engineer at Red Hat
>   Author of http://www.iptv-analyzer.org
>   LinkedIn: http://www.linkedin.com/in/brouer

[toc] | [prev] | [next] | [standalone]

#1406291

On Tue, 24 May 2016 12:28:09 +0200
Jesper Dangaard Brouer <brouer@redhat.com> wrote:

> I do like perf, but it does not answer my questions about the
> performance of this queue. I will code something up in my own
> framework[2] to answer my own performance questions.
> 
> Like what is be minimum overhead (in cycles) achievable with this type
> of queue, in the most optimal situation (e.g. same CPU enq+deq cache hot)
> for fastpath usage.

Coded it up here:
 https://github.com/netoptimizer/prototype-kernel/commit/b16a3332184
 https://github.com/netoptimizer/prototype-kernel/blob/master/kernel/lib/skb_array_bench01.c

This is a really fake benchmark, but it sort of shows the minimum
overhead achievable with this type of queue, where it is the same
CPU enqueuing and dequeuing, and cache is guaranteed to be hot.

Measured on a i7-4790K CPU @ 4.00GHz, the average cost of
enqueue+dequeue of a single object is around 102 cycles(tsc).

To compare this with below, where enq and deq is measured separately:
 102 / 2 = 51 cycles

 
> Then I also want to know how this performs when two CPUs are involved.
> As this is also a primary use-case, for you when sending packets into a
> guest.

Coded it up here:
 https://github.com/netoptimizer/prototype-kernel/commit/75fe31ef62e
 https://github.com/netoptimizer/prototype-kernel/blob/master/kernel/lib/skb_array_parallel01.c
 
This parallel benchmark try to keep two (or more) CPUs busy enqueuing or
dequeuing on the same skb_array queue.  It prefills the queue,
and stops the test as soon as queue is empty or full, or
completes a number of "loops"/cycles.

For two CPUs the results are really good:
 enqueue: 54 cycles(tsc)
 dequeue: 53 cycles(tsc)

Going to 4 CPUs, things break down (but it was not primary use-case?):
 CPU(0) 927 cycles(tsc) enqueue
 CPU(1) 921 cycles(tsc) dequeue
 CPU(2) 927 cycles(tsc) enqueue
 CPU(3) 898 cycles(tsc) dequeue


Next on my todo-list is to implement same tests for e.g. alf_queue, so
we can compare the concurrency part (which is the important part).
But FYI I'll be busy the next days at conf http://fosd2016.itu.dk/

-- 
Best regards,
  Jesper Dangaard Brouer
  MSc.CS, Principal Kernel Engineer at Red Hat
  Author of http://www.iptv-analyzer.org
  LinkedIn: http://www.linkedin.com/in/brouer


[1] https://git.kernel.org/cgit/linux/kernel/git/mst/vhost.git/log/?h=vhost
[2] https://github.com/netoptimizer/prototype-kernel

[toc] | [prev] | [next] | [standalone]

#1406443

On Tue, May 24, 2016 at 07:03:20PM +0200, Jesper Dangaard Brouer wrote:
> 
> On Tue, 24 May 2016 12:28:09 +0200
> Jesper Dangaard Brouer <brouer@redhat.com> wrote:
> 
> > I do like perf, but it does not answer my questions about the
> > performance of this queue. I will code something up in my own
> > framework[2] to answer my own performance questions.
> > 
> > Like what is be minimum overhead (in cycles) achievable with this type
> > of queue, in the most optimal situation (e.g. same CPU enq+deq cache hot)
> > for fastpath usage.
> 
> Coded it up here:
>  https://github.com/netoptimizer/prototype-kernel/commit/b16a3332184
>  https://github.com/netoptimizer/prototype-kernel/blob/master/kernel/lib/skb_array_bench01.c
> 
> This is a really fake benchmark, but it sort of shows the minimum
> overhead achievable with this type of queue, where it is the same
> CPU enqueuing and dequeuing, and cache is guaranteed to be hot.
> 
> Measured on a i7-4790K CPU @ 4.00GHz, the average cost of
> enqueue+dequeue of a single object is around 102 cycles(tsc).
> 
> To compare this with below, where enq and deq is measured separately:
>  102 / 2 = 51 cycles
> 
>  
> > Then I also want to know how this performs when two CPUs are involved.
> > As this is also a primary use-case, for you when sending packets into a
> > guest.
> 
> Coded it up here:
>  https://github.com/netoptimizer/prototype-kernel/commit/75fe31ef62e
>  https://github.com/netoptimizer/prototype-kernel/blob/master/kernel/lib/skb_array_parallel01.c
>  
> This parallel benchmark try to keep two (or more) CPUs busy enqueuing or
> dequeuing on the same skb_array queue.  It prefills the queue,
> and stops the test as soon as queue is empty or full, or
> completes a number of "loops"/cycles.
> 
> For two CPUs the results are really good:
>  enqueue: 54 cycles(tsc)
>  dequeue: 53 cycles(tsc)
> 
> Going to 4 CPUs, things break down (but it was not primary use-case?):
>  CPU(0) 927 cycles(tsc) enqueue
>  CPU(1) 921 cycles(tsc) dequeue
>  CPU(2) 927 cycles(tsc) enqueue
>  CPU(3) 898 cycles(tsc) dequeue

It's mostly the spinlock contention I guess.
Maybe we don't need fair spinlocks in this case.
Try replacing spinlocks with simple cmpxchg
and see what happens?

[toc] | [prev] | [standalone]

Back to top | Article view | linux.kernel

csiph-web