Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1478824 > unrolled thread

[PATCH v2 00/33] Enable Intel Resource Allocation in Resource Director Technology

Back to article view | Back to linux.kernel

Contents

  [PATCH v2 00/33] Enable Intel Resource Allocation in Resource Director Technology  "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:00 +0200
    [PATCH v2 32/33] MAINTAINERS: Add maintainer for Intel RDT resource allocation "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:00 +0200
    [PATCH v2 30/33] x86/intel_rdt_rdtgroup.c: Process schemata input from resctrl interface "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:00 +0200
      Re: [PATCH v2 30/33] x86/intel_rdt_rdtgroup.c: Process schemata  input from resctrl interface Thomas Gleixner <tglx@linutronix.de> - 2016-09-09 00:30 +0200
    [PATCH v2 31/33] Documentation/kernel-parameters: Add kernel parameter "resctrl" for CAT "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:00 +0200
      Re: [PATCH v2 31/33] Documentation/kernel-parameters: Add kernel  parameter "resctrl" for CAT Thomas Gleixner <tglx@linutronix.de> - 2016-09-09 00:30 +0200
    [PATCH v2 28/33] x86/intel_rdt_rdtgroup.c: Read and write cpus "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 28/33] x86/intel_rdt_rdtgroup.c: Read and write cpus Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 22:30 +0200
    [PATCH v2 24/33] x86/intel_rdt_rdtgroup.c: Create info directory "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 24/33] x86/intel_rdt_rdtgroup.c: Create info  directory Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 18:10 +0200
    [PATCH v2 11/33] x86/intel_rdt: Hot cpu support for Cache Allocation "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 11/33] x86/intel_rdt: Hot cpu support for Cache  Allocation Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 12:10 +0200
    [PATCH v2 13/33] Define CONFIG_INTEL_RDT "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 13/33] Define CONFIG_INTEL_RDT Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 12:20 +0200
    [PATCH v2 08/33] x86/intel_rdt: Add Class of service management "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 08/33] x86/intel_rdt: Add Class of service  management Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 11:00 +0200
    [PATCH v2 22/33] x86/intel_rdt.c: Extend RDT to per cache and per resources "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 22/33] x86/intel_rdt.c: Extend RDT to per cache and  per resources Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 17:10 +0200
    [PATCH v2 20/33] x86/intel_rdt.h: Header for inter_rdt.c "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 20/33] x86/intel_rdt.h: Header for inter_rdt.c Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 14:40 +0200
    [PATCH v2 10/33] x86/intel_rdt: Implement scheduling support for Intel RDT "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 10/33] x86/intel_rdt: Implement scheduling support  for Intel RDT Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 12:00 +0200
    [PATCH v2 16/33] x86/intel_rdt: Class of service and capacity bitmask management for CDP "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 16/33] x86/intel_rdt: Class of service and capacity  bitmask management for CDP Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 12:40 +0200
    [PATCH v2 04/33] drivers/base/cacheinfo.c: Export some cacheinfo functions for others to use "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 04/33] drivers/base/cacheinfo.c: Export some cacheinfo  functions for others to use Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 10:30 +0200
    [PATCH v2 25/33] include/linux/resctrl.h: Define fork and exit functions in a new header file "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 25/33] include/linux/resctrl.h: Define fork and exit  functions in a new header file Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 18:20 +0200
    [PATCH v2 27/33] x86/intel_rdt_rdtgroup.c: Implement resctrl file system commands "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 27/33] x86/intel_rdt_rdtgroup.c: Implement resctrl  file system commands Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 22:20 +0200
      Re: [PATCH v2 27/33] x86/intel_rdt_rdtgroup.c: Implement resctrl  file system commands Fenghua Yu <fenghua.yu@intel.com> - 2016-09-09 00:30 +0200
    [PATCH v2 29/33] x86/intel_rdt_rdtgroup.c: Tasks iterator and write "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 29/33] x86/intel_rdt_rdtgroup.c: Tasks iterator and  write Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 23:00 +0200
    [PATCH v2 17/33] x86/intel_rdt: Hot cpu update for code data prioritization "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 17/33] x86/intel_rdt: Hot cpu update for code data  prioritization Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 12:40 +0200
    [PATCH v2 06/33] Documentation, x86: Documentation for Intel resource allocation user interface "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 06/33] Documentation, x86: Documentation for Intel  resource allocation user interface Borislav Petkov <bp@suse.de> - 2016-09-08 13:30 +0200
      Re: [PATCH v2 06/33] Documentation, x86: Documentation for Intel  resource allocation user interface Fenghua Yu <fenghua.yu@intel.com> - 2016-09-09 00:20 +0200
        Re: [PATCH v2 06/33] Documentation, x86: Documentation for Intel  resource allocation user interface Fenghua Yu <fenghua.yu@intel.com> - 2016-09-09 06:30 +0200
    [PATCH v2 03/33] x86, intel_cacheinfo: Enable cache id in x86 "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
    [PATCH v2 09/33] x86/intel_rdt: Add L3 cache capacity bitmask management "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 09/33] x86/intel_rdt: Add L3 cache capacity bitmask  management Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 11:50 +0200
    [PATCH v2 07/33] x86/intel_rdt: Add support for Cache Allocation detection "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 07/33] x86/intel_rdt: Add support for Cache Allocation  detection Borislav Petkov <bp@suse.de> - 2016-09-08 14:00 +0200
        RE: [PATCH v2 07/33] x86/intel_rdt: Add support for Cache  Allocation detection "Yu, Fenghua" <fenghua.yu@intel.com> - 2016-09-08 19:00 +0200
          Re: [PATCH v2 07/33] x86/intel_rdt: Add support for Cache Allocation  detection Borislav Petkov <bp@suse.de> - 2016-09-08 19:20 +0200
      Re: [PATCH v2 07/33] x86/intel_rdt: Add support for Cache Allocation  detection Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 15:20 +0200
        RE: [PATCH v2 07/33] x86/intel_rdt: Add support for Cache  Allocation detection "Yu, Fenghua" <fenghua.yu@intel.com> - 2016-09-08 16:00 +0200
    [PATCH v2 01/33] cacheinfo: Introduce cache id "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
    [PATCH v2 05/33] x86/intel_rdt: Cache Allocation documentation "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
    [PATCH v2 19/33] magic number for resctrl file system "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 19/33] magic number for resctrl file system Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 12:50 +0200
        Re: [PATCH v2 19/33] magic number for resctrl file system Borislav Petkov <bp@alien8.de> - 2016-09-08 12:50 +0200
    [PATCH v2 21/33] x86/intel_rdt_rdtgroup.h: Header for user interface "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 21/33] x86/intel_rdt_rdtgroup.h: Header for user  interface Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 14:50 +0200
    [PATCH v2 26/33] Task fork and exit for rdtgroup "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 26/33] Task fork and exit for rdtgroup Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 21:50 +0200
    [PATCH v2 18/33] sched.h: Add rg_list and rdtgroup in task_struct "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 18/33] sched.h: Add rg_list and rdtgroup in  task_struct Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 12:40 +0200
    [PATCH v2 02/33] Documentation, ABI: Add a document entry for cache id "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 02/33] Documentation, ABI: Add a document entry for  cache id Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 21:40 +0200
    [PATCH v2 12/33] x86/intel_rdt: Intel haswell Cache Allocation enumeration "Fenghua Yu" <fenghua.yu@intel.com> - 2016-09-08 09:10 +0200
      Re: [PATCH v2 12/33] x86/intel_rdt: Intel haswell Cache Allocation  enumeration Thomas Gleixner <tglx@linutronix.de> - 2016-09-08 12:20 +0200

Page 2 of 4 — ← Prev page 1 [2] 3 4  Next page →

#1478836 — [PATCH v2 10/33] x86/intel_rdt: Implement scheduling support for Intel RDT

From: Vikas Shivappa <vikas.shivappa@linux.intel.com>

Adds support for IA32_PQR_ASSOC MSR writes during task scheduling. For
Cache Allocation, MSR write would let the task fill in the cache
'subset' represented by the task's capacity bit mask.

The high 32 bits in the per processor MSR IA32_PQR_ASSOC represents the
CLOSid. During context switch kernel implements this by writing the
CLOSid of the task belongs to the CPU's IA32_PQR_ASSOC MSR.

This patch also implements a common software cache for IA32_PQR_MSR
(RMID 0:9, CLOSId 32:63) to be used by both Cache monitoring (CMT) and
Cache allocation. CMT updates the RMID where as cache_alloc updates the
CLOSid in the software cache. During scheduling when the new RMID/CLOSid
value is different from the cached values, IA32_PQR_MSR is updated.
Since the measured rdmsr latency for IA32_PQR_MSR is very high (~250
 cycles) this software cache is necessary to avoid reading the MSR to
compare the current CLOSid value.

The following considerations are done for the PQR MSR write so that it
minimally impacts scheduler hot path:
 - This path does not exist on any non-intel platforms.
 - On Intel platforms, this would not exist by default unless INTEL_RDT
 is enabled.
 - remains a no-op when INTEL_RDT is enabled and intel SKU does not
 support the feature.
 - When feature is available and enabled, never does MSR write till the
 user manually starts using one of the capacity bit masks.
 - MSR write is only done when there is a task with different Closid is
 scheduled on the CPU. Typically if the task groups are bound to be
 scheduled on a set of CPUs, the number of MSR writes is greatly
 reduced.
 - A per CPU cache of CLOSids is maintained to do the check so that we
 don't have to do a rdmsr which actually costs a lot of cycles.

Signed-off-by: Vikas Shivappa <vikas.shivappa@linux.intel.com>
Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
---
 arch/x86/events/intel/cqm.c       | 24 ++----------------------
 arch/x86/include/asm/intel_rdt.h  | 36 ++++++++++++++++++++++++++++++++++++
 arch/x86/include/asm/pqr_common.h | 27 +++++++++++++++++++++++++++
 arch/x86/kernel/cpu/intel_rdt.c   | 18 ++++++++++++++++++
 arch/x86/kernel/process_64.c      |  6 ++++++
 5 files changed, 89 insertions(+), 22 deletions(-)
 create mode 100644 arch/x86/include/asm/pqr_common.h

diff --git a/arch/x86/events/intel/cqm.c b/arch/x86/events/intel/cqm.c
index 783c49d..0ed56ae 100644
--- a/arch/x86/events/intel/cqm.c
+++ b/arch/x86/events/intel/cqm.c
@@ -7,9 +7,9 @@
 #include <linux/perf_event.h>
 #include <linux/slab.h>
 #include <asm/cpu_device_id.h>
+#include <asm/pqr_common.h>
 #include "../perf_event.h"
 
-#define MSR_IA32_PQR_ASSOC	0x0c8f
 #define MSR_IA32_QM_CTR		0x0c8e
 #define MSR_IA32_QM_EVTSEL	0x0c8d
 
@@ -24,32 +24,13 @@ static unsigned int cqm_l3_scale; /* supposedly cacheline size */
 static bool cqm_enabled, mbm_enabled;
 unsigned int mbm_socket_max;
 
-/**
- * struct intel_pqr_state - State cache for the PQR MSR
- * @rmid:		The cached Resource Monitoring ID
- * @closid:		The cached Class Of Service ID
- * @rmid_usecnt:	The usage counter for rmid
- *
- * The upper 32 bits of MSR_IA32_PQR_ASSOC contain closid and the
- * lower 10 bits rmid. The update to MSR_IA32_PQR_ASSOC always
- * contains both parts, so we need to cache them.
- *
- * The cache also helps to avoid pointless updates if the value does
- * not change.
- */
-struct intel_pqr_state {
-	u32			rmid;
-	u32			closid;
-	int			rmid_usecnt;
-};
-
 /*
  * The cached intel_pqr_state is strictly per CPU and can never be
  * updated from a remote CPU. Both functions which modify the state
  * (intel_cqm_event_start and intel_cqm_event_stop) are called with
  * interrupts disabled, which is sufficient for the protection.
  */
-static DEFINE_PER_CPU(struct intel_pqr_state, pqr_state);
+DEFINE_PER_CPU(struct intel_pqr_state, pqr_state);
 static struct hrtimer *mbm_timers;
 /**
  * struct sample - mbm event's (local or total) data
@@ -1583,7 +1564,6 @@ static int intel_cqm_cpu_starting(unsigned int cpu)
 	struct cpuinfo_x86 *c = &cpu_data(cpu);
 
 	state->rmid = 0;
-	state->closid = 0;
 	state->rmid_usecnt = 0;
 
 	WARN_ON(c->x86_cache_max_rmid != cqm_max_rmid);
diff --git a/arch/x86/include/asm/intel_rdt.h b/arch/x86/include/asm/intel_rdt.h
index 68c9a79..8512174 100644
--- a/arch/x86/include/asm/intel_rdt.h
+++ b/arch/x86/include/asm/intel_rdt.h
@@ -3,14 +3,50 @@
 
 #ifdef CONFIG_INTEL_RDT
 
+#include <linux/jump_label.h>
+
 #define MAX_CBM_LENGTH			32
 #define IA32_L3_CBM_BASE		0xc90
 #define CBM_FROM_INDEX(x)		(IA32_L3_CBM_BASE + x)
 
+extern struct static_key rdt_enable_key;
+void __intel_rdt_sched_in(void *dummy);
+
 struct clos_cbm_table {
 	unsigned long cbm;
 	unsigned int clos_refcnt;
 };
 
+/*
+ * intel_rdt_sched_in() - Writes the task's CLOSid to IA32_PQR_MSR
+ *
+ * Following considerations are made so that this has minimal impact
+ * on scheduler hot path:
+ * - This will stay as no-op unless we are running on an Intel SKU
+ * which supports L3 cache allocation.
+ * - When support is present and enabled, does not do any
+ * IA32_PQR_MSR writes until the user starts really using the feature
+ * ie creates a rdtgroup directory and assigns a cache_mask thats
+ * different from the root rdtgroup's cache_mask.
+ * - Caches the per cpu CLOSid values and does the MSR write only
+ * when a task with a different CLOSid is scheduled in. That
+ * means the task belongs to a different rdtgroup.
+ * - Closids are allocated so that different rdtgroup directories
+ * with same cache_mask gets the same CLOSid. This minimizes CLOSids
+ * used and reduces MSR write frequency.
+ */
+static inline void intel_rdt_sched_in(void)
+{
+	/*
+	 * Call the schedule in code only when RDT is enabled.
+	 */
+	if (static_key_false(&rdt_enable_key))
+		__intel_rdt_sched_in(NULL);
+}
+
+#else
+
+static inline void intel_rdt_sched_in(void) {}
+
 #endif
 #endif
diff --git a/arch/x86/include/asm/pqr_common.h b/arch/x86/include/asm/pqr_common.h
new file mode 100644
index 0000000..11e985c
--- /dev/null
+++ b/arch/x86/include/asm/pqr_common.h
@@ -0,0 +1,27 @@
+#ifndef _X86_RDT_H_
+#define _X86_RDT_H_
+
+#define MSR_IA32_PQR_ASSOC	0x0c8f
+
+/**
+ * struct intel_pqr_state - State cache for the PQR MSR
+ * @rmid:		The cached Resource Monitoring ID
+ * @closid:		The cached Class Of Service ID
+ * @rmid_usecnt:	The usage counter for rmid
+ *
+ * The upper 32 bits of MSR_IA32_PQR_ASSOC contain closid and the
+ * lower 10 bits rmid. The update to MSR_IA32_PQR_ASSOC always
+ * contains both parts, so we need to cache them.
+ *
+ * The cache also helps to avoid pointless updates if the value does
+ * not change.
+ */
+struct intel_pqr_state {
+	u32			rmid;
+	u32			closid;
+	int			rmid_usecnt;
+};
+
+DECLARE_PER_CPU(struct intel_pqr_state, pqr_state);
+
+#endif
diff --git a/arch/x86/kernel/cpu/intel_rdt.c b/arch/x86/kernel/cpu/intel_rdt.c
index 9cf3a7d..9f30492 100644
--- a/arch/x86/kernel/cpu/intel_rdt.c
+++ b/arch/x86/kernel/cpu/intel_rdt.c
@@ -21,6 +21,8 @@
  */
 #include <linux/slab.h>
 #include <linux/err.h>
+#include <linux/sched.h>
+#include <asm/pqr_common.h>
 #include <asm/intel_rdt.h>
 
 /*
@@ -41,12 +43,28 @@ static cpumask_t rdt_cpumask;
  */
 static cpumask_t tmp_cpumask;
 static DEFINE_MUTEX(rdtgroup_mutex);
+struct static_key __read_mostly rdt_enable_key = STATIC_KEY_INIT_FALSE;
 
 struct rdt_remote_data {
 	int msr;
 	u64 val;
 };
 
+void __intel_rdt_sched_in(void *dummy)
+{
+	struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
+
+	/*
+	 * Currently closid is always 0. When  user interface is added,
+	 * closid will come from user interface.
+	 */
+	if (state->closid == 0)
+		return;
+
+	wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, 0);
+	state->closid = 0;
+}
+
 static inline void closid_get(u32 closid)
 {
 	struct clos_cbm_table *cct = &cctable[closid];
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 63236d8..1c98f80 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -48,6 +48,7 @@
 #include <asm/syscalls.h>
 #include <asm/debugreg.h>
 #include <asm/switch_to.h>
+#include <asm/intel_rdt.h>
 #include <asm/xen/hypervisor.h>
 
 asmlinkage extern void ret_from_fork(void);
@@ -472,6 +473,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 			loadsegment(ss, __KERNEL_DS);
 	}
 
+	/*
+	 * Load the Intel cache allocation PQR MSR.
+	 */
+	intel_rdt_sched_in();
+
 	return prev_p;
 }
 
-- 
2.5.0

[toc] | [prev] | [next] | [standalone]

#1479013 — Re: [PATCH v2 10/33] x86/intel_rdt: Implement scheduling support for Intel RDT

On Thu, 8 Sep 2016, Fenghua Yu wrote:
> +extern struct static_key rdt_enable_key;
> +void __intel_rdt_sched_in(void *dummy);
> +
>  struct clos_cbm_table {
>  	unsigned long cbm;
>  	unsigned int clos_refcnt;
>  };
>  
> +/*
> + * intel_rdt_sched_in() - Writes the task's CLOSid to IA32_PQR_MSR
> + *
> + * Following considerations are made so that this has minimal impact
> + * on scheduler hot path:
> + * - This will stay as no-op unless we are running on an Intel SKU
> + * which supports L3 cache allocation.
> + * - When support is present and enabled, does not do any
> + * IA32_PQR_MSR writes until the user starts really using the feature
> + * ie creates a rdtgroup directory and assigns a cache_mask thats
> + * different from the root rdtgroup's cache_mask.
> + * - Caches the per cpu CLOSid values and does the MSR write only
> + * when a task with a different CLOSid is scheduled in. That
> + * means the task belongs to a different rdtgroup.
> + * - Closids are allocated so that different rdtgroup directories
> + * with same cache_mask gets the same CLOSid. This minimizes CLOSids
> + * used and reduces MSR write frequency.
> + */
> +static inline void intel_rdt_sched_in(void)
> +{
> +	/*
> +	 * Call the schedule in code only when RDT is enabled.
> +	 */
> +	if (static_key_false(&rdt_enable_key))

static_branch_[un]likely() is the proper function to use.

> +		__intel_rdt_sched_in(NULL);
> +
> +void __intel_rdt_sched_in(void *dummy)

What's the purpose of this dummy argument?

> +{
> +	struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
> +
> +	/*
> +	 * Currently closid is always 0. When  user interface is added,
> +	 * closid will come from user interface.
> +	 */
> +	if (state->closid == 0)
> +		return;
> +
> +	wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, 0);
> +	state->closid = 0;
> +}

Thanks,

	tglx

[toc] | [prev] | [next] | [standalone]

#1478837 — [PATCH v2 16/33] x86/intel_rdt: Class of service and capacity bitmask management for CDP

From: Vikas Shivappa <vikas.shivappa@linux.intel.com>

Add support to manage CLOSid(CLass Of Service id) and capacity
bitmask(cbm) for code data prioritization(CDP).

Closid management includes changes to allocating, freeing closid and
closid_get and closid_put and changes to closid availability map during
CDP set up. CDP has a separate cbm for code and data.

Each closid is mapped to a (dcache_cbm, icache_cbm) pair when cdp mode
is enabled.

Signed-off-by: Vikas Shivappa <vikas.shivappa@linux.intel.com>
Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
---
 arch/x86/kernel/cpu/intel_rdt.c | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/cpu/intel_rdt.c b/arch/x86/kernel/cpu/intel_rdt.c
index e0f23b6..9cee3fe 100644
--- a/arch/x86/kernel/cpu/intel_rdt.c
+++ b/arch/x86/kernel/cpu/intel_rdt.c
@@ -27,7 +27,13 @@
 #include <asm/intel_rdt.h>
 
 /*
- * cctable maintains 1:1 mapping between CLOSid and cache bitmask.
+ * During cache alloc mode cctable maintains 1:1 mapping between
+ * CLOSid and cache bitmask.
+ *
+ * During CDP mode, the cctable maintains a 1:2 mapping between the closid
+ * and (dcache_cbm, icache_cbm) pair.
+ * index of a dcache_cbm for CLOSid 'n' = n << 1.
+ * index of a icache_cbm for CLOSid 'n' = n << 1 + 1
  */
 static struct clos_cbm_table *cctable;
 /*
@@ -50,6 +56,13 @@ bool cdp_enabled;
 
 #define __DCBM_TABLE_INDEX(x)	(x << 1)
 #define __ICBM_TABLE_INDEX(x)	((x << 1) + 1)
+#define __DCBM_MSR_INDEX(x)			\
+	CBM_FROM_INDEX(__DCBM_TABLE_INDEX(x))
+#define __ICBM_MSR_INDEX(x)			\
+	CBM_FROM_INDEX(__ICBM_TABLE_INDEX(x))
+
+#define DCBM_TABLE_INDEX(x)	(x << cdp_enabled)
+#define ICBM_TABLE_INDEX(x)	((x << cdp_enabled) + cdp_enabled)
 
 struct rdt_remote_data {
 	int msr;
@@ -107,9 +120,12 @@ void __intel_rdt_sched_in(void *dummy)
 	state->closid = 0;
 }
 
+/*
+ * When cdp mode is enabled, refcnt is maintained in the dcache_cbm entry.
+ */
 static inline void closid_get(u32 closid)
 {
-	struct clos_cbm_table *cct = &cctable[closid];
+	struct clos_cbm_table *cct = &cctable[DCBM_TABLE_INDEX(closid)];
 
 	lockdep_assert_held(&rdtgroup_mutex);
 
@@ -139,7 +155,7 @@ static int closid_alloc(u32 *closid)
 static inline void closid_free(u32 closid)
 {
 	clear_bit(closid, cconfig.closmap);
-	cctable[closid].cbm = 0;
+	cctable[DCBM_TABLE_INDEX(closid)].cbm = 0;
 
 	if (WARN_ON(!cconfig.closids_used))
 		return;
@@ -149,7 +165,7 @@ static inline void closid_free(u32 closid)
 
 static void closid_put(u32 closid)
 {
-	struct clos_cbm_table *cct = &cctable[closid];
+	struct clos_cbm_table *cct = &cctable[DCBM_TABLE_INDEX(closid)];
 
 	lockdep_assert_held(&rdtgroup_mutex);
 	if (WARN_ON(!cct->clos_refcnt))
-- 
2.5.0

[toc] | [prev] | [next] | [standalone]

#1479066 — Re: [PATCH v2 16/33] x86/intel_rdt: Class of service and capacity bitmask management for CDP

On Thu, 8 Sep 2016, Fenghua Yu wrote:

> Add support to manage CLOSid(CLass Of Service id) and capacity
> bitmask(cbm) for code data prioritization(CDP).

I manage to understand that.

> Closid management includes changes to allocating, freeing closid and
> closid_get and closid_put and changes to closid availability map during
> CDP set up. 

But this is just a random sequence of word, function names and a reference
to the availability map which is not touched at all in this patch.

> CDP has a separate cbm for code and data.

> +/*
> + * When cdp mode is enabled, refcnt is maintained in the dcache_cbm entry.

Sorry. I really cannot figure out what that means.

> + */
>  static inline void closid_get(u32 closid)
>  {
> -	struct clos_cbm_table *cct = &cctable[closid];
> +	struct clos_cbm_table *cct = &cctable[DCBM_TABLE_INDEX(closid)];
>  
>  	lockdep_assert_held(&rdtgroup_mutex);
>  
> @@ -139,7 +155,7 @@ static int closid_alloc(u32 *closid)
>  static inline void closid_free(u32 closid)
>  {
>  	clear_bit(closid, cconfig.closmap);
> -	cctable[closid].cbm = 0;
> +	cctable[DCBM_TABLE_INDEX(closid)].cbm = 0;
>  
>  	if (WARN_ON(!cconfig.closids_used))
>  		return;
> @@ -149,7 +165,7 @@ static inline void closid_free(u32 closid)
>  
>  static void closid_put(u32 closid)
>  {
> -	struct clos_cbm_table *cct = &cctable[closid];
> +	struct clos_cbm_table *cct = &cctable[DCBM_TABLE_INDEX(closid)];

So if CDP is disabled we look at table[closid] and if it's enabled we look
at table[closid << 1]. What is managing the interleaved entries in the
table?

Thanks,

	tglx

[toc] | [prev] | [next] | [standalone]

#1478838 — [PATCH v2 04/33] drivers/base/cacheinfo.c: Export some cacheinfo functions for others to use

From: Fenghua Yu <fenghua.yu@intel.com>

We use ci_cpu_cacheinfo in CAT. Export this function for CAT to reuse.

Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
---
 drivers/base/cacheinfo.c  | 2 +-
 include/linux/cacheinfo.h | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index 2a21c15..f6e269a 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -29,7 +29,7 @@
 #include <linux/sysfs.h>
 
 /* pointer to per cpu cacheinfo */
-static DEFINE_PER_CPU(struct cpu_cacheinfo, ci_cpu_cacheinfo);
+DEFINE_PER_CPU(struct cpu_cacheinfo, ci_cpu_cacheinfo);
 #define ci_cacheinfo(cpu)	(&per_cpu(ci_cpu_cacheinfo, cpu))
 #define cache_leaves(cpu)	(ci_cacheinfo(cpu)->num_leaves)
 #define per_cpu_cacheinfo(cpu)	(ci_cacheinfo(cpu)->info_list)
diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h
index cf6984d..fa5e829 100644
--- a/include/linux/cacheinfo.h
+++ b/include/linux/cacheinfo.h
@@ -94,6 +94,8 @@ int func(unsigned int cpu)					\
 	return ret;						\
 }
 
+#define ci_cacheinfo(cpu)       (&per_cpu(ci_cpu_cacheinfo, cpu))
+
 struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu);
 int init_cache_level(unsigned int cpu);
 int populate_cache_leaves(unsigned int cpu);
-- 
2.5.0

[toc] | [prev] | [next] | [standalone]

#1478908 — Re: [PATCH v2 04/33] drivers/base/cacheinfo.c: Export some cacheinfo functions for others to use

On Thu, 8 Sep 2016, Fenghua Yu wrote:

> From: Fenghua Yu <fenghua.yu@intel.com>
> 
> We use ci_cpu_cacheinfo in CAT. Export this function for CAT to reuse.

So ci_cpu_cacheinfo is a function? AFAICT it's a struct.
 
> +#define ci_cacheinfo(cpu)       (&per_cpu(ci_cpu_cacheinfo, cpu))

Why a define and not an inline? &per_cpu should be per_cpu_ptr ....

And a define is not a function either and certainly that whole thing has
nothing to do with an export.

Furthermore $subject talks about some functions. I still have to see one.

It's an art to get a onliner patch screwed up in more than one way so
badly.

No bisquit!

	tglx

[toc] | [prev] | [next] | [standalone]

#1478839 — [PATCH v2 25/33] include/linux/resctrl.h: Define fork and exit functions in a new header file

From: Fenghua Yu <fenghua.yu@intel.com>

A new header file is created in include/linux/resctrl.h. It contains
defintions of rdtgroup_fork and rdtgroup_exit for x86. The functions
are empty for other architectures.

Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
---
 include/linux/resctrl.h | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 include/linux/resctrl.h

diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
new file mode 100644
index 0000000..68dabc4
--- /dev/null
+++ b/include/linux/resctrl.h
@@ -0,0 +1,12 @@
+#ifndef _LINUX_RESCTRL_H
+#define _LINUX_RESCTRL_H
+
+#ifdef CONFIG_INTEL_RDT
+extern void rdtgroup_fork(struct task_struct *child);
+extern void rdtgroup_exit(struct task_struct *tsk);
+#else
+static inline void rdtgroup_fork(struct task_struct *child) {}
+static inline void rdtgroup_exit(struct task_struct *tsk) {}
+#endif /* CONFIG_X86 */
+
+#endif /* _LINUX_RESCTRL_H */
-- 
2.5.0

[toc] | [prev] | [next] | [standalone]

#1479364 — Re: [PATCH v2 25/33] include/linux/resctrl.h: Define fork and exit functions in a new header file

On Thu, 8 Sep 2016, Fenghua Yu wrote:

> From: Fenghua Yu <fenghua.yu@intel.com>
> 
> A new header file is created in include/linux/resctrl.h. It contains

No comment. You should be able to guess what I'm trying not to say.

> defintions of rdtgroup_fork and rdtgroup_exit for x86. The functions
> are empty for other architectures.
> 
> Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
> Reviewed-by: Tony Luck <tony.luck@intel.com>
> ---
>  include/linux/resctrl.h | 12 ++++++++++++
>  1 file changed, 12 insertions(+)
>  create mode 100644 include/linux/resctrl.h
> 
> diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
> new file mode 100644
> index 0000000..68dabc4
> --- /dev/null
> +++ b/include/linux/resctrl.h
> @@ -0,0 +1,12 @@
> +#ifndef _LINUX_RESCTRL_H
> +#define _LINUX_RESCTRL_H
> +
> +#ifdef CONFIG_INTEL_RDT
> +extern void rdtgroup_fork(struct task_struct *child);
> +extern void rdtgroup_exit(struct task_struct *tsk);
> +#else
> +static inline void rdtgroup_fork(struct task_struct *child) {}
> +static inline void rdtgroup_exit(struct task_struct *tsk) {}
> +#endif /* CONFIG_X86 */
> +
> +#endif /* _LINUX_RESCTRL_H */
> -- 
> 2.5.0
> 
> 

[toc] | [prev] | [next] | [standalone]

#1478840 — [PATCH v2 27/33] x86/intel_rdt_rdtgroup.c: Implement resctrl file system commands

From: Fenghua Yu <fenghua.yu@intel.com>

Four basic file system commands are implement for resctrl.
mount, umount, mkdir, and rmdir.

Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
---
 arch/x86/include/asm/intel_rdt_rdtgroup.h |   1 +
 arch/x86/kernel/cpu/intel_rdt_rdtgroup.c  | 211 ++++++++++++++++++++++++++++++
 2 files changed, 212 insertions(+)

diff --git a/arch/x86/include/asm/intel_rdt_rdtgroup.h b/arch/x86/include/asm/intel_rdt_rdtgroup.h
index 92208a2..43a3b83 100644
--- a/arch/x86/include/asm/intel_rdt_rdtgroup.h
+++ b/arch/x86/include/asm/intel_rdt_rdtgroup.h
@@ -10,6 +10,7 @@
 /* Defined in intel_rdt_rdtgroup.c.*/
 extern int __init rdtgroup_init(void);
 extern void rdtgroup_exit(struct task_struct *tsk);
+extern bool rdtgroup_mounted;
 
 /* Defined in intel_rdt.c. */
 extern struct list_head rdtgroup_lists;
diff --git a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
index acea62c..71231ba 100644
--- a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
+++ b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
@@ -51,6 +51,13 @@ static int rdt_info_show(struct seq_file *seq, void *v);
 static int rdt_max_closid_show(struct seq_file *seq, void *v);
 static int rdt_max_cbm_len_show(struct seq_file *seq, void *v);
 static int domain_to_cache_id_show(struct seq_file *seq, void *v);
+static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
+			umode_t mode);
+static int rdtgroup_rmdir(struct kernfs_node *kn);
+static struct dentry *rdt_mount(struct file_system_type *fs_type,
+			 int flags, const char *unused_dev_name,
+			 void *data);
+static void rdt_kill_sb(struct super_block *sb);
 
 /* rdtgroup core interface files */
 static struct rftype rdtgroup_root_base_files[] = {
@@ -112,12 +119,24 @@ static struct rftype rdtgroup_partition_base_files[] = {
 	},
 };
 
+static struct kernfs_syscall_ops rdtgroup_kf_syscall_ops = {
+	.mkdir          = rdtgroup_mkdir,
+	.rmdir          = rdtgroup_rmdir,
+};
+
+static struct file_system_type rdt_fs_type = {
+	.name = "resctrl",
+	.mount = rdt_mount,
+	.kill_sb = rdt_kill_sb,
+};
+
 struct rdtgroup *root_rdtgrp;
 static struct rftype rdtgroup_partition_base_files[];
 struct cache_domain cache_domains[MAX_CACHE_LEAVES];
 /* The default hierarchy. */
 struct rdtgroup_root rdtgrp_dfl_root;
 static struct list_head rdtgroups;
+bool rdtgroup_mounted;
 
 /*
  * kernfs_root - find out the kernfs_root a kernfs_node belongs to
@@ -730,6 +749,110 @@ static void rdtgroup_destroy_locked(struct rdtgroup *rdtgrp)
 	kernfs_remove(rdtgrp->kn);
 }
 
+static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
+			umode_t mode)
+{
+	struct rdtgroup *parent, *rdtgrp;
+	struct rdtgroup_root *root;
+	struct kernfs_node *kn;
+	int ret;
+
+	if (parent_kn != root_rdtgrp->kn)
+		return -EPERM;
+
+	/* Do not accept '\n' to avoid unparsable situation.
+	 */
+	if (strchr(name, '\n'))
+		return -EINVAL;
+
+	parent = rdtgroup_kn_lock_live(parent_kn);
+	if (!parent)
+		return -ENODEV;
+	root = parent->root;
+
+	/* allocate the rdtgroup. */
+	rdtgrp = kzalloc(sizeof(*rdtgrp), GFP_KERNEL);
+	if (!rdtgrp) {
+		ret = -ENOMEM;
+		goto out_unlock;
+	}
+
+	INIT_LIST_HEAD(&rdtgrp->pset.tasks);
+
+	cpumask_clear(&rdtgrp->cpu_mask);
+
+	rdtgrp->root = root;
+
+	/* create the directory */
+	kn = kernfs_create_dir(parent->kn, name, mode, rdtgrp);
+	if (IS_ERR(kn)) {
+		ret = PTR_ERR(kn);
+		goto out_cancel_ref;
+	}
+	rdtgrp->kn = kn;
+
+	/*
+	 * This extra ref will be put in kernfs_remove() and guarantees
+	 * that @rdtgrp->kn is always accessible.
+	 */
+	kernfs_get(kn);
+
+	atomic_inc(&root->nr_rdtgrps);
+
+	ret = rdtgroup_kn_set_ugid(kn);
+	if (ret)
+		goto out_destroy;
+
+	ret = rdtgroup_partition_populate_dir(kn);
+	if (ret)
+		goto out_destroy;
+
+	kernfs_activate(kn);
+
+	list_add_tail(&rdtgrp->rdtgroup_list, &rdtgroup_lists);
+	/* Generate default schema for rdtgrp. */
+	ret = get_default_resources(rdtgrp);
+	if (ret)
+		goto out_destroy;
+
+	ret = 0;
+	goto out_unlock;
+
+out_cancel_ref:
+	kfree(rdtgrp);
+out_unlock:
+	rdtgroup_kn_unlock(parent_kn);
+	return ret;
+
+out_destroy:
+	rdtgroup_destroy_locked(rdtgrp);
+	goto out_unlock;
+}
+
+static int rdtgroup_rmdir(struct kernfs_node *kn)
+{
+	struct rdtgroup *rdtgrp;
+	int cpu;
+	int ret = 0;
+
+	rdtgrp = rdtgroup_kn_lock_live(kn);
+	if (!rdtgrp)
+		return -ENODEV;
+
+	if (!list_empty(&rdtgrp->pset.tasks)) {
+		ret = -EBUSY;
+		goto out;
+	}
+
+	for_each_cpu(cpu, &rdtgrp->cpu_mask)
+		per_cpu(cpu_rdtgroup, cpu) = root_rdtgrp;
+
+	rdtgroup_destroy_locked(rdtgrp);
+
+out:
+	rdtgroup_kn_unlock(kn);
+	return ret;
+}
 static int
 rdtgroup_move_task_all(struct rdtgroup *src_rdtgrp, struct rdtgroup *dst_rdtgrp)
 {
@@ -978,3 +1101,91 @@ void rdtgroup_fork(struct task_struct *child)
 out:
 	mutex_unlock(&rdtgroup_mutex);
 }
+
+static struct dentry *rdt_mount(struct file_system_type *fs_type,
+			 int flags, const char *unused_dev_name,
+			 void *data)
+{
+	struct super_block *pinned_sb = NULL;
+	struct rdtgroup_root *root;
+	struct dentry *dentry;
+	int ret;
+	bool new_sb;
+
+	/*
+	 * The first time anyone tries to mount a rdtgroup, enable the list
+	 * linking tasks and fix up all existing tasks.
+	 */
+	if (rdtgroup_mounted)
+		return ERR_PTR(-EBUSY);
+
+	rdt_opts.cdp_enabled = false;
+	rdt_opts.verbose = false;
+	cdp_enabled = false;
+
+	ret = parse_rdtgroupfs_options(data);
+	if (ret)
+		goto out_mount;
+
+	if (rdt_opts.cdp_enabled) {
+		cdp_enabled = true;
+		cconfig.max_closid >>= cdp_enabled;
+		pr_info("CDP is enabled\n");
+	}
+
+	init_msrs(cdp_enabled);
+
+	root = &rdtgrp_dfl_root;
+
+	ret = get_default_resources(&root->rdtgrp);
+	if (ret)
+		return ERR_PTR(-ENOSPC);
+
+out_mount:
+	dentry = kernfs_mount(fs_type, flags, root->kf_root,
+			      RDTGROUP_SUPER_MAGIC,
+			      &new_sb);
+	if (IS_ERR(dentry) || !new_sb)
+		goto out_unlock;
+
+	/*
+	 * If @pinned_sb, we're reusing an existing root and holding an
+	 * extra ref on its sb.  Mount is complete.  Put the extra ref.
+	 */
+	if (pinned_sb) {
+		WARN_ON(new_sb);
+		deactivate_super(pinned_sb);
+	}
+
+	INIT_LIST_HEAD(&root->rdtgrp.pset.tasks);
+
+	cpumask_copy(&root->rdtgrp.cpu_mask, cpu_online_mask);
+	static_key_slow_inc(&rdt_enable_key);
+	rdtgroup_mounted = true;
+
+	return dentry;
+
+out_unlock:
+	return ERR_PTR(ret);
+}
+
+static void rdt_kill_sb(struct super_block *sb)
+{
+	mutex_lock(&rdtgroup_mutex);
+
+	rmdir_all_sub();
+
+	static_key_slow_dec(&rdt_enable_key);
+
+	release_root_closid();
+	root_rdtgrp->resource.valid = false;
+
+	/* Restore max_closid to original value. */
+	cconfig.max_closid <<= cdp_enabled;
+
+	kernfs_kill_sb(sb);
+	INIT_LIST_HEAD(&root_rdtgrp->pset.tasks);
+	rdtgroup_mounted = false;
+
+	mutex_unlock(&rdtgroup_mutex);
+}
-- 
2.5.0

[toc] | [prev] | [next] | [standalone]

#1479486 — Re: [PATCH v2 27/33] x86/intel_rdt_rdtgroup.c: Implement resctrl file system commands

On Thu, 8 Sep 2016, Fenghua Yu wrote:
> +static struct kernfs_syscall_ops rdtgroup_kf_syscall_ops = {
> +	.mkdir          = rdtgroup_mkdir,
> +	.rmdir          = rdtgroup_rmdir,
> +};
> +
> +static struct file_system_type rdt_fs_type = {
> +	.name = "resctrl",
> +	.mount = rdt_mount,
> +	.kill_sb = rdt_kill_sb,
> +};

So the above struct is nicely aligned and readable. While this one is
not. Sigh,

>  struct rdtgroup *root_rdtgrp;
>  static struct rftype rdtgroup_partition_base_files[];
>  struct cache_domain cache_domains[MAX_CACHE_LEAVES];
>  /* The default hierarchy. */
>  struct rdtgroup_root rdtgrp_dfl_root;
>  static struct list_head rdtgroups;
> +bool rdtgroup_mounted;

Your choice of global/static visible variables is driven by a random
generator or what?
  
>  /*
>   * kernfs_root - find out the kernfs_root a kernfs_node belongs to
> @@ -730,6 +749,110 @@ static void rdtgroup_destroy_locked(struct rdtgroup *rdtgrp)
>  	kernfs_remove(rdtgrp->kn);
>  }
>  
> +static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
> +			umode_t mode)
> +{
> +	struct rdtgroup *parent, *rdtgrp;
> +	struct rdtgroup_root *root;
> +	struct kernfs_node *kn;
> +	int ret;
> +
> +	if (parent_kn != root_rdtgrp->kn)
> +		return -EPERM;
> +
> +	/* Do not accept '\n' to avoid unparsable situation.
> +	 */

Where did copy you this comment style from? Its' horrible and here is a
lengthy explanation why: https://lkml.org/lkml/2016/7/8/625

> +static struct dentry *rdt_mount(struct file_system_type *fs_type,
> +			 int flags, const char *unused_dev_name,
> +			 void *data)
> +{
> +	struct super_block *pinned_sb = NULL;
> +	struct rdtgroup_root *root;
> +	struct dentry *dentry;
> +	int ret;
> +	bool new_sb;
> +
> +	/*
> +	 * The first time anyone tries to mount a rdtgroup, enable the list
> +	 * linking tasks and fix up all existing tasks.

What are 'list linking tasks'? What is fixed up here?

> +	 */
> +	if (rdtgroup_mounted)
> +		return ERR_PTR(-EBUSY);

How is this serialized against concurrent mounts? Oh well....

> +	rdt_opts.cdp_enabled = false;
> +	rdt_opts.verbose = false;
> +	cdp_enabled = false;
> +
> +	ret = parse_rdtgroupfs_options(data);
> +	if (ret)
> +		goto out_mount;
> +
> +	if (rdt_opts.cdp_enabled) {
> +		cdp_enabled = true;
> +		cconfig.max_closid >>= cdp_enabled;
> +		pr_info("CDP is enabled\n");
> +	}
> +
> +	init_msrs(cdp_enabled);
> +
> +	root = &rdtgrp_dfl_root;
> +
> +	ret = get_default_resources(&root->rdtgrp);
> +	if (ret)
> +		return ERR_PTR(-ENOSPC);
> +
> +out_mount:
> +	dentry = kernfs_mount(fs_type, flags, root->kf_root,
> +			      RDTGROUP_SUPER_MAGIC,
> +			      &new_sb);
> +	if (IS_ERR(dentry) || !new_sb)
> +		goto out_unlock;

And ret , which is returned @out_unlock is 0. So instead of returning the
error code encoded in dentry you return a NULL pointer.

> +	/*
> +	 * If @pinned_sb, we're reusing an existing root and holding an
> +	 * extra ref on its sb.  Mount is complete.  Put the extra ref.
> +	 */
> +	if (pinned_sb) {

And how exactly becomes pinned_sb != NULL?

> +		WARN_ON(new_sb);
> +		deactivate_super(pinned_sb);
> +	}

Not at all.

> +	INIT_LIST_HEAD(&root->rdtgrp.pset.tasks);
> +
> +	cpumask_copy(&root->rdtgrp.cpu_mask, cpu_online_mask);
> +	static_key_slow_inc(&rdt_enable_key);
> +	rdtgroup_mounted = true;
> +
> +	return dentry;
> +
> +out_unlock:
> +	return ERR_PTR(ret);

So a return magically unlocks stuff. Or does this happen in ERR_PTR()?

This jump label is not only pointless it's also named badly.

Thanks,

	tglx

[toc] | [prev] | [next] | [standalone]

#1479557 — Re: [PATCH v2 27/33] x86/intel_rdt_rdtgroup.c: Implement resctrl file system commands

On Thu, Sep 08, 2016 at 03:04:12PM -0700, Shaohua Li wrote:
> On Thu, Sep 08, 2016 at 02:57:21AM -0700, Fenghua Yu wrote:
> >  /*
> >   * kernfs_root - find out the kernfs_root a kernfs_node belongs to
> > @@ -730,6 +749,110 @@ static void rdtgroup_destroy_locked(struct rdtgroup *rdtgrp)
> >  	kernfs_remove(rdtgrp->kn);
> >  }
> >  
> > +static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
> > +			umode_t mode)
> > +{
> > +	struct rdtgroup *parent, *rdtgrp;
> > +	struct rdtgroup_root *root;
> > +	struct kernfs_node *kn;
> > +	int ret;
> > +
> > +	if (parent_kn != root_rdtgrp->kn)
> > +		return -EPERM;
> > +
> 
> So we can't create nested groups. Is this limitation temporary? I don't see
> this is mentioned in the interface document.

Yes, we cannot create nested groups, i.e. up to one level of sub-directory
in the resctrl file system. This is different from cgroup.

I can add this info in the intel_rdt_ui.txt.

Thanks.

-Fenghua

[toc] | [prev] | [next] | [standalone]

#1478841 — [PATCH v2 29/33] x86/intel_rdt_rdtgroup.c: Tasks iterator and write

From: Fenghua Yu <fenghua.yu@intel.com>

"tasks" file in rdtgroup contains task pids. User can move a task pid
to one directory. A task can only stay in one directory at the same
time.

Each rdtgroup contains a rg_list. When a pid is written to this
rdtgroup's tasks, the task's rg_list is added in the rdtgroup's
linked list and deleted from its previous rdtgroup's linked list.

When user reads the "tasks" file, all pids are shown in the order
from small to large.

Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
---
 arch/x86/kernel/cpu/intel_rdt_rdtgroup.c | 161 +++++++++++++++++++++++--------
 1 file changed, 120 insertions(+), 41 deletions(-)

diff --git a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
index 6c3161a..16f7195 100644
--- a/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
+++ b/arch/x86/kernel/cpu/intel_rdt_rdtgroup.c
@@ -63,6 +63,9 @@ static int rdtgroup_cpus_show(struct seq_file *s, void *v);
 static ssize_t rdtgroup_cpus_write(struct kernfs_open_file *of,
 			char *buf, size_t nbytes, loff_t off);
 
+static int rdtgroup_tasks_show(struct seq_file *s, void *v);
+static ssize_t rdtgroup_tasks_write(struct kernfs_open_file *of,
+				  char *buf, size_t nbytes, loff_t off);
 /* rdtgroup core interface files */
 static struct rftype rdtgroup_root_base_files[] = {
 	{
@@ -720,14 +723,41 @@ static __init void init_cache_domains(void)
 
 void rdtgroup_exit(struct task_struct *tsk)
 {
+	if (tsk->rdtgroup) {
+		atomic_dec(&tsk->rdtgroup->refcount);
+		tsk->rdtgroup = NULL;
+	}
+}
 
-	if (!list_empty(&tsk->rg_list)) {
-		struct rdtgroup *rdtgrp = tsk->rdtgroup;
+static void show_rdt_tasks(struct rdtgroup *r, struct seq_file *s)
+{
+	struct task_struct *p;
+	struct rdtgroup *this = r;
 
-		list_del_init(&tsk->rg_list);
-		tsk->rdtgroup = NULL;
-		atomic_dec(&rdtgrp->refcount);
+
+	if (r == root_rdtgrp)
+		return;
+
+	rcu_read_lock();
+	for_each_process(p) {
+		if (p->rdtgroup == this)
+			seq_printf(s, "%d\n", p->pid);
 	}
+	rcu_read_unlock();
+}
+
+static int rdtgroup_tasks_show(struct seq_file *s, void *v)
+{
+	struct kernfs_open_file *of = s->private;
+	struct rdtgroup *rdtgrp;
+
+	rdtgrp = rdtgroup_kn_lock_live(of->kn);
+	if (rdtgrp == NULL)
+		return -ENODEV;
+
+	show_rdt_tasks(rdtgrp, s);
+	rdtgroup_kn_unlock(of->kn);
+	return 0;
 }
 
 static void rdtgroup_destroy_locked(struct rdtgroup *rdtgrp)
@@ -781,8 +811,6 @@ static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
 		goto out_unlock;
 	}
 
-	INIT_LIST_HEAD(&rdtgrp->pset.tasks);
-
 	cpumask_clear(&rdtgrp->cpu_mask);
 
 	rdtgrp->root = root;
@@ -843,7 +871,7 @@ static int rdtgroup_rmdir(struct kernfs_node *kn)
 	if (!rdtgrp)
 		return -ENODEV;
 
-	if (!list_empty(&rdtgrp->pset.tasks)) {
+	if (atomic_read(&rdtgrp->refcount)) {
 		ret = -EBUSY;
 		goto out;
 	}
@@ -857,28 +885,6 @@ out:
 	rdtgroup_kn_unlock(kn);
 	return ret;
 }
-static int
-rdtgroup_move_task_all(struct rdtgroup *src_rdtgrp, struct rdtgroup *dst_rdtgrp)
-{
-	struct list_head *tasks;
-
-	tasks = &src_rdtgrp->pset.tasks;
-	while (!list_empty(tasks)) {
-		struct task_struct *tsk;
-		struct list_head *pos;
-		pid_t pid;
-		int ret;
-
-		pos = tasks->next;
-		tsk = list_entry(pos, struct task_struct, rg_list);
-		pid = tsk->pid;
-		ret = rdtgroup_move_task(pid, dst_rdtgrp, false, NULL);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
 
 /*
  * Forcibly remove all of subdirectories under root.
@@ -1088,22 +1094,16 @@ void rdtgroup_fork(struct task_struct *child)
 {
 	struct rdtgroup *rdtgrp;
 
-	INIT_LIST_HEAD(&child->rg_list);
+	child->rdtgroup = NULL;
 	if (!rdtgroup_mounted)
 		return;
 
-	mutex_lock(&rdtgroup_mutex);
-
 	rdtgrp = current->rdtgroup;
 	if (!rdtgrp)
-		goto out;
+		return;
 
-	list_add_tail(&child->rg_list, &rdtgrp->pset.tasks);
 	child->rdtgroup = rdtgrp;
 	atomic_inc(&rdtgrp->refcount);
-
-out:
-	mutex_unlock(&rdtgroup_mutex);
 }
 
 static struct dentry *rdt_mount(struct file_system_type *fs_type,
@@ -1161,8 +1161,6 @@ out_mount:
 		deactivate_super(pinned_sb);
 	}
 
-	INIT_LIST_HEAD(&root->rdtgrp.pset.tasks);
-
 	cpumask_copy(&root->rdtgrp.cpu_mask, cpu_online_mask);
 	static_key_slow_inc(&rdt_enable_key);
 	rdtgroup_mounted = true;
@@ -1188,7 +1186,6 @@ static void rdt_kill_sb(struct super_block *sb)
 	cconfig.max_closid <<= cdp_enabled;
 
 	kernfs_kill_sb(sb);
-	INIT_LIST_HEAD(&root_rdtgrp->pset.tasks);
 	rdtgroup_mounted = false;
 
 	mutex_unlock(&rdtgroup_mutex);
@@ -1281,3 +1278,85 @@ end:
 
 	return ret ?: nbytes;
 }
+
+static int _rdtgroup_move_task(struct task_struct *tsk, struct rdtgroup *rdtgrp)
+{
+	if (tsk->rdtgroup)
+		atomic_dec(&tsk->rdtgroup->refcount);
+
+	if (rdtgrp == root_rdtgrp)
+		tsk->rdtgroup = NULL;
+	else
+		tsk->rdtgroup = rdtgrp;
+
+	atomic_inc(&rdtgrp->refcount);
+
+	return 0;
+}
+
+static int rdtgroup_move_task(pid_t pid, struct rdtgroup *rdtgrp,
+			      bool threadgroup, struct kernfs_open_file *of)
+{
+	struct task_struct *tsk;
+	int ret;
+
+	rcu_read_lock();
+	if (pid) {
+		tsk = find_task_by_vpid(pid);
+		if (!tsk) {
+			ret = -ESRCH;
+			goto out_unlock_rcu;
+		}
+	} else {
+		tsk = current;
+	}
+
+	if (threadgroup)
+		tsk = tsk->group_leader;
+
+	get_task_struct(tsk);
+	rcu_read_unlock();
+
+	ret = rdtgroup_procs_write_permission(tsk, of);
+	if (!ret)
+		_rdtgroup_move_task(tsk, rdtgrp);
+
+	put_task_struct(tsk);
+	goto out_unlock_threadgroup;
+
+out_unlock_rcu:
+	rcu_read_unlock();
+out_unlock_threadgroup:
+	return ret;
+}
+
+ssize_t _rdtgroup_procs_write(struct rdtgroup *rdtgrp,
+			   struct kernfs_open_file *of, char *buf,
+			   size_t nbytes, loff_t off, bool threadgroup)
+{
+	pid_t pid;
+	int ret;
+
+	if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
+		return -EINVAL;
+
+	ret = rdtgroup_move_task(pid, rdtgrp, threadgroup, of);
+
+	return ret ?: nbytes;
+}
+
+static ssize_t rdtgroup_tasks_write(struct kernfs_open_file *of,
+				  char *buf, size_t nbytes, loff_t off)
+{
+	struct rdtgroup *rdtgrp;
+	int ret;
+
+	rdtgrp = rdtgroup_kn_lock_live(of->kn);
+	if (!rdtgrp)
+		return -ENODEV;
+
+	ret = _rdtgroup_procs_write(rdtgrp, of, buf, nbytes, off, false);
+
+	rdtgroup_kn_unlock(of->kn);
+	return ret;
+}
-- 
2.5.0

[toc] | [prev] | [next] | [standalone]

#1479516 — Re: [PATCH v2 29/33] x86/intel_rdt_rdtgroup.c: Tasks iterator and write

On Thu, 8 Sep 2016, Fenghua Yu wrote:
>  void rdtgroup_exit(struct task_struct *tsk)
>  {
> +	if (tsk->rdtgroup) {
> +		atomic_dec(&tsk->rdtgroup->refcount);
> +		tsk->rdtgroup = NULL;

The changelog still talks about this list stuff while this patch seems to
remove it and solely rely on the tsk->rdtgroup pointer, which is a sensible
thing to do.

Writing sensible changelogs would make the life of reviewers too easy,
right?

> +	}
> +}
>  
> -	if (!list_empty(&tsk->rg_list)) {
> -		struct rdtgroup *rdtgrp = tsk->rdtgroup;
> +static void show_rdt_tasks(struct rdtgroup *r, struct seq_file *s)
> +{
> +	struct task_struct *p;
> +	struct rdtgroup *this = r;
>  
> -		list_del_init(&tsk->rg_list);
> -		tsk->rdtgroup = NULL;
> -		atomic_dec(&rdtgrp->refcount);
> +
> +	if (r == root_rdtgrp)
> +		return;

Why has the root_rdtgroup a task file in the first place?

>  static void rdtgroup_destroy_locked(struct rdtgroup *rdtgrp)
> @@ -781,8 +811,6 @@ static int rdtgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
>  		goto out_unlock;
>  	}
>  
> -	INIT_LIST_HEAD(&rdtgrp->pset.tasks);
> -
>  	cpumask_clear(&rdtgrp->cpu_mask);
>  
>  	rdtgrp->root = root;
> @@ -843,7 +871,7 @@ static int rdtgroup_rmdir(struct kernfs_node *kn)
>  	if (!rdtgrp)
>  		return -ENODEV;
>  
> -	if (!list_empty(&rdtgrp->pset.tasks)) {
> +	if (atomic_read(&rdtgrp->refcount)) {

So you rely on rdtgrp->refcount completely now.

>  /*
>   * Forcibly remove all of subdirectories under root.
> @@ -1088,22 +1094,16 @@ void rdtgroup_fork(struct task_struct *child)
>  {
>  	struct rdtgroup *rdtgrp;
>  
> -	INIT_LIST_HEAD(&child->rg_list);
> +	child->rdtgroup = NULL;
>  	if (!rdtgroup_mounted)
>  		return;
>  
> -	mutex_lock(&rdtgroup_mutex);
> -
>  	rdtgrp = current->rdtgroup;
>  	if (!rdtgrp)
> -		goto out;
> +		return;
>  
> -	list_add_tail(&child->rg_list, &rdtgrp->pset.tasks);
>  	child->rdtgroup = rdtgrp;
>  	atomic_inc(&rdtgrp->refcount);

This lacks any form of documentation WHY this is correct and works. I asked
you last time to document the locking and serialization rules ....

>  	cpumask_copy(&root->rdtgrp.cpu_mask, cpu_online_mask);

Can't remember if I told you already, but this is racy against hotplug.

> +static int _rdtgroup_move_task(struct task_struct *tsk, struct rdtgroup *rdtgrp)
> +{
> +	if (tsk->rdtgroup)
> +		atomic_dec(&tsk->rdtgroup->refcount);
> +
> +	if (rdtgrp == root_rdtgrp)
> +		tsk->rdtgroup = NULL;
> +	else
> +		tsk->rdtgroup = rdtgrp;
> +
> +	atomic_inc(&rdtgrp->refcount);
> +
> +	return 0;
> +}
> +
> +static int rdtgroup_move_task(pid_t pid, struct rdtgroup *rdtgrp,
> +			      bool threadgroup, struct kernfs_open_file *of)
> +{
> +	struct task_struct *tsk;
> +	int ret;
> +
> +	rcu_read_lock();
> +	if (pid) {
> +		tsk = find_task_by_vpid(pid);
> +		if (!tsk) {
> +			ret = -ESRCH;
> +			goto out_unlock_rcu;
> +		}
> +	} else {
> +		tsk = current;
> +	}
> +
> +	if (threadgroup)
> +		tsk = tsk->group_leader;
> +
> +	get_task_struct(tsk);
> +	rcu_read_unlock();
> +
> +	ret = rdtgroup_procs_write_permission(tsk, of);
> +	if (!ret)
> +		_rdtgroup_move_task(tsk, rdtgrp);
> +
> +	put_task_struct(tsk);
> +	goto out_unlock_threadgroup;
> +
> +out_unlock_rcu:
> +	rcu_read_unlock();
> +out_unlock_threadgroup:
> +	return ret;
> +}
> +
> +ssize_t _rdtgroup_procs_write(struct rdtgroup *rdtgrp,
> +			   struct kernfs_open_file *of, char *buf,
> +			   size_t nbytes, loff_t off, bool threadgroup)

global visible? And what is the underscore for?

> +{
> +	pid_t pid;
> +	int ret;
> +
> +	if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0)
> +		return -EINVAL;

Why do you evaluate the buffer inside the lock held region?

This function split is completely bogus and artificial.

> +
> +	ret = rdtgroup_move_task(pid, rdtgrp, threadgroup, of);
> +
> +	return ret ?: nbytes;
> +}
> +
> +static ssize_t rdtgroup_tasks_write(struct kernfs_open_file *of,
> +				  char *buf, size_t nbytes, loff_t off)
> +{
> +	struct rdtgroup *rdtgrp;
> +	int ret;
> +
> +	rdtgrp = rdtgroup_kn_lock_live(of->kn);
> +	if (!rdtgrp)
> +		return -ENODEV;
> +
> +	ret = _rdtgroup_procs_write(rdtgrp, of, buf, nbytes, off, false);
> +
> +	rdtgroup_kn_unlock(of->kn);
> +	return ret;
> +}

More sigh.

     tglx

[toc] | [prev] | [next] | [standalone]

#1478842 — [PATCH v2 17/33] x86/intel_rdt: Hot cpu update for code data prioritization

From: Vikas Shivappa <vikas.shivappa@linux.intel.com>

Updates hot cpu notification handling for code data prioritization(cdp).
The capacity bitmask(cbm) is global for both data and instruction and we
need to update the new online package with all the cbms by writing to
the IA32_L3_QOS_n MSRs.

Signed-off-by: Vikas Shivappa <vikas.shivappa@linux.intel.com>
Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
---
 arch/x86/kernel/cpu/intel_rdt.c | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/cpu/intel_rdt.c b/arch/x86/kernel/cpu/intel_rdt.c
index 9cee3fe..1bcff29 100644
--- a/arch/x86/kernel/cpu/intel_rdt.c
+++ b/arch/x86/kernel/cpu/intel_rdt.c
@@ -223,6 +223,26 @@ static inline bool rdt_cpumask_update(int cpu)
 	return false;
 }
 
+static void cbm_update_msr(u32 index)
+{
+	struct rdt_remote_data info;
+	int dindex;
+
+	dindex = DCBM_TABLE_INDEX(index);
+	if (cctable[dindex].clos_refcnt) {
+
+		info.msr = CBM_FROM_INDEX(dindex);
+		info.val = cctable[dindex].cbm;
+		msr_cpu_update((void *) &info);
+
+		if (cdp_enabled) {
+			info.msr = __ICBM_MSR_INDEX(index);
+			info.val = cctable[dindex + 1].cbm;
+			msr_cpu_update((void *) &info);
+		}
+	}
+}
+
 /*
  * cbm_update_msrs() - Updates all the existing IA32_L3_MASK_n MSRs
  * which are one per CLOSid on the current package.
@@ -230,15 +250,10 @@ static inline bool rdt_cpumask_update(int cpu)
 static void cbm_update_msrs(void *dummy)
 {
 	int maxid = cconfig.max_closid;
-	struct rdt_remote_data info;
 	unsigned int i;
 
 	for (i = 0; i < maxid; i++) {
-		if (cctable[i].clos_refcnt) {
-			info.msr = CBM_FROM_INDEX(i);
-			info.val = cctable[i].cbm;
-			msr_cpu_update((void *) &info);
-		}
+		cbm_update_msr(i);
 	}
 }
 
-- 
2.5.0

[toc] | [prev] | [next] | [standalone]

#1479061 — Re: [PATCH v2 17/33] x86/intel_rdt: Hot cpu update for code data prioritization

On Thu, 8 Sep 2016, Fenghua Yu wrote:
> Updates hot cpu notification handling for code data prioritization(cdp).

Some more useless information

> The capacity bitmask(cbm) is global for both data and instruction and we
> need to update the new online package with all the cbms by writing to
> the IA32_L3_QOS_n MSRs.

If I wouldn't know the details of the hardware then this explanation would
make me run away screaming ....

> +static void cbm_update_msr(u32 index)
> +{
> +	struct rdt_remote_data info;
> +	int dindex;
> +
> +	dindex = DCBM_TABLE_INDEX(index);
> +	if (cctable[dindex].clos_refcnt) {
> +
> +		info.msr = CBM_FROM_INDEX(dindex);
> +		info.val = cctable[dindex].cbm;
> +		msr_cpu_update((void *) &info);
> +
> +		if (cdp_enabled) {
> +			info.msr = __ICBM_MSR_INDEX(index);
> +			info.val = cctable[dindex + 1].cbm;
> +			msr_cpu_update((void *) &info);
> +		}
> +	}

As usual there is a complete lack of comments here.

> +}
> +
>  /*
>   * cbm_update_msrs() - Updates all the existing IA32_L3_MASK_n MSRs
>   * which are one per CLOSid on the current package.
> @@ -230,15 +250,10 @@ static inline bool rdt_cpumask_update(int cpu)
>  static void cbm_update_msrs(void *dummy)
>  {
>  	int maxid = cconfig.max_closid;
> -	struct rdt_remote_data info;
>  	unsigned int i;
>  
>  	for (i = 0; i < maxid; i++) {
> -		if (cctable[i].clos_refcnt) {
> -			info.msr = CBM_FROM_INDEX(i);
> -			info.val = cctable[i].cbm;
> -			msr_cpu_update((void *) &info);
> -		}
> +		cbm_update_msr(i);
>  	}

The curly braces can go as well.

Thanks,

	tglx

[toc] | [prev] | [next] | [standalone]

#1478844 — [PATCH v2 06/33] Documentation, x86: Documentation for Intel resource allocation user interface

From: Fenghua Yu <fenghua.yu@intel.com>

The documentation describes user interface of how to allocate resource
in Intel RDT.

Please note that the documentation covers generic user interface. Current
patch set code only implemente CAT L3. CAT L2 code will be sent later.

Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
---
 Documentation/x86/intel_rdt_ui.txt | 164 +++++++++++++++++++++++++++++++++++++
 1 file changed, 164 insertions(+)
 create mode 100644 Documentation/x86/intel_rdt_ui.txt

diff --git a/Documentation/x86/intel_rdt_ui.txt b/Documentation/x86/intel_rdt_ui.txt
new file mode 100644
index 0000000..27de386
--- /dev/null
+++ b/Documentation/x86/intel_rdt_ui.txt
@@ -0,0 +1,164 @@
+User Interface for Resource Allocation in Intel Resource Director Technology
+
+Copyright (C) 2016 Intel Corporation
+
+Fenghua Yu <fenghua.yu@intel.com>
+Tony Luck <tony.luck@intel.com>
+
+This feature is enabled by the CONFIG_INTEL_RDT Kconfig and the
+X86 /proc/cpuinfo flag bits "rdt", "cat_l3" and "cdp_l3".
+
+To use the feature mount the file system:
+
+ # mount -t resctrl resctrl [-o cdp,verbose] /sys/fs/resctrl
+
+mount options are:
+
+"cdp": Enable code/data prioritization in L3 cache allocations.
+
+"verbose": Output more info in the "info" file under info directory
+	and in dmesg. This is mainly for debug.
+
+
+Resource groups
+---------------
+Resource groups are represented as directories in the resctrl file
+system. The default group is the root directory. Other groups may be
+created as desired by the system administrator using the "mkdir(1)"
+command, and removed using "rmdir(1)".
+
+There are three files associated with each group:
+
+"tasks": A list of tasks that belongs to this group. Tasks can be
+	added to a group by writing the task ID to the "tasks" file
+	(which will automatically remove them from the previous
+	group to which they belonged). New tasks created by fork(2)
+	and clone(2) are added to the same group as their parent.
+	If a pid is not in any sub partition, it is in root partition
+	(i.e. default partition).
+
+"cpus": A bitmask of logical CPUs assigned to this group. Writing
+	a new mask can add/remove CPUs from this group. Added CPUs
+	are removed from their previous group. Removed ones are
+	given to the default (root) group.
+
+"schemata": A list of all the resources available to this group.
+	Each resource has its own line and format - see below for
+	details.
+
+When a task is running the following rules define which resources
+are available to it:
+
+1) If the task is a member of a non-default group, then the schemata
+for that group is used.
+
+2) Else if the task belongs to the default group, but is running on a
+CPU that is assigned to some specific group, then the schemata for
+the CPU's group is used.
+
+3) Otherwise the schemata for the default group is used.
+
+
+Schemata files - general concepts
+---------------------------------
+Each line in the file describes one resource. The line starts with
+the name of the resource, followed by specific values to be applied
+in each of the instances of that resource on the system.
+
+Cache IDs
+---------
+On current generation systems there is one L3 cache per socket and L2
+caches are generally just shared by the hyperthreads on a core, but this
+isn't an architectural requirement. We could have multiple separate L3
+caches on a socket, multiple cores could share an L2 cache. So instead
+of using "socket" or "core" to define the set of logical cpus sharing
+a resource we use a "Cache ID". At a given cache level this will be a
+unique number across the whole system (but it isn't guaranteed to be a
+contiguous sequence, there may be gaps).  To find the ID for each logical
+CPU look in /sys/devices/system/cpu/cpu*/cache/index*/id
+
+Cache Bit Masks (CBM)
+---------------------
+For cache resources we describe the portion of the cache that is available
+for allocation using a bitmask. The number of bits in the mask is defined
+by each cpu model (and may be different for different cache levels). It
+is found using CPUID, but is also provided in the "info" directory of
+the resctrl file system in "info/{resource}/max_cbm_len". X86 hardware
+requires that these masks have all the '1' bits in a contiguous block. So
+0x3, 0x6 and 0xC are legal 4-bit masks with two bits set, but 0x5, 0x9
+and 0xA are not.  On a system with a 20-bit mask each bit represents 5%
+of the capacity of the cache. You could partition the cache into four
+equal parts with masks: 0x1f, 0x3e0, 0x7c00, 0xf8000.
+
+
+L3 details (code and data prioritization disabled)
+--------------------------------------------------
+With CDP disabled the L3 schemata format is:
+
+	L3:<cache_id0>=<cbm>;<cache_id1>=<cbm>;...
+
+L3 details (CDP enabled via mount option to resctrl)
+----------------------------------------------------
+When CDP is enabled, you need to specify separate cache bit masks for
+code and data access. The generic format is:
+
+	L3:<cache_id0>=<d_cbm>,<i_cbm>;<cache_id1>=<d_cbm>,<i_cbm>;...
+
+where the d_cbm masks are for data access, and the i_cbm masks for code.
+
+
+Example 1
+---------
+On a two socket machine (one L3 cache per socket) with just four bits
+for cache bit masks
+
+# mount -t resctrl resctrl /sys/fs/resctrl
+# cd /sys/fs/resctrl
+# mkdir p0 p1
+# echo "L3:0=3;1=c" > /sys/fs/resctrl/p0/schemata
+# echo "L3:0=3;1=3" > /sys/fs/resctrl/p1/schemata
+
+The default resource group is unmodified, so we have access to all parts
+of all caches (its schemata file reads "L3:0=f;1=f").
+
+Tasks that are under the control of group "p0" may only allocate from the
+"lower" 50% on cache ID 0, and the "upper" 50% of cache ID 1.
+Tasks in group "p1" use the "lower" 50% of cache on both sockets.
+
+Example 2
+---------
+Again two sockets, but this time with a more realistic 20-bit mask.
+
+Two real time tasks pid=1234 running on processor 0 and pid=5678 running on
+processor 1 on socket 0 on a 2-socket and dual core machine. To avoid noisy
+neighbors, each of the two real-time tasks exclusively occupies one quarter
+of L3 cache on socket 0.
+
+# mount -t resctrl resctrl /sys/fs/resctrl
+# cd /sys/fs/resctrl
+
+First we reset the schemata for the default group so that the "upper"
+50% of the L3 cache on socket 0 cannot be used by ordinary tasks:
+
+# echo "L3:0=3ff;1=fffff" > schemata
+
+Next we make a resource group for our first real time task and give
+it access to the "top" 25% of the cache on socket 0.
+
+# mkdir p0
+# echo "L3:0=f8000;1=fffff" > p0/schemata
+
+Finally we move our first real time task into this resource group. We
+also use taskset(1) to ensure the task always runs on a dedicated CPU
+on socket 0. Most uses of resource groups will also constrain which
+processors tasks run on.
+
+# echo 1234 > p0/tasks
+# taskset -cp 1 1234
+
+Ditto for the second real time task (with the remaining 25% of cache):
+
+# mkdir p1
+# echo "L3:0=7c00;1=fffff" > p1/schemata
+# echo 5678 > p1/tasks
+# taskset -cp 2 5678
-- 
2.5.0

[toc] | [prev] | [next] | [standalone]

#1479115 — Re: [PATCH v2 06/33] Documentation, x86: Documentation for Intel resource allocation user interface

On Thu, Sep 08, 2016 at 02:57:00AM -0700, Fenghua Yu wrote:
> From: Fenghua Yu <fenghua.yu@intel.com>
> 
> The documentation describes user interface of how to allocate resource
> in Intel RDT.
> 
> Please note that the documentation covers generic user interface. Current
> patch set code only implemente CAT L3. CAT L2 code will be sent later.
> 
> Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
> Reviewed-by: Tony Luck <tony.luck@intel.com>
> ---
>  Documentation/x86/intel_rdt_ui.txt | 164 +++++++++++++++++++++++++++++++++++++
>  1 file changed, 164 insertions(+)
>  create mode 100644 Documentation/x86/intel_rdt_ui.txt
> 
> diff --git a/Documentation/x86/intel_rdt_ui.txt b/Documentation/x86/intel_rdt_ui.txt
> new file mode 100644
> index 0000000..27de386
> --- /dev/null
> +++ b/Documentation/x86/intel_rdt_ui.txt

Why isn't this part of Documentation/x86/intel_rdt.txt and needs to be a
separate file?

-- 
Regards/Gruss,
    Boris.

SUSE Linux GmbH, GF: Felix Imendörffer, Jane Smithard, Graham Norton, HRB 21284 (AG Nürnberg)
-- 

[toc] | [prev] | [next] | [standalone]

#1479545 — Re: [PATCH v2 06/33] Documentation, x86: Documentation for Intel resource allocation user interface

On Thu, Sep 08, 2016 at 03:01:20PM -0700, Shaohua Li wrote:
> On Thu, Sep 08, 2016 at 02:57:00AM -0700, Fenghua Yu wrote:
> > From: Fenghua Yu <fenghua.yu@intel.com>
> > 
> > The documentation describes user interface of how to allocate resource
> > in Intel RDT.
> > 
> > Please note that the documentation covers generic user interface. Current
> > patch set code only implemente CAT L3. CAT L2 code will be sent later.
> > 
> > Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
> > Reviewed-by: Tony Luck <tony.luck@intel.com>
> > ---
> >  Documentation/x86/intel_rdt_ui.txt | 164 +++++++++++++++++++++++++++++++++++++
> >  1 file changed, 164 insertions(+)
> >  create mode 100644 Documentation/x86/intel_rdt_ui.txt
> > 
> > diff --git a/Documentation/x86/intel_rdt_ui.txt b/Documentation/x86/intel_rdt_ui.txt
> > new file mode 100644
> > index 0000000..27de386
> > --- /dev/null
> > +++ b/Documentation/x86/intel_rdt_ui.txt
> > @@ -0,0 +1,164 @@
> > +User Interface for Resource Allocation in Intel Resource Director Technology
> > +
> > +Copyright (C) 2016 Intel Corporation
> > +
> > +Fenghua Yu <fenghua.yu@intel.com>
> > +Tony Luck <tony.luck@intel.com>
> > +
> > +This feature is enabled by the CONFIG_INTEL_RDT Kconfig and the
> > +X86 /proc/cpuinfo flag bits "rdt", "cat_l3" and "cdp_l3".
> > +
> > +To use the feature mount the file system:
> > +
> > + # mount -t resctrl resctrl [-o cdp,verbose] /sys/fs/resctrl
> > +
> > +mount options are:
> > +
> > +"cdp": Enable code/data prioritization in L3 cache allocations.
> > +
> > +"verbose": Output more info in the "info" file under info directory
> > +	and in dmesg. This is mainly for debug.
> > +
> > +
> > +Resource groups
> > +---------------
> > +Resource groups are represented as directories in the resctrl file
> > +system. The default group is the root directory. Other groups may be
> > +created as desired by the system administrator using the "mkdir(1)"
> > +command, and removed using "rmdir(1)".
> > +
> > +There are three files associated with each group:
> > +
> > +"tasks": A list of tasks that belongs to this group. Tasks can be
> > +	added to a group by writing the task ID to the "tasks" file
> > +	(which will automatically remove them from the previous
> > +	group to which they belonged). New tasks created by fork(2)
> > +	and clone(2) are added to the same group as their parent.
> > +	If a pid is not in any sub partition, it is in root partition
> > +	(i.e. default partition).
> Hi Fenghua,
> 
> Will you add a 'procs' interface to allow move a process into a group? Using
> the 'tasks' interface to move process is inconvenient and has race conditions
> (eg, some new threads could be escaped).

We don't plan to add a 'procs' interface for rdtgroup. We only use resctrl
interface to allocate resources.

Why the "tasks" is inconvenient? If sysadmin wants to allocte a portion of L3
for a pid, the operation in resctl is to write the pid to a "tasks". While
in 'procs', the operation is to write a partition to a pid. If considering
convenience, they are same, right?

A thread uses either default partition (in root dir) or a sub partition (in
sub-directory). Sysadmin can control that. Kernel handles race condition.
Any issue with that?

Thanks.

-Fenghua
The same 

[toc] | [prev] | [next] | [standalone]

#1479649 — Re: [PATCH v2 06/33] Documentation, x86: Documentation for Intel resource allocation user interface

On Thu, Sep 08, 2016 at 03:45:14PM -0700, Shaohua Li wrote:
> On Thu, Sep 08, 2016 at 06:17:47PM -0700, Fenghua Yu wrote:
> > On Thu, Sep 08, 2016 at 03:01:20PM -0700, Shaohua Li wrote:
> > > On Thu, Sep 08, 2016 at 02:57:00AM -0700, Fenghua Yu wrote:
> > > > From: Fenghua Yu <fenghua.yu@intel.com>
> > > > 
> > > > The documentation describes user interface of how to allocate resource
> > > > in Intel RDT.
> > > > 
> > > > Please note that the documentation covers generic user interface. Current
> > > > patch set code only implemente CAT L3. CAT L2 code will be sent later.
> > > > 
> > > > Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
> > > > Reviewed-by: Tony Luck <tony.luck@intel.com>
> > > > ---
> > > >  Documentation/x86/intel_rdt_ui.txt | 164 +++++++++++++++++++++++++++++++++++++
> > > >  1 file changed, 164 insertions(+)
> > > >  create mode 100644 Documentation/x86/intel_rdt_ui.txt
> > > > 
> > > > diff --git a/Documentation/x86/intel_rdt_ui.txt b/Documentation/x86/intel_rdt_ui.txt
> > > > new file mode 100644
> > > > index 0000000..27de386
> > > > --- /dev/null
> > > > +++ b/Documentation/x86/intel_rdt_ui.txt
> > > > @@ -0,0 +1,164 @@
> > > > +User Interface for Resource Allocation in Intel Resource Director Technology
> > > > +
> > > > +Copyright (C) 2016 Intel Corporation
> > > > +
> > > > +Fenghua Yu <fenghua.yu@intel.com>
> > > > +Tony Luck <tony.luck@intel.com>
> > > > +
> > > > +This feature is enabled by the CONFIG_INTEL_RDT Kconfig and the
> > > > +X86 /proc/cpuinfo flag bits "rdt", "cat_l3" and "cdp_l3".
> > > > +
> > > > +To use the feature mount the file system:
> > > > +
> > > > + # mount -t resctrl resctrl [-o cdp,verbose] /sys/fs/resctrl
> > > > +
> > > > +mount options are:
> > > > +
> > > > +"cdp": Enable code/data prioritization in L3 cache allocations.
> > > > +
> > > > +"verbose": Output more info in the "info" file under info directory
> > > > +	and in dmesg. This is mainly for debug.
> > > > +
> > > > +
> > > > +Resource groups
> > > > +---------------
> > > > +Resource groups are represented as directories in the resctrl file
> > > > +system. The default group is the root directory. Other groups may be
> > > > +created as desired by the system administrator using the "mkdir(1)"
> > > > +command, and removed using "rmdir(1)".
> > > > +
> > > > +There are three files associated with each group:
> > > > +
> > > > +"tasks": A list of tasks that belongs to this group. Tasks can be
> > > > +	added to a group by writing the task ID to the "tasks" file
> > > > +	(which will automatically remove them from the previous
> > > > +	group to which they belonged). New tasks created by fork(2)
> > > > +	and clone(2) are added to the same group as their parent.
> > > > +	If a pid is not in any sub partition, it is in root partition
> > > > +	(i.e. default partition).
> > > Hi Fenghua,
> > > 
> > > Will you add a 'procs' interface to allow move a process into a group? Using
> > > the 'tasks' interface to move process is inconvenient and has race conditions
> > > (eg, some new threads could be escaped).
> > 
> > We don't plan to add a 'procs' interface for rdtgroup. We only use resctrl
> > interface to allocate resources.
> > 
> > Why the "tasks" is inconvenient? If sysadmin wants to allocte a portion of L3
> > for a pid, the operation in resctl is to write the pid to a "tasks". While
> > in 'procs', the operation is to write a partition to a pid. If considering
> > convenience, they are same, right?
> > 
> > A thread uses either default partition (in root dir) or a sub partition (in
> > sub-directory). Sysadmin can control that. Kernel handles race condition.
> > Any issue with that?
> 
> I don't mean writing the 'tasks' file is inconvenient. So to move a process to
> a group, we do:
> 1. get all thread pid of the process
> 2. write every pid to 'tasks'
> 
> this is inconvenient. And if a new thread is created between 1 and 2, we don't
> put the thread to the group. Am I missing anything?

As said in this doc, "New tasks created by fork(2) and clone(2) are added
to the same group as their parent.". So the new thread created b/w 1 and 2
will automatically go to the "tasks" as the process. Later sysadming can
still move any pid to any group.

Is this convenient?

Thanks.

-Fenghua

[toc] | [prev] | [next] | [standalone]

#1478845 — [PATCH v2 03/33] x86, intel_cacheinfo: Enable cache id in x86

From: Fenghua Yu <fenghua.yu@intel.com>

Enable cache id in x86. Cache id comes from APIC ID and CPUID4.

Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Tony Luck <tony.luck@intel.com>
Acked-by: Borislav Petkov <bp@suse.com>
---
 arch/x86/kernel/cpu/intel_cacheinfo.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c
index de6626c..8dc5720 100644
--- a/arch/x86/kernel/cpu/intel_cacheinfo.c
+++ b/arch/x86/kernel/cpu/intel_cacheinfo.c
@@ -153,6 +153,7 @@ struct _cpuid4_info_regs {
 	union _cpuid4_leaf_eax eax;
 	union _cpuid4_leaf_ebx ebx;
 	union _cpuid4_leaf_ecx ecx;
+	unsigned int id;
 	unsigned long size;
 	struct amd_northbridge *nb;
 };
@@ -894,6 +895,8 @@ static void __cache_cpumap_setup(unsigned int cpu, int index,
 static void ci_leaf_init(struct cacheinfo *this_leaf,
 			 struct _cpuid4_info_regs *base)
 {
+	this_leaf->id = base->id;
+	this_leaf->attributes = CACHE_ID;
 	this_leaf->level = base->eax.split.level;
 	this_leaf->type = cache_type_map[base->eax.split.type];
 	this_leaf->coherency_line_size =
@@ -920,6 +923,22 @@ static int __init_cache_level(unsigned int cpu)
 	return 0;
 }
 
+/*
+ * The max shared threads number comes from CPUID.4:EAX[25-14] with input
+ * ECX as cache index. Then right shift apicid by the number's order to get
+ * cache id for this cache node.
+ */
+static void get_cache_id(int cpu, struct _cpuid4_info_regs *id4_regs)
+{
+	struct cpuinfo_x86 *c = &cpu_data(cpu);
+	unsigned long num_threads_sharing;
+	int index_msb;
+
+	num_threads_sharing = 1 + id4_regs->eax.split.num_threads_sharing;
+	index_msb = get_count_order(num_threads_sharing);
+	id4_regs->id = c->apicid >> index_msb;
+}
+
 static int __populate_cache_leaves(unsigned int cpu)
 {
 	unsigned int idx, ret;
@@ -931,6 +950,7 @@ static int __populate_cache_leaves(unsigned int cpu)
 		ret = cpuid4_cache_lookup_regs(idx, &id4_regs);
 		if (ret)
 			return ret;
+		get_cache_id(cpu, &id4_regs);
 		ci_leaf_init(this_leaf++, &id4_regs);
 		__cache_cpumap_setup(cpu, idx, &id4_regs);
 	}
-- 
2.5.0

[toc] | [prev] | [next] | [standalone]

Page 2 of 4 — ← Prev page 1 [2] 3 4  Next page →

Back to top | Article view | linux.kernel

csiph-web