Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1707256 > unrolled thread

[PATCH v3] iommu/s390: Add support for iommu_device handling

Back to article view | Back to linux.kernel

Contents

  [PATCH v3] iommu/s390: Add support for iommu_device handling Joerg Roedel <joro@8bytes.org> - 2017-08-09 12:30 +0200
    Re: [PATCH v3] iommu/s390: Add support for iommu_device handling Sebastian Ott <sebott@linux.vnet.ibm.com> - 2017-08-10 21:10 +0200
      Re: [PATCH v3] iommu/s390: Add support for iommu_device handling Joerg Roedel <joro@8bytes.org> - 2017-08-10 23:30 +0200
      Re: [PATCH v3] iommu/s390: Add support for iommu_device handling Joerg Roedel <joro@8bytes.org> - 2017-08-11 09:40 +0200
        Re: [PATCH v3] iommu/s390: Add support for iommu_device handling Sebastian Ott <sebott@linux.vnet.ibm.com> - 2017-08-11 19:10 +0200
          Re: [PATCH v3] iommu/s390: Add support for iommu_device handling Sebastian Ott <sebott@linux.vnet.ibm.com> - 2017-08-11 19:20 +0200
          Re: [PATCH v3] iommu/s390: Add support for iommu_device handling Sebastian Ott <sebott@linux.vnet.ibm.com> - 2017-08-11 19:30 +0200

#1707256 — [PATCH v3] iommu/s390: Add support for iommu_device handling

From: Joerg Roedel <jroedel@suse.de>

Add support for the iommu_device_register interface to make
the s390 hardware iommus visible to the iommu core and in
sysfs.

Signed-off-by: Joerg Roedel <jroedel@suse.de>
---
 arch/s390/include/asm/pci.h |  7 +++++++
 arch/s390/pci/pci.c         |  9 ++++++++-
 drivers/iommu/s390-iommu.c  | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 50 insertions(+), 1 deletion(-)

v2->v3:
	- Rebased to v4.13-rc4
	- Fixed error path in zpci_create_device() to
	  destroy the iommu instance too (noted by
	  Gerald Schaefer)

diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
index f36b4b726057..386df9adef0a 100644
--- a/arch/s390/include/asm/pci.h
+++ b/arch/s390/include/asm/pci.h
@@ -8,6 +8,7 @@
 
 #include <linux/pci.h>
 #include <linux/mutex.h>
+#include <linux/iommu.h>
 #include <asm-generic/pci.h>
 #include <asm/pci_clp.h>
 #include <asm/pci_debug.h>
@@ -122,6 +123,8 @@ struct zpci_dev {
 	unsigned long	iommu_pages;
 	unsigned int	next_bit;
 
+	struct iommu_device iommu_dev;  /* IOMMU core handle */
+
 	char res_name[16];
 	struct zpci_bar_struct bars[PCI_BAR_COUNT];
 
@@ -174,6 +177,10 @@ int clp_enable_fh(struct zpci_dev *, u8);
 int clp_disable_fh(struct zpci_dev *);
 int clp_get_state(u32 fid, enum zpci_state *state);
 
+/* IOMMU Interface */
+int zpci_init_iommu(struct zpci_dev *zdev);
+void zpci_destroy_iommu(struct zpci_dev *zdev);
+
 #ifdef CONFIG_PCI
 /* Error handling and recovery */
 void zpci_event_error(void *);
diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c
index 7b30af5da222..001ca80fa2fe 100644
--- a/arch/s390/pci/pci.c
+++ b/arch/s390/pci/pci.c
@@ -776,6 +776,7 @@ void pcibios_remove_bus(struct pci_bus *bus)
 
 	zpci_exit_slot(zdev);
 	zpci_cleanup_bus_resources(zdev);
+	zpci_destroy_iommu(zdev);
 	zpci_free_domain(zdev);
 
 	spin_lock(&zpci_list_lock);
@@ -848,11 +849,15 @@ int zpci_create_device(struct zpci_dev *zdev)
 	if (rc)
 		goto out;
 
+	rc = zpci_init_iommu(zdev);
+	if (rc)
+		goto out_free;
+
 	mutex_init(&zdev->lock);
 	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
 		rc = zpci_enable_device(zdev);
 		if (rc)
-			goto out_free;
+			goto out_destroy_iommu;
 	}
 	rc = zpci_scan_bus(zdev);
 	if (rc)
@@ -869,6 +874,8 @@ int zpci_create_device(struct zpci_dev *zdev)
 out_disable:
 	if (zdev->state == ZPCI_FN_STATE_ONLINE)
 		zpci_disable_device(zdev);
+out_destroy_iommu:
+	zpci_destroy_iommu(zdev);
 out_free:
 	zpci_free_domain(zdev);
 out:
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 8788640756a7..85f3bc52efc2 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -18,6 +18,8 @@
  */
 #define S390_IOMMU_PGSIZES	(~0xFFFUL)
 
+static struct iommu_ops s390_iommu_ops;
+
 struct s390_domain {
 	struct iommu_domain	domain;
 	struct list_head	devices;
@@ -166,11 +168,13 @@ static void s390_iommu_detach_device(struct iommu_domain *domain,
 static int s390_iommu_add_device(struct device *dev)
 {
 	struct iommu_group *group = iommu_group_get_for_dev(dev);
+	struct zpci_dev *zdev = to_pci_dev(dev)->sysdata;
 
 	if (IS_ERR(group))
 		return PTR_ERR(group);
 
 	iommu_group_put(group);
+	iommu_device_link(&zdev->iommu_dev, dev);
 
 	return 0;
 }
@@ -197,6 +201,7 @@ static void s390_iommu_remove_device(struct device *dev)
 			s390_iommu_detach_device(domain, dev);
 	}
 
+	iommu_device_unlink(&zdev->iommu_dev, dev);
 	iommu_group_remove_device(dev);
 }
 
@@ -327,6 +332,36 @@ static size_t s390_iommu_unmap(struct iommu_domain *domain,
 	return size;
 }
 
+int zpci_init_iommu(struct zpci_dev *zdev)
+{
+	int rc = 0;
+
+	rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
+				    "s390-iommu.%08x", zdev->fid);
+	if (rc)
+		goto out_err;
+
+	iommu_device_set_ops(&zdev->iommu_dev, &s390_iommu_ops);
+
+	rc = iommu_device_register(&zdev->iommu_dev);
+	if (rc)
+		goto out_sysfs;
+
+	return 0;
+
+out_sysfs:
+	iommu_device_sysfs_remove(&zdev->iommu_dev);
+
+out_err:
+	return rc;
+}
+
+void zpci_destroy_iommu(struct zpci_dev *zdev)
+{
+	iommu_device_unregister(&zdev->iommu_dev);
+	iommu_device_sysfs_remove(&zdev->iommu_dev);
+}
+
 static struct iommu_ops s390_iommu_ops = {
 	.capable = s390_iommu_capable,
 	.domain_alloc = s390_domain_alloc,
-- 
2.12.3

[toc] | [next] | [standalone]

#1708969

On Wed, 9 Aug 2017, Joerg Roedel wrote:
> From: Joerg Roedel <jroedel@suse.de>
> 
> Add support for the iommu_device_register interface to make
> the s390 hardware iommus visible to the iommu core and in
> sysfs.
> 

With this patch pci hot-unplug fails with a use after free or refcounting
issue - I'm currently trying to understand what's going on...

Sebastian

[toc] | [prev] | [next] | [standalone]

#1709030

Hi Sebastian,

On Thu, Aug 10, 2017 at 09:07:06PM +0200, Sebastian Ott wrote:
> On Wed, 9 Aug 2017, Joerg Roedel wrote:
> > From: Joerg Roedel <jroedel@suse.de>
> > 
> > Add support for the iommu_device_register interface to make
> > the s390 hardware iommus visible to the iommu core and in
> > sysfs.
> > 
> 
> With this patch pci hot-unplug fails with a use after free or refcounting
> issue - I'm currently trying to understand what's going on...

Thanks for testing and looking into the issue.

Regards,

	Joerg

[toc] | [prev] | [next] | [standalone]

#1709338

Hey Sebastian,

On Thu, Aug 10, 2017 at 09:07:06PM +0200, Sebastian Ott wrote:
> With this patch pci hot-unplug fails with a use after free or refcounting
> issue - I'm currently trying to understand what's going on...

Let me know if I can help with debugging the issue, do you have a
backtrace for me to look at?


Regards,

	Joerg

[toc] | [prev] | [next] | [standalone]

#1709864

Hello Joerg,

On Fri, 11 Aug 2017, Joerg Roedel wrote:
> Hey Sebastian,
> 
> On Thu, Aug 10, 2017 at 09:07:06PM +0200, Sebastian Ott wrote:
> > With this patch pci hot-unplug fails with a use after free or refcounting
> > issue - I'm currently trying to understand what's going on...
> 
> Let me know if I can help with debugging the issue, do you have a
> backtrace for me to look at?

I would have send backtraces but everyone looked different: random mem
corruptions, panic during unrelated allocations and stuff like that.

..but I found the bug, actually 2 bugs:

* That patch embedded a struct iommu_device within struct zpci_dev but
the iommu_device has a release function (via its class) - so when
the release function gets called it frees memory that was never allocated.
The fix is to not embedd struct iommu_device in zpci_dev (see below)

* iommu_release_device must not release the struct device but the
structure it is embedded in: struct iommu_device (I'll send a patch
for that)

With these fixed it works fine.

Sebastian

---
 arch/s390/include/asm/pci.h |  2 +-
 drivers/iommu/s390-iommu.c  | 20 ++++++++++++--------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
index 386df9a..de3129e 100644
--- a/arch/s390/include/asm/pci.h
+++ b/arch/s390/include/asm/pci.h
@@ -123,7 +123,7 @@ struct zpci_dev {
 	unsigned long	iommu_pages;
 	unsigned int	next_bit;
 
-	struct iommu_device iommu_dev;  /* IOMMU core handle */
+	struct iommu_device *iommu_dev;  /* IOMMU core handle */
 
 	char res_name[16];
 	struct zpci_bar_struct bars[PCI_BAR_COUNT];
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 85f3bc5..58a7414 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -174,7 +174,7 @@ static int s390_iommu_add_device(struct device *dev)
 		return PTR_ERR(group);
 
 	iommu_group_put(group);
-	iommu_device_link(&zdev->iommu_dev, dev);
+	iommu_device_link(zdev->iommu_dev, dev);
 
 	return 0;
 }
@@ -201,7 +201,7 @@ static void s390_iommu_remove_device(struct device *dev)
 			s390_iommu_detach_device(domain, dev);
 	}
 
-	iommu_device_unlink(&zdev->iommu_dev, dev);
+	iommu_device_unlink(zdev->iommu_dev, dev);
 	iommu_group_remove_device(dev);
 }
 
@@ -336,21 +336,25 @@ int zpci_init_iommu(struct zpci_dev *zdev)
 {
 	int rc = 0;
 
-	rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL,
+	zdev->iommu_dev = kzalloc(sizeof(*zdev->iommu_dev), GFP_KERNEL);
+	if (!zdev->iommu_dev)
+		return -ENOMEM;
+
+	rc = iommu_device_sysfs_add(zdev->iommu_dev, NULL, NULL,
 				    "s390-iommu.%08x", zdev->fid);
 	if (rc)
 		goto out_err;
 
-	iommu_device_set_ops(&zdev->iommu_dev, &s390_iommu_ops);
+	iommu_device_set_ops(zdev->iommu_dev, &s390_iommu_ops);
 
-	rc = iommu_device_register(&zdev->iommu_dev);
+	rc = iommu_device_register(zdev->iommu_dev);
 	if (rc)
 		goto out_sysfs;
 
 	return 0;
 
 out_sysfs:
-	iommu_device_sysfs_remove(&zdev->iommu_dev);
+	iommu_device_sysfs_remove(zdev->iommu_dev);
 
 out_err:
 	return rc;
@@ -358,8 +362,8 @@ int zpci_init_iommu(struct zpci_dev *zdev)
 
 void zpci_destroy_iommu(struct zpci_dev *zdev)
 {
-	iommu_device_unregister(&zdev->iommu_dev);
-	iommu_device_sysfs_remove(&zdev->iommu_dev);
+	iommu_device_unregister(zdev->iommu_dev);
+	iommu_device_sysfs_remove(zdev->iommu_dev);
 }
 
 static struct iommu_ops s390_iommu_ops = {
-- 
2.5.5

[toc] | [prev] | [next] | [standalone]

#1709872

On Fri, 11 Aug 2017, Sebastian Ott wrote:
> * iommu_release_device must not release the struct device but the
> structure it is embedded in: struct iommu_device (I'll send a patch
> for that)

--->8

From 2839c92e038af47b8cb569f84d571878c54d1815 Mon Sep 17 00:00:00 2001
From: Sebastian Ott <sebott@linux.vnet.ibm.com>
Date: Fri, 11 Aug 2017 19:04:00 +0200
Subject: [PATCH] iommu: fix the release function of iommu_class

The release function of iommu_class must not only free the device
structure it gets called with but the whole structure the device
is embedded in - struct iommu_device.

Signed-off-by: Sebastian Ott <sebott@linux.vnet.ibm.com>
---
 drivers/iommu/iommu-sysfs.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/iommu/iommu-sysfs.c b/drivers/iommu/iommu-sysfs.c
index c58351e..b65a501 100644
--- a/drivers/iommu/iommu-sysfs.c
+++ b/drivers/iommu/iommu-sysfs.c
@@ -34,7 +34,9 @@ static const struct attribute_group *iommu_dev_groups[] = {
 
 static void iommu_release_device(struct device *dev)
 {
-	kfree(dev);
+	struct iommu_device *iommu = container_of(dev, struct iommu_device, dev);
+
+	kfree(iommu);
 }
 
 static struct class iommu_class = {
-- 
2.5.5

[toc] | [prev] | [next] | [standalone]

#1709883

On Fri, 11 Aug 2017, Sebastian Ott wrote:
> * That patch embedded a struct iommu_device within struct zpci_dev but
> the iommu_device has a release function (via its class) - so when
> the release function gets called it frees memory that was never allocated.
> The fix is to not embedd struct iommu_device in zpci_dev (see below)

While discussing the problem with Gerald he noticed that
struct iommu_device is embedded in other structs. So that
is potentially broken in other cases, too.

Sebastian

[toc] | [prev] | [standalone]

Back to top | Article view | linux.kernel

csiph-web