Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1458155
| From | Greg Kroah-Hartman <gregkh@linuxfoundation.org> |
|---|---|
| Newsgroups | linux.kernel |
| Subject | [PATCH 4.4 64/68] libceph: apply new_state before new_up_client on incrementals |
| Date | 2016-08-08 21:50 +0200 |
| Message-ID | <s3YIG-5FD-1@gated-at.bofh.it> (permalink) |
| References | <s3YfD-5sp-3@gated-at.bofh.it> |
| Organization | linux.* mail to news gateway |
4.4-stable review patch. If anyone has any objections, please let me know.
------------------
From: Ilya Dryomov <idryomov@gmail.com>
commit 930c532869774ebf8af9efe9484c597f896a7d46 upstream.
Currently, osd_weight and osd_state fields are updated in the encoding
order. This is wrong, because an incremental map may look like e.g.
new_up_client: { osd=6, addr=... } # set osd_state and addr
new_state: { osd=6, xorstate=EXISTS } # clear osd_state
Suppose osd6's current osd_state is EXISTS (i.e. osd6 is down). After
applying new_up_client, osd_state is changed to EXISTS | UP. Carrying
on with the new_state update, we flip EXISTS and leave osd6 in a weird
"!EXISTS but UP" state. A non-existent OSD is considered down by the
mapping code
2087 for (i = 0; i < pg->pg_temp.len; i++) {
2088 if (ceph_osd_is_down(osdmap, pg->pg_temp.osds[i])) {
2089 if (ceph_can_shift_osds(pi))
2090 continue;
2091
2092 temp->osds[temp->size++] = CRUSH_ITEM_NONE;
and so requests get directed to the second OSD in the set instead of
the first, resulting in OSD-side errors like:
[WRN] : client.4239 192.168.122.21:0/2444980242 misdirected client.4239.1:2827 pg 2.5df899f2 to osd.4 not [1,4,6] in e680/680
and hung rbds on the client:
[ 493.566367] rbd: rbd0: write 400000 at 11cc00000 (0)
[ 493.566805] rbd: rbd0: result -6 xferred 400000
[ 493.567011] blk_update_request: I/O error, dev rbd0, sector 9330688
The fix is to decouple application from the decoding and:
- apply new_weight first
- apply new_state before new_up_client
- twiddle osd_state flags if marking in
- clear out some of the state if osd is destroyed
Fixes: http://tracker.ceph.com/issues/14901
Signed-off-by: Ilya Dryomov <idryomov@gmail.com>
Reviewed-by: Josh Durgin <jdurgin@redhat.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
net/ceph/osdmap.c | 156 +++++++++++++++++++++++++++++++++++++++---------------
1 file changed, 113 insertions(+), 43 deletions(-)
--- a/net/ceph/osdmap.c
+++ b/net/ceph/osdmap.c
@@ -1192,6 +1192,115 @@ struct ceph_osdmap *ceph_osdmap_decode(v
}
/*
+ * Encoding order is (new_up_client, new_state, new_weight). Need to
+ * apply in the (new_weight, new_state, new_up_client) order, because
+ * an incremental map may look like e.g.
+ *
+ * new_up_client: { osd=6, addr=... } # set osd_state and addr
+ * new_state: { osd=6, xorstate=EXISTS } # clear osd_state
+ */
+static int decode_new_up_state_weight(void **p, void *end,
+ struct ceph_osdmap *map)
+{
+ void *new_up_client;
+ void *new_state;
+ void *new_weight_end;
+ u32 len;
+
+ new_up_client = *p;
+ ceph_decode_32_safe(p, end, len, e_inval);
+ len *= sizeof(u32) + sizeof(struct ceph_entity_addr);
+ ceph_decode_need(p, end, len, e_inval);
+ *p += len;
+
+ new_state = *p;
+ ceph_decode_32_safe(p, end, len, e_inval);
+ len *= sizeof(u32) + sizeof(u8);
+ ceph_decode_need(p, end, len, e_inval);
+ *p += len;
+
+ /* new_weight */
+ ceph_decode_32_safe(p, end, len, e_inval);
+ while (len--) {
+ s32 osd;
+ u32 w;
+
+ ceph_decode_need(p, end, 2*sizeof(u32), e_inval);
+ osd = ceph_decode_32(p);
+ w = ceph_decode_32(p);
+ BUG_ON(osd >= map->max_osd);
+ pr_info("osd%d weight 0x%x %s\n", osd, w,
+ w == CEPH_OSD_IN ? "(in)" :
+ (w == CEPH_OSD_OUT ? "(out)" : ""));
+ map->osd_weight[osd] = w;
+
+ /*
+ * If we are marking in, set the EXISTS, and clear the
+ * AUTOOUT and NEW bits.
+ */
+ if (w) {
+ map->osd_state[osd] |= CEPH_OSD_EXISTS;
+ map->osd_state[osd] &= ~(CEPH_OSD_AUTOOUT |
+ CEPH_OSD_NEW);
+ }
+ }
+ new_weight_end = *p;
+
+ /* new_state (up/down) */
+ *p = new_state;
+ len = ceph_decode_32(p);
+ while (len--) {
+ s32 osd;
+ u8 xorstate;
+ int ret;
+
+ osd = ceph_decode_32(p);
+ xorstate = ceph_decode_8(p);
+ if (xorstate == 0)
+ xorstate = CEPH_OSD_UP;
+ BUG_ON(osd >= map->max_osd);
+ if ((map->osd_state[osd] & CEPH_OSD_UP) &&
+ (xorstate & CEPH_OSD_UP))
+ pr_info("osd%d down\n", osd);
+ if ((map->osd_state[osd] & CEPH_OSD_EXISTS) &&
+ (xorstate & CEPH_OSD_EXISTS)) {
+ pr_info("osd%d does not exist\n", osd);
+ map->osd_weight[osd] = CEPH_OSD_IN;
+ ret = set_primary_affinity(map, osd,
+ CEPH_OSD_DEFAULT_PRIMARY_AFFINITY);
+ if (ret)
+ return ret;
+ memset(map->osd_addr + osd, 0, sizeof(*map->osd_addr));
+ map->osd_state[osd] = 0;
+ } else {
+ map->osd_state[osd] ^= xorstate;
+ }
+ }
+
+ /* new_up_client */
+ *p = new_up_client;
+ len = ceph_decode_32(p);
+ while (len--) {
+ s32 osd;
+ struct ceph_entity_addr addr;
+
+ osd = ceph_decode_32(p);
+ ceph_decode_copy(p, &addr, sizeof(addr));
+ ceph_decode_addr(&addr);
+ BUG_ON(osd >= map->max_osd);
+ pr_info("osd%d up\n", osd);
+ map->osd_state[osd] |= CEPH_OSD_EXISTS | CEPH_OSD_UP;
+ map->osd_addr[osd] = addr;
+ }
+
+ *p = new_weight_end;
+ return 0;
+
+e_inval:
+ return -EINVAL;
+}
+
+/*
* decode and apply an incremental map update.
*/
struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end,
@@ -1290,49 +1399,10 @@ struct ceph_osdmap *osdmap_apply_increme
__remove_pg_pool(&map->pg_pools, pi);
}
- /* new_up */
- ceph_decode_32_safe(p, end, len, e_inval);
- while (len--) {
- u32 osd;
- struct ceph_entity_addr addr;
- ceph_decode_32_safe(p, end, osd, e_inval);
- ceph_decode_copy_safe(p, end, &addr, sizeof(addr), e_inval);
- ceph_decode_addr(&addr);
- pr_info("osd%d up\n", osd);
- BUG_ON(osd >= map->max_osd);
- map->osd_state[osd] |= CEPH_OSD_UP | CEPH_OSD_EXISTS;
- map->osd_addr[osd] = addr;
- }
-
- /* new_state */
- ceph_decode_32_safe(p, end, len, e_inval);
- while (len--) {
- u32 osd;
- u8 xorstate;
- ceph_decode_32_safe(p, end, osd, e_inval);
- xorstate = **(u8 **)p;
- (*p)++; /* clean flag */
- if (xorstate == 0)
- xorstate = CEPH_OSD_UP;
- if (xorstate & CEPH_OSD_UP)
- pr_info("osd%d down\n", osd);
- if (osd < map->max_osd)
- map->osd_state[osd] ^= xorstate;
- }
-
- /* new_weight */
- ceph_decode_32_safe(p, end, len, e_inval);
- while (len--) {
- u32 osd, off;
- ceph_decode_need(p, end, sizeof(u32)*2, e_inval);
- osd = ceph_decode_32(p);
- off = ceph_decode_32(p);
- pr_info("osd%d weight 0x%x %s\n", osd, off,
- off == CEPH_OSD_IN ? "(in)" :
- (off == CEPH_OSD_OUT ? "(out)" : ""));
- if (osd < map->max_osd)
- map->osd_weight[osd] = off;
- }
+ /* new_up_client, new_state, new_weight */
+ err = decode_new_up_state_weight(p, end, map);
+ if (err)
+ goto bad;
/* new_pg_temp */
err = decode_new_pg_temp(p, end, map);
Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread
[PATCH 4.4 00/68] 4.4.17-stable review Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 36/68] locks: use file_inode() Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 32/68] pinctrl: single: Fix missing flush of posted write for a wakeirq Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 15/68] pps: do not crash when failed to register Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 58/68] ipr: Clear interrupt on croc/crocodile when running with LSI Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 10/68] fs/nilfs2: fix potential underflow in call to crc32_le Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 17/68] arc: unwind: warn only once if DW2_UNWIND is disabled Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 35/68] power_supply: power_supply_read_temp only if use_cnt > 0 Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 56/68] RDS: fix rds_tcp_init() error path Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 68/68] vfs: fix deadlock in file_remove_privs() on overlayfs Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 26/68] Input: elantech - add more IC body types to the list Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 42/68] spi: sunxi: fix transfer timeout Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 47/68] block: fix use-after-free in sys_ioprio_get() Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 37/68] Revert "ecryptfs: forbid opening files without mmap handler" Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 25/68] Input: vmmouse - remove port reservation Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 49/68] sched/fair: Fix effective_load() to consistently use smoothed load Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 21/68] xenbus: dont bail early from xenbus_dev_request_and_reply() Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 28/68] Input: wacom_w8001 - w8001_MAX_LENGTH should be 13 Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 09/68] mm, compaction: abort free scanner if split fails Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 07/68] dmaengine: at_xdmac: double FIFO flush needed to compute residue Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 55/68] can: fix oops caused by wrong rtnl dellink usage Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 52/68] can: at91_can: RX queue could get stuck at high bus load Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 06/68] dmaengine: at_xdmac: fix residue corruption Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 61/68] i2c: qup: Fix wrong value of index variable Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 45/68] platform/chrome: cros_ec_dev - double fetch bug in ioctl Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:20 +0200
[PATCH 4.4 64/68] libceph: apply new_state before new_up_client on incrementals Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 16/68] kernel/sysrq, watchdog, sched/core: Reset watchdog on all CPUs while processing sysrq-w Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 51/68] perf/x86: fix PEBS issues on Intel Atom/Core2 Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 14/68] vmlinux.lds: account for destructor sections Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 38/68] ecryptfs: dont allow mmap when the lower fs doesnt support it Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 46/68] qeth: delete napi struct when removing a qeth device Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 08/68] mm, sl[au]b: add __GFP_ATOMIC to the GFP reclaim mask Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 66/68] intel_th: pci: Add Kaby Lake PCH-H support Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 41/68] namespace: update event counter when umounting a deleted dentry Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 60/68] posix_cpu_timer: Exit early when process has been reaped Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 54/68] can: fix handling of unmodifiable configuration options fix Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 67/68] intel_th: Fix a deadlock in modprobing Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 43/68] spi: sun4i: fix FIFO limit Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 11/68] mm, compaction: prevent VM_BUG_ON when terminating freeing scanner Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 39/68] ext4: verify extent header depth Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 12/68] mm, meminit: always return a valid node from early_pfn_to_nid Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 63/68] libata: LITE-ON CX1-JB256-HP needs lower max_sectors Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 40/68] 9p: use file_dentry() Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 57/68] SCSI: fix new bug in scsi_dev_info_list string matching Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 50/68] ovl: handle ATTR_KILL* Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 44/68] clk: rockchip: initialize flags of clk_init_data in mmc-phase clock Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 59/68] media: fix airspy usb probe error path Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 53/68] can: c_can: Update D_CAN TX and RX functions to 32 bit - fix Altera Cyclone access Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 13/68] mm, meminit: ensure node is online before checking whether pages are uninitialised Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 48/68] mmc: block: fix packed command header endianness Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 21:50 +0200
[PATCH 4.4 34/68] cgroup: set css->id to -1 during init Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 22:00 +0200
[PATCH 4.4 18/68] ARC: unwind: ensure that .debug_frame is generated (vs. .eh_frame) Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 22:00 +0200
[PATCH 4.4 27/68] Input: xpad - fix oops when attaching an unknown Xbox One gamepad Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 22:00 +0200
[PATCH 4.4 29/68] Input: xpad - validate USB endpoint count during probe Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 22:00 +0200
[PATCH 4.4 19/68] xen/pciback: Fix conf_space read/write overlap check. Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 22:00 +0200
[PATCH 4.4 03/68] x86/quirks: Reintroduce scanning of secondary buses Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 22:00 +0200
[PATCH 4.4 31/68] pvclock: Add CPU barriers to get correct version value Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-08 22:00 +0200
Re: [PATCH 4.4 00/68] 4.4.17-stable review Guenter Roeck <linux@roeck-us.net> - 2016-08-09 06:30 +0200
Re: [PATCH 4.4 00/68] 4.4.17-stable review Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-09 10:30 +0200
Re: [PATCH 4.4 00/68] 4.4.17-stable review Guenter Roeck <linux@roeck-us.net> - 2016-08-09 18:20 +0200
Re: [PATCH 4.4 00/68] 4.4.17-stable review Greg Kroah-Hartman <gregkh@linuxfoundation.org> - 2016-08-09 19:30 +0200
Re: [PATCH 4.4 00/68] 4.4.17-stable review Shuah Khan <shuah.kh@samsung.com> - 2016-08-09 17:20 +0200
csiph-web