Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]

Groups > linux.kernel > #1224956 > unrolled thread

Test program: check if fsync() can detect I/O error (1/2)

Back to article view | Back to linux.kernel

This discussion starts older than the indexed window; earlier articles aren't shown. The article labeled Started by below is the oldest one visible, not the original post.

Contents

  Test program: check if fsync() can detect I/O error (1/2) Junichi Nomura <j-nomura@ce.jp.nec.com> - 2015-09-15 12:00 +0200
    Test program: check if fsync() can detect I/O error (2/2) Junichi Nomura <j-nomura@ce.jp.nec.com> - 2015-09-15 12:00 +0200

#1224956 — Test program: check if fsync() can detect I/O error (1/2)

> However if admins run a command such as sync or fsfreeze along side,
> fsync/fdatasync may return success even if writeback has failed.
> That could lead to data corruption.

For reproducing the problem, compile the attached C program (iogen.c)
and run with 'runtest.sh' script in the next mail:
  # gcc -o iogen iogen.c
  # bash ./runtest.sh

"iogen" does write(), fsync() and checks if on-disk data is same
as application's buffer after successful fsync.
"runtest.sh" injects failure for the file being written by "iogen".
(You need to enable CONFIG_HWPOISON_INJECT=m for the memory error
 injection to work.)

Without the patch, fsync returns success even though data is not on
disk.

  TEST: ext4 / ioerr / sync-command
  (iogen): inject
  (admin): Injecting I/O error
  (admin): Calling sync(2)
  (iogen): remove
  FAIL: corruption!
  DIFF 00000200: de de de de de de de de  | 00 00 00 00 00 00 00 00
  ...

With the patch, fsync detects error correctly.

  TEST: ext4 / ioerr / sync-command
  (iogen): inject
  (admin): Injecting I/O error
  (admin): Calling sync(2)
  INFO: App fsync: Input/output error
  (iogen): remove
  PASS: detected error right
  (iogen): end

-- cut here --
#define _GNU_SOURCE
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

unsigned char *app_buf;
unsigned char *ondisk_data;
char *testfile;
size_t buflen;
int fd;
int rfd;

void dumpdiff(unsigned char *buf1, unsigned char *buf2, int len)
{
	int i, j;
	for(i = 0; i < len; i += 8) {
		if (!memcmp(&buf1[i], &buf2[i], 8))
			continue;
		fprintf(stderr, "DIFF %08x: ", i);
		for(j = 0; j < 8 && i + j < len; j++)
			fprintf(stderr, "%02x ", buf1[i]);
                fprintf(stderr, " | ");
		for(j = 0; j < 8 && i + j < len; j++)
			fprintf(stderr, "%02x ", buf2[i]);
                fprintf(stderr, "\n");
        }
}

void notify_injector(char *str)
{
        if (str)
                fprintf(stderr, "(iogen): %s\n", str);
        write(1, "\n", 2);
        sleep(1);
}

void open_fds(void)
{
	fd = open(testfile, O_RDWR);
	if (fd < 0) {
		perror("????: App open");
		exit(1);
	}
	rfd = open(testfile, O_RDONLY|O_DIRECT); /* for verification */
	if (rfd < 0) {
		perror("????: App open rfd");
		exit(1);
	}
}

void init_fd_status(void)
{
	int r;

	r = fsync(fd); /* flush and clean */
	if (r) {
		perror("????: App fsync0");
		exit(1);
	}
	r = pread(fd, app_buf, buflen, 0); /* stage onto cache */
	if (r != buflen) {
		perror("????: App read1");
		exit(1);
	}
}

void close_fds(void)
{
	int r;

	r = close(rfd);
	if (r)
		perror("????: App close read fd");
	r = close(fd);
	if (r)
		perror("????: App close write fd");
}

void write_data(int cnt)
{
	int r;

	memset(app_buf, cnt, buflen);
	r = pwrite(fd, app_buf, buflen, 0);
	if (r != buflen)
		perror("????: App write1");
}

int sync_data(void)
{
	int r, r2;

	r = fsync(fd);
	if (r)
		perror("INFO: App fsync");
	r2 = fsync(fd);
	if (r2)
		perror("????: App fsync (redo)");

	return r;
}

void read_data_direct(void)
{
	int r;

	r = pread(rfd, ondisk_data, buflen, 0);
	if (r != buflen) {
		perror("????: App direct read");
		r = pread(rfd, ondisk_data, buflen, 0);
		if (r != buflen)
			perror("FAIL: App direct read (retry)");
	}
}

void check_data(int fsync_result)
{
	int r;

	r = memcmp(app_buf, ondisk_data, buflen);
	if (r) {
		/* data is different */
		if (fsync_result == 0) {
			fprintf(stderr, "FAIL: corruption!\n");
			dumpdiff(app_buf, ondisk_data, buflen);
		} else
			fprintf(stderr, "PASS: detected error right\n");
	} else {
		/* data is same */
		if (fsync_result == 0)
			fprintf(stderr, "PASS: no error, data is ok\n");
		else
			fprintf(stderr, "????: sync failed, data is ok\n");
	}
}

void cleanup_data(int cnt)
{
	int r;

	/* write-fsync-read without error injection */
	memset(app_buf, cnt, buflen);
	r = pwrite(fd, app_buf, buflen, 0);
	if (r != buflen)
		perror("BUG : App write (w/o failure)");
	r = fsync(fd);
	if (r)
		perror("BUG : App fsync (w/o failure)");
	r = pread(rfd, ondisk_data, buflen, 0);
	if (r != buflen)
		perror("BUG : App read (w/o failure)");
	r = memcmp(app_buf, ondisk_data, buflen);
	if (r)
		fprintf(stderr, "BUG : memcmp failed\n");
}

/*
 * Do this:
 *   1) write
 *   2) inject failure
 *   3) fsync (should return error)
 *   4) remove failure
 *   5) check on-disk data (using direct read)
 */
void runtest(void)
{
	int fsync_result;

	notify_injector("start");
	open_fds();
	init_fd_status();

	write_data(0xde);
	notify_injector("inject");
	fsync_result = sync_data();
	notify_injector("remove");

	/* re-read and compare */
	read_data_direct();
	check_data(fsync_result);

	cleanup_data(0);
	close_fds();
	notify_injector("end");
}

int main(int argc, char **argv)
{
	testfile = argv[1];
	buflen = atoi(argv[2]);

	app_buf = malloc(buflen);
	if (!app_buf)
		exit(1);
	if (posix_memalign((void **) &ondisk_data, 4096, buflen))
		exit(1);

	runtest();

	free(app_buf);
	free(ondisk_data);

	return 0;
}--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

[toc] | [next] | [standalone]

#1224964 — Test program: check if fsync() can detect I/O error (2/2)

On 09/15/15 17:39, Jun'ichi Nomura wrote:
>> However if admins run a command such as sync or fsfreeze along side,
>> fsync/fdatasync may return success even if writeback has failed.
>> That could lead to data corruption.
>
> For reproducing the problem, compile the attached C program (iogen.c)
> and run with 'runtest.sh' script in the next mail:
>   # gcc -o iogen iogen.c
>   # bash ./runtest.sh

-- cut here --
#!/bin/bash

# preparation for hwpoison injection
export KERNEL_SRC=/lib/modules/$(uname -r)/build
[ -d "$KERNEL_SRC" ] || exit 1 # no kernel source given
make vm -C $KERNEL_SRC/tools || exit 1 # tools/vm failed to build
pagetypes=$KERNEL_SRC/tools/vm/page-types
[ -x $pagetypes ] || exit 1
modprobe hwpoison-inject

# -------------------------------------------------------------------
fstype=ext4

# file name of loopback image
loopfile=test.img
imgsize=16M
lodev=/dev/loop0

# filesystem to use
mkfs=mkfs.$fstype

# device-mapper map name
testmap=testmap

# file name to store device-mapper table data
mapok=testmap.ok
maperr=testmap.err

# mount point and file name used for testing
testdir=/mnt/test
testfile=$testdir/x

# test file size
filesize=16384

# -------------------------------------------------------------------
# Set up
#

endtest() {
	sleep 3
	umount $testdir
	dmsetup remove $testmap
	losetup -d $lodev
	exit
}

# Create loopback device for testing
dd if=/dev/zero of=$loopfile bs=$imgsize count=1
losetup $lodev $loopfile || endtest
if [ ! -b $lodev ]; then
	endtest
fi


# Layer DM device for error injection
echo "0 $(blockdev --getsz $lodev) linear $lodev 0" | dmsetup create $testmap
dmsetup table $testmap > $mapok || endtest
if [ ! -b /dev/mapper/$testmap ]; then
	endtest
fi

# Mount and create target file
mkdir -p $testdir
$mkfs /dev/mapper/$testmap
mount /dev/mapper/$testmap $testdir || endtest
dd if=/dev/zero of=$testfile bs=$filesize count=1 oflag=direct || endtest

# Find physical location of the target file
find_location() {
	# pick up physical block number of file offset 0
	filefrag -v $1 | \
		awk '$1 == "0" {print $3} $1 == "0:" {print $4}' | \
		sed 's/\.//g'
}
filefrag -v $testfile
block=$(find_location $testfile)
if [ -z "$block" ]; then
	endtest
fi
blocksize=$(stat -c %s -f $testfile)
secsize=512
sector=$((block * blocksize / secsize + 1))

# Create error mapping: inject error at $sector
next=$((sector + 1))
total=$(blockdev --getsz $lodev)
remainder=$((total - next))
cat <<EOF > $maperr
0 $sector linear $lodev 0
$sector 1 error
$next $remainder linear $lodev $next
EOF

map_replace() {
	cat $1 | dmsetup load $testmap
	dmsetup suspend --nolockfs $testmap
	dmsetup resume $testmap
}

inject_memory_error() {
	local pfn=0x$($pagetypes -f $testfile -Nl | grep ^1$'\t' | cut -f2)
	[ "$pfn" = 0x ] && return 1 # target pfn not found
	$pagetypes -a $pfn -X -N
}

# -------------------------------------------------------------------
# Test
#

msg() {
	echo $* > /dev/kmsg
	echo $*
}

injector_ioerr_nop() {
	# start
	read x
	msg "TEST: $fstype / ioerr / (no admin action)"

	# inject
	read x
	msg "(admin): Injecting I/O error"
	map_replace $maperr
	msg "(admin): Do nothing"

	# remove
	read x
	map_replace $mapok

	# end
	read x
	umount /dev/mapper/$testmap || endtest
	mount /dev/mapper/$testmap $testdir || endtest
}

injector_ioerr_synccmd() {
	# start
	read x
	msg "TEST: $fstype / ioerr / sync-command"

	# inject
	read x
	msg "(admin): Injecting I/O error"
	map_replace $maperr
	msg "(admin): Calling sync(2)"
	sync

	# remove
	read x
	map_replace $mapok

	# end
	read x
	umount /dev/mapper/$testmap || endtest
	mount /dev/mapper/$testmap $testdir || endtest
}

injector_hwpoison_synccmd() {
	# start
	read x
	msg "TEST: $fstype / memory-error / sync-command"

	# inject
	read x
	msg "(admin): Injecting memory error"
	inject_memory_error
	msg "(admin): Calling sync(2)"
	sync

	# remove
	read x

	# end
	read x
	umount /dev/mapper/$testmap || endtest
	mount /dev/mapper/$testmap $testdir || endtest
}

msg '============'
./iogen $testfile $filesize | injector_ioerr_nop
msg '============'
./iogen $testfile $filesize | injector_ioerr_synccmd
msg '============'
./iogen $testfile $filesize | injector_hwpoison_synccmd

# -------------------------------------------------------------------
# Clean up
#
endtest
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

[toc] | [prev] | [standalone]

Back to top | Article view | linux.kernel

csiph-web