Groups | Search | Server Info | Keyboard shortcuts | Login | Register [http] [https] [nntp] [nntps]
Groups > linux.kernel > #1507870
| From | "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> |
|---|---|
| Newsgroups | linux.kernel |
| Subject | [PATCHv4 30/43] mm: account huge pages to dirty, writaback, reclaimable, etc. |
| Date | 2016-10-25 02:30 +0200 |
| Message-ID | <svXMR-6y6-3@gated-at.bofh.it> (permalink) |
| References | <svXDb-6uQ-5@gated-at.bofh.it> |
| Organization | linux.* mail to news gateway |
We need to account huge pages according to its size to get background
writaback work properly.
Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
---
fs/fs-writeback.c | 10 ++++---
include/linux/backing-dev.h | 10 +++++++
include/linux/memcontrol.h | 5 ++--
mm/migrate.c | 1 +
mm/page-writeback.c | 67 +++++++++++++++++++++++++++++----------------
5 files changed, 64 insertions(+), 29 deletions(-)
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c
index 05713a5da083..2feb8677e69e 100644
--- a/fs/fs-writeback.c
+++ b/fs/fs-writeback.c
@@ -366,8 +366,9 @@ static void inode_switch_wbs_work_fn(struct work_struct *work)
struct page *page = radix_tree_deref_slot_protected(slot,
&mapping->tree_lock);
if (likely(page) && PageDirty(page)) {
- __dec_wb_stat(old_wb, WB_RECLAIMABLE);
- __inc_wb_stat(new_wb, WB_RECLAIMABLE);
+ int nr = hpage_nr_pages(page);
+ __add_wb_stat(old_wb, WB_RECLAIMABLE, -nr);
+ __add_wb_stat(new_wb, WB_RECLAIMABLE, nr);
}
}
@@ -376,9 +377,10 @@ static void inode_switch_wbs_work_fn(struct work_struct *work)
struct page *page = radix_tree_deref_slot_protected(slot,
&mapping->tree_lock);
if (likely(page)) {
+ int nr = hpage_nr_pages(page);
WARN_ON_ONCE(!PageWriteback(page));
- __dec_wb_stat(old_wb, WB_WRITEBACK);
- __inc_wb_stat(new_wb, WB_WRITEBACK);
+ __add_wb_stat(old_wb, WB_WRITEBACK, -nr);
+ __add_wb_stat(new_wb, WB_WRITEBACK, nr);
}
}
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index 43b93a947e61..e63487f78824 100644
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -61,6 +61,16 @@ static inline void __add_wb_stat(struct bdi_writeback *wb,
__percpu_counter_add(&wb->stat[item], amount, WB_STAT_BATCH);
}
+static inline void add_wb_stat(struct bdi_writeback *wb,
+ enum wb_stat_item item, s64 amount)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ __add_wb_stat(wb, item, amount);
+ local_irq_restore(flags);
+}
+
static inline void __inc_wb_stat(struct bdi_writeback *wb,
enum wb_stat_item item)
{
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 61d20c17f3b7..d24092581442 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -29,6 +29,7 @@
#include <linux/mmzone.h>
#include <linux/writeback.h>
#include <linux/page-flags.h>
+#include <linux/mm.h>
struct mem_cgroup;
struct page;
@@ -506,13 +507,13 @@ static inline void mem_cgroup_update_page_stat(struct page *page,
static inline void mem_cgroup_inc_page_stat(struct page *page,
enum mem_cgroup_stat_index idx)
{
- mem_cgroup_update_page_stat(page, idx, 1);
+ mem_cgroup_update_page_stat(page, idx, hpage_nr_pages(page));
}
static inline void mem_cgroup_dec_page_stat(struct page *page,
enum mem_cgroup_stat_index idx)
{
- mem_cgroup_update_page_stat(page, idx, -1);
+ mem_cgroup_update_page_stat(page, idx, -hpage_nr_pages(page));
}
unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
diff --git a/mm/migrate.c b/mm/migrate.c
index 99250aee1ac1..bfc722959d3e 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -505,6 +505,7 @@ int migrate_page_move_mapping(struct address_space *mapping,
* are mapped to swap space.
*/
if (newzone != oldzone) {
+ BUG_ON(PageTransHuge(page));
__dec_node_state(oldzone->zone_pgdat, NR_FILE_PAGES);
__inc_node_state(newzone->zone_pgdat, NR_FILE_PAGES);
if (PageSwapBacked(page) && !PageSwapCache(page)) {
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index c76fc90b7039..f903c09940c4 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -2421,19 +2421,22 @@ void account_page_dirtied(struct page *page, struct address_space *mapping)
if (mapping_cap_account_dirty(mapping)) {
struct bdi_writeback *wb;
+ struct zone *zone = page_zone(page);
+ pg_data_t *pgdat = page_pgdat(page);
+ int nr = hpage_nr_pages(page);
inode_attach_wb(inode, page);
wb = inode_to_wb(inode);
mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_DIRTY);
- __inc_node_page_state(page, NR_FILE_DIRTY);
- __inc_zone_page_state(page, NR_ZONE_WRITE_PENDING);
- __inc_node_page_state(page, NR_DIRTIED);
- __inc_wb_stat(wb, WB_RECLAIMABLE);
- __inc_wb_stat(wb, WB_DIRTIED);
- task_io_account_write(PAGE_SIZE);
- current->nr_dirtied++;
- this_cpu_inc(bdp_ratelimits);
+ __mod_node_page_state(pgdat, NR_FILE_DIRTY, nr);
+ __mod_zone_page_state(zone, NR_ZONE_WRITE_PENDING, nr);
+ __mod_node_page_state(pgdat, NR_DIRTIED, nr);
+ __add_wb_stat(wb, WB_RECLAIMABLE, nr);
+ __add_wb_stat(wb, WB_DIRTIED, nr);
+ task_io_account_write(nr * PAGE_SIZE);
+ current->nr_dirtied += nr;
+ this_cpu_add(bdp_ratelimits, nr);
}
}
EXPORT_SYMBOL(account_page_dirtied);
@@ -2447,11 +2450,15 @@ void account_page_cleaned(struct page *page, struct address_space *mapping,
struct bdi_writeback *wb)
{
if (mapping_cap_account_dirty(mapping)) {
+ struct zone *zone = page_zone(page);
+ pg_data_t *pgdat = page_pgdat(page);
+ int nr = hpage_nr_pages(page);
+
mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_DIRTY);
- dec_node_page_state(page, NR_FILE_DIRTY);
- dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
- dec_wb_stat(wb, WB_RECLAIMABLE);
- task_io_account_cancelled_write(PAGE_SIZE);
+ mod_node_page_state(pgdat, NR_FILE_DIRTY, -nr);
+ mod_zone_page_state(zone, NR_ZONE_WRITE_PENDING, -nr);
+ add_wb_stat(wb, WB_RECLAIMABLE, -nr);
+ task_io_account_cancelled_write(PAGE_SIZE * nr);
}
}
@@ -2511,14 +2518,16 @@ void account_page_redirty(struct page *page)
struct address_space *mapping = page->mapping;
if (mapping && mapping_cap_account_dirty(mapping)) {
+ pg_data_t *pgdat = page_pgdat(page);
+ int nr = hpage_nr_pages(page);
struct inode *inode = mapping->host;
struct bdi_writeback *wb;
bool locked;
wb = unlocked_inode_to_wb_begin(inode, &locked);
- current->nr_dirtied--;
- dec_node_page_state(page, NR_DIRTIED);
- dec_wb_stat(wb, WB_DIRTIED);
+ current->nr_dirtied -= nr;
+ mod_node_page_state(pgdat, NR_DIRTIED, -nr);
+ add_wb_stat(wb, WB_DIRTIED, -nr);
unlocked_inode_to_wb_end(inode, locked);
}
}
@@ -2704,10 +2713,14 @@ int clear_page_dirty_for_io(struct page *page)
*/
wb = unlocked_inode_to_wb_begin(inode, &locked);
if (TestClearPageDirty(page)) {
+ struct zone *zone = page_zone(page);
+ pg_data_t *pgdat = page_pgdat(page);
+ int nr = hpage_nr_pages(page);
+
mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_DIRTY);
- dec_node_page_state(page, NR_FILE_DIRTY);
- dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
- dec_wb_stat(wb, WB_RECLAIMABLE);
+ mod_node_page_state(pgdat, NR_FILE_DIRTY, -nr);
+ mod_zone_page_state(zone, NR_ZONE_WRITE_PENDING, -nr);
+ add_wb_stat(wb, WB_RECLAIMABLE, -nr);
ret = 1;
}
unlocked_inode_to_wb_end(inode, locked);
@@ -2751,10 +2764,14 @@ int test_clear_page_writeback(struct page *page)
ret = TestClearPageWriteback(page);
}
if (ret) {
+ struct zone *zone = page_zone(page);
+ pg_data_t *pgdat = page_pgdat(page);
+ int nr = hpage_nr_pages(page);
+
mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_WRITEBACK);
- dec_node_page_state(page, NR_WRITEBACK);
- dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
- inc_node_page_state(page, NR_WRITTEN);
+ mod_node_page_state(pgdat, NR_WRITEBACK, -nr);
+ mod_zone_page_state(zone, NR_ZONE_WRITE_PENDING, -nr);
+ mod_node_page_state(pgdat, NR_WRITTEN, nr);
}
unlock_page_memcg(page);
return ret;
@@ -2806,9 +2823,13 @@ int __test_set_page_writeback(struct page *page, bool keep_write)
ret = TestSetPageWriteback(page);
}
if (!ret) {
+ struct zone *zone = page_zone(page);
+ pg_data_t *pgdat = page_pgdat(page);
+ int nr = hpage_nr_pages(page);
+
mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_WRITEBACK);
- inc_node_page_state(page, NR_WRITEBACK);
- inc_zone_page_state(page, NR_ZONE_WRITE_PENDING);
+ mod_node_page_state(pgdat, NR_WRITEBACK, nr);
+ mod_zone_page_state(zone, NR_ZONE_WRITE_PENDING, nr);
}
unlock_page_memcg(page);
return ret;
--
2.9.3
Back to linux.kernel | Previous | Next — Previous in thread | Next in thread | Find similar | Unroll thread
[PATCHv4 00/43] ext4: support of huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 02/43] radix tree test suite: Allow GFP_ATOMIC allocations to fail "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 34/43] ext4: handle huge pages in __ext4_block_zero_page_range() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 26/43] fs: make block_page_mkwrite() aware about huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 40/43] ext4: fix SEEK_DATA/SEEK_HOLE for huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 42/43] mm, fs, ext4: expand use of page_mapping() and page_to_pgoff() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 41/43] ext4: make fallocate() operations work with huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 09/43] mm, rmap: account file thp pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 35/43] ext4: make ext4_block_write_begin() aware about huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 06/43] mm, shmem: swich huge tmpfs to multi-order radix-tree entries "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 36/43] ext4: handle huge pages in ext4_da_write_end() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 10/43] thp: try to free page's buffers before attempt split "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 11/43] thp: handle write-protection faults for file THP "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:20 +0200
[PATCHv4 18/43] block: define BIO_MAX_PAGES to HPAGE_PMD_NR if huge page cache enabled "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
Re: [PATCHv4 18/43] block: define BIO_MAX_PAGES to HPAGE_PMD_NR if huge page cache enabled Christoph Hellwig <hch@infradead.org> - 2016-10-25 09:30 +0200
Re: [PATCHv4 18/43] block: define BIO_MAX_PAGES to HPAGE_PMD_NR if huge page cache enabled "Kirill A. Shutemov" <kirill@shutemov.name> - 2016-10-26 00:40 +0200
Re: [PATCHv4 18/43] block: define BIO_MAX_PAGES to HPAGE_PMD_NR if huge page cache enabled Andreas Dilger <adilger@dilger.ca> - 2016-10-26 06:20 +0200
Re: [PATCHv4 18/43] block: define BIO_MAX_PAGES to HPAGE_PMD_NR if huge page cache enabled Ming Lei <tom.leiming@gmail.com> - 2016-10-26 09:40 +0200
Re: [PATCHv4 18/43] block: define BIO_MAX_PAGES to HPAGE_PMD_NR if huge page cache enabled Christoph Hellwig <hch@infradead.org> - 2016-10-26 09:40 +0200
Re: [PATCHv4 18/43] block: define BIO_MAX_PAGES to HPAGE_PMD_NR if huge page cache enabled Christoph Hellwig <hch@infradead.org> - 2016-10-26 09:40 +0200
Re: [PATCHv4 18/43] block: define BIO_MAX_PAGES to HPAGE_PMD_NR if huge page cache enabled Christoph Hellwig <hch@infradead.org> - 2016-10-26 09:40 +0200
[PATCHv4 30/43] mm: account huge pages to dirty, writaback, reclaimable, etc. "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 31/43] ext4: make ext4_mpage_readpages() hugepage-aware "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 17/43] HACK: readahead: alloc huge pages, if allowed "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 20/43] mm: make write_cache_pages() work on huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 21/43] thp: introduce hpage_size() and hpage_mask() "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 01/43] tools: Add WARN_ON_ONCE "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 08/43] page-flags: relax page flag policy for few flags "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 39/43] ext4: make EXT4_IOC_MOVE_EXT work with huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 32/43] ext4: make ext4_writepage() work on huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 13/43] filemap: allocate huge page in page_cache_read(), if allowed "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 28/43] truncate: make invalidate_inode_pages2_range() aware about huge pages "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 03/43] radix-tree: Add radix_tree_join "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 15/43] filemap: allocate huge page in pagecache_get_page(), if allowed "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 29/43] mm, hugetlb: switch hugetlbfs to multi-order radix-tree entries "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 23/43] thp: make thp_get_unmapped_area() respect S_HUGE_MODE "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
[PATCHv4 24/43] fs: make block_read_full_page() be able to read huge page "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com> - 2016-10-25 02:30 +0200
csiph-web