mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
403 lines
9.7 KiB
403 lines
9.7 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* |
|
* linux/mm/page_io.c |
|
* |
|
* Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds |
|
* |
|
* Swap reorganised 29.12.95, |
|
* Asynchronous swapping added 30.12.95. Stephen Tweedie |
|
* Removed race in async swapping. 14.4.1996. Bruno Haible |
|
* Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie |
|
* Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman |
|
*/ |
|
|
|
#include <linux/mm.h> |
|
#include <linux/kernel_stat.h> |
|
#include <linux/gfp.h> |
|
#include <linux/pagemap.h> |
|
#include <linux/swap.h> |
|
#include <linux/bio.h> |
|
#include <linux/swapops.h> |
|
#include <linux/buffer_head.h> |
|
#include <linux/writeback.h> |
|
#include <linux/frontswap.h> |
|
#include <linux/blkdev.h> |
|
#include <linux/psi.h> |
|
#include <linux/uio.h> |
|
#include <linux/sched/task.h> |
|
#include <linux/delayacct.h> |
|
|
|
void end_swap_bio_write(struct bio *bio) |
|
{ |
|
struct page *page = bio_first_page_all(bio); |
|
|
|
if (bio->bi_status) { |
|
SetPageError(page); |
|
/* |
|
* We failed to write the page out to swap-space. |
|
* Re-dirty the page in order to avoid it being reclaimed. |
|
* Also print a dire warning that things will go BAD (tm) |
|
* very quickly. |
|
* |
|
* Also clear PG_reclaim to avoid folio_rotate_reclaimable() |
|
*/ |
|
set_page_dirty(page); |
|
pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n", |
|
MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), |
|
(unsigned long long)bio->bi_iter.bi_sector); |
|
ClearPageReclaim(page); |
|
} |
|
end_page_writeback(page); |
|
bio_put(bio); |
|
} |
|
|
|
static void end_swap_bio_read(struct bio *bio) |
|
{ |
|
struct page *page = bio_first_page_all(bio); |
|
struct task_struct *waiter = bio->bi_private; |
|
|
|
if (bio->bi_status) { |
|
SetPageError(page); |
|
ClearPageUptodate(page); |
|
pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n", |
|
MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), |
|
(unsigned long long)bio->bi_iter.bi_sector); |
|
goto out; |
|
} |
|
|
|
SetPageUptodate(page); |
|
out: |
|
unlock_page(page); |
|
WRITE_ONCE(bio->bi_private, NULL); |
|
bio_put(bio); |
|
if (waiter) { |
|
blk_wake_io_task(waiter); |
|
put_task_struct(waiter); |
|
} |
|
} |
|
|
|
int generic_swapfile_activate(struct swap_info_struct *sis, |
|
struct file *swap_file, |
|
sector_t *span) |
|
{ |
|
struct address_space *mapping = swap_file->f_mapping; |
|
struct inode *inode = mapping->host; |
|
unsigned blocks_per_page; |
|
unsigned long page_no; |
|
unsigned blkbits; |
|
sector_t probe_block; |
|
sector_t last_block; |
|
sector_t lowest_block = -1; |
|
sector_t highest_block = 0; |
|
int nr_extents = 0; |
|
int ret; |
|
|
|
blkbits = inode->i_blkbits; |
|
blocks_per_page = PAGE_SIZE >> blkbits; |
|
|
|
/* |
|
* Map all the blocks into the extent tree. This code doesn't try |
|
* to be very smart. |
|
*/ |
|
probe_block = 0; |
|
page_no = 0; |
|
last_block = i_size_read(inode) >> blkbits; |
|
while ((probe_block + blocks_per_page) <= last_block && |
|
page_no < sis->max) { |
|
unsigned block_in_page; |
|
sector_t first_block; |
|
|
|
cond_resched(); |
|
|
|
first_block = probe_block; |
|
ret = bmap(inode, &first_block); |
|
if (ret || !first_block) |
|
goto bad_bmap; |
|
|
|
/* |
|
* It must be PAGE_SIZE aligned on-disk |
|
*/ |
|
if (first_block & (blocks_per_page - 1)) { |
|
probe_block++; |
|
goto reprobe; |
|
} |
|
|
|
for (block_in_page = 1; block_in_page < blocks_per_page; |
|
block_in_page++) { |
|
sector_t block; |
|
|
|
block = probe_block + block_in_page; |
|
ret = bmap(inode, &block); |
|
if (ret || !block) |
|
goto bad_bmap; |
|
|
|
if (block != first_block + block_in_page) { |
|
/* Discontiguity */ |
|
probe_block++; |
|
goto reprobe; |
|
} |
|
} |
|
|
|
first_block >>= (PAGE_SHIFT - blkbits); |
|
if (page_no) { /* exclude the header page */ |
|
if (first_block < lowest_block) |
|
lowest_block = first_block; |
|
if (first_block > highest_block) |
|
highest_block = first_block; |
|
} |
|
|
|
/* |
|
* We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks |
|
*/ |
|
ret = add_swap_extent(sis, page_no, 1, first_block); |
|
if (ret < 0) |
|
goto out; |
|
nr_extents += ret; |
|
page_no++; |
|
probe_block += blocks_per_page; |
|
reprobe: |
|
continue; |
|
} |
|
ret = nr_extents; |
|
*span = 1 + highest_block - lowest_block; |
|
if (page_no == 0) |
|
page_no = 1; /* force Empty message */ |
|
sis->max = page_no; |
|
sis->pages = page_no - 1; |
|
sis->highest_bit = page_no - 1; |
|
out: |
|
return ret; |
|
bad_bmap: |
|
pr_err("swapon: swapfile has holes\n"); |
|
ret = -EINVAL; |
|
goto out; |
|
} |
|
|
|
/* |
|
* We may have stale swap cache pages in memory: notice |
|
* them here and get rid of the unnecessary final write. |
|
*/ |
|
int swap_writepage(struct page *page, struct writeback_control *wbc) |
|
{ |
|
int ret = 0; |
|
|
|
if (try_to_free_swap(page)) { |
|
unlock_page(page); |
|
goto out; |
|
} |
|
/* |
|
* Arch code may have to preserve more data than just the page |
|
* contents, e.g. memory tags. |
|
*/ |
|
ret = arch_prepare_to_swap(page); |
|
if (ret) { |
|
set_page_dirty(page); |
|
unlock_page(page); |
|
goto out; |
|
} |
|
if (frontswap_store(page) == 0) { |
|
set_page_writeback(page); |
|
unlock_page(page); |
|
end_page_writeback(page); |
|
goto out; |
|
} |
|
ret = __swap_writepage(page, wbc, end_swap_bio_write); |
|
out: |
|
return ret; |
|
} |
|
|
|
static inline void count_swpout_vm_event(struct page *page) |
|
{ |
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE |
|
if (unlikely(PageTransHuge(page))) |
|
count_vm_event(THP_SWPOUT); |
|
#endif |
|
count_vm_events(PSWPOUT, thp_nr_pages(page)); |
|
} |
|
|
|
#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) |
|
static void bio_associate_blkg_from_page(struct bio *bio, struct page *page) |
|
{ |
|
struct cgroup_subsys_state *css; |
|
struct mem_cgroup *memcg; |
|
|
|
memcg = page_memcg(page); |
|
if (!memcg) |
|
return; |
|
|
|
rcu_read_lock(); |
|
css = cgroup_e_css(memcg->css.cgroup, &io_cgrp_subsys); |
|
bio_associate_blkg_from_css(bio, css); |
|
rcu_read_unlock(); |
|
} |
|
#else |
|
#define bio_associate_blkg_from_page(bio, page) do { } while (0) |
|
#endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */ |
|
|
|
int __swap_writepage(struct page *page, struct writeback_control *wbc, |
|
bio_end_io_t end_write_func) |
|
{ |
|
struct bio *bio; |
|
int ret; |
|
struct swap_info_struct *sis = page_swap_info(page); |
|
|
|
VM_BUG_ON_PAGE(!PageSwapCache(page), page); |
|
if (data_race(sis->flags & SWP_FS_OPS)) { |
|
struct kiocb kiocb; |
|
struct file *swap_file = sis->swap_file; |
|
struct address_space *mapping = swap_file->f_mapping; |
|
struct bio_vec bv = { |
|
.bv_page = page, |
|
.bv_len = PAGE_SIZE, |
|
.bv_offset = 0 |
|
}; |
|
struct iov_iter from; |
|
|
|
iov_iter_bvec(&from, WRITE, &bv, 1, PAGE_SIZE); |
|
init_sync_kiocb(&kiocb, swap_file); |
|
kiocb.ki_pos = page_file_offset(page); |
|
|
|
set_page_writeback(page); |
|
unlock_page(page); |
|
ret = mapping->a_ops->direct_IO(&kiocb, &from); |
|
if (ret == PAGE_SIZE) { |
|
count_vm_event(PSWPOUT); |
|
ret = 0; |
|
} else { |
|
/* |
|
* In the case of swap-over-nfs, this can be a |
|
* temporary failure if the system has limited |
|
* memory for allocating transmit buffers. |
|
* Mark the page dirty and avoid |
|
* folio_rotate_reclaimable but rate-limit the |
|
* messages but do not flag PageError like |
|
* the normal direct-to-bio case as it could |
|
* be temporary. |
|
*/ |
|
set_page_dirty(page); |
|
ClearPageReclaim(page); |
|
pr_err_ratelimited("Write error on dio swapfile (%llu)\n", |
|
page_file_offset(page)); |
|
} |
|
end_page_writeback(page); |
|
return ret; |
|
} |
|
|
|
ret = bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc); |
|
if (!ret) { |
|
count_swpout_vm_event(page); |
|
return 0; |
|
} |
|
|
|
bio = bio_alloc(sis->bdev, 1, |
|
REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc), |
|
GFP_NOIO); |
|
bio->bi_iter.bi_sector = swap_page_sector(page); |
|
bio->bi_end_io = end_write_func; |
|
bio_add_page(bio, page, thp_size(page), 0); |
|
|
|
bio_associate_blkg_from_page(bio, page); |
|
count_swpout_vm_event(page); |
|
set_page_writeback(page); |
|
unlock_page(page); |
|
submit_bio(bio); |
|
|
|
return 0; |
|
} |
|
|
|
int swap_readpage(struct page *page, bool synchronous) |
|
{ |
|
struct bio *bio; |
|
int ret = 0; |
|
struct swap_info_struct *sis = page_swap_info(page); |
|
bool workingset = PageWorkingset(page); |
|
unsigned long pflags; |
|
|
|
VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page); |
|
VM_BUG_ON_PAGE(!PageLocked(page), page); |
|
VM_BUG_ON_PAGE(PageUptodate(page), page); |
|
|
|
/* |
|
* Count submission time as memory stall. When the device is congested, |
|
* or the submitting cgroup IO-throttled, submission can be a |
|
* significant part of overall IO time. |
|
*/ |
|
if (workingset) |
|
psi_memstall_enter(&pflags); |
|
delayacct_swapin_start(); |
|
|
|
if (frontswap_load(page) == 0) { |
|
SetPageUptodate(page); |
|
unlock_page(page); |
|
goto out; |
|
} |
|
|
|
if (data_race(sis->flags & SWP_FS_OPS)) { |
|
struct file *swap_file = sis->swap_file; |
|
struct address_space *mapping = swap_file->f_mapping; |
|
|
|
ret = mapping->a_ops->readpage(swap_file, page); |
|
if (!ret) |
|
count_vm_event(PSWPIN); |
|
goto out; |
|
} |
|
|
|
if (sis->flags & SWP_SYNCHRONOUS_IO) { |
|
ret = bdev_read_page(sis->bdev, swap_page_sector(page), page); |
|
if (!ret) { |
|
count_vm_event(PSWPIN); |
|
goto out; |
|
} |
|
} |
|
|
|
ret = 0; |
|
bio = bio_alloc(sis->bdev, 1, REQ_OP_READ, GFP_KERNEL); |
|
bio->bi_iter.bi_sector = swap_page_sector(page); |
|
bio->bi_end_io = end_swap_bio_read; |
|
bio_add_page(bio, page, thp_size(page), 0); |
|
/* |
|
* Keep this task valid during swap readpage because the oom killer may |
|
* attempt to access it in the page fault retry time check. |
|
*/ |
|
if (synchronous) { |
|
bio->bi_opf |= REQ_POLLED; |
|
get_task_struct(current); |
|
bio->bi_private = current; |
|
} |
|
count_vm_event(PSWPIN); |
|
bio_get(bio); |
|
submit_bio(bio); |
|
while (synchronous) { |
|
set_current_state(TASK_UNINTERRUPTIBLE); |
|
if (!READ_ONCE(bio->bi_private)) |
|
break; |
|
|
|
if (!bio_poll(bio, NULL, 0)) |
|
blk_io_schedule(); |
|
} |
|
__set_current_state(TASK_RUNNING); |
|
bio_put(bio); |
|
|
|
out: |
|
if (workingset) |
|
psi_memstall_leave(&pflags); |
|
delayacct_swapin_end(); |
|
return ret; |
|
} |
|
|
|
bool swap_dirty_folio(struct address_space *mapping, struct folio *folio) |
|
{ |
|
struct swap_info_struct *sis = swp_swap_info(folio_swap_entry(folio)); |
|
|
|
if (data_race(sis->flags & SWP_FS_OPS)) { |
|
const struct address_space_operations *aops; |
|
|
|
mapping = sis->swap_file->f_mapping; |
|
aops = mapping->a_ops; |
|
|
|
VM_BUG_ON_FOLIO(!folio_test_swapcache(folio), folio); |
|
return aops->dirty_folio(mapping, folio); |
|
} else { |
|
return noop_dirty_folio(mapping, folio); |
|
} |
|
}
|
|
|