mirror of https://github.com/Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
333 lines
7.8 KiB
333 lines
7.8 KiB
// SPDX-License-Identifier: GPL-2.0-only |
|
/* |
|
* Copyright (C) 2017-2018 HUAWEI, Inc. |
|
* https://www.huawei.com/ |
|
* Created by Gao Xiang <[email protected]> |
|
*/ |
|
#include "internal.h" |
|
#include <linux/prefetch.h> |
|
|
|
#include <trace/events/erofs.h> |
|
|
|
static void erofs_readendio(struct bio *bio) |
|
{ |
|
struct bio_vec *bvec; |
|
blk_status_t err = bio->bi_status; |
|
struct bvec_iter_all iter_all; |
|
|
|
bio_for_each_segment_all(bvec, bio, iter_all) { |
|
struct page *page = bvec->bv_page; |
|
|
|
/* page is already locked */ |
|
DBG_BUGON(PageUptodate(page)); |
|
|
|
if (err) |
|
SetPageError(page); |
|
else |
|
SetPageUptodate(page); |
|
|
|
unlock_page(page); |
|
/* page could be reclaimed now */ |
|
} |
|
bio_put(bio); |
|
} |
|
|
|
struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr) |
|
{ |
|
struct address_space *const mapping = sb->s_bdev->bd_inode->i_mapping; |
|
struct page *page; |
|
|
|
page = read_cache_page_gfp(mapping, blkaddr, |
|
mapping_gfp_constraint(mapping, ~__GFP_FS)); |
|
/* should already be PageUptodate */ |
|
if (!IS_ERR(page)) |
|
lock_page(page); |
|
return page; |
|
} |
|
|
|
static int erofs_map_blocks_flatmode(struct inode *inode, |
|
struct erofs_map_blocks *map, |
|
int flags) |
|
{ |
|
int err = 0; |
|
erofs_blk_t nblocks, lastblk; |
|
u64 offset = map->m_la; |
|
struct erofs_inode *vi = EROFS_I(inode); |
|
bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE); |
|
|
|
trace_erofs_map_blocks_flatmode_enter(inode, map, flags); |
|
|
|
nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE); |
|
lastblk = nblocks - tailendpacking; |
|
|
|
if (offset >= inode->i_size) { |
|
/* leave out-of-bound access unmapped */ |
|
map->m_flags = 0; |
|
map->m_plen = 0; |
|
goto out; |
|
} |
|
|
|
/* there is no hole in flatmode */ |
|
map->m_flags = EROFS_MAP_MAPPED; |
|
|
|
if (offset < blknr_to_addr(lastblk)) { |
|
map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la; |
|
map->m_plen = blknr_to_addr(lastblk) - offset; |
|
} else if (tailendpacking) { |
|
/* 2 - inode inline B: inode, [xattrs], inline last blk... */ |
|
struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); |
|
|
|
map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize + |
|
vi->xattr_isize + erofs_blkoff(map->m_la); |
|
map->m_plen = inode->i_size - offset; |
|
|
|
/* inline data should be located in one meta block */ |
|
if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) { |
|
erofs_err(inode->i_sb, |
|
"inline data cross block boundary @ nid %llu", |
|
vi->nid); |
|
DBG_BUGON(1); |
|
err = -EFSCORRUPTED; |
|
goto err_out; |
|
} |
|
|
|
map->m_flags |= EROFS_MAP_META; |
|
} else { |
|
erofs_err(inode->i_sb, |
|
"internal error @ nid: %llu (size %llu), m_la 0x%llx", |
|
vi->nid, inode->i_size, map->m_la); |
|
DBG_BUGON(1); |
|
err = -EIO; |
|
goto err_out; |
|
} |
|
|
|
out: |
|
map->m_llen = map->m_plen; |
|
|
|
err_out: |
|
trace_erofs_map_blocks_flatmode_exit(inode, map, flags, 0); |
|
return err; |
|
} |
|
|
|
int erofs_map_blocks(struct inode *inode, |
|
struct erofs_map_blocks *map, int flags) |
|
{ |
|
if (erofs_inode_is_data_compressed(EROFS_I(inode)->datalayout)) { |
|
int err = z_erofs_map_blocks_iter(inode, map, flags); |
|
|
|
if (map->mpage) { |
|
put_page(map->mpage); |
|
map->mpage = NULL; |
|
} |
|
return err; |
|
} |
|
return erofs_map_blocks_flatmode(inode, map, flags); |
|
} |
|
|
|
static inline struct bio *erofs_read_raw_page(struct bio *bio, |
|
struct address_space *mapping, |
|
struct page *page, |
|
erofs_off_t *last_block, |
|
unsigned int nblocks, |
|
unsigned int *eblks, |
|
bool ra) |
|
{ |
|
struct inode *const inode = mapping->host; |
|
struct super_block *const sb = inode->i_sb; |
|
erofs_off_t current_block = (erofs_off_t)page->index; |
|
int err; |
|
|
|
DBG_BUGON(!nblocks); |
|
|
|
if (PageUptodate(page)) { |
|
err = 0; |
|
goto has_updated; |
|
} |
|
|
|
/* note that for readpage case, bio also equals to NULL */ |
|
if (bio && |
|
(*last_block + 1 != current_block || !*eblks)) { |
|
submit_bio_retry: |
|
submit_bio(bio); |
|
bio = NULL; |
|
} |
|
|
|
if (!bio) { |
|
struct erofs_map_blocks map = { |
|
.m_la = blknr_to_addr(current_block), |
|
}; |
|
erofs_blk_t blknr; |
|
unsigned int blkoff; |
|
|
|
err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW); |
|
if (err) |
|
goto err_out; |
|
|
|
/* zero out the holed page */ |
|
if (!(map.m_flags & EROFS_MAP_MAPPED)) { |
|
zero_user_segment(page, 0, PAGE_SIZE); |
|
SetPageUptodate(page); |
|
|
|
/* imply err = 0, see erofs_map_blocks */ |
|
goto has_updated; |
|
} |
|
|
|
/* for RAW access mode, m_plen must be equal to m_llen */ |
|
DBG_BUGON(map.m_plen != map.m_llen); |
|
|
|
blknr = erofs_blknr(map.m_pa); |
|
blkoff = erofs_blkoff(map.m_pa); |
|
|
|
/* deal with inline page */ |
|
if (map.m_flags & EROFS_MAP_META) { |
|
void *vsrc, *vto; |
|
struct page *ipage; |
|
|
|
DBG_BUGON(map.m_plen > PAGE_SIZE); |
|
|
|
ipage = erofs_get_meta_page(inode->i_sb, blknr); |
|
|
|
if (IS_ERR(ipage)) { |
|
err = PTR_ERR(ipage); |
|
goto err_out; |
|
} |
|
|
|
vsrc = kmap_atomic(ipage); |
|
vto = kmap_atomic(page); |
|
memcpy(vto, vsrc + blkoff, map.m_plen); |
|
memset(vto + map.m_plen, 0, PAGE_SIZE - map.m_plen); |
|
kunmap_atomic(vto); |
|
kunmap_atomic(vsrc); |
|
flush_dcache_page(page); |
|
|
|
SetPageUptodate(page); |
|
/* TODO: could we unlock the page earlier? */ |
|
unlock_page(ipage); |
|
put_page(ipage); |
|
|
|
/* imply err = 0, see erofs_map_blocks */ |
|
goto has_updated; |
|
} |
|
|
|
/* pa must be block-aligned for raw reading */ |
|
DBG_BUGON(erofs_blkoff(map.m_pa)); |
|
|
|
/* max # of continuous pages */ |
|
if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE)) |
|
nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE); |
|
|
|
*eblks = bio_max_segs(nblocks); |
|
bio = bio_alloc(GFP_NOIO, *eblks); |
|
|
|
bio->bi_end_io = erofs_readendio; |
|
bio_set_dev(bio, sb->s_bdev); |
|
bio->bi_iter.bi_sector = (sector_t)blknr << |
|
LOG_SECTORS_PER_BLOCK; |
|
bio->bi_opf = REQ_OP_READ | (ra ? REQ_RAHEAD : 0); |
|
} |
|
|
|
err = bio_add_page(bio, page, PAGE_SIZE, 0); |
|
/* out of the extent or bio is full */ |
|
if (err < PAGE_SIZE) |
|
goto submit_bio_retry; |
|
--*eblks; |
|
*last_block = current_block; |
|
return bio; |
|
|
|
err_out: |
|
/* for sync reading, set page error immediately */ |
|
if (!ra) { |
|
SetPageError(page); |
|
ClearPageUptodate(page); |
|
} |
|
has_updated: |
|
unlock_page(page); |
|
|
|
/* if updated manually, continuous pages has a gap */ |
|
if (bio) |
|
submit_bio(bio); |
|
return err ? ERR_PTR(err) : NULL; |
|
} |
|
|
|
/* |
|
* since we dont have write or truncate flows, so no inode |
|
* locking needs to be held at the moment. |
|
*/ |
|
static int erofs_raw_access_readpage(struct file *file, struct page *page) |
|
{ |
|
erofs_off_t last_block; |
|
unsigned int eblks; |
|
struct bio *bio; |
|
|
|
trace_erofs_readpage(page, true); |
|
|
|
bio = erofs_read_raw_page(NULL, page->mapping, |
|
page, &last_block, 1, &eblks, false); |
|
|
|
if (IS_ERR(bio)) |
|
return PTR_ERR(bio); |
|
|
|
if (bio) |
|
submit_bio(bio); |
|
return 0; |
|
} |
|
|
|
static void erofs_raw_access_readahead(struct readahead_control *rac) |
|
{ |
|
erofs_off_t last_block; |
|
unsigned int eblks; |
|
struct bio *bio = NULL; |
|
struct page *page; |
|
|
|
trace_erofs_readpages(rac->mapping->host, readahead_index(rac), |
|
readahead_count(rac), true); |
|
|
|
while ((page = readahead_page(rac))) { |
|
prefetchw(&page->flags); |
|
|
|
bio = erofs_read_raw_page(bio, rac->mapping, page, &last_block, |
|
readahead_count(rac), &eblks, true); |
|
|
|
/* all the page errors are ignored when readahead */ |
|
if (IS_ERR(bio)) { |
|
pr_err("%s, readahead error at page %lu of nid %llu\n", |
|
__func__, page->index, |
|
EROFS_I(rac->mapping->host)->nid); |
|
|
|
bio = NULL; |
|
} |
|
|
|
put_page(page); |
|
} |
|
|
|
if (bio) |
|
submit_bio(bio); |
|
} |
|
|
|
static sector_t erofs_bmap(struct address_space *mapping, sector_t block) |
|
{ |
|
struct inode *inode = mapping->host; |
|
struct erofs_map_blocks map = { |
|
.m_la = blknr_to_addr(block), |
|
}; |
|
|
|
if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE) { |
|
erofs_blk_t blks = i_size_read(inode) >> LOG_BLOCK_SIZE; |
|
|
|
if (block >> LOG_SECTORS_PER_BLOCK >= blks) |
|
return 0; |
|
} |
|
|
|
if (!erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW)) |
|
return erofs_blknr(map.m_pa); |
|
|
|
return 0; |
|
} |
|
|
|
/* for uncompressed (aligned) files and raw access for other files */ |
|
const struct address_space_operations erofs_raw_access_aops = { |
|
.readpage = erofs_raw_access_readpage, |
|
.readahead = erofs_raw_access_readahead, |
|
.bmap = erofs_bmap, |
|
}; |
|
|
|
|