forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
616 lines
17 KiB
616 lines
17 KiB
// SPDX-License-Identifier: GPL-2.0+ |
|
/* |
|
* Generic Error-Correcting Code (ECC) engine |
|
* |
|
* Copyright (C) 2019 Macronix |
|
* Author: |
|
* Miquèl RAYNAL <[email protected]> |
|
* |
|
* |
|
* This file describes the abstraction of any NAND ECC engine. It has been |
|
* designed to fit most cases, including parallel NANDs and SPI-NANDs. |
|
* |
|
* There are three main situations where instantiating this ECC engine makes |
|
* sense: |
|
* - external: The ECC engine is outside the NAND pipeline, typically this |
|
* is a software ECC engine, or an hardware engine that is |
|
* outside the NAND controller pipeline. |
|
* - pipelined: The ECC engine is inside the NAND pipeline, ie. on the |
|
* controller's side. This is the case of most of the raw NAND |
|
* controllers. In the pipeline case, the ECC bytes are |
|
* generated/data corrected on the fly when a page is |
|
* written/read. |
|
* - ondie: The ECC engine is inside the NAND pipeline, on the chip's side. |
|
* Some NAND chips can correct themselves the data. |
|
* |
|
* Besides the initial setup and final cleanups, the interfaces are rather |
|
* simple: |
|
* - prepare: Prepare an I/O request. Enable/disable the ECC engine based on |
|
* the I/O request type. In case of software correction or external |
|
* engine, this step may involve to derive the ECC bytes and place |
|
* them in the OOB area before a write. |
|
* - finish: Finish an I/O request. Correct the data in case of a read |
|
* request and report the number of corrected bits/uncorrectable |
|
* errors. Most likely empty for write operations, unless you have |
|
* hardware specific stuff to do, like shutting down the engine to |
|
* save power. |
|
* |
|
* The I/O request should be enclosed in a prepare()/finish() pair of calls |
|
* and will behave differently depending on the requested I/O type: |
|
* - raw: Correction disabled |
|
* - ecc: Correction enabled |
|
* |
|
* The request direction is impacting the logic as well: |
|
* - read: Load data from the NAND chip |
|
* - write: Store data in the NAND chip |
|
* |
|
* Mixing all this combinations together gives the following behavior. |
|
* Those are just examples, drivers are free to add custom steps in their |
|
* prepare/finish hook. |
|
* |
|
* [external ECC engine] |
|
* - external + prepare + raw + read: do nothing |
|
* - external + finish + raw + read: do nothing |
|
* - external + prepare + raw + write: do nothing |
|
* - external + finish + raw + write: do nothing |
|
* - external + prepare + ecc + read: do nothing |
|
* - external + finish + ecc + read: calculate expected ECC bytes, extract |
|
* ECC bytes from OOB buffer, correct |
|
* and report any bitflip/error |
|
* - external + prepare + ecc + write: calculate ECC bytes and store them at |
|
* the right place in the OOB buffer based |
|
* on the OOB layout |
|
* - external + finish + ecc + write: do nothing |
|
* |
|
* [pipelined ECC engine] |
|
* - pipelined + prepare + raw + read: disable the controller's ECC engine if |
|
* activated |
|
* - pipelined + finish + raw + read: do nothing |
|
* - pipelined + prepare + raw + write: disable the controller's ECC engine if |
|
* activated |
|
* - pipelined + finish + raw + write: do nothing |
|
* - pipelined + prepare + ecc + read: enable the controller's ECC engine if |
|
* deactivated |
|
* - pipelined + finish + ecc + read: check the status, report any |
|
* error/bitflip |
|
* - pipelined + prepare + ecc + write: enable the controller's ECC engine if |
|
* deactivated |
|
* - pipelined + finish + ecc + write: do nothing |
|
* |
|
* [ondie ECC engine] |
|
* - ondie + prepare + raw + read: send commands to disable the on-chip ECC |
|
* engine if activated |
|
* - ondie + finish + raw + read: do nothing |
|
* - ondie + prepare + raw + write: send commands to disable the on-chip ECC |
|
* engine if activated |
|
* - ondie + finish + raw + write: do nothing |
|
* - ondie + prepare + ecc + read: send commands to enable the on-chip ECC |
|
* engine if deactivated |
|
* - ondie + finish + ecc + read: send commands to check the status, report |
|
* any error/bitflip |
|
* - ondie + prepare + ecc + write: send commands to enable the on-chip ECC |
|
* engine if deactivated |
|
* - ondie + finish + ecc + write: do nothing |
|
*/ |
|
|
|
#include <linux/module.h> |
|
#include <linux/mtd/nand.h> |
|
#include <linux/slab.h> |
|
|
|
/** |
|
* nand_ecc_init_ctx - Init the ECC engine context |
|
* @nand: the NAND device |
|
* |
|
* On success, the caller is responsible of calling @nand_ecc_cleanup_ctx(). |
|
*/ |
|
int nand_ecc_init_ctx(struct nand_device *nand) |
|
{ |
|
if (!nand->ecc.engine || !nand->ecc.engine->ops->init_ctx) |
|
return 0; |
|
|
|
return nand->ecc.engine->ops->init_ctx(nand); |
|
} |
|
EXPORT_SYMBOL(nand_ecc_init_ctx); |
|
|
|
/** |
|
* nand_ecc_cleanup_ctx - Cleanup the ECC engine context |
|
* @nand: the NAND device |
|
*/ |
|
void nand_ecc_cleanup_ctx(struct nand_device *nand) |
|
{ |
|
if (nand->ecc.engine && nand->ecc.engine->ops->cleanup_ctx) |
|
nand->ecc.engine->ops->cleanup_ctx(nand); |
|
} |
|
EXPORT_SYMBOL(nand_ecc_cleanup_ctx); |
|
|
|
/** |
|
* nand_ecc_prepare_io_req - Prepare an I/O request |
|
* @nand: the NAND device |
|
* @req: the I/O request |
|
*/ |
|
int nand_ecc_prepare_io_req(struct nand_device *nand, |
|
struct nand_page_io_req *req) |
|
{ |
|
if (!nand->ecc.engine || !nand->ecc.engine->ops->prepare_io_req) |
|
return 0; |
|
|
|
return nand->ecc.engine->ops->prepare_io_req(nand, req); |
|
} |
|
EXPORT_SYMBOL(nand_ecc_prepare_io_req); |
|
|
|
/** |
|
* nand_ecc_finish_io_req - Finish an I/O request |
|
* @nand: the NAND device |
|
* @req: the I/O request |
|
*/ |
|
int nand_ecc_finish_io_req(struct nand_device *nand, |
|
struct nand_page_io_req *req) |
|
{ |
|
if (!nand->ecc.engine || !nand->ecc.engine->ops->finish_io_req) |
|
return 0; |
|
|
|
return nand->ecc.engine->ops->finish_io_req(nand, req); |
|
} |
|
EXPORT_SYMBOL(nand_ecc_finish_io_req); |
|
|
|
/* Define default OOB placement schemes for large and small page devices */ |
|
static int nand_ooblayout_ecc_sp(struct mtd_info *mtd, int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
struct nand_device *nand = mtd_to_nanddev(mtd); |
|
unsigned int total_ecc_bytes = nand->ecc.ctx.total; |
|
|
|
if (section > 1) |
|
return -ERANGE; |
|
|
|
if (!section) { |
|
oobregion->offset = 0; |
|
if (mtd->oobsize == 16) |
|
oobregion->length = 4; |
|
else |
|
oobregion->length = 3; |
|
} else { |
|
if (mtd->oobsize == 8) |
|
return -ERANGE; |
|
|
|
oobregion->offset = 6; |
|
oobregion->length = total_ecc_bytes - 4; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static int nand_ooblayout_free_sp(struct mtd_info *mtd, int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
if (section > 1) |
|
return -ERANGE; |
|
|
|
if (mtd->oobsize == 16) { |
|
if (section) |
|
return -ERANGE; |
|
|
|
oobregion->length = 8; |
|
oobregion->offset = 8; |
|
} else { |
|
oobregion->length = 2; |
|
if (!section) |
|
oobregion->offset = 3; |
|
else |
|
oobregion->offset = 6; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static const struct mtd_ooblayout_ops nand_ooblayout_sp_ops = { |
|
.ecc = nand_ooblayout_ecc_sp, |
|
.free = nand_ooblayout_free_sp, |
|
}; |
|
|
|
const struct mtd_ooblayout_ops *nand_get_small_page_ooblayout(void) |
|
{ |
|
return &nand_ooblayout_sp_ops; |
|
} |
|
EXPORT_SYMBOL_GPL(nand_get_small_page_ooblayout); |
|
|
|
static int nand_ooblayout_ecc_lp(struct mtd_info *mtd, int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
struct nand_device *nand = mtd_to_nanddev(mtd); |
|
unsigned int total_ecc_bytes = nand->ecc.ctx.total; |
|
|
|
if (section || !total_ecc_bytes) |
|
return -ERANGE; |
|
|
|
oobregion->length = total_ecc_bytes; |
|
oobregion->offset = mtd->oobsize - oobregion->length; |
|
|
|
return 0; |
|
} |
|
|
|
static int nand_ooblayout_free_lp(struct mtd_info *mtd, int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
struct nand_device *nand = mtd_to_nanddev(mtd); |
|
unsigned int total_ecc_bytes = nand->ecc.ctx.total; |
|
|
|
if (section) |
|
return -ERANGE; |
|
|
|
oobregion->length = mtd->oobsize - total_ecc_bytes - 2; |
|
oobregion->offset = 2; |
|
|
|
return 0; |
|
} |
|
|
|
static const struct mtd_ooblayout_ops nand_ooblayout_lp_ops = { |
|
.ecc = nand_ooblayout_ecc_lp, |
|
.free = nand_ooblayout_free_lp, |
|
}; |
|
|
|
const struct mtd_ooblayout_ops *nand_get_large_page_ooblayout(void) |
|
{ |
|
return &nand_ooblayout_lp_ops; |
|
} |
|
EXPORT_SYMBOL_GPL(nand_get_large_page_ooblayout); |
|
|
|
/* |
|
* Support the old "large page" layout used for 1-bit Hamming ECC where ECC |
|
* are placed at a fixed offset. |
|
*/ |
|
static int nand_ooblayout_ecc_lp_hamming(struct mtd_info *mtd, int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
struct nand_device *nand = mtd_to_nanddev(mtd); |
|
unsigned int total_ecc_bytes = nand->ecc.ctx.total; |
|
|
|
if (section) |
|
return -ERANGE; |
|
|
|
switch (mtd->oobsize) { |
|
case 64: |
|
oobregion->offset = 40; |
|
break; |
|
case 128: |
|
oobregion->offset = 80; |
|
break; |
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
oobregion->length = total_ecc_bytes; |
|
if (oobregion->offset + oobregion->length > mtd->oobsize) |
|
return -ERANGE; |
|
|
|
return 0; |
|
} |
|
|
|
static int nand_ooblayout_free_lp_hamming(struct mtd_info *mtd, int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
struct nand_device *nand = mtd_to_nanddev(mtd); |
|
unsigned int total_ecc_bytes = nand->ecc.ctx.total; |
|
int ecc_offset = 0; |
|
|
|
if (section < 0 || section > 1) |
|
return -ERANGE; |
|
|
|
switch (mtd->oobsize) { |
|
case 64: |
|
ecc_offset = 40; |
|
break; |
|
case 128: |
|
ecc_offset = 80; |
|
break; |
|
default: |
|
return -EINVAL; |
|
} |
|
|
|
if (section == 0) { |
|
oobregion->offset = 2; |
|
oobregion->length = ecc_offset - 2; |
|
} else { |
|
oobregion->offset = ecc_offset + total_ecc_bytes; |
|
oobregion->length = mtd->oobsize - oobregion->offset; |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static const struct mtd_ooblayout_ops nand_ooblayout_lp_hamming_ops = { |
|
.ecc = nand_ooblayout_ecc_lp_hamming, |
|
.free = nand_ooblayout_free_lp_hamming, |
|
}; |
|
|
|
const struct mtd_ooblayout_ops *nand_get_large_page_hamming_ooblayout(void) |
|
{ |
|
return &nand_ooblayout_lp_hamming_ops; |
|
} |
|
EXPORT_SYMBOL_GPL(nand_get_large_page_hamming_ooblayout); |
|
|
|
static enum nand_ecc_engine_type |
|
of_get_nand_ecc_engine_type(struct device_node *np) |
|
{ |
|
struct device_node *eng_np; |
|
|
|
if (of_property_read_bool(np, "nand-no-ecc-engine")) |
|
return NAND_ECC_ENGINE_TYPE_NONE; |
|
|
|
if (of_property_read_bool(np, "nand-use-soft-ecc-engine")) |
|
return NAND_ECC_ENGINE_TYPE_SOFT; |
|
|
|
eng_np = of_parse_phandle(np, "nand-ecc-engine", 0); |
|
of_node_put(eng_np); |
|
|
|
if (eng_np) { |
|
if (eng_np == np) |
|
return NAND_ECC_ENGINE_TYPE_ON_DIE; |
|
else |
|
return NAND_ECC_ENGINE_TYPE_ON_HOST; |
|
} |
|
|
|
return NAND_ECC_ENGINE_TYPE_INVALID; |
|
} |
|
|
|
static const char * const nand_ecc_placement[] = { |
|
[NAND_ECC_PLACEMENT_OOB] = "oob", |
|
[NAND_ECC_PLACEMENT_INTERLEAVED] = "interleaved", |
|
}; |
|
|
|
static enum nand_ecc_placement of_get_nand_ecc_placement(struct device_node *np) |
|
{ |
|
enum nand_ecc_placement placement; |
|
const char *pm; |
|
int err; |
|
|
|
err = of_property_read_string(np, "nand-ecc-placement", &pm); |
|
if (!err) { |
|
for (placement = NAND_ECC_PLACEMENT_OOB; |
|
placement < ARRAY_SIZE(nand_ecc_placement); placement++) { |
|
if (!strcasecmp(pm, nand_ecc_placement[placement])) |
|
return placement; |
|
} |
|
} |
|
|
|
return NAND_ECC_PLACEMENT_UNKNOWN; |
|
} |
|
|
|
static const char * const nand_ecc_algos[] = { |
|
[NAND_ECC_ALGO_HAMMING] = "hamming", |
|
[NAND_ECC_ALGO_BCH] = "bch", |
|
[NAND_ECC_ALGO_RS] = "rs", |
|
}; |
|
|
|
static enum nand_ecc_algo of_get_nand_ecc_algo(struct device_node *np) |
|
{ |
|
enum nand_ecc_algo ecc_algo; |
|
const char *pm; |
|
int err; |
|
|
|
err = of_property_read_string(np, "nand-ecc-algo", &pm); |
|
if (!err) { |
|
for (ecc_algo = NAND_ECC_ALGO_HAMMING; |
|
ecc_algo < ARRAY_SIZE(nand_ecc_algos); |
|
ecc_algo++) { |
|
if (!strcasecmp(pm, nand_ecc_algos[ecc_algo])) |
|
return ecc_algo; |
|
} |
|
} |
|
|
|
return NAND_ECC_ALGO_UNKNOWN; |
|
} |
|
|
|
static int of_get_nand_ecc_step_size(struct device_node *np) |
|
{ |
|
int ret; |
|
u32 val; |
|
|
|
ret = of_property_read_u32(np, "nand-ecc-step-size", &val); |
|
return ret ? ret : val; |
|
} |
|
|
|
static int of_get_nand_ecc_strength(struct device_node *np) |
|
{ |
|
int ret; |
|
u32 val; |
|
|
|
ret = of_property_read_u32(np, "nand-ecc-strength", &val); |
|
return ret ? ret : val; |
|
} |
|
|
|
void of_get_nand_ecc_user_config(struct nand_device *nand) |
|
{ |
|
struct device_node *dn = nanddev_get_of_node(nand); |
|
int strength, size; |
|
|
|
nand->ecc.user_conf.engine_type = of_get_nand_ecc_engine_type(dn); |
|
nand->ecc.user_conf.algo = of_get_nand_ecc_algo(dn); |
|
nand->ecc.user_conf.placement = of_get_nand_ecc_placement(dn); |
|
|
|
strength = of_get_nand_ecc_strength(dn); |
|
if (strength >= 0) |
|
nand->ecc.user_conf.strength = strength; |
|
|
|
size = of_get_nand_ecc_step_size(dn); |
|
if (size >= 0) |
|
nand->ecc.user_conf.step_size = size; |
|
|
|
if (of_property_read_bool(dn, "nand-ecc-maximize")) |
|
nand->ecc.user_conf.flags |= NAND_ECC_MAXIMIZE_STRENGTH; |
|
} |
|
EXPORT_SYMBOL(of_get_nand_ecc_user_config); |
|
|
|
/** |
|
* nand_ecc_is_strong_enough - Check if the chip configuration meets the |
|
* datasheet requirements. |
|
* |
|
* @nand: Device to check |
|
* |
|
* If our configuration corrects A bits per B bytes and the minimum |
|
* required correction level is X bits per Y bytes, then we must ensure |
|
* both of the following are true: |
|
* |
|
* (1) A / B >= X / Y |
|
* (2) A >= X |
|
* |
|
* Requirement (1) ensures we can correct for the required bitflip density. |
|
* Requirement (2) ensures we can correct even when all bitflips are clumped |
|
* in the same sector. |
|
*/ |
|
bool nand_ecc_is_strong_enough(struct nand_device *nand) |
|
{ |
|
const struct nand_ecc_props *reqs = nanddev_get_ecc_requirements(nand); |
|
const struct nand_ecc_props *conf = nanddev_get_ecc_conf(nand); |
|
struct mtd_info *mtd = nanddev_to_mtd(nand); |
|
int corr, ds_corr; |
|
|
|
if (conf->step_size == 0 || reqs->step_size == 0) |
|
/* Not enough information */ |
|
return true; |
|
|
|
/* |
|
* We get the number of corrected bits per page to compare |
|
* the correction density. |
|
*/ |
|
corr = (mtd->writesize * conf->strength) / conf->step_size; |
|
ds_corr = (mtd->writesize * reqs->strength) / reqs->step_size; |
|
|
|
return corr >= ds_corr && conf->strength >= reqs->strength; |
|
} |
|
EXPORT_SYMBOL(nand_ecc_is_strong_enough); |
|
|
|
/* ECC engine driver internal helpers */ |
|
int nand_ecc_init_req_tweaking(struct nand_ecc_req_tweak_ctx *ctx, |
|
struct nand_device *nand) |
|
{ |
|
unsigned int total_buffer_size; |
|
|
|
ctx->nand = nand; |
|
|
|
/* Let the user decide the exact length of each buffer */ |
|
if (!ctx->page_buffer_size) |
|
ctx->page_buffer_size = nanddev_page_size(nand); |
|
if (!ctx->oob_buffer_size) |
|
ctx->oob_buffer_size = nanddev_per_page_oobsize(nand); |
|
|
|
total_buffer_size = ctx->page_buffer_size + ctx->oob_buffer_size; |
|
|
|
ctx->spare_databuf = kzalloc(total_buffer_size, GFP_KERNEL); |
|
if (!ctx->spare_databuf) |
|
return -ENOMEM; |
|
|
|
ctx->spare_oobbuf = ctx->spare_databuf + ctx->page_buffer_size; |
|
|
|
return 0; |
|
} |
|
EXPORT_SYMBOL_GPL(nand_ecc_init_req_tweaking); |
|
|
|
void nand_ecc_cleanup_req_tweaking(struct nand_ecc_req_tweak_ctx *ctx) |
|
{ |
|
kfree(ctx->spare_databuf); |
|
} |
|
EXPORT_SYMBOL_GPL(nand_ecc_cleanup_req_tweaking); |
|
|
|
/* |
|
* Ensure data and OOB area is fully read/written otherwise the correction might |
|
* not work as expected. |
|
*/ |
|
void nand_ecc_tweak_req(struct nand_ecc_req_tweak_ctx *ctx, |
|
struct nand_page_io_req *req) |
|
{ |
|
struct nand_device *nand = ctx->nand; |
|
struct nand_page_io_req *orig, *tweak; |
|
|
|
/* Save the original request */ |
|
ctx->orig_req = *req; |
|
ctx->bounce_data = false; |
|
ctx->bounce_oob = false; |
|
orig = &ctx->orig_req; |
|
tweak = req; |
|
|
|
/* Ensure the request covers the entire page */ |
|
if (orig->datalen < nanddev_page_size(nand)) { |
|
ctx->bounce_data = true; |
|
tweak->dataoffs = 0; |
|
tweak->datalen = nanddev_page_size(nand); |
|
tweak->databuf.in = ctx->spare_databuf; |
|
memset(tweak->databuf.in, 0xFF, ctx->page_buffer_size); |
|
} |
|
|
|
if (orig->ooblen < nanddev_per_page_oobsize(nand)) { |
|
ctx->bounce_oob = true; |
|
tweak->ooboffs = 0; |
|
tweak->ooblen = nanddev_per_page_oobsize(nand); |
|
tweak->oobbuf.in = ctx->spare_oobbuf; |
|
memset(tweak->oobbuf.in, 0xFF, ctx->oob_buffer_size); |
|
} |
|
|
|
/* Copy the data that must be writen in the bounce buffers, if needed */ |
|
if (orig->type == NAND_PAGE_WRITE) { |
|
if (ctx->bounce_data) |
|
memcpy((void *)tweak->databuf.out + orig->dataoffs, |
|
orig->databuf.out, orig->datalen); |
|
|
|
if (ctx->bounce_oob) |
|
memcpy((void *)tweak->oobbuf.out + orig->ooboffs, |
|
orig->oobbuf.out, orig->ooblen); |
|
} |
|
} |
|
EXPORT_SYMBOL_GPL(nand_ecc_tweak_req); |
|
|
|
void nand_ecc_restore_req(struct nand_ecc_req_tweak_ctx *ctx, |
|
struct nand_page_io_req *req) |
|
{ |
|
struct nand_page_io_req *orig, *tweak; |
|
|
|
orig = &ctx->orig_req; |
|
tweak = req; |
|
|
|
/* Restore the data read from the bounce buffers, if needed */ |
|
if (orig->type == NAND_PAGE_READ) { |
|
if (ctx->bounce_data) |
|
memcpy(orig->databuf.in, |
|
tweak->databuf.in + orig->dataoffs, |
|
orig->datalen); |
|
|
|
if (ctx->bounce_oob) |
|
memcpy(orig->oobbuf.in, |
|
tweak->oobbuf.in + orig->ooboffs, |
|
orig->ooblen); |
|
} |
|
|
|
/* Ensure the original request is restored */ |
|
*req = *orig; |
|
} |
|
EXPORT_SYMBOL_GPL(nand_ecc_restore_req); |
|
|
|
struct nand_ecc_engine *nand_ecc_get_sw_engine(struct nand_device *nand) |
|
{ |
|
unsigned int algo = nand->ecc.user_conf.algo; |
|
|
|
if (algo == NAND_ECC_ALGO_UNKNOWN) |
|
algo = nand->ecc.defaults.algo; |
|
|
|
switch (algo) { |
|
case NAND_ECC_ALGO_HAMMING: |
|
return nand_ecc_sw_hamming_get_engine(); |
|
case NAND_ECC_ALGO_BCH: |
|
return nand_ecc_sw_bch_get_engine(); |
|
default: |
|
break; |
|
} |
|
|
|
return NULL; |
|
} |
|
EXPORT_SYMBOL(nand_ecc_get_sw_engine); |
|
|
|
struct nand_ecc_engine *nand_ecc_get_on_die_hw_engine(struct nand_device *nand) |
|
{ |
|
return nand->ecc.ondie_engine; |
|
} |
|
EXPORT_SYMBOL(nand_ecc_get_on_die_hw_engine); |
|
|
|
MODULE_LICENSE("GPL"); |
|
MODULE_AUTHOR("Miquel Raynal <[email protected]>"); |
|
MODULE_DESCRIPTION("Generic ECC engine");
|
|
|