forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
599 lines
15 KiB
599 lines
15 KiB
// SPDX-License-Identifier: GPL-2.0-or-later |
|
/* |
|
* Copyright (C) 2017 Free Electrons |
|
* Copyright (C) 2017 NextThing Co |
|
* |
|
* Author: Boris Brezillon <[email protected]> |
|
*/ |
|
|
|
#include <linux/slab.h> |
|
|
|
#include "internals.h" |
|
|
|
/* |
|
* Special Micron status bit 3 indicates that the block has been |
|
* corrected by on-die ECC and should be rewritten. |
|
*/ |
|
#define NAND_ECC_STATUS_WRITE_RECOMMENDED BIT(3) |
|
|
|
/* |
|
* On chips with 8-bit ECC and additional bit can be used to distinguish |
|
* cases where a errors were corrected without needing a rewrite |
|
* |
|
* Bit 4 Bit 3 Bit 0 Description |
|
* ----- ----- ----- ----------- |
|
* 0 0 0 No Errors |
|
* 0 0 1 Multiple uncorrected errors |
|
* 0 1 0 4 - 6 errors corrected, recommend rewrite |
|
* 0 1 1 Reserved |
|
* 1 0 0 1 - 3 errors corrected |
|
* 1 0 1 Reserved |
|
* 1 1 0 7 - 8 errors corrected, recommend rewrite |
|
*/ |
|
#define NAND_ECC_STATUS_MASK (BIT(4) | BIT(3) | BIT(0)) |
|
#define NAND_ECC_STATUS_UNCORRECTABLE BIT(0) |
|
#define NAND_ECC_STATUS_4_6_CORRECTED BIT(3) |
|
#define NAND_ECC_STATUS_1_3_CORRECTED BIT(4) |
|
#define NAND_ECC_STATUS_7_8_CORRECTED (BIT(4) | BIT(3)) |
|
|
|
struct nand_onfi_vendor_micron { |
|
u8 two_plane_read; |
|
u8 read_cache; |
|
u8 read_unique_id; |
|
u8 dq_imped; |
|
u8 dq_imped_num_settings; |
|
u8 dq_imped_feat_addr; |
|
u8 rb_pulldown_strength; |
|
u8 rb_pulldown_strength_feat_addr; |
|
u8 rb_pulldown_strength_num_settings; |
|
u8 otp_mode; |
|
u8 otp_page_start; |
|
u8 otp_data_prot_addr; |
|
u8 otp_num_pages; |
|
u8 otp_feat_addr; |
|
u8 read_retry_options; |
|
u8 reserved[72]; |
|
u8 param_revision; |
|
} __packed; |
|
|
|
struct micron_on_die_ecc { |
|
bool forced; |
|
bool enabled; |
|
void *rawbuf; |
|
}; |
|
|
|
struct micron_nand { |
|
struct micron_on_die_ecc ecc; |
|
}; |
|
|
|
static int micron_nand_setup_read_retry(struct nand_chip *chip, int retry_mode) |
|
{ |
|
u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = {retry_mode}; |
|
|
|
return nand_set_features(chip, ONFI_FEATURE_ADDR_READ_RETRY, feature); |
|
} |
|
|
|
/* |
|
* Configure chip properties from Micron vendor-specific ONFI table |
|
*/ |
|
static int micron_nand_onfi_init(struct nand_chip *chip) |
|
{ |
|
struct nand_parameters *p = &chip->parameters; |
|
|
|
if (p->onfi) { |
|
struct nand_onfi_vendor_micron *micron = (void *)p->onfi->vendor; |
|
|
|
chip->read_retries = micron->read_retry_options; |
|
chip->ops.setup_read_retry = micron_nand_setup_read_retry; |
|
} |
|
|
|
if (p->supports_set_get_features) { |
|
set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->set_feature_list); |
|
set_bit(ONFI_FEATURE_ON_DIE_ECC, p->set_feature_list); |
|
set_bit(ONFI_FEATURE_ADDR_READ_RETRY, p->get_feature_list); |
|
set_bit(ONFI_FEATURE_ON_DIE_ECC, p->get_feature_list); |
|
} |
|
|
|
return 0; |
|
} |
|
|
|
static int micron_nand_on_die_4_ooblayout_ecc(struct mtd_info *mtd, |
|
int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
if (section >= 4) |
|
return -ERANGE; |
|
|
|
oobregion->offset = (section * 16) + 8; |
|
oobregion->length = 8; |
|
|
|
return 0; |
|
} |
|
|
|
static int micron_nand_on_die_4_ooblayout_free(struct mtd_info *mtd, |
|
int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
if (section >= 4) |
|
return -ERANGE; |
|
|
|
oobregion->offset = (section * 16) + 2; |
|
oobregion->length = 6; |
|
|
|
return 0; |
|
} |
|
|
|
static const struct mtd_ooblayout_ops micron_nand_on_die_4_ooblayout_ops = { |
|
.ecc = micron_nand_on_die_4_ooblayout_ecc, |
|
.free = micron_nand_on_die_4_ooblayout_free, |
|
}; |
|
|
|
static int micron_nand_on_die_8_ooblayout_ecc(struct mtd_info *mtd, |
|
int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
struct nand_chip *chip = mtd_to_nand(mtd); |
|
|
|
if (section) |
|
return -ERANGE; |
|
|
|
oobregion->offset = mtd->oobsize - chip->ecc.total; |
|
oobregion->length = chip->ecc.total; |
|
|
|
return 0; |
|
} |
|
|
|
static int micron_nand_on_die_8_ooblayout_free(struct mtd_info *mtd, |
|
int section, |
|
struct mtd_oob_region *oobregion) |
|
{ |
|
struct nand_chip *chip = mtd_to_nand(mtd); |
|
|
|
if (section) |
|
return -ERANGE; |
|
|
|
oobregion->offset = 2; |
|
oobregion->length = mtd->oobsize - chip->ecc.total - 2; |
|
|
|
return 0; |
|
} |
|
|
|
static const struct mtd_ooblayout_ops micron_nand_on_die_8_ooblayout_ops = { |
|
.ecc = micron_nand_on_die_8_ooblayout_ecc, |
|
.free = micron_nand_on_die_8_ooblayout_free, |
|
}; |
|
|
|
static int micron_nand_on_die_ecc_setup(struct nand_chip *chip, bool enable) |
|
{ |
|
struct micron_nand *micron = nand_get_manufacturer_data(chip); |
|
u8 feature[ONFI_SUBFEATURE_PARAM_LEN] = { 0, }; |
|
int ret; |
|
|
|
if (micron->ecc.forced) |
|
return 0; |
|
|
|
if (micron->ecc.enabled == enable) |
|
return 0; |
|
|
|
if (enable) |
|
feature[0] |= ONFI_FEATURE_ON_DIE_ECC_EN; |
|
|
|
ret = nand_set_features(chip, ONFI_FEATURE_ON_DIE_ECC, feature); |
|
if (!ret) |
|
micron->ecc.enabled = enable; |
|
|
|
return ret; |
|
} |
|
|
|
static int micron_nand_on_die_ecc_status_4(struct nand_chip *chip, u8 status, |
|
void *buf, int page, |
|
int oob_required) |
|
{ |
|
struct micron_nand *micron = nand_get_manufacturer_data(chip); |
|
struct mtd_info *mtd = nand_to_mtd(chip); |
|
unsigned int step, max_bitflips = 0; |
|
bool use_datain = false; |
|
int ret; |
|
|
|
if (!(status & NAND_ECC_STATUS_WRITE_RECOMMENDED)) { |
|
if (status & NAND_STATUS_FAIL) |
|
mtd->ecc_stats.failed++; |
|
|
|
return 0; |
|
} |
|
|
|
/* |
|
* The internal ECC doesn't tell us the number of bitflips that have |
|
* been corrected, but tells us if it recommends to rewrite the block. |
|
* If it's the case, we need to read the page in raw mode and compare |
|
* its content to the corrected version to extract the actual number of |
|
* bitflips. |
|
* But before we do that, we must make sure we have all OOB bytes read |
|
* in non-raw mode, even if the user did not request those bytes. |
|
*/ |
|
if (!oob_required) { |
|
/* |
|
* We first check which operation is supported by the controller |
|
* before running it. This trick makes it possible to support |
|
* all controllers, even the most constraints, without almost |
|
* any performance hit. |
|
* |
|
* TODO: could be enhanced to avoid repeating the same check |
|
* over and over in the fast path. |
|
*/ |
|
if (!nand_has_exec_op(chip) || |
|
!nand_read_data_op(chip, chip->oob_poi, mtd->oobsize, false, |
|
true)) |
|
use_datain = true; |
|
|
|
if (use_datain) |
|
ret = nand_read_data_op(chip, chip->oob_poi, |
|
mtd->oobsize, false, false); |
|
else |
|
ret = nand_change_read_column_op(chip, mtd->writesize, |
|
chip->oob_poi, |
|
mtd->oobsize, false); |
|
if (ret) |
|
return ret; |
|
} |
|
|
|
micron_nand_on_die_ecc_setup(chip, false); |
|
|
|
ret = nand_read_page_op(chip, page, 0, micron->ecc.rawbuf, |
|
mtd->writesize + mtd->oobsize); |
|
if (ret) |
|
return ret; |
|
|
|
for (step = 0; step < chip->ecc.steps; step++) { |
|
unsigned int offs, i, nbitflips = 0; |
|
u8 *rawbuf, *corrbuf; |
|
|
|
offs = step * chip->ecc.size; |
|
rawbuf = micron->ecc.rawbuf + offs; |
|
corrbuf = buf + offs; |
|
|
|
for (i = 0; i < chip->ecc.size; i++) |
|
nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); |
|
|
|
offs = (step * 16) + 4; |
|
rawbuf = micron->ecc.rawbuf + mtd->writesize + offs; |
|
corrbuf = chip->oob_poi + offs; |
|
|
|
for (i = 0; i < chip->ecc.bytes + 4; i++) |
|
nbitflips += hweight8(corrbuf[i] ^ rawbuf[i]); |
|
|
|
if (WARN_ON(nbitflips > chip->ecc.strength)) |
|
return -EINVAL; |
|
|
|
max_bitflips = max(nbitflips, max_bitflips); |
|
mtd->ecc_stats.corrected += nbitflips; |
|
} |
|
|
|
return max_bitflips; |
|
} |
|
|
|
static int micron_nand_on_die_ecc_status_8(struct nand_chip *chip, u8 status) |
|
{ |
|
struct mtd_info *mtd = nand_to_mtd(chip); |
|
|
|
/* |
|
* With 8/512 we have more information but still don't know precisely |
|
* how many bit-flips were seen. |
|
*/ |
|
switch (status & NAND_ECC_STATUS_MASK) { |
|
case NAND_ECC_STATUS_UNCORRECTABLE: |
|
mtd->ecc_stats.failed++; |
|
return 0; |
|
case NAND_ECC_STATUS_1_3_CORRECTED: |
|
mtd->ecc_stats.corrected += 3; |
|
return 3; |
|
case NAND_ECC_STATUS_4_6_CORRECTED: |
|
mtd->ecc_stats.corrected += 6; |
|
/* rewrite recommended */ |
|
return 6; |
|
case NAND_ECC_STATUS_7_8_CORRECTED: |
|
mtd->ecc_stats.corrected += 8; |
|
/* rewrite recommended */ |
|
return 8; |
|
default: |
|
return 0; |
|
} |
|
} |
|
|
|
static int |
|
micron_nand_read_page_on_die_ecc(struct nand_chip *chip, uint8_t *buf, |
|
int oob_required, int page) |
|
{ |
|
struct mtd_info *mtd = nand_to_mtd(chip); |
|
bool use_datain = false; |
|
u8 status; |
|
int ret, max_bitflips = 0; |
|
|
|
ret = micron_nand_on_die_ecc_setup(chip, true); |
|
if (ret) |
|
return ret; |
|
|
|
ret = nand_read_page_op(chip, page, 0, NULL, 0); |
|
if (ret) |
|
goto out; |
|
|
|
ret = nand_status_op(chip, &status); |
|
if (ret) |
|
goto out; |
|
|
|
/* |
|
* We first check which operation is supported by the controller before |
|
* running it. This trick makes it possible to support all controllers, |
|
* even the most constraints, without almost any performance hit. |
|
* |
|
* TODO: could be enhanced to avoid repeating the same check over and |
|
* over in the fast path. |
|
*/ |
|
if (!nand_has_exec_op(chip) || |
|
!nand_read_data_op(chip, buf, mtd->writesize, false, true)) |
|
use_datain = true; |
|
|
|
if (use_datain) { |
|
ret = nand_exit_status_op(chip); |
|
if (ret) |
|
goto out; |
|
|
|
ret = nand_read_data_op(chip, buf, mtd->writesize, false, |
|
false); |
|
if (!ret && oob_required) |
|
ret = nand_read_data_op(chip, chip->oob_poi, |
|
mtd->oobsize, false, false); |
|
} else { |
|
ret = nand_change_read_column_op(chip, 0, buf, mtd->writesize, |
|
false); |
|
if (!ret && oob_required) |
|
ret = nand_change_read_column_op(chip, mtd->writesize, |
|
chip->oob_poi, |
|
mtd->oobsize, false); |
|
} |
|
|
|
if (chip->ecc.strength == 4) |
|
max_bitflips = micron_nand_on_die_ecc_status_4(chip, status, |
|
buf, page, |
|
oob_required); |
|
else |
|
max_bitflips = micron_nand_on_die_ecc_status_8(chip, status); |
|
|
|
out: |
|
micron_nand_on_die_ecc_setup(chip, false); |
|
|
|
return ret ? ret : max_bitflips; |
|
} |
|
|
|
static int |
|
micron_nand_write_page_on_die_ecc(struct nand_chip *chip, const uint8_t *buf, |
|
int oob_required, int page) |
|
{ |
|
int ret; |
|
|
|
ret = micron_nand_on_die_ecc_setup(chip, true); |
|
if (ret) |
|
return ret; |
|
|
|
ret = nand_write_page_raw(chip, buf, oob_required, page); |
|
micron_nand_on_die_ecc_setup(chip, false); |
|
|
|
return ret; |
|
} |
|
|
|
enum { |
|
/* The NAND flash doesn't support on-die ECC */ |
|
MICRON_ON_DIE_UNSUPPORTED, |
|
|
|
/* |
|
* The NAND flash supports on-die ECC and it can be |
|
* enabled/disabled by a set features command. |
|
*/ |
|
MICRON_ON_DIE_SUPPORTED, |
|
|
|
/* |
|
* The NAND flash supports on-die ECC, and it cannot be |
|
* disabled. |
|
*/ |
|
MICRON_ON_DIE_MANDATORY, |
|
}; |
|
|
|
#define MICRON_ID_INTERNAL_ECC_MASK GENMASK(1, 0) |
|
#define MICRON_ID_ECC_ENABLED BIT(7) |
|
|
|
/* |
|
* Try to detect if the NAND support on-die ECC. To do this, we enable |
|
* the feature, and read back if it has been enabled as expected. We |
|
* also check if it can be disabled, because some Micron NANDs do not |
|
* allow disabling the on-die ECC and we don't support such NANDs for |
|
* now. |
|
* |
|
* This function also has the side effect of disabling on-die ECC if |
|
* it had been left enabled by the firmware/bootloader. |
|
*/ |
|
static int micron_supports_on_die_ecc(struct nand_chip *chip) |
|
{ |
|
const struct nand_ecc_props *requirements = |
|
nanddev_get_ecc_requirements(&chip->base); |
|
u8 id[5]; |
|
int ret; |
|
|
|
if (!chip->parameters.onfi) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
if (nanddev_bits_per_cell(&chip->base) != 1) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
/* |
|
* We only support on-die ECC of 4/512 or 8/512 |
|
*/ |
|
if (requirements->strength != 4 && requirements->strength != 8) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
/* 0x2 means on-die ECC is available. */ |
|
if (chip->id.len != 5 || |
|
(chip->id.data[4] & MICRON_ID_INTERNAL_ECC_MASK) != 0x2) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
/* |
|
* It seems that there are devices which do not support ECC officially. |
|
* At least the MT29F2G08ABAGA / MT29F2G08ABBGA devices supports |
|
* enabling the ECC feature but don't reflect that to the READ_ID table. |
|
* So we have to guarantee that we disable the ECC feature directly |
|
* after we did the READ_ID table command. Later we can evaluate the |
|
* ECC_ENABLE support. |
|
*/ |
|
ret = micron_nand_on_die_ecc_setup(chip, true); |
|
if (ret) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
ret = nand_readid_op(chip, 0, id, sizeof(id)); |
|
if (ret) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
ret = micron_nand_on_die_ecc_setup(chip, false); |
|
if (ret) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
if (!(id[4] & MICRON_ID_ECC_ENABLED)) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
ret = nand_readid_op(chip, 0, id, sizeof(id)); |
|
if (ret) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
if (id[4] & MICRON_ID_ECC_ENABLED) |
|
return MICRON_ON_DIE_MANDATORY; |
|
|
|
/* |
|
* We only support on-die ECC of 4/512 or 8/512 |
|
*/ |
|
if (requirements->strength != 4 && requirements->strength != 8) |
|
return MICRON_ON_DIE_UNSUPPORTED; |
|
|
|
return MICRON_ON_DIE_SUPPORTED; |
|
} |
|
|
|
static int micron_nand_init(struct nand_chip *chip) |
|
{ |
|
struct nand_device *base = &chip->base; |
|
const struct nand_ecc_props *requirements = |
|
nanddev_get_ecc_requirements(base); |
|
struct mtd_info *mtd = nand_to_mtd(chip); |
|
struct micron_nand *micron; |
|
int ondie; |
|
int ret; |
|
|
|
micron = kzalloc(sizeof(*micron), GFP_KERNEL); |
|
if (!micron) |
|
return -ENOMEM; |
|
|
|
nand_set_manufacturer_data(chip, micron); |
|
|
|
ret = micron_nand_onfi_init(chip); |
|
if (ret) |
|
goto err_free_manuf_data; |
|
|
|
chip->options |= NAND_BBM_FIRSTPAGE; |
|
|
|
if (mtd->writesize == 2048) |
|
chip->options |= NAND_BBM_SECONDPAGE; |
|
|
|
ondie = micron_supports_on_die_ecc(chip); |
|
|
|
if (ondie == MICRON_ON_DIE_MANDATORY && |
|
chip->ecc.engine_type != NAND_ECC_ENGINE_TYPE_ON_DIE) { |
|
pr_err("On-die ECC forcefully enabled, not supported\n"); |
|
ret = -EINVAL; |
|
goto err_free_manuf_data; |
|
} |
|
|
|
if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE) { |
|
if (ondie == MICRON_ON_DIE_UNSUPPORTED) { |
|
pr_err("On-die ECC selected but not supported\n"); |
|
ret = -EINVAL; |
|
goto err_free_manuf_data; |
|
} |
|
|
|
if (ondie == MICRON_ON_DIE_MANDATORY) { |
|
micron->ecc.forced = true; |
|
micron->ecc.enabled = true; |
|
} |
|
|
|
/* |
|
* In case of 4bit on-die ECC, we need a buffer to store a |
|
* page dumped in raw mode so that we can compare its content |
|
* to the same page after ECC correction happened and extract |
|
* the real number of bitflips from this comparison. |
|
* That's not needed for 8-bit ECC, because the status expose |
|
* a better approximation of the number of bitflips in a page. |
|
*/ |
|
if (requirements->strength == 4) { |
|
micron->ecc.rawbuf = kmalloc(mtd->writesize + |
|
mtd->oobsize, |
|
GFP_KERNEL); |
|
if (!micron->ecc.rawbuf) { |
|
ret = -ENOMEM; |
|
goto err_free_manuf_data; |
|
} |
|
} |
|
|
|
if (requirements->strength == 4) |
|
mtd_set_ooblayout(mtd, |
|
µn_nand_on_die_4_ooblayout_ops); |
|
else |
|
mtd_set_ooblayout(mtd, |
|
µn_nand_on_die_8_ooblayout_ops); |
|
|
|
chip->ecc.bytes = requirements->strength * 2; |
|
chip->ecc.size = 512; |
|
chip->ecc.strength = requirements->strength; |
|
chip->ecc.algo = NAND_ECC_ALGO_BCH; |
|
chip->ecc.read_page = micron_nand_read_page_on_die_ecc; |
|
chip->ecc.write_page = micron_nand_write_page_on_die_ecc; |
|
|
|
if (ondie == MICRON_ON_DIE_MANDATORY) { |
|
chip->ecc.read_page_raw = nand_read_page_raw_notsupp; |
|
chip->ecc.write_page_raw = nand_write_page_raw_notsupp; |
|
} else { |
|
if (!chip->ecc.read_page_raw) |
|
chip->ecc.read_page_raw = nand_read_page_raw; |
|
if (!chip->ecc.write_page_raw) |
|
chip->ecc.write_page_raw = nand_write_page_raw; |
|
} |
|
} |
|
|
|
return 0; |
|
|
|
err_free_manuf_data: |
|
kfree(micron->ecc.rawbuf); |
|
kfree(micron); |
|
|
|
return ret; |
|
} |
|
|
|
static void micron_nand_cleanup(struct nand_chip *chip) |
|
{ |
|
struct micron_nand *micron = nand_get_manufacturer_data(chip); |
|
|
|
kfree(micron->ecc.rawbuf); |
|
kfree(micron); |
|
} |
|
|
|
static void micron_fixup_onfi_param_page(struct nand_chip *chip, |
|
struct nand_onfi_params *p) |
|
{ |
|
/* |
|
* MT29F1G08ABAFAWP-ITE:F and possibly others report 00 00 for the |
|
* revision number field of the ONFI parameter page. Assume ONFI |
|
* version 1.0 if the revision number is 00 00. |
|
*/ |
|
if (le16_to_cpu(p->revision) == 0) |
|
p->revision = cpu_to_le16(ONFI_VERSION_1_0); |
|
} |
|
|
|
const struct nand_manufacturer_ops micron_nand_manuf_ops = { |
|
.init = micron_nand_init, |
|
.cleanup = micron_nand_cleanup, |
|
.fixup_onfi_param_page = micron_fixup_onfi_param_page, |
|
};
|
|
|