forked from Qortal/qortal
AlphaX-Projects
3 months ago
11 changed files with 8541 additions and 24 deletions
@ -0,0 +1,101 @@
|
||||
package org.qortal.block; |
||||
|
||||
import org.apache.logging.log4j.LogManager; |
||||
import org.apache.logging.log4j.Logger; |
||||
import org.eclipse.persistence.jaxb.JAXBContextFactory; |
||||
import org.eclipse.persistence.jaxb.UnmarshallerProperties; |
||||
import org.qortal.data.account.AccountBalanceData; |
||||
import org.qortal.repository.DataException; |
||||
|
||||
import javax.xml.bind.JAXBContext; |
||||
import javax.xml.bind.JAXBException; |
||||
import javax.xml.bind.UnmarshalException; |
||||
import javax.xml.bind.Unmarshaller; |
||||
import javax.xml.transform.stream.StreamSource; |
||||
import java.io.InputStream; |
||||
import java.util.List; |
||||
import java.util.stream.Collectors; |
||||
|
||||
/** |
||||
* Block 1333492 |
||||
* <p> |
||||
* As described in InvalidBalanceBlocks.java, legacy bugs caused a small drift in account balances. |
||||
* This block adjusts any remaining differences between a clean reindex/resync and a recent bootstrap. |
||||
* <p> |
||||
* The block height 1333492 isn't significant - it's simply the height of a recent bootstrap at the |
||||
* time of development, so that the account balances could be accessed and compared against the same |
||||
* block in a reindexed db. |
||||
* <p> |
||||
* As with InvalidBalanceBlocks, the discrepancies are insignificant, except for a single |
||||
* account which has a 3.03 QORT discrepancy. This was due to the account being the first recipient |
||||
* of a name sale and encountering an early bug in this area. |
||||
* <p> |
||||
* The total offset for this block is 3.02816514 QORT. |
||||
*/ |
||||
public final class Block1333492 { |
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger(Block1333492.class); |
||||
private static final String ACCOUNT_DELTAS_SOURCE = "block-1333492-deltas.json"; |
||||
|
||||
private static final List<AccountBalanceData> accountDeltas = readAccountDeltas(); |
||||
|
||||
private Block1333492() { |
||||
/* Do not instantiate */ |
||||
} |
||||
|
||||
@SuppressWarnings("unchecked") |
||||
private static List<AccountBalanceData> readAccountDeltas() { |
||||
Unmarshaller unmarshaller; |
||||
|
||||
try { |
||||
// Create JAXB context aware of classes we need to unmarshal
|
||||
JAXBContext jc = JAXBContextFactory.createContext(new Class[] { |
||||
AccountBalanceData.class |
||||
}, null); |
||||
|
||||
// Create unmarshaller
|
||||
unmarshaller = jc.createUnmarshaller(); |
||||
|
||||
// Set the unmarshaller media type to JSON
|
||||
unmarshaller.setProperty(UnmarshallerProperties.MEDIA_TYPE, "application/json"); |
||||
|
||||
// Tell unmarshaller that there's no JSON root element in the JSON input
|
||||
unmarshaller.setProperty(UnmarshallerProperties.JSON_INCLUDE_ROOT, false); |
||||
} catch (JAXBException e) { |
||||
String message = "Failed to setup unmarshaller to read block 1333492 deltas"; |
||||
LOGGER.error(message, e); |
||||
throw new RuntimeException(message, e); |
||||
} |
||||
|
||||
ClassLoader classLoader = BlockChain.class.getClassLoader(); |
||||
InputStream in = classLoader.getResourceAsStream(ACCOUNT_DELTAS_SOURCE); |
||||
StreamSource jsonSource = new StreamSource(in); |
||||
|
||||
try { |
||||
// Attempt to unmarshal JSON stream to BlockChain config
|
||||
return (List<AccountBalanceData>) unmarshaller.unmarshal(jsonSource, AccountBalanceData.class).getValue(); |
||||
} catch (UnmarshalException e) { |
||||
String message = "Failed to parse block 1333492 deltas"; |
||||
LOGGER.error(message, e); |
||||
throw new RuntimeException(message, e); |
||||
} catch (JAXBException e) { |
||||
String message = "Unexpected JAXB issue while processing block 1333492 deltas"; |
||||
LOGGER.error(message, e); |
||||
throw new RuntimeException(message, e); |
||||
} |
||||
} |
||||
|
||||
public static void processFix(Block block) throws DataException { |
||||
block.repository.getAccountRepository().modifyAssetBalances(accountDeltas); |
||||
} |
||||
|
||||
public static void orphanFix(Block block) throws DataException { |
||||
// Create inverse deltas
|
||||
List<AccountBalanceData> inverseDeltas = accountDeltas.stream() |
||||
.map(delta -> new AccountBalanceData(delta.getAddress(), delta.getAssetId(), 0 - delta.getBalance())) |
||||
.collect(Collectors.toList()); |
||||
|
||||
block.repository.getAccountRepository().modifyAssetBalances(inverseDeltas); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,134 @@
|
||||
package org.qortal.block; |
||||
|
||||
import org.apache.logging.log4j.LogManager; |
||||
import org.apache.logging.log4j.Logger; |
||||
import org.eclipse.persistence.jaxb.JAXBContextFactory; |
||||
import org.eclipse.persistence.jaxb.UnmarshallerProperties; |
||||
import org.qortal.data.account.AccountBalanceData; |
||||
import org.qortal.repository.DataException; |
||||
|
||||
import javax.xml.bind.JAXBContext; |
||||
import javax.xml.bind.JAXBException; |
||||
import javax.xml.bind.UnmarshalException; |
||||
import javax.xml.bind.Unmarshaller; |
||||
import javax.xml.transform.stream.StreamSource; |
||||
import java.io.InputStream; |
||||
import java.util.*; |
||||
import java.util.stream.Collectors; |
||||
|
||||
|
||||
/** |
||||
* Due to various bugs - which have been fixed - a small amount of balance drift occurred |
||||
* in the chainstate of running nodes and bootstraps, when compared with a clean sync from genesis. |
||||
* This resulted in a significant number of invalid transactions in the chain history due to |
||||
* subtle balance discrepancies. The sum of all discrepancies that resulted in an invalid |
||||
* transaction is 0.00198322 QORT, so despite the large quantity of transactions, they |
||||
* represent an insignificant amount when summed. |
||||
* <p> |
||||
* This class is responsible for retroactively fixing all the past transactions which |
||||
* are invalid due to the balance discrepancies. |
||||
*/ |
||||
|
||||
|
||||
public final class InvalidBalanceBlocks { |
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger(InvalidBalanceBlocks.class); |
||||
|
||||
private static final String ACCOUNT_DELTAS_SOURCE = "invalid-transaction-balance-deltas.json"; |
||||
|
||||
private static final List<AccountBalanceData> accountDeltas = readAccountDeltas(); |
||||
private static final List<Integer> affectedHeights = getAffectedHeights(); |
||||
|
||||
private InvalidBalanceBlocks() { |
||||
/* Do not instantiate */ |
||||
} |
||||
|
||||
@SuppressWarnings("unchecked") |
||||
private static List<AccountBalanceData> readAccountDeltas() { |
||||
Unmarshaller unmarshaller; |
||||
|
||||
try { |
||||
// Create JAXB context aware of classes we need to unmarshal
|
||||
JAXBContext jc = JAXBContextFactory.createContext(new Class[] { |
||||
AccountBalanceData.class |
||||
}, null); |
||||
|
||||
// Create unmarshaller
|
||||
unmarshaller = jc.createUnmarshaller(); |
||||
|
||||
// Set the unmarshaller media type to JSON
|
||||
unmarshaller.setProperty(UnmarshallerProperties.MEDIA_TYPE, "application/json"); |
||||
|
||||
// Tell unmarshaller that there's no JSON root element in the JSON input
|
||||
unmarshaller.setProperty(UnmarshallerProperties.JSON_INCLUDE_ROOT, false); |
||||
} catch (JAXBException e) { |
||||
String message = "Failed to setup unmarshaller to read block 212937 deltas"; |
||||
LOGGER.error(message, e); |
||||
throw new RuntimeException(message, e); |
||||
} |
||||
|
||||
ClassLoader classLoader = BlockChain.class.getClassLoader(); |
||||
InputStream in = classLoader.getResourceAsStream(ACCOUNT_DELTAS_SOURCE); |
||||
StreamSource jsonSource = new StreamSource(in); |
||||
|
||||
try { |
||||
// Attempt to unmarshal JSON stream to BlockChain config
|
||||
return (List<AccountBalanceData>) unmarshaller.unmarshal(jsonSource, AccountBalanceData.class).getValue(); |
||||
} catch (UnmarshalException e) { |
||||
String message = "Failed to parse balance deltas"; |
||||
LOGGER.error(message, e); |
||||
throw new RuntimeException(message, e); |
||||
} catch (JAXBException e) { |
||||
String message = "Unexpected JAXB issue while processing balance deltas"; |
||||
LOGGER.error(message, e); |
||||
throw new RuntimeException(message, e); |
||||
} |
||||
} |
||||
|
||||
private static List<Integer> getAffectedHeights() { |
||||
List<Integer> heights = new ArrayList<>(); |
||||
for (AccountBalanceData accountBalanceData : accountDeltas) { |
||||
if (!heights.contains(accountBalanceData.getHeight())) { |
||||
heights.add(accountBalanceData.getHeight()); |
||||
} |
||||
} |
||||
return heights; |
||||
} |
||||
|
||||
private static List<AccountBalanceData> getAccountDeltasAtHeight(int height) { |
||||
return accountDeltas.stream().filter(a -> a.getHeight() == height).collect(Collectors.toList()); |
||||
} |
||||
|
||||
public static boolean isAffectedBlock(int height) { |
||||
return affectedHeights.contains(Integer.valueOf(height)); |
||||
} |
||||
|
||||
public static void processFix(Block block) throws DataException { |
||||
Integer blockHeight = block.getBlockData().getHeight(); |
||||
List<AccountBalanceData> deltas = getAccountDeltasAtHeight(blockHeight); |
||||
if (deltas == null) { |
||||
throw new DataException(String.format("Unable to lookup invalid balance data for block height %d", blockHeight)); |
||||
} |
||||
|
||||
block.repository.getAccountRepository().modifyAssetBalances(deltas); |
||||
|
||||
LOGGER.info("Applied balance patch for block {}", blockHeight); |
||||
} |
||||
|
||||
public static void orphanFix(Block block) throws DataException { |
||||
Integer blockHeight = block.getBlockData().getHeight(); |
||||
List<AccountBalanceData> deltas = getAccountDeltasAtHeight(blockHeight); |
||||
if (deltas == null) { |
||||
throw new DataException(String.format("Unable to lookup invalid balance data for block height %d", blockHeight)); |
||||
} |
||||
|
||||
// Create inverse delta(s)
|
||||
for (AccountBalanceData accountBalanceData : deltas) { |
||||
AccountBalanceData inverseBalanceData = new AccountBalanceData(accountBalanceData.getAddress(), accountBalanceData.getAssetId(), -accountBalanceData.getBalance()); |
||||
block.repository.getAccountRepository().modifyAssetBalances(List.of(inverseBalanceData)); |
||||
} |
||||
|
||||
LOGGER.info("Reverted balance patch for block {}", blockHeight); |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,213 @@
|
||||
package org.qortal.repository; |
||||
|
||||
import org.apache.logging.log4j.LogManager; |
||||
import org.apache.logging.log4j.Logger; |
||||
import org.qortal.block.Block; |
||||
import org.qortal.block.GenesisBlock; |
||||
import org.qortal.controller.Controller; |
||||
import org.qortal.data.block.BlockArchiveData; |
||||
import org.qortal.data.block.BlockData; |
||||
import org.qortal.data.transaction.TransactionData; |
||||
import org.qortal.settings.Settings; |
||||
import org.qortal.transaction.Transaction; |
||||
import org.qortal.transform.block.BlockTransformation; |
||||
import org.qortal.utils.Base58; |
||||
import org.qortal.utils.NTP; |
||||
|
||||
import java.util.concurrent.TimeoutException; |
||||
|
||||
public class ReindexManager { |
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger(ReindexManager.class); |
||||
|
||||
private Repository repository; |
||||
|
||||
private final int pruneAndTrimBlockInterval = 2000; |
||||
private final int maintenanceBlockInterval = 50000; |
||||
|
||||
private boolean resume = false; |
||||
|
||||
public ReindexManager() { |
||||
|
||||
} |
||||
|
||||
public void reindex() throws DataException { |
||||
try { |
||||
this.runPreChecks(); |
||||
this.rebuildRepository(); |
||||
|
||||
try (final Repository repository = RepositoryManager.getRepository()) { |
||||
this.repository = repository; |
||||
this.requestCheckpoint(); |
||||
this.processGenesisBlock(); |
||||
this.processBlocks(); |
||||
} |
||||
|
||||
} catch (InterruptedException e) { |
||||
throw new DataException("Interrupted before complete"); |
||||
} |
||||
} |
||||
|
||||
private void runPreChecks() throws DataException, InterruptedException { |
||||
LOGGER.info("Running pre-checks..."); |
||||
if (Settings.getInstance().isTopOnly()) { |
||||
throw new DataException("Reindexing not supported in top-only mode. Please bootstrap or resync from genesis."); |
||||
} |
||||
if (Settings.getInstance().isLite()) { |
||||
throw new DataException("Reindexing not supported in lite mode."); |
||||
} |
||||
|
||||
while (NTP.getTime() == null) { |
||||
LOGGER.info("Waiting for NTP..."); |
||||
Thread.sleep(5000L); |
||||
} |
||||
} |
||||
|
||||
private void rebuildRepository() throws DataException { |
||||
if (resume) { |
||||
return; |
||||
} |
||||
|
||||
LOGGER.info("Rebuilding repository..."); |
||||
RepositoryManager.rebuild(); |
||||
} |
||||
|
||||
private void requestCheckpoint() { |
||||
RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); |
||||
} |
||||
|
||||
private void processGenesisBlock() throws DataException, InterruptedException { |
||||
if (resume) { |
||||
return; |
||||
} |
||||
|
||||
LOGGER.info("Processing genesis block..."); |
||||
|
||||
GenesisBlock genesisBlock = GenesisBlock.getInstance(repository); |
||||
|
||||
// Add Genesis Block to blockchain
|
||||
genesisBlock.process(); |
||||
|
||||
this.repository.saveChanges(); |
||||
} |
||||
|
||||
private void processBlocks() throws DataException { |
||||
LOGGER.info("Processing blocks..."); |
||||
|
||||
int height = this.repository.getBlockRepository().getBlockchainHeight(); |
||||
while (true) { |
||||
height++; |
||||
|
||||
boolean processed = this.processBlock(height); |
||||
if (!processed) { |
||||
LOGGER.info("Block {} couldn't be processed. If this is the last archived block, then the process is complete.", height); |
||||
break; // TODO: check if complete
|
||||
} |
||||
|
||||
// Prune and trim regularly, leaving a buffer
|
||||
if (height >= pruneAndTrimBlockInterval*2 && height % pruneAndTrimBlockInterval == 0) { |
||||
int startHeight = Math.max(height - pruneAndTrimBlockInterval*2, 2); |
||||
int endHeight = height - pruneAndTrimBlockInterval; |
||||
LOGGER.info("Pruning and trimming blocks {} to {}...", startHeight, endHeight); |
||||
this.repository.getATRepository().rebuildLatestAtStates(height - 250); |
||||
this.repository.saveChanges(); |
||||
this.prune(startHeight, endHeight); |
||||
this.trim(startHeight, endHeight); |
||||
} |
||||
|
||||
// Run repository maintenance regularly, to keep blockchain.data size down
|
||||
if (height % maintenanceBlockInterval == 0) { |
||||
this.runRepositoryMaintenance(); |
||||
} |
||||
} |
||||
} |
||||
|
||||
private boolean processBlock(int height) throws DataException { |
||||
Block block = this.fetchBlock(height); |
||||
if (block == null) { |
||||
return false; |
||||
} |
||||
|
||||
// Transactions are stored without approval status so determine that now
|
||||
for (Transaction transaction : block.getTransactions()) |
||||
transaction.setInitialApprovalStatus(); |
||||
|
||||
// It's best not to run preProcess() until there is a reason to
|
||||
// block.preProcess();
|
||||
|
||||
Block.ValidationResult validationResult = block.isValid(); |
||||
if (validationResult != Block.ValidationResult.OK) { |
||||
throw new DataException(String.format("Invalid block at height %d: %s", height, validationResult)); |
||||
} |
||||
|
||||
// Save transactions attached to this block
|
||||
for (Transaction transaction : block.getTransactions()) { |
||||
TransactionData transactionData = transaction.getTransactionData(); |
||||
this.repository.getTransactionRepository().save(transactionData); |
||||
} |
||||
|
||||
block.process(); |
||||
|
||||
LOGGER.info(String.format("Reindexed block height %d, sig %.8s", block.getBlockData().getHeight(), Base58.encode(block.getBlockData().getSignature()))); |
||||
|
||||
// Add to block archive table, since this originated from the archive but the chainstate has to be rebuilt
|
||||
this.addToBlockArchive(block.getBlockData()); |
||||
|
||||
this.repository.saveChanges(); |
||||
|
||||
Controller.getInstance().onNewBlock(block.getBlockData()); |
||||
|
||||
return true; |
||||
} |
||||
|
||||
private Block fetchBlock(int height) { |
||||
BlockTransformation b = BlockArchiveReader.getInstance().fetchBlockAtHeight(height); |
||||
if (b != null) { |
||||
if (b.getAtStatesHash() != null) { |
||||
return new Block(this.repository, b.getBlockData(), b.getTransactions(), b.getAtStatesHash()); |
||||
} |
||||
else { |
||||
return new Block(this.repository, b.getBlockData(), b.getTransactions(), b.getAtStates()); |
||||
} |
||||
} |
||||
|
||||
return null; |
||||
} |
||||
|
||||
private void addToBlockArchive(BlockData blockData) throws DataException { |
||||
// Write the signature and height into the BlockArchive table
|
||||
BlockArchiveData blockArchiveData = new BlockArchiveData(blockData); |
||||
this.repository.getBlockArchiveRepository().save(blockArchiveData); |
||||
this.repository.getBlockArchiveRepository().setBlockArchiveHeight(blockData.getHeight()+1); |
||||
this.repository.saveChanges(); |
||||
} |
||||
|
||||
private void prune(int startHeight, int endHeight) throws DataException { |
||||
this.repository.getBlockRepository().pruneBlocks(startHeight, endHeight); |
||||
this.repository.getATRepository().pruneAtStates(startHeight, endHeight); |
||||
this.repository.getATRepository().setAtPruneHeight(endHeight+1); |
||||
this.repository.saveChanges(); |
||||
} |
||||
|
||||
private void trim(int startHeight, int endHeight) throws DataException { |
||||
this.repository.getBlockRepository().trimOldOnlineAccountsSignatures(startHeight, endHeight); |
||||
|
||||
int count = 1; // Any number greater than 0
|
||||
while (count > 0) { |
||||
count = this.repository.getATRepository().trimAtStates(startHeight, endHeight, Settings.getInstance().getAtStatesTrimLimit()); |
||||
} |
||||
|
||||
this.repository.getBlockRepository().setBlockPruneHeight(endHeight+1); |
||||
this.repository.getATRepository().setAtTrimHeight(endHeight+1); |
||||
this.repository.saveChanges(); |
||||
} |
||||
|
||||
private void runRepositoryMaintenance() throws DataException { |
||||
try { |
||||
this.repository.performPeriodicMaintenance(1000L); |
||||
} catch (TimeoutException e) { |
||||
LOGGER.info("Timed out waiting for repository before running maintenance"); |
||||
} |
||||
} |
||||
|
||||
} |
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue