From 5824f75669b43eb6f0d86a9056416f9238bcad58 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 15 May 2021 12:19:15 +0100 Subject: [PATCH] Rework of the repository export and import functions. The existing HSQL export/import (PERFORM EXPORT SCRIPT and PERFORM IMPORT SCRIPT) have been replaced with a custom JSON import and export. Whilst this is less generic, it has some significant advantages: - When exporting data, it is now able to combine the exported data with any data that already exists in the backup file. This prevents a backup after a bootstrap from overwriting data from before the bootstrap, and removes the need for all of the "archive" files that we currently create. - Adds support for partial imports, and updates. Previously an import would fail if any of the data being imported already existed in the db. It will now add new rows and update existing ones. - The format and contents of the exported trade bot data now matches the output of the /crosschain/tradebot API. - Data is retrieved without the need for a database lock, and therefore the export process is much faster and less invasive. This should prevent the lockups and other problems seen when using the trade portal. For now, there are a couple of trade-offs to using this new approach: - The minting key import/export has been temporarily removed until there is more time to transition it to this new format. - Existing .script backups can no longer be imported using versions higher than 1.5.1. Both of these can be solved by temporarily running version 1.5.1, performing the necessary imports/exports, then returning to the latest version. Longer term the minting keys export/import will be reimplemented using the JSON format. --- .../qortal/api/resource/AdminResource.java | 19 +--- .../qortal/controller/tradebot/TradeBot.java | 12 +-- .../qortal/data/crosschain/TradeBotData.java | 55 ++++++++++ .../org/qortal/repository/Repository.java | 2 +- .../repository/hsqldb/HSQLDBRepository.java | 101 +++++++++--------- 5 files changed, 115 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index c295b90b..719a3b9d 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -542,19 +542,8 @@ public class AdminResource { Security.checkApiCallAllowed(request); try (final Repository repository = RepositoryManager.getRepository()) { - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - - blockchainLock.lockInterruptibly(); - - try { - repository.exportNodeLocalData(true); - return "true"; - } finally { - blockchainLock.unlock(); - } - } catch (InterruptedException e) { - // We couldn't lock blockchain to perform export - return "false"; + repository.exportNodeLocalData(); + return "true"; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -564,7 +553,7 @@ public class AdminResource { @Path("/repository/data") @Operation( summary = "Import data into repository.", - description = "Imports data from file on local machine. Filename is forced to 'import.script' if apiKey is not set.", + description = "Imports data from file on local machine. Filename is forced to 'import.json' if apiKey is not set.", requestBody = @RequestBody( required = true, content = @Content( @@ -588,7 +577,7 @@ public class AdminResource { // Hard-coded because it's too dangerous to allow user-supplied filenames in weaker security contexts if (Settings.getInstance().getApiKey() == null) - filename = "import.script"; + filename = "import.json"; try (final Repository repository = RepositoryManager.getRepository()) { ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 94c7cefb..fa3b599e 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -272,15 +272,9 @@ public class TradeBot implements Listener { // Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure try { LOGGER.info("About to backup trade bot data..."); - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - blockchainLock.lockInterruptibly(); - try { - repository.exportNodeLocalData(true); - } finally { - blockchainLock.unlock(); - } - } catch (InterruptedException | DataException e) { - LOGGER.info(String.format("Failed to obtain blockchain lock when exporting trade bot data: %s", e.getMessage())); + repository.exportNodeLocalData(); + } catch (DataException e) { + LOGGER.info(String.format("Repository issue when exporting trade bot data: %s", e.getMessage())); } } diff --git a/src/main/java/org/qortal/data/crosschain/TradeBotData.java b/src/main/java/org/qortal/data/crosschain/TradeBotData.java index b360c53e..19481466 100644 --- a/src/main/java/org/qortal/data/crosschain/TradeBotData.java +++ b/src/main/java/org/qortal/data/crosschain/TradeBotData.java @@ -6,6 +6,9 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; import io.swagger.v3.oas.annotations.media.Schema; +import org.json.JSONObject; + +import org.qortal.utils.Base58; // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) @@ -205,6 +208,58 @@ public class TradeBotData { return this.receivingAccountInfo; } + public JSONObject toJson() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("tradePrivateKey", Base58.encode(this.getTradePrivateKey())); + jsonObject.put("acctName", this.getAcctName()); + jsonObject.put("tradeState", this.getState()); + jsonObject.put("tradeStateValue", this.getStateValue()); + jsonObject.put("creatorAddress", this.getCreatorAddress()); + jsonObject.put("atAddress", this.getAtAddress()); + jsonObject.put("timestamp", this.getTimestamp()); + jsonObject.put("qortAmount", this.getQortAmount()); + if (this.getTradeNativePublicKey() != null) jsonObject.put("tradeNativePublicKey", Base58.encode(this.getTradeNativePublicKey())); + if (this.getTradeNativePublicKeyHash() != null) jsonObject.put("tradeNativePublicKeyHash", Base58.encode(this.getTradeNativePublicKeyHash())); + jsonObject.put("tradeNativeAddress", this.getTradeNativeAddress()); + if (this.getSecret() != null) jsonObject.put("secret", Base58.encode(this.getSecret())); + if (this.getHashOfSecret() != null) jsonObject.put("hashOfSecret", Base58.encode(this.getHashOfSecret())); + jsonObject.put("foreignBlockchain", this.getForeignBlockchain()); + if (this.getTradeForeignPublicKey() != null) jsonObject.put("tradeForeignPublicKey", Base58.encode(this.getTradeForeignPublicKey())); + if (this.getTradeForeignPublicKeyHash() != null) jsonObject.put("tradeForeignPublicKeyHash", Base58.encode(this.getTradeForeignPublicKeyHash())); + jsonObject.put("foreignKey", this.getForeignKey()); + jsonObject.put("foreignAmount", this.getForeignAmount()); + if (this.getLastTransactionSignature() != null) jsonObject.put("lastTransactionSignature", Base58.encode(this.getLastTransactionSignature())); + jsonObject.put("lockTimeA", this.getLockTimeA()); + if (this.getReceivingAccountInfo() != null) jsonObject.put("receivingAccountInfo", Base58.encode(this.getReceivingAccountInfo())); + return jsonObject; + } + + public static TradeBotData fromJson(JSONObject json) { + return new TradeBotData( + json.isNull("tradePrivateKey") ? null : Base58.decode(json.getString("tradePrivateKey")), + json.isNull("acctName") ? null : json.getString("acctName"), + json.isNull("tradeState") ? null : json.getString("tradeState"), + json.isNull("tradeStateValue") ? null : json.getInt("tradeStateValue"), + json.isNull("creatorAddress") ? null : json.getString("creatorAddress"), + json.isNull("atAddress") ? null : json.getString("atAddress"), + json.isNull("timestamp") ? null : json.getLong("timestamp"), + json.isNull("qortAmount") ? null : json.getLong("qortAmount"), + json.isNull("tradeNativePublicKey") ? null : Base58.decode(json.getString("tradeNativePublicKey")), + json.isNull("tradeNativePublicKeyHash") ? null : Base58.decode(json.getString("tradeNativePublicKeyHash")), + json.isNull("tradeNativeAddress") ? null : json.getString("tradeNativeAddress"), + json.isNull("secret") ? null : Base58.decode(json.getString("secret")), + json.isNull("hashOfSecret") ? null : Base58.decode(json.getString("hashOfSecret")), + json.isNull("foreignBlockchain") ? null : json.getString("foreignBlockchain"), + json.isNull("tradeForeignPublicKey") ? null : Base58.decode(json.getString("tradeForeignPublicKey")), + json.isNull("tradeForeignPublicKeyHash") ? null : Base58.decode(json.getString("tradeForeignPublicKeyHash")), + json.isNull("foreignAmount") ? null : json.getLong("foreignAmount"), + json.isNull("foreignKey") ? null : json.getString("foreignKey"), + json.isNull("lastTransactionSignature") ? null : Base58.decode(json.getString("lastTransactionSignature")), + json.isNull("lockTimeA") ? null : json.getInt("lockTimeA"), + json.isNull("receivingAccountInfo") ? null : Base58.decode(json.getString("receivingAccountInfo")) + ); + } + // Mostly for debugging public String toString() { return String.format("%s: %s (%d)", this.atAddress, this.tradeState, this.tradeStateValue); diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index 5438f1d9..656e6e1e 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -49,7 +49,7 @@ public interface Repository extends AutoCloseable { public void performPeriodicMaintenance() throws DataException; - public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException; + public void exportNodeLocalData() throws DataException; public void importDataFromFile(String filename) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 5557c13e..09c6a6d4 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -2,6 +2,7 @@ package org.qortal.repository.hsqldb; import java.awt.TrayIcon.MessageType; import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Files; @@ -15,23 +16,19 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Savepoint; import java.sql.Statement; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; +import org.json.JSONArray; +import org.json.JSONObject; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.PrivateKeyAccount; import org.qortal.crypto.Crypto; +import org.qortal.data.crosschain.TradeBotData; import org.qortal.globalization.Translator; import org.qortal.gui.SysTray; import org.qortal.repository.ATRepository; @@ -52,7 +49,7 @@ import org.qortal.repository.TransactionRepository; import org.qortal.repository.VotingRepository; import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository; import org.qortal.settings.Settings; -import org.qortal.utils.NTP; +import org.qortal.utils.Base58; public class HSQLDBRepository implements Repository { @@ -460,8 +457,7 @@ public class HSQLDBRepository implements Repository { } @Override - public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException { - + public void exportNodeLocalData() throws DataException { // Create the qortal-backup folder if it doesn't exist Path backupPath = Paths.get("qortal-backup"); try { @@ -471,52 +467,59 @@ public class HSQLDBRepository implements Repository { throw new DataException("Unable to create backup folder"); } - // We need to rename or delete an existing TradeBotStates backup before creating a new one - File tradeBotStatesBackupFile = new File("qortal-backup/TradeBotStates.script"); - if (tradeBotStatesBackupFile.exists()) { - if (keepArchivedCopy) { - // Rename existing TradeBotStates backup, to make sure that we're not overwriting any keys - File archivedBackupFile = new File(String.format("qortal-backup/TradeBotStates-archive-%d.script", NTP.getTime())); - if (tradeBotStatesBackupFile.renameTo(archivedBackupFile)) - LOGGER.info(String.format("Moved existing TradeBotStates backup file to %s", archivedBackupFile.getPath())); - else - throw new DataException("Unable to rename existing TradeBotStates backup"); - } else { - // Delete existing copy - LOGGER.info("Deleting existing TradeBotStates backup because it is being replaced with a new one"); - tradeBotStatesBackupFile.delete(); + try { + // Load trade bot data + List allTradeBotData = this.getCrossChainRepository().getAllTradeBotData(); + JSONArray allTradeBotDataJson = new JSONArray(); + for (TradeBotData tradeBotData : allTradeBotData) { + JSONObject tradeBotDataJson = tradeBotData.toJson(); + allTradeBotDataJson.put(tradeBotDataJson); } - } - // There's currently no need to take an archived copy of the MintingAccounts data - just delete the old one if it exists - File mintingAccountsBackupFile = new File("qortal-backup/MintingAccounts.script"); - if (mintingAccountsBackupFile.exists()) { - LOGGER.info("Deleting existing MintingAccounts backup because it is being replaced with a new one"); - mintingAccountsBackupFile.delete(); - } + // We need to combine existing TradeBotStates data before overwriting + String fileName = "qortal-backup/TradeBotStates.json"; + File tradeBotStatesBackupFile = new File(fileName); + if (tradeBotStatesBackupFile.exists()) { + String jsonString = new String(Files.readAllBytes(Paths.get(fileName))); + JSONArray allExistingTradeBotData = new JSONArray(jsonString); + Iterator iterator = allExistingTradeBotData.iterator(); + while(iterator.hasNext()) { + JSONObject existingTradeBotData = (JSONObject)iterator.next(); + String existingTradePrivateKey = (String) existingTradeBotData.get("tradePrivateKey"); + // Check if we already have an entry for this trade + boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey)); + if (found == false) + // We need to add this to our list + allTradeBotDataJson.put(existingTradeBotData); + } + } - try (Statement stmt = this.connection.createStatement()) { - stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'qortal-backup/MintingAccounts.script'"); - stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'qortal-backup/TradeBotStates.script'"); - LOGGER.info("Exported sensitive/node-local data: minting keys and trade bot states"); - } catch (SQLException e) { - throw new DataException("Unable to export sensitive/node-local data from repository"); + FileWriter writer = new FileWriter(fileName); + writer.write(allTradeBotDataJson.toString()); + writer.close(); + LOGGER.info("Exported sensitive/node-local data: trade bot states"); + + } catch (DataException | IOException e) { + throw new DataException("Unable to export trade bot states from repository"); } } @Override public void importDataFromFile(String filename) throws DataException { - try (Statement stmt = this.connection.createStatement()) { - LOGGER.info(() -> String.format("Importing data into repository from %s", filename)); - - String escapedFilename = stmt.enquoteLiteral(filename); - stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " CONTINUE ON ERROR"); - - LOGGER.info(() -> String.format("Imported data into repository from %s", filename)); - } catch (SQLException e) { - LOGGER.info(() -> String.format("Failed to import data into repository from %s: %s", filename, e.getMessage())); - throw new DataException("Unable to import sensitive/node-local data to repository: " + e.getMessage()); + LOGGER.info(() -> String.format("Importing data into repository from %s", filename)); + try { + String jsonString = new String(Files.readAllBytes(Paths.get(filename))); + JSONArray tradeBotDataToImport = new JSONArray(jsonString); + Iterator iterator = tradeBotDataToImport.iterator(); + while(iterator.hasNext()) { + JSONObject tradeBotDataJson = (JSONObject)iterator.next(); + TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson); + this.getCrossChainRepository().save(tradeBotData); + } + } catch (IOException e) { + throw new DataException("Unable to import sensitive/node-local trade bot states to repository: " + e.getMessage()); } + LOGGER.info(() -> String.format("Imported trade bot states into repository from %s", filename)); } @Override