From 8926d2a73c6613653ee5da6553323fa830a4ae8b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 28 Sep 2021 09:24:21 +0100 Subject: [PATCH] Rework of import/export process. - Adds support for minting accounts as well as trade bot states - Includes automatic import of both types on node startup, and automatic export on node shutdown - Retains legacy trade bot states in a separate "TradeBotStatesArchive.json" file, whilst keeping the current active ones in "TradeBotStates.json". This prevents states being re-imported after they have been removed, but still keeps a copy of the data in case a key is ever needed. - Uses indentation in the JSON files for easier readability. --- .../qortal/api/resource/AdminResource.java | 4 + .../org/qortal/controller/Controller.java | 48 +++ .../qortal/controller/tradebot/TradeBot.java | 6 +- .../data/account/MintingAccountData.java | 19 + .../qortal/repository/AccountRepository.java | 2 + .../org/qortal/repository/Repository.java | 8 +- .../hsqldb/HSQLDBAccountRepository.java | 19 + .../repository/hsqldb/HSQLDBImportExport.java | 298 +++++++++++++ .../repository/hsqldb/HSQLDBRepository.java | 69 +--- .../java/org/qortal/settings/Settings.java | 7 + .../org/qortal/test/ImportExportTests.java | 390 ++++++++++++++++++ .../test-settings-v2-bitcoin-regtest.json | 1 + .../test-settings-v2-block-archive.json | 1 + .../test-settings-v2-founder-rewards.json | 1 + .../test-settings-v2-leftover-reward.json | 1 + .../resources/test-settings-v2-minting.json | 1 + ...test-settings-v2-qora-holder-extremes.json | 1 + .../test-settings-v2-qora-holder.json | 1 + .../test-settings-v2-reward-levels.json | 1 + .../test-settings-v2-reward-scaling.json | 1 + src/test/resources/test-settings-v2.json | 1 + 21 files changed, 810 insertions(+), 70 deletions(-) create mode 100644 src/main/java/org/qortal/repository/hsqldb/HSQLDBImportExport.java create mode 100644 src/test/java/org/qortal/test/ImportExportTests.java diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index 3e666fe4..bfcd54ca 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -609,6 +609,10 @@ public class AdminResource { repository.saveChanges(); return "true"; + + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e); + } finally { blockchainLock.unlock(); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 7755fd4d..fad244f4 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -424,6 +424,9 @@ public class Controller extends Thread { return; // Not System.exit() so that GUI can display error } + // Import current trade bot states and minting accounts if they exist + Controller.importRepositoryData(); + // Rebuild Names table and check database integrity NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); namesDatabaseIntegrityCheck.rebuildAllNames(); @@ -601,6 +604,47 @@ public class Controller extends Thread { } } + /** + * Import current trade bot states and minting accounts. + * This is needed because the user may have bootstrapped, or there could be a database inconsistency + * if the core crashed when computing the nonce during the start of the trade process. + */ + private static void importRepositoryData() { + try (final Repository repository = RepositoryManager.getRepository()) { + + String exportPath = Settings.getInstance().getExportPath(); + try { + Path importPath = Paths.get(exportPath, "TradeBotStates.json"); + repository.importDataFromFile(importPath.toString()); + } catch (FileNotFoundException e) { + // Do nothing, as the files will only exist in certain cases + } + + try { + Path importPath = Paths.get(exportPath, "MintingAccounts.json"); + repository.importDataFromFile(importPath.toString()); + } catch (FileNotFoundException e) { + // Do nothing, as the files will only exist in certain cases + } + repository.saveChanges(); + } + catch (DataException | IOException e) { + LOGGER.info("Unable to import data into repository: {}", e.getMessage()); + } + } + + /** + * Export current trade bot states and minting accounts. + */ + private void exportRepositoryData() { + try (final Repository repository = RepositoryManager.getRepository()) { + repository.exportNodeLocalData(); + + } catch (DataException e) { + // Fail silently as this is an optional step + } + } + public static final Predicate hasMisbehaved = peer -> { final Long lastMisbehaved = peer.getPeerData().getLastMisbehaved(); return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF; @@ -952,6 +996,10 @@ public class Controller extends Thread { } } + // Export local data + LOGGER.info("Backing up local data"); + this.exportRepositoryData(); + LOGGER.info("Shutting down networking"); Network.getInstance().shutdown(); diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 6e9d1474..36351927 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -245,17 +245,17 @@ public class TradeBot implements Listener { } } - /*package*/ static byte[] generateTradePrivateKey() { + public static byte[] generateTradePrivateKey() { // The private key is used for both Curve25519 and secp256k1 so needs to be valid for both. // Curve25519 accepts any seed, so generate a valid secp256k1 key and use that. return new ECKey().getPrivKeyBytes(); } - /*package*/ static byte[] deriveTradeNativePublicKey(byte[] privateKey) { + public static byte[] deriveTradeNativePublicKey(byte[] privateKey) { return PrivateKeyAccount.toPublicKey(privateKey); } - /*package*/ static byte[] deriveTradeForeignPublicKey(byte[] privateKey) { + public static byte[] deriveTradeForeignPublicKey(byte[] privateKey) { return ECKey.fromPrivate(privateKey).getPubKey(); } diff --git a/src/main/java/org/qortal/data/account/MintingAccountData.java b/src/main/java/org/qortal/data/account/MintingAccountData.java index 02b4c0f8..63c6c723 100644 --- a/src/main/java/org/qortal/data/account/MintingAccountData.java +++ b/src/main/java/org/qortal/data/account/MintingAccountData.java @@ -4,10 +4,12 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlTransient; +import org.json.JSONObject; import org.qortal.crypto.Crypto; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.media.Schema.AccessMode; +import org.qortal.utils.Base58; // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) @@ -61,4 +63,21 @@ public class MintingAccountData { return this.publicKey; } + + // JSON + + public JSONObject toJson() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("privateKey", Base58.encode(this.getPrivateKey())); + jsonObject.put("publicKey", Base58.encode(this.getPublicKey())); + return jsonObject; + } + + public static MintingAccountData fromJson(JSONObject json) { + return new MintingAccountData( + json.isNull("privateKey") ? null : Base58.decode(json.getString("privateKey")), + json.isNull("publicKey") ? null : Base58.decode(json.getString("publicKey")) + ); + } + } diff --git a/src/main/java/org/qortal/repository/AccountRepository.java b/src/main/java/org/qortal/repository/AccountRepository.java index a23771f9..256f9556 100644 --- a/src/main/java/org/qortal/repository/AccountRepository.java +++ b/src/main/java/org/qortal/repository/AccountRepository.java @@ -191,6 +191,8 @@ public interface AccountRepository { public List getMintingAccounts() throws DataException; + public MintingAccountData getMintingAccount(byte[] mintingAccountKey) throws DataException; + public void save(MintingAccountData mintingAccountData) throws DataException; /** Delete minting account info, used by BlockMinter, from repository using passed public or private key. */ diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index fab48a14..f6728968 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -1,5 +1,7 @@ package org.qortal.repository; +import java.io.IOException; + public interface Repository extends AutoCloseable { public ATRepository getATRepository(); @@ -47,14 +49,16 @@ public interface Repository extends AutoCloseable { public void setDebug(boolean debugState); - public void backup(boolean quick) throws DataException; + public void backup(boolean quick, String name) throws DataException; public void performPeriodicMaintenance() throws DataException; public void exportNodeLocalData() throws DataException; - public void importDataFromFile(String filename) throws DataException; + public void importDataFromFile(String filename) throws DataException, IOException; public void checkConsistency() throws DataException; + public static void attemptRecovery(String connectionUrl, String name) throws DataException {} + } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java index 0dca46eb..b28a224c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java @@ -904,6 +904,25 @@ public class HSQLDBAccountRepository implements AccountRepository { } } + @Override + public MintingAccountData getMintingAccount(byte[] mintingAccountKey) throws DataException { + try (ResultSet resultSet = this.repository.checkedExecute("SELECT minter_private_key, minter_public_key " + + "FROM MintingAccounts WHERE minter_private_key = ? OR minter_public_key = ?", + mintingAccountKey, mintingAccountKey)) { + + if (resultSet == null) + return null; + + byte[] minterPrivateKey = resultSet.getBytes(1); + byte[] minterPublicKey = resultSet.getBytes(2); + + return new MintingAccountData(minterPrivateKey, minterPublicKey); + + } catch (SQLException e) { + throw new DataException("Unable to fetch minting accounts from repository", e); + } + } + @Override public void save(MintingAccountData mintingAccountData) throws DataException { HSQLDBSaver saveHelper = new HSQLDBSaver("MintingAccounts"); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBImportExport.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBImportExport.java new file mode 100644 index 00000000..c5881c01 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBImportExport.java @@ -0,0 +1,298 @@ +package org.qortal.repository.hsqldb; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.qortal.data.account.MintingAccountData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.Bootstrap; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.settings.Settings; +import org.qortal.utils.Base58; +import org.qortal.utils.Triple; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; +import java.util.List; + +public class HSQLDBImportExport { + + private static final Logger LOGGER = LogManager.getLogger(Bootstrap.class); + + public static void backupTradeBotStates(Repository repository) throws DataException { + HSQLDBImportExport.backupCurrentTradeBotStates(repository); + HSQLDBImportExport.backupArchivedTradeBotStates(repository); + + LOGGER.info("Exported sensitive/node-local data: trade bot states"); + } + + public static void backupMintingAccounts(Repository repository) throws DataException { + HSQLDBImportExport.backupCurrentMintingAccounts(repository); + + LOGGER.info("Exported sensitive/node-local data: minting accounts"); + } + + + /* Trade bot states */ + + /** + * Backs up the trade bot states currently in the repository, without combining them with past ones + * @param repository + * @throws DataException + */ + private static void backupCurrentTradeBotStates(Repository repository) throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // Load current trade bot data + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + JSONArray currentTradeBotDataJson = new JSONArray(); + for (TradeBotData tradeBotData : allTradeBotData) { + JSONObject tradeBotDataJson = tradeBotData.toJson(); + currentTradeBotDataJson.put(tradeBotDataJson); + } + + // Wrap current trade bot data in an object to indicate the type + JSONObject currentTradeBotDataJsonWrapper = new JSONObject(); + currentTradeBotDataJsonWrapper.put("type", "tradeBotStates"); + currentTradeBotDataJsonWrapper.put("dataset", "current"); + currentTradeBotDataJsonWrapper.put("data", currentTradeBotDataJson); + + // Write current trade bot data (just the ones currently in the database) + String fileName = Paths.get(backupDirectory.toString(), "TradeBotStates.json").toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(currentTradeBotDataJsonWrapper.toString(2)); + writer.close(); + + } catch (DataException | IOException e) { + throw new DataException("Unable to export trade bot states from repository"); + } + } + + /** + * Backs up the trade bot states currently in the repository to a separate "archive" file, + * making sure to combine them with any unique states already present in the archive. + * @param repository + * @throws DataException + */ + private static void backupArchivedTradeBotStates(Repository repository) throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // Load current trade bot data + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + JSONArray allTradeBotDataJson = new JSONArray(); + for (TradeBotData tradeBotData : allTradeBotData) { + JSONObject tradeBotDataJson = tradeBotData.toJson(); + allTradeBotDataJson.put(tradeBotDataJson); + } + + // We need to combine existing archived TradeBotStates data before overwriting + String fileName = Paths.get(backupDirectory.toString(), "TradeBotStatesArchive.json").toString(); + File tradeBotStatesBackupFile = new File(fileName); + if (tradeBotStatesBackupFile.exists()) { + + String jsonString = new String(Files.readAllBytes(Paths.get(fileName))); + Triple parsedJSON = HSQLDBImportExport.parseJSONString(jsonString); + if (parsedJSON.getA() == null || parsedJSON.getC() == null) { + throw new DataException("Missing data when exporting archived trade bot states"); + } + String type = parsedJSON.getA(); + String dataset = parsedJSON.getB(); + JSONArray data = parsedJSON.getC(); + + if (!type.equals("tradeBotStates") || !dataset.equals("archive")) { + throw new DataException("Format mismatch when exporting archived trade bot states"); + } + + Iterator iterator = data.iterator(); + while(iterator.hasNext()) { + JSONObject existingTradeBotDataItem = (JSONObject)iterator.next(); + String existingTradePrivateKey = (String) existingTradeBotDataItem.get("tradePrivateKey"); + // Check if we already have an entry for this trade + boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey)); + if (found == false) + // Add the data from the backup file to our "allTradeBotDataJson" array as it's not currently in the db + allTradeBotDataJson.put(existingTradeBotDataItem); + } + } + + // Wrap all trade bot data in an object to indicate the type + JSONObject allTradeBotDataJsonWrapper = new JSONObject(); + allTradeBotDataJsonWrapper.put("type", "tradeBotStates"); + allTradeBotDataJsonWrapper.put("dataset", "archive"); + allTradeBotDataJsonWrapper.put("data", allTradeBotDataJson); + + // Write ALL trade bot data to archive (current plus states that are no longer in the database) + FileWriter writer = new FileWriter(fileName); + writer.write(allTradeBotDataJsonWrapper.toString(2)); + writer.close(); + + } catch (DataException | IOException e) { + throw new DataException("Unable to export trade bot states from repository"); + } + } + + + /* Minting accounts */ + + /** + * Backs up the minting accounts currently in the repository, without combining them with past ones + * @param repository + * @throws DataException + */ + private static void backupCurrentMintingAccounts(Repository repository) throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // Load current trade bot data + List allMintingAccountData = repository.getAccountRepository().getMintingAccounts(); + JSONArray currentMintingAccountJson = new JSONArray(); + for (MintingAccountData mintingAccountData : allMintingAccountData) { + JSONObject mintingAccountDataJson = mintingAccountData.toJson(); + currentMintingAccountJson.put(mintingAccountDataJson); + } + + // Wrap current trade bot data in an object to indicate the type + JSONObject currentMintingAccountDataJsonWrapper = new JSONObject(); + currentMintingAccountDataJsonWrapper.put("type", "mintingAccounts"); + currentMintingAccountDataJsonWrapper.put("dataset", "current"); + currentMintingAccountDataJsonWrapper.put("data", currentMintingAccountJson); + + // Write current trade bot data (just the ones currently in the database) + String fileName = Paths.get(backupDirectory.toString(), "MintingAccounts.json").toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(currentMintingAccountDataJsonWrapper.toString(2)); + writer.close(); + + } catch (DataException | IOException e) { + throw new DataException("Unable to export minting accounts from repository"); + } + } + + + /* Utils */ + + /** + * Imports data from supplied file + * Data type is loaded from the file itself, and if missing, TradeBotStates is assumed + * + * @param filename + * @param repository + * @throws DataException + * @throws IOException + */ + public static void importDataFromFile(String filename, Repository repository) throws DataException, IOException { + Path path = Paths.get(filename); + if (!path.toFile().exists()) { + throw new FileNotFoundException(String.format("File doesn't exist: %s", filename)); + } + byte[] fileContents = Files.readAllBytes(path); + if (fileContents == null) { + throw new FileNotFoundException(String.format("Unable to read file contents: %s", filename)); + } + + LOGGER.info(String.format("Importing %s into repository ...", filename)); + + String jsonString = new String(fileContents); + Triple parsedJSON = HSQLDBImportExport.parseJSONString(jsonString); + if (parsedJSON.getA() == null || parsedJSON.getC() == null) { + throw new DataException(String.format("Missing data when importing %s into repository", filename)); + } + String type = parsedJSON.getA(); + JSONArray data = parsedJSON.getC(); + + Iterator iterator = data.iterator(); + while(iterator.hasNext()) { + JSONObject dataJsonObject = (JSONObject)iterator.next(); + + if (type.equals("tradeBotStates")) { + HSQLDBImportExport.importTradeBotDataJSON(dataJsonObject, repository); + } + else if (type.equals("mintingAccounts")) { + HSQLDBImportExport.importMintingAccountDataJSON(dataJsonObject, repository); + } + else { + throw new DataException(String.format("Unrecognized data type when importing %s into repository", filename)); + } + + } + LOGGER.info(String.format("Imported %s into repository from %s", type, filename)); + } + + private static void importTradeBotDataJSON(JSONObject tradeBotDataJson, Repository repository) throws DataException { + TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson); + repository.getCrossChainRepository().save(tradeBotData); + } + + private static void importMintingAccountDataJSON(JSONObject mintingAccountDataJson, Repository repository) throws DataException { + MintingAccountData mintingAccountData = MintingAccountData.fromJson(mintingAccountDataJson); + repository.getAccountRepository().save(mintingAccountData); + } + + public static Path getExportDirectory(boolean createIfNotExists) throws DataException { + Path backupPath = Paths.get(Settings.getInstance().getExportPath()); + + if (createIfNotExists) { + // Create the qortal-backup folder if it doesn't exist + try { + Files.createDirectories(backupPath); + } catch (IOException e) { + LOGGER.info(String.format("Unable to create %s folder", backupPath.toString())); + throw new DataException(String.format("Unable to create %s folder", backupPath.toString())); + } + } + + return backupPath; + } + + /** + * Parses a JSON string and returns "data", "type", and "dataset" fields. + * In the case of legacy JSON files with no type, they are assumed to be TradeBotStates archives, + * as we had never implemented this for any other types. + * + * @param jsonString + * @return Triple (type, dataset, data) + */ + private static Triple parseJSONString(String jsonString) throws DataException { + String type = null; + String dataset = null; + JSONArray data = null; + + try { + // Firstly try importing the new format + JSONObject jsonData = new JSONObject(jsonString); + if (jsonData != null && jsonData.getString("type") != null) { + + type = jsonData.getString("type"); + dataset = jsonData.getString("dataset"); + data = jsonData.getJSONArray("data"); + } + + } catch (JSONException e) { + // Could be a legacy format which didn't contain a type or any other outer keys, so try importing that + // Treat these as TradeBotStates archives, given that this was the only type previously implemented + try { + type = "tradeBotStates"; + dataset = "archive"; + data = new JSONArray(jsonString); + + } catch (JSONException e2) { + // Still failed, so give up + throw new DataException("Couldn't import JSON file"); + } + } + + return new Triple(type, dataset, data); + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 6807c100..c1f8a2d5 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -2,7 +2,6 @@ package org.qortal.repository.hsqldb; import java.awt.TrayIcon.MessageType; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Files; @@ -21,20 +20,15 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; -import org.json.JSONArray; -import org.json.JSONObject; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.PrivateKeyAccount; import org.qortal.crypto.Crypto; -import org.qortal.data.crosschain.TradeBotData; import org.qortal.globalization.Translator; import org.qortal.gui.SysTray; import org.qortal.repository.*; import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository; import org.qortal.settings.Settings; -import org.qortal.utils.Base58; public class HSQLDBRepository implements Repository { @@ -450,68 +444,13 @@ public class HSQLDBRepository implements Repository { @Override public void exportNodeLocalData() throws DataException { - // Create the qortal-backup folder if it doesn't exist - Path backupPath = Paths.get("qortal-backup"); - try { - Files.createDirectories(backupPath); - } catch (IOException e) { - LOGGER.info("Unable to create backup folder"); - throw new DataException("Unable to create backup folder"); - } - - try { - // Load trade bot data - List allTradeBotData = this.getCrossChainRepository().getAllTradeBotData(); - JSONArray allTradeBotDataJson = new JSONArray(); - for (TradeBotData tradeBotData : allTradeBotData) { - JSONObject tradeBotDataJson = tradeBotData.toJson(); - allTradeBotDataJson.put(tradeBotDataJson); - } - - // We need to combine existing TradeBotStates data before overwriting - String fileName = "qortal-backup/TradeBotStates.json"; - File tradeBotStatesBackupFile = new File(fileName); - if (tradeBotStatesBackupFile.exists()) { - String jsonString = new String(Files.readAllBytes(Paths.get(fileName))); - JSONArray allExistingTradeBotData = new JSONArray(jsonString); - Iterator iterator = allExistingTradeBotData.iterator(); - while(iterator.hasNext()) { - JSONObject existingTradeBotData = (JSONObject)iterator.next(); - String existingTradePrivateKey = (String) existingTradeBotData.get("tradePrivateKey"); - // Check if we already have an entry for this trade - boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey)); - if (found == false) - // We need to add this to our list - allTradeBotDataJson.put(existingTradeBotData); - } - } - - FileWriter writer = new FileWriter(fileName); - writer.write(allTradeBotDataJson.toString()); - writer.close(); - LOGGER.info("Exported sensitive/node-local data: trade bot states"); - - } catch (DataException | IOException e) { - throw new DataException("Unable to export trade bot states from repository"); - } + HSQLDBImportExport.backupTradeBotStates(this); + HSQLDBImportExport.backupMintingAccounts(this); } @Override - public void importDataFromFile(String filename) throws DataException { - LOGGER.info(() -> String.format("Importing data into repository from %s", filename)); - try { - String jsonString = new String(Files.readAllBytes(Paths.get(filename))); - JSONArray tradeBotDataToImport = new JSONArray(jsonString); - Iterator iterator = tradeBotDataToImport.iterator(); - while(iterator.hasNext()) { - JSONObject tradeBotDataJson = (JSONObject)iterator.next(); - TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson); - this.getCrossChainRepository().save(tradeBotData); - } - } catch (IOException e) { - throw new DataException("Unable to import sensitive/node-local trade bot states to repository: " + e.getMessage()); - } - LOGGER.info(() -> String.format("Imported trade bot states into repository from %s", filename)); + public void importDataFromFile(String filename) throws DataException, IOException { + HSQLDBImportExport.importDataFromFile(filename, this); } @Override diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 15ead8e7..d284d59d 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -184,6 +184,9 @@ public class Settings { /** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */ private int repositoryConnectionPoolSize = 100; + // Export/import + private String exportPath = "qortal-backup"; + // Auto-update sources private String[] autoUpdateRepos = new String[] { "https://github.com/Qortal/qortal/raw/%s/qortal.update", @@ -502,6 +505,10 @@ public class Settings { return this.repositoryConnectionPoolSize; } + public String getExportPath() { + return this.exportPath; + } + public boolean isAutoUpdateEnabled() { return this.autoUpdateEnabled; } diff --git a/src/test/java/org/qortal/test/ImportExportTests.java b/src/test/java/org/qortal/test/ImportExportTests.java new file mode 100644 index 00000000..c7a5062f --- /dev/null +++ b/src/test/java/org/qortal/test/ImportExportTests.java @@ -0,0 +1,390 @@ +package org.qortal.test; + +import org.apache.commons.io.FileUtils; +import org.bitcoinj.core.Address; +import org.bitcoinj.core.AddressFormatException; +import org.bitcoinj.core.ECKey; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PublicKeyAccount; +import org.qortal.controller.tradebot.LitecoinACCTv1TradeBot; +import org.qortal.controller.tradebot.TradeBot; +import org.qortal.crosschain.Litecoin; +import org.qortal.crosschain.LitecoinACCTv1; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crypto.Crypto; +import org.qortal.data.account.MintingAccountData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBImportExport; +import org.qortal.settings.Settings; +import org.qortal.test.common.Common; +import org.qortal.utils.NTP; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class ImportExportTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + this.deleteExportDirectory(); + } + + @After + public void afterTest() throws DataException { + this.deleteExportDirectory(); + } + + + @Test + public void testExportAndImportTradeBotStates() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure no trade bots exist + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Create some trade bots + List tradeBots = new ArrayList<>(); + for (int i=0; i<10; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + tradeBots.add(tradeBotData); + } + + // Ensure they have been added + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Export them + HSQLDBImportExport.backupTradeBotStates(repository); + + // Delete them from the repository + for (TradeBotData tradeBotData : tradeBots) { + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + } + + // Ensure they have been deleted + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Import them + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Ensure all the data matches + for (TradeBotData tradeBotData : tradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNotNull(repositoryTradeBotData); + assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString()); + } + + repository.saveChanges(); + } + } + + @Test + public void testExportAndImportCurrentTradeBotStates() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure no trade bots exist + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Create some trade bots + List tradeBots = new ArrayList<>(); + for (int i=0; i<10; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + tradeBots.add(tradeBotData); + } + + // Ensure they have been added + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Export them + HSQLDBImportExport.backupTradeBotStates(repository); + + // Delete them from the repository + for (TradeBotData tradeBotData : tradeBots) { + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + } + + // Ensure they have been deleted + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Add some more trade bots + List additionalTradeBots = new ArrayList<>(); + for (int i=0; i<5; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + additionalTradeBots.add(tradeBotData); + } + + // Export again + HSQLDBImportExport.backupTradeBotStates(repository); + + // Import current states only + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(5, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Ensure that only the additional trade bots have been imported and that the data matches + for (TradeBotData tradeBotData : additionalTradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNotNull(repositoryTradeBotData); + assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString()); + } + + // None of the original trade bots should exist in the repository + for (TradeBotData tradeBotData : tradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNull(repositoryTradeBotData); + } + + repository.saveChanges(); + } + } + + @Test + public void testExportAndImportAllTradeBotStates() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure no trade bots exist + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Create some trade bots + List tradeBots = new ArrayList<>(); + for (int i=0; i<10; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + tradeBots.add(tradeBotData); + } + + // Ensure they have been added + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Export them + HSQLDBImportExport.backupTradeBotStates(repository); + + // Delete them from the repository + for (TradeBotData tradeBotData : tradeBots) { + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + } + + // Ensure they have been deleted + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Add some more trade bots + List additionalTradeBots = new ArrayList<>(); + for (int i=0; i<5; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + repository.getCrossChainRepository().save(tradeBotData); + additionalTradeBots.add(tradeBotData); + } + + // Export again + HSQLDBImportExport.backupTradeBotStates(repository); + + // Import all states from the archive + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "TradeBotStatesArchive.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(15, repository.getCrossChainRepository().getAllTradeBotData().size()); + + // Ensure that all known trade bots have been imported and that the data matches + tradeBots.addAll(additionalTradeBots); + + for (TradeBotData tradeBotData : tradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNotNull(repositoryTradeBotData); + assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString()); + } + + repository.saveChanges(); + } + } + + @Test + public void testExportAndImportLegacyTradeBotStates() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Create some trade bots, but don't save them in the repository + List tradeBots = new ArrayList<>(); + for (int i=0; i<10; i++) { + TradeBotData tradeBotData = this.createTradeBotData(repository); + tradeBots.add(tradeBotData); + } + + // Create a legacy format TradeBotStates.json backup file + this.exportLegacyTradeBotStatesJson(tradeBots); + + // Ensure no trade bots exist in repository + assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty()); + + // Import the legacy format file + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size()); + + for (TradeBotData tradeBotData : tradeBots) { + byte[] tradePrivateKey = tradeBotData.getTradePrivateKey(); + TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey); + assertNotNull(repositoryTradeBotData); + assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString()); + } + + repository.saveChanges(); + } + } + + @Test + public void testExportAndImportMintingAccountData() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure no minting accounts exist + assertTrue(repository.getAccountRepository().getMintingAccounts().isEmpty()); + + // Create some minting accounts + List mintingAccounts = new ArrayList<>(); + for (int i=0; i<10; i++) { + MintingAccountData mintingAccountData = this.createMintingAccountData(); + repository.getAccountRepository().save(mintingAccountData); + mintingAccounts.add(mintingAccountData); + } + + // Ensure they have been added + assertEquals(10, repository.getAccountRepository().getMintingAccounts().size()); + + // Export them + HSQLDBImportExport.backupMintingAccounts(repository); + + // Delete them from the repository + for (MintingAccountData mintingAccountData : mintingAccounts) { + repository.getAccountRepository().delete(mintingAccountData.getPrivateKey()); + } + + // Ensure they have been deleted + assertTrue(repository.getAccountRepository().getMintingAccounts().isEmpty()); + + // Import them + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + Path filePath = Paths.get(exportPath.toString(), "MintingAccounts.json"); + HSQLDBImportExport.importDataFromFile(filePath.toString(), repository); + + // Ensure they have been imported + assertEquals(10, repository.getAccountRepository().getMintingAccounts().size()); + + // Ensure all the data matches + for (MintingAccountData mintingAccountData : mintingAccounts) { + byte[] privateKey = mintingAccountData.getPrivateKey(); + MintingAccountData repositoryMintingAccountData = repository.getAccountRepository().getMintingAccount(privateKey); + assertNotNull(repositoryMintingAccountData); + assertEquals(mintingAccountData.toJson().toString(), repositoryMintingAccountData.toJson().toString()); + } + + repository.saveChanges(); + } + } + + + private TradeBotData createTradeBotData(Repository repository) throws DataException { + byte[] tradePrivateKey = TradeBot.generateTradePrivateKey(); + + byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey); + byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey); + String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey); + + byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey); + byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey); + + String receivingAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE"; + + // Convert Litecoin receiving address into public key hash (we only support P2PKH at this time) + Address litecoinReceivingAddress; + try { + litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), receivingAddress); + } catch (AddressFormatException e) { + throw new DataException("Unsupported Litecoin receiving address: " + receivingAddress); + } + + byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash(); + + byte[] creatorPublicKey = new byte[32]; + PublicKeyAccount creator = new PublicKeyAccount(repository, creatorPublicKey); + + long timestamp = NTP.getTime(); + String atAddress = "AT_ADDRESS"; + long foreignAmount = 1234; + long qortAmount= 5678; + + TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME, + LitecoinACCTv1TradeBot.State.BOB_WAITING_FOR_AT_CONFIRM.name(), LitecoinACCTv1TradeBot.State.BOB_WAITING_FOR_AT_CONFIRM.value, + creator.getAddress(), atAddress, timestamp, qortAmount, + tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress, + null, null, + SupportedBlockchain.LITECOIN.name(), + tradeForeignPublicKey, tradeForeignPublicKeyHash, + foreignAmount, null, null, null, litecoinReceivingAccountInfo); + + return tradeBotData; + } + + private MintingAccountData createMintingAccountData() { + // These don't need to be valid keys - just 32 byte strings for the purposes of testing + byte[] privateKey = new ECKey().getPrivKeyBytes(); + byte[] publicKey = new ECKey().getPrivKeyBytes(); + + return new MintingAccountData(privateKey, publicKey); + } + + private void exportLegacyTradeBotStatesJson(List allTradeBotData) throws IOException, DataException { + JSONArray allTradeBotDataJson = new JSONArray(); + for (TradeBotData tradeBotData : allTradeBotData) { + JSONObject tradeBotDataJson = tradeBotData.toJson(); + allTradeBotDataJson.put(tradeBotDataJson); + } + + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + String fileName = Paths.get(backupDirectory.toString(), "TradeBotStates.json").toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(allTradeBotDataJson.toString()); + writer.close(); + } + + private void deleteExportDirectory() { + // Delete archive directory if exists + Path archivePath = Paths.get(Settings.getInstance().getExportPath()); + try { + FileUtils.deleteDirectory(archivePath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/resources/test-settings-v2-bitcoin-regtest.json b/src/test/resources/test-settings-v2-bitcoin-regtest.json index 86379ae7..f0a993e2 100644 --- a/src/test/resources/test-settings-v2-bitcoin-regtest.json +++ b/src/test/resources/test-settings-v2-bitcoin-regtest.json @@ -3,6 +3,7 @@ "litecoinNet": "REGTEST", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0 diff --git a/src/test/resources/test-settings-v2-block-archive.json b/src/test/resources/test-settings-v2-block-archive.json index 612c8658..b71b2679 100644 --- a/src/test/resources/test-settings-v2-block-archive.json +++ b/src/test/resources/test-settings-v2-block-archive.json @@ -3,6 +3,7 @@ "litecoinNet": "TEST3", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0, diff --git a/src/test/resources/test-settings-v2-founder-rewards.json b/src/test/resources/test-settings-v2-founder-rewards.json index c89df187..b73544ea 100644 --- a/src/test/resources/test-settings-v2-founder-rewards.json +++ b/src/test/resources/test-settings-v2-founder-rewards.json @@ -1,6 +1,7 @@ { "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-founder-rewards.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0 diff --git a/src/test/resources/test-settings-v2-leftover-reward.json b/src/test/resources/test-settings-v2-leftover-reward.json index bdbc1d52..5c87cc94 100644 --- a/src/test/resources/test-settings-v2-leftover-reward.json +++ b/src/test/resources/test-settings-v2-leftover-reward.json @@ -1,6 +1,7 @@ { "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-leftover-reward.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0 diff --git a/src/test/resources/test-settings-v2-minting.json b/src/test/resources/test-settings-v2-minting.json index 9c72c375..abff27e3 100644 --- a/src/test/resources/test-settings-v2-minting.json +++ b/src/test/resources/test-settings-v2-minting.json @@ -1,6 +1,7 @@ { "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-minting.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0 diff --git a/src/test/resources/test-settings-v2-qora-holder-extremes.json b/src/test/resources/test-settings-v2-qora-holder-extremes.json index b311fbf2..dbf55170 100644 --- a/src/test/resources/test-settings-v2-qora-holder-extremes.json +++ b/src/test/resources/test-settings-v2-qora-holder-extremes.json @@ -1,6 +1,7 @@ { "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder-extremes.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0 diff --git a/src/test/resources/test-settings-v2-qora-holder.json b/src/test/resources/test-settings-v2-qora-holder.json index 83b23287..c9b995a6 100644 --- a/src/test/resources/test-settings-v2-qora-holder.json +++ b/src/test/resources/test-settings-v2-qora-holder.json @@ -1,6 +1,7 @@ { "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0 diff --git a/src/test/resources/test-settings-v2-reward-levels.json b/src/test/resources/test-settings-v2-reward-levels.json index 1c6862ad..4cc8de14 100644 --- a/src/test/resources/test-settings-v2-reward-levels.json +++ b/src/test/resources/test-settings-v2-reward-levels.json @@ -1,6 +1,7 @@ { "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-reward-levels.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0 diff --git a/src/test/resources/test-settings-v2-reward-scaling.json b/src/test/resources/test-settings-v2-reward-scaling.json index 262938b7..e1958d63 100644 --- a/src/test/resources/test-settings-v2-reward-scaling.json +++ b/src/test/resources/test-settings-v2-reward-scaling.json @@ -1,6 +1,7 @@ { "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-reward-scaling.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0 diff --git a/src/test/resources/test-settings-v2.json b/src/test/resources/test-settings-v2.json index a8983d3d..13c0a60f 100644 --- a/src/test/resources/test-settings-v2.json +++ b/src/test/resources/test-settings-v2.json @@ -3,6 +3,7 @@ "litecoinNet": "TEST3", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2.json", + "exportPath": "qortal-backup-test", "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0