Browse Source

Rework of import/export process.

- Adds support for minting accounts as well as trade bot states
- Includes automatic import of both types on node startup, and automatic export on node shutdown
- Retains legacy trade bot states in a separate "TradeBotStatesArchive.json" file, whilst keeping the current active ones in "TradeBotStates.json". This prevents states being re-imported after they have been removed, but still keeps a copy of the data in case a key is ever needed.
- Uses indentation in the JSON files for easier readability.
bootstrap
CalDescent 3 years ago
parent
commit
8926d2a73c
  1. 4
      src/main/java/org/qortal/api/resource/AdminResource.java
  2. 48
      src/main/java/org/qortal/controller/Controller.java
  3. 6
      src/main/java/org/qortal/controller/tradebot/TradeBot.java
  4. 19
      src/main/java/org/qortal/data/account/MintingAccountData.java
  5. 2
      src/main/java/org/qortal/repository/AccountRepository.java
  6. 8
      src/main/java/org/qortal/repository/Repository.java
  7. 19
      src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java
  8. 298
      src/main/java/org/qortal/repository/hsqldb/HSQLDBImportExport.java
  9. 69
      src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java
  10. 7
      src/main/java/org/qortal/settings/Settings.java
  11. 390
      src/test/java/org/qortal/test/ImportExportTests.java
  12. 1
      src/test/resources/test-settings-v2-bitcoin-regtest.json
  13. 1
      src/test/resources/test-settings-v2-block-archive.json
  14. 1
      src/test/resources/test-settings-v2-founder-rewards.json
  15. 1
      src/test/resources/test-settings-v2-leftover-reward.json
  16. 1
      src/test/resources/test-settings-v2-minting.json
  17. 1
      src/test/resources/test-settings-v2-qora-holder-extremes.json
  18. 1
      src/test/resources/test-settings-v2-qora-holder.json
  19. 1
      src/test/resources/test-settings-v2-reward-levels.json
  20. 1
      src/test/resources/test-settings-v2-reward-scaling.json
  21. 1
      src/test/resources/test-settings-v2.json

4
src/main/java/org/qortal/api/resource/AdminResource.java

@ -609,6 +609,10 @@ public class AdminResource {
repository.saveChanges();
return "true";
} catch (IOException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e);
} finally {
blockchainLock.unlock();
}

48
src/main/java/org/qortal/controller/Controller.java

@ -424,6 +424,9 @@ public class Controller extends Thread {
return; // Not System.exit() so that GUI can display error
}
// Import current trade bot states and minting accounts if they exist
Controller.importRepositoryData();
// Rebuild Names table and check database integrity
NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck();
namesDatabaseIntegrityCheck.rebuildAllNames();
@ -601,6 +604,47 @@ public class Controller extends Thread {
}
}
/**
* Import current trade bot states and minting accounts.
* This is needed because the user may have bootstrapped, or there could be a database inconsistency
* if the core crashed when computing the nonce during the start of the trade process.
*/
private static void importRepositoryData() {
try (final Repository repository = RepositoryManager.getRepository()) {
String exportPath = Settings.getInstance().getExportPath();
try {
Path importPath = Paths.get(exportPath, "TradeBotStates.json");
repository.importDataFromFile(importPath.toString());
} catch (FileNotFoundException e) {
// Do nothing, as the files will only exist in certain cases
}
try {
Path importPath = Paths.get(exportPath, "MintingAccounts.json");
repository.importDataFromFile(importPath.toString());
} catch (FileNotFoundException e) {
// Do nothing, as the files will only exist in certain cases
}
repository.saveChanges();
}
catch (DataException | IOException e) {
LOGGER.info("Unable to import data into repository: {}", e.getMessage());
}
}
/**
* Export current trade bot states and minting accounts.
*/
private void exportRepositoryData() {
try (final Repository repository = RepositoryManager.getRepository()) {
repository.exportNodeLocalData();
} catch (DataException e) {
// Fail silently as this is an optional step
}
}
public static final Predicate<Peer> hasMisbehaved = peer -> {
final Long lastMisbehaved = peer.getPeerData().getLastMisbehaved();
return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF;
@ -952,6 +996,10 @@ public class Controller extends Thread {
}
}
// Export local data
LOGGER.info("Backing up local data");
this.exportRepositoryData();
LOGGER.info("Shutting down networking");
Network.getInstance().shutdown();

6
src/main/java/org/qortal/controller/tradebot/TradeBot.java

@ -245,17 +245,17 @@ public class TradeBot implements Listener {
}
}
/*package*/ static byte[] generateTradePrivateKey() {
public static byte[] generateTradePrivateKey() {
// The private key is used for both Curve25519 and secp256k1 so needs to be valid for both.
// Curve25519 accepts any seed, so generate a valid secp256k1 key and use that.
return new ECKey().getPrivKeyBytes();
}
/*package*/ static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
public static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
return PrivateKeyAccount.toPublicKey(privateKey);
}
/*package*/ static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {
public static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {
return ECKey.fromPrivate(privateKey).getPubKey();
}

19
src/main/java/org/qortal/data/account/MintingAccountData.java

@ -4,10 +4,12 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlTransient;
import org.json.JSONObject;
import org.qortal.crypto.Crypto;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
import org.qortal.utils.Base58;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
@ -61,4 +63,21 @@ public class MintingAccountData {
return this.publicKey;
}
// JSON
public JSONObject toJson() {
JSONObject jsonObject = new JSONObject();
jsonObject.put("privateKey", Base58.encode(this.getPrivateKey()));
jsonObject.put("publicKey", Base58.encode(this.getPublicKey()));
return jsonObject;
}
public static MintingAccountData fromJson(JSONObject json) {
return new MintingAccountData(
json.isNull("privateKey") ? null : Base58.decode(json.getString("privateKey")),
json.isNull("publicKey") ? null : Base58.decode(json.getString("publicKey"))
);
}
}

2
src/main/java/org/qortal/repository/AccountRepository.java

@ -191,6 +191,8 @@ public interface AccountRepository {
public List<MintingAccountData> getMintingAccounts() throws DataException;
public MintingAccountData getMintingAccount(byte[] mintingAccountKey) throws DataException;
public void save(MintingAccountData mintingAccountData) throws DataException;
/** Delete minting account info, used by BlockMinter, from repository using passed public or private key. */

8
src/main/java/org/qortal/repository/Repository.java

@ -1,5 +1,7 @@
package org.qortal.repository;
import java.io.IOException;
public interface Repository extends AutoCloseable {
public ATRepository getATRepository();
@ -47,14 +49,16 @@ public interface Repository extends AutoCloseable {
public void setDebug(boolean debugState);
public void backup(boolean quick) throws DataException;
public void backup(boolean quick, String name) throws DataException;
public void performPeriodicMaintenance() throws DataException;
public void exportNodeLocalData() throws DataException;
public void importDataFromFile(String filename) throws DataException;
public void importDataFromFile(String filename) throws DataException, IOException;
public void checkConsistency() throws DataException;
public static void attemptRecovery(String connectionUrl, String name) throws DataException {}
}

19
src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java

@ -904,6 +904,25 @@ public class HSQLDBAccountRepository implements AccountRepository {
}
}
@Override
public MintingAccountData getMintingAccount(byte[] mintingAccountKey) throws DataException {
try (ResultSet resultSet = this.repository.checkedExecute("SELECT minter_private_key, minter_public_key " +
"FROM MintingAccounts WHERE minter_private_key = ? OR minter_public_key = ?",
mintingAccountKey, mintingAccountKey)) {
if (resultSet == null)
return null;
byte[] minterPrivateKey = resultSet.getBytes(1);
byte[] minterPublicKey = resultSet.getBytes(2);
return new MintingAccountData(minterPrivateKey, minterPublicKey);
} catch (SQLException e) {
throw new DataException("Unable to fetch minting accounts from repository", e);
}
}
@Override
public void save(MintingAccountData mintingAccountData) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("MintingAccounts");

298
src/main/java/org/qortal/repository/hsqldb/HSQLDBImportExport.java

@ -0,0 +1,298 @@
package org.qortal.repository.hsqldb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.crosschain.TradeBotData;
import org.qortal.repository.Bootstrap;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import org.qortal.utils.Triple;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
import java.util.List;
public class HSQLDBImportExport {
private static final Logger LOGGER = LogManager.getLogger(Bootstrap.class);
public static void backupTradeBotStates(Repository repository) throws DataException {
HSQLDBImportExport.backupCurrentTradeBotStates(repository);
HSQLDBImportExport.backupArchivedTradeBotStates(repository);
LOGGER.info("Exported sensitive/node-local data: trade bot states");
}
public static void backupMintingAccounts(Repository repository) throws DataException {
HSQLDBImportExport.backupCurrentMintingAccounts(repository);
LOGGER.info("Exported sensitive/node-local data: minting accounts");
}
/* Trade bot states */
/**
* Backs up the trade bot states currently in the repository, without combining them with past ones
* @param repository
* @throws DataException
*/
private static void backupCurrentTradeBotStates(Repository repository) throws DataException {
try {
Path backupDirectory = HSQLDBImportExport.getExportDirectory(true);
// Load current trade bot data
List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
JSONArray currentTradeBotDataJson = new JSONArray();
for (TradeBotData tradeBotData : allTradeBotData) {
JSONObject tradeBotDataJson = tradeBotData.toJson();
currentTradeBotDataJson.put(tradeBotDataJson);
}
// Wrap current trade bot data in an object to indicate the type
JSONObject currentTradeBotDataJsonWrapper = new JSONObject();
currentTradeBotDataJsonWrapper.put("type", "tradeBotStates");
currentTradeBotDataJsonWrapper.put("dataset", "current");
currentTradeBotDataJsonWrapper.put("data", currentTradeBotDataJson);
// Write current trade bot data (just the ones currently in the database)
String fileName = Paths.get(backupDirectory.toString(), "TradeBotStates.json").toString();
FileWriter writer = new FileWriter(fileName);
writer.write(currentTradeBotDataJsonWrapper.toString(2));
writer.close();
} catch (DataException | IOException e) {
throw new DataException("Unable to export trade bot states from repository");
}
}
/**
* Backs up the trade bot states currently in the repository to a separate "archive" file,
* making sure to combine them with any unique states already present in the archive.
* @param repository
* @throws DataException
*/
private static void backupArchivedTradeBotStates(Repository repository) throws DataException {
try {
Path backupDirectory = HSQLDBImportExport.getExportDirectory(true);
// Load current trade bot data
List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
JSONArray allTradeBotDataJson = new JSONArray();
for (TradeBotData tradeBotData : allTradeBotData) {
JSONObject tradeBotDataJson = tradeBotData.toJson();
allTradeBotDataJson.put(tradeBotDataJson);
}
// We need to combine existing archived TradeBotStates data before overwriting
String fileName = Paths.get(backupDirectory.toString(), "TradeBotStatesArchive.json").toString();
File tradeBotStatesBackupFile = new File(fileName);
if (tradeBotStatesBackupFile.exists()) {
String jsonString = new String(Files.readAllBytes(Paths.get(fileName)));
Triple<String, String, JSONArray> parsedJSON = HSQLDBImportExport.parseJSONString(jsonString);
if (parsedJSON.getA() == null || parsedJSON.getC() == null) {
throw new DataException("Missing data when exporting archived trade bot states");
}
String type = parsedJSON.getA();
String dataset = parsedJSON.getB();
JSONArray data = parsedJSON.getC();
if (!type.equals("tradeBotStates") || !dataset.equals("archive")) {
throw new DataException("Format mismatch when exporting archived trade bot states");
}
Iterator<Object> iterator = data.iterator();
while(iterator.hasNext()) {
JSONObject existingTradeBotDataItem = (JSONObject)iterator.next();
String existingTradePrivateKey = (String) existingTradeBotDataItem.get("tradePrivateKey");
// Check if we already have an entry for this trade
boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey));
if (found == false)
// Add the data from the backup file to our "allTradeBotDataJson" array as it's not currently in the db
allTradeBotDataJson.put(existingTradeBotDataItem);
}
}
// Wrap all trade bot data in an object to indicate the type
JSONObject allTradeBotDataJsonWrapper = new JSONObject();
allTradeBotDataJsonWrapper.put("type", "tradeBotStates");
allTradeBotDataJsonWrapper.put("dataset", "archive");
allTradeBotDataJsonWrapper.put("data", allTradeBotDataJson);
// Write ALL trade bot data to archive (current plus states that are no longer in the database)
FileWriter writer = new FileWriter(fileName);
writer.write(allTradeBotDataJsonWrapper.toString(2));
writer.close();
} catch (DataException | IOException e) {
throw new DataException("Unable to export trade bot states from repository");
}
}
/* Minting accounts */
/**
* Backs up the minting accounts currently in the repository, without combining them with past ones
* @param repository
* @throws DataException
*/
private static void backupCurrentMintingAccounts(Repository repository) throws DataException {
try {
Path backupDirectory = HSQLDBImportExport.getExportDirectory(true);
// Load current trade bot data
List<MintingAccountData> allMintingAccountData = repository.getAccountRepository().getMintingAccounts();
JSONArray currentMintingAccountJson = new JSONArray();
for (MintingAccountData mintingAccountData : allMintingAccountData) {
JSONObject mintingAccountDataJson = mintingAccountData.toJson();
currentMintingAccountJson.put(mintingAccountDataJson);
}
// Wrap current trade bot data in an object to indicate the type
JSONObject currentMintingAccountDataJsonWrapper = new JSONObject();
currentMintingAccountDataJsonWrapper.put("type", "mintingAccounts");
currentMintingAccountDataJsonWrapper.put("dataset", "current");
currentMintingAccountDataJsonWrapper.put("data", currentMintingAccountJson);
// Write current trade bot data (just the ones currently in the database)
String fileName = Paths.get(backupDirectory.toString(), "MintingAccounts.json").toString();
FileWriter writer = new FileWriter(fileName);
writer.write(currentMintingAccountDataJsonWrapper.toString(2));
writer.close();
} catch (DataException | IOException e) {
throw new DataException("Unable to export minting accounts from repository");
}
}
/* Utils */
/**
* Imports data from supplied file
* Data type is loaded from the file itself, and if missing, TradeBotStates is assumed
*
* @param filename
* @param repository
* @throws DataException
* @throws IOException
*/
public static void importDataFromFile(String filename, Repository repository) throws DataException, IOException {
Path path = Paths.get(filename);
if (!path.toFile().exists()) {
throw new FileNotFoundException(String.format("File doesn't exist: %s", filename));
}
byte[] fileContents = Files.readAllBytes(path);
if (fileContents == null) {
throw new FileNotFoundException(String.format("Unable to read file contents: %s", filename));
}
LOGGER.info(String.format("Importing %s into repository ...", filename));
String jsonString = new String(fileContents);
Triple<String, String, JSONArray> parsedJSON = HSQLDBImportExport.parseJSONString(jsonString);
if (parsedJSON.getA() == null || parsedJSON.getC() == null) {
throw new DataException(String.format("Missing data when importing %s into repository", filename));
}
String type = parsedJSON.getA();
JSONArray data = parsedJSON.getC();
Iterator<Object> iterator = data.iterator();
while(iterator.hasNext()) {
JSONObject dataJsonObject = (JSONObject)iterator.next();
if (type.equals("tradeBotStates")) {
HSQLDBImportExport.importTradeBotDataJSON(dataJsonObject, repository);
}
else if (type.equals("mintingAccounts")) {
HSQLDBImportExport.importMintingAccountDataJSON(dataJsonObject, repository);
}
else {
throw new DataException(String.format("Unrecognized data type when importing %s into repository", filename));
}
}
LOGGER.info(String.format("Imported %s into repository from %s", type, filename));
}
private static void importTradeBotDataJSON(JSONObject tradeBotDataJson, Repository repository) throws DataException {
TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson);
repository.getCrossChainRepository().save(tradeBotData);
}
private static void importMintingAccountDataJSON(JSONObject mintingAccountDataJson, Repository repository) throws DataException {
MintingAccountData mintingAccountData = MintingAccountData.fromJson(mintingAccountDataJson);
repository.getAccountRepository().save(mintingAccountData);
}
public static Path getExportDirectory(boolean createIfNotExists) throws DataException {
Path backupPath = Paths.get(Settings.getInstance().getExportPath());
if (createIfNotExists) {
// Create the qortal-backup folder if it doesn't exist
try {
Files.createDirectories(backupPath);
} catch (IOException e) {
LOGGER.info(String.format("Unable to create %s folder", backupPath.toString()));
throw new DataException(String.format("Unable to create %s folder", backupPath.toString()));
}
}
return backupPath;
}
/**
* Parses a JSON string and returns "data", "type", and "dataset" fields.
* In the case of legacy JSON files with no type, they are assumed to be TradeBotStates archives,
* as we had never implemented this for any other types.
*
* @param jsonString
* @return Triple<String, String, JSONArray> (type, dataset, data)
*/
private static Triple<String, String, JSONArray> parseJSONString(String jsonString) throws DataException {
String type = null;
String dataset = null;
JSONArray data = null;
try {
// Firstly try importing the new format
JSONObject jsonData = new JSONObject(jsonString);
if (jsonData != null && jsonData.getString("type") != null) {
type = jsonData.getString("type");
dataset = jsonData.getString("dataset");
data = jsonData.getJSONArray("data");
}
} catch (JSONException e) {
// Could be a legacy format which didn't contain a type or any other outer keys, so try importing that
// Treat these as TradeBotStates archives, given that this was the only type previously implemented
try {
type = "tradeBotStates";
dataset = "archive";
data = new JSONArray(jsonString);
} catch (JSONException e2) {
// Still failed, so give up
throw new DataException("Couldn't import JSON file");
}
}
return new Triple(type, dataset, data);
}
}

69
src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java

@ -2,7 +2,6 @@ package org.qortal.repository.hsqldb;
import java.awt.TrayIcon.MessageType;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.file.Files;
@ -21,20 +20,15 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import org.json.JSONArray;
import org.json.JSONObject;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.crypto.Crypto;
import org.qortal.data.crosschain.TradeBotData;
import org.qortal.globalization.Translator;
import org.qortal.gui.SysTray;
import org.qortal.repository.*;
import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
public class HSQLDBRepository implements Repository {
@ -450,68 +444,13 @@ public class HSQLDBRepository implements Repository {
@Override
public void exportNodeLocalData() throws DataException {
// Create the qortal-backup folder if it doesn't exist
Path backupPath = Paths.get("qortal-backup");
try {
Files.createDirectories(backupPath);
} catch (IOException e) {
LOGGER.info("Unable to create backup folder");
throw new DataException("Unable to create backup folder");
}
try {
// Load trade bot data
List<TradeBotData> allTradeBotData = this.getCrossChainRepository().getAllTradeBotData();
JSONArray allTradeBotDataJson = new JSONArray();
for (TradeBotData tradeBotData : allTradeBotData) {
JSONObject tradeBotDataJson = tradeBotData.toJson();
allTradeBotDataJson.put(tradeBotDataJson);
}
// We need to combine existing TradeBotStates data before overwriting
String fileName = "qortal-backup/TradeBotStates.json";
File tradeBotStatesBackupFile = new File(fileName);
if (tradeBotStatesBackupFile.exists()) {
String jsonString = new String(Files.readAllBytes(Paths.get(fileName)));
JSONArray allExistingTradeBotData = new JSONArray(jsonString);
Iterator<Object> iterator = allExistingTradeBotData.iterator();
while(iterator.hasNext()) {
JSONObject existingTradeBotData = (JSONObject)iterator.next();
String existingTradePrivateKey = (String) existingTradeBotData.get("tradePrivateKey");
// Check if we already have an entry for this trade
boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey));
if (found == false)
// We need to add this to our list
allTradeBotDataJson.put(existingTradeBotData);
}
}
FileWriter writer = new FileWriter(fileName);
writer.write(allTradeBotDataJson.toString());
writer.close();
LOGGER.info("Exported sensitive/node-local data: trade bot states");
} catch (DataException | IOException e) {
throw new DataException("Unable to export trade bot states from repository");
}
HSQLDBImportExport.backupTradeBotStates(this);
HSQLDBImportExport.backupMintingAccounts(this);
}
@Override
public void importDataFromFile(String filename) throws DataException {
LOGGER.info(() -> String.format("Importing data into repository from %s", filename));
try {
String jsonString = new String(Files.readAllBytes(Paths.get(filename)));
JSONArray tradeBotDataToImport = new JSONArray(jsonString);
Iterator<Object> iterator = tradeBotDataToImport.iterator();
while(iterator.hasNext()) {
JSONObject tradeBotDataJson = (JSONObject)iterator.next();
TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson);
this.getCrossChainRepository().save(tradeBotData);
}
} catch (IOException e) {
throw new DataException("Unable to import sensitive/node-local trade bot states to repository: " + e.getMessage());
}
LOGGER.info(() -> String.format("Imported trade bot states into repository from %s", filename));
public void importDataFromFile(String filename) throws DataException, IOException {
HSQLDBImportExport.importDataFromFile(filename, this);
}
@Override

7
src/main/java/org/qortal/settings/Settings.java

@ -184,6 +184,9 @@ public class Settings {
/** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */
private int repositoryConnectionPoolSize = 100;
// Export/import
private String exportPath = "qortal-backup";
// Auto-update sources
private String[] autoUpdateRepos = new String[] {
"https://github.com/Qortal/qortal/raw/%s/qortal.update",
@ -502,6 +505,10 @@ public class Settings {
return this.repositoryConnectionPoolSize;
}
public String getExportPath() {
return this.exportPath;
}
public boolean isAutoUpdateEnabled() {
return this.autoUpdateEnabled;
}

390
src/test/java/org/qortal/test/ImportExportTests.java

@ -0,0 +1,390 @@
package org.qortal.test;
import org.apache.commons.io.FileUtils;
import org.bitcoinj.core.Address;
import org.bitcoinj.core.AddressFormatException;
import org.bitcoinj.core.ECKey;
import org.json.JSONArray;
import org.json.JSONObject;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PublicKeyAccount;
import org.qortal.controller.tradebot.LitecoinACCTv1TradeBot;
import org.qortal.controller.tradebot.TradeBot;
import org.qortal.crosschain.Litecoin;
import org.qortal.crosschain.LitecoinACCTv1;
import org.qortal.crosschain.SupportedBlockchain;
import org.qortal.crypto.Crypto;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.crosschain.TradeBotData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.repository.hsqldb.HSQLDBImportExport;
import org.qortal.settings.Settings;
import org.qortal.test.common.Common;
import org.qortal.utils.NTP;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.*;
public class ImportExportTests extends Common {
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
this.deleteExportDirectory();
}
@After
public void afterTest() throws DataException {
this.deleteExportDirectory();
}
@Test
public void testExportAndImportTradeBotStates() throws DataException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Ensure no trade bots exist
assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
// Create some trade bots
List<TradeBotData> tradeBots = new ArrayList<>();
for (int i=0; i<10; i++) {
TradeBotData tradeBotData = this.createTradeBotData(repository);
repository.getCrossChainRepository().save(tradeBotData);
tradeBots.add(tradeBotData);
}
// Ensure they have been added
assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
// Export them
HSQLDBImportExport.backupTradeBotStates(repository);
// Delete them from the repository
for (TradeBotData tradeBotData : tradeBots) {
repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
}
// Ensure they have been deleted
assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
// Import them
Path exportPath = HSQLDBImportExport.getExportDirectory(false);
Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json");
HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
// Ensure they have been imported
assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
// Ensure all the data matches
for (TradeBotData tradeBotData : tradeBots) {
byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
assertNotNull(repositoryTradeBotData);
assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString());
}
repository.saveChanges();
}
}
@Test
public void testExportAndImportCurrentTradeBotStates() throws DataException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Ensure no trade bots exist
assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
// Create some trade bots
List<TradeBotData> tradeBots = new ArrayList<>();
for (int i=0; i<10; i++) {
TradeBotData tradeBotData = this.createTradeBotData(repository);
repository.getCrossChainRepository().save(tradeBotData);
tradeBots.add(tradeBotData);
}
// Ensure they have been added
assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
// Export them
HSQLDBImportExport.backupTradeBotStates(repository);
// Delete them from the repository
for (TradeBotData tradeBotData : tradeBots) {
repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
}
// Ensure they have been deleted
assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
// Add some more trade bots
List<TradeBotData> additionalTradeBots = new ArrayList<>();
for (int i=0; i<5; i++) {
TradeBotData tradeBotData = this.createTradeBotData(repository);
repository.getCrossChainRepository().save(tradeBotData);
additionalTradeBots.add(tradeBotData);
}
// Export again
HSQLDBImportExport.backupTradeBotStates(repository);
// Import current states only
Path exportPath = HSQLDBImportExport.getExportDirectory(false);
Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json");
HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
// Ensure they have been imported
assertEquals(5, repository.getCrossChainRepository().getAllTradeBotData().size());
// Ensure that only the additional trade bots have been imported and that the data matches
for (TradeBotData tradeBotData : additionalTradeBots) {
byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
assertNotNull(repositoryTradeBotData);
assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString());
}
// None of the original trade bots should exist in the repository
for (TradeBotData tradeBotData : tradeBots) {
byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
assertNull(repositoryTradeBotData);
}
repository.saveChanges();
}
}
@Test
public void testExportAndImportAllTradeBotStates() throws DataException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Ensure no trade bots exist
assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
// Create some trade bots
List<TradeBotData> tradeBots = new ArrayList<>();
for (int i=0; i<10; i++) {
TradeBotData tradeBotData = this.createTradeBotData(repository);
repository.getCrossChainRepository().save(tradeBotData);
tradeBots.add(tradeBotData);
}
// Ensure they have been added
assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
// Export them
HSQLDBImportExport.backupTradeBotStates(repository);
// Delete them from the repository
for (TradeBotData tradeBotData : tradeBots) {
repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
}
// Ensure they have been deleted
assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
// Add some more trade bots
List<TradeBotData> additionalTradeBots = new ArrayList<>();
for (int i=0; i<5; i++) {
TradeBotData tradeBotData = this.createTradeBotData(repository);
repository.getCrossChainRepository().save(tradeBotData);
additionalTradeBots.add(tradeBotData);
}
// Export again
HSQLDBImportExport.backupTradeBotStates(repository);
// Import all states from the archive
Path exportPath = HSQLDBImportExport.getExportDirectory(false);
Path filePath = Paths.get(exportPath.toString(), "TradeBotStatesArchive.json");
HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
// Ensure they have been imported
assertEquals(15, repository.getCrossChainRepository().getAllTradeBotData().size());
// Ensure that all known trade bots have been imported and that the data matches
tradeBots.addAll(additionalTradeBots);
for (TradeBotData tradeBotData : tradeBots) {
byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
assertNotNull(repositoryTradeBotData);
assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString());
}
repository.saveChanges();
}
}
@Test
public void testExportAndImportLegacyTradeBotStates() throws DataException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Create some trade bots, but don't save them in the repository
List<TradeBotData> tradeBots = new ArrayList<>();
for (int i=0; i<10; i++) {
TradeBotData tradeBotData = this.createTradeBotData(repository);
tradeBots.add(tradeBotData);
}
// Create a legacy format TradeBotStates.json backup file
this.exportLegacyTradeBotStatesJson(tradeBots);
// Ensure no trade bots exist in repository
assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
// Import the legacy format file
Path exportPath = HSQLDBImportExport.getExportDirectory(false);
Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json");
HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
// Ensure they have been imported
assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
for (TradeBotData tradeBotData : tradeBots) {
byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
assertNotNull(repositoryTradeBotData);
assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString());
}
repository.saveChanges();
}
}
@Test
public void testExportAndImportMintingAccountData() throws DataException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Ensure no minting accounts exist
assertTrue(repository.getAccountRepository().getMintingAccounts().isEmpty());
// Create some minting accounts
List<MintingAccountData> mintingAccounts = new ArrayList<>();
for (int i=0; i<10; i++) {
MintingAccountData mintingAccountData = this.createMintingAccountData();
repository.getAccountRepository().save(mintingAccountData);
mintingAccounts.add(mintingAccountData);
}
// Ensure they have been added
assertEquals(10, repository.getAccountRepository().getMintingAccounts().size());
// Export them
HSQLDBImportExport.backupMintingAccounts(repository);
// Delete them from the repository
for (MintingAccountData mintingAccountData : mintingAccounts) {
repository.getAccountRepository().delete(mintingAccountData.getPrivateKey());
}
// Ensure they have been deleted
assertTrue(repository.getAccountRepository().getMintingAccounts().isEmpty());
// Import them
Path exportPath = HSQLDBImportExport.getExportDirectory(false);
Path filePath = Paths.get(exportPath.toString(), "MintingAccounts.json");
HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
// Ensure they have been imported
assertEquals(10, repository.getAccountRepository().getMintingAccounts().size());
// Ensure all the data matches
for (MintingAccountData mintingAccountData : mintingAccounts) {
byte[] privateKey = mintingAccountData.getPrivateKey();
MintingAccountData repositoryMintingAccountData = repository.getAccountRepository().getMintingAccount(privateKey);
assertNotNull(repositoryMintingAccountData);
assertEquals(mintingAccountData.toJson().toString(), repositoryMintingAccountData.toJson().toString());
}
repository.saveChanges();
}
}
private TradeBotData createTradeBotData(Repository repository) throws DataException {
byte[] tradePrivateKey = TradeBot.generateTradePrivateKey();
byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey);
byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey);
String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey);
byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey);
byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey);
String receivingAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
// Convert Litecoin receiving address into public key hash (we only support P2PKH at this time)
Address litecoinReceivingAddress;
try {
litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), receivingAddress);
} catch (AddressFormatException e) {
throw new DataException("Unsupported Litecoin receiving address: " + receivingAddress);
}
byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash();
byte[] creatorPublicKey = new byte[32];
PublicKeyAccount creator = new PublicKeyAccount(repository, creatorPublicKey);
long timestamp = NTP.getTime();
String atAddress = "AT_ADDRESS";
long foreignAmount = 1234;
long qortAmount= 5678;
TradeBotData tradeBotData = new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME,
LitecoinACCTv1TradeBot.State.BOB_WAITING_FOR_AT_CONFIRM.name(), LitecoinACCTv1TradeBot.State.BOB_WAITING_FOR_AT_CONFIRM.value,
creator.getAddress(), atAddress, timestamp, qortAmount,
tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress,
null, null,
SupportedBlockchain.LITECOIN.name(),
tradeForeignPublicKey, tradeForeignPublicKeyHash,
foreignAmount, null, null, null, litecoinReceivingAccountInfo);
return tradeBotData;
}
private MintingAccountData createMintingAccountData() {
// These don't need to be valid keys - just 32 byte strings for the purposes of testing
byte[] privateKey = new ECKey().getPrivKeyBytes();
byte[] publicKey = new ECKey().getPrivKeyBytes();
return new MintingAccountData(privateKey, publicKey);
}
private void exportLegacyTradeBotStatesJson(List<TradeBotData> allTradeBotData) throws IOException, DataException {
JSONArray allTradeBotDataJson = new JSONArray();
for (TradeBotData tradeBotData : allTradeBotData) {
JSONObject tradeBotDataJson = tradeBotData.toJson();
allTradeBotDataJson.put(tradeBotDataJson);
}
Path backupDirectory = HSQLDBImportExport.getExportDirectory(true);
String fileName = Paths.get(backupDirectory.toString(), "TradeBotStates.json").toString();
FileWriter writer = new FileWriter(fileName);
writer.write(allTradeBotDataJson.toString());
writer.close();
}
private void deleteExportDirectory() {
// Delete archive directory if exists
Path archivePath = Paths.get(Settings.getInstance().getExportPath());
try {
FileUtils.deleteDirectory(archivePath.toFile());
} catch (IOException e) {
}
}
}

1
src/test/resources/test-settings-v2-bitcoin-regtest.json

@ -3,6 +3,7 @@
"litecoinNet": "REGTEST",
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0

1
src/test/resources/test-settings-v2-block-archive.json

@ -3,6 +3,7 @@
"litecoinNet": "TEST3",
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0,

1
src/test/resources/test-settings-v2-founder-rewards.json

@ -1,6 +1,7 @@
{
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2-founder-rewards.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0

1
src/test/resources/test-settings-v2-leftover-reward.json

@ -1,6 +1,7 @@
{
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2-leftover-reward.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0

1
src/test/resources/test-settings-v2-minting.json

@ -1,6 +1,7 @@
{
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2-minting.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0

1
src/test/resources/test-settings-v2-qora-holder-extremes.json

@ -1,6 +1,7 @@
{
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2-qora-holder-extremes.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0

1
src/test/resources/test-settings-v2-qora-holder.json

@ -1,6 +1,7 @@
{
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2-qora-holder.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0

1
src/test/resources/test-settings-v2-reward-levels.json

@ -1,6 +1,7 @@
{
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2-reward-levels.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0

1
src/test/resources/test-settings-v2-reward-scaling.json

@ -1,6 +1,7 @@
{
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2-reward-scaling.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0

1
src/test/resources/test-settings-v2.json

@ -3,6 +3,7 @@
"litecoinNet": "TEST3",
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2.json",
"exportPath": "qortal-backup-test",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0

Loading…
Cancel
Save