Browse Source

Rework of the repository export and import functions.

The existing HSQL export/import (PERFORM EXPORT SCRIPT and PERFORM IMPORT SCRIPT) have been replaced with a custom JSON import and export. Whilst this is less generic, it has some significant advantages:

- When exporting data, it is now able to combine the exported data with any data that already exists in the backup file. This prevents a backup after a bootstrap from overwriting data from before the bootstrap, and removes the need for all of the "archive" files that we currently create.
- Adds support for partial imports, and updates. Previously an import would fail if any of the data being imported already existed in the db. It will now add new rows and update existing ones.
- The format and contents of the exported trade bot data now matches the output of the /crosschain/tradebot API.
- Data is retrieved without the need for a database lock, and therefore the export process is much faster and less invasive. This should prevent the lockups and other problems seen when using the trade portal.

For now, there are a couple of trade-offs to using this new approach:
- The minting key import/export has been temporarily removed until there is more time to transition it to this new format.
- Existing .script backups can no longer be imported using versions higher than 1.5.1.

Both of these can be solved by temporarily running version 1.5.1, performing the necessary imports/exports, then returning to the latest version. Longer term the minting keys export/import will be reimplemented using the JSON format.
networking
CalDescent 3 years ago
parent
commit
5824f75669
  1. 19
      src/main/java/org/qortal/api/resource/AdminResource.java
  2. 12
      src/main/java/org/qortal/controller/tradebot/TradeBot.java
  3. 55
      src/main/java/org/qortal/data/crosschain/TradeBotData.java
  4. 2
      src/main/java/org/qortal/repository/Repository.java
  5. 101
      src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java

19
src/main/java/org/qortal/api/resource/AdminResource.java

@ -542,19 +542,8 @@ public class AdminResource {
Security.checkApiCallAllowed(request);
try (final Repository repository = RepositoryManager.getRepository()) {
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
blockchainLock.lockInterruptibly();
try {
repository.exportNodeLocalData(true);
return "true";
} finally {
blockchainLock.unlock();
}
} catch (InterruptedException e) {
// We couldn't lock blockchain to perform export
return "false";
repository.exportNodeLocalData();
return "true";
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
@ -564,7 +553,7 @@ public class AdminResource {
@Path("/repository/data")
@Operation(
summary = "Import data into repository.",
description = "Imports data from file on local machine. Filename is forced to 'import.script' if apiKey is not set.",
description = "Imports data from file on local machine. Filename is forced to 'import.json' if apiKey is not set.",
requestBody = @RequestBody(
required = true,
content = @Content(
@ -588,7 +577,7 @@ public class AdminResource {
// Hard-coded because it's too dangerous to allow user-supplied filenames in weaker security contexts
if (Settings.getInstance().getApiKey() == null)
filename = "import.script";
filename = "import.json";
try (final Repository repository = RepositoryManager.getRepository()) {
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();

12
src/main/java/org/qortal/controller/tradebot/TradeBot.java

@ -272,15 +272,9 @@ public class TradeBot implements Listener {
// Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure
try {
LOGGER.info("About to backup trade bot data...");
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
blockchainLock.lockInterruptibly();
try {
repository.exportNodeLocalData(true);
} finally {
blockchainLock.unlock();
}
} catch (InterruptedException | DataException e) {
LOGGER.info(String.format("Failed to obtain blockchain lock when exporting trade bot data: %s", e.getMessage()));
repository.exportNodeLocalData();
} catch (DataException e) {
LOGGER.info(String.format("Repository issue when exporting trade bot data: %s", e.getMessage()));
}
}

55
src/main/java/org/qortal/data/crosschain/TradeBotData.java

@ -6,6 +6,9 @@ import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import io.swagger.v3.oas.annotations.media.Schema;
import org.json.JSONObject;
import org.qortal.utils.Base58;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
@ -205,6 +208,58 @@ public class TradeBotData {
return this.receivingAccountInfo;
}
public JSONObject toJson() {
JSONObject jsonObject = new JSONObject();
jsonObject.put("tradePrivateKey", Base58.encode(this.getTradePrivateKey()));
jsonObject.put("acctName", this.getAcctName());
jsonObject.put("tradeState", this.getState());
jsonObject.put("tradeStateValue", this.getStateValue());
jsonObject.put("creatorAddress", this.getCreatorAddress());
jsonObject.put("atAddress", this.getAtAddress());
jsonObject.put("timestamp", this.getTimestamp());
jsonObject.put("qortAmount", this.getQortAmount());
if (this.getTradeNativePublicKey() != null) jsonObject.put("tradeNativePublicKey", Base58.encode(this.getTradeNativePublicKey()));
if (this.getTradeNativePublicKeyHash() != null) jsonObject.put("tradeNativePublicKeyHash", Base58.encode(this.getTradeNativePublicKeyHash()));
jsonObject.put("tradeNativeAddress", this.getTradeNativeAddress());
if (this.getSecret() != null) jsonObject.put("secret", Base58.encode(this.getSecret()));
if (this.getHashOfSecret() != null) jsonObject.put("hashOfSecret", Base58.encode(this.getHashOfSecret()));
jsonObject.put("foreignBlockchain", this.getForeignBlockchain());
if (this.getTradeForeignPublicKey() != null) jsonObject.put("tradeForeignPublicKey", Base58.encode(this.getTradeForeignPublicKey()));
if (this.getTradeForeignPublicKeyHash() != null) jsonObject.put("tradeForeignPublicKeyHash", Base58.encode(this.getTradeForeignPublicKeyHash()));
jsonObject.put("foreignKey", this.getForeignKey());
jsonObject.put("foreignAmount", this.getForeignAmount());
if (this.getLastTransactionSignature() != null) jsonObject.put("lastTransactionSignature", Base58.encode(this.getLastTransactionSignature()));
jsonObject.put("lockTimeA", this.getLockTimeA());
if (this.getReceivingAccountInfo() != null) jsonObject.put("receivingAccountInfo", Base58.encode(this.getReceivingAccountInfo()));
return jsonObject;
}
public static TradeBotData fromJson(JSONObject json) {
return new TradeBotData(
json.isNull("tradePrivateKey") ? null : Base58.decode(json.getString("tradePrivateKey")),
json.isNull("acctName") ? null : json.getString("acctName"),
json.isNull("tradeState") ? null : json.getString("tradeState"),
json.isNull("tradeStateValue") ? null : json.getInt("tradeStateValue"),
json.isNull("creatorAddress") ? null : json.getString("creatorAddress"),
json.isNull("atAddress") ? null : json.getString("atAddress"),
json.isNull("timestamp") ? null : json.getLong("timestamp"),
json.isNull("qortAmount") ? null : json.getLong("qortAmount"),
json.isNull("tradeNativePublicKey") ? null : Base58.decode(json.getString("tradeNativePublicKey")),
json.isNull("tradeNativePublicKeyHash") ? null : Base58.decode(json.getString("tradeNativePublicKeyHash")),
json.isNull("tradeNativeAddress") ? null : json.getString("tradeNativeAddress"),
json.isNull("secret") ? null : Base58.decode(json.getString("secret")),
json.isNull("hashOfSecret") ? null : Base58.decode(json.getString("hashOfSecret")),
json.isNull("foreignBlockchain") ? null : json.getString("foreignBlockchain"),
json.isNull("tradeForeignPublicKey") ? null : Base58.decode(json.getString("tradeForeignPublicKey")),
json.isNull("tradeForeignPublicKeyHash") ? null : Base58.decode(json.getString("tradeForeignPublicKeyHash")),
json.isNull("foreignAmount") ? null : json.getLong("foreignAmount"),
json.isNull("foreignKey") ? null : json.getString("foreignKey"),
json.isNull("lastTransactionSignature") ? null : Base58.decode(json.getString("lastTransactionSignature")),
json.isNull("lockTimeA") ? null : json.getInt("lockTimeA"),
json.isNull("receivingAccountInfo") ? null : Base58.decode(json.getString("receivingAccountInfo"))
);
}
// Mostly for debugging
public String toString() {
return String.format("%s: %s (%d)", this.atAddress, this.tradeState, this.tradeStateValue);

2
src/main/java/org/qortal/repository/Repository.java

@ -49,7 +49,7 @@ public interface Repository extends AutoCloseable {
public void performPeriodicMaintenance() throws DataException;
public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException;
public void exportNodeLocalData() throws DataException;
public void importDataFromFile(String filename) throws DataException;

101
src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java

@ -2,6 +2,7 @@ package org.qortal.repository.hsqldb;
import java.awt.TrayIcon.MessageType;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.file.Files;
@ -15,23 +16,19 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Savepoint;
import java.sql.Statement;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import org.json.JSONArray;
import org.json.JSONObject;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.crypto.Crypto;
import org.qortal.data.crosschain.TradeBotData;
import org.qortal.globalization.Translator;
import org.qortal.gui.SysTray;
import org.qortal.repository.ATRepository;
@ -52,7 +49,7 @@ import org.qortal.repository.TransactionRepository;
import org.qortal.repository.VotingRepository;
import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository;
import org.qortal.settings.Settings;
import org.qortal.utils.NTP;
import org.qortal.utils.Base58;
public class HSQLDBRepository implements Repository {
@ -460,8 +457,7 @@ public class HSQLDBRepository implements Repository {
}
@Override
public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException {
public void exportNodeLocalData() throws DataException {
// Create the qortal-backup folder if it doesn't exist
Path backupPath = Paths.get("qortal-backup");
try {
@ -471,52 +467,59 @@ public class HSQLDBRepository implements Repository {
throw new DataException("Unable to create backup folder");
}
// We need to rename or delete an existing TradeBotStates backup before creating a new one
File tradeBotStatesBackupFile = new File("qortal-backup/TradeBotStates.script");
if (tradeBotStatesBackupFile.exists()) {
if (keepArchivedCopy) {
// Rename existing TradeBotStates backup, to make sure that we're not overwriting any keys
File archivedBackupFile = new File(String.format("qortal-backup/TradeBotStates-archive-%d.script", NTP.getTime()));
if (tradeBotStatesBackupFile.renameTo(archivedBackupFile))
LOGGER.info(String.format("Moved existing TradeBotStates backup file to %s", archivedBackupFile.getPath()));
else
throw new DataException("Unable to rename existing TradeBotStates backup");
} else {
// Delete existing copy
LOGGER.info("Deleting existing TradeBotStates backup because it is being replaced with a new one");
tradeBotStatesBackupFile.delete();
try {
// Load trade bot data
List<TradeBotData> allTradeBotData = this.getCrossChainRepository().getAllTradeBotData();
JSONArray allTradeBotDataJson = new JSONArray();
for (TradeBotData tradeBotData : allTradeBotData) {
JSONObject tradeBotDataJson = tradeBotData.toJson();
allTradeBotDataJson.put(tradeBotDataJson);
}
}
// There's currently no need to take an archived copy of the MintingAccounts data - just delete the old one if it exists
File mintingAccountsBackupFile = new File("qortal-backup/MintingAccounts.script");
if (mintingAccountsBackupFile.exists()) {
LOGGER.info("Deleting existing MintingAccounts backup because it is being replaced with a new one");
mintingAccountsBackupFile.delete();
}
// We need to combine existing TradeBotStates data before overwriting
String fileName = "qortal-backup/TradeBotStates.json";
File tradeBotStatesBackupFile = new File(fileName);
if (tradeBotStatesBackupFile.exists()) {
String jsonString = new String(Files.readAllBytes(Paths.get(fileName)));
JSONArray allExistingTradeBotData = new JSONArray(jsonString);
Iterator<Object> iterator = allExistingTradeBotData.iterator();
while(iterator.hasNext()) {
JSONObject existingTradeBotData = (JSONObject)iterator.next();
String existingTradePrivateKey = (String) existingTradeBotData.get("tradePrivateKey");
// Check if we already have an entry for this trade
boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey));
if (found == false)
// We need to add this to our list
allTradeBotDataJson.put(existingTradeBotData);
}
}
try (Statement stmt = this.connection.createStatement()) {
stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'qortal-backup/MintingAccounts.script'");
stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'qortal-backup/TradeBotStates.script'");
LOGGER.info("Exported sensitive/node-local data: minting keys and trade bot states");
} catch (SQLException e) {
throw new DataException("Unable to export sensitive/node-local data from repository");
FileWriter writer = new FileWriter(fileName);
writer.write(allTradeBotDataJson.toString());
writer.close();
LOGGER.info("Exported sensitive/node-local data: trade bot states");
} catch (DataException | IOException e) {
throw new DataException("Unable to export trade bot states from repository");
}
}
@Override
public void importDataFromFile(String filename) throws DataException {
try (Statement stmt = this.connection.createStatement()) {
LOGGER.info(() -> String.format("Importing data into repository from %s", filename));
String escapedFilename = stmt.enquoteLiteral(filename);
stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " CONTINUE ON ERROR");
LOGGER.info(() -> String.format("Imported data into repository from %s", filename));
} catch (SQLException e) {
LOGGER.info(() -> String.format("Failed to import data into repository from %s: %s", filename, e.getMessage()));
throw new DataException("Unable to import sensitive/node-local data to repository: " + e.getMessage());
LOGGER.info(() -> String.format("Importing data into repository from %s", filename));
try {
String jsonString = new String(Files.readAllBytes(Paths.get(filename)));
JSONArray tradeBotDataToImport = new JSONArray(jsonString);
Iterator<Object> iterator = tradeBotDataToImport.iterator();
while(iterator.hasNext()) {
JSONObject tradeBotDataJson = (JSONObject)iterator.next();
TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson);
this.getCrossChainRepository().save(tradeBotData);
}
} catch (IOException e) {
throw new DataException("Unable to import sensitive/node-local trade bot states to repository: " + e.getMessage());
}
LOGGER.info(() -> String.format("Imported trade bot states into repository from %s", filename));
}
@Override

Loading…
Cancel
Save