diff --git a/.gitignore b/.gitignore
index cf1e7ed2..8f2de896 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
/db*
+/lists/
/bin/
/target/
/qortal-backup/
@@ -15,8 +16,8 @@
/settings.json
/testnet*
/settings*.json
-/testchain.json
-/run-testnet.sh
+/testchain*.json
+/run-testnet*.sh
/.idea
/qortal.iml
.DS_Store
@@ -25,4 +26,6 @@
/run.pid
/run.log
/WindowsInstaller/Install Files/qortal.jar
+/*.7z
+/tmp
/data*
diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip
index 7d69ffb9..61ee6934 100755
--- a/WindowsInstaller/Qortal.aip
+++ b/WindowsInstaller/Qortal.aip
@@ -17,10 +17,10 @@
-
+
-
+
@@ -212,7 +212,7 @@
-
+
diff --git a/WindowsInstaller/qortal.ico b/WindowsInstaller/qortal.ico
old mode 100755
new mode 100644
index b0f8f5fb..a44ed445
Binary files a/WindowsInstaller/qortal.ico and b/WindowsInstaller/qortal.ico differ
diff --git a/pom.xml b/pom.xml
index f91d4cfa..1495dea7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -14,6 +14,9 @@
1.3.83.61.8
+ 2.6
+ 1.21
+ 1.91.2.228.1-jre2.5.1
@@ -454,7 +457,17 @@
commons-iocommons-io
- 2.6
+ ${commons-io.version}
+
+
+ org.apache.commons
+ commons-compress
+ ${commons-compress.version}
+
+
+ org.tukaani
+ xz
+ ${xz.version}
diff --git a/src/main/java/org/qortal/RepositoryMaintenance.java b/src/main/java/org/qortal/RepositoryMaintenance.java
index c3ae0616..b085822b 100644
--- a/src/main/java/org/qortal/RepositoryMaintenance.java
+++ b/src/main/java/org/qortal/RepositoryMaintenance.java
@@ -1,6 +1,7 @@
package org.qortal;
import java.security.Security;
+import java.util.concurrent.TimeoutException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -57,10 +58,10 @@ public class RepositoryMaintenance {
LOGGER.info("Starting repository periodic maintenance. This can take a while...");
try (final Repository repository = RepositoryManager.getRepository()) {
- repository.performPeriodicMaintenance();
+ repository.performPeriodicMaintenance(null);
LOGGER.info("Repository periodic maintenance completed");
- } catch (DataException e) {
+ } catch (DataException | TimeoutException e) {
LOGGER.error("Repository periodic maintenance failed", e);
}
diff --git a/src/main/java/org/qortal/api/ApiExceptionFactory.java b/src/main/java/org/qortal/api/ApiExceptionFactory.java
index e66c6e84..294cef83 100644
--- a/src/main/java/org/qortal/api/ApiExceptionFactory.java
+++ b/src/main/java/org/qortal/api/ApiExceptionFactory.java
@@ -16,4 +16,8 @@ public enum ApiExceptionFactory {
return createException(request, apiError, null);
}
+ public ApiException createCustomException(HttpServletRequest request, ApiError apiError, String message) {
+ return new ApiException(apiError.getStatus(), apiError.getCode(), message, null);
+ }
+
}
diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java
index 5baf2c5d..cafba4ae 100644
--- a/src/main/java/org/qortal/api/ApiService.java
+++ b/src/main/java/org/qortal/api/ApiService.java
@@ -14,6 +14,8 @@ import java.security.SecureRandom;
import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLContext;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.eclipse.jetty.http.HttpVersion;
import org.eclipse.jetty.rewrite.handler.RedirectPatternRule;
import org.eclipse.jetty.rewrite.handler.RewriteHandler;
@@ -50,6 +52,8 @@ import org.qortal.settings.Settings;
public class ApiService {
+ private static final Logger LOGGER = LogManager.getLogger(ApiService.class);
+
private static ApiService instance;
private final ResourceConfig config;
@@ -203,6 +207,9 @@ public class ApiService {
context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot");
context.addServlet(PresenceWebSocket.class, "/websockets/presence");
+ // Warn about API security if needed
+ this.checkApiSecurity();
+
// Start server
this.server.start();
} catch (Exception e) {
@@ -222,4 +229,23 @@ public class ApiService {
this.server = null;
}
+ private void checkApiSecurity() {
+ // Warn about API security if needed
+ boolean allConnectionsAllowed = false;
+ if (Settings.getInstance().isApiKeyDisabled()) {
+ for (String pattern : Settings.getInstance().getApiWhitelist()) {
+ if (pattern.startsWith("0.0.0.0/") || pattern.startsWith("::/") || pattern.endsWith("/0")) {
+ allConnectionsAllowed = true;
+ }
+ }
+
+ if (allConnectionsAllowed) {
+ LOGGER.warn("Warning: API key validation is currently disabled, and the API whitelist " +
+ "is allowing all connections. This can be a security risk.");
+ LOGGER.warn("To fix, set the apiKeyDisabled setting to false, or allow only specific local " +
+ "IP addresses using the apiWhitelist setting.");
+ }
+ }
+ }
+
}
diff --git a/src/main/java/org/qortal/api/Security.java b/src/main/java/org/qortal/api/Security.java
index 448f951a..4e25b03b 100644
--- a/src/main/java/org/qortal/api/Security.java
+++ b/src/main/java/org/qortal/api/Security.java
@@ -12,6 +12,11 @@ public abstract class Security {
public static final String API_KEY_HEADER = "X-API-KEY";
public static void checkApiCallAllowed(HttpServletRequest request) {
+ // If API key checking has been disabled, we will allow the request in all cases
+ boolean isApiKeyDisabled = Settings.getInstance().isApiKeyDisabled();
+ if (isApiKeyDisabled)
+ return;
+
String expectedApiKey = Settings.getInstance().getApiKey();
String passedApiKey = request.getHeader(API_KEY_HEADER);
diff --git a/src/main/java/org/qortal/api/model/AddressListRequest.java b/src/main/java/org/qortal/api/model/AddressListRequest.java
new file mode 100644
index 00000000..c600609f
--- /dev/null
+++ b/src/main/java/org/qortal/api/model/AddressListRequest.java
@@ -0,0 +1,18 @@
+package org.qortal.api.model;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.List;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AddressListRequest {
+
+ @Schema(description = "A list of addresses")
+ public List addresses;
+
+ public AddressListRequest() {
+ }
+
+}
diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java
index 35fccd96..2fdacf9d 100644
--- a/src/main/java/org/qortal/api/resource/AdminResource.java
+++ b/src/main/java/org/qortal/api/resource/AdminResource.java
@@ -22,6 +22,7 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
@@ -35,6 +36,7 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.RollingFileAppender;
import org.qortal.account.Account;
@@ -67,6 +69,8 @@ import com.google.common.collect.Lists;
@Tag(name = "Admin")
public class AdminResource {
+ private static final Logger LOGGER = LogManager.getLogger(AdminResource.class);
+
private static final int MAX_LOG_LINES = 500;
@Context
@@ -460,6 +464,23 @@ public class AdminResource {
if (targetHeight <= 0 || targetHeight > Controller.getInstance().getChainHeight())
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT);
+ // Make sure we're not orphaning as far back as the archived blocks
+ // FUTURE: we could support this by first importing earlier blocks from the archive
+ if (Settings.getInstance().isTopOnly() ||
+ Settings.getInstance().isArchiveEnabled()) {
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ // Find the first unarchived block
+ int oldestBlock = repository.getBlockArchiveRepository().getBlockArchiveHeight();
+ // Add some extra blocks just in case we're currently archiving/pruning
+ oldestBlock += 100;
+ if (targetHeight <= oldestBlock) {
+ LOGGER.info("Unable to orphan beyond block {} because it is archived", oldestBlock);
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT);
+ }
+ }
+ }
+
if (BlockChain.orphan(targetHeight))
return "true";
else
@@ -554,13 +575,13 @@ public class AdminResource {
@Path("/repository/data")
@Operation(
summary = "Import data into repository.",
- description = "Imports data from file on local machine. Filename is forced to 'import.json' if apiKey is not set.",
+ description = "Imports data from file on local machine. Filename is forced to 'qortal-backup/TradeBotStates.json' if apiKey is not set.",
requestBody = @RequestBody(
required = true,
content = @Content(
mediaType = MediaType.TEXT_PLAIN,
schema = @Schema(
- type = "string", example = "MintingAccounts.script"
+ type = "string", example = "qortal-backup/TradeBotStates.json"
)
)
),
@@ -578,7 +599,7 @@ public class AdminResource {
// Hard-coded because it's too dangerous to allow user-supplied filenames in weaker security contexts
if (Settings.getInstance().getApiKey() == null)
- filename = "import.json";
+ filename = "qortal-backup/TradeBotStates.json";
try (final Repository repository = RepositoryManager.getRepository()) {
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
@@ -590,6 +611,10 @@ public class AdminResource {
repository.saveChanges();
return "true";
+
+ } catch (IOException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e);
+
} finally {
blockchainLock.unlock();
}
@@ -645,14 +670,16 @@ public class AdminResource {
blockchainLock.lockInterruptibly();
try {
- repository.backup(true);
+ // Timeout if the database isn't ready for backing up after 60 seconds
+ long timeout = 60 * 1000L;
+ repository.backup(true, "backup", timeout);
repository.saveChanges();
return "true";
} finally {
blockchainLock.unlock();
}
- } catch (InterruptedException e) {
+ } catch (InterruptedException | TimeoutException e) {
// We couldn't lock blockchain to perform backup
return "false";
} catch (DataException e) {
@@ -677,13 +704,15 @@ public class AdminResource {
blockchainLock.lockInterruptibly();
try {
- repository.performPeriodicMaintenance();
+ // Timeout if the database isn't ready to start after 60 seconds
+ long timeout = 60 * 1000L;
+ repository.performPeriodicMaintenance(timeout);
} finally {
blockchainLock.unlock();
}
} catch (InterruptedException e) {
// No big deal
- } catch (DataException e) {
+ } catch (DataException | TimeoutException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java
index 8920ecc1..b8163c7d 100644
--- a/src/main/java/org/qortal/api/resource/BlocksResource.java
+++ b/src/main/java/org/qortal/api/resource/BlocksResource.java
@@ -15,6 +15,8 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
@@ -33,11 +35,13 @@ import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.model.BlockMintingInfo;
import org.qortal.api.model.BlockSignerSummary;
import org.qortal.block.Block;
+import org.qortal.controller.Controller;
import org.qortal.crypto.Crypto;
import org.qortal.data.account.AccountData;
import org.qortal.data.block.BlockData;
import org.qortal.data.block.BlockSummaryData;
import org.qortal.data.transaction.TransactionData;
+import org.qortal.repository.BlockArchiveReader;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
@@ -81,11 +85,19 @@ public class BlocksResource {
}
try (final Repository repository = RepositoryManager.getRepository()) {
+ // Check the database first
BlockData blockData = repository.getBlockRepository().fromSignature(signature);
- if (blockData == null)
- throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ if (blockData != null) {
+ return blockData;
+ }
- return blockData;
+ // Not found, so try the block archive
+ blockData = repository.getBlockArchiveRepository().fromSignature(signature);
+ if (blockData != null) {
+ return blockData;
+ }
+
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
@@ -116,16 +128,24 @@ public class BlocksResource {
}
try (final Repository repository = RepositoryManager.getRepository()) {
+
+ // Check the database first
BlockData blockData = repository.getBlockRepository().fromSignature(signature);
- if (blockData == null)
- throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ if (blockData != null) {
+ Block block = new Block(repository, blockData);
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
+ bytes.write(BlockTransformer.toBytes(block));
+ return Base58.encode(bytes.toByteArray());
+ }
- Block block = new Block(repository, blockData);
- ByteArrayOutputStream bytes = new ByteArrayOutputStream();
- bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
- bytes.write(BlockTransformer.toBytes(block));
- return Base58.encode(bytes.toByteArray());
+ // Not found, so try the block archive
+ byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, false, repository);
+ if (bytes != null) {
+ return Base58.encode(bytes);
+ }
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
} catch (TransformationException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e);
} catch (DataException | IOException e) {
@@ -170,8 +190,12 @@ public class BlocksResource {
}
try (final Repository repository = RepositoryManager.getRepository()) {
- if (repository.getBlockRepository().getHeightFromSignature(signature) == 0)
+ // Check if the block exists in either the database or archive
+ if (repository.getBlockRepository().getHeightFromSignature(signature) == 0 &&
+ repository.getBlockArchiveRepository().getHeightFromSignature(signature) == 0) {
+ // Not found in either the database or archive
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ }
return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse);
} catch (DataException e) {
@@ -200,7 +224,19 @@ public class BlocksResource {
})
public BlockData getFirstBlock() {
try (final Repository repository = RepositoryManager.getRepository()) {
- return repository.getBlockRepository().fromHeight(1);
+ // Check the database first
+ BlockData blockData = repository.getBlockRepository().fromHeight(1);
+ if (blockData != null) {
+ return blockData;
+ }
+
+ // Try the archive
+ blockData = repository.getBlockArchiveRepository().fromHeight(1);
+ if (blockData != null) {
+ return blockData;
+ }
+
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
@@ -262,17 +298,28 @@ public class BlocksResource {
}
try (final Repository repository = RepositoryManager.getRepository()) {
+ BlockData childBlockData = null;
+
+ // Check if block exists in database
BlockData blockData = repository.getBlockRepository().fromSignature(signature);
+ if (blockData != null) {
+ return repository.getBlockRepository().fromReference(signature);
+ }
- // Check block exists
- if (blockData == null)
- throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
-
- BlockData childBlockData = repository.getBlockRepository().fromReference(signature);
+ // Not found, so try the archive
+ // This also checks that the parent block exists
+ // It will return null if either the parent or child don't exit
+ childBlockData = repository.getBlockArchiveRepository().fromReference(signature);
// Check child block exists
- if (childBlockData == null)
+ if (childBlockData == null) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ }
+
+ // Check child block's reference matches the supplied signature
+ if (!Arrays.equals(childBlockData.getReference(), signature)) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ }
return childBlockData;
} catch (DataException e) {
@@ -338,13 +385,20 @@ public class BlocksResource {
}
try (final Repository repository = RepositoryManager.getRepository()) {
+ // Firstly check the database
BlockData blockData = repository.getBlockRepository().fromSignature(signature);
+ if (blockData != null) {
+ return blockData.getHeight();
+ }
- // Check block exists
- if (blockData == null)
- throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ // Not found, so try the archive
+ blockData = repository.getBlockArchiveRepository().fromSignature(signature);
+ if (blockData != null) {
+ return blockData.getHeight();
+ }
+
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
- return blockData.getHeight();
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
@@ -371,11 +425,20 @@ public class BlocksResource {
})
public BlockData getByHeight(@PathParam("height") int height) {
try (final Repository repository = RepositoryManager.getRepository()) {
+ // Firstly check the database
BlockData blockData = repository.getBlockRepository().fromHeight(height);
- if (blockData == null)
- throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ if (blockData != null) {
+ return blockData;
+ }
+
+ // Not found, so try the archive
+ blockData = repository.getBlockArchiveRepository().fromHeight(height);
+ if (blockData != null) {
+ return blockData;
+ }
+
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
- return blockData;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
@@ -402,12 +465,31 @@ public class BlocksResource {
})
public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) {
try (final Repository repository = RepositoryManager.getRepository()) {
+ // Try the database
BlockData blockData = repository.getBlockRepository().fromHeight(height);
- if (blockData == null)
- throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ if (blockData == null) {
+
+ // Not found, so try the archive
+ blockData = repository.getBlockArchiveRepository().fromHeight(height);
+ if (blockData == null) {
+
+ // Still not found
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ }
+ }
Block block = new Block(repository, blockData);
BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference());
+ if (parentBlockData == null) {
+ // Parent block not found - try the archive
+ parentBlockData = repository.getBlockArchiveRepository().fromSignature(blockData.getReference());
+ if (parentBlockData == null) {
+
+ // Still not found
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ }
+ }
+
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
if (minterLevel == 0)
// This may be unavailable when requesting a trimmed block
@@ -454,13 +536,26 @@ public class BlocksResource {
})
public BlockData getByTimestamp(@PathParam("timestamp") long timestamp) {
try (final Repository repository = RepositoryManager.getRepository()) {
- int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp);
- if (height == 0)
- throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ BlockData blockData = null;
- BlockData blockData = repository.getBlockRepository().fromHeight(height);
- if (blockData == null)
+ // Try the Blocks table
+ int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp);
+ if (height > 0) {
+ // Found match in Blocks table
+ return repository.getBlockRepository().fromHeight(height);
+ }
+
+ // Not found in Blocks table, so try the archive
+ height = repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp);
+ if (height > 0) {
+ // Found match in archive
+ blockData = repository.getBlockArchiveRepository().fromHeight(height);
+ }
+
+ // Ensure block exists
+ if (blockData == null) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+ }
return blockData;
} catch (DataException e) {
@@ -497,9 +592,14 @@ public class BlocksResource {
for (/* count already set */; count > 0; --count, ++height) {
BlockData blockData = repository.getBlockRepository().fromHeight(height);
- if (blockData == null)
- // Run out of blocks!
- break;
+ if (blockData == null) {
+ // Not found - try the archive
+ blockData = repository.getBlockArchiveRepository().fromHeight(height);
+ if (blockData == null) {
+ // Run out of blocks!
+ break;
+ }
+ }
blocks.add(blockData);
}
@@ -544,7 +644,29 @@ public class BlocksResource {
if (accountData == null || accountData.getPublicKey() == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.PUBLIC_KEY_NOT_FOUND);
- return repository.getBlockRepository().getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse);
+
+ List summaries = repository.getBlockRepository()
+ .getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse);
+
+ // Add any from the archive
+ List archivedSummaries = repository.getBlockArchiveRepository()
+ .getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse);
+ if (archivedSummaries != null && !archivedSummaries.isEmpty()) {
+ summaries.addAll(archivedSummaries);
+ }
+ else {
+ summaries = archivedSummaries;
+ }
+
+ // Sort the results (because they may have been obtained from two places)
+ if (reverse != null && reverse) {
+ summaries.sort((s1, s2) -> Integer.valueOf(s2.getHeight()).compareTo(Integer.valueOf(s1.getHeight())));
+ }
+ else {
+ summaries.sort(Comparator.comparing(s -> Integer.valueOf(s.getHeight())));
+ }
+
+ return summaries;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
@@ -580,7 +702,8 @@ public class BlocksResource {
if (!Crypto.isValidAddress(address))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
- return repository.getBlockRepository().getBlockSigners(addresses, limit, offset, reverse);
+ // This method pulls data from both Blocks and BlockArchive, so no need to query serparately
+ return repository.getBlockArchiveRepository().getBlockSigners(addresses, limit, offset, reverse);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
@@ -620,7 +743,76 @@ public class BlocksResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
try (final Repository repository = RepositoryManager.getRepository()) {
- return repository.getBlockRepository().getBlockSummaries(startHeight, endHeight, count);
+
+ /*
+ * start end count result
+ * 10 40 null blocks 10 to 39 (excludes end block, ignore count)
+ *
+ * null null null blocks 1 to 50 (assume count=50, maybe start=1)
+ * 30 null null blocks 30 to 79 (assume count=50)
+ * 30 null 10 blocks 30 to 39
+ *
+ * null null 50 last 50 blocks? so if max(blocks.height) is 200, then blocks 151 to 200
+ * null 200 null blocks 150 to 199 (excludes end block, assume count=50)
+ * null 200 10 blocks 190 to 199 (excludes end block)
+ */
+
+ List blockSummaries = new ArrayList<>();
+
+ // Use the latest X blocks if only a count is specified
+ if (startHeight == null && endHeight == null && count != null) {
+ BlockData chainTip = repository.getBlockRepository().getLastBlock();
+ startHeight = chainTip.getHeight() - count;
+ endHeight = chainTip.getHeight();
+ }
+
+ // ... otherwise default the start height to 1
+ if (startHeight == null && endHeight == null) {
+ startHeight = 1;
+ }
+
+ // Default the count to 50
+ if (count == null) {
+ count = 50;
+ }
+
+ // If both a start and end height exist, ignore the count
+ if (startHeight != null && endHeight != null) {
+ if (startHeight > 0 && endHeight > 0) {
+ count = Integer.MAX_VALUE;
+ }
+ }
+
+ // Derive start height from end height if missing
+ if (startHeight == null || startHeight == 0) {
+ if (endHeight != null && endHeight > 0) {
+ if (count != null) {
+ startHeight = endHeight - count;
+ }
+ }
+ }
+
+ for (/* count already set */; count > 0; --count, ++startHeight) {
+ if (endHeight != null && startHeight >= endHeight) {
+ break;
+ }
+ BlockData blockData = repository.getBlockRepository().fromHeight(startHeight);
+ if (blockData == null) {
+ // Not found - try the archive
+ blockData = repository.getBlockArchiveRepository().fromHeight(startHeight);
+ if (blockData == null) {
+ // Run out of blocks!
+ break;
+ }
+ }
+
+ if (blockData != null) {
+ BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
+ blockSummaries.add(blockSummaryData);
+ }
+ }
+
+ return blockSummaries;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
diff --git a/src/main/java/org/qortal/api/resource/BootstrapResource.java b/src/main/java/org/qortal/api/resource/BootstrapResource.java
new file mode 100644
index 00000000..9b9b7f2a
--- /dev/null
+++ b/src/main/java/org/qortal/api/resource/BootstrapResource.java
@@ -0,0 +1,92 @@
+package org.qortal.api.resource;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.api.ApiError;
+import org.qortal.api.ApiExceptionFactory;
+import org.qortal.api.Security;
+import org.qortal.repository.Bootstrap;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+
+
+@Path("/bootstrap")
+@Tag(name = "Bootstrap")
+public class BootstrapResource {
+
+ private static final Logger LOGGER = LogManager.getLogger(BootstrapResource.class);
+
+ @Context
+ HttpServletRequest request;
+
+ @POST
+ @Path("/create")
+ @Operation(
+ summary = "Create bootstrap",
+ description = "Builds a bootstrap file for distribution",
+ responses = {
+ @ApiResponse(
+ description = "path to file on success, an exception on failure",
+ content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string"))
+ )
+ }
+ )
+ public String createBootstrap() {
+ Security.checkApiCallAllowed(request);
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+
+ Bootstrap bootstrap = new Bootstrap(repository);
+ try {
+ bootstrap.checkRepositoryState();
+ } catch (DataException e) {
+ LOGGER.info("Not ready to create bootstrap: {}", e.getMessage());
+ throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
+ }
+ bootstrap.validateBlockchain();
+ return bootstrap.create();
+
+ } catch (DataException | InterruptedException | IOException e) {
+ LOGGER.info("Unable to create bootstrap", e);
+ throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
+ }
+ }
+
+ @GET
+ @Path("/validate")
+ @Operation(
+ summary = "Validate blockchain",
+ description = "Useful to check database integrity prior to creating or after installing a bootstrap. " +
+ "This process is intensive and can take over an hour to run.",
+ responses = {
+ @ApiResponse(
+ description = "true if valid, false if invalid",
+ content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
+ )
+ }
+ )
+ public boolean validateBootstrap() {
+ Security.checkApiCallAllowed(request);
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+
+ Bootstrap bootstrap = new Bootstrap(repository);
+ return bootstrap.validateCompleteBlockchain();
+
+ } catch (DataException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
+ }
+ }
+}
diff --git a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java
index 0076609a..46d7ebc6 100644
--- a/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java
+++ b/src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java
@@ -11,6 +11,7 @@ import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.GET;
+import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.core.Context;
@@ -173,7 +174,7 @@ public class CrossChainHtlcResource {
}
}
- @GET
+ @POST
@Path("/redeem/{ataddress}")
@Operation(
summary = "Redeems HTLC associated with supplied AT",
@@ -231,7 +232,7 @@ public class CrossChainHtlcResource {
}
}
- @GET
+ @POST
@Path("/redeemAll")
@Operation(
summary = "Redeems HTLC for all applicable ATs in tradebot data",
@@ -415,7 +416,7 @@ public class CrossChainHtlcResource {
return false;
}
- @GET
+ @POST
@Path("/refund/{ataddress}")
@Operation(
summary = "Refunds HTLC associated with supplied AT",
@@ -463,7 +464,7 @@ public class CrossChainHtlcResource {
}
- @GET
+ @POST
@Path("/refundAll")
@Operation(
summary = "Refunds HTLC for all applicable ATs in tradebot data",
@@ -478,8 +479,6 @@ public class CrossChainHtlcResource {
)
@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
public boolean refundAllHtlc() {
- Security.checkApiCallAllowed(request);
-
Security.checkApiCallAllowed(request);
boolean success = false;
@@ -568,6 +567,13 @@ public class CrossChainHtlcResource {
if (crossChainTradeData == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+ // If the AT is "finished" then it will have a zero balance
+ // In these cases we should avoid HTLC refunds if tbe QORT haven't been returned to the seller
+ if (atData.getIsFinished() && crossChainTradeData.mode != AcctMode.REFUNDED && crossChainTradeData.mode != AcctMode.CANCELLED) {
+ LOGGER.info(String.format("Skipping AT %s because the QORT has already been redemed", atAddress));
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+ }
+
List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null);
if (tradeBotData == null)
diff --git a/src/main/java/org/qortal/api/resource/ListsResource.java b/src/main/java/org/qortal/api/resource/ListsResource.java
new file mode 100644
index 00000000..dea6690c
--- /dev/null
+++ b/src/main/java/org/qortal/api/resource/ListsResource.java
@@ -0,0 +1,298 @@
+package org.qortal.api.resource;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.ArraySchema;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+import org.qortal.api.*;
+import org.qortal.api.model.AddressListRequest;
+import org.qortal.crypto.Crypto;
+import org.qortal.data.account.AccountData;
+import org.qortal.list.ResourceListManager;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+
+
+@Path("/lists")
+@Tag(name = "Lists")
+public class ListsResource {
+
+ @Context
+ HttpServletRequest request;
+
+ @POST
+ @Path("/blacklist/address/{address}")
+ @Operation(
+ summary = "Add a QORT address to the local blacklist",
+ responses = {
+ @ApiResponse(
+ description = "Returns true on success, or an exception on failure",
+ content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
+ )
+ }
+ )
+ @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, ApiError.REPOSITORY_ISSUE})
+ public String addAddressToBlacklist(@PathParam("address") String address) {
+ Security.checkApiCallAllowed(request);
+
+ if (!Crypto.isValidAddress(address))
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ AccountData accountData = repository.getAccountRepository().getAccount(address);
+ // Not found?
+ if (accountData == null)
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN);
+
+ // Valid address, so go ahead and blacklist it
+ boolean success = ResourceListManager.getInstance().addAddressToBlacklist(address, true);
+
+ return success ? "true" : "false";
+ } catch (DataException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+ }
+ }
+
+ @POST
+ @Path("/blacklist/addresses")
+ @Operation(
+ summary = "Add one or more QORT addresses to the local blacklist",
+ requestBody = @RequestBody(
+ required = true,
+ content = @Content(
+ mediaType = MediaType.APPLICATION_JSON,
+ schema = @Schema(
+ implementation = AddressListRequest.class
+ )
+ )
+ ),
+ responses = {
+ @ApiResponse(
+ description = "Returns true if all addresses were processed, false if any couldn't be " +
+ "processed, or an exception on failure. If false or an exception is returned, " +
+ "the list will not be updated, and the request will need to be re-issued.",
+ content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
+ )
+ }
+ )
+ @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, ApiError.REPOSITORY_ISSUE})
+ public String addAddressesToBlacklist(AddressListRequest addressListRequest) {
+ Security.checkApiCallAllowed(request);
+
+ if (addressListRequest == null || addressListRequest.addresses == null) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+ }
+
+ int successCount = 0;
+ int errorCount = 0;
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+
+ for (String address : addressListRequest.addresses) {
+
+ if (!Crypto.isValidAddress(address)) {
+ errorCount++;
+ continue;
+ }
+
+ AccountData accountData = repository.getAccountRepository().getAccount(address);
+ // Not found?
+ if (accountData == null) {
+ errorCount++;
+ continue;
+ }
+
+ // Valid address, so go ahead and blacklist it
+ boolean success = ResourceListManager.getInstance().addAddressToBlacklist(address, false);
+ if (success) {
+ successCount++;
+ }
+ else {
+ errorCount++;
+ }
+ }
+ } catch (DataException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+ }
+
+ if (successCount > 0 && errorCount == 0) {
+ // All were successful, so save the blacklist
+ ResourceListManager.getInstance().saveBlacklist();
+ return "true";
+ }
+ else {
+ // Something went wrong, so revert
+ ResourceListManager.getInstance().revertBlacklist();
+ return "false";
+ }
+ }
+
+
+ @DELETE
+ @Path("/blacklist/address/{address}")
+ @Operation(
+ summary = "Remove a QORT address from the local blacklist",
+ responses = {
+ @ApiResponse(
+ description = "Returns true on success, or an exception on failure",
+ content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
+ )
+ }
+ )
+ @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, ApiError.REPOSITORY_ISSUE})
+ public String removeAddressFromBlacklist(@PathParam("address") String address) {
+ Security.checkApiCallAllowed(request);
+
+ if (!Crypto.isValidAddress(address))
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ AccountData accountData = repository.getAccountRepository().getAccount(address);
+ // Not found?
+ if (accountData == null)
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN);
+
+ // Valid address, so go ahead and blacklist it
+ boolean success = ResourceListManager.getInstance().removeAddressFromBlacklist(address, true);
+
+ return success ? "true" : "false";
+ } catch (DataException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+ }
+ }
+
+ @DELETE
+ @Path("/blacklist/addresses")
+ @Operation(
+ summary = "Remove one or more QORT addresses from the local blacklist",
+ requestBody = @RequestBody(
+ required = true,
+ content = @Content(
+ mediaType = MediaType.APPLICATION_JSON,
+ schema = @Schema(
+ implementation = AddressListRequest.class
+ )
+ )
+ ),
+ responses = {
+ @ApiResponse(
+ description = "Returns true if all addresses were processed, false if any couldn't be " +
+ "processed, or an exception on failure. If false or an exception is returned, " +
+ "the list will not be updated, and the request will need to be re-issued.",
+ content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
+ )
+ }
+ )
+ @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, ApiError.REPOSITORY_ISSUE})
+ public String removeAddressesFromBlacklist(AddressListRequest addressListRequest) {
+ Security.checkApiCallAllowed(request);
+
+ if (addressListRequest == null || addressListRequest.addresses == null) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+ }
+
+ int successCount = 0;
+ int errorCount = 0;
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+
+ for (String address : addressListRequest.addresses) {
+
+ if (!Crypto.isValidAddress(address)) {
+ errorCount++;
+ continue;
+ }
+
+ AccountData accountData = repository.getAccountRepository().getAccount(address);
+ // Not found?
+ if (accountData == null) {
+ errorCount++;
+ continue;
+ }
+
+ // Valid address, so go ahead and blacklist it
+ // Don't save as we will do this at the end of the process
+ boolean success = ResourceListManager.getInstance().removeAddressFromBlacklist(address, false);
+ if (success) {
+ successCount++;
+ }
+ else {
+ errorCount++;
+ }
+ }
+ } catch (DataException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+ }
+
+ if (successCount > 0 && errorCount == 0) {
+ // All were successful, so save the blacklist
+ ResourceListManager.getInstance().saveBlacklist();
+ return "true";
+ }
+ else {
+ // Something went wrong, so revert
+ ResourceListManager.getInstance().revertBlacklist();
+ return "false";
+ }
+ }
+
+ @GET
+ @Path("/blacklist/addresses")
+ @Operation(
+ summary = "Fetch the list of blacklisted addresses",
+ responses = {
+ @ApiResponse(
+ description = "A JSON array of addresses",
+ content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = String.class)))
+ )
+ }
+ )
+ public String getAddressBlacklist() {
+ Security.checkApiCallAllowed(request);
+ return ResourceListManager.getInstance().getBlacklistJSONString();
+ }
+
+ @GET
+ @Path("/blacklist/address/{address}")
+ @Operation(
+ summary = "Check if an address is present in the local blacklist",
+ responses = {
+ @ApiResponse(
+ description = "Returns true or false if the list was queried, or an exception on failure",
+ content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
+ )
+ }
+ )
+ @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, ApiError.REPOSITORY_ISSUE})
+ public String checkAddressInBlacklist(@PathParam("address") String address) {
+ Security.checkApiCallAllowed(request);
+
+ if (!Crypto.isValidAddress(address))
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ AccountData accountData = repository.getAccountRepository().getAccount(address);
+ // Not found?
+ if (accountData == null)
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN);
+
+ // Valid address, so go ahead and blacklist it
+ boolean blacklisted = ResourceListManager.getInstance().isAddressInBlacklist(address);
+
+ return blacklisted ? "true" : "false";
+ } catch (DataException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+ }
+ }
+
+}
diff --git a/src/main/java/org/qortal/at/AT.java b/src/main/java/org/qortal/at/AT.java
index e82ab14e..005bb0cd 100644
--- a/src/main/java/org/qortal/at/AT.java
+++ b/src/main/java/org/qortal/at/AT.java
@@ -1,5 +1,7 @@
package org.qortal.at;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.ciyam.at.MachineState;
@@ -56,12 +58,12 @@ public class AT {
this.atData = new ATData(atAddress, creatorPublicKey, creation, machineState.version, assetId, codeBytes, codeHash,
machineState.isSleeping(), machineState.getSleepUntilHeight(), machineState.isFinished(), machineState.hadFatalError(),
- machineState.isFrozen(), machineState.getFrozenBalance());
+ machineState.isFrozen(), machineState.getFrozenBalance(), null);
byte[] stateData = machineState.toBytes();
byte[] stateHash = Crypto.digest(stateData);
- this.atStateData = new ATStateData(atAddress, height, stateData, stateHash, 0L, true);
+ this.atStateData = new ATStateData(atAddress, height, stateData, stateHash, 0L, true, null);
}
// Getters / setters
@@ -84,13 +86,28 @@ public class AT {
this.repository.getATRepository().delete(this.atData.getATAddress());
}
+ /**
+ * Potentially execute AT.
+ *
+ * Note that sleep-until-message support might set/reset
+ * sleep-related flags/values.
+ *
+ * {@link #getATStateData()} will return null if nothing happened.
+ *
+ * @param blockHeight
+ * @param blockTimestamp
+ * @return AT-generated transactions, possibly empty
+ * @throws DataException
+ */
public List run(int blockHeight, long blockTimestamp) throws DataException {
String atAddress = this.atData.getATAddress();
QortalATAPI api = new QortalATAPI(repository, this.atData, blockTimestamp);
QortalAtLoggerFactory loggerFactory = QortalAtLoggerFactory.getInstance();
- byte[] codeBytes = this.atData.getCodeBytes();
+ if (!api.willExecute(blockHeight))
+ // this.atStateData will be null
+ return Collections.emptyList();
// Fetch latest ATStateData for this AT
ATStateData latestAtStateData = this.repository.getATRepository().getLatestATState(atAddress);
@@ -100,8 +117,10 @@ public class AT {
throw new IllegalStateException("No previous AT state data found");
// [Re]create AT machine state using AT state data or from scratch as applicable
+ byte[] codeBytes = this.atData.getCodeBytes();
MachineState state = MachineState.fromBytes(api, loggerFactory, latestAtStateData.getStateData(), codeBytes);
try {
+ api.preExecute(state);
state.execute();
} catch (Exception e) {
throw new DataException(String.format("Uncaught exception while running AT '%s'", atAddress), e);
@@ -109,9 +128,18 @@ public class AT {
byte[] stateData = state.toBytes();
byte[] stateHash = Crypto.digest(stateData);
- long atFees = api.calcFinalFees(state);
- this.atStateData = new ATStateData(atAddress, blockHeight, stateData, stateHash, atFees, false);
+ // Nothing happened?
+ if (state.getSteps() == 0 && Arrays.equals(stateHash, latestAtStateData.getStateHash()))
+ // We currently want to execute frozen ATs, to maintain backwards support.
+ if (state.isFrozen() == false)
+ // this.atStateData will be null
+ return Collections.emptyList();
+
+ long atFees = api.calcFinalFees(state);
+ Long sleepUntilMessageTimestamp = this.atData.getSleepUntilMessageTimestamp();
+
+ this.atStateData = new ATStateData(atAddress, blockHeight, stateData, stateHash, atFees, false, sleepUntilMessageTimestamp);
return api.getTransactions();
}
@@ -130,6 +158,10 @@ public class AT {
this.atData.setHadFatalError(state.hadFatalError());
this.atData.setIsFrozen(state.isFrozen());
this.atData.setFrozenBalance(state.getFrozenBalance());
+
+ // Special sleep-until-message support
+ this.atData.setSleepUntilMessageTimestamp(this.atStateData.getSleepUntilMessageTimestamp());
+
this.repository.getATRepository().save(this.atData);
}
@@ -157,6 +189,10 @@ public class AT {
this.atData.setHadFatalError(state.hadFatalError());
this.atData.setIsFrozen(state.isFrozen());
this.atData.setFrozenBalance(state.getFrozenBalance());
+
+ // Special sleep-until-message support
+ this.atData.setSleepUntilMessageTimestamp(previousStateData.getSleepUntilMessageTimestamp());
+
this.repository.getATRepository().save(this.atData);
}
diff --git a/src/main/java/org/qortal/at/QortalATAPI.java b/src/main/java/org/qortal/at/QortalATAPI.java
index 6a379d59..c393a684 100644
--- a/src/main/java/org/qortal/at/QortalATAPI.java
+++ b/src/main/java/org/qortal/at/QortalATAPI.java
@@ -32,6 +32,7 @@ import org.qortal.group.Group;
import org.qortal.repository.ATRepository;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
+import org.qortal.repository.ATRepository.NextTransactionInfo;
import org.qortal.transaction.AtTransaction;
import org.qortal.transaction.Transaction.TransactionType;
import org.qortal.utils.Base58;
@@ -74,8 +75,45 @@ public class QortalATAPI extends API {
return this.transactions;
}
- public long calcFinalFees(MachineState state) {
- return state.getSteps() * this.ciyamAtSettings.feePerStep;
+ public boolean willExecute(int blockHeight) throws DataException {
+ // Sleep-until-message/height checking
+ Long sleepUntilMessageTimestamp = this.atData.getSleepUntilMessageTimestamp();
+
+ if (sleepUntilMessageTimestamp != null) {
+ // Quicker to check height, if sleep-until-height also active
+ Integer sleepUntilHeight = this.atData.getSleepUntilHeight();
+
+ boolean wakeDueToHeight = sleepUntilHeight != null && sleepUntilHeight != 0 && blockHeight >= sleepUntilHeight;
+
+ boolean wakeDueToMessage = false;
+ if (!wakeDueToHeight) {
+ // No avoiding asking repository
+ Timestamp previousTxTimestamp = new Timestamp(sleepUntilMessageTimestamp);
+ NextTransactionInfo nextTransactionInfo = this.repository.getATRepository().findNextTransaction(this.atData.getATAddress(),
+ previousTxTimestamp.blockHeight,
+ previousTxTimestamp.transactionSequence);
+
+ wakeDueToMessage = nextTransactionInfo != null;
+ }
+
+ // Can we skip?
+ if (!wakeDueToHeight && !wakeDueToMessage)
+ return false;
+ }
+
+ return true;
+ }
+
+ public void preExecute(MachineState state) {
+ // Sleep-until-message/height checking
+ Long sleepUntilMessageTimestamp = this.atData.getSleepUntilMessageTimestamp();
+
+ if (sleepUntilMessageTimestamp != null) {
+ // We've passed checks, so clear sleep-related flags/values
+ this.setIsSleeping(state, false);
+ this.setSleepUntilHeight(state, 0);
+ this.atData.setSleepUntilMessageTimestamp(null);
+ }
}
// Inherited methods from CIYAM AT API
@@ -412,6 +450,10 @@ public class QortalATAPI extends API {
// Utility methods
+ public long calcFinalFees(MachineState state) {
+ return state.getSteps() * this.ciyamAtSettings.feePerStep;
+ }
+
/** Returns partial transaction signature, used to verify we're operating on the same transaction and not naively using block height & sequence. */
public static byte[] partialSignature(byte[] fullSignature) {
return Arrays.copyOfRange(fullSignature, 8, 32);
@@ -460,6 +502,15 @@ public class QortalATAPI extends API {
}
}
+ /*package*/ void sleepUntilMessageOrHeight(MachineState state, long txTimestamp, Long sleepUntilHeight) {
+ this.setIsSleeping(state, true);
+
+ this.atData.setSleepUntilMessageTimestamp(txTimestamp);
+
+ if (sleepUntilHeight != null)
+ this.setSleepUntilHeight(state, sleepUntilHeight.intValue());
+ }
+
/** Returns AT's account */
/* package */ Account getATAccount() {
return new Account(this.repository, this.atData.getATAddress());
diff --git a/src/main/java/org/qortal/at/QortalFunctionCode.java b/src/main/java/org/qortal/at/QortalFunctionCode.java
index 0d11e488..7069290a 100644
--- a/src/main/java/org/qortal/at/QortalFunctionCode.java
+++ b/src/main/java/org/qortal/at/QortalFunctionCode.java
@@ -84,6 +84,43 @@ public enum QortalFunctionCode {
api.setB(state, bBytes);
}
},
+ /**
+ * Sleep AT until a new message arrives after 'tx-timestamp'.
+ * 0x0503 tx-timestamp
+ */
+ SLEEP_UNTIL_MESSAGE(0x0503, 1, false) {
+ @Override
+ protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException {
+ if (functionData.value1 <= 0)
+ return;
+
+ long txTimestamp = functionData.value1;
+
+ QortalATAPI api = (QortalATAPI) state.getAPI();
+ api.sleepUntilMessageOrHeight(state, txTimestamp, null);
+ }
+ },
+ /**
+ * Sleep AT until a new message arrives, after 'tx-timestamp', or height reached.
+ * 0x0504 tx-timestamp height
+ */
+ SLEEP_UNTIL_MESSAGE_OR_HEIGHT(0x0504, 2, false) {
+ @Override
+ protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException {
+ if (functionData.value1 <= 0)
+ return;
+
+ long txTimestamp = functionData.value1;
+
+ if (functionData.value2 <= 0)
+ return;
+
+ long sleepUntilHeight = functionData.value2;
+
+ QortalATAPI api = (QortalATAPI) state.getAPI();
+ api.sleepUntilMessageOrHeight(state, txTimestamp, sleepUntilHeight);
+ }
+ },
/**
* Convert address in B to 20-byte value in LSB of B1, and all of B2 & B3.
* 0x0510
diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java
index 798a4f91..d6bb11f3 100644
--- a/src/main/java/org/qortal/block/Block.java
+++ b/src/main/java/org/qortal/block/Block.java
@@ -1104,9 +1104,14 @@ public class Block {
// Create repository savepoint here so we can rollback to it after testing transactions
repository.setSavepoint();
- if (this.blockData.getHeight() == 212937)
+ if (this.blockData.getHeight() == 212937) {
// Apply fix for block 212937 but fix will be rolled back before we exit method
Block212937.processFix(this);
+ }
+ else if (InvalidNameRegistrationBlocks.isAffectedBlock(this.blockData.getHeight())) {
+ // Apply fix for affected name registration blocks, but fix will be rolled back before we exit method
+ InvalidNameRegistrationBlocks.processFix(this);
+ }
for (Transaction transaction : this.getTransactions()) {
TransactionData transactionData = transaction.getTransactionData();
@@ -1145,7 +1150,7 @@ public class Block {
// Check transaction can even be processed
validationResult = transaction.isProcessable();
if (validationResult != Transaction.ValidationResult.OK) {
- LOGGER.debug(String.format("Error during transaction validation, tx %s: %s", Base58.encode(transactionData.getSignature()), validationResult.name()));
+ LOGGER.info(String.format("Error during transaction validation, tx %s: %s", Base58.encode(transactionData.getSignature()), validationResult.name()));
return ValidationResult.TRANSACTION_INVALID;
}
@@ -1259,12 +1264,13 @@ public class Block {
for (ATData atData : executableATs) {
AT at = new AT(this.repository, atData);
List atTransactions = at.run(this.blockData.getHeight(), this.blockData.getTimestamp());
+ ATStateData atStateData = at.getATStateData();
+ // Didn't execute? (e.g. sleeping)
+ if (atStateData == null)
+ continue;
allAtTransactions.addAll(atTransactions);
-
- ATStateData atStateData = at.getATStateData();
this.ourAtStates.add(atStateData);
-
this.ourAtFees += atStateData.getFees();
}
@@ -1293,6 +1299,21 @@ public class Block {
return mintingAccount.canMint();
}
+ /**
+ * Pre-process block, and its transactions.
+ * This allows for any database integrity checks prior to validation.
+ * This is called before isValid() and process()
+ *
+ * @throws DataException
+ */
+ public void preProcess() throws DataException {
+ List blocksTransactions = this.getTransactions();
+
+ for (Transaction transaction : blocksTransactions) {
+ transaction.preProcess();
+ }
+ }
+
/**
* Process block, and its transactions, adding them to the blockchain.
*
diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java
index e6b8db4e..7a6d6605 100644
--- a/src/main/java/org/qortal/block/BlockChain.java
+++ b/src/main/java/org/qortal/block/BlockChain.java
@@ -4,10 +4,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.concurrent.locks.ReentrantLock;
import javax.xml.bind.JAXBContext;
@@ -27,11 +24,9 @@ import org.eclipse.persistence.jaxb.UnmarshallerProperties;
import org.qortal.controller.Controller;
import org.qortal.data.block.BlockData;
import org.qortal.network.Network;
-import org.qortal.repository.BlockRepository;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
+import org.qortal.repository.*;
import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
import org.qortal.utils.StringLongMapXmlAdapter;
/**
@@ -506,29 +501,105 @@ public class BlockChain {
* @throws SQLException
*/
public static void validate() throws DataException {
- // Check first block is Genesis Block
- if (!isGenesisBlockValid())
- rebuildBlockchain();
+ boolean isTopOnly = Settings.getInstance().isTopOnly();
+ boolean archiveEnabled = Settings.getInstance().isArchiveEnabled();
+ boolean canBootstrap = Settings.getInstance().getBootstrap();
+ boolean needsArchiveRebuild = false;
+ BlockData chainTip;
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ chainTip = repository.getBlockRepository().getLastBlock();
+
+ // Ensure archive is (at least partially) intact, and force a bootstrap if it isn't
+ if (!isTopOnly && archiveEnabled && canBootstrap) {
+ needsArchiveRebuild = (repository.getBlockArchiveRepository().fromHeight(2) == null);
+ if (needsArchiveRebuild) {
+ LOGGER.info("Couldn't retrieve block 2 from archive. Bootstrapping...");
+
+ // If there are minting accounts, make sure to back them up
+ // Don't backup if there are no minting accounts, as this can cause problems
+ if (!repository.getAccountRepository().getMintingAccounts().isEmpty()) {
+ Controller.getInstance().exportRepositoryData();
+ }
+ }
+ }
+ }
+
+ boolean hasBlocks = (chainTip != null && chainTip.getHeight() > 1);
+
+ if (isTopOnly && hasBlocks) {
+ // Top-only mode is enabled and we have blocks, so it's possible that the genesis block has been pruned
+ // It's best not to validate it, and there's no real need to
+ } else {
+ // Check first block is Genesis Block
+ if (!isGenesisBlockValid() || needsArchiveRebuild) {
+ try {
+ rebuildBlockchain();
+
+ } catch (InterruptedException e) {
+ throw new DataException(String.format("Interrupted when trying to rebuild blockchain: %s", e.getMessage()));
+ }
+ }
+ }
+
+ // We need to create a new connection, as the previous repository and its connections may be been
+ // closed by rebuildBlockchain() if a bootstrap was applied
try (final Repository repository = RepositoryManager.getRepository()) {
repository.checkConsistency();
- int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - 1440, 1);
+ // Set the number of blocks to validate based on the pruned state of the chain
+ // If pruned, subtract an extra 10 to allow room for error
+ int blocksToValidate = (isTopOnly || archiveEnabled) ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440;
+ int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - blocksToValidate, 1);
BlockData detachedBlockData = repository.getBlockRepository().getDetachedBlockSignature(startHeight);
if (detachedBlockData != null) {
- LOGGER.error(String.format("Block %d's reference does not match any block's signature", detachedBlockData.getHeight()));
+ LOGGER.error(String.format("Block %d's reference does not match any block's signature",
+ detachedBlockData.getHeight()));
+ LOGGER.error(String.format("Your chain may be invalid and you should consider bootstrapping" +
+ " or re-syncing from genesis."));
+ }
+ }
+ }
- // Wait for blockchain lock (whereas orphan() only tries to get lock)
- ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
- blockchainLock.lock();
- try {
- LOGGER.info(String.format("Orphaning back to block %d", detachedBlockData.getHeight() - 1));
- orphan(detachedBlockData.getHeight() - 1);
- } finally {
- blockchainLock.unlock();
+ /**
+ * More thorough blockchain validation method. Useful for validating bootstraps.
+ * A DataException is thrown if anything is invalid.
+ *
+ * @throws DataException
+ */
+ public static void validateAllBlocks() throws DataException {
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ BlockData chainTip = repository.getBlockRepository().getLastBlock();
+ final int chainTipHeight = chainTip.getHeight();
+ final int oldestBlock = 1; // TODO: increase if in pruning mode
+ byte[] lastReference = null;
+
+ for (int height = chainTipHeight; height > oldestBlock; height--) {
+ BlockData blockData = repository.getBlockRepository().fromHeight(height);
+ if (blockData == null) {
+ blockData = repository.getBlockArchiveRepository().fromHeight(height);
}
+
+ if (blockData == null) {
+ String error = String.format("Missing block at height %d", height);
+ LOGGER.error(error);
+ throw new DataException(error);
+ }
+
+ if (height != chainTipHeight) {
+ // Check reference
+ if (!Arrays.equals(blockData.getSignature(), lastReference)) {
+ String error = String.format("Invalid reference for block at height %d: %s (should be %s)",
+ height, Base58.encode(blockData.getReference()), Base58.encode(lastReference));
+ LOGGER.error(error);
+ throw new DataException(error);
+ }
+ }
+
+ lastReference = blockData.getReference();
}
}
}
@@ -551,7 +622,15 @@ public class BlockChain {
}
}
- private static void rebuildBlockchain() throws DataException {
+ private static void rebuildBlockchain() throws DataException, InterruptedException {
+ boolean shouldBootstrap = Settings.getInstance().getBootstrap();
+ if (shouldBootstrap) {
+ // Settings indicate that we should apply a bootstrap rather than rebuilding and syncing from genesis
+ Bootstrap bootstrap = new Bootstrap();
+ bootstrap.startImport();
+ return;
+ }
+
// (Re)build repository
if (!RepositoryManager.wasPristineAtOpen())
RepositoryManager.rebuild();
diff --git a/src/main/java/org/qortal/block/InvalidNameRegistrationBlocks.java b/src/main/java/org/qortal/block/InvalidNameRegistrationBlocks.java
new file mode 100644
index 00000000..ebef366f
--- /dev/null
+++ b/src/main/java/org/qortal/block/InvalidNameRegistrationBlocks.java
@@ -0,0 +1,114 @@
+package org.qortal.block;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.naming.Name;
+import org.qortal.repository.DataException;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Invalid Name Registration Blocks
+ *
+ * A node minted a version of block 535658 that contained one transaction:
+ * a REGISTER_NAME transaction that attempted to register a name that was already registered.
+ *
+ * This invalid transaction made block 535658 (rightly) invalid to several nodes,
+ * which refused to use that block.
+ * However, it seems there were no other nodes minting an alternative, valid block at that time
+ * and so the chain stalled for several nodes in the network.
+ *
+ * Additionally, the invalid block 535658 affected all new installations, regardless of whether
+ * they synchronized from scratch (block 1) or used an 'official release' bootstrap.
+ *
+ * The diagnosis found the following:
+ * - The original problem occurred in block 535205 where for some unknown reason many nodes didn't
+ * add the name from a REGISTER_NAME transaction to their Names table.
+ * - As a result, those nodes had a corrupt db, because they weren't holding a record of the name.
+ * - This invalid db then caused them to treat a candidate for block 535658 as valid when it
+ * should have been invalid.
+ * - As such, the chain continued on with a technically invalid block in it, for a subset of the network
+ *
+ * As with block 212937, there were three options, but the only feasible one was to apply edits to block
+ * 535658 to make it valid. There were several cross-chain trades completed after this block, so doing
+ * any kind of rollback was out of the question.
+ *
+ * To complicate things further, a custom data field was used for the first REGISTER_NAME transaction,
+ * and the default data field was used for the second. So it was important that all nodes ended up with
+ * the exact same data regardless of how they arrived there.
+ *
+ * The invalid block 535658 signature is: 3oiuDhok...NdXvCLEV.
+ *
+ * The invalid transaction in block 212937 is:
+ *
+ * Account Qbx9ojxv7XNi1xDMWzzw7xDvd1zYW6SKFB attempted to register the name Qplay
+ * when they had already registered it 12 hours before in block 535205.
+ *
+ * However, on the broken DB nodes, their Names table was missing a record for the `Qplay` name
+ * which was sufficient to make the transaction valid.
+ *
+ * This problem then occurred two more times, in blocks 536140 and 541334
+ * To reduce duplication, I have combined all three block fixes into a single class
+ *
+ */
+public final class InvalidNameRegistrationBlocks {
+
+ private static final Logger LOGGER = LogManager.getLogger(InvalidNameRegistrationBlocks.class);
+
+ public static Map invalidBlocksNamesMap = new HashMap()
+ {
+ {
+ put(535658, "Qplay");
+ put(536140, "Qweb");
+ put(541334, "Qithub");
+ }
+ };
+
+ private InvalidNameRegistrationBlocks() {
+ /* Do not instantiate */
+ }
+
+ public static boolean isAffectedBlock(int height) {
+ return (invalidBlocksNamesMap.containsKey(height));
+ }
+
+ public static void processFix(Block block) throws DataException {
+ Integer blockHeight = block.getBlockData().getHeight();
+ String invalidName = invalidBlocksNamesMap.get(blockHeight);
+ if (invalidName == null) {
+ throw new DataException(String.format("Unable to lookup invalid name for block height %d", blockHeight));
+ }
+
+ // Unregister the existing name record if it exists
+ // This ensures that the duplicate name is considered valid, and therefore
+ // the second (i.e. duplicate) REGISTER_NAME transaction data is applied.
+ // Both were issued by the same user account, so there is no conflict.
+ Name name = new Name(block.repository, invalidName);
+ name.unregister();
+
+ LOGGER.debug("Applied name registration patch for block {}", blockHeight);
+ }
+
+ // Note:
+ // There is no need to write an orphanFix() method, as we do not have
+ // the necessary ATStatesData to orphan back this far anyway
+
+}
diff --git a/src/main/java/org/qortal/controller/AutoUpdate.java b/src/main/java/org/qortal/controller/AutoUpdate.java
index 6c1dd928..f07e82d1 100644
--- a/src/main/java/org/qortal/controller/AutoUpdate.java
+++ b/src/main/java/org/qortal/controller/AutoUpdate.java
@@ -14,6 +14,7 @@ import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.concurrent.TimeoutException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -215,8 +216,17 @@ public class AutoUpdate extends Thread {
}
// Give repository a chance to backup in case things go badly wrong (if enabled)
- if (Settings.getInstance().getRepositoryBackupInterval() > 0)
- RepositoryManager.backup(true);
+ if (Settings.getInstance().getRepositoryBackupInterval() > 0) {
+ try {
+ // Timeout if the database isn't ready for backing up after 60 seconds
+ long timeout = 60 * 1000L;
+ RepositoryManager.backup(true, "backup", timeout);
+
+ } catch (TimeoutException e) {
+ LOGGER.info("Attempt to backup repository failed due to timeout: {}", e.getMessage());
+ // Continue with the auto update anyway...
+ }
+ }
// Call ApplyUpdate to end this process (unlocking current JAR so it can be replaced)
String javaHome = System.getProperty("java.home");
diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java
index 8b6563f2..33431258 100644
--- a/src/main/java/org/qortal/controller/BlockMinter.java
+++ b/src/main/java/org/qortal/controller/BlockMinter.java
@@ -44,6 +44,9 @@ public class BlockMinter extends Thread {
private static Long lastLogTimestamp;
private static Long logTimeout;
+ // Recovery
+ public static final long INVALID_BLOCK_RECOVERY_TIMEOUT = 10 * 60 * 1000L; // ms
+
// Constructors
public BlockMinter() {
@@ -144,9 +147,25 @@ public class BlockMinter extends Thread {
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
continue;
+ // If we are stuck on an invalid block, we should allow an alternative to be minted
+ boolean recoverInvalidBlock = false;
+ if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
+ // We've had at least one invalid block
+ long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
+ long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
+ if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
+ if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
+ // Last valid block was more than 10 mins ago, but we've had an invalid block since then
+ // Assume that the chain has stalled because there is no alternative valid candidate
+ // Enter recovery mode to allow alternative, valid candidates to be minted
+ recoverInvalidBlock = true;
+ }
+ }
+ }
+
// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
- if (Controller.getInstance().getRecoveryMode() == false)
+ if (Controller.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
continue;
// There are enough peers with a recent block and our latest block is recent
@@ -230,6 +249,8 @@ public class BlockMinter extends Thread {
if (testBlock.isTimestampValid() != ValidationResult.OK)
continue;
+ testBlock.preProcess();
+
// Is new block valid yet? (Before adding unconfirmed transactions)
ValidationResult result = testBlock.isValid();
if (result != ValidationResult.OK) {
@@ -421,7 +442,8 @@ public class BlockMinter extends Thread {
// Add to blockchain
newBlock.process();
- LOGGER.info(String.format("Minted new test block: %d", newBlock.getBlockData().getHeight()));
+ LOGGER.info(String.format("Minted new test block: %d sig: %.8s",
+ newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature())));
repository.saveChanges();
diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java
index 6bab9d42..5b04aae5 100644
--- a/src/main/java/org/qortal/controller/Controller.java
+++ b/src/main/java/org/qortal/controller/Controller.java
@@ -1,10 +1,47 @@
package org.qortal.controller;
-import com.google.common.primitives.Longs;
+import java.awt.TrayIcon.MessageType;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.SecureRandom;
+import java.security.Security;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
+import com.google.common.primitives.Longs;
import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.account.PublicKeyAccount;
@@ -13,9 +50,11 @@ import org.qortal.api.DomainMapService;
import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.block.BlockChain.BlockTimingByHeight;
-import org.qortal.controller.Synchronizer.SynchronizationResult;
import org.qortal.controller.arbitrary.ArbitraryDataCleanupManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
+import org.qortal.controller.Synchronizer.SynchronizationResult;
+import org.qortal.controller.repository.PruneManager;
+import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
import org.qortal.controller.tradebot.TradeBot;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.RewardShareData;
@@ -34,10 +73,7 @@ import org.qortal.gui.SysTray;
import org.qortal.network.Network;
import org.qortal.network.Peer;
import org.qortal.network.message.*;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryFactory;
-import org.qortal.repository.RepositoryManager;
+import org.qortal.repository.*;
import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
import org.qortal.settings.Settings;
import org.qortal.transaction.Transaction;
@@ -123,6 +159,7 @@ public class Controller extends Thread {
};
private long repositoryBackupTimestamp = startTime; // ms
+ private long repositoryMaintenanceTimestamp = startTime; // ms
private long repositoryCheckpointTimestamp = startTime; // ms
private long ntpCheckTimestamp = startTime; // ms
private long deleteExpiredTimestamp = startTime + DELETE_EXPIRED_INTERVAL; // ms
@@ -291,6 +328,10 @@ public class Controller extends Thread {
return this.buildVersion;
}
+ public String getVersionStringWithoutPrefix() {
+ return this.buildVersion.replaceFirst(VERSION_PREFIX, "");
+ }
+
/** Returns current blockchain height, or 0 if it's not available. */
public int getChainHeight() {
synchronized (this.latestBlocks) {
@@ -334,7 +375,7 @@ public class Controller extends Thread {
return this.savedArgs;
}
- /* package */ public static boolean isStopping() {
+ public static boolean isStopping() {
return isStopping;
}
@@ -392,6 +433,12 @@ public class Controller extends Thread {
try {
RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl());
RepositoryManager.setRepositoryFactory(repositoryFactory);
+ RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ RepositoryManager.archive(repository);
+ RepositoryManager.prune(repository);
+ }
} catch (DataException e) {
// If exception has no cause then repository is in use by some other process.
if (e.getCause() == null) {
@@ -405,6 +452,11 @@ public class Controller extends Thread {
return; // Not System.exit() so that GUI can display error
}
+ // Rebuild Names table and check database integrity
+ NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck();
+ namesDatabaseIntegrityCheck.rebuildAllNames();
+ namesDatabaseIntegrityCheck.runIntegrityCheck();
+
LOGGER.info("Validating blockchain");
try {
BlockChain.validate();
@@ -417,6 +469,12 @@ public class Controller extends Thread {
return; // Not System.exit() so that GUI can display error
}
+ // Import current trade bot states and minting accounts if they exist
+ Controller.importRepositoryData();
+
+ // Add the initial peers to the repository if we don't have any
+ Controller.installInitialPeers();
+
LOGGER.info("Starting controller");
Controller.getInstance().start();
@@ -500,10 +558,10 @@ public class Controller extends Thread {
final long repositoryBackupInterval = Settings.getInstance().getRepositoryBackupInterval();
final long repositoryCheckpointInterval = Settings.getInstance().getRepositoryCheckpointInterval();
+ long repositoryMaintenanceInterval = getRandomRepositoryMaintenanceInterval();
- ExecutorService trimExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory());
- trimExecutor.execute(new AtStatesTrimmer());
- trimExecutor.execute(new OnlineAccountsSignaturesTrimmer());
+ // Start executor service for trimming or pruning
+ PruneManager.getInstance().start();
try {
while (!isStopping) {
@@ -562,7 +620,39 @@ public class Controller extends Thread {
Translator.INSTANCE.translate("SysTray", "CREATING_BACKUP_OF_DB_FILES"),
MessageType.INFO);
- RepositoryManager.backup(true);
+ try {
+ // Timeout if the database isn't ready for backing up after 60 seconds
+ long timeout = 60 * 1000L;
+ RepositoryManager.backup(true, "backup", timeout);
+
+ } catch (TimeoutException e) {
+ LOGGER.info("Attempt to backup repository failed due to timeout: {}", e.getMessage());
+ }
+ }
+
+ // Give repository a chance to perform maintenance (if enabled)
+ if (repositoryMaintenanceInterval > 0 && now >= repositoryMaintenanceTimestamp + repositoryMaintenanceInterval) {
+ repositoryMaintenanceTimestamp = now + repositoryMaintenanceInterval;
+
+ if (Settings.getInstance().getShowMaintenanceNotification())
+ SysTray.getInstance().showMessage(Translator.INSTANCE.translate("SysTray", "DB_MAINTENANCE"),
+ Translator.INSTANCE.translate("SysTray", "PERFORMING_DB_MAINTENANCE"),
+ MessageType.INFO);
+
+ LOGGER.info("Starting scheduled repository maintenance. This can take a while...");
+ try (final Repository repository = RepositoryManager.getRepository()) {
+
+ // Timeout if the database isn't ready for maintenance after 60 seconds
+ long timeout = 60 * 1000L;
+ repository.performPeriodicMaintenance(timeout);
+
+ LOGGER.info("Scheduled repository maintenance completed");
+ } catch (DataException | TimeoutException e) {
+ LOGGER.error("Scheduled repository maintenance failed", e);
+ }
+
+ // Get a new random interval
+ repositoryMaintenanceInterval = getRandomRepositoryMaintenanceInterval();
}
// Prune stuck/slow/old peers
@@ -589,13 +679,68 @@ public class Controller extends Thread {
Thread.interrupted();
// Fall-through to exit
} finally {
- trimExecutor.shutdownNow();
+ PruneManager.getInstance().stop();
+ }
+ }
+
+ /**
+ * Import current trade bot states and minting accounts.
+ * This is needed because the user may have bootstrapped, or there could be a database inconsistency
+ * if the core crashed when computing the nonce during the start of the trade process.
+ */
+ private static void importRepositoryData() {
+ try (final Repository repository = RepositoryManager.getRepository()) {
+
+ String exportPath = Settings.getInstance().getExportPath();
+ try {
+ Path importPath = Paths.get(exportPath, "TradeBotStates.json");
+ repository.importDataFromFile(importPath.toString());
+ } catch (FileNotFoundException e) {
+ // Do nothing, as the files will only exist in certain cases
+ }
try {
- trimExecutor.awaitTermination(2L, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- // We tried...
+ Path importPath = Paths.get(exportPath, "MintingAccounts.json");
+ repository.importDataFromFile(importPath.toString());
+ } catch (FileNotFoundException e) {
+ // Do nothing, as the files will only exist in certain cases
}
+ repository.saveChanges();
+ }
+ catch (DataException | IOException e) {
+ LOGGER.info("Unable to import data into repository: {}", e.getMessage());
+ }
+ }
+
+ private static void installInitialPeers() {
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ if (repository.getNetworkRepository().getAllPeers().isEmpty()) {
+ Network.installInitialPeers(repository);
+ }
+
+ } catch (DataException e) {
+ // Fail silently as this is an optional step
+ }
+ }
+
+ private long getRandomRepositoryMaintenanceInterval() {
+ final long minInterval = Settings.getInstance().getRepositoryMaintenanceMinInterval();
+ final long maxInterval = Settings.getInstance().getRepositoryMaintenanceMaxInterval();
+ if (maxInterval == 0) {
+ return 0;
+ }
+ return (new Random().nextLong() % (maxInterval - minInterval)) + minInterval;
+ }
+
+ /**
+ * Export current trade bot states and minting accounts.
+ */
+ public void exportRepositoryData() {
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ repository.exportNodeLocalData();
+
+ } catch (DataException e) {
+ // Fail silently as this is an optional step
}
}
@@ -878,7 +1023,7 @@ public class Controller extends Thread {
}
}
- String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height) + "\n" + String.format("Build version: %s", this.buildVersion);
+ String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height) + "\n" + String.format("%s: %s", Translator.INSTANCE.translate("SysTray", "BUILD_VERSION"), this.buildVersion);
SysTray.getInstance().setToolTipText(tooltip);
this.callbackExecutor.execute(() -> {
@@ -951,6 +1096,10 @@ public class Controller extends Thread {
}
}
+ // Export local data
+ LOGGER.info("Backing up local data");
+ this.exportRepositoryData();
+
LOGGER.info("Shutting down networking");
Network.getInstance().shutdown();
@@ -1291,6 +1440,34 @@ public class Controller extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) {
BlockData blockData = repository.getBlockRepository().fromSignature(signature);
+ if (blockData != null) {
+ if (PruneManager.getInstance().isBlockPruned(blockData.getHeight())) {
+ // If this is a pruned block, we likely only have partial data, so best not to sent it
+ blockData = null;
+ }
+ }
+
+ // If we have no block data, we should check the archive in case it's there
+ if (blockData == null) {
+ if (Settings.getInstance().isArchiveEnabled()) {
+ byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, true, repository);
+ if (bytes != null) {
+ CachedBlockMessage blockMessage = new CachedBlockMessage(bytes);
+ blockMessage.setId(message.getId());
+
+ // This call also causes the other needed data to be pulled in from repository
+ if (!peer.sendMessage(blockMessage)) {
+ peer.disconnect("failed to send block");
+ // Don't fall-through to caching because failure to send might be from failure to build message
+ return;
+ }
+
+ // Sent successfully from archive, so nothing more to do
+ return;
+ }
+ }
+ }
+
if (blockData == null) {
// We don't have this block
this.stats.getBlockMessageStats.unknownBlocks.getAndIncrement();
@@ -1459,12 +1636,29 @@ public class Controller extends Thread {
int numberRequested = Math.min(Network.MAX_BLOCK_SUMMARIES_PER_REPLY, getBlockSummariesMessage.getNumberRequested());
BlockData blockData = repository.getBlockRepository().fromReference(parentSignature);
+ if (blockData == null) {
+ // Try the archive
+ blockData = repository.getBlockArchiveRepository().fromReference(parentSignature);
+ }
+
+ if (blockData != null) {
+ if (PruneManager.getInstance().isBlockPruned(blockData.getHeight())) {
+ // If this request contains a pruned block, we likely only have partial data, so best not to sent anything
+ // We always prune from the oldest first, so it's fine to just check the first block requested
+ blockData = null;
+ }
+ }
while (blockData != null && blockSummaries.size() < numberRequested) {
BlockSummaryData blockSummary = new BlockSummaryData(blockData);
blockSummaries.add(blockSummary);
- blockData = repository.getBlockRepository().fromReference(blockData.getSignature());
+ byte[] previousSignature = blockData.getSignature();
+ blockData = repository.getBlockRepository().fromReference(previousSignature);
+ if (blockData == null) {
+ // Try the archive
+ blockData = repository.getBlockArchiveRepository().fromReference(previousSignature);
+ }
}
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while sending block summaries after %s to peer %s", Base58.encode(parentSignature), peer), e);
@@ -1513,11 +1707,20 @@ public class Controller extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) {
int numberRequested = getSignaturesMessage.getNumberRequested();
BlockData blockData = repository.getBlockRepository().fromReference(parentSignature);
+ if (blockData == null) {
+ // Try the archive
+ blockData = repository.getBlockArchiveRepository().fromReference(parentSignature);
+ }
while (blockData != null && signatures.size() < numberRequested) {
signatures.add(blockData.getSignature());
- blockData = repository.getBlockRepository().fromReference(blockData.getSignature());
+ byte[] previousSignature = blockData.getSignature();
+ blockData = repository.getBlockRepository().fromReference(previousSignature);
+ if (blockData == null) {
+ // Try the archive
+ blockData = repository.getBlockArchiveRepository().fromReference(previousSignature);
+ }
}
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while sending V2 signatures after %s to peer %s", Base58.encode(parentSignature), peer), e);
diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java
index 6ddbad16..fde89851 100644
--- a/src/main/java/org/qortal/controller/Synchronizer.java
+++ b/src/main/java/org/qortal/controller/Synchronizer.java
@@ -3,12 +3,9 @@ package org.qortal.controller;
import java.math.BigInteger;
import java.text.DecimalFormat;
import java.text.NumberFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
-import java.util.Iterator;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -71,6 +68,11 @@ public class Synchronizer {
// Keep track of the size of the last re-org, so it can be logged
private int lastReorgSize;
+ // Keep track of invalid blocks so that we don't keep trying to sync them
+ private Map invalidBlockSignatures = Collections.synchronizedMap(new HashMap<>());
+ public Long timeValidBlockLastReceived = null;
+ public Long timeInvalidBlockLastReceived = null;
+
private static Synchronizer instance;
public enum SynchronizationResult {
@@ -346,6 +348,12 @@ public class Synchronizer {
}
}
+ // Ignore this peer if it holds an invalid block
+ if (this.containsInvalidBlockSummary(peer.getCommonBlockData().getBlockSummariesAfterCommonBlock())) {
+ LOGGER.debug("Ignoring peer %s because it holds an invalid block", peer);
+ peers.remove(peer);
+ }
+
// Reduce minChainLength if needed. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength
List peerBlockSummaries = peer.getCommonBlockData().getBlockSummariesAfterCommonBlock();
if (peerBlockSummaries != null && peerBlockSummaries.size() > 0)
@@ -489,6 +497,71 @@ public class Synchronizer {
}
+
+ /* Invalid block signature tracking */
+
+ private void addInvalidBlockSignature(byte[] signature) {
+ Long now = NTP.getTime();
+ if (now == null) {
+ return;
+ }
+
+ // Add or update existing entry
+ String sig58 = Base58.encode(signature);
+ invalidBlockSignatures.put(sig58, now);
+ }
+ private void deleteOlderInvalidSignatures(Long now) {
+ if (now == null) {
+ return;
+ }
+
+ // Delete signatures with older timestamps
+ Iterator it = invalidBlockSignatures.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry pair = (Map.Entry)it.next();
+ Long lastSeen = (Long) pair.getValue();
+
+ // Remove signature if we haven't seen it for more than 1 hour
+ if (now - lastSeen > 60 * 60 * 1000L) {
+ it.remove();
+ }
+ }
+ }
+ private boolean containsInvalidBlockSummary(List blockSummaries) {
+ if (blockSummaries == null || invalidBlockSignatures == null) {
+ return false;
+ }
+
+ // Loop through our known invalid blocks and check each one against supplied block summaries
+ for (String invalidSignature58 : invalidBlockSignatures.keySet()) {
+ byte[] invalidSignature = Base58.decode(invalidSignature58);
+ for (BlockSummaryData blockSummary : blockSummaries) {
+ byte[] signature = blockSummary.getSignature();
+ if (Arrays.equals(signature, invalidSignature)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+ private boolean containsInvalidBlockSignature(List blockSignatures) {
+ if (blockSignatures == null || invalidBlockSignatures == null) {
+ return false;
+ }
+
+ // Loop through our known invalid blocks and check each one against supplied block signatures
+ for (String invalidSignature58 : invalidBlockSignatures.keySet()) {
+ byte[] invalidSignature = Base58.decode(invalidSignature58);
+ for (byte[] signature : blockSignatures) {
+ if (Arrays.equals(signature, invalidSignature)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+
/**
* Attempt to synchronize blockchain with peer.
*
@@ -535,6 +608,15 @@ public class Synchronizer {
// Reset last re-org size as we are starting a new sync round
this.lastReorgSize = 0;
+ // Set the initial value of timeValidBlockLastReceived if it's null
+ Long now = NTP.getTime();
+ if (this.timeValidBlockLastReceived == null) {
+ this.timeValidBlockLastReceived = now;
+ }
+
+ // Delete invalid signatures with older timestamps
+ this.deleteOlderInvalidSignatures(now);
+
List peerBlockSummaries = new ArrayList<>();
SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, force, peerBlockSummaries, true);
if (findCommonBlockResult != SynchronizationResult.OK) {
@@ -883,6 +965,12 @@ public class Synchronizer {
break;
}
+ // Catch a block with an invalid signature before orphaning, so that we retain our existing valid candidate
+ if (this.containsInvalidBlockSignature(peerBlockSignatures)) {
+ LOGGER.info(String.format("Peer %s sent invalid block signature: %.8s", peer, Base58.encode(latestPeerSignature)));
+ return SynchronizationResult.INVALID_DATA;
+ }
+
byte[] nextPeerSignature = peerBlockSignatures.get(0);
int nextHeight = height + 1;
@@ -985,13 +1073,20 @@ public class Synchronizer {
if (Controller.isStopping())
return SynchronizationResult.SHUTTING_DOWN;
+ newBlock.preProcess();
+
ValidationResult blockResult = newBlock.isValid();
if (blockResult != ValidationResult.OK) {
LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer,
newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getSignature()), blockResult.name()));
+ this.addInvalidBlockSignature(newBlock.getSignature());
+ this.timeInvalidBlockLastReceived = NTP.getTime();
return SynchronizationResult.INVALID_DATA;
}
+ // Block is valid
+ this.timeValidBlockLastReceived = NTP.getTime();
+
// Save transactions attached to this block
for (Transaction transaction : newBlock.getTransactions()) {
TransactionData transactionData = transaction.getTransactionData();
@@ -1173,13 +1268,20 @@ public class Synchronizer {
for (Transaction transaction : newBlock.getTransactions())
transaction.setInitialApprovalStatus();
+ newBlock.preProcess();
+
ValidationResult blockResult = newBlock.isValid();
if (blockResult != ValidationResult.OK) {
LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer,
ourHeight, Base58.encode(latestPeerSignature), blockResult.name()));
+ this.addInvalidBlockSignature(newBlock.getSignature());
+ this.timeInvalidBlockLastReceived = NTP.getTime();
return SynchronizationResult.INVALID_DATA;
}
+ // Block is valid
+ this.timeValidBlockLastReceived = NTP.getTime();
+
// Save transactions attached to this block
for (Transaction transaction : newBlock.getTransactions()) {
TransactionData transactionData = transaction.getTransactionData();
diff --git a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java
new file mode 100644
index 00000000..3b92db51
--- /dev/null
+++ b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java
@@ -0,0 +1,109 @@
+package org.qortal.controller.repository;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.NTP;
+
+public class AtStatesPruner implements Runnable {
+
+ private static final Logger LOGGER = LogManager.getLogger(AtStatesPruner.class);
+
+ @Override
+ public void run() {
+ Thread.currentThread().setName("AT States pruner");
+
+ boolean archiveMode = false;
+ if (!Settings.getInstance().isTopOnly()) {
+ // Top-only mode isn't enabled, but we might want to prune for the purposes of archiving
+ if (!Settings.getInstance().isArchiveEnabled()) {
+ // No pruning or archiving, so we must not prune anything
+ return;
+ }
+ else {
+ // We're allowed to prune blocks that have already been archived
+ archiveMode = true;
+ }
+ }
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ int pruneStartHeight = repository.getATRepository().getAtPruneHeight();
+
+ repository.discardChanges();
+ repository.getATRepository().rebuildLatestAtStates();
+
+ while (!Controller.isStopping()) {
+ repository.discardChanges();
+
+ Thread.sleep(Settings.getInstance().getAtStatesPruneInterval());
+
+ BlockData chainTip = Controller.getInstance().getChainTip();
+ if (chainTip == null || NTP.getTime() == null)
+ continue;
+
+ // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
+ if (Controller.getInstance().isSynchronizing())
+ continue;
+
+ // Prune AT states for all blocks up until our latest minus pruneBlockLimit
+ final int ourLatestHeight = chainTip.getHeight();
+ int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit();
+
+ // In archive mode we are only allowed to trim blocks that have already been archived
+ if (archiveMode) {
+ upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1;
+
+ // TODO: validate that the actual archived data exists before pruning it?
+ }
+
+ int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize();
+ int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);
+
+ if (pruneStartHeight >= upperPruneHeight)
+ continue;
+
+ LOGGER.debug(String.format("Pruning AT states between blocks %d and %d...", pruneStartHeight, upperPruneHeight));
+
+ int numAtStatesPruned = repository.getATRepository().pruneAtStates(pruneStartHeight, upperPruneHeight);
+ repository.saveChanges();
+ int numAtStateDataRowsTrimmed = repository.getATRepository().trimAtStates(
+ pruneStartHeight, upperPruneHeight, Settings.getInstance().getAtStatesTrimLimit());
+ repository.saveChanges();
+
+ if (numAtStatesPruned > 0 || numAtStateDataRowsTrimmed > 0) {
+ final int finalPruneStartHeight = pruneStartHeight;
+ LOGGER.debug(() -> String.format("Pruned %d AT state%s between blocks %d and %d",
+ numAtStatesPruned, (numAtStatesPruned != 1 ? "s" : ""),
+ finalPruneStartHeight, upperPruneHeight));
+ } else {
+ // Can we move onto next batch?
+ if (upperPrunableHeight > upperBatchHeight) {
+ pruneStartHeight = upperBatchHeight;
+ repository.getATRepository().setAtPruneHeight(pruneStartHeight);
+ repository.getATRepository().rebuildLatestAtStates();
+ repository.saveChanges();
+
+ final int finalPruneStartHeight = pruneStartHeight;
+ LOGGER.debug(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight));
+ }
+ else {
+ // We've pruned up to the upper prunable height
+ // Back off for a while to save CPU for syncing
+ repository.discardChanges();
+ Thread.sleep(5*60*1000L);
+ }
+ }
+ }
+ } catch (DataException e) {
+ LOGGER.warn(String.format("Repository issue trying to prune AT states: %s", e.getMessage()));
+ } catch (InterruptedException e) {
+ // Time to exit
+ }
+ }
+
+}
diff --git a/src/main/java/org/qortal/controller/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
similarity index 92%
rename from src/main/java/org/qortal/controller/AtStatesTrimmer.java
rename to src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
index b452b3cc..98a1a889 100644
--- a/src/main/java/org/qortal/controller/AtStatesTrimmer.java
+++ b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
@@ -1,7 +1,8 @@
-package org.qortal.controller;
+package org.qortal.controller.repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
import org.qortal.data.block.BlockData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
@@ -20,8 +21,8 @@ public class AtStatesTrimmer implements Runnable {
try (final Repository repository = RepositoryManager.getRepository()) {
int trimStartHeight = repository.getATRepository().getAtTrimHeight();
- repository.getATRepository().prepareForAtStateTrimming();
- repository.saveChanges();
+ repository.discardChanges();
+ repository.getATRepository().rebuildLatestAtStates();
while (!Controller.isStopping()) {
repository.discardChanges();
@@ -62,7 +63,7 @@ public class AtStatesTrimmer implements Runnable {
if (upperTrimmableHeight > upperBatchHeight) {
trimStartHeight = upperBatchHeight;
repository.getATRepository().setAtTrimHeight(trimStartHeight);
- repository.getATRepository().prepareForAtStateTrimming();
+ repository.getATRepository().rebuildLatestAtStates();
repository.saveChanges();
final int finalTrimStartHeight = trimStartHeight;
diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java
new file mode 100644
index 00000000..2a987d97
--- /dev/null
+++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java
@@ -0,0 +1,113 @@
+package org.qortal.controller.repository;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.*;
+import org.qortal.settings.Settings;
+import org.qortal.transform.TransformationException;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+
+public class BlockArchiver implements Runnable {
+
+ private static final Logger LOGGER = LogManager.getLogger(BlockArchiver.class);
+
+ private static final long INITIAL_SLEEP_PERIOD = 0L; // TODO: 5 * 60 * 1000L + 1234L; // ms
+
+ public void run() {
+ Thread.currentThread().setName("Block archiver");
+
+ if (!Settings.getInstance().isArchiveEnabled()) {
+ return;
+ }
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ // Don't even start building until initial rush has ended
+ Thread.sleep(INITIAL_SLEEP_PERIOD);
+
+ int startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight();
+
+ // Don't attempt to archive if we have no ATStatesHeightIndex, as it will be too slow
+ boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex();
+ if (!hasAtStatesHeightIndex) {
+ LOGGER.info("Unable to start block archiver due to missing ATStatesHeightIndex. Bootstrapping is recommended.");
+ repository.discardChanges();
+ return;
+ }
+
+ LOGGER.info("Starting block archiver...");
+
+ while (!Controller.isStopping()) {
+ repository.discardChanges();
+
+ Thread.sleep(Settings.getInstance().getArchiveInterval());
+
+ BlockData chainTip = Controller.getInstance().getChainTip();
+ if (chainTip == null || NTP.getTime() == null) {
+ continue;
+ }
+
+ // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
+ if (Controller.getInstance().isSynchronizing()) {
+ continue;
+ }
+
+ // Don't attempt to archive if we're not synced yet
+ final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+ if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) {
+ continue;
+ }
+
+
+ // Build cache of blocks
+ try {
+ final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
+ BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository);
+ BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
+ switch (result) {
+ case OK:
+ // Increment block archive height
+ startHeight += writer.getWrittenCount();
+ repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight);
+ repository.saveChanges();
+ break;
+
+ case STOPPING:
+ return;
+
+ // We've reached the limit of the blocks we can archive
+ // Sleep for a while to allow more to become available
+ case NOT_ENOUGH_BLOCKS:
+ // We didn't reach our file size target, so that must mean that we don't have enough blocks
+ // yet or something went wrong. Sleep for a while and then try again.
+ repository.discardChanges();
+ Thread.sleep(60 * 60 * 1000L); // 1 hour
+ break;
+
+ case BLOCK_NOT_FOUND:
+ // We tried to archive a block that didn't exist. This is a major failure and likely means
+ // that a bootstrap or re-sync is needed. Try again every minute until then.
+ LOGGER.info("Error: block not found when building archive. If this error persists, " +
+ "a bootstrap or re-sync may be needed.");
+ repository.discardChanges();
+ Thread.sleep( 60 * 1000L); // 1 minute
+ break;
+ }
+
+ } catch (IOException | TransformationException e) {
+ LOGGER.info("Caught exception when creating block cache", e);
+ }
+
+ }
+ } catch (DataException e) {
+ LOGGER.info("Caught exception when creating block cache", e);
+ } catch (InterruptedException e) {
+ // Do nothing
+ }
+
+ }
+
+}
diff --git a/src/main/java/org/qortal/controller/repository/BlockPruner.java b/src/main/java/org/qortal/controller/repository/BlockPruner.java
new file mode 100644
index 00000000..1258ee38
--- /dev/null
+++ b/src/main/java/org/qortal/controller/repository/BlockPruner.java
@@ -0,0 +1,114 @@
+package org.qortal.controller.repository;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.NTP;
+
+public class BlockPruner implements Runnable {
+
+ private static final Logger LOGGER = LogManager.getLogger(BlockPruner.class);
+
+ @Override
+ public void run() {
+ Thread.currentThread().setName("Block pruner");
+
+ boolean archiveMode = false;
+ if (!Settings.getInstance().isTopOnly()) {
+ // Top-only mode isn't enabled, but we might want to prune for the purposes of archiving
+ if (!Settings.getInstance().isArchiveEnabled()) {
+ // No pruning or archiving, so we must not prune anything
+ return;
+ }
+ else {
+ // We're allowed to prune blocks that have already been archived
+ archiveMode = true;
+ }
+ }
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ int pruneStartHeight = repository.getBlockRepository().getBlockPruneHeight();
+
+ // Don't attempt to prune if we have no ATStatesHeightIndex, as it will be too slow
+ boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex();
+ if (!hasAtStatesHeightIndex) {
+ LOGGER.info("Unable to start block pruner due to missing ATStatesHeightIndex. Bootstrapping is recommended.");
+ return;
+ }
+
+ while (!Controller.isStopping()) {
+ repository.discardChanges();
+
+ Thread.sleep(Settings.getInstance().getBlockPruneInterval());
+
+ BlockData chainTip = Controller.getInstance().getChainTip();
+ if (chainTip == null || NTP.getTime() == null)
+ continue;
+
+ // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
+ if (Controller.getInstance().isSynchronizing()) {
+ continue;
+ }
+
+ // Don't attempt to prune if we're not synced yet
+ final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+ if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) {
+ continue;
+ }
+
+ // Prune all blocks up until our latest minus pruneBlockLimit
+ final int ourLatestHeight = chainTip.getHeight();
+ int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit();
+
+ // In archive mode we are only allowed to trim blocks that have already been archived
+ if (archiveMode) {
+ upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1;
+ }
+
+ int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize();
+ int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);
+
+ if (pruneStartHeight >= upperPruneHeight) {
+ continue;
+ }
+
+ LOGGER.debug(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight));
+
+ int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight);
+ repository.saveChanges();
+
+ if (numBlocksPruned > 0) {
+ LOGGER.debug(String.format("Pruned %d block%s between %d and %d",
+ numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""),
+ pruneStartHeight, upperPruneHeight));
+ } else {
+ final int nextPruneHeight = upperPruneHeight + 1;
+ repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight);
+ repository.saveChanges();
+ LOGGER.debug(String.format("Bumping block base prune height to %d", pruneStartHeight));
+
+ // Can we move onto next batch?
+ if (upperPrunableHeight > nextPruneHeight) {
+ pruneStartHeight = nextPruneHeight;
+ }
+ else {
+ // We've pruned up to the upper prunable height
+ // Back off for a while to save CPU for syncing
+ repository.discardChanges();
+ Thread.sleep(10*60*1000L);
+ }
+ }
+ }
+ } catch (DataException e) {
+ LOGGER.warn(String.format("Repository issue trying to prune blocks: %s", e.getMessage()));
+ } catch (InterruptedException e) {
+ // Time to exit
+ }
+ }
+
+}
diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java
new file mode 100644
index 00000000..f12bd14a
--- /dev/null
+++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java
@@ -0,0 +1,410 @@
+package org.qortal.controller.repository;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.account.PublicKeyAccount;
+import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
+import org.qortal.data.naming.NameData;
+import org.qortal.data.transaction.*;
+import org.qortal.naming.Name;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.transaction.Transaction.TransactionType;
+import org.qortal.utils.Unicode;
+
+import java.util.*;
+
+public class NamesDatabaseIntegrityCheck {
+
+ private static final Logger LOGGER = LogManager.getLogger(NamesDatabaseIntegrityCheck.class);
+
+ private static final List ALL_NAME_TX_TYPE = Arrays.asList(
+ TransactionType.REGISTER_NAME,
+ TransactionType.UPDATE_NAME,
+ TransactionType.BUY_NAME,
+ TransactionType.SELL_NAME
+ );
+
+ private List nameTransactions = new ArrayList<>();
+
+ public int rebuildName(String name, Repository repository) {
+ int modificationCount = 0;
+ try {
+ List transactions = this.fetchAllTransactionsInvolvingName(name, repository);
+ if (transactions.isEmpty()) {
+ // This name was never registered, so there's nothing to do
+ return modificationCount;
+ }
+
+ // Loop through each past transaction and re-apply it to the Names table
+ for (TransactionData currentTransaction : transactions) {
+
+ // Process REGISTER_NAME transactions
+ if (currentTransaction.getType() == TransactionType.REGISTER_NAME) {
+ RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) currentTransaction;
+ Name nameObj = new Name(repository, registerNameTransactionData);
+ nameObj.register();
+ modificationCount++;
+ LOGGER.trace("Processed REGISTER_NAME transaction for name {}", name);
+ }
+
+ // Process UPDATE_NAME transactions
+ if (currentTransaction.getType() == TransactionType.UPDATE_NAME) {
+ UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) currentTransaction;
+
+ if (Objects.equals(updateNameTransactionData.getNewName(), name) &&
+ !Objects.equals(updateNameTransactionData.getName(), updateNameTransactionData.getNewName())) {
+ // This renames an existing name, so we need to process that instead
+ this.rebuildName(updateNameTransactionData.getName(), repository);
+ }
+ else {
+ Name nameObj = new Name(repository, name);
+ if (nameObj != null && nameObj.getNameData() != null) {
+ nameObj.update(updateNameTransactionData);
+ modificationCount++;
+ LOGGER.trace("Processed UPDATE_NAME transaction for name {}", name);
+ } else {
+ // Something went wrong
+ throw new DataException(String.format("Name data not found for name %s", updateNameTransactionData.getName()));
+ }
+ }
+ }
+
+ // Process SELL_NAME transactions
+ if (currentTransaction.getType() == TransactionType.SELL_NAME) {
+ SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) currentTransaction;
+ Name nameObj = new Name(repository, sellNameTransactionData.getName());
+ if (nameObj != null && nameObj.getNameData() != null) {
+ nameObj.sell(sellNameTransactionData);
+ modificationCount++;
+ LOGGER.trace("Processed SELL_NAME transaction for name {}", name);
+ }
+ else {
+ // Something went wrong
+ throw new DataException(String.format("Name data not found for name %s", sellNameTransactionData.getName()));
+ }
+ }
+
+ // Process BUY_NAME transactions
+ if (currentTransaction.getType() == TransactionType.BUY_NAME) {
+ BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) currentTransaction;
+ Name nameObj = new Name(repository, buyNameTransactionData.getName());
+ if (nameObj != null && nameObj.getNameData() != null) {
+ nameObj.buy(buyNameTransactionData);
+ modificationCount++;
+ LOGGER.trace("Processed BUY_NAME transaction for name {}", name);
+ }
+ else {
+ // Something went wrong
+ throw new DataException(String.format("Name data not found for name %s", buyNameTransactionData.getName()));
+ }
+ }
+ }
+
+ } catch (DataException e) {
+ LOGGER.info("Unable to run integrity check for name {}: {}", name, e.getMessage());
+ }
+
+ return modificationCount;
+ }
+
+ public int rebuildAllNames() {
+ int modificationCount = 0;
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ List names = this.fetchAllNames(repository);
+ for (String name : names) {
+ modificationCount += this.rebuildName(name, repository);
+ }
+ repository.saveChanges();
+ }
+ catch (DataException e) {
+ LOGGER.info("Error when running integrity check for all names: {}", e.getMessage());
+ }
+
+ //LOGGER.info("modificationCount: {}", modificationCount);
+ return modificationCount;
+ }
+
+ public void runIntegrityCheck() {
+ boolean integrityCheckFailed = false;
+ try (final Repository repository = RepositoryManager.getRepository()) {
+
+ // Fetch all the (confirmed) REGISTER_NAME transactions
+ List registerNameTransactions = this.fetchRegisterNameTransactions();
+
+ // Loop through each REGISTER_NAME txn signature and request the full transaction data
+ for (RegisterNameTransactionData registerNameTransactionData : registerNameTransactions) {
+ String registeredName = registerNameTransactionData.getName();
+ NameData nameData = repository.getNameRepository().fromName(registeredName);
+
+ // Check to see if this name has been updated or bought at any point
+ TransactionData latestUpdate = this.fetchLatestModificationTransactionInvolvingName(registeredName, repository);
+ if (latestUpdate == null) {
+ // Name was never updated once registered
+ // We expect this name to still be registered to this transaction's creator
+
+ if (nameData == null) {
+ LOGGER.info("Error: registered name {} doesn't exist in Names table. Adding...", registeredName);
+ integrityCheckFailed = true;
+ }
+ else {
+ LOGGER.trace("Registered name {} is correctly registered", registeredName);
+ }
+
+ // Check the owner is correct
+ PublicKeyAccount creator = new PublicKeyAccount(repository, registerNameTransactionData.getCreatorPublicKey());
+ if (!Objects.equals(creator.getAddress(), nameData.getOwner())) {
+ LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
+ registeredName, nameData.getOwner(), creator.getAddress());
+ integrityCheckFailed = true;
+ }
+ else {
+ LOGGER.trace("Registered name {} has the correct owner", registeredName);
+ }
+ }
+ else {
+ // Check if owner is correct after update
+
+ // Check for name updates
+ if (latestUpdate.getType() == TransactionType.UPDATE_NAME) {
+ UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) latestUpdate;
+ PublicKeyAccount creator = new PublicKeyAccount(repository, updateNameTransactionData.getCreatorPublicKey());
+
+ // When this name is the "new name", we expect the current owner to match the txn creator
+ if (Objects.equals(updateNameTransactionData.getNewName(), registeredName)) {
+ if (!Objects.equals(creator.getAddress(), nameData.getOwner())) {
+ LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
+ registeredName, nameData.getOwner(), creator.getAddress());
+ integrityCheckFailed = true;
+ }
+ else {
+ LOGGER.trace("Registered name {} has the correct owner after being updated", registeredName);
+ }
+ }
+
+ // When this name is the old name, we expect the "new name"'s owner to match the txn creator
+ // The old name will then be unregistered, or re-registered.
+ // FUTURE: check database integrity for names that have been updated and then the original name re-registered
+ else if (Objects.equals(updateNameTransactionData.getName(), registeredName)) {
+ NameData newNameData = repository.getNameRepository().fromName(updateNameTransactionData.getNewName());
+ if (!Objects.equals(creator.getAddress(), newNameData.getOwner())) {
+ LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
+ updateNameTransactionData.getNewName(), newNameData.getOwner(), creator.getAddress());
+ integrityCheckFailed = true;
+ }
+ else {
+ LOGGER.trace("Registered name {} has the correct owner after being updated", updateNameTransactionData.getNewName());
+ }
+ }
+
+ else {
+ LOGGER.info("Unhandled update case for name {}", registeredName);
+ }
+ }
+
+ // Check for name buys
+ else if (latestUpdate.getType() == TransactionType.BUY_NAME) {
+ BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) latestUpdate;
+ PublicKeyAccount creator = new PublicKeyAccount(repository, buyNameTransactionData.getCreatorPublicKey());
+ if (!Objects.equals(creator.getAddress(), nameData.getOwner())) {
+ LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
+ registeredName, nameData.getOwner(), creator.getAddress());
+ integrityCheckFailed = true;
+ }
+ else {
+ LOGGER.trace("Registered name {} has the correct owner after being bought", registeredName);
+ }
+ }
+
+ // Check for name sells
+ else if (latestUpdate.getType() == TransactionType.SELL_NAME) {
+ SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) latestUpdate;
+ PublicKeyAccount creator = new PublicKeyAccount(repository, sellNameTransactionData.getCreatorPublicKey());
+ if (!Objects.equals(creator.getAddress(), nameData.getOwner())) {
+ LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
+ registeredName, nameData.getOwner(), creator.getAddress());
+ integrityCheckFailed = true;
+ }
+ else {
+ LOGGER.trace("Registered name {} has the correct owner after being listed for sale", registeredName);
+ }
+ }
+
+ else {
+ LOGGER.info("Unhandled case for name {}", registeredName);
+ }
+
+ }
+
+ }
+
+ } catch (DataException e) {
+ LOGGER.warn(String.format("Repository issue trying to trim online accounts signatures: %s", e.getMessage()));
+ integrityCheckFailed = true;
+ }
+
+ if (integrityCheckFailed) {
+ LOGGER.info("Registered names database integrity check failed. Bootstrapping is recommended.");
+ } else {
+ LOGGER.info("Registered names database integrity check passed.");
+ }
+ }
+
+ private List fetchRegisterNameTransactions() {
+ List registerNameTransactions = new ArrayList<>();
+
+ for (TransactionData transactionData : this.nameTransactions) {
+ if (transactionData.getType() == TransactionType.REGISTER_NAME) {
+ RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData;
+ registerNameTransactions.add(registerNameTransactionData);
+ }
+ }
+ return registerNameTransactions;
+ }
+
+ private List fetchUpdateNameTransactions() {
+ List updateNameTransactions = new ArrayList<>();
+
+ for (TransactionData transactionData : this.nameTransactions) {
+ if (transactionData.getType() == TransactionType.UPDATE_NAME) {
+ UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData;
+ updateNameTransactions.add(updateNameTransactionData);
+ }
+ }
+ return updateNameTransactions;
+ }
+
+ private List fetchSellNameTransactions() {
+ List sellNameTransactions = new ArrayList<>();
+
+ for (TransactionData transactionData : this.nameTransactions) {
+ if (transactionData.getType() == TransactionType.SELL_NAME) {
+ SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData;
+ sellNameTransactions.add(sellNameTransactionData);
+ }
+ }
+ return sellNameTransactions;
+ }
+
+ private List fetchBuyNameTransactions() {
+ List buyNameTransactions = new ArrayList<>();
+
+ for (TransactionData transactionData : this.nameTransactions) {
+ if (transactionData.getType() == TransactionType.BUY_NAME) {
+ BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData;
+ buyNameTransactions.add(buyNameTransactionData);
+ }
+ }
+ return buyNameTransactions;
+ }
+
+ private void fetchAllNameTransactions(Repository repository) throws DataException {
+ List nameTransactions = new ArrayList<>();
+
+ // Fetch all the confirmed REGISTER_NAME transaction signatures
+ List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(
+ null, null, null, ALL_NAME_TX_TYPE, null, null,
+ ConfirmationStatus.CONFIRMED, null, null, false);
+
+ for (byte[] signature : signatures) {
+ TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+ nameTransactions.add(transactionData);
+ }
+ this.nameTransactions = nameTransactions;
+ }
+
+ private List fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException {
+ List transactions = new ArrayList<>();
+ String reducedName = Unicode.sanitize(name);
+
+ // Fetch all the confirmed name-modification transactions
+ if (this.nameTransactions.isEmpty()) {
+ this.fetchAllNameTransactions(repository);
+ }
+
+ for (TransactionData transactionData : this.nameTransactions) {
+
+ if ((transactionData instanceof RegisterNameTransactionData)) {
+ RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData;
+ if (Objects.equals(registerNameTransactionData.getReducedName(), reducedName)) {
+ transactions.add(transactionData);
+ }
+ }
+ if ((transactionData instanceof UpdateNameTransactionData)) {
+ UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData;
+ if (Objects.equals(updateNameTransactionData.getName(), name) ||
+ Objects.equals(updateNameTransactionData.getReducedNewName(), reducedName)) {
+ transactions.add(transactionData);
+ }
+ }
+ if ((transactionData instanceof BuyNameTransactionData)) {
+ BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData;
+ if (Objects.equals(buyNameTransactionData.getName(), name)) {
+ transactions.add(transactionData);
+ }
+ }
+ if ((transactionData instanceof SellNameTransactionData)) {
+ SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData;
+ if (Objects.equals(sellNameTransactionData.getName(), name)) {
+ transactions.add(transactionData);
+ }
+ }
+ }
+ return transactions;
+ }
+
+ private TransactionData fetchLatestModificationTransactionInvolvingName(String registeredName, Repository repository) throws DataException {
+ List transactionsInvolvingName = this.fetchAllTransactionsInvolvingName(registeredName, repository);
+
+ // Get the latest update for this name (excluding REGISTER_NAME transactions)
+ TransactionData latestUpdateToName = transactionsInvolvingName.stream()
+ .filter(txn -> txn.getType() != TransactionType.REGISTER_NAME)
+ .max(Comparator.comparing(TransactionData::getTimestamp))
+ .orElse(null);
+
+ return latestUpdateToName;
+ }
+
+ private List fetchAllNames(Repository repository) throws DataException {
+ List names = new ArrayList<>();
+
+ // Fetch all the confirmed name transactions
+ if (this.nameTransactions.isEmpty()) {
+ this.fetchAllNameTransactions(repository);
+ }
+
+ for (TransactionData transactionData : this.nameTransactions) {
+
+ if ((transactionData instanceof RegisterNameTransactionData)) {
+ RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData;
+ if (!names.contains(registerNameTransactionData.getName())) {
+ names.add(registerNameTransactionData.getName());
+ }
+ }
+ if ((transactionData instanceof UpdateNameTransactionData)) {
+ UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData;
+ if (!names.contains(updateNameTransactionData.getName())) {
+ names.add(updateNameTransactionData.getName());
+ }
+ if (!names.contains(updateNameTransactionData.getNewName())) {
+ names.add(updateNameTransactionData.getNewName());
+ }
+ }
+ if ((transactionData instanceof BuyNameTransactionData)) {
+ BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData;
+ if (!names.contains(buyNameTransactionData.getName())) {
+ names.add(buyNameTransactionData.getName());
+ }
+ }
+ if ((transactionData instanceof SellNameTransactionData)) {
+ SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData;
+ if (!names.contains(sellNameTransactionData.getName())) {
+ names.add(sellNameTransactionData.getName());
+ }
+ }
+ }
+ return names;
+ }
+
+}
diff --git a/src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java
similarity index 97%
rename from src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java
rename to src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java
index b32a2b06..c7f248d5 100644
--- a/src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java
+++ b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java
@@ -1,8 +1,9 @@
-package org.qortal.controller;
+package org.qortal.controller.repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.block.BlockChain;
+import org.qortal.controller.Controller;
import org.qortal.data.block.BlockData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
diff --git a/src/main/java/org/qortal/controller/repository/PruneManager.java b/src/main/java/org/qortal/controller/repository/PruneManager.java
new file mode 100644
index 00000000..ec27456f
--- /dev/null
+++ b/src/main/java/org/qortal/controller/repository/PruneManager.java
@@ -0,0 +1,160 @@
+package org.qortal.controller.repository;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.settings.Settings;
+import org.qortal.utils.DaemonThreadFactory;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+public class PruneManager {
+
+ private static final Logger LOGGER = LogManager.getLogger(PruneManager.class);
+
+ private static PruneManager instance;
+
+ private boolean isTopOnly = Settings.getInstance().isTopOnly();
+ private int pruneBlockLimit = Settings.getInstance().getPruneBlockLimit();
+
+ private ExecutorService executorService;
+
+ private PruneManager() {
+
+ }
+
+ public static synchronized PruneManager getInstance() {
+ if (instance == null)
+ instance = new PruneManager();
+
+ return instance;
+ }
+
+ public void start() {
+ this.executorService = Executors.newCachedThreadPool(new DaemonThreadFactory());
+
+ if (Settings.getInstance().isTopOnly()) {
+ // Top-only-sync
+ this.startTopOnlySyncMode();
+ }
+ else if (Settings.getInstance().isArchiveEnabled()) {
+ // Full node with block archive
+ this.startFullNodeWithBlockArchive();
+ }
+ else {
+ // Full node with full SQL support
+ this.startFullSQLNode();
+ }
+ }
+
+ /**
+ * Top-only-sync
+ * In this mode, we delete (prune) all blocks except
+ * a small number of recent ones. There is no need for
+ * trimming or archiving, because all relevant blocks
+ * are deleted.
+ */
+ private void startTopOnlySyncMode() {
+ this.startPruning();
+
+ // We don't need the block archive in top-only mode
+ this.deleteArchive();
+ }
+
+ /**
+ * Full node with block archive
+ * In this mode we archive trimmed blocks, and then
+ * prune archived blocks to keep the database small
+ */
+ private void startFullNodeWithBlockArchive() {
+ this.startTrimming();
+ this.startArchiving();
+ this.startPruning();
+ }
+
+ /**
+ * Full node with full SQL support
+ * In this mode we trim the database but don't prune
+ * or archive any data, because we want to maintain
+ * full SQL support of old blocks. This mode will not
+ * be actively maintained but can be used by those who
+ * need to perform SQL analysis on older blocks.
+ */
+ private void startFullSQLNode() {
+ this.startTrimming();
+ }
+
+
+ private void startPruning() {
+ this.executorService.execute(new AtStatesPruner());
+ this.executorService.execute(new BlockPruner());
+ }
+
+ private void startTrimming() {
+ this.executorService.execute(new AtStatesTrimmer());
+ this.executorService.execute(new OnlineAccountsSignaturesTrimmer());
+ }
+
+ private void startArchiving() {
+ this.executorService.execute(new BlockArchiver());
+ }
+
+ private void deleteArchive() {
+ if (!Settings.getInstance().isTopOnly()) {
+ LOGGER.error("Refusing to delete archive when not in top-only mode");
+ }
+
+ try {
+ Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive");
+ if (archivePath.toFile().exists()) {
+ LOGGER.info("Deleting block archive because we are in top-only mode...");
+ FileUtils.deleteDirectory(archivePath.toFile());
+ }
+
+ } catch (IOException e) {
+ LOGGER.info("Couldn't delete archive: {}", e.getMessage());
+ }
+ }
+
+ public void stop() {
+ this.executorService.shutdownNow();
+
+ try {
+ this.executorService.awaitTermination(2L, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ // We tried...
+ }
+ }
+
+ public boolean isBlockPruned(int height) throws DataException {
+ if (!this.isTopOnly) {
+ return false;
+ }
+
+ BlockData chainTip = Controller.getInstance().getChainTip();
+ if (chainTip == null) {
+ throw new DataException("Unable to determine chain tip when checking if a block is pruned");
+ }
+
+ if (height == 1) {
+ // We don't prune the genesis block
+ return false;
+ }
+
+ final int ourLatestHeight = chainTip.getHeight();
+ final int latestUnprunedHeight = ourLatestHeight - this.pruneBlockLimit;
+
+ return (height < latestUnprunedHeight);
+ }
+
+}
diff --git a/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java
index 790584d3..038ecded 100644
--- a/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java
+++ b/src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java
@@ -360,6 +360,7 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot {
case BOB_DONE:
case ALICE_REFUNDED:
case BOB_REFUNDED:
+ case ALICE_REFUNDING_A:
return true;
default:
diff --git a/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv1TradeBot.java
index 516fa621..e7b60b25 100644
--- a/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv1TradeBot.java
+++ b/src/main/java/org/qortal/controller/tradebot/DogecoinACCTv1TradeBot.java
@@ -353,6 +353,7 @@ public class DogecoinACCTv1TradeBot implements AcctTradeBot {
case BOB_DONE:
case ALICE_REFUNDED:
case BOB_REFUNDED:
+ case ALICE_REFUNDING_A:
return true;
default:
diff --git a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java
index 0246c199..686b675e 100644
--- a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java
+++ b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java
@@ -364,6 +364,7 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
case BOB_DONE:
case ALICE_REFUNDED:
case BOB_REFUNDED:
+ case ALICE_REFUNDING_A:
return true;
default:
diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java
index 6e9d1474..36351927 100644
--- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java
+++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java
@@ -245,17 +245,17 @@ public class TradeBot implements Listener {
}
}
- /*package*/ static byte[] generateTradePrivateKey() {
+ public static byte[] generateTradePrivateKey() {
// The private key is used for both Curve25519 and secp256k1 so needs to be valid for both.
// Curve25519 accepts any seed, so generate a valid secp256k1 key and use that.
return new ECKey().getPrivKeyBytes();
}
- /*package*/ static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
+ public static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
return PrivateKeyAccount.toPublicKey(privateKey);
}
- /*package*/ static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {
+ public static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {
return ECKey.fromPrivate(privateKey).getPubKey();
}
diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java
index d4693818..3665f4ba 100644
--- a/src/main/java/org/qortal/crosschain/Bitcoiny.java
+++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java
@@ -406,14 +406,24 @@ public abstract class Bitcoiny implements ForeignBlockchain {
protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set keySet) {
long amount = 0;
long total = 0L;
+ long totalInputAmount = 0L;
+ long totalOutputAmount = 0L;
+ List inputs = new ArrayList<>();
+ List outputs = new ArrayList<>();
+
for (BitcoinyTransaction.Input input : t.inputs) {
try {
BitcoinyTransaction t2 = getTransaction(input.outputTxHash);
List senders = t2.outputs.get(input.outputVout).addresses;
+ long inputAmount = t2.outputs.get(input.outputVout).value;
+ totalInputAmount += inputAmount;
for (String sender : senders) {
+ boolean addressInWallet = false;
if (keySet.contains(sender)) {
- total += t2.outputs.get(input.outputVout).value;
+ total += inputAmount;
+ addressInWallet = true;
}
+ inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet));
}
} catch (ForeignBlockchainException e) {
LOGGER.trace("Failed to retrieve transaction information {}", input.outputTxHash);
@@ -422,17 +432,22 @@ public abstract class Bitcoiny implements ForeignBlockchain {
if (t.outputs != null && !t.outputs.isEmpty()) {
for (BitcoinyTransaction.Output output : t.outputs) {
for (String address : output.addresses) {
+ boolean addressInWallet = false;
if (keySet.contains(address)) {
if (total > 0L) {
amount -= (total - output.value);
} else {
amount += output.value;
}
+ addressInWallet = true;
}
+ outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet));
}
+ totalOutputAmount += output.value;
}
}
- return new SimpleTransaction(t.txHash, t.timestamp, amount);
+ long fee = totalInputAmount - totalOutputAmount;
+ return new SimpleTransaction(t.txHash, t.timestamp, amount, fee, inputs, outputs);
}
/**
diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java
index 8f41ed86..4ab7e0b1 100644
--- a/src/main/java/org/qortal/crosschain/ElectrumX.java
+++ b/src/main/java/org/qortal/crosschain/ElectrumX.java
@@ -653,18 +653,27 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
Object errorObj = responseJson.get("error");
if (errorObj != null) {
- if (errorObj instanceof String)
- throw new ForeignBlockchainException.NetworkException(String.format("Unexpected error message from ElectrumX RPC %s: %s", method, (String) errorObj), this.currentServer);
+ if (errorObj instanceof String) {
+ LOGGER.debug(String.format("Unexpected error message from ElectrumX server %s for RPC method %s: %s", this.currentServer, method, (String) errorObj));
+ // Try another server
+ return null;
+ }
- if (!(errorObj instanceof JSONObject))
- throw new ForeignBlockchainException.NetworkException(String.format("Unexpected error response from ElectrumX RPC %s", method), this.currentServer);
+ if (!(errorObj instanceof JSONObject)) {
+ LOGGER.debug(String.format("Unexpected error response from ElectrumX server %s for RPC method %s", this.currentServer, method));
+ // Try another server
+ return null;
+ }
JSONObject errorJson = (JSONObject) errorObj;
Object messageObj = errorJson.get("message");
- if (!(messageObj instanceof String))
- throw new ForeignBlockchainException.NetworkException(String.format("Missing/invalid message in error response from ElectrumX RPC %s", method), this.currentServer);
+ if (!(messageObj instanceof String)) {
+ LOGGER.debug(String.format("Missing/invalid message in error response from ElectrumX server %s for RPC method %s", this.currentServer, method));
+ // Try another server
+ return null;
+ }
String message = (String) messageObj;
diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java
index 0c04243c..42ee70de 100644
--- a/src/main/java/org/qortal/crosschain/Litecoin.java
+++ b/src/main/java/org/qortal/crosschain/Litecoin.java
@@ -21,6 +21,8 @@ public class Litecoin extends Bitcoiny {
private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(10000); // 0.0001 LTC per 1000 bytes
+ private static final long MINIMUM_ORDER_AMOUNT = 1000000; // 0.01 LTC minimum order, to avoid dust errors
+
// Temporary values until a dynamic fee system is written.
private static final long MAINNET_FEE = 1000L;
private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST
@@ -164,6 +166,11 @@ public class Litecoin extends Bitcoiny {
return DEFAULT_FEE_PER_KB;
}
+ @Override
+ public long getMinimumOrderAmount() {
+ return MINIMUM_ORDER_AMOUNT;
+ }
+
/**
* Returns estimated LTC fee, in sats per 1000bytes, optionally for historic timestamp.
*
diff --git a/src/main/java/org/qortal/crosschain/SimpleTransaction.java b/src/main/java/org/qortal/crosschain/SimpleTransaction.java
index 0fae20a5..27c9f9e3 100644
--- a/src/main/java/org/qortal/crosschain/SimpleTransaction.java
+++ b/src/main/java/org/qortal/crosschain/SimpleTransaction.java
@@ -2,20 +2,85 @@ package org.qortal.crosschain;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.List;
@XmlAccessorType(XmlAccessType.FIELD)
public class SimpleTransaction {
private String txHash;
private Integer timestamp;
private long totalAmount;
+ private long feeAmount;
+ private List inputs;
+ private List