Improved detection of an existing arbitrary resources cache.

src/main/java/org/qortal/controller/Controller.java

@@ -450,6 +450,13 @@ public class Controller extends Thread {
 				Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process.");
 				return;
 			}
+			if (ArbitraryDataCacheManager.getInstance().needsArbitraryResourcesCacheRebuild(repository)) {
+				// Don't allow the node to start if arbitrary resources cache hasn't been built yet
+				// This is needed to handle a case when bootstrapping
+				LOGGER.error("Database upgrade needed. Please restart the core to complete the upgrade process.");
+				Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process.");
+				return;
+			}
 		} catch (DataException e) {
 			LOGGER.error("Error checking transaction sequences in repository", e);
 			return;

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java

@@ -103,6 +103,28 @@ public class ArbitraryDataCacheManager extends Thread {
         LOGGER.debug(() -> String.format("Transaction %.8s added to queue", Base58.encode(transactionData.getSignature())));
     }
 
+    public boolean needsArbitraryResourcesCacheRebuild(Repository repository) throws DataException {
+        // Check if we have an entry in the cache for the oldest ARBITRARY transaction with a name
+        List<ArbitraryTransactionData> oldestCacheableTransactions = repository.getArbitraryRepository().getArbitraryTransactions(true, 1, 0, false);
+        if (oldestCacheableTransactions == null || oldestCacheableTransactions.isEmpty()) {
+            // No relevant arbitrary transactions yet on this chain
+            LOGGER.debug("No relevant arbitrary transactions exist to build cache from");
+            return false;
+        }
+        // We have an arbitrary transaction, so check if it's in the cache
+        ArbitraryTransactionData txn = oldestCacheableTransactions.get(0);
+        ArbitraryResourceData cachedResource = repository.getArbitraryRepository().getArbitraryResource(txn.getService(), txn.getName(), txn.getIdentifier());
+        if (cachedResource != null) {
+            // Earliest resource exists in the cache, so assume it has been built.
+            // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so
+            // we shouldn't ever be left in a partially rebuilt state.
+            LOGGER.debug("Arbitrary resources cache already built");
+            return false;
+        }
+
+        return true;
+    }
+
     public boolean buildArbitraryResourcesCache(Repository repository, boolean forceRebuild) throws DataException {
         if (Settings.getInstance().isLite()) {
             // Lite nodes have no blockchain
@@ -110,12 +132,8 @@ public class ArbitraryDataCacheManager extends Thread {
         }
 
         try {
-            // Check if QDNResources table is empty
+            // Skip if already built
-            List<ArbitraryResourceData> resources = repository.getArbitraryRepository().getArbitraryResources(10, 0, false);
+            if (!needsArbitraryResourcesCacheRebuild(repository) && !forceRebuild) {
-            if (!resources.isEmpty() && !forceRebuild) {
-                // Resources exist in the cache, so assume complete.
-                // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so
-                // we shouldn't ever be left in a partially rebuilt state.
                 LOGGER.debug("Arbitrary resources cache already built");
                 return false;
             }

src/main/java/org/qortal/repository/ArbitraryRepository.java

@@ -31,6 +31,8 @@ public interface ArbitraryRepository {
 
 	public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
 
+	public List<ArbitraryTransactionData> getArbitraryTransactions(boolean requireName, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
 
 	// Resource related
 

src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java

@@ -316,6 +316,85 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		return this.getSingleTransaction(name, service, method, identifier, false);
 	}
 
+	public List<ArbitraryTransactionData> getArbitraryTransactions(boolean requireName, Integer limit, Integer offset, Boolean reverse) throws DataException {
+		StringBuilder sql = new StringBuilder(512);
+		sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
+			"tx_group_id, block_height, approval_status, approval_height, " +
+			"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
+			"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
+			"JOIN Transactions USING (signature)");
+
+		if (requireName) {
+			sql.append(" WHERE name IS NOT NULL");
+		}
+
+		sql.append(" ORDER BY created_when");
+
+		if (reverse != null && reverse) {
+			sql.append(" DESC");
+		}
+
+		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
+
+		List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) {
+			if (resultSet == null)
+				return null;
+
+			do {
+				//TransactionType type = TransactionType.valueOf(resultSet.getInt(1));
+
+				byte[] reference = resultSet.getBytes(2);
+				byte[] signature = resultSet.getBytes(3);
+				byte[] creatorPublicKey = resultSet.getBytes(4);
+				long timestamp = resultSet.getLong(5);
+
+				Long fee = resultSet.getLong(6);
+				if (fee == 0 && resultSet.wasNull())
+					fee = null;
+
+				int txGroupId = resultSet.getInt(7);
+
+				Integer blockHeight = resultSet.getInt(8);
+				if (blockHeight == 0 && resultSet.wasNull())
+					blockHeight = null;
+
+				ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
+				Integer approvalHeight = resultSet.getInt(10);
+				if (approvalHeight == 0 && resultSet.wasNull())
+					approvalHeight = null;
+
+				BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
+
+				int version = resultSet.getInt(11);
+				int nonce = resultSet.getInt(12);
+				int serviceInt = resultSet.getInt(13);
+				int size = resultSet.getInt(14);
+				boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
+				DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
+				byte[] data = resultSet.getBytes(16);
+				byte[] metadataHash = resultSet.getBytes(17);
+				String nameResult = resultSet.getString(18);
+				String identifierResult = resultSet.getString(19);
+				Method method = Method.valueOf(resultSet.getInt(20));
+				byte[] secret = resultSet.getBytes(21);
+				Compression compression = Compression.valueOf(resultSet.getInt(22));
+				// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
+
+				ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
+						version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
+						compression, data, dataType, metadataHash, null);
+
+				arbitraryTransactionData.add(transactionData);
+			} while (resultSet.next());
+
+			return arbitraryTransactionData;
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+		}
+	}
+
 
 	// Resource related