Merge remote-tracking branch 'kenny/master' into master-kenny3

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -2089,7 +2089,7 @@ public String finalizeUpload(
 	
 			} catch (IOException e) {
 				// Streaming errors should not rethrow — just log
-				LOGGER.warn(String.format("Streaming error for %s %s: %s", service, name, e.getMessage()), e);
+				LOGGER.warn(String.format("Streaming error for %s %s: %s", service, name, e.getMessage()));
 			}
 	
 		} catch (IOException | ApiException | DataException e) {

src/main/java/org/qortal/api/restricted/resource/AdminResource.java

@@ -1092,25 +1092,4 @@ public class AdminResource {
 
 		return info;
 	}
-
-	@GET
-	@Path("/dbstates")
-	@Operation(
-			summary = "Get DB States",
-			description = "Get DB States",
-			responses = {
-					@ApiResponse(
-							content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = DbConnectionInfo.class)))
-					)
-			}
-	)
-	public List<DbConnectionInfo> getDbConnectionsStates() {
-
-		try {
-			return Controller.REPOSITORY_FACTORY.getDbConnectionsStates();
-		} catch (Exception e) {
-			LOGGER.error(e.getMessage(), e);
-			return new ArrayList<>(0);
-		}
-	}
 }

src/main/java/org/qortal/arbitrary/misc/Service.java

@@ -62,7 +62,17 @@ public enum Service {
 
             // Custom validation function to require an index HTML file in the root directory
             List<String> fileNames = ArbitraryDataRenderer.indexFiles();
-            String[] files = path.toFile().list();
+            List<String> files;
+
+            // single files are paackaged differently
+            if( path.toFile().isFile() ) {
+                files = new ArrayList<>(1);
+                files.add(path.getFileName().toString());
+            }
+            else {
+                files = new ArrayList<>(Arrays.asList(path.toFile().list()));
+            }
+
             if (files != null) {
                 for (String file : files) {
                     Path fileName = Paths.get(file).getFileName();

src/main/java/org/qortal/block/Block.java

@@ -1735,7 +1735,7 @@ public class Block {
 
 			for (int newLevel = maximumLevel; newLevel >= 0; --newLevel)
 				if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) {
-					if (newLevel > accountData.getLevel()) {
+					if (newLevel != accountData.getLevel()) {
 						// Account has increased in level!
 						accountData.setLevel(newLevel);
 						bumpedAccounts.put(accountData.getAddress(), newLevel);
@@ -2141,7 +2141,7 @@ public class Block {
 
 			int blocksMintedAdjustment
 				=
-				(this.blockData.getHeight() > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight())
+				(this.blockData.getHeight() -1 > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight())
 				?
 				0
 				:
@@ -2151,7 +2151,7 @@ public class Block {
 
 			for (int newLevel = maximumLevel; newLevel >= 0; --newLevel)
 				if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) {
-					if (newLevel < accountData.getLevel()) {
+					if (newLevel != accountData.getLevel()) {
 						// Account has decreased in level!
 						accountData.setLevel(newLevel);
 						repository.getAccountRepository().setLevel(accountData);

src/main/java/org/qortal/controller/Controller.java

@@ -76,8 +76,6 @@ import java.util.stream.Stream;
 
 public class Controller extends Thread {
 
-	public static HSQLDBRepositoryFactory REPOSITORY_FACTORY;
-
 	static {
 		// This must go before any calls to LogManager/Logger
 		System.setProperty("log4j2.formatMsgNoLookups", "true");
@@ -411,8 +409,8 @@ public class Controller extends Thread {
 
 		LOGGER.info("Starting repository");
 		try {
-			REPOSITORY_FACTORY = new HSQLDBRepositoryFactory(getRepositoryUrl());
+			HSQLDBRepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl());
-			RepositoryManager.setRepositoryFactory(REPOSITORY_FACTORY);
+			RepositoryManager.setRepositoryFactory(repositoryFactory);
 			RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
 
 			try (final Repository repository = RepositoryManager.getRepository()) {

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -47,15 +47,15 @@ public class ArbitraryDataStorageManager extends Thread {
 
     private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
 
-    /** Treat storage as full at 90% usage, to reduce risk of going over the limit.
+    /** Treat storage as full at 80% usage, to reduce risk of going over the limit.
      * This is necessary because we don't calculate total storage values before every write.
      * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit.
      * This must be lower than DELETION_THRESHOLD. */
-    private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90%
+    private static final double STORAGE_FULL_THRESHOLD = 0.8f; // 80%
 
-    /** Start deleting files once we reach 98% usage.
+    /** Start deleting files once we reach 90% usage.
      * This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */
-    public static final double DELETION_THRESHOLD = 0.98f; // 98%
+    public static final double DELETION_THRESHOLD = 0.9f; // 90%
 
     private static final long PER_NAME_STORAGE_MULTIPLIER = 4L;
 

src/main/java/org/qortal/repository/Repository.java

@@ -1,6 +1,7 @@
 package org.qortal.repository;
 
 import java.io.IOException;
+import java.sql.Connection;
 import java.util.concurrent.TimeoutException;
 
 public interface Repository extends AutoCloseable {
@@ -62,4 +63,5 @@ public interface Repository extends AutoCloseable {
 
 	public static void attemptRecovery(String connectionUrl, String name) throws DataException {}
 
+    public Connection getConnection();
 }

src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java

@@ -468,7 +468,7 @@ public class HSQLDBCacheUtils {
 
                 Thread.currentThread().setName(DB_CACHE_TIMER_TASK);
 
-                try (final HSQLDBRepository respository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) {
+                try (final Repository respository = RepositoryManager.getRepository()) {
                     fillCache(ArbitraryResourceCache.getInstance(), respository);
                 }
                 catch( DataException e ) {
@@ -611,7 +611,7 @@ public class HSQLDBCacheUtils {
     private static int recordCurrentBalances(ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight) {
         int currentHeight;
 
-        try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) {
+        try (final Repository repository = RepositoryManager.getRepository()) {
 
             // get current balances
             List<AccountBalanceData> accountBalances = getAccountBalances(repository);
@@ -675,7 +675,7 @@ public class HSQLDBCacheUtils {
      * @param cache the cache to fill
      * @param repository the data source to fill the cache with
      */
-    public static void fillCache(ArbitraryResourceCache cache, HSQLDBRepository repository) {
+    public static void fillCache(ArbitraryResourceCache cache, Repository repository) {
 
         try {
             // ensure all data is committed in, before we query it
@@ -713,7 +713,7 @@ public class HSQLDBCacheUtils {
      *
      * @throws SQLException
      */
-    private static void fillNamepMap(ConcurrentHashMap<String, Integer> levelByName, HSQLDBRepository repository ) throws SQLException {
+    private static void fillNamepMap(ConcurrentHashMap<String, Integer> levelByName, Repository repository ) throws SQLException {
 
         StringBuilder sql = new StringBuilder(512);
 
@@ -721,7 +721,7 @@ public class HSQLDBCacheUtils {
         sql.append("FROM NAMES ");
         sql.append("INNER JOIN ACCOUNTS on owner = account ");
 
-        Statement statement = repository.connection.createStatement();
+        Statement statement = repository.getConnection().createStatement();
 
         ResultSet resultSet = statement.executeQuery(sql.toString());
 
@@ -744,7 +744,7 @@ public class HSQLDBCacheUtils {
      * @return the resources
      * @throws SQLException
      */
-    private static List<ArbitraryResourceData> getResources( HSQLDBRepository repository) throws SQLException {
+    private static List<ArbitraryResourceData> getResources( Repository repository) throws SQLException {
 
         List<ArbitraryResourceData> resources = new ArrayList<>();
 
@@ -756,7 +756,7 @@ public class HSQLDBCacheUtils {
         sql.append("LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) WHERE name IS NOT NULL");
 
         List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
-        Statement statement = repository.connection.createStatement();
+        Statement statement = repository.getConnection().createStatement();
 
         ResultSet resultSet = statement.executeQuery(sql.toString());
 
@@ -822,7 +822,7 @@ public class HSQLDBCacheUtils {
         return resources;
     }
 
-    public static List<AccountBalanceData> getAccountBalances(HSQLDBRepository repository) {
+    public static List<AccountBalanceData> getAccountBalances(Repository repository) {
 
         StringBuilder sql = new StringBuilder();
 
@@ -836,7 +836,7 @@ public class HSQLDBCacheUtils {
         LOGGER.info( "Getting account balances ...");
 
         try {
-            Statement statement = repository.connection.createStatement();
+            Statement statement = repository.getConnection().createStatement();
 
             ResultSet resultSet = statement.executeQuery(sql.toString());
 

src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java

@@ -174,6 +174,11 @@ public class HSQLDBRepository implements Repository {
 
 	// Transaction COMMIT / ROLLBACK / savepoints
 
+	@Override
+	public Connection getConnection() {
+		return this.connection;
+	}
+
 	@Override
 	public void saveChanges() throws DataException {
 		long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis();

src/main/java/org/qortal/settings/Settings.java

@@ -756,14 +756,14 @@ public class Settings {
 	private void setAdditionalDefaults() {
 		// Populate defaults for maxThreadsPerMessageType. If any are specified in settings.json, they will take priority.
 		maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE", 5));
-		maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 5));
+		maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 20));
 		maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA", 5));
 		maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA", 5));
 		maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE_LIST", 50));
 		maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE_LIST", 50));
 		maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_SIGNATURES", 5));
 		maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_METADATA", 5));
-		maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 50));
+		maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 100));
 		maxThreadsPerMessageType.add(new ThreadLimit("GET_TRANSACTION", 50));
 		maxThreadsPerMessageType.add(new ThreadLimit("TRANSACTION_SIGNATURES", 50));
 		maxThreadsPerMessageType.add(new ThreadLimit("TRADE_PRESENCES", 50));

src/main/java/org/qortal/transaction/BuyNameTransaction.java

@@ -102,7 +102,7 @@ public class BuyNameTransaction extends Transaction {
 			return ValidationResult.INVALID_AMOUNT;
 
 		// Check buyer has enough funds
-		if (buyer.getConfirmedBalance(Asset.QORT) < this.buyNameTransactionData.getFee())
+		if (buyer.getConfirmedBalance(Asset.QORT) < this.buyNameTransactionData.getFee() + this.buyNameTransactionData.getAmount())
 			return ValidationResult.NO_BALANCE;
 
 		return ValidationResult.OK;

src/main/java/org/qortal/utils/ArbitraryIndexUtils.java

@@ -53,7 +53,7 @@ public class ArbitraryIndexUtils {
                 try {
                     fillCache(IndexCache.getInstance());
                 } catch (IOException | DataException e) {
-                    LOGGER.error(e.getMessage(), e);
+                    LOGGER.warn(e.getMessage());
                 }
             }
         };
@@ -111,6 +111,8 @@ public class ArbitraryIndexUtils {
 
                         indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1), indexResource.identifier ));
                     }
+                } catch (MissingDataException e) {
+                    LOGGER.warn( e.getMessage() );
                 } catch (InvalidFormatException e) {
                    LOGGER.debug("invalid format, skipping: " + indexResource);
                 } catch (UnrecognizedPropertyException e) {
@@ -191,7 +193,7 @@ public class ArbitraryIndexUtils {
         }
     }
 
-    public static String getJson(String name, String identifier) throws IOException {
+    public static String getJson(String name, String identifier) throws IOException, MissingDataException {
 
         try {
             ArbitraryDataReader arbitraryDataReader
@@ -209,7 +211,7 @@ public class ArbitraryIndexUtils {
                     } catch (MissingDataException e) {
                         if (attempts > maxAttempts) {
                             // Give up after 5 attempts
-                            throw new IOException("Data unavailable. Please try again later.");
+                            throw e;
                         }
                     }
                 }

src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java

@@ -4,8 +4,10 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.qortal.account.PrivateKeyAccount;
+import org.qortal.block.BlockChain;
 import org.qortal.controller.BlockMinter;
 import org.qortal.controller.OnlineAccountsManager;
+import org.qortal.data.account.AccountData;
 import org.qortal.data.account.RewardShareData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
@@ -15,8 +17,9 @@ import org.qortal.test.common.BlockUtils;
 import org.qortal.test.common.Common;
 import org.qortal.test.common.TestAccount;
 
-import static org.junit.Assert.assertEquals;
+import java.util.List;
-import static org.junit.Assert.assertNotNull;
+
+import static org.junit.Assert.*;
 
 public class BlocksMintedCountTests extends Common {
 
@@ -85,6 +88,121 @@ public class BlocksMintedCountTests extends Common {
 		}
 	}
 
+	@Test
+	public void testLevelSetting() {
+
+		boolean exceptionThrown = false;
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+
+			// get the Alice's reward share account
+			PrivateKeyAccount aliceMintingAccount = Common.getTestAccount(repository, "alice-reward-share");
+
+			// give Alice an 8 blocks minted adjustment
+			int blocksMintedAdjustmentForAlice = 8;
+			adjustMintingData(repository, "alice", blocksMintedAdjustmentForAlice);
+
+			// Confirm reward-share exists
+			RewardShareData aliceRewardShareData = repository.getAccountRepository().getRewardShare(aliceMintingAccount.getPublicKey());
+			assertNotNull(aliceRewardShareData);
+
+			// mint 40 blocks
+			for( int i = 0; i < 40; i++ ) {
+				// Create signed timestamps
+				OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(aliceMintingAccount);
+
+				// Mint another block
+				BlockMinter.mintTestingBlockRetainingTimestamps(repository, aliceMintingAccount);
+
+				// assert Alice's minting data after another block minted
+				assertMintingData(repository, "alice", blocksMintedAdjustmentForAlice);
+
+				// orphan the block
+				BlockUtils.orphanLastBlock(repository);
+
+				// assert the orphaning
+				assertMintingData(repository, "alice", blocksMintedAdjustmentForAlice);
+
+				// mint another block to reverse the orpaning
+				BlockMinter.mintTestingBlockRetainingTimestamps(repository, aliceMintingAccount);
+			}
+		}
+		catch (DataException e) {
+			exceptionThrown = true;
+		}
+
+		assertFalse(exceptionThrown);
+	}
+
+	/**
+	 * Assert Minting Data
+	 *
+	 * @param repository the data repository
+	 * @param name the name of the minting account
+	 * @param adjustment the blocks minted adjustment
+	 *
+	 * @throws DataException
+	 */
+	private static void assertMintingData(Repository repository, String name, int adjustment ) throws DataException {
+
+		// get the test account data
+		TestAccount testAccount = Common.getTestAccount(repository, name);
+		AccountData testAccountData = repository.getAccountRepository().getAccount(testAccount.getAddress());
+
+		List<Integer> blocksNeededByLevel = BlockChain.getInstance().getBlocksNeededByLevel();
+
+		// determine current height and adjustment ability
+		int height = repository.getBlockRepository().getBlockchainHeight();
+		int adjustmentRemovalHeight = BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight();
+		boolean isAdjustingEnabled = height <= adjustmentRemovalHeight;
+
+		// initialize loop variables
+		int blocksLeft;
+
+		if( isAdjustingEnabled )
+			blocksLeft = testAccountData.getBlocksMinted() + adjustment;
+		else
+			blocksLeft = testAccountData.getBlocksMinted();
+
+		int index = 0;
+		int expectedLevel = 0;
+
+		// update expected level based on the blocks needed by level list entries
+		while( blocksNeededByLevel.size() > index ) {
+
+			Integer blocksNeededByThisLevel = blocksNeededByLevel.get(index);
+			if( blocksNeededByThisLevel <= blocksLeft ) {
+				expectedLevel++;
+				blocksLeft -= blocksNeededByThisLevel;
+			}
+			else {
+				break;
+			}
+
+			index++;
+		}
+
+		// print and assert the expected and derived numbers
+		System.out.println(String.format("height = %s,expectedLevel = %s, adjustment = %s, blocksMinted = %s", height, expectedLevel, adjustment, testAccountData.getBlocksMinted()) );
+		assertEquals( expectedLevel, testAccountData.getLevel() );
+	}
+
+	/**
+	 * Adjust Minting Data
+	 *
+	 * @param repository the data repository
+	 * @param name the name of the account to adjust
+	 * @param blocksMintedAdjustment the number of blocks to adjust
+	 *
+	 * @throws DataException
+	 */
+	private static void adjustMintingData(Repository repository, String name, int blocksMintedAdjustment) throws DataException {
+		TestAccount testAccount = Common.getTestAccount(repository, name);
+		AccountData testAccountData = repository.getAccountRepository().getAccount(testAccount.getAddress());
+		testAccountData.setBlocksMintedAdjustment(blocksMintedAdjustment);
+		repository.getAccountRepository().setBlocksMintedAdjustment(testAccountData);
+	}
+
 	private void testRewardShare(Repository repository, PrivateKeyAccount testRewardShareAccount, int aliceDelta, int bobDelta) throws DataException {
 		// Create signed timestamps
 		OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(testRewardShareAccount);
@@ -124,5 +242,4 @@ public class BlocksMintedCountTests extends Common {
 		TestAccount testAccount = Common.getTestAccount(repository, name);
 		return repository.getAccountRepository().getAccount(testAccount.getAddress()).getBlocksMinted();
 	}
-
 }

src/test/resources/test-chain-v2.json

@@ -116,7 +116,7 @@
 		"nullGroupMembershipHeight": 20,
 		"adminQueryFixHeight": 9999999999999,
 		"multipleNamesPerAccountHeight": 10,
-		"mintedBlocksAdjustmentRemovalHeight": 9999999999999
+		"mintedBlocksAdjustmentRemovalHeight": 27
 	},
 	"genesisInfo": {
 		"version": 4,