From fd37c2b76b5e10d7d010703347c0cae7c50e9769 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Mar 2022 16:24:04 +0000 Subject: [PATCH] Moved all online accounts code to a new OnlineAccountsManager controller class There are no logic changes here other than moving performOnlineAccountsTasks() onto its own thread, so that it's not subject to anything that might be slowing down the main controller thread. --- .../api/resource/AddressesResource.java | 6 +- src/main/java/org/qortal/block/Block.java | 14 +- .../org/qortal/controller/BlockMinter.java | 4 +- .../org/qortal/controller/Controller.java | 392 +--------------- .../controller/OnlineAccountsManager.java | 444 ++++++++++++++++++ .../transaction/PresenceTransaction.java | 3 +- .../test/minting/BlocksMintedCountTests.java | 6 +- .../test/minting/DisagreementTests.java | 4 +- 8 files changed, 473 insertions(+), 400 deletions(-) create mode 100644 src/main/java/org/qortal/controller/OnlineAccountsManager.java diff --git a/src/main/java/org/qortal/api/resource/AddressesResource.java b/src/main/java/org/qortal/api/resource/AddressesResource.java index 39c76cf4..b5268db7 100644 --- a/src/main/java/org/qortal/api/resource/AddressesResource.java +++ b/src/main/java/org/qortal/api/resource/AddressesResource.java @@ -30,7 +30,7 @@ import org.qortal.api.Security; import org.qortal.api.model.ApiOnlineAccount; import org.qortal.api.model.RewardShareKeyRequest; import org.qortal.asset.Asset; -import org.qortal.controller.Controller; +import org.qortal.controller.OnlineAccountsManager; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; import org.qortal.data.account.RewardShareData; @@ -156,7 +156,7 @@ public class AddressesResource { ) @ApiErrors({ApiError.PUBLIC_KEY_NOT_FOUND, ApiError.REPOSITORY_ISSUE}) public List getOnlineAccounts() { - List onlineAccounts = Controller.getInstance().getOnlineAccounts(); + List onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts(); // Map OnlineAccountData entries to OnlineAccount via reward-share data try (final Repository repository = RepositoryManager.getRepository()) { @@ -191,7 +191,7 @@ public class AddressesResource { ) @ApiErrors({ApiError.PUBLIC_KEY_NOT_FOUND, ApiError.REPOSITORY_ISSUE}) public List getOnlineAccountsByLevel() { - List onlineAccounts = Controller.getInstance().getOnlineAccounts(); + List onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts(); try (final Repository repository = RepositoryManager.getRepository()) { List onlineAccountLevels = new ArrayList<>(); diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index d6bb11f3..927249fa 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -28,7 +28,7 @@ import org.qortal.asset.Asset; import org.qortal.at.AT; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.block.BlockChain.AccountLevelShareBin; -import org.qortal.controller.Controller; +import org.qortal.controller.OnlineAccountsManager; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; @@ -320,7 +320,7 @@ public class Block { byte[] reference = parentBlockData.getSignature(); // Fetch our list of online accounts - List onlineAccounts = Controller.getInstance().getOnlineAccounts(); + List onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts(); if (onlineAccounts.isEmpty()) { LOGGER.error("No online accounts - not even our own?"); return null; @@ -988,10 +988,10 @@ public class Block { byte[] onlineTimestampBytes = Longs.toByteArray(onlineTimestamp); // If this block is much older than current online timestamp, then there's no point checking current online accounts - List currentOnlineAccounts = onlineTimestamp < NTP.getTime() - Controller.ONLINE_TIMESTAMP_MODULUS + List currentOnlineAccounts = onlineTimestamp < NTP.getTime() - OnlineAccountsManager.ONLINE_TIMESTAMP_MODULUS ? null - : Controller.getInstance().getOnlineAccounts(); - List latestBlocksOnlineAccounts = Controller.getInstance().getLatestBlocksOnlineAccounts(); + : OnlineAccountsManager.getInstance().getOnlineAccounts(); + List latestBlocksOnlineAccounts = OnlineAccountsManager.getInstance().getLatestBlocksOnlineAccounts(); // Extract online accounts' timestamp signatures from block data List onlineAccountsSignatures = BlockTransformer.decodeTimestampSignatures(this.blockData.getOnlineAccountsSignatures()); @@ -1369,7 +1369,7 @@ public class Block { postBlockTidy(); // Give Controller our cached, valid online accounts data (if any) to help reduce CPU load for next block - Controller.getInstance().pushLatestBlocksOnlineAccounts(this.cachedValidOnlineAccounts); + OnlineAccountsManager.getInstance().pushLatestBlocksOnlineAccounts(this.cachedValidOnlineAccounts); // Log some debugging info relating to the block weight calculation this.logDebugInfo(); @@ -1588,7 +1588,7 @@ public class Block { postBlockTidy(); // Remove any cached, valid online accounts data from Controller - Controller.getInstance().popLatestBlocksOnlineAccounts(); + OnlineAccountsManager.getInstance().popLatestBlocksOnlineAccounts(); } protected void orphanTransactionsFromBlock() throws DataException { diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 4369ec91..583facd0 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -110,7 +110,7 @@ public class BlockMinter extends Thread { continue; // No online accounts? (e.g. during startup) - if (Controller.getInstance().getOnlineAccounts().isEmpty()) + if (OnlineAccountsManager.getInstance().getOnlineAccounts().isEmpty()) continue; List mintingAccountsData = repository.getAccountRepository().getMintingAccounts(); @@ -479,7 +479,7 @@ public class BlockMinter extends Thread { throw new DataException("Ignoring attempt to mint testing block for non-test chain!"); // Ensure mintingAccount is 'online' so blocks can be minted - Controller.getInstance().ensureTestingAccountsOnline(mintingAndOnlineAccounts); + OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(mintingAndOnlineAccounts); PrivateKeyAccount mintingAccount = mintingAndOnlineAccounts[0]; diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index b506ccf8..bb89c4a5 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -98,16 +98,6 @@ public class Controller extends Thread { * This mainly exists to stop expired transactions from bloating the list */ public static final long EXPIRED_TRANSACTION_RECHECK_INTERVAL = 10 * 60 * 1000L; // ms - // To do with online accounts list - private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms - private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 1 * 60 * 1000L; // ms - public static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000L; - private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L); - /** How many (latest) blocks' worth of online accounts we cache */ - private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2; - private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L; - - private static volatile boolean isStopping = false; private static BlockMinter blockMinter = null; private static volatile boolean requestSysTrayUpdate = true; @@ -138,8 +128,6 @@ public class Controller extends Thread { private long ntpCheckTimestamp = startTime; // ms private long deleteExpiredTimestamp = startTime + DELETE_EXPIRED_INTERVAL; // ms - private long onlineAccountsTasksTimestamp = startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms - /** Whether we can mint new blocks, as reported by BlockMinter. */ private volatile boolean isMintingPossible = false; @@ -152,11 +140,6 @@ public class Controller extends Thread { /** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly minted block. */ private final ReentrantLock blockchainLock = new ReentrantLock(); - /** Cache of current 'online accounts' */ - List onlineAccounts = new ArrayList<>(); - /** Cache of latest blocks' online accounts */ - Deque> latestBlocksOnlineAccounts = new ArrayDeque<>(MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS); - // Stats @XmlAccessorType(XmlAccessType.FIELD) public static class StatsSnapshot { @@ -469,6 +452,9 @@ public class Controller extends Thread { ArbitraryDataStorageManager.getInstance().start(); ArbitraryDataRenderManager.getInstance().start(); + LOGGER.info("Starting online accounts manager"); + OnlineAccountsManager.getInstance().start(); + // Auto-update service? if (Settings.getInstance().isAutoUpdateEnabled()) { LOGGER.info("Starting auto-update"); @@ -639,12 +625,6 @@ public class Controller extends Thread { deleteExpiredTimestamp = now + DELETE_EXPIRED_INTERVAL; deleteExpiredTransactions(); } - - // Perform tasks to do with managing online accounts list - if (now >= onlineAccountsTasksTimestamp) { - onlineAccountsTasksTimestamp = now + ONLINE_ACCOUNTS_TASKS_INTERVAL; - performOnlineAccountsTasks(); - } } } catch (InterruptedException e) { // Clear interrupted flag so we can shutdown trim threads @@ -1042,6 +1022,9 @@ public class Controller extends Thread { ArbitraryDataStorageManager.getInstance().shutdown(); ArbitraryDataRenderManager.getInstance().shutdown(); + LOGGER.info("Shutting down online accounts manager"); + OnlineAccountsManager.getInstance().shutdown(); + if (blockMinter != null) { LOGGER.info("Shutting down block minter"); blockMinter.shutdown(); @@ -1353,19 +1336,19 @@ public class Controller extends Thread { break; case GET_ONLINE_ACCOUNTS: - onNetworkGetOnlineAccountsMessage(peer, message); + OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsMessage(peer, message); break; case ONLINE_ACCOUNTS: - onNetworkOnlineAccountsMessage(peer, message); + OnlineAccountsManager.getInstance().onNetworkOnlineAccountsMessage(peer, message); break; case GET_ONLINE_ACCOUNTS_V2: - onNetworkGetOnlineAccountsV2Message(peer, message); + OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV2Message(peer, message); break; case ONLINE_ACCOUNTS_V2: - onNetworkOnlineAccountsV2Message(peer, message); + OnlineAccountsManager.getInstance().onNetworkOnlineAccountsV2Message(peer, message); break; case GET_ARBITRARY_DATA: @@ -1745,363 +1728,8 @@ public class Controller extends Thread { } } - private void onNetworkGetOnlineAccountsMessage(Peer peer, Message message) { - GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message; - - List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); - - // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts - List accountsToSend; - synchronized (this.onlineAccounts) { - accountsToSend = new ArrayList<>(this.onlineAccounts); - } - - Iterator iterator = accountsToSend.iterator(); - - SEND_ITERATOR: - while (iterator.hasNext()) { - OnlineAccountData onlineAccountData = iterator.next(); - - for (int i = 0; i < excludeAccounts.size(); ++i) { - OnlineAccountData excludeAccountData = excludeAccounts.get(i); - - if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { - iterator.remove(); - continue SEND_ITERATOR; - } - } - } - - Message onlineAccountsMessage = new OnlineAccountsMessage(accountsToSend); - peer.sendMessage(onlineAccountsMessage); - - LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer)); - } - - private void onNetworkOnlineAccountsMessage(Peer peer, Message message) { - OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message; - - List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); - LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer)); - - try (final Repository repository = RepositoryManager.getRepository()) { - for (OnlineAccountData onlineAccountData : peersOnlineAccounts) - this.verifyAndAddAccount(repository, onlineAccountData); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e); - } - } - - private void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) { - GetOnlineAccountsV2Message getOnlineAccountsMessage = (GetOnlineAccountsV2Message) message; - - List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); - - // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts - List accountsToSend; - synchronized (this.onlineAccounts) { - accountsToSend = new ArrayList<>(this.onlineAccounts); - } - - Iterator iterator = accountsToSend.iterator(); - - SEND_ITERATOR: - while (iterator.hasNext()) { - OnlineAccountData onlineAccountData = iterator.next(); - - for (int i = 0; i < excludeAccounts.size(); ++i) { - OnlineAccountData excludeAccountData = excludeAccounts.get(i); - - if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { - iterator.remove(); - continue SEND_ITERATOR; - } - } - } - - Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend); - peer.sendMessage(onlineAccountsMessage); - - LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer)); - } - - private void onNetworkOnlineAccountsV2Message(Peer peer, Message message) { - OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message; - - List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); - LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer)); - - try (final Repository repository = RepositoryManager.getRepository()) { - for (OnlineAccountData onlineAccountData : peersOnlineAccounts) - this.verifyAndAddAccount(repository, onlineAccountData); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e); - } - } - // Utilities - private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException { - final Long now = NTP.getTime(); - if (now == null) - return; - - PublicKeyAccount otherAccount = new PublicKeyAccount(repository, onlineAccountData.getPublicKey()); - - // Check timestamp is 'recent' here - if (Math.abs(onlineAccountData.getTimestamp() - now) > ONLINE_TIMESTAMP_MODULUS * 2) { - LOGGER.trace(() -> String.format("Rejecting online account %s with out of range timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp())); - return; - } - - // Verify - byte[] data = Longs.toByteArray(onlineAccountData.getTimestamp()); - if (!otherAccount.verify(onlineAccountData.getSignature(), data)) { - LOGGER.trace(() -> String.format("Rejecting invalid online account %s", otherAccount.getAddress())); - return; - } - - // Qortal: check online account is actually reward-share - RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(onlineAccountData.getPublicKey()); - if (rewardShareData == null) { - // Reward-share doesn't even exist - probably not a good sign - LOGGER.trace(() -> String.format("Rejecting unknown online reward-share public key %s", Base58.encode(onlineAccountData.getPublicKey()))); - return; - } - - Account mintingAccount = new Account(repository, rewardShareData.getMinter()); - if (!mintingAccount.canMint()) { - // Minting-account component of reward-share can no longer mint - disregard - LOGGER.trace(() -> String.format("Rejecting online reward-share with non-minting account %s", mintingAccount.getAddress())); - return; - } - - synchronized (this.onlineAccounts) { - OnlineAccountData existingAccountData = this.onlineAccounts.stream().filter(account -> Arrays.equals(account.getPublicKey(), onlineAccountData.getPublicKey())).findFirst().orElse(null); - - if (existingAccountData != null) { - if (existingAccountData.getTimestamp() < onlineAccountData.getTimestamp()) { - this.onlineAccounts.remove(existingAccountData); - - LOGGER.trace(() -> String.format("Updated online account %s with timestamp %d (was %d)", otherAccount.getAddress(), onlineAccountData.getTimestamp(), existingAccountData.getTimestamp())); - } else { - LOGGER.trace(() -> String.format("Not updating existing online account %s", otherAccount.getAddress())); - - return; - } - } else { - LOGGER.trace(() -> String.format("Added online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp())); - } - - this.onlineAccounts.add(onlineAccountData); - } - } - - public void ensureTestingAccountsOnline(PrivateKeyAccount... onlineAccounts) { - if (!BlockChain.getInstance().isTestChain()) { - LOGGER.warn("Ignoring attempt to ensure test account is online for non-test chain!"); - return; - } - - final Long now = NTP.getTime(); - if (now == null) - return; - - final long onlineAccountsTimestamp = Controller.toOnlineAccountTimestamp(now); - byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); - - synchronized (this.onlineAccounts) { - this.onlineAccounts.clear(); - - for (PrivateKeyAccount onlineAccount : onlineAccounts) { - // Check mintingAccount is actually reward-share? - - byte[] signature = onlineAccount.sign(timestampBytes); - byte[] publicKey = onlineAccount.getPublicKey(); - - OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); - this.onlineAccounts.add(ourOnlineAccountData); - } - } - } - - private void performOnlineAccountsTasks() { - final Long now = NTP.getTime(); - if (now == null) - return; - - // Expire old entries - final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD; - synchronized (this.onlineAccounts) { - Iterator iterator = this.onlineAccounts.iterator(); - while (iterator.hasNext()) { - OnlineAccountData onlineAccountData = iterator.next(); - - if (onlineAccountData.getTimestamp() < cutoffThreshold) { - iterator.remove(); - - LOGGER.trace(() -> { - PublicKeyAccount otherAccount = new PublicKeyAccount(null, onlineAccountData.getPublicKey()); - return String.format("Removed expired online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp()); - }); - } - } - } - - // Request data from other peers? - if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) { - List safeOnlineAccounts; - synchronized (this.onlineAccounts) { - safeOnlineAccounts = new ArrayList<>(this.onlineAccounts); - } - - Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts); - Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts); - - Network.getInstance().broadcast(peer -> - peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 - ); - } - - // Refresh our online accounts signatures? - sendOurOnlineAccountsInfo(); - } - - private void sendOurOnlineAccountsInfo() { - final Long now = NTP.getTime(); - if (now != null) { - - List mintingAccounts; - try (final Repository repository = RepositoryManager.getRepository()) { - mintingAccounts = repository.getAccountRepository().getMintingAccounts(); - - // We have no accounts, but don't reset timestamp - if (mintingAccounts.isEmpty()) - return; - - // Only reward-share accounts allowed - Iterator iterator = mintingAccounts.iterator(); - int i = 0; - while (iterator.hasNext()) { - MintingAccountData mintingAccountData = iterator.next(); - - RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey()); - if (rewardShareData == null) { - // Reward-share doesn't even exist - probably not a good sign - iterator.remove(); - continue; - } - - Account mintingAccount = new Account(repository, rewardShareData.getMinter()); - if (!mintingAccount.canMint()) { - // Minting-account component of reward-share can no longer mint - disregard - iterator.remove(); - continue; - } - - if (++i > 2) { - iterator.remove(); - continue; - } - } - } catch (DataException e) { - LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage())); - return; - } - - // 'current' timestamp - final long onlineAccountsTimestamp = Controller.toOnlineAccountTimestamp(now); - boolean hasInfoChanged = false; - - byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); - List ourOnlineAccounts = new ArrayList<>(); - - MINTING_ACCOUNTS: - for (MintingAccountData mintingAccountData : mintingAccounts) { - PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey()); - - byte[] signature = mintingAccount.sign(timestampBytes); - byte[] publicKey = mintingAccount.getPublicKey(); - - // Our account is online - OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); - synchronized (this.onlineAccounts) { - Iterator iterator = this.onlineAccounts.iterator(); - while (iterator.hasNext()) { - OnlineAccountData existingOnlineAccountData = iterator.next(); - - if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) { - // If our online account is already present, with same timestamp, then move on to next mintingAccount - if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp) - continue MINTING_ACCOUNTS; - - // If our online account is already present, but with older timestamp, then remove it - iterator.remove(); - break; - } - } - - this.onlineAccounts.add(ourOnlineAccountData); - } - - LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp)); - ourOnlineAccounts.add(ourOnlineAccountData); - hasInfoChanged = true; - } - - if (!hasInfoChanged) - return; - - Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts); - Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts); - - Network.getInstance().broadcast(peer -> - peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 - ); - - LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); - } - } - - public static long toOnlineAccountTimestamp(long timestamp) { - return (timestamp / ONLINE_TIMESTAMP_MODULUS) * ONLINE_TIMESTAMP_MODULUS; - } - - /** Returns list of online accounts with timestamp recent enough to be considered currently online. */ - public List getOnlineAccounts() { - final long onlineTimestamp = Controller.toOnlineAccountTimestamp(NTP.getTime()); - - synchronized (this.onlineAccounts) { - return this.onlineAccounts.stream().filter(account -> account.getTimestamp() == onlineTimestamp).collect(Collectors.toList()); - } - } - - /** Returns cached, unmodifiable list of latest block's online accounts. */ - public List getLatestBlocksOnlineAccounts() { - synchronized (this.latestBlocksOnlineAccounts) { - return this.latestBlocksOnlineAccounts.peekFirst(); - } - } - - /** Caches list of latest block's online accounts. Typically called by Block.process() */ - public void pushLatestBlocksOnlineAccounts(List latestBlocksOnlineAccounts) { - synchronized (this.latestBlocksOnlineAccounts) { - if (this.latestBlocksOnlineAccounts.size() == MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS) - this.latestBlocksOnlineAccounts.pollLast(); - - this.latestBlocksOnlineAccounts.addFirst(latestBlocksOnlineAccounts == null - ? Collections.emptyList() - : Collections.unmodifiableList(latestBlocksOnlineAccounts)); - } - } - - /** Reverts list of latest block's online accounts. Typically called by Block.orphan() */ - public void popLatestBlocksOnlineAccounts() { - synchronized (this.latestBlocksOnlineAccounts) { - this.latestBlocksOnlineAccounts.pollFirst(); - } - } - /** Returns a list of peers that are not misbehaving, and have a recent block. */ public List getRecentBehavingPeers() { final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java new file mode 100644 index 00000000..07bbb019 --- /dev/null +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -0,0 +1,444 @@ +package org.qortal.controller; + +import com.google.common.primitives.Longs; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.account.Account; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.account.PublicKeyAccount; +import org.qortal.block.BlockChain; +import org.qortal.data.account.MintingAccountData; +import org.qortal.data.account.RewardShareData; +import org.qortal.data.network.OnlineAccountData; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.*; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; + +import java.util.*; +import java.util.stream.Collectors; + +public class OnlineAccountsManager extends Thread { + + private static final Logger LOGGER = LogManager.getLogger(OnlineAccountsManager.class); + + private static OnlineAccountsManager instance; + private volatile boolean isStopping = false; + + // To do with online accounts list + private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms + private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 1 * 60 * 1000L; // ms + public static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000L; + private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L); + /** How many (latest) blocks' worth of online accounts we cache */ + private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2; + private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L; + + private long onlineAccountsTasksTimestamp = Controller.startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms + + /** Cache of current 'online accounts' */ + List onlineAccounts = new ArrayList<>(); + /** Cache of latest blocks' online accounts */ + Deque> latestBlocksOnlineAccounts = new ArrayDeque<>(MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS); + + public OnlineAccountsManager() { + + } + + public static synchronized OnlineAccountsManager getInstance() { + if (instance == null) { + instance = new OnlineAccountsManager(); + } + + return instance; + } + + public void run() { + try { + while (!Controller.isStopping()) { + Thread.sleep(1000L); + + final Long now = NTP.getTime(); + + // Perform tasks to do with managing online accounts list + if (now >= onlineAccountsTasksTimestamp) { + onlineAccountsTasksTimestamp = now + ONLINE_ACCOUNTS_TASKS_INTERVAL; + performOnlineAccountsTasks(); + } + } + } catch (InterruptedException e) { + // Fall through to exit thread + } + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + + // Utilities + + private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException { + final Long now = NTP.getTime(); + if (now == null) + return; + + PublicKeyAccount otherAccount = new PublicKeyAccount(repository, onlineAccountData.getPublicKey()); + + // Check timestamp is 'recent' here + if (Math.abs(onlineAccountData.getTimestamp() - now) > ONLINE_TIMESTAMP_MODULUS * 2) { + LOGGER.trace(() -> String.format("Rejecting online account %s with out of range timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp())); + return; + } + + // Verify + byte[] data = Longs.toByteArray(onlineAccountData.getTimestamp()); + if (!otherAccount.verify(onlineAccountData.getSignature(), data)) { + LOGGER.trace(() -> String.format("Rejecting invalid online account %s", otherAccount.getAddress())); + return; + } + + // Qortal: check online account is actually reward-share + RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(onlineAccountData.getPublicKey()); + if (rewardShareData == null) { + // Reward-share doesn't even exist - probably not a good sign + LOGGER.trace(() -> String.format("Rejecting unknown online reward-share public key %s", Base58.encode(onlineAccountData.getPublicKey()))); + return; + } + + Account mintingAccount = new Account(repository, rewardShareData.getMinter()); + if (!mintingAccount.canMint()) { + // Minting-account component of reward-share can no longer mint - disregard + LOGGER.trace(() -> String.format("Rejecting online reward-share with non-minting account %s", mintingAccount.getAddress())); + return; + } + + synchronized (this.onlineAccounts) { + OnlineAccountData existingAccountData = this.onlineAccounts.stream().filter(account -> Arrays.equals(account.getPublicKey(), onlineAccountData.getPublicKey())).findFirst().orElse(null); + + if (existingAccountData != null) { + if (existingAccountData.getTimestamp() < onlineAccountData.getTimestamp()) { + this.onlineAccounts.remove(existingAccountData); + + LOGGER.trace(() -> String.format("Updated online account %s with timestamp %d (was %d)", otherAccount.getAddress(), onlineAccountData.getTimestamp(), existingAccountData.getTimestamp())); + } else { + LOGGER.trace(() -> String.format("Not updating existing online account %s", otherAccount.getAddress())); + + return; + } + } else { + LOGGER.trace(() -> String.format("Added online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp())); + } + + this.onlineAccounts.add(onlineAccountData); + } + } + + public void ensureTestingAccountsOnline(PrivateKeyAccount... onlineAccounts) { + if (!BlockChain.getInstance().isTestChain()) { + LOGGER.warn("Ignoring attempt to ensure test account is online for non-test chain!"); + return; + } + + final Long now = NTP.getTime(); + if (now == null) + return; + + final long onlineAccountsTimestamp = toOnlineAccountTimestamp(now); + byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); + + synchronized (this.onlineAccounts) { + this.onlineAccounts.clear(); + + for (PrivateKeyAccount onlineAccount : onlineAccounts) { + // Check mintingAccount is actually reward-share? + + byte[] signature = onlineAccount.sign(timestampBytes); + byte[] publicKey = onlineAccount.getPublicKey(); + + OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); + this.onlineAccounts.add(ourOnlineAccountData); + } + } + } + + private void performOnlineAccountsTasks() { + final Long now = NTP.getTime(); + if (now == null) + return; + + // Expire old entries + final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD; + synchronized (this.onlineAccounts) { + Iterator iterator = this.onlineAccounts.iterator(); + while (iterator.hasNext()) { + OnlineAccountData onlineAccountData = iterator.next(); + + if (onlineAccountData.getTimestamp() < cutoffThreshold) { + iterator.remove(); + + LOGGER.trace(() -> { + PublicKeyAccount otherAccount = new PublicKeyAccount(null, onlineAccountData.getPublicKey()); + return String.format("Removed expired online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp()); + }); + } + } + } + + // Request data from other peers? + if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) { + List safeOnlineAccounts; + synchronized (this.onlineAccounts) { + safeOnlineAccounts = new ArrayList<>(this.onlineAccounts); + } + + Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts); + Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts); + + Network.getInstance().broadcast(peer -> + peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 + ); + } + + // Refresh our online accounts signatures? + sendOurOnlineAccountsInfo(); + } + + private void sendOurOnlineAccountsInfo() { + final Long now = NTP.getTime(); + if (now != null) { + + List mintingAccounts; + try (final Repository repository = RepositoryManager.getRepository()) { + mintingAccounts = repository.getAccountRepository().getMintingAccounts(); + + // We have no accounts, but don't reset timestamp + if (mintingAccounts.isEmpty()) + return; + + // Only reward-share accounts allowed + Iterator iterator = mintingAccounts.iterator(); + int i = 0; + while (iterator.hasNext()) { + MintingAccountData mintingAccountData = iterator.next(); + + RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey()); + if (rewardShareData == null) { + // Reward-share doesn't even exist - probably not a good sign + iterator.remove(); + continue; + } + + Account mintingAccount = new Account(repository, rewardShareData.getMinter()); + if (!mintingAccount.canMint()) { + // Minting-account component of reward-share can no longer mint - disregard + iterator.remove(); + continue; + } + + if (++i > 2) { + iterator.remove(); + continue; + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage())); + return; + } + + // 'current' timestamp + final long onlineAccountsTimestamp = toOnlineAccountTimestamp(now); + boolean hasInfoChanged = false; + + byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); + List ourOnlineAccounts = new ArrayList<>(); + + MINTING_ACCOUNTS: + for (MintingAccountData mintingAccountData : mintingAccounts) { + PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey()); + + byte[] signature = mintingAccount.sign(timestampBytes); + byte[] publicKey = mintingAccount.getPublicKey(); + + // Our account is online + OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); + synchronized (this.onlineAccounts) { + Iterator iterator = this.onlineAccounts.iterator(); + while (iterator.hasNext()) { + OnlineAccountData existingOnlineAccountData = iterator.next(); + + if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) { + // If our online account is already present, with same timestamp, then move on to next mintingAccount + if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp) + continue MINTING_ACCOUNTS; + + // If our online account is already present, but with older timestamp, then remove it + iterator.remove(); + break; + } + } + + this.onlineAccounts.add(ourOnlineAccountData); + } + + LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp)); + ourOnlineAccounts.add(ourOnlineAccountData); + hasInfoChanged = true; + } + + if (!hasInfoChanged) + return; + + Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts); + Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts); + + Network.getInstance().broadcast(peer -> + peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1 + ); + + LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); + } + } + + public static long toOnlineAccountTimestamp(long timestamp) { + return (timestamp / ONLINE_TIMESTAMP_MODULUS) * ONLINE_TIMESTAMP_MODULUS; + } + + /** Returns list of online accounts with timestamp recent enough to be considered currently online. */ + public List getOnlineAccounts() { + final long onlineTimestamp = toOnlineAccountTimestamp(NTP.getTime()); + + synchronized (this.onlineAccounts) { + return this.onlineAccounts.stream().filter(account -> account.getTimestamp() == onlineTimestamp).collect(Collectors.toList()); + } + } + + + /** Returns cached, unmodifiable list of latest block's online accounts. */ + public List getLatestBlocksOnlineAccounts() { + synchronized (this.latestBlocksOnlineAccounts) { + return this.latestBlocksOnlineAccounts.peekFirst(); + } + } + + /** Caches list of latest block's online accounts. Typically called by Block.process() */ + public void pushLatestBlocksOnlineAccounts(List latestBlocksOnlineAccounts) { + synchronized (this.latestBlocksOnlineAccounts) { + if (this.latestBlocksOnlineAccounts.size() == MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS) + this.latestBlocksOnlineAccounts.pollLast(); + + this.latestBlocksOnlineAccounts.addFirst(latestBlocksOnlineAccounts == null + ? Collections.emptyList() + : Collections.unmodifiableList(latestBlocksOnlineAccounts)); + } + } + + /** Reverts list of latest block's online accounts. Typically called by Block.orphan() */ + public void popLatestBlocksOnlineAccounts() { + synchronized (this.latestBlocksOnlineAccounts) { + this.latestBlocksOnlineAccounts.pollFirst(); + } + } + + + // Network handlers + + public void onNetworkGetOnlineAccountsMessage(Peer peer, Message message) { + GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message; + + List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); + + // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts + List accountsToSend; + synchronized (this.onlineAccounts) { + accountsToSend = new ArrayList<>(this.onlineAccounts); + } + + Iterator iterator = accountsToSend.iterator(); + + SEND_ITERATOR: + while (iterator.hasNext()) { + OnlineAccountData onlineAccountData = iterator.next(); + + for (int i = 0; i < excludeAccounts.size(); ++i) { + OnlineAccountData excludeAccountData = excludeAccounts.get(i); + + if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { + iterator.remove(); + continue SEND_ITERATOR; + } + } + } + + Message onlineAccountsMessage = new OnlineAccountsMessage(accountsToSend); + peer.sendMessage(onlineAccountsMessage); + + LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer)); + } + + public void onNetworkOnlineAccountsMessage(Peer peer, Message message) { + OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message; + + List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); + LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer)); + + try (final Repository repository = RepositoryManager.getRepository()) { + for (OnlineAccountData onlineAccountData : peersOnlineAccounts) + this.verifyAndAddAccount(repository, onlineAccountData); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e); + } + } + + public void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) { + GetOnlineAccountsV2Message getOnlineAccountsMessage = (GetOnlineAccountsV2Message) message; + + List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); + + // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts + List accountsToSend; + synchronized (this.onlineAccounts) { + accountsToSend = new ArrayList<>(this.onlineAccounts); + } + + Iterator iterator = accountsToSend.iterator(); + + SEND_ITERATOR: + while (iterator.hasNext()) { + OnlineAccountData onlineAccountData = iterator.next(); + + for (int i = 0; i < excludeAccounts.size(); ++i) { + OnlineAccountData excludeAccountData = excludeAccounts.get(i); + + if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { + iterator.remove(); + continue SEND_ITERATOR; + } + } + } + + Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend); + peer.sendMessage(onlineAccountsMessage); + + LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer)); + } + + public void onNetworkOnlineAccountsV2Message(Peer peer, Message message) { + OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message; + + List peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts(); + LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer)); + + try (final Repository repository = RepositoryManager.getRepository()) { + for (OnlineAccountData onlineAccountData : peersOnlineAccounts) + this.verifyAndAddAccount(repository, onlineAccountData); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e); + } + } +} diff --git a/src/main/java/org/qortal/transaction/PresenceTransaction.java b/src/main/java/org/qortal/transaction/PresenceTransaction.java index e1350dfe..9bdbe3c7 100644 --- a/src/main/java/org/qortal/transaction/PresenceTransaction.java +++ b/src/main/java/org/qortal/transaction/PresenceTransaction.java @@ -13,6 +13,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.account.Account; import org.qortal.controller.Controller; +import org.qortal.controller.OnlineAccountsManager; import org.qortal.controller.tradebot.TradeBot; import org.qortal.crosschain.ACCT; import org.qortal.crosschain.SupportedBlockchain; @@ -48,7 +49,7 @@ public class PresenceTransaction extends Transaction { REWARD_SHARE(0) { @Override public long getLifetime() { - return Controller.ONLINE_TIMESTAMP_MODULUS; + return OnlineAccountsManager.ONLINE_TIMESTAMP_MODULUS; } }, TRADE_BOT(1) { diff --git a/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java b/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java index 88f63ddf..f318a667 100644 --- a/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java +++ b/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java @@ -8,7 +8,7 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.controller.BlockMinter; -import org.qortal.controller.Controller; +import org.qortal.controller.OnlineAccountsManager; import org.qortal.data.account.RewardShareData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -77,7 +77,7 @@ public class BlocksMintedCountTests extends Common { assertNotNull(testRewardShareData); // Create signed timestamps - Controller.getInstance().ensureTestingAccountsOnline(mintingAccount, testRewardShareAccount); + OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(mintingAccount, testRewardShareAccount); // Even though Alice features in two online reward-shares, she should only gain +1 blocksMinted // Bob only features in one online reward-share, so should also only gain +1 blocksMinted @@ -87,7 +87,7 @@ public class BlocksMintedCountTests extends Common { private void testRewardShare(Repository repository, PrivateKeyAccount testRewardShareAccount, int aliceDelta, int bobDelta) throws DataException { // Create signed timestamps - Controller.getInstance().ensureTestingAccountsOnline(testRewardShareAccount); + OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(testRewardShareAccount); testRewardShareRetainingTimestamps(repository, testRewardShareAccount, aliceDelta, bobDelta); } diff --git a/src/test/java/org/qortal/test/minting/DisagreementTests.java b/src/test/java/org/qortal/test/minting/DisagreementTests.java index 4dea75c7..d32256b7 100644 --- a/src/test/java/org/qortal/test/minting/DisagreementTests.java +++ b/src/test/java/org/qortal/test/minting/DisagreementTests.java @@ -11,7 +11,7 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.controller.BlockMinter; -import org.qortal.controller.Controller; +import org.qortal.controller.OnlineAccountsManager; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; @@ -73,7 +73,7 @@ public class DisagreementTests extends Common { assertNotNull(testRewardShareData); // Create signed timestamps - Controller.getInstance().ensureTestingAccountsOnline(mintingAccount, testRewardShareAccount); + OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(mintingAccount, testRewardShareAccount); // Mint another block BlockMinter.mintTestingBlockRetainingTimestamps(repository, mintingAccount);