From 17ae7acc6df73c1da54bdd8b6b580a7e9759f0c0 Mon Sep 17 00:00:00 2001 From: catbref Date: Fri, 25 Sep 2020 15:25:57 +0100 Subject: [PATCH] Reduce DB storage of AT states Drop created_when column from ATStates as it never changes and can be fetched from ATs table. This takes about 50s on a fast machine. Correspondingly rebuild height-based index on ATStates. This takes about 3 minutes on a fast machine. Modify AT-related repository methods and callers. Aggressively remove 'old' (> 2 weeks) actual AT state binary data, leaving only the hash in DB (for syncing purposes). Seems to keep up with syncing from another node on localhost. --- .../api/websocket/TradeOffersWebSocket.java | 2 +- src/main/java/org/qortal/at/AT.java | 5 +- .../org/qortal/controller/Controller.java | 50 +++++++++++ .../java/org/qortal/crosschain/BTCACCT.java | 10 +-- .../java/org/qortal/data/at/ATStateData.java | 14 +-- .../org/qortal/repository/ATRepository.java | 6 ++ .../repository/hsqldb/HSQLDBATRepository.java | 89 +++++++++++++------ .../hsqldb/HSQLDBDatabaseUpdates.java | 12 +++ .../java/org/qortal/settings/Settings.java | 6 ++ 9 files changed, 149 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java index 740d7f5d..a2cf3cac 100644 --- a/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java +++ b/src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java @@ -256,7 +256,7 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener { if (crossChainTradeData.mode == BTCACCT.Mode.OFFERING) // We want when trade was created, not when it was last updated - atStateTimestamp = atState.getCreation(); + atStateTimestamp = crossChainTradeData.creationTimestamp; else atStateTimestamp = timestamp != null ? timestamp : repository.getBlockRepository().getTimestampFromHeight(atState.getHeight()); diff --git a/src/main/java/org/qortal/at/AT.java b/src/main/java/org/qortal/at/AT.java index bae1593a..e82ab14e 100644 --- a/src/main/java/org/qortal/at/AT.java +++ b/src/main/java/org/qortal/at/AT.java @@ -61,7 +61,7 @@ public class AT { byte[] stateData = machineState.toBytes(); byte[] stateHash = Crypto.digest(stateData); - this.atStateData = new ATStateData(atAddress, height, creation, stateData, stateHash, 0L, true); + this.atStateData = new ATStateData(atAddress, height, stateData, stateHash, 0L, true); } // Getters / setters @@ -107,12 +107,11 @@ public class AT { throw new DataException(String.format("Uncaught exception while running AT '%s'", atAddress), e); } - long creation = this.atData.getCreation(); byte[] stateData = state.toBytes(); byte[] stateHash = Crypto.digest(stateData); long atFees = api.calcFinalFees(state); - this.atStateData = new ATStateData(atAddress, blockHeight, creation, stateData, stateHash, atFees, false); + this.atStateData = new ATStateData(atAddress, blockHeight, stateData, stateHash, atFees, false); return api.getTransactions(); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 4e9c6e76..2683fb2d 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -111,6 +111,8 @@ public class Controller extends Thread { private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms + private static final long TRIM_AT_STATES_INTERVAL = 2 * 1000L; // ms + private static final int TRIM_AT_BATCH_SIZE = 200; // blocks // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms @@ -138,6 +140,8 @@ public class Controller extends Thread { private long repositoryBackupTimestamp = startTime; // ms private long ntpCheckTimestamp = startTime; // ms private long deleteExpiredTimestamp = startTime + DELETE_EXPIRED_INTERVAL; // ms + private long trimAtStatesTimestamp = startTime + TRIM_AT_STATES_INTERVAL; // ms + private Integer trimAtStatesStartHeight = null; private long onlineAccountsTasksTimestamp = startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms /** Whether we can mint new blocks, as reported by BlockMinter. */ @@ -483,6 +487,11 @@ public class Controller extends Thread { onlineAccountsTasksTimestamp = now + ONLINE_ACCOUNTS_TASKS_INTERVAL; performOnlineAccountsTasks(); } + + if (now >= trimAtStatesTimestamp) { + trimAtStatesTimestamp = now + TRIM_AT_STATES_INTERVAL; + trimAtStates(); + } } } catch (InterruptedException e) { // Fall-through to exit @@ -1427,6 +1436,47 @@ public class Controller extends Thread { } } + private void trimAtStates() { + if (this.getChainTip() == null) + return; + + try (final Repository repository = RepositoryManager.tryRepository()) { + if (repository == null) + return; + + if (trimAtStatesStartHeight == null) { + trimAtStatesStartHeight = repository.getATRepository().findFirstTrimmableStateHeight(); + // The above will probably take enough time by itself + return; + } + + long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); + // We want to keep AT states near the tip of our copy of blockchain so we can process/orphan nearby blocks + long chainTrimmableTimestamp = this.getChainTip().getTimestamp() - Settings.getInstance().getAtStatesMaxLifetime(); + + long upperTrimmableTimestamp = Math.min(currentTrimmableTimestamp, chainTrimmableTimestamp); + + int upperTrimmableHeight = repository.getBlockRepository().getHeightFromTimestamp(upperTrimmableTimestamp); + int upperBatchHeight = Math.min(trimAtStatesStartHeight + TRIM_AT_BATCH_SIZE, upperTrimmableHeight); + + if (trimAtStatesStartHeight >= upperBatchHeight) + return; + + int numAtStatesTrimmed = repository.getATRepository().trimAtStates(trimAtStatesStartHeight, upperBatchHeight); + repository.saveChanges(); + + if (numAtStatesTrimmed > 0) { + LOGGER.debug(() -> String.format("Trimmed %d AT state%s between blocks %d and %d", + numAtStatesTrimmed, (numAtStatesTrimmed != 1 ? "s" : ""), + trimAtStatesStartHeight, upperBatchHeight)); + } else { + trimAtStatesStartHeight = upperBatchHeight; + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to trim AT states: %s", e.getMessage())); + } + } + private void sendOurOnlineAccountsInfo() { final Long now = NTP.getTime(); if (now == null) diff --git a/src/main/java/org/qortal/crosschain/BTCACCT.java b/src/main/java/org/qortal/crosschain/BTCACCT.java index f3db8587..1e803c52 100644 --- a/src/main/java/org/qortal/crosschain/BTCACCT.java +++ b/src/main/java/org/qortal/crosschain/BTCACCT.java @@ -611,7 +611,7 @@ public class BTCACCT { */ public static CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); } /** @@ -622,8 +622,8 @@ public class BTCACCT { * @throws DataException */ public static CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { - byte[] creatorPublicKey = repository.getATRepository().getCreatorPublicKey(atStateData.getATAddress()); - return populateTradeData(repository, creatorPublicKey, atStateData); + ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); } /** @@ -633,7 +633,7 @@ public class BTCACCT { * @param atAddress * @throws DataException */ - public static CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, ATStateData atStateData) throws DataException { + public static CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -641,7 +641,7 @@ public class BTCACCT { tradeData.qortalAtAddress = atAddress; tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); - tradeData.creationTimestamp = atStateData.getCreation(); + tradeData.creationTimestamp = creationTimestamp; Account atAccount = new Account(repository, atAddress); tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); diff --git a/src/main/java/org/qortal/data/at/ATStateData.java b/src/main/java/org/qortal/data/at/ATStateData.java index b8c13e0d..e689f5ae 100644 --- a/src/main/java/org/qortal/data/at/ATStateData.java +++ b/src/main/java/org/qortal/data/at/ATStateData.java @@ -5,7 +5,6 @@ public class ATStateData { // Properties private String ATAddress; private Integer height; - private Long creation; private byte[] stateData; private byte[] stateHash; private Long fees; @@ -14,10 +13,9 @@ public class ATStateData { // Constructors /** Create new ATStateData */ - public ATStateData(String ATAddress, Integer height, Long creation, byte[] stateData, byte[] stateHash, Long fees, boolean isInitial) { + public ATStateData(String ATAddress, Integer height, byte[] stateData, byte[] stateHash, Long fees, boolean isInitial) { this.ATAddress = ATAddress; this.height = height; - this.creation = creation; this.stateData = stateData; this.stateHash = stateHash; this.fees = fees; @@ -26,21 +24,21 @@ public class ATStateData { /** For recreating per-block ATStateData from repository where not all info is needed */ public ATStateData(String ATAddress, int height, byte[] stateHash, Long fees, boolean isInitial) { - this(ATAddress, height, null, null, stateHash, fees, isInitial); + this(ATAddress, height, null, stateHash, fees, isInitial); } /** For creating ATStateData from serialized bytes when we don't have all the info */ public ATStateData(String ATAddress, byte[] stateHash) { // This won't ever be initial AT state from deployment as that's never serialized over the network, // but generated when the DeployAtTransaction is processed locally. - this(ATAddress, null, null, null, stateHash, null, false); + this(ATAddress, null, null, stateHash, null, false); } /** For creating ATStateData from serialized bytes when we don't have all the info */ public ATStateData(String ATAddress, byte[] stateHash, Long fees) { // This won't ever be initial AT state from deployment as that's never serialized over the network, // but generated when the DeployAtTransaction is processed locally. - this(ATAddress, null, null, null, stateHash, fees, false); + this(ATAddress, null, null, stateHash, fees, false); } // Getters / setters @@ -58,10 +56,6 @@ public class ATStateData { this.height = height; } - public Long getCreation() { - return this.creation; - } - public byte[] getStateData() { return this.stateData; } diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index f3c2b16d..9abe8c3d 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -87,6 +87,12 @@ public interface ATRepository { */ public List getBlockATStatesAtHeight(int height) throws DataException; + /** Returns height of first trimmable AT state, or null if not found. */ + public Integer findFirstTrimmableStateHeight() throws DataException; + + /** Trims non-initial full AT state data between passed heights. Returns number of trimmed rows. */ + public int trimAtStates(int minHeight, int maxHeight) throws DataException; + /** * Save ATStateData into repository. *

diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index f8b6ce4d..6685896c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -248,7 +248,7 @@ public class HSQLDBATRepository implements ATRepository { @Override public ATStateData getATStateAtHeight(String atAddress, int height) throws DataException { - String sql = "SELECT created_when, state_data, state_hash, fees, is_initial " + String sql = "SELECT state_data, state_hash, fees, is_initial " + "FROM ATStates " + "WHERE AT_address = ? AND height = ? " + "LIMIT 1"; @@ -257,13 +257,12 @@ public class HSQLDBATRepository implements ATRepository { if (resultSet == null) return null; - long created = resultSet.getLong(1); - byte[] stateData = resultSet.getBytes(2); // Actually BLOB - byte[] stateHash = resultSet.getBytes(3); - long fees = resultSet.getLong(4); - boolean isInitial = resultSet.getBoolean(5); + byte[] stateData = resultSet.getBytes(1); // Actually BLOB + byte[] stateHash = resultSet.getBytes(2); + long fees = resultSet.getLong(3); + boolean isInitial = resultSet.getBoolean(4); - return new ATStateData(atAddress, height, created, stateData, stateHash, fees, isInitial); + return new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial); } catch (SQLException e) { throw new DataException("Unable to fetch AT state from repository", e); } @@ -271,7 +270,7 @@ public class HSQLDBATRepository implements ATRepository { @Override public ATStateData getLatestATState(String atAddress) throws DataException { - String sql = "SELECT height, created_when, state_data, state_hash, fees, is_initial " + String sql = "SELECT height, state_data, state_hash, fees, is_initial " + "FROM ATStates " + "WHERE AT_address = ? " // AT_address then height so the compound primary key is used as an index @@ -284,13 +283,12 @@ public class HSQLDBATRepository implements ATRepository { return null; int height = resultSet.getInt(1); - long created = resultSet.getLong(2); - byte[] stateData = resultSet.getBytes(3); // Actually BLOB - byte[] stateHash = resultSet.getBytes(4); - long fees = resultSet.getLong(5); - boolean isInitial = resultSet.getBoolean(6); + byte[] stateData = resultSet.getBytes(2); // Actually BLOB + byte[] stateHash = resultSet.getBytes(3); + long fees = resultSet.getLong(4); + boolean isInitial = resultSet.getBoolean(5); - return new ATStateData(atAddress, height, created, stateData, stateHash, fees, isInitial); + return new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial); } catch (SQLException e) { throw new DataException("Unable to fetch latest AT state from repository", e); } @@ -303,10 +301,10 @@ public class HSQLDBATRepository implements ATRepository { StringBuilder sql = new StringBuilder(1024); List bindParams = new ArrayList<>(); - sql.append("SELECT AT_address, height, created_when, state_data, state_hash, fees, is_initial " + sql.append("SELECT AT_address, height, state_data, state_hash, fees, is_initial " + "FROM ATs " + "CROSS JOIN LATERAL(" - + "SELECT height, created_when, state_data, state_hash, fees, is_initial " + + "SELECT height, state_data, state_hash, fees, is_initial " + "FROM ATStates " + "WHERE ATStates.AT_address = ATs.AT_address "); @@ -354,13 +352,12 @@ public class HSQLDBATRepository implements ATRepository { do { String atAddress = resultSet.getString(1); int height = resultSet.getInt(2); - long created = resultSet.getLong(3); - byte[] stateData = resultSet.getBytes(4); // Actually BLOB - byte[] stateHash = resultSet.getBytes(5); - long fees = resultSet.getLong(6); - boolean isInitial = resultSet.getBoolean(7); + byte[] stateData = resultSet.getBytes(3); // Actually BLOB + byte[] stateHash = resultSet.getBytes(4); + long fees = resultSet.getLong(5); + boolean isInitial = resultSet.getBoolean(6); - ATStateData atStateData = new ATStateData(atAddress, height, created, stateData, stateHash, fees, isInitial); + ATStateData atStateData = new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial); atStates.add(atStateData); } while (resultSet.next()); @@ -375,6 +372,7 @@ public class HSQLDBATRepository implements ATRepository { public List getBlockATStatesAtHeight(int height) throws DataException { String sql = "SELECT AT_address, state_hash, fees, is_initial " + "FROM ATStates " + + "LEFT OUTER JOIN ATs USING (AT_address) " + "WHERE height = ? " + "ORDER BY created_when ASC"; @@ -401,18 +399,57 @@ public class HSQLDBATRepository implements ATRepository { return atStates; } + @Override + public Integer findFirstTrimmableStateHeight() throws DataException { + String sql = "SELECT MIN(height) FROM ATStates " + + "WHERE is_initial = FALSE AND state_data IS NOT NULL"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return null; + + int height = resultSet.getInt(1); + if (height == 0 && resultSet.wasNull()) + return null; + + return height; + } catch (SQLException e) { + throw new DataException("Unable to find first trimmable AT state in repository", e); + } + } + + @Override + public int trimAtStates(int minHeight, int maxHeight) throws DataException { + if (minHeight >= maxHeight) + return 0; + + // We're often called so no need to trim all states in one go. + // Limit updates to reduce CPU and memory load. + String sql = "UPDATE ATStates SET state_data = NULL " + + "WHERE is_initial = FALSE " + + "AND state_data IS NOT NULL " + + "AND height BETWEEN ? AND ? " + + "LIMIT 4000"; + + try { + return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to trim AT states in repository", e); + } + } + @Override public void save(ATStateData atStateData) throws DataException { // We shouldn't ever save partial ATStateData - if (atStateData.getCreation() == null || atStateData.getStateHash() == null || atStateData.getHeight() == null) + if (atStateData.getStateHash() == null || atStateData.getHeight() == null) throw new IllegalArgumentException("Refusing to save partial AT state into repository!"); HSQLDBSaver saveHelper = new HSQLDBSaver("ATStates"); saveHelper.bind("AT_address", atStateData.getATAddress()).bind("height", atStateData.getHeight()) - .bind("created_when", atStateData.getCreation()).bind("state_data", atStateData.getStateData()) - .bind("state_hash", atStateData.getStateHash()).bind("fees", atStateData.getFees()) - .bind("is_initial", atStateData.isInitial()); + .bind("state_data", atStateData.getStateData()).bind("state_hash", atStateData.getStateHash()) + .bind("fees", atStateData.getFees()).bind("is_initial", atStateData.isInitial()); try { saveHelper.execute(this.repository); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index c6356e5d..6db40f21 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -212,6 +212,8 @@ public class HSQLDBDatabaseUpdates { + "PRIMARY KEY (account))"); // For looking up an account by public key stmt.execute("CREATE INDEX AccountPublicKeyIndex on Accounts (public_key)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE Accounts NEW SPACE"); // Account balances stmt.execute("CREATE TABLE AccountBalances (account QortalAddress, asset_id AssetID, balance QortalAmount NOT NULL, " @@ -220,6 +222,8 @@ public class HSQLDBDatabaseUpdates { stmt.execute("CREATE INDEX AccountBalancesAssetBalanceIndex ON AccountBalances (asset_id, balance)"); // Add CHECK constraint to account balances stmt.execute("ALTER TABLE AccountBalances ADD CONSTRAINT CheckBalanceNotNegative CHECK (balance >= 0)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE AccountBalances NEW SPACE"); // Keeping track of QORT gained from holding legacy QORA stmt.execute("CREATE TABLE AccountQortFromQoraInfo (account QortalAddress, final_qort_from_qora QortalAmount, final_block_height INT, " @@ -417,6 +421,8 @@ public class HSQLDBDatabaseUpdates { + "PRIMARY KEY (AT_address, height), FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)"); // For finding per-block AT states, ordered by creation timestamp stmt.execute("CREATE INDEX BlockATStateIndex on ATStates (height, created_when)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE ATStates NEW SPACE"); // Deploy CIYAM AT Transactions stmt.execute("CREATE TABLE DeployATTransactions (signature Signature, creator QortalPublicKey NOT NULL, AT_name ATName NOT NULL, " @@ -653,6 +659,12 @@ public class HSQLDBDatabaseUpdates { stmt.execute("DROP TABLE IF EXISTS NextBlockHeight"); break; + case 25: + // Remove excess created_when from ATStates + stmt.execute("ALTER TABLE ATStates DROP created_when"); + stmt.execute("CREATE INDEX ATStateHeightIndex on ATStates (height)"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index b42675c5..38a5f3c6 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -79,6 +79,8 @@ public class Settings { private long repositoryBackupInterval = 0; // ms /** Whether to show a notification when we backup repository. */ private boolean showBackupNotification = false; + /** How long to keep old, full, AT state data (ms). */ + private long atStatesMaxLifetime = 2 * 7 * 24 * 60 * 60 * 1000L; // milliseconds // Peer-to-peer related private boolean isTestNet = false; @@ -406,4 +408,8 @@ public class Settings { return this.showBackupNotification; } + public long getAtStatesMaxLifetime() { + return this.atStatesMaxLifetime; + } + }