From 0da21356c7ccb1f9a24efcbf08c9d2f021b789ee Mon Sep 17 00:00:00 2001 From: catbref Date: Wed, 3 Jul 2019 11:18:10 +0100 Subject: [PATCH] Repository backup/recovery Controller requests 'quick' repository backup every 123 minutes. On start-up, if repository fails to load, recovery is attempted using backup (if present). AutoUpdate also requests 'slow' repository backup just before calling ApplyUpdate. ('Slow' means perform "CHECKPOINT DEFRAG" first). --- .../java/org/qora/controller/AutoUpdate.java | 3 + .../java/org/qora/controller/Controller.java | 13 +- .../java/org/qora/repository/Repository.java | 6 +- .../qora/repository/RepositoryManager.java | 8 ++ .../repository/hsqldb/HSQLDBRepository.java | 123 +++++++++++++++++- .../hsqldb/HSQLDBRepositoryFactory.java | 15 ++- 6 files changed, 162 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qora/controller/AutoUpdate.java b/src/main/java/org/qora/controller/AutoUpdate.java index 124b988b..e8077548 100644 --- a/src/main/java/org/qora/controller/AutoUpdate.java +++ b/src/main/java/org/qora/controller/AutoUpdate.java @@ -212,6 +212,9 @@ public class AutoUpdate extends Thread { return false; // failed - try another repo } + // Give repository a chance to backup in case things go badly wrong + RepositoryManager.backup(false); + // Call ApplyUpdate to end this process (unlocking current JAR so it can be replaced) String javaHome = System.getProperty("java.home"); LOGGER.debug(String.format("Java home: %s", javaHome)); diff --git a/src/main/java/org/qora/controller/Controller.java b/src/main/java/org/qora/controller/Controller.java index 1ac62177..9885e795 100644 --- a/src/main/java/org/qora/controller/Controller.java +++ b/src/main/java/org/qora/controller/Controller.java @@ -84,8 +84,9 @@ public class Controller extends Thread { private static final long MISBEHAVIOUR_COOLOFF = 60 * 60 * 1000; // ms private static final int MAX_BLOCKCHAIN_TIP_AGE = 5; // blocks private static final Object shutdownLock = new Object(); - private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s/blockchain;create=true"; + private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s/blockchain;create=true;hsqldb.full_log_replay=true"; private static final long ARBITRARY_REQUEST_TIMEOUT = 5 * 1000; // ms + private static final long REPOSITORY_BACKUP_PERIOD = 123 * 60 * 1000; // ms private static volatile boolean isStopping = false; private static BlockGenerator blockGenerator = null; @@ -95,6 +96,8 @@ public class Controller extends Thread { private final String buildVersion; private final long buildTimestamp; // seconds + private long repositoryBackupTimestamp = startTime + REPOSITORY_BACKUP_PERIOD; + /** * Map of recent requests for ARBITRARY transaction data payloads. *

@@ -117,6 +120,8 @@ public class Controller extends Thread { /** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly generated block. */ private final ReentrantLock blockchainLock = new ReentrantLock(); + // Constructors + private Controller() { Properties properties = new Properties(); try (InputStream in = this.getClass().getResourceAsStream("/build.properties")) { @@ -296,6 +301,12 @@ public class Controller extends Thread { // Clean up arbitrary data request cache final long requestMinimumTimestamp = System.currentTimeMillis() - ARBITRARY_REQUEST_TIMEOUT; arbitraryDataRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); + + // Give repository a chance to backup + if (System.currentTimeMillis() >= repositoryBackupTimestamp) { + repositoryBackupTimestamp += REPOSITORY_BACKUP_PERIOD; + RepositoryManager.backup(true); + } } } catch (InterruptedException e) { // Fall-through to exit diff --git a/src/main/java/org/qora/repository/Repository.java b/src/main/java/org/qora/repository/Repository.java index 192bec76..3f88432f 100644 --- a/src/main/java/org/qora/repository/Repository.java +++ b/src/main/java/org/qora/repository/Repository.java @@ -26,9 +26,9 @@ public interface Repository extends AutoCloseable { public void discardChanges() throws DataException; - void setSavepoint() throws DataException; + public void setSavepoint() throws DataException; - void rollbackToSavepoint() throws DataException; + public void rollbackToSavepoint() throws DataException; @Override public void close() throws DataException; @@ -39,4 +39,6 @@ public interface Repository extends AutoCloseable { public void setDebug(boolean debugState); + public void backup(boolean quick) throws DataException; + } diff --git a/src/main/java/org/qora/repository/RepositoryManager.java b/src/main/java/org/qora/repository/RepositoryManager.java index a03e1cc5..e8c3aa5b 100644 --- a/src/main/java/org/qora/repository/RepositoryManager.java +++ b/src/main/java/org/qora/repository/RepositoryManager.java @@ -20,4 +20,12 @@ public abstract class RepositoryManager { repositoryFactory = null; } + public static void backup(boolean quick) { + try (final Repository repository = getRepository()) { + repository.backup(quick); + } catch (DataException e) { + // Backup is best-effort so don't complain + } + } + } diff --git a/src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java index 32b895cc..d544af89 100644 --- a/src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java @@ -1,7 +1,13 @@ package org.qora.repository.hsqldb; +import java.io.File; +import java.io.IOException; import java.math.BigDecimal; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.Connection; +import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -12,9 +18,12 @@ import java.time.OffsetDateTime; import java.time.ZoneOffset; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Comparator; import java.util.Deque; import java.util.List; import java.util.TimeZone; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,7 +56,7 @@ public class HSQLDBRepository implements Repository { protected long sessionId; // NB: no visibility modifier so only callable from within same package - HSQLDBRepository(Connection connection) throws DataException { + /* package */ HSQLDBRepository(Connection connection) throws DataException { this.connection = connection; this.savepoints = new ArrayDeque<>(3); @@ -225,6 +234,118 @@ public class HSQLDBRepository implements Repository { this.debugState = debugState; } + @Override + public void backup(boolean quick) throws DataException { + // First perform a CHECKPOINT + try { + if (quick) + this.connection.createStatement().execute("CHECKPOINT"); + else + this.connection.createStatement().execute("CHECKPOINT DEFRAG"); + } catch (SQLException e) { + throw new DataException("Unable to prepare repository for backup"); + } + + // Clean out any previous backup + try { + String connectionUrl = this.connection.getMetaData().getURL(); + String dbPathname = getDbPathname(connectionUrl); + if (dbPathname == null) + throw new DataException("Unable to locate repository for backup?"); + + String backupUrl = buildBackupUrl(dbPathname); + String backupPathname = getDbPathname(backupUrl); + if (backupPathname == null) + throw new DataException("Unable to determine location for repository backup?"); + + Path backupDirPath = Paths.get(backupPathname).getParent(); + String backupDirPathname = backupDirPath.toString(); + + Files.walk(backupDirPath) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .filter(file -> file.getPath().startsWith(backupDirPathname)) + .forEach(File::delete); + } catch (SQLException | IOException e) { + throw new DataException("Unable to remove previous repository backup"); + } + + // Actually create backup + try { + this.connection.createStatement().execute("BACKUP DATABASE TO 'backup/' BLOCKING AS FILES"); + } catch (SQLException e) { + throw new DataException("Unable to backup repository"); + } + } + + /** Returns DB pathname from passed connection URL. */ + private static String getDbPathname(String connectionUrl) { + Pattern pattern = Pattern.compile("file:(.*?);"); + Matcher matcher = pattern.matcher(connectionUrl); + + if (!matcher.find()) + return null; + + String pathname = matcher.group(1); + return pathname; + } + + private static String buildBackupUrl(String dbPathname) { + Path oldRepoPath = Paths.get(dbPathname); + Path oldRepoDirPath = oldRepoPath.getParent(); + Path oldRepoFilePath = oldRepoPath.getFileName(); + + // Try to open backup. We need to remove "create=true" and insert "backup" dir before final filename. + String backupUrlTemplate = "jdbc:hsqldb:file:%s/backup/%s;create=false;hsqldb.full_log_replay=true"; + String backupUrl = String.format(backupUrlTemplate, oldRepoDirPath.toString(), oldRepoFilePath.toString()); + return backupUrl; + } + + /* package */ static void attemptRecovery(String connectionUrl) throws DataException { + String dbPathname = getDbPathname(connectionUrl); + if (dbPathname == null) + throw new DataException("Unable to locate repository for backup?"); + + String backupUrl = buildBackupUrl(dbPathname); + Path oldRepoDirPath = Paths.get(dbPathname).getParent(); + + // Attempt connection to backup to see if it is viable + try (Connection connection = DriverManager.getConnection(backupUrl)) { + LOGGER.info("Attempting repository recovery using backup"); + + // Move old repository files out the way + Files.walk(oldRepoDirPath) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .filter(file -> file.getPath().startsWith(dbPathname)) + .forEach(File::delete); + + try { + // Now "backup" the backup back to original repository location (the parent) + // NOTE: trailing / is OK because HSQLDB checks for both / and O/S-specific separator + // textdb.allow_full_path connection property is required to be able to use '..' + connection.createStatement().execute("BACKUP DATABASE TO '../' BLOCKING AS FILES"); + } catch (SQLException e) { + // We really failed + throw new DataException("Failed to recover repository to original location"); + } + + // Close backup + } catch (SQLException e) { + // We really failed + throw new DataException("Failed to open repository or perform recovery"); + } catch (IOException e) { + throw new DataException("Failed to delete old repository to perform recovery"); + } + + // Now attempt to open recovered repository, just to check + try (Connection connection = DriverManager.getConnection(connectionUrl)) { + } catch (SQLException e) { + // We really failed + throw new DataException("Failed to open recovered repository"); + } + } + /** * Returns prepared statement using passed SQL, logging query if necessary. */ diff --git a/src/main/java/org/qora/repository/hsqldb/HSQLDBRepositoryFactory.java b/src/main/java/org/qora/repository/hsqldb/HSQLDBRepositoryFactory.java index fc3a12de..6845b8c9 100644 --- a/src/main/java/org/qora/repository/hsqldb/HSQLDBRepositoryFactory.java +++ b/src/main/java/org/qora/repository/hsqldb/HSQLDBRepositoryFactory.java @@ -5,6 +5,8 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.util.Properties; +import org.hsqldb.HsqlException; +import org.hsqldb.error.ErrorCode; import org.hsqldb.jdbc.JDBCPool; import org.qora.repository.DataException; import org.qora.repository.Repository; @@ -22,7 +24,16 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory { // Check no-one else is accessing database try (Connection connection = DriverManager.getConnection(this.connectionUrl)) { } catch (SQLException e) { - throw new DataException("Unable to open repository: " + e.getMessage()); + Throwable cause = e.getCause(); + if (cause == null || !(cause instanceof HsqlException)) + throw new DataException("Unable to open repository: " + e.getMessage()); + + HsqlException he = (HsqlException) cause; + if (he.getErrorCode() != -ErrorCode.ERROR_IN_LOG_FILE && he.getErrorCode() != -ErrorCode.M_DatabaseScriptReader_read) + throw new DataException("Unable to open repository: " + e.getMessage()); + + // Attempt recovery? + HSQLDBRepository.attemptRecovery(connectionUrl); } this.connectionPool = new JDBCPool(); @@ -45,7 +56,7 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory { try { return new HSQLDBRepository(this.getConnection()); } catch (SQLException e) { - throw new DataException("Repository initialization error", e); + throw new DataException("Repository instantiation error", e); } }