Repository backup/recovery

Controller requests 'quick' repository backup every 123 minutes.

On start-up, if repository fails to load, recovery is attempted using
backup (if present).

AutoUpdate also requests 'slow' repository backup just before
calling ApplyUpdate. ('Slow' means perform "CHECKPOINT DEFRAG" first).

src/main/java/org/qora/controller/AutoUpdate.java

@@ -212,6 +212,9 @@ public class AutoUpdate extends Thread {
 			return false; // failed - try another repo
 		}
 
+		// Give repository a chance to backup in case things go badly wrong
+		RepositoryManager.backup(false);
+
 		// Call ApplyUpdate to end this process (unlocking current JAR so it can be replaced)
 		String javaHome = System.getProperty("java.home");
 		LOGGER.debug(String.format("Java home: %s", javaHome));

src/main/java/org/qora/controller/Controller.java

@@ -84,8 +84,9 @@ public class Controller extends Thread {
 	private static final long MISBEHAVIOUR_COOLOFF = 60 * 60 * 1000; // ms
 	private static final int MAX_BLOCKCHAIN_TIP_AGE = 5; // blocks
 	private static final Object shutdownLock = new Object();
-	private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s/blockchain;create=true";
+	private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s/blockchain;create=true;hsqldb.full_log_replay=true";
 	private static final long ARBITRARY_REQUEST_TIMEOUT = 5 * 1000; // ms
+	private static final long REPOSITORY_BACKUP_PERIOD = 123 * 60 * 1000; // ms
 
 	private static volatile boolean isStopping = false;
 	private static BlockGenerator blockGenerator = null;
@@ -95,6 +96,8 @@ public class Controller extends Thread {
 	private final String buildVersion;
 	private final long buildTimestamp; // seconds
 
+	private long repositoryBackupTimestamp = startTime + REPOSITORY_BACKUP_PERIOD;
+
 	/**
 	 * Map of recent requests for ARBITRARY transaction data payloads.
 	 * <p>
@@ -117,6 +120,8 @@ public class Controller extends Thread {
 	/** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly generated block. */
 	private final ReentrantLock blockchainLock = new ReentrantLock();
 
+	// Constructors
+
 	private Controller() {
 		Properties properties = new Properties();
 		try (InputStream in = this.getClass().getResourceAsStream("/build.properties")) {
@@ -296,6 +301,12 @@ public class Controller extends Thread {
 				// Clean up arbitrary data request cache
 				final long requestMinimumTimestamp = System.currentTimeMillis() - ARBITRARY_REQUEST_TIMEOUT;
 				arbitraryDataRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp);
+
+				// Give repository a chance to backup
+				if (System.currentTimeMillis() >= repositoryBackupTimestamp) {
+					repositoryBackupTimestamp += REPOSITORY_BACKUP_PERIOD;
+					RepositoryManager.backup(true);
+				}
 			}
 		} catch (InterruptedException e) {
 			// Fall-through to exit

src/main/java/org/qora/repository/Repository.java

@@ -26,9 +26,9 @@ public interface Repository extends AutoCloseable {
 
 	public void discardChanges() throws DataException;
 
-	void setSavepoint() throws DataException;
+	public void setSavepoint() throws DataException;
 
-	void rollbackToSavepoint() throws DataException;
+	public void rollbackToSavepoint() throws DataException;
 
 	@Override
 	public void close() throws DataException;
@@ -39,4 +39,6 @@ public interface Repository extends AutoCloseable {
 
 	public void setDebug(boolean debugState);
 
+	public void backup(boolean quick) throws DataException;
+
 }

src/main/java/org/qora/repository/RepositoryManager.java

@@ -20,4 +20,12 @@ public abstract class RepositoryManager {
 		repositoryFactory = null;
 	}
 
+	public static void backup(boolean quick) {
+		try (final Repository repository = getRepository()) {
+			repository.backup(quick);
+		} catch (DataException e) {
+			// Backup is best-effort so don't complain
+		}
+	}
+
 }

src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java

@@ -1,7 +1,13 @@
 package org.qora.repository.hsqldb;
 
+import java.io.File;
+import java.io.IOException;
 import java.math.BigDecimal;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.Connection;
+import java.sql.DriverManager;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -12,9 +18,12 @@ import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.Deque;
 import java.util.List;
 import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -47,7 +56,7 @@ public class HSQLDBRepository implements Repository {
 	protected long sessionId;
 
 	// NB: no visibility modifier so only callable from within same package
-	HSQLDBRepository(Connection connection) throws DataException {
+	/* package */ HSQLDBRepository(Connection connection) throws DataException {
 		this.connection = connection;
 		this.savepoints = new ArrayDeque<>(3);
 
@@ -225,6 +234,118 @@ public class HSQLDBRepository implements Repository {
 		this.debugState = debugState;
 	}
 
+	@Override
+	public void backup(boolean quick) throws DataException {
+		// First perform a CHECKPOINT
+		try {
+			if (quick)
+				this.connection.createStatement().execute("CHECKPOINT");
+			else
+				this.connection.createStatement().execute("CHECKPOINT DEFRAG");
+		} catch (SQLException e) {
+			throw new DataException("Unable to prepare repository for backup");
+		}
+
+		// Clean out any previous backup
+		try {
+			String connectionUrl = this.connection.getMetaData().getURL();
+			String dbPathname = getDbPathname(connectionUrl);
+			if (dbPathname == null)
+				throw new DataException("Unable to locate repository for backup?");
+
+			String backupUrl = buildBackupUrl(dbPathname);
+			String backupPathname = getDbPathname(backupUrl);
+			if (backupPathname == null)
+				throw new DataException("Unable to determine location for repository backup?");
+
+			Path backupDirPath = Paths.get(backupPathname).getParent();
+			String backupDirPathname = backupDirPath.toString();
+
+			Files.walk(backupDirPath)
+					.sorted(Comparator.reverseOrder())
+					.map(Path::toFile)
+					.filter(file -> file.getPath().startsWith(backupDirPathname))
+					.forEach(File::delete);
+		} catch (SQLException | IOException e) {
+			throw new DataException("Unable to remove previous repository backup");
+		}
+
+		// Actually create backup
+		try {
+			this.connection.createStatement().execute("BACKUP DATABASE TO 'backup/' BLOCKING AS FILES");
+		} catch (SQLException e) {
+			throw new DataException("Unable to backup repository");
+		}
+	}
+
+	/** Returns DB pathname from passed connection URL. */
+	private static String getDbPathname(String connectionUrl) {
+		Pattern pattern = Pattern.compile("file:(.*?);");
+		Matcher matcher = pattern.matcher(connectionUrl);
+
+		if (!matcher.find())
+			return null;
+
+		String pathname = matcher.group(1);
+		return pathname;
+	}
+
+	private static String buildBackupUrl(String dbPathname) {
+		Path oldRepoPath = Paths.get(dbPathname);
+		Path oldRepoDirPath = oldRepoPath.getParent();
+		Path oldRepoFilePath = oldRepoPath.getFileName();
+
+		// Try to open backup. We need to remove "create=true" and insert "backup" dir before final filename.
+		String backupUrlTemplate = "jdbc:hsqldb:file:%s/backup/%s;create=false;hsqldb.full_log_replay=true";
+		String backupUrl = String.format(backupUrlTemplate, oldRepoDirPath.toString(), oldRepoFilePath.toString());
+		return backupUrl;
+	}
+
+	/* package */ static void attemptRecovery(String connectionUrl) throws DataException {
+		String dbPathname = getDbPathname(connectionUrl);
+		if (dbPathname == null)
+			throw new DataException("Unable to locate repository for backup?");
+
+		String backupUrl = buildBackupUrl(dbPathname);
+		Path oldRepoDirPath = Paths.get(dbPathname).getParent();
+
+		// Attempt connection to backup to see if it is viable
+		try (Connection connection = DriverManager.getConnection(backupUrl)) {
+			LOGGER.info("Attempting repository recovery using backup");
+
+			// Move old repository files out the way
+			Files.walk(oldRepoDirPath)
+					.sorted(Comparator.reverseOrder())
+					.map(Path::toFile)
+					.filter(file -> file.getPath().startsWith(dbPathname))
+					.forEach(File::delete);
+
+			try {
+				// Now "backup" the backup back to original repository location (the parent)
+				// NOTE: trailing / is OK because HSQLDB checks for both / and O/S-specific separator
+				// textdb.allow_full_path connection property is required to be able to use '..'
+				connection.createStatement().execute("BACKUP DATABASE TO '../' BLOCKING AS FILES");
+			} catch (SQLException e) {
+				// We really failed
+				throw new DataException("Failed to recover repository to original location");
+			}
+
+			// Close backup
+		} catch (SQLException e) {
+			// We really failed
+			throw new DataException("Failed to open repository or perform recovery");
+		} catch (IOException e) {
+			throw new DataException("Failed to delete old repository to perform recovery");
+		}
+
+		// Now attempt to open recovered repository, just to check
+		try (Connection connection = DriverManager.getConnection(connectionUrl)) {
+		} catch (SQLException e) {
+			// We really failed
+			throw new DataException("Failed to open recovered repository");
+		}
+	}
+
 	/**
 	 * Returns prepared statement using passed SQL, logging query if necessary.
 	 */

src/main/java/org/qora/repository/hsqldb/HSQLDBRepositoryFactory.java

@@ -5,6 +5,8 @@ import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.Properties;
 
+import org.hsqldb.HsqlException;
+import org.hsqldb.error.ErrorCode;
 import org.hsqldb.jdbc.JDBCPool;
 import org.qora.repository.DataException;
 import org.qora.repository.Repository;
@@ -22,7 +24,16 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory {
 		// Check no-one else is accessing database
 		try (Connection connection = DriverManager.getConnection(this.connectionUrl)) {
 		} catch (SQLException e) {
-			throw new DataException("Unable to open repository: " + e.getMessage());
+			Throwable cause = e.getCause();
+			if (cause == null || !(cause instanceof HsqlException))
+				throw new DataException("Unable to open repository: " + e.getMessage());
+
+			HsqlException he = (HsqlException) cause;
+			if (he.getErrorCode() != -ErrorCode.ERROR_IN_LOG_FILE && he.getErrorCode() != -ErrorCode.M_DatabaseScriptReader_read)
+				throw new DataException("Unable to open repository: " + e.getMessage());
+
+			// Attempt recovery?
+			HSQLDBRepository.attemptRecovery(connectionUrl);
 		}
 
 		this.connectionPool = new JDBCPool();
@@ -45,7 +56,7 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory {
 		try {
 			return new HSQLDBRepository(this.getConnection());
 		} catch (SQLException e) {
-			throw new DataException("Repository initialization error", e);
+			throw new DataException("Repository instantiation error", e);
 		}
 	}