diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java index ea99afba..24c7f30d 100644 --- a/src/main/java/org/qortal/api/HTMLParser.java +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -13,9 +13,9 @@ public class HTMLParser { private String linkPrefix; - public HTMLParser(String resourceId, String inPath, boolean usePrefix) { + public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix) { String inPathWithoutFilename = inPath.substring(0, inPath.lastIndexOf('/')); - this.linkPrefix = usePrefix ? String.format("/site/%s%s", resourceId, inPathWithoutFilename) : ""; + this.linkPrefix = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : ""; } /** diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 12deed16..63157320 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -272,10 +272,10 @@ public class ArbitraryResource { Service service = Service.ARBITRARY_DATA; Compression compression = Compression.NONE; - DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), method, compression); + DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), name, service, method, compression); try { dataFileWriter.save(); - } catch (IOException e) { + } catch (IOException | DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index c4773333..01232d2d 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -98,15 +98,15 @@ public class WebsiteResource { } byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58); - String name = null; - ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT; + String name = "CalDescentTest1"; // TODO: dynamic + ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PATCH; ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE; ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; - DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), method, compression); + DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), name, service, method, compression); try { dataFileWriter.save(); - } catch (IOException e) { + } catch (IOException | DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); @@ -144,6 +144,7 @@ public class WebsiteResource { secret, compression, digest, dataType, chunkHashes, payments); ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData); + LOGGER.info("Computing nonce..."); transaction.computeNonce(); Transaction.ValidationResult result = transaction.isValidUnconfirmed(); @@ -197,13 +198,15 @@ public class WebsiteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION); } + String name = null; + Service service = Service.WEBSITE; Method method = Method.PUT; Compression compression = Compression.ZIP; - DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(directoryPath), method, compression); + DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(directoryPath), name, service, method, compression); try { dataFileWriter.save(); - } catch (IOException e) { + } catch (IOException | DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); } catch (IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); @@ -222,26 +225,38 @@ public class WebsiteResource { @GET @Path("{signature}") public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) { - return this.get(signature, ResourceIdType.SIGNATURE, "/", null,true); + return this.get(signature, ResourceIdType.SIGNATURE, "/", null, "/site", true); } @GET @Path("{signature}/{path:.*}") public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath) { - return this.get(signature, ResourceIdType.SIGNATURE, inPath,null,true); + return this.get(signature, ResourceIdType.SIGNATURE, inPath,null, "/site", true); } @GET @Path("/hash/{hash}") public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58) { - return this.get(hash58, ResourceIdType.FILE_HASH, "/", secret58,true); + return this.get(hash58, ResourceIdType.FILE_HASH, "/", secret58, "/site/hash", true); + } + + @GET + @Path("/name/{name}/{path:.*}") + public HttpServletResponse getPathByName(@PathParam("name") String name, @PathParam("path") String inPath) { + return this.get(name, ResourceIdType.NAME, inPath, null, "/site/name", true); + } + + @GET + @Path("/name/{name}") + public HttpServletResponse getIndexByName(@PathParam("name") String name) { + return this.get(name, ResourceIdType.NAME, "/", null, "/site/name", true); } @GET @Path("/hash/{hash}/{path:.*}") public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath, @QueryParam("secret") String secret58) { - return this.get(hash58, ResourceIdType.FILE_HASH, inPath, secret58,true); + return this.get(hash58, ResourceIdType.FILE_HASH, inPath, secret58, "/site/hash", true); } @GET @@ -259,19 +274,23 @@ public class WebsiteResource { private HttpServletResponse getDomainMap(String inPath) { Map domainMap = Settings.getInstance().getSimpleDomainMap(); if (domainMap != null && domainMap.containsKey(request.getServerName())) { - return this.get(domainMap.get(request.getServerName()), ResourceIdType.SIGNATURE, inPath, null, false); + return this.get(domainMap.get(request.getServerName()), ResourceIdType.SIGNATURE, inPath, null, "", false); } return this.get404Response(); } - private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, String inPath, String secret58, boolean usePrefix) { + private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, String inPath, String secret58, + String prefix, boolean usePrefix) { if (!inPath.startsWith(File.separator)) { inPath = File.separator + inPath; } - DataFileReader dataFileReader = new DataFileReader(resourceId, resourceIdType); + Service service = Service.WEBSITE; + DataFileReader dataFileReader = new DataFileReader(resourceId, resourceIdType, service); dataFileReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only try { + // TODO: overwrite if new transaction arrives, to invalidate cache + // We could store the latest transaction signature in the extracted folder dataFileReader.load(false); } catch (Exception e) { return this.get404Response(); @@ -289,7 +308,7 @@ public class WebsiteResource { if (HTMLParser.isHtmlFile(filename)) { // HTML file - needs to be parsed byte[] data = Files.readAllBytes(Paths.get(filePath)); // TODO: limit file size that can be read into memory - HTMLParser htmlParser = new HTMLParser(resourceId, inPath, usePrefix); + HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix); data = htmlParser.replaceRelativeLinks(filename, data); response.setContentType(context.getMimeType(filename)); response.setContentLength(data.length); @@ -311,7 +330,7 @@ public class WebsiteResource { } return response; } catch (FileNotFoundException | NoSuchFileException e) { - LOGGER.info("File not found at path: {}", unzippedPath); + LOGGER.info("Unable to serve file: {}", e.getMessage()); if (inPath.equals("/")) { // Delete the unzipped folder if no index file was found try { diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 2968db3d..29827e30 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -368,7 +368,7 @@ public class ArbitraryDataManager extends Thread { // Load file(s) and add any that exist to the list of hashes DataFile dataFile = DataFile.fromHash(hash); - if (chunkHashes.length > 0) { + if (chunkHashes != null && chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); for (DataFileChunk dataFileChunk : dataFile.getChunks()) { if (dataFileChunk.exists()) { diff --git a/src/main/java/org/qortal/repository/ArbitraryRepository.java b/src/main/java/org/qortal/repository/ArbitraryRepository.java index 80f8c1e3..5e3e657a 100644 --- a/src/main/java/org/qortal/repository/ArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/ArbitraryRepository.java @@ -1,6 +1,9 @@ package org.qortal.repository; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; + +import java.util.List; public interface ArbitraryRepository { @@ -12,4 +15,8 @@ public interface ArbitraryRepository { public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException; + public List getArbitraryTransactions(String name, Service service, long since) throws DataException; + + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException; + } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index b3edf41a..5bc174e2 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -3,12 +3,20 @@ package org.qortal.repository.hsqldb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.crypto.Crypto; +import org.qortal.data.PaymentData; import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.TransactionData; -import org.qortal.data.transaction.ArbitraryTransactionData.DataType; import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.DataException; import org.qortal.storage.DataFile; +import org.qortal.transaction.Transaction.ApprovalStatus; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; public class HSQLDBArbitraryRepository implements ArbitraryRepository { @@ -48,7 +56,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { // Load data file(s) DataFile dataFile = DataFile.fromHash(digest); - if (chunkHashes.length > 0) { + if (chunkHashes != null && chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); } @@ -83,7 +91,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { // Load data file(s) DataFile dataFile = DataFile.fromHash(digest); - if (chunkHashes.length > 0) { + if (chunkHashes != null && chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); } @@ -168,7 +176,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { // Load data file(s) DataFile dataFile = DataFile.fromHash(digest); - if (chunkHashes.length > 0) { + if (chunkHashes != null && chunkHashes.length > 0) { dataFile.addChunkHashes(chunkHashes); } @@ -176,4 +184,133 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { dataFile.deleteAll(); } + @Override + public List getArbitraryTransactions(String name, Service service, long since) throws DataException { + String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, chunk_hashes, " + + "name, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature) " + + "WHERE name = ? AND service = ? AND created_when >= ?" + + "ORDER BY created_when ASC"; + List arbitraryTransactionData = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql, name, service.value, since)) { + if (resultSet == null) + return null; + + do { + //TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + Service serviceResult = Service.valueOf(resultSet.getInt(13)); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] chunkHashes = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + Method method = Method.valueOf(resultSet.getInt(19)); + byte[] secret = resultSet.getBytes(20); + Compression compression = Compression.valueOf(resultSet.getInt(21)); + + List payments = new ArrayList<>(); // TODO: this.getPaymentsFromSignature(baseTransactionData.getSignature()); + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceResult, nonce, size, nameResult, method, secret, compression, data, + dataType, chunkHashes, payments); + + arbitraryTransactionData.add(transactionData); + } while (resultSet.next()); + + return arbitraryTransactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } + } + + @Override + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException { + String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + "tx_group_id, block_height, approval_status, approval_height, " + + "version, nonce, service, size, is_data_raw, data, chunk_hashes, " + + "name, update_method, secret, compression FROM ArbitraryTransactions " + + "JOIN Transactions USING (signature) " + + "WHERE name = ? AND service = ? AND update_method = ? " + + "ORDER BY created_when DESC LIMIT 1"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, name, service.value, method.value)) { + if (resultSet == null) + return null; + + //TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] signature = resultSet.getBytes(3); + byte[] creatorPublicKey = resultSet.getBytes(4); + long timestamp = resultSet.getLong(5); + + Long fee = resultSet.getLong(6); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(7); + + Integer blockHeight = resultSet.getInt(8); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9)); + Integer approvalHeight = resultSet.getInt(10); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + int version = resultSet.getInt(11); + int nonce = resultSet.getInt(12); + Service serviceResult = Service.valueOf(resultSet.getInt(13)); + int size = resultSet.getInt(14); + boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(16); + byte[] chunkHashes = resultSet.getBytes(17); + String nameResult = resultSet.getString(18); + Method methodResult = Method.valueOf(resultSet.getInt(19)); + byte[] secret = resultSet.getBytes(20); + Compression compression = Compression.valueOf(resultSet.getInt(21)); + + List payments = new ArrayList<>(); // TODO: this.getPaymentsFromSignature(baseTransactionData.getSignature()); + ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, + version, serviceResult, nonce, size, nameResult, methodResult, secret, compression, data, + dataType, chunkHashes, payments); + + return transactionData; + } catch (SQLException e) { + throw new DataException("Unable to fetch arbitrary transactions from repository", e); + } + } + } diff --git a/src/main/java/org/qortal/storage/DataFile.java b/src/main/java/org/qortal/storage/DataFile.java index 568674d6..75579e1a 100644 --- a/src/main/java/org/qortal/storage/DataFile.java +++ b/src/main/java/org/qortal/storage/DataFile.java @@ -44,7 +44,9 @@ public class DataFile { // Resource ID types public enum ResourceIdType { SIGNATURE, - FILE_HASH + FILE_HASH, + TRANSACTION_DATA, + NAME }; private static final Logger LOGGER = LogManager.getLogger(DataFile.class); diff --git a/src/main/java/org/qortal/storage/DataFileBuilder.java b/src/main/java/org/qortal/storage/DataFileBuilder.java new file mode 100644 index 00000000..b4faba02 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileBuilder.java @@ -0,0 +1,131 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.Method; +import org.qortal.data.transaction.ArbitraryTransactionData.Service; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.storage.DataFile.ResourceIdType; +import org.qortal.utils.Base58; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class DataFileBuilder { + + private static final Logger LOGGER = LogManager.getLogger(DataFileBuilder.class); + + private String name; + private Service service; + + private List transactions; + private ArbitraryTransactionData latestPutTransaction; + private List paths; + private Path finalPath; + + public DataFileBuilder(String name, Service service) { + this.name = name; + this.service = service; + this.paths = new ArrayList<>(); + } + + public void build() throws DataException, IOException { + this.fetchTransactions(); + this.validateTransactions(); + this.processTransactions(); + this.buildLatestState(); + } + + private void fetchTransactions() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Get the most recent PUT + ArbitraryTransactionData latestPut = repository.getArbitraryRepository() + .getLatestTransaction(this.name, this.service, Method.PUT); + if (latestPut == null) { + throw new IllegalStateException("Cannot PATCH without existing PUT. Deploy using PUT first."); + } + this.latestPutTransaction = latestPut; + + // Load all transactions since the latest PUT + List transactionDataList = repository.getArbitraryRepository() + .getArbitraryTransactions(this.name, this.service, latestPut.getTimestamp()); + this.transactions = transactionDataList; + } + } + + private void validateTransactions() { + List transactionDataList = new ArrayList<>(this.transactions); + ArbitraryTransactionData latestPut = this.latestPutTransaction; + + if (latestPut == null) { + throw new IllegalStateException("Cannot PATCH without existing PUT. Deploy using PUT first."); + } + if (latestPut.getMethod() != Method.PUT) { + throw new IllegalStateException("Expected PUT but received PATCH"); + } + if (transactionDataList.size() == 0) { + throw new IllegalStateException(String.format("No transactions found for name %s, service %s, since %d", + name, service, latestPut.getTimestamp())); + } + + // Verify that the signature of the first transaction matches the latest PUT + ArbitraryTransactionData firstTransaction = transactionDataList.get(0); + if (!Objects.equals(firstTransaction.getSignature(), latestPut.getSignature())) { + throw new IllegalStateException("First transaction did not match latest PUT transaction"); + } + + // Remove the first transaction, as it should be the only PUT + transactionDataList.remove(0); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + if (!(transactionData instanceof ArbitraryTransactionData)) { + String sig58 = Base58.encode(transactionData.getSignature()); + throw new IllegalStateException(String.format("Received non-arbitrary transaction: %s", sig58)); + } + if (transactionData.getMethod() != Method.PATCH) { + throw new IllegalStateException("Expected PATCH but received PUT"); + } + } + } + + private void processTransactions() throws IOException, DataException { + List transactionDataList = new ArrayList<>(this.transactions); + + for (ArbitraryTransactionData transactionData : transactionDataList) { + LOGGER.trace("Found arbitrary transaction {}", Base58.encode(transactionData.getSignature())); + + // Build the data file, overwriting anything that was previously there + String sig58 = Base58.encode(transactionData.getSignature()); + DataFileReader dataFileReader = new DataFileReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service); + dataFileReader.setTransactionData(transactionData); + dataFileReader.load(true); + Path path = dataFileReader.getFilePath(); + if (path == null) { + throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58)); + } + if (!Files.exists(path)) { + throw new IllegalStateException(String.format("Path doesn't exist when building data from transaction %s", sig58)); + } + paths.add(path); + } + } + + private void buildLatestState() throws IOException, DataException { + DataFilePatches dataFilePatches = new DataFilePatches(this.paths); + dataFilePatches.applyPatches(); + this.finalPath = dataFilePatches.getFinalPath(); + } + + public Path getFinalPath() { + return this.finalPath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileCombiner.java b/src/main/java/org/qortal/storage/DataFileCombiner.java new file mode 100644 index 00000000..edb7b362 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileCombiner.java @@ -0,0 +1,56 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class DataFileCombiner { + + private static final Logger LOGGER = LogManager.getLogger(DataFileCombiner.class); + + private Path pathBefore; + private Path pathAfter; + private Path finalPath; + + public DataFileCombiner(Path pathBefore, Path pathAfter) { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + } + + public void combine() throws IOException { + try { + this.preExecute(); + this.process(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + if (this.pathBefore == null || this.pathAfter == null) { + throw new IllegalStateException(String.format("No paths available to build patch")); + } + if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) { + throw new IllegalStateException(String.format("Unable to create patch because at least one path doesn't exist")); + } + } + + private void postExecute() { + + } + + private void process() throws IOException { + DataFileMerge merge = new DataFileMerge(this.pathBefore, this.pathAfter); + merge.compute(); + this.finalPath = merge.getMergePath(); + } + + public Path getFinalPath() { + return this.finalPath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileCreatePatch.java b/src/main/java/org/qortal/storage/DataFileCreatePatch.java new file mode 100644 index 00000000..67ecf9cb --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileCreatePatch.java @@ -0,0 +1,58 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.repository.DataException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class DataFileCreatePatch { + + private static final Logger LOGGER = LogManager.getLogger(DataFileCreatePatch.class); + + private Path pathBefore; + private Path pathAfter; + private Path finalPath; + + public DataFileCreatePatch(Path pathBefore, Path pathAfter) { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + } + + public void create() throws DataException, IOException { + try { + this.preExecute(); + this.process(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + if (this.pathBefore == null || this.pathAfter == null) { + throw new IllegalStateException(String.format("No paths available to build patch")); + } + if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) { + throw new IllegalStateException(String.format("Unable to create patch because at least one path doesn't exist")); + } + } + + private void postExecute() { + + } + + private void process() { + + DataFileDiff diff = new DataFileDiff(this.pathBefore, this.pathAfter); + diff.compute(); + this.finalPath = diff.getDiffPath(); + } + + public Path getFinalPath() { + return this.finalPath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileDiff.java b/src/main/java/org/qortal/storage/DataFileDiff.java new file mode 100644 index 00000000..e6534f79 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileDiff.java @@ -0,0 +1,218 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.crypto.Crypto; + +import java.io.File; +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; + +public class DataFileDiff { + + private static final Logger LOGGER = LogManager.getLogger(DataFileDiff.class); + + private Path pathBefore; + private Path pathAfter; + private Path diffPath; + + public DataFileDiff(Path pathBefore, Path pathAfter) { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + } + + public void compute() { + try { + this.preExecute(); + this.findAddedOrModifiedFiles(); + this.findRemovedFiles(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + this.createOutputDirectory(); + } + + private void postExecute() { + + } + + private void createOutputDirectory() { + // Ensure temp folder exists + Path tempDir; + try { + tempDir = Files.createTempDirectory("qortal-diff"); + } catch (IOException e) { + throw new IllegalStateException("Unable to create temp directory"); + } + this.diffPath = tempDir; + } + + private void findAddedOrModifiedFiles() { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); + +// LOGGER.info("this.pathBefore: {}", this.pathBefore); +// LOGGER.info("this.pathAfter: {}", this.pathAfter); +// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute); +// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute); +// LOGGER.info("diffPathAbsolute: {}", diffPathAbsolute); + + + try { + // Check for additions or modifications + Files.walkFileTree(this.pathAfter, new FileVisitor() { + + @Override + public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException { + Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath()); + Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter); + + boolean wasAdded = false; + boolean wasModified = false; + + if (!Files.exists(filePathBefore)) { + LOGGER.info("File was added: {}", after.toString()); + wasAdded = true; + } + else if (Files.size(after) != Files.size(filePathBefore)) { + // Check file size first because it's quicker + LOGGER.info("File size was modified: {}", after.toString()); + wasModified = true; + } + else if (!Arrays.equals(DataFileDiff.digestFromPath(after), DataFileDiff.digestFromPath(filePathBefore))) { + // Check hashes as a last resort + LOGGER.info("File contents were modified: {}", after.toString()); + wasModified = true; + } + + if (wasAdded | wasModified) { + DataFileDiff.copyFilePathToBaseDir(after, diffPathAbsolute, filePathAfter); + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e){ + LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage()); + // TODO: throw exception? + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) { + return FileVisitResult.CONTINUE; + } + + }); + } catch (IOException e) { + LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + } + } + + private void findRemovedFiles() { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); + try { + // Check for removals + Files.walkFileTree(this.pathBefore, new FileVisitor() { + + @Override + public FileVisitResult preVisitDirectory(Path before, BasicFileAttributes attrs) throws IOException { + Path directoryPathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath()); + Path directoryPathAfter = pathAfterAbsolute.resolve(directoryPathBefore); + + if (!Files.exists(directoryPathAfter)) { + LOGGER.info("Directory was removed: {}", directoryPathAfter.toString()); + + DataFileDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore); + // TODO: we might need to mark directories differently to files + // TODO: add path to manifest JSON + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path before, BasicFileAttributes attrs) throws IOException { + Path filePathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath()); + Path filePathAfter = pathAfterAbsolute.resolve(filePathBefore); + + if (!Files.exists(filePathAfter)) { + LOGGER.trace("File was removed: {}", before.toString()); + + DataFileDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore); + // TODO: add path to manifest JSON + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e){ + LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage()); + // TODO: throw exception? + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) { + return FileVisitResult.CONTINUE; + } + + }); + } catch (IOException e) { + LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + } + } + + + private static byte[] digestFromPath(Path path) { + try { + return Crypto.digest(Files.readAllBytes(path)); + } catch (IOException e) { + return null; + } + } + + private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + if (!Files.exists(source)) { + throw new IOException(String.format("File not found: %s", source.toString())); + } + + Path dest = Paths.get(base.toString(), relativePath.toString()); + LOGGER.trace("Copying {} to {}", source, dest); + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + } + + private static void markFilePathAsRemoved(Path base, Path relativePath) throws IOException { + String newFilename = relativePath.toString().concat(".removed"); + Path dest = Paths.get(base.toString(), newFilename); + File file = new File(dest.toString()); + File parent = file.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + LOGGER.info("Creating file {}", dest); + file.createNewFile(); + } + + + public Path getDiffPath() { + return this.diffPath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFileMerge.java b/src/main/java/org/qortal/storage/DataFileMerge.java new file mode 100644 index 00000000..c03018cf --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFileMerge.java @@ -0,0 +1,190 @@ +package org.qortal.storage; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.crypto.Crypto; +import org.qortal.utils.FilesystemUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; + +public class DataFileMerge { + + private static final Logger LOGGER = LogManager.getLogger(DataFileMerge.class); + + private Path pathBefore; + private Path pathAfter; + private Path mergePath; + + public DataFileMerge(Path pathBefore, Path pathAfter) { + this.pathBefore = pathBefore; + this.pathAfter = pathAfter; + } + + public void compute() throws IOException { + try { + this.preExecute(); + this.copyPreviousStateToMergePath(); + this.findDifferences(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + this.createOutputDirectory(); + } + + private void postExecute() { + + } + + private void createOutputDirectory() { + // Ensure temp folder exists + Path tempDir; + try { + tempDir = Files.createTempDirectory("qortal-diff"); + } catch (IOException e) { + throw new IllegalStateException("Unable to create temp directory"); + } + this.mergePath = tempDir; + } + + private void copyPreviousStateToMergePath() throws IOException { + DataFileMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get("")); + } + + private void findDifferences() { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path mergePathAbsolute = this.mergePath.toAbsolutePath(); + +// LOGGER.info("this.pathBefore: {}", this.pathBefore); +// LOGGER.info("this.pathAfter: {}", this.pathAfter); +// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute); +// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute); +// LOGGER.info("mergePathAbsolute: {}", mergePathAbsolute); + + + try { + // Check for additions or modifications + Files.walkFileTree(this.pathAfter, new FileVisitor() { + + @Override + public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException { + Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath()); + Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter); + + boolean wasAdded = false; + boolean wasModified = false; + boolean wasRemoved = false; + + if (after.toString().endsWith(".removed")) { + LOGGER.trace("File was removed: {}", after.toString()); + wasRemoved = true; + } + else if (!Files.exists(filePathBefore)) { + LOGGER.trace("File was added: {}", after.toString()); + wasAdded = true; + } + else if (Files.size(after) != Files.size(filePathBefore)) { + // Check file size first because it's quicker + LOGGER.trace("File size was modified: {}", after.toString()); + wasModified = true; + } + else if (!Arrays.equals(DataFileMerge.digestFromPath(after), DataFileMerge.digestFromPath(filePathBefore))) { + // Check hashes as a last resort + LOGGER.trace("File contents were modified: {}", after.toString()); + wasModified = true; + } + + if (wasAdded | wasModified) { + DataFileMerge.copyFilePathToBaseDir(after, mergePathAbsolute, filePathAfter); + } + + if (wasRemoved) { + if (filePathAfter.toString().endsWith(".removed")) { + // Trim the ".removed" + Path filePathAfterTrimmed = Paths.get(filePathAfter.toString().substring(0, filePathAfter.toString().length()-8)); + DataFileMerge.deletePathInBaseDir(mergePathAbsolute, filePathAfterTrimmed); + } + } + + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e){ + LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage()); + // TODO: throw exception? + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) { + return FileVisitResult.CONTINUE; + } + + }); + } catch (IOException e) { + LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + } + } + + + private static byte[] digestFromPath(Path path) { + try { + return Crypto.digest(Files.readAllBytes(path)); + } catch (IOException e) { + return null; + } + } + + private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + if (!Files.exists(source)) { + throw new IOException(String.format("File not found: %s", source.toString())); + } + + Path dest = Paths.get(base.toString(), relativePath.toString()); + LOGGER.trace("Copying {} to {}", source, dest); + Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + } + + private static void copyDirPathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + if (!Files.exists(source)) { + throw new IOException(String.format("File not found: %s", source.toString())); + } + + Path dest = Paths.get(base.toString(), relativePath.toString()); + LOGGER.trace("Copying {} to {}", source, dest); + FilesystemUtils.copyDirectory(source.toString(), dest.toString()); + } + + private static void deletePathInBaseDir(Path base, Path relativePath) throws IOException { + Path dest = Paths.get(base.toString(), relativePath.toString()); + File file = new File(dest.toString()); + if (file.exists() && file.isFile()) { + LOGGER.trace("Deleting file {}", dest); + Files.delete(dest); + } + if (file.exists() && file.isDirectory()) { + LOGGER.trace("Deleting directory {}", dest); + FileUtils.deleteDirectory(file); + } + } + + public Path getMergePath() { + return this.mergePath; + } + +} diff --git a/src/main/java/org/qortal/storage/DataFilePatches.java b/src/main/java/org/qortal/storage/DataFilePatches.java new file mode 100644 index 00000000..98fe2f07 --- /dev/null +++ b/src/main/java/org/qortal/storage/DataFilePatches.java @@ -0,0 +1,66 @@ +package org.qortal.storage; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.qortal.repository.DataException; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +public class DataFilePatches { + + private static final Logger LOGGER = LogManager.getLogger(DataFilePatches.class); + + private List paths; + private Path finalPath; + + public DataFilePatches(List paths) { + this.paths = paths; + } + + public void applyPatches() throws DataException, IOException { + try { + this.preExecute(); + this.process(); + + } finally { + this.postExecute(); + } + } + + private void preExecute() { + if (this.paths == null || this.paths.isEmpty()) { + throw new IllegalStateException(String.format("No paths available to build latest state")); + } + } + + private void postExecute() { + + } + + private void process() throws IOException { + if (this.paths.size() == 1) { + // No patching needed + this.finalPath = this.paths.get(0); + return; + } + + Path pathBefore = this.paths.get(0); + + // Loop from the second path onwards + for (int i=1; i() { + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { + // Don't delete the parent directory, as we want to leave an empty folder + if (dir.compareTo(uncompressedPath) == 0) { + return FileVisitResult.CONTINUE; + } + + if (e == null) { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } else { + throw e; + } + } + + }); + } catch (IOException e) { + LOGGER.info("Unable to delete file or directory: {}", e.getMessage()); + } + } + } + } + private void fetch() throws IllegalStateException, IOException, DataException { switch (resourceIdType) { + case FILE_HASH: + this.fetchFromFileHash(); + break; + + case NAME: + this.fetchFromName(); + break; + case SIGNATURE: this.fetchFromSignature(); break; - case FILE_HASH: - this.fetchFromFileHash(); + case TRANSACTION_DATA: + this.fetchFromTransactionData(this.transactionData); break; default: @@ -101,9 +163,30 @@ public class DataFileReader { } } + private void fetchFromFileHash() { + // Load data file directly from the hash + DataFile dataFile = DataFile.fromHash58(resourceId); + // Set filePath to the location of the DataFile + this.filePath = Paths.get(dataFile.getFilePath()); + } + + private void fetchFromName() throws IllegalStateException, IOException, DataException { + + // Build the existing state using past transactions + DataFileBuilder builder = new DataFileBuilder(this.resourceId, this.service); + builder.build(); + Path builtPath = builder.getFinalPath(); + if (builtPath == null) { + throw new IllegalStateException("Unable to build path"); + } + + // Set filePath to the builtPath + this.filePath = builtPath; + } + private void fetchFromSignature() throws IllegalStateException, IOException, DataException { - // Load the full transaction data so we can access the file hashes + // Load the full transaction data from the database so we can access the file hashes ArbitraryTransactionData transactionData; try (final Repository repository = RepositoryManager.getRepository()) { transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId)); @@ -112,6 +195,14 @@ public class DataFileReader { throw new IllegalStateException(String.format("Transaction data not found for signature %s", this.resourceId)); } + this.fetchFromTransactionData(transactionData); + } + + private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws IllegalStateException, IOException, DataException { + if (!(transactionData instanceof ArbitraryTransactionData)) { + throw new IllegalStateException(String.format("Transaction data not found for signature %s", this.resourceId)); + } + // Load hashes byte[] digest = transactionData.getData(); byte[] chunkHashes = transactionData.getChunkHashes(); @@ -123,19 +214,19 @@ public class DataFileReader { } // Load data file(s) - this.dataFile = DataFile.fromHash(digest); - if (!this.dataFile.exists()) { - if (!this.dataFile.allChunksExist(chunkHashes)) { + DataFile dataFile = DataFile.fromHash(digest); + if (!dataFile.exists()) { + if (!dataFile.allChunksExist(chunkHashes)) { // TODO: fetch them? throw new IllegalStateException(String.format("Missing chunks for file {}", dataFile)); } // We have all the chunks but not the complete file, so join them - this.dataFile.addChunkHashes(chunkHashes); - this.dataFile.join(); + dataFile.addChunkHashes(chunkHashes); + dataFile.join(); } // If the complete file still doesn't exist then something went wrong - if (!this.dataFile.exists()) { + if (!dataFile.exists()) { throw new IOException(String.format("File doesn't exist: %s", dataFile)); } // Ensure the complete hash matches the joined chunks @@ -146,13 +237,6 @@ public class DataFileReader { this.filePath = Paths.get(dataFile.getFilePath()); } - private void fetchFromFileHash() { - // Load data file directly from the hash - this.dataFile = DataFile.fromHash58(resourceId); - // Set filePath to the location of the DataFile - this.filePath = Paths.get(dataFile.getFilePath()); - } - private void decrypt() { // Decrypt if we have the secret key. byte[] secret = this.secret58 != null ? Base58.decode(this.secret58) : null; @@ -168,15 +252,26 @@ public class DataFileReader { } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) { - throw new IllegalStateException(String.format("Unable to decrypt file %s: %s", dataFile, e.getMessage())); + throw new IllegalStateException(String.format("Unable to decrypt file at path %s: %s", this.filePath, e.getMessage())); } } else { - // Assume it is unencrypted. We may block this in the future. - this.filePath = Paths.get(this.dataFile.getFilePath()); + // Assume it is unencrypted. This will be the case when we have built a custom path by combining + // multiple decrypted archives into a single state. } } private void uncompress() throws IOException { + if (this.filePath == null || !Files.exists(this.filePath)) { + throw new IllegalStateException("Can't uncompress non-existent file path"); + } + File file = new File(this.filePath.toString()); + if (file.isDirectory()) { + // Already a directory - nothing to uncompress + // We still need to copy the directory to its final destination if it's not already there + this.copyFilePathToFinalDestination(); + return; + } + try { // TODO: compression types //if (transactionData.getCompression() == ArbitraryTransactionData.Compression.ZIP) { @@ -191,6 +286,20 @@ public class DataFileReader { this.filePath = this.uncompressedPath; } + private void copyFilePathToFinalDestination() throws IOException { + if (this.filePath.compareTo(this.uncompressedPath) != 0) { + File source = new File(this.filePath.toString()); + File dest = new File(this.uncompressedPath.toString()); + if (source == null || !source.exists()) { + throw new IllegalStateException("Source directory doesn't exist"); + } + if (dest == null || !dest.exists()) { + throw new IllegalStateException("Destination directory doesn't exist"); + } + FilesystemUtils.copyDirectory(source.toString(), dest.toString()); + } + } + private void cleanupFilesystem() throws IOException { // Clean up if (this.uncompressedPath != null) { @@ -202,6 +311,10 @@ public class DataFileReader { } + public void setTransactionData(ArbitraryTransactionData transactionData) { + this.transactionData = transactionData; + } + public void setSecret58(String secret58) { this.secret58 = secret58; } diff --git a/src/main/java/org/qortal/storage/DataFileWriter.java b/src/main/java/org/qortal/storage/DataFileWriter.java index 539cb199..a0771f25 100644 --- a/src/main/java/org/qortal/storage/DataFileWriter.java +++ b/src/main/java/org/qortal/storage/DataFileWriter.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.crypto.AES; +import org.qortal.repository.DataException; import org.qortal.storage.DataFile.*; import org.qortal.utils.ZipUtils; @@ -26,6 +27,8 @@ public class DataFileWriter { private static final Logger LOGGER = LogManager.getLogger(DataFileWriter.class); private Path filePath; + private String name; + private Service service; private Method method; private Compression compression; @@ -37,15 +40,18 @@ public class DataFileWriter { private Path compressedPath; private Path encryptedPath; - public DataFileWriter(Path filePath, Method method, Compression compression) { + public DataFileWriter(Path filePath, String name, Service service, Method method, Compression compression) { this.filePath = filePath; + this.name = name; + this.service = service; this.method = method; this.compression = compression; } - public void save() throws IllegalStateException, IOException { + public void save() throws IllegalStateException, IOException, DataException { try { this.preExecute(); + this.process(); this.compress(); this.encrypt(); this.split(); @@ -82,6 +88,36 @@ public class DataFileWriter { this.workingPath = tempDir; } + private void process() throws DataException, IOException { + switch (this.method) { + + case PUT: + // Nothing to do + break; + + case PATCH: + this.processPatch(); + break; + + default: + throw new IllegalStateException(String.format("Unknown method specified: %s", method.toString())); + } + } + + private void processPatch() throws DataException, IOException { + + // Build the existing state using past transactions + DataFileBuilder builder = new DataFileBuilder(this.name, this.service); + builder.build(); + Path builtPath = builder.getFinalPath(); + + // Compute a diff of the latest changes on top of the previous state + // Then use only the differences as our data payload + DataFileCreatePatch patch = new DataFileCreatePatch(builtPath, this.filePath); + patch.create(); + this.filePath = patch.getFinalPath(); + } + private void compress() { // Compress the data if requested if (this.compression != Compression.NONE) { diff --git a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java index 6ee8d5ff..cb9142d8 100644 --- a/src/main/java/org/qortal/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qortal/transaction/ArbitraryTransaction.java @@ -108,7 +108,8 @@ public class ArbitraryTransaction extends Transaction { if (chunkHashes == null && expectedChunkHashesSize > 0) { return ValidationResult.INVALID_DATA_LENGTH; } - if (chunkHashes.length != expectedChunkHashesSize) { + int chunkHashesLength = chunkHashes != null ? chunkHashes.length : 0; + if (chunkHashesLength != expectedChunkHashesSize) { return ValidationResult.INVALID_DATA_LENGTH; } } diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java new file mode 100644 index 00000000..a87e18a0 --- /dev/null +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -0,0 +1,34 @@ +package org.qortal.utils; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class FilesystemUtils { + + public static boolean isDirectoryEmpty(Path path) throws IOException { + if (Files.isDirectory(path)) { + try (DirectoryStream directory = Files.newDirectoryStream(path)) { + return !directory.iterator().hasNext(); + } + } + + return false; + } + + public static void copyDirectory(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException { + Files.walk(Paths.get(sourceDirectoryLocation)) + .forEach(source -> { + Path destination = Paths.get(destinationDirectoryLocation, source.toString() + .substring(sourceDirectoryLocation.length())); + try { + Files.copy(source, destination); + } catch (IOException e) { + e.printStackTrace(); + } + }); + } + +}