diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 6e82257a..05e2696b 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -59,29 +59,32 @@ public class ArbitraryDataFile { protected Path filePath; protected String hash58; + protected byte[] signature; private ArrayList chunks; private byte[] secret; public ArbitraryDataFile() { } - public ArbitraryDataFile(String hash58) throws DataException { + public ArbitraryDataFile(String hash58, byte[] signature) throws DataException { this.createDataDirectory(); - this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, false); + this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, signature, false); this.chunks = new ArrayList<>(); this.hash58 = hash58; + this.signature = signature; } - public ArbitraryDataFile(byte[] fileContent) throws DataException { + public ArbitraryDataFile(byte[] fileContent, byte[] signature) throws DataException { if (fileContent == null) { LOGGER.error("fileContent is null"); return; } this.hash58 = Base58.encode(Crypto.digest(fileContent)); + this.signature = signature; LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length)); - Path outputFilePath = getOutputFilePath(this.hash58, true); + Path outputFilePath = getOutputFilePath(this.hash58, signature, true); File outputFile = outputFilePath.toFile(); try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { outputStream.write(fileContent); @@ -97,15 +100,15 @@ public class ArbitraryDataFile { } } - public static ArbitraryDataFile fromHash58(String hash58) throws DataException { - return new ArbitraryDataFile(hash58); + public static ArbitraryDataFile fromHash58(String hash58, byte[] signature) throws DataException { + return new ArbitraryDataFile(hash58, signature); } - public static ArbitraryDataFile fromHash(byte[] hash) throws DataException { - return ArbitraryDataFile.fromHash58(Base58.encode(hash)); + public static ArbitraryDataFile fromHash(byte[] hash, byte[] signature) throws DataException { + return ArbitraryDataFile.fromHash58(Base58.encode(hash), signature); } - public static ArbitraryDataFile fromPath(Path path) { + public static ArbitraryDataFile fromPath(Path path, byte[] signature) { if (path == null) { return null; } @@ -113,11 +116,11 @@ public class ArbitraryDataFile { if (file.exists()) { try { byte[] digest = Crypto.digest(file); - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); // Copy file to data directory if needed if (Files.exists(path) && !arbitraryDataFile.isInBaseDirectory(path)) { - arbitraryDataFile.copyToDataDirectory(path); + arbitraryDataFile.copyToDataDirectory(path, signature); } // Or, if it's already in the data directory, we may need to move it else if (!path.equals(arbitraryDataFile.getFilePath())) { @@ -134,8 +137,8 @@ public class ArbitraryDataFile { return null; } - public static ArbitraryDataFile fromFile(File file) { - return ArbitraryDataFile.fromPath(Paths.get(file.getPath())); + public static ArbitraryDataFile fromFile(File file, byte[] signature) { + return ArbitraryDataFile.fromPath(Paths.get(file.getPath()), signature); } private boolean createDataDirectory() { @@ -151,11 +154,11 @@ public class ArbitraryDataFile { return true; } - private Path copyToDataDirectory(Path sourcePath) throws DataException { + private Path copyToDataDirectory(Path sourcePath, byte[] signature) throws DataException { if (this.hash58 == null || this.filePath == null) { return null; } - Path outputFilePath = getOutputFilePath(this.hash58, true); + Path outputFilePath = getOutputFilePath(this.hash58, signature, true); sourcePath = sourcePath.toAbsolutePath(); Path destPath = outputFilePath.toAbsolutePath(); try { @@ -165,13 +168,25 @@ public class ArbitraryDataFile { } } - public static Path getOutputFilePath(String hash58, boolean createDirectories) throws DataException { + public static Path getOutputFilePath(String hash58, byte[] signature, boolean createDirectories) throws DataException { + Path directory; + if (hash58 == null) { return null; } - String hash58First2Chars = hash58.substring(0, 2).toLowerCase(); - String hash58Next2Chars = hash58.substring(2, 4).toLowerCase(); - Path directory = Paths.get(Settings.getInstance().getDataPath(), hash58First2Chars, hash58Next2Chars); + if (signature != null) { + // Key by signature + String signature58 = Base58.encode(signature); + String sig58First2Chars = signature58.substring(0, 2).toLowerCase(); + String sig58Next2Chars = signature58.substring(2, 4).toLowerCase(); + directory = Paths.get(Settings.getInstance().getDataPath(), sig58First2Chars, sig58Next2Chars, signature58); + } + else { + // Put files without signatures in a "_misc" directory, and the files will be relocated later + String hash58First2Chars = hash58.substring(0, 2).toLowerCase(); + String hash58Next2Chars = hash58.substring(2, 4).toLowerCase(); + directory = Paths.get(Settings.getInstance().getDataPath(), "_misc", hash58First2Chars, hash58Next2Chars); + } if (createDirectories) { try { @@ -217,7 +232,7 @@ public class ArbitraryDataFile { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH]; byteBuffer.get(chunkDigest); - ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkDigest); + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkDigest, this.signature); this.addChunk(chunk); } } @@ -252,7 +267,7 @@ public class ArbitraryDataFile { out.write(buffer, 0, numberOfBytes); out.flush(); - ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray()); + ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray(), this.signature); ValidationResult validationResult = chunk.isValid(); if (validationResult == ValidationResult.OK) { this.chunks.add(chunk); @@ -301,7 +316,7 @@ public class ArbitraryDataFile { out.close(); // Copy temporary file to data directory - this.filePath = this.copyToDataDirectory(outputPath); + this.filePath = this.copyToDataDirectory(outputPath, this.signature); if (FilesystemUtils.pathInsideDataOrTempPath(outputPath)) { Files.delete(outputPath); } @@ -425,7 +440,7 @@ public class ArbitraryDataFile { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH]; byteBuffer.get(chunkHash); - ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash); + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature); if (!chunk.exists()) { return false; } @@ -441,7 +456,7 @@ public class ArbitraryDataFile { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH]; byteBuffer.get(chunkHash); - ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash); + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature); if (chunk.exists()) { return true; } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java index b5adcace..b113fbba 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java @@ -13,20 +13,20 @@ public class ArbitraryDataFileChunk extends ArbitraryDataFile { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class); - public ArbitraryDataFileChunk(String hash58) throws DataException { - super(hash58); + public ArbitraryDataFileChunk(String hash58, byte[] signature) throws DataException { + super(hash58, signature); } - public ArbitraryDataFileChunk(byte[] fileContent) throws DataException { - super(fileContent); + public ArbitraryDataFileChunk(byte[] fileContent, byte[] signature) throws DataException { + super(fileContent, signature); } - public static ArbitraryDataFileChunk fromHash58(String hash58) throws DataException { - return new ArbitraryDataFileChunk(hash58); + public static ArbitraryDataFileChunk fromHash58(String hash58, byte[] signature) throws DataException { + return new ArbitraryDataFileChunk(hash58, signature); } - public static ArbitraryDataFileChunk fromHash(byte[] hash) throws DataException { - return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash)); + public static ArbitraryDataFileChunk fromHash(byte[] hash, byte[] signature) throws DataException { + return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash), signature); } @Override diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index cc701d85..62fc83db 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -18,6 +18,7 @@ import org.qortal.repository.RepositoryManager; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.settings.Settings; import org.qortal.transform.Transformer; +import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; import org.qortal.utils.FilesystemUtils; import org.qortal.utils.ZipUtils; @@ -75,7 +76,7 @@ public class ArbitraryDataReader { this.identifier = identifier; this.workingPath = this.buildWorkingPath(); - this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); + this.uncompressedPath = Paths.get(this.workingPath.toString(), "data"); // By default we can request missing files // Callers can use setCanRequestMissingFiles(false) to prevent it @@ -249,8 +250,8 @@ public class ArbitraryDataReader { } private void fetchFromFileHash() throws DataException { - // Load data file directly from the hash - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId); + // Load data file directly from the hash (without a signature) + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId, null); // Set filePath to the location of the ArbitraryDataFile this.filePath = arbitraryDataFile.getFilePath(); } @@ -303,6 +304,7 @@ public class ArbitraryDataReader { // Load hashes byte[] digest = transactionData.getData(); byte[] chunkHashes = transactionData.getChunkHashes(); + byte[] signature = transactionData.getSignature(); // Load secret byte[] secret = transactionData.getSecret(); @@ -311,7 +313,8 @@ public class ArbitraryDataReader { } // Load data file(s) - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData); if (!arbitraryDataFile.exists()) { if (!arbitraryDataFile.allChunksExist(chunkHashes) || chunkHashes == null) { if (ArbitraryDataStorageManager.getInstance().isNameInBlacklist(transactionData.getName())) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index d5b55f2b..49f698ed 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -56,7 +56,7 @@ public class ArbitraryDataWriter { this.compression = compression; } - public void save() throws DataException, IOException, DataException, InterruptedException, MissingDataException { + public void save() throws IOException, DataException, InterruptedException, MissingDataException { try { this.preExecute(); this.validateService(); @@ -251,7 +251,8 @@ public class ArbitraryDataWriter { } private void split() throws IOException, DataException { - this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath); + // We don't have a signature yet, so use null to put the file in a generic folder + this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath, null); if (this.arbitraryDataFile == null) { throw new IOException("No file available when trying to split"); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index cf7058d8..2ab904ab 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -493,11 +493,11 @@ public class ArbitraryDataManager extends Thread { // Fetch data files by hash - private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, byte[] hash) { + private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, byte[] signature, byte[] hash) { String hash58 = Base58.encode(hash); LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); arbitraryDataFileRequests.put(hash58, NTP.getTime()); - Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(hash); + Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash); Message message = null; try { @@ -624,7 +624,7 @@ public class ArbitraryDataManager extends Thread { List hashes) throws DataException { // Load data file(s) - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData()); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature); arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); // If hashes are null, we will treat this to mean all data hashes associated with this file @@ -646,13 +646,13 @@ public class ArbitraryDataManager extends Thread { if (!arbitraryDataFile.chunkExists(hash)) { // Only request the file if we aren't already requesting it from someone else if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) { - ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, hash); + ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, signature, hash); if (receivedArbitraryDataFile != null) { LOGGER.info("Received data file {} from peer {}", receivedArbitraryDataFile, peer); receivedAtLeastOneFile = true; } else { - LOGGER.info("Peer {} didn't respond with data file {}", peer, hash); + LOGGER.info("Peer {} didn't respond with data file {}", peer, Base58.encode(hash)); } } else { @@ -769,7 +769,7 @@ public class ArbitraryDataManager extends Thread { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; // Load data file(s) - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData()); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature); arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); // Check all hashes exist @@ -809,13 +809,14 @@ public class ArbitraryDataManager extends Thread { public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) { GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message; byte[] hash = getArbitraryDataFileMessage.getHash(); + byte[] signature = getArbitraryDataFileMessage.getSignature(); Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet(); try { - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); if (arbitraryDataFile.exists()) { - ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(arbitraryDataFile); + ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile); arbitraryDataFileMessage.setId(message.getId()); if (!peer.sendMessage(arbitraryDataFileMessage)) { LOGGER.info("Couldn't sent file"); @@ -829,7 +830,7 @@ public class ArbitraryDataManager extends Thread { Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement(); // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout - LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile)); + LOGGER.debug(String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile)); // We'll send empty block summaries message as it's very short // TODO: use a different message type here @@ -869,7 +870,7 @@ public class ArbitraryDataManager extends Thread { byte[] chunkHashes = transactionData.getChunkHashes(); // Load file(s) and add any that exist to the list of hashes - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); if (chunkHashes != null && chunkHashes.length > 0) { arbitraryDataFile.addChunkHashes(chunkHashes); for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) { diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java index b1fc21eb..d87e9685 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -3,6 +3,7 @@ package org.qortal.network.message; import com.google.common.primitives.Ints; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.repository.DataException; +import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -10,18 +11,23 @@ import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class ArbitraryDataFileMessage extends Message { - + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + + private final byte[] signature; private final ArbitraryDataFile arbitraryDataFile; - public ArbitraryDataFileMessage(ArbitraryDataFile arbitraryDataFile) { + public ArbitraryDataFileMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) { super(MessageType.ARBITRARY_DATA_FILE); + this.signature = signature; this.arbitraryDataFile = arbitraryDataFile; } - public ArbitraryDataFileMessage(int id, ArbitraryDataFile arbitraryDataFile) { + public ArbitraryDataFileMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) { super(id, MessageType.ARBITRARY_DATA_FILE); + this.signature = signature; this.arbitraryDataFile = arbitraryDataFile; } @@ -30,6 +36,9 @@ public class ArbitraryDataFileMessage extends Message { } public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + byte[] signature = new byte[SIGNATURE_LENGTH]; + byteBuffer.get(signature); + int dataLength = byteBuffer.getInt(); if (byteBuffer.remaining() != dataLength) @@ -39,8 +48,8 @@ public class ArbitraryDataFileMessage extends Message { byteBuffer.get(data); try { - ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data); - return new ArbitraryDataFileMessage(id, arbitraryDataFile); + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data, signature); + return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile); } catch (DataException e) { return null; @@ -61,6 +70,8 @@ public class ArbitraryDataFileMessage extends Message { try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + bytes.write(signature); + bytes.write(Ints.toByteArray(data.length)); bytes.write(data); @@ -72,7 +83,7 @@ public class ArbitraryDataFileMessage extends Message { } public ArbitraryDataFileMessage cloneWithNewId(int newId) { - ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.arbitraryDataFile); + ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.signature, this.arbitraryDataFile); clone.setId(newId); return clone; } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java index f13951b7..809b983d 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java @@ -1,5 +1,6 @@ package org.qortal.network.message; +import org.qortal.transform.Transformer; import org.qortal.transform.transaction.TransactionTransformer; import java.io.ByteArrayOutputStream; @@ -9,33 +10,42 @@ import java.nio.ByteBuffer; public class GetArbitraryDataFileMessage extends Message { + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; + private final byte[] signature; private final byte[] hash; - public GetArbitraryDataFileMessage(byte[] hash) { - this(-1, hash); + public GetArbitraryDataFileMessage(byte[] signature, byte[] hash) { + this(-1, signature, hash); } - private GetArbitraryDataFileMessage(int id, byte[] hash) { + private GetArbitraryDataFileMessage(int id, byte[] signature, byte[] hash) { super(id, MessageType.GET_ARBITRARY_DATA_FILE); + this.signature = signature; this.hash = hash; } + public byte[] getSignature() { + return this.signature; + } + public byte[] getHash() { return this.hash; } public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != HASH_LENGTH) + if (bytes.remaining() != HASH_LENGTH + SIGNATURE_LENGTH) return null; - byte[] hash = new byte[HASH_LENGTH]; + byte[] signature = new byte[SIGNATURE_LENGTH]; + bytes.get(signature); + byte[] hash = new byte[HASH_LENGTH]; bytes.get(hash); - return new GetArbitraryDataFileMessage(id, hash); + return new GetArbitraryDataFileMessage(id, signature, hash); } @Override @@ -43,6 +53,8 @@ public class GetArbitraryDataFileMessage extends Message { try { ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + bytes.write(this.signature); + bytes.write(this.hash); return bytes.toByteArray(); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index dd377a3b..7e82b800 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -12,6 +12,7 @@ import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.DataException; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.transaction.Transaction.ApprovalStatus; +import org.qortal.utils.ArbitraryTransactionUtils; import java.sql.ResultSet; import java.sql.SQLException; @@ -53,24 +54,23 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { byte[] chunkHashes = transactionData.getChunkHashes(); // Load data file(s) - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); if (chunkHashes != null && chunkHashes.length > 0) { arbitraryDataFile.addChunkHashes(chunkHashes); } - // Check if we already have the complete data file - if (arbitraryDataFile.exists()) { + // Check if we already have the complete data file or all chunks + if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(chunkHashes)) { return true; } - // If this transaction doesn't have any chunks, then we require the complete file - if (chunkHashes == null) { - return false; - } - - // Alternatively, if we have all the chunks, then it's safe to assume the data is local - if (arbitraryDataFile.allChunksExist(chunkHashes)) { - return true; + // We may need to relocate files from the "misc_" folder to the signature folder + int relocatedCount = ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData); + if (relocatedCount > 0) { + // Files were relocated, so check again to see if they exist in the correct place + if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(chunkHashes)) { + return true; + } } return false; @@ -93,7 +93,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { byte[] chunkHashes = transactionData.getChunkHashes(); // Load data file(s) - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); if (chunkHashes != null && chunkHashes.length > 0) { arbitraryDataFile.addChunkHashes(chunkHashes); } @@ -143,7 +143,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); // Load data file(s) - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + byte[] signature = arbitraryTransactionData.getSignature(); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); if (chunkHashes != null && chunkHashes.length > 0) { arbitraryDataFile.addChunkHashes(chunkHashes); } diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index f9cb5f83..066f602a 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -3,6 +3,7 @@ package org.qortal.utils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataFileChunk; import org.qortal.arbitrary.misc.Service; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; @@ -15,6 +16,9 @@ import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + + public class ArbitraryTransactionUtils { private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransactionUtils.class); @@ -83,9 +87,10 @@ public class ArbitraryTransactionUtils { } byte[] digest = transactionData.getData(); + byte[] signature = transactionData.getSignature(); // Load complete file - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); return arbitraryDataFile.exists(); } @@ -97,6 +102,7 @@ public class ArbitraryTransactionUtils { byte[] digest = transactionData.getData(); byte[] chunkHashes = transactionData.getChunkHashes(); + byte[] signature = transactionData.getSignature(); if (chunkHashes == null) { // This file doesn't have any chunks, which is the same as us having them all @@ -104,7 +110,7 @@ public class ArbitraryTransactionUtils { } // Load complete file and chunks - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); if (chunkHashes != null && chunkHashes.length > 0) { arbitraryDataFile.addChunkHashes(chunkHashes); } @@ -118,6 +124,7 @@ public class ArbitraryTransactionUtils { byte[] digest = transactionData.getData(); byte[] chunkHashes = transactionData.getChunkHashes(); + byte[] signature = transactionData.getSignature(); if (chunkHashes == null) { // This file doesn't have any chunks, which means none exist @@ -125,7 +132,7 @@ public class ArbitraryTransactionUtils { } // Load complete file and chunks - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); if (chunkHashes != null && chunkHashes.length > 0) { arbitraryDataFile.addChunkHashes(chunkHashes); } @@ -139,6 +146,7 @@ public class ArbitraryTransactionUtils { byte[] digest = transactionData.getData(); byte[] chunkHashes = transactionData.getChunkHashes(); + byte[] signature = transactionData.getSignature(); if (chunkHashes == null) { // This file doesn't have any chunks @@ -146,7 +154,7 @@ public class ArbitraryTransactionUtils { } // Load complete file and chunks - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); if (chunkHashes != null && chunkHashes.length > 0) { arbitraryDataFile.addChunkHashes(chunkHashes); } @@ -174,8 +182,8 @@ public class ArbitraryTransactionUtils { return true; } - public static boolean isFileHashRecent(byte[] hash, long now, long cleanupAfter) throws DataException { - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); + public static boolean isFileHashRecent(byte[] hash, byte[] signature, long now, long cleanupAfter) throws DataException { + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); if (arbitraryDataFile == null || !arbitraryDataFile.exists()) { // No hash, or file doesn't exist, so it's not recent return false; @@ -188,11 +196,12 @@ public class ArbitraryTransactionUtils { public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { byte[] completeHash = arbitraryTransactionData.getData(); byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + byte[] signature = arbitraryTransactionData.getSignature(); - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature); arbitraryDataFile.addChunkHashes(chunkHashes); - if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, now, cleanupAfter)) { + if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, signature, now, cleanupAfter)) { LOGGER.info("Deleting file {} because it can be rebuilt from chunks " + "if needed", Base58.encode(completeHash)); @@ -203,8 +212,9 @@ public class ArbitraryTransactionUtils { public static void deleteCompleteFileAndChunks(ArbitraryTransactionData arbitraryTransactionData) throws DataException { byte[] completeHash = arbitraryTransactionData.getData(); byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + byte[] signature = arbitraryTransactionData.getSignature(); - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature); arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.deleteAll(); } @@ -212,9 +222,10 @@ public class ArbitraryTransactionUtils { public static void convertFileToChunks(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { byte[] completeHash = arbitraryTransactionData.getData(); byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + byte[] signature = arbitraryTransactionData.getSignature(); // Split the file into chunks - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature); int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); if (chunkCount > 1) { LOGGER.info(String.format("Successfully split %s into %d chunk%s", @@ -226,7 +237,7 @@ public class ArbitraryTransactionUtils { if (arbitraryDataFile.allChunksExist(chunkHashes)) { // Now delete the original file if it's not recent - if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, now, cleanupAfter)) { + if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, signature, now, cleanupAfter)) { LOGGER.info("Deleting file {} because it can now be rebuilt from " + "chunks if needed", Base58.encode(completeHash)); @@ -240,4 +251,73 @@ public class ArbitraryTransactionUtils { } } + /** + * When first uploaded, files go into a _misc folder as they are not yet associated with a + * transaction signature. Once the transaction is broadcast, they need to be moved to the + * correct location, keyed by the transaction signature. + * @param arbitraryTransactionData + * @return + * @throws DataException + */ + public static int checkAndRelocateMiscFiles(ArbitraryTransactionData arbitraryTransactionData) { + int filesRelocatedCount = 0; + + try { + // Load hashes + byte[] digest = arbitraryTransactionData.getData(); + byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); + + // Load signature + byte[] signature = arbitraryTransactionData.getSignature(); + + // Check if any files for this transaction exist in the misc folder + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, null); + if (chunkHashes != null && chunkHashes.length > 0) { + arbitraryDataFile.addChunkHashes(chunkHashes); + } + if (arbitraryDataFile.anyChunksExist(chunkHashes)) { + // At least one chunk exists in the misc folder - move them + for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) { + if (chunk.exists()) { + // Determine the correct path by initializing a new ArbitraryDataFile instance with the signature + ArbitraryDataFile newChunk = ArbitraryDataFile.fromHash(chunk.getHash(), signature); + Path oldPath = chunk.getFilePath(); + Path newPath = newChunk.getFilePath(); + + // Ensure parent directories exist, then copy the file + LOGGER.info("Relocating chunk from {} to {}...", oldPath, newPath); + Files.createDirectories(newPath.getParent()); + Files.move(oldPath, newPath, REPLACE_EXISTING); + filesRelocatedCount++; + + // Delete empty parent directories + FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath); + } + } + } + // Also move the complete file if it exists + if (arbitraryDataFile.exists()) { + // Determine the correct path by initializing a new ArbitraryDataFile instance with the signature + ArbitraryDataFile newCompleteFile = ArbitraryDataFile.fromHash(arbitraryDataFile.getHash(), signature); + Path oldPath = arbitraryDataFile.getFilePath(); + Path newPath = newCompleteFile.getFilePath(); + + // Ensure parent directories exist, then copy the file + LOGGER.info("Relocating complete file from {} to {}...", oldPath, newPath); + Files.createDirectories(newPath.getParent()); + Files.move(oldPath, newPath, REPLACE_EXISTING); + filesRelocatedCount++; + + // Delete empty parent directories + FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath); + } + } + catch (DataException | IOException e) { + LOGGER.info("Unable to check and relocate all files for signature {}: {}", + Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage()); + } + + return filesRelocatedCount; + } + } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java index 2bbbc776..aabbe502 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java @@ -20,7 +20,7 @@ public class ArbitraryDataFileTests extends Common { @Test public void testSplitAndJoin() throws DataException { String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; - ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes()); + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes(), null); assertTrue(arbitraryDataFile.exists()); assertEquals(62, arbitraryDataFile.size()); assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58()); @@ -50,7 +50,7 @@ public class ArbitraryDataFileTests extends Common { byte[] randomData = new byte[fileSize]; new Random().nextBytes(randomData); // No need for SecureRandom here - ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData); + ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData, null); assertTrue(arbitraryDataFile.exists()); assertEquals(fileSize, arbitraryDataFile.size()); String originalFileDigest = arbitraryDataFile.digest58();