mirror of
https://github.com/Qortal/qortal.git
synced 2025-02-12 10:15:49 +00:00
Major rework of local data directory structure
Files are now keyed by signature, in the format: data/si/gn/signature/hash For times when there is no signature available (i.e. at the time of initial upload), files are keyed by hash, in the format: data/_misc/ha/sh/hash Files in the _misc folder are subsequently relocated to a path that is keyed by the resulting signature. The end result is that chunks are now grouped on the filesystem by signature. This allows more transparency as to what is being hosted, and will also help simplify the reporting and management of local files.
This commit is contained in:
CalDescent
parent
0dd43d5c9a
commit
bc38184ebf
@ -59,29 +59,32 @@ public class ArbitraryDataFile {
|
||||
|
||||
protected Path filePath;
|
||||
protected String hash58;
|
||||
protected byte[] signature;
|
||||
private ArrayList<ArbitraryDataFileChunk> chunks;
|
||||
private byte[] secret;
|
||||
|
||||
public ArbitraryDataFile() {
|
||||
}
|
||||
|
||||
public ArbitraryDataFile(String hash58) throws DataException {
|
||||
public ArbitraryDataFile(String hash58, byte[] signature) throws DataException {
|
||||
this.createDataDirectory();
|
||||
this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, false);
|
||||
this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, signature, false);
|
||||
this.chunks = new ArrayList<>();
|
||||
this.hash58 = hash58;
|
||||
this.signature = signature;
|
||||
}
|
||||
|
||||
public ArbitraryDataFile(byte[] fileContent) throws DataException {
|
||||
public ArbitraryDataFile(byte[] fileContent, byte[] signature) throws DataException {
|
||||
if (fileContent == null) {
|
||||
LOGGER.error("fileContent is null");
|
||||
return;
|
||||
}
|
||||
|
||||
this.hash58 = Base58.encode(Crypto.digest(fileContent));
|
||||
this.signature = signature;
|
||||
LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length));
|
||||
|
||||
Path outputFilePath = getOutputFilePath(this.hash58, true);
|
||||
Path outputFilePath = getOutputFilePath(this.hash58, signature, true);
|
||||
File outputFile = outputFilePath.toFile();
|
||||
try (FileOutputStream outputStream = new FileOutputStream(outputFile)) {
|
||||
outputStream.write(fileContent);
|
||||
@ -97,15 +100,15 @@ public class ArbitraryDataFile {
|
||||
}
|
||||
}
|
||||
|
||||
public static ArbitraryDataFile fromHash58(String hash58) throws DataException {
|
||||
return new ArbitraryDataFile(hash58);
|
||||
public static ArbitraryDataFile fromHash58(String hash58, byte[] signature) throws DataException {
|
||||
return new ArbitraryDataFile(hash58, signature);
|
||||
}
|
||||
|
||||
public static ArbitraryDataFile fromHash(byte[] hash) throws DataException {
|
||||
return ArbitraryDataFile.fromHash58(Base58.encode(hash));
|
||||
public static ArbitraryDataFile fromHash(byte[] hash, byte[] signature) throws DataException {
|
||||
return ArbitraryDataFile.fromHash58(Base58.encode(hash), signature);
|
||||
}
|
||||
|
||||
public static ArbitraryDataFile fromPath(Path path) {
|
||||
public static ArbitraryDataFile fromPath(Path path, byte[] signature) {
|
||||
if (path == null) {
|
||||
return null;
|
||||
}
|
||||
@ -113,11 +116,11 @@ public class ArbitraryDataFile {
|
||||
if (file.exists()) {
|
||||
try {
|
||||
byte[] digest = Crypto.digest(file);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
|
||||
|
||||
// Copy file to data directory if needed
|
||||
if (Files.exists(path) && !arbitraryDataFile.isInBaseDirectory(path)) {
|
||||
arbitraryDataFile.copyToDataDirectory(path);
|
||||
arbitraryDataFile.copyToDataDirectory(path, signature);
|
||||
}
|
||||
// Or, if it's already in the data directory, we may need to move it
|
||||
else if (!path.equals(arbitraryDataFile.getFilePath())) {
|
||||
@ -134,8 +137,8 @@ public class ArbitraryDataFile {
|
||||
return null;
|
||||
}
|
||||
|
||||
public static ArbitraryDataFile fromFile(File file) {
|
||||
return ArbitraryDataFile.fromPath(Paths.get(file.getPath()));
|
||||
public static ArbitraryDataFile fromFile(File file, byte[] signature) {
|
||||
return ArbitraryDataFile.fromPath(Paths.get(file.getPath()), signature);
|
||||
}
|
||||
|
||||
private boolean createDataDirectory() {
|
||||
@ -151,11 +154,11 @@ public class ArbitraryDataFile {
|
||||
return true;
|
||||
}
|
||||
|
||||
private Path copyToDataDirectory(Path sourcePath) throws DataException {
|
||||
private Path copyToDataDirectory(Path sourcePath, byte[] signature) throws DataException {
|
||||
if (this.hash58 == null || this.filePath == null) {
|
||||
return null;
|
||||
}
|
||||
Path outputFilePath = getOutputFilePath(this.hash58, true);
|
||||
Path outputFilePath = getOutputFilePath(this.hash58, signature, true);
|
||||
sourcePath = sourcePath.toAbsolutePath();
|
||||
Path destPath = outputFilePath.toAbsolutePath();
|
||||
try {
|
||||
@ -165,13 +168,25 @@ public class ArbitraryDataFile {
|
||||
}
|
||||
}
|
||||
|
||||
public static Path getOutputFilePath(String hash58, boolean createDirectories) throws DataException {
|
||||
public static Path getOutputFilePath(String hash58, byte[] signature, boolean createDirectories) throws DataException {
|
||||
Path directory;
|
||||
|
||||
if (hash58 == null) {
|
||||
return null;
|
||||
}
|
||||
String hash58First2Chars = hash58.substring(0, 2).toLowerCase();
|
||||
String hash58Next2Chars = hash58.substring(2, 4).toLowerCase();
|
||||
Path directory = Paths.get(Settings.getInstance().getDataPath(), hash58First2Chars, hash58Next2Chars);
|
||||
if (signature != null) {
|
||||
// Key by signature
|
||||
String signature58 = Base58.encode(signature);
|
||||
String sig58First2Chars = signature58.substring(0, 2).toLowerCase();
|
||||
String sig58Next2Chars = signature58.substring(2, 4).toLowerCase();
|
||||
directory = Paths.get(Settings.getInstance().getDataPath(), sig58First2Chars, sig58Next2Chars, signature58);
|
||||
}
|
||||
else {
|
||||
// Put files without signatures in a "_misc" directory, and the files will be relocated later
|
||||
String hash58First2Chars = hash58.substring(0, 2).toLowerCase();
|
||||
String hash58Next2Chars = hash58.substring(2, 4).toLowerCase();
|
||||
directory = Paths.get(Settings.getInstance().getDataPath(), "_misc", hash58First2Chars, hash58Next2Chars);
|
||||
}
|
||||
|
||||
if (createDirectories) {
|
||||
try {
|
||||
@ -217,7 +232,7 @@ public class ArbitraryDataFile {
|
||||
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
|
||||
byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH];
|
||||
byteBuffer.get(chunkDigest);
|
||||
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkDigest);
|
||||
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkDigest, this.signature);
|
||||
this.addChunk(chunk);
|
||||
}
|
||||
}
|
||||
@ -252,7 +267,7 @@ public class ArbitraryDataFile {
|
||||
out.write(buffer, 0, numberOfBytes);
|
||||
out.flush();
|
||||
|
||||
ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray());
|
||||
ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray(), this.signature);
|
||||
ValidationResult validationResult = chunk.isValid();
|
||||
if (validationResult == ValidationResult.OK) {
|
||||
this.chunks.add(chunk);
|
||||
@ -301,7 +316,7 @@ public class ArbitraryDataFile {
|
||||
out.close();
|
||||
|
||||
// Copy temporary file to data directory
|
||||
this.filePath = this.copyToDataDirectory(outputPath);
|
||||
this.filePath = this.copyToDataDirectory(outputPath, this.signature);
|
||||
if (FilesystemUtils.pathInsideDataOrTempPath(outputPath)) {
|
||||
Files.delete(outputPath);
|
||||
}
|
||||
@ -425,7 +440,7 @@ public class ArbitraryDataFile {
|
||||
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
|
||||
byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH];
|
||||
byteBuffer.get(chunkHash);
|
||||
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash);
|
||||
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
|
||||
if (!chunk.exists()) {
|
||||
return false;
|
||||
}
|
||||
@ -441,7 +456,7 @@ public class ArbitraryDataFile {
|
||||
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
|
||||
byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH];
|
||||
byteBuffer.get(chunkHash);
|
||||
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash);
|
||||
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
|
||||
if (chunk.exists()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -13,20 +13,20 @@ public class ArbitraryDataFileChunk extends ArbitraryDataFile {
|
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class);
|
||||
|
||||
public ArbitraryDataFileChunk(String hash58) throws DataException {
|
||||
super(hash58);
|
||||
public ArbitraryDataFileChunk(String hash58, byte[] signature) throws DataException {
|
||||
super(hash58, signature);
|
||||
}
|
||||
|
||||
public ArbitraryDataFileChunk(byte[] fileContent) throws DataException {
|
||||
super(fileContent);
|
||||
public ArbitraryDataFileChunk(byte[] fileContent, byte[] signature) throws DataException {
|
||||
super(fileContent, signature);
|
||||
}
|
||||
|
||||
public static ArbitraryDataFileChunk fromHash58(String hash58) throws DataException {
|
||||
return new ArbitraryDataFileChunk(hash58);
|
||||
public static ArbitraryDataFileChunk fromHash58(String hash58, byte[] signature) throws DataException {
|
||||
return new ArbitraryDataFileChunk(hash58, signature);
|
||||
}
|
||||
|
||||
public static ArbitraryDataFileChunk fromHash(byte[] hash) throws DataException {
|
||||
return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash));
|
||||
public static ArbitraryDataFileChunk fromHash(byte[] hash, byte[] signature) throws DataException {
|
||||
return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash), signature);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -18,6 +18,7 @@ import org.qortal.repository.RepositoryManager;
|
||||
import org.qortal.arbitrary.ArbitraryDataFile.*;
|
||||
import org.qortal.settings.Settings;
|
||||
import org.qortal.transform.Transformer;
|
||||
import org.qortal.utils.ArbitraryTransactionUtils;
|
||||
import org.qortal.utils.Base58;
|
||||
import org.qortal.utils.FilesystemUtils;
|
||||
import org.qortal.utils.ZipUtils;
|
||||
@ -75,7 +76,7 @@ public class ArbitraryDataReader {
|
||||
this.identifier = identifier;
|
||||
|
||||
this.workingPath = this.buildWorkingPath();
|
||||
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
|
||||
this.uncompressedPath = Paths.get(this.workingPath.toString(), "data");
|
||||
|
||||
// By default we can request missing files
|
||||
// Callers can use setCanRequestMissingFiles(false) to prevent it
|
||||
@ -249,8 +250,8 @@ public class ArbitraryDataReader {
|
||||
}
|
||||
|
||||
private void fetchFromFileHash() throws DataException {
|
||||
// Load data file directly from the hash
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId);
|
||||
// Load data file directly from the hash (without a signature)
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId, null);
|
||||
// Set filePath to the location of the ArbitraryDataFile
|
||||
this.filePath = arbitraryDataFile.getFilePath();
|
||||
}
|
||||
@ -303,6 +304,7 @@ public class ArbitraryDataReader {
|
||||
// Load hashes
|
||||
byte[] digest = transactionData.getData();
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
byte[] signature = transactionData.getSignature();
|
||||
|
||||
// Load secret
|
||||
byte[] secret = transactionData.getSecret();
|
||||
@ -311,7 +313,8 @@ public class ArbitraryDataReader {
|
||||
}
|
||||
|
||||
// Load data file(s)
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
|
||||
ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
|
||||
if (!arbitraryDataFile.exists()) {
|
||||
if (!arbitraryDataFile.allChunksExist(chunkHashes) || chunkHashes == null) {
|
||||
if (ArbitraryDataStorageManager.getInstance().isNameInBlacklist(transactionData.getName())) {
|
||||
|
||||
@ -56,7 +56,7 @@ public class ArbitraryDataWriter {
|
||||
this.compression = compression;
|
||||
}
|
||||
|
||||
public void save() throws DataException, IOException, DataException, InterruptedException, MissingDataException {
|
||||
public void save() throws IOException, DataException, InterruptedException, MissingDataException {
|
||||
try {
|
||||
this.preExecute();
|
||||
this.validateService();
|
||||
@ -251,7 +251,8 @@ public class ArbitraryDataWriter {
|
||||
}
|
||||
|
||||
private void split() throws IOException, DataException {
|
||||
this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath);
|
||||
// We don't have a signature yet, so use null to put the file in a generic folder
|
||||
this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath, null);
|
||||
if (this.arbitraryDataFile == null) {
|
||||
throw new IOException("No file available when trying to split");
|
||||
}
|
||||
|
||||
@ -493,11 +493,11 @@ public class ArbitraryDataManager extends Thread {
|
||||
|
||||
// Fetch data files by hash
|
||||
|
||||
private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, byte[] hash) {
|
||||
private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, byte[] signature, byte[] hash) {
|
||||
String hash58 = Base58.encode(hash);
|
||||
LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer));
|
||||
arbitraryDataFileRequests.put(hash58, NTP.getTime());
|
||||
Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(hash);
|
||||
Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash);
|
||||
|
||||
Message message = null;
|
||||
try {
|
||||
@ -624,7 +624,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
List<byte[]> hashes) throws DataException {
|
||||
|
||||
// Load data file(s)
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData());
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature);
|
||||
arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes());
|
||||
|
||||
// If hashes are null, we will treat this to mean all data hashes associated with this file
|
||||
@ -646,13 +646,13 @@ public class ArbitraryDataManager extends Thread {
|
||||
if (!arbitraryDataFile.chunkExists(hash)) {
|
||||
// Only request the file if we aren't already requesting it from someone else
|
||||
if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
|
||||
ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, hash);
|
||||
ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, signature, hash);
|
||||
if (receivedArbitraryDataFile != null) {
|
||||
LOGGER.info("Received data file {} from peer {}", receivedArbitraryDataFile, peer);
|
||||
receivedAtLeastOneFile = true;
|
||||
}
|
||||
else {
|
||||
LOGGER.info("Peer {} didn't respond with data file {}", peer, hash);
|
||||
LOGGER.info("Peer {} didn't respond with data file {}", peer, Base58.encode(hash));
|
||||
}
|
||||
}
|
||||
else {
|
||||
@ -769,7 +769,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
|
||||
|
||||
// Load data file(s)
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData());
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature);
|
||||
arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes());
|
||||
|
||||
// Check all hashes exist
|
||||
@ -809,13 +809,14 @@ public class ArbitraryDataManager extends Thread {
|
||||
public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) {
|
||||
GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message;
|
||||
byte[] hash = getArbitraryDataFileMessage.getHash();
|
||||
byte[] signature = getArbitraryDataFileMessage.getSignature();
|
||||
Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet();
|
||||
|
||||
try {
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
|
||||
|
||||
if (arbitraryDataFile.exists()) {
|
||||
ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(arbitraryDataFile);
|
||||
ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile);
|
||||
arbitraryDataFileMessage.setId(message.getId());
|
||||
if (!peer.sendMessage(arbitraryDataFileMessage)) {
|
||||
LOGGER.info("Couldn't sent file");
|
||||
@ -829,7 +830,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement();
|
||||
|
||||
// Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout
|
||||
LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile));
|
||||
LOGGER.debug(String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile));
|
||||
|
||||
// We'll send empty block summaries message as it's very short
|
||||
// TODO: use a different message type here
|
||||
@ -869,7 +870,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
|
||||
// Load file(s) and add any that exist to the list of hashes
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) {
|
||||
|
||||
@ -3,6 +3,7 @@ package org.qortal.network.message;
|
||||
import com.google.common.primitives.Ints;
|
||||
import org.qortal.arbitrary.ArbitraryDataFile;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.transform.Transformer;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
@ -11,17 +12,22 @@ import java.nio.ByteBuffer;
|
||||
|
||||
public class ArbitraryDataFileMessage extends Message {
|
||||
|
||||
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
|
||||
|
||||
private final byte[] signature;
|
||||
private final ArbitraryDataFile arbitraryDataFile;
|
||||
|
||||
public ArbitraryDataFileMessage(ArbitraryDataFile arbitraryDataFile) {
|
||||
public ArbitraryDataFileMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) {
|
||||
super(MessageType.ARBITRARY_DATA_FILE);
|
||||
|
||||
this.signature = signature;
|
||||
this.arbitraryDataFile = arbitraryDataFile;
|
||||
}
|
||||
|
||||
public ArbitraryDataFileMessage(int id, ArbitraryDataFile arbitraryDataFile) {
|
||||
public ArbitraryDataFileMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) {
|
||||
super(id, MessageType.ARBITRARY_DATA_FILE);
|
||||
|
||||
this.signature = signature;
|
||||
this.arbitraryDataFile = arbitraryDataFile;
|
||||
}
|
||||
|
||||
@ -30,6 +36,9 @@ public class ArbitraryDataFileMessage extends Message {
|
||||
}
|
||||
|
||||
public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
|
||||
byte[] signature = new byte[SIGNATURE_LENGTH];
|
||||
byteBuffer.get(signature);
|
||||
|
||||
int dataLength = byteBuffer.getInt();
|
||||
|
||||
if (byteBuffer.remaining() != dataLength)
|
||||
@ -39,8 +48,8 @@ public class ArbitraryDataFileMessage extends Message {
|
||||
byteBuffer.get(data);
|
||||
|
||||
try {
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data);
|
||||
return new ArbitraryDataFileMessage(id, arbitraryDataFile);
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data, signature);
|
||||
return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile);
|
||||
}
|
||||
catch (DataException e) {
|
||||
return null;
|
||||
@ -61,6 +70,8 @@ public class ArbitraryDataFileMessage extends Message {
|
||||
try {
|
||||
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
|
||||
|
||||
bytes.write(signature);
|
||||
|
||||
bytes.write(Ints.toByteArray(data.length));
|
||||
|
||||
bytes.write(data);
|
||||
@ -72,7 +83,7 @@ public class ArbitraryDataFileMessage extends Message {
|
||||
}
|
||||
|
||||
public ArbitraryDataFileMessage cloneWithNewId(int newId) {
|
||||
ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.arbitraryDataFile);
|
||||
ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.signature, this.arbitraryDataFile);
|
||||
clone.setId(newId);
|
||||
return clone;
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
package org.qortal.network.message;
|
||||
|
||||
import org.qortal.transform.Transformer;
|
||||
import org.qortal.transform.transaction.TransactionTransformer;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
@ -9,33 +10,42 @@ import java.nio.ByteBuffer;
|
||||
|
||||
public class GetArbitraryDataFileMessage extends Message {
|
||||
|
||||
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
|
||||
private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH;
|
||||
|
||||
private final byte[] signature;
|
||||
private final byte[] hash;
|
||||
|
||||
public GetArbitraryDataFileMessage(byte[] hash) {
|
||||
this(-1, hash);
|
||||
public GetArbitraryDataFileMessage(byte[] signature, byte[] hash) {
|
||||
this(-1, signature, hash);
|
||||
}
|
||||
|
||||
private GetArbitraryDataFileMessage(int id, byte[] hash) {
|
||||
private GetArbitraryDataFileMessage(int id, byte[] signature, byte[] hash) {
|
||||
super(id, MessageType.GET_ARBITRARY_DATA_FILE);
|
||||
|
||||
this.signature = signature;
|
||||
this.hash = hash;
|
||||
}
|
||||
|
||||
public byte[] getSignature() {
|
||||
return this.signature;
|
||||
}
|
||||
|
||||
public byte[] getHash() {
|
||||
return this.hash;
|
||||
}
|
||||
|
||||
public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
|
||||
if (bytes.remaining() != HASH_LENGTH)
|
||||
if (bytes.remaining() != HASH_LENGTH + SIGNATURE_LENGTH)
|
||||
return null;
|
||||
|
||||
byte[] hash = new byte[HASH_LENGTH];
|
||||
byte[] signature = new byte[SIGNATURE_LENGTH];
|
||||
bytes.get(signature);
|
||||
|
||||
byte[] hash = new byte[HASH_LENGTH];
|
||||
bytes.get(hash);
|
||||
|
||||
return new GetArbitraryDataFileMessage(id, hash);
|
||||
return new GetArbitraryDataFileMessage(id, signature, hash);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -43,6 +53,8 @@ public class GetArbitraryDataFileMessage extends Message {
|
||||
try {
|
||||
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
|
||||
|
||||
bytes.write(this.signature);
|
||||
|
||||
bytes.write(this.hash);
|
||||
|
||||
return bytes.toByteArray();
|
||||
|
||||
@ -12,6 +12,7 @@ import org.qortal.repository.ArbitraryRepository;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.arbitrary.ArbitraryDataFile;
|
||||
import org.qortal.transaction.Transaction.ApprovalStatus;
|
||||
import org.qortal.utils.ArbitraryTransactionUtils;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
@ -53,24 +54,23 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
|
||||
// Load data file(s)
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
|
||||
// Check if we already have the complete data file
|
||||
if (arbitraryDataFile.exists()) {
|
||||
// Check if we already have the complete data file or all chunks
|
||||
if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(chunkHashes)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// If this transaction doesn't have any chunks, then we require the complete file
|
||||
if (chunkHashes == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Alternatively, if we have all the chunks, then it's safe to assume the data is local
|
||||
if (arbitraryDataFile.allChunksExist(chunkHashes)) {
|
||||
return true;
|
||||
// We may need to relocate files from the "misc_" folder to the signature folder
|
||||
int relocatedCount = ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
|
||||
if (relocatedCount > 0) {
|
||||
// Files were relocated, so check again to see if they exist in the correct place
|
||||
if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(chunkHashes)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
@ -93,7 +93,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
|
||||
// Load data file(s)
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
@ -143,7 +143,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
||||
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
|
||||
|
||||
// Load data file(s)
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
byte[] signature = arbitraryTransactionData.getSignature();
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ package org.qortal.utils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.qortal.arbitrary.ArbitraryDataFile;
|
||||
import org.qortal.arbitrary.ArbitraryDataFileChunk;
|
||||
import org.qortal.arbitrary.misc.Service;
|
||||
import org.qortal.data.transaction.ArbitraryTransactionData;
|
||||
import org.qortal.data.transaction.TransactionData;
|
||||
@ -15,6 +16,9 @@ import java.nio.file.Path;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.Arrays;
|
||||
|
||||
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
|
||||
|
||||
|
||||
public class ArbitraryTransactionUtils {
|
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransactionUtils.class);
|
||||
@ -83,9 +87,10 @@ public class ArbitraryTransactionUtils {
|
||||
}
|
||||
|
||||
byte[] digest = transactionData.getData();
|
||||
byte[] signature = transactionData.getSignature();
|
||||
|
||||
// Load complete file
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
|
||||
return arbitraryDataFile.exists();
|
||||
|
||||
}
|
||||
@ -97,6 +102,7 @@ public class ArbitraryTransactionUtils {
|
||||
|
||||
byte[] digest = transactionData.getData();
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
byte[] signature = transactionData.getSignature();
|
||||
|
||||
if (chunkHashes == null) {
|
||||
// This file doesn't have any chunks, which is the same as us having them all
|
||||
@ -104,7 +110,7 @@ public class ArbitraryTransactionUtils {
|
||||
}
|
||||
|
||||
// Load complete file and chunks
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
@ -118,6 +124,7 @@ public class ArbitraryTransactionUtils {
|
||||
|
||||
byte[] digest = transactionData.getData();
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
byte[] signature = transactionData.getSignature();
|
||||
|
||||
if (chunkHashes == null) {
|
||||
// This file doesn't have any chunks, which means none exist
|
||||
@ -125,7 +132,7 @@ public class ArbitraryTransactionUtils {
|
||||
}
|
||||
|
||||
// Load complete file and chunks
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
@ -139,6 +146,7 @@ public class ArbitraryTransactionUtils {
|
||||
|
||||
byte[] digest = transactionData.getData();
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
byte[] signature = transactionData.getSignature();
|
||||
|
||||
if (chunkHashes == null) {
|
||||
// This file doesn't have any chunks
|
||||
@ -146,7 +154,7 @@ public class ArbitraryTransactionUtils {
|
||||
}
|
||||
|
||||
// Load complete file and chunks
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
@ -174,8 +182,8 @@ public class ArbitraryTransactionUtils {
|
||||
return true;
|
||||
}
|
||||
|
||||
public static boolean isFileHashRecent(byte[] hash, long now, long cleanupAfter) throws DataException {
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash);
|
||||
public static boolean isFileHashRecent(byte[] hash, byte[] signature, long now, long cleanupAfter) throws DataException {
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
|
||||
if (arbitraryDataFile == null || !arbitraryDataFile.exists()) {
|
||||
// No hash, or file doesn't exist, so it's not recent
|
||||
return false;
|
||||
@ -188,11 +196,12 @@ public class ArbitraryTransactionUtils {
|
||||
public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
|
||||
byte[] completeHash = arbitraryTransactionData.getData();
|
||||
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
|
||||
byte[] signature = arbitraryTransactionData.getSignature();
|
||||
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature);
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
|
||||
if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, now, cleanupAfter)) {
|
||||
if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, signature, now, cleanupAfter)) {
|
||||
LOGGER.info("Deleting file {} because it can be rebuilt from chunks " +
|
||||
"if needed", Base58.encode(completeHash));
|
||||
|
||||
@ -203,8 +212,9 @@ public class ArbitraryTransactionUtils {
|
||||
public static void deleteCompleteFileAndChunks(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
|
||||
byte[] completeHash = arbitraryTransactionData.getData();
|
||||
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
|
||||
byte[] signature = arbitraryTransactionData.getSignature();
|
||||
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature);
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
arbitraryDataFile.deleteAll();
|
||||
}
|
||||
@ -212,9 +222,10 @@ public class ArbitraryTransactionUtils {
|
||||
public static void convertFileToChunks(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
|
||||
byte[] completeHash = arbitraryTransactionData.getData();
|
||||
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
|
||||
byte[] signature = arbitraryTransactionData.getSignature();
|
||||
|
||||
// Split the file into chunks
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature);
|
||||
int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE);
|
||||
if (chunkCount > 1) {
|
||||
LOGGER.info(String.format("Successfully split %s into %d chunk%s",
|
||||
@ -226,7 +237,7 @@ public class ArbitraryTransactionUtils {
|
||||
if (arbitraryDataFile.allChunksExist(chunkHashes)) {
|
||||
|
||||
// Now delete the original file if it's not recent
|
||||
if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, now, cleanupAfter)) {
|
||||
if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, signature, now, cleanupAfter)) {
|
||||
LOGGER.info("Deleting file {} because it can now be rebuilt from " +
|
||||
"chunks if needed", Base58.encode(completeHash));
|
||||
|
||||
@ -240,4 +251,73 @@ public class ArbitraryTransactionUtils {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When first uploaded, files go into a _misc folder as they are not yet associated with a
|
||||
* transaction signature. Once the transaction is broadcast, they need to be moved to the
|
||||
* correct location, keyed by the transaction signature.
|
||||
* @param arbitraryTransactionData
|
||||
* @return
|
||||
* @throws DataException
|
||||
*/
|
||||
public static int checkAndRelocateMiscFiles(ArbitraryTransactionData arbitraryTransactionData) {
|
||||
int filesRelocatedCount = 0;
|
||||
|
||||
try {
|
||||
// Load hashes
|
||||
byte[] digest = arbitraryTransactionData.getData();
|
||||
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
|
||||
|
||||
// Load signature
|
||||
byte[] signature = arbitraryTransactionData.getSignature();
|
||||
|
||||
// Check if any files for this transaction exist in the misc folder
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, null);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
if (arbitraryDataFile.anyChunksExist(chunkHashes)) {
|
||||
// At least one chunk exists in the misc folder - move them
|
||||
for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) {
|
||||
if (chunk.exists()) {
|
||||
// Determine the correct path by initializing a new ArbitraryDataFile instance with the signature
|
||||
ArbitraryDataFile newChunk = ArbitraryDataFile.fromHash(chunk.getHash(), signature);
|
||||
Path oldPath = chunk.getFilePath();
|
||||
Path newPath = newChunk.getFilePath();
|
||||
|
||||
// Ensure parent directories exist, then copy the file
|
||||
LOGGER.info("Relocating chunk from {} to {}...", oldPath, newPath);
|
||||
Files.createDirectories(newPath.getParent());
|
||||
Files.move(oldPath, newPath, REPLACE_EXISTING);
|
||||
filesRelocatedCount++;
|
||||
|
||||
// Delete empty parent directories
|
||||
FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Also move the complete file if it exists
|
||||
if (arbitraryDataFile.exists()) {
|
||||
// Determine the correct path by initializing a new ArbitraryDataFile instance with the signature
|
||||
ArbitraryDataFile newCompleteFile = ArbitraryDataFile.fromHash(arbitraryDataFile.getHash(), signature);
|
||||
Path oldPath = arbitraryDataFile.getFilePath();
|
||||
Path newPath = newCompleteFile.getFilePath();
|
||||
|
||||
// Ensure parent directories exist, then copy the file
|
||||
LOGGER.info("Relocating complete file from {} to {}...", oldPath, newPath);
|
||||
Files.createDirectories(newPath.getParent());
|
||||
Files.move(oldPath, newPath, REPLACE_EXISTING);
|
||||
filesRelocatedCount++;
|
||||
|
||||
// Delete empty parent directories
|
||||
FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath);
|
||||
}
|
||||
}
|
||||
catch (DataException | IOException e) {
|
||||
LOGGER.info("Unable to check and relocate all files for signature {}: {}",
|
||||
Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage());
|
||||
}
|
||||
|
||||
return filesRelocatedCount;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -20,7 +20,7 @@ public class ArbitraryDataFileTests extends Common {
|
||||
@Test
|
||||
public void testSplitAndJoin() throws DataException {
|
||||
String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes());
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes(), null);
|
||||
assertTrue(arbitraryDataFile.exists());
|
||||
assertEquals(62, arbitraryDataFile.size());
|
||||
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58());
|
||||
@ -50,7 +50,7 @@ public class ArbitraryDataFileTests extends Common {
|
||||
byte[] randomData = new byte[fileSize];
|
||||
new Random().nextBytes(randomData); // No need for SecureRandom here
|
||||
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData);
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData, null);
|
||||
assertTrue(arbitraryDataFile.exists());
|
||||
assertEquals(fileSize, arbitraryDataFile.size());
|
||||
String originalFileDigest = arbitraryDataFile.digest58();
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user