Qortal/qortal
Qortal/qortal
3
0
Fork 3
mirror of https://github.com/Qortal/qortal.git synced 2025-02-12 10:15:49 +00:00
Code Issues Projects Releases Wiki Activity

Major rework of local data directory structure

Browse Source 
Files are now keyed by signature, in the format:
data/si/gn/signature/hash

For times when there is no signature available (i.e. at the time of initial upload), files are keyed by hash, in the format:
data/_misc/ha/sh/hash

Files in the _misc folder are subsequently relocated to a path that is keyed by the resulting signature.

The end result is that chunks are now grouped on the filesystem by signature. This allows more transparency as to what is being hosted, and will also help simplify the reporting and management of local files.
This commit is contained in:
CalDescent 2021-11-27 13:00:32 +00:00
parent 0dd43d5c9a
commit bc38184ebf
10 changed files with 210 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
View File

@ -59,29 +59,32 @@ public class ArbitraryDataFile {
protected Path filePath; protected Path filePath;
protected String hash58; protected String hash58;
protected byte[] signature;
private ArrayList<ArbitraryDataFileChunk> chunks; private ArrayList<ArbitraryDataFileChunk> chunks;
private byte[] secret; private byte[] secret;
public ArbitraryDataFile() { public ArbitraryDataFile() {
} }
public ArbitraryDataFile(String hash58) throws DataException { public ArbitraryDataFile(String hash58, byte[] signature) throws DataException {
this.createDataDirectory(); this.createDataDirectory();
this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, false); this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, signature, false);
this.chunks = new ArrayList<>(); this.chunks = new ArrayList<>();
this.hash58 = hash58; this.hash58 = hash58;
this.signature = signature;
} }
public ArbitraryDataFile(byte[] fileContent) throws DataException { public ArbitraryDataFile(byte[] fileContent, byte[] signature) throws DataException {
if (fileContent == null) { if (fileContent == null) {
LOGGER.error("fileContent is null"); LOGGER.error("fileContent is null");
return; return;
} }
this.hash58 = Base58.encode(Crypto.digest(fileContent)); this.hash58 = Base58.encode(Crypto.digest(fileContent));
this.signature = signature;
LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length)); LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length));
Path outputFilePath = getOutputFilePath(this.hash58, true); Path outputFilePath = getOutputFilePath(this.hash58, signature, true);
File outputFile = outputFilePath.toFile(); File outputFile = outputFilePath.toFile();
try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { try (FileOutputStream outputStream = new FileOutputStream(outputFile)) {
outputStream.write(fileContent); outputStream.write(fileContent);
@ -97,15 +100,15 @@ public class ArbitraryDataFile {
} }
} }
public static ArbitraryDataFile fromHash58(String hash58) throws DataException { public static ArbitraryDataFile fromHash58(String hash58, byte[] signature) throws DataException {
return new ArbitraryDataFile(hash58); return new ArbitraryDataFile(hash58, signature);
} }
public static ArbitraryDataFile fromHash(byte[] hash) throws DataException { public static ArbitraryDataFile fromHash(byte[] hash, byte[] signature) throws DataException {
return ArbitraryDataFile.fromHash58(Base58.encode(hash)); return ArbitraryDataFile.fromHash58(Base58.encode(hash), signature);
} }
public static ArbitraryDataFile fromPath(Path path) { public static ArbitraryDataFile fromPath(Path path, byte[] signature) {
if (path == null) { if (path == null) {
return null; return null;
} }
@ -113,11 +116,11 @@ public class ArbitraryDataFile {
if (file.exists()) { if (file.exists()) {
try { try {
byte[] digest = Crypto.digest(file); byte[] digest = Crypto.digest(file);
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
// Copy file to data directory if needed // Copy file to data directory if needed
if (Files.exists(path) && !arbitraryDataFile.isInBaseDirectory(path)) { if (Files.exists(path) && !arbitraryDataFile.isInBaseDirectory(path)) {
arbitraryDataFile.copyToDataDirectory(path); arbitraryDataFile.copyToDataDirectory(path, signature);
} }
// Or, if it's already in the data directory, we may need to move it // Or, if it's already in the data directory, we may need to move it
else if (!path.equals(arbitraryDataFile.getFilePath())) { else if (!path.equals(arbitraryDataFile.getFilePath())) {
@ -134,8 +137,8 @@ public class ArbitraryDataFile {
return null; return null;
} }
public static ArbitraryDataFile fromFile(File file) { public static ArbitraryDataFile fromFile(File file, byte[] signature) {
return ArbitraryDataFile.fromPath(Paths.get(file.getPath())); return ArbitraryDataFile.fromPath(Paths.get(file.getPath()), signature);
} }
private boolean createDataDirectory() { private boolean createDataDirectory() {
@ -151,11 +154,11 @@ public class ArbitraryDataFile {
return true; return true;
} }
private Path copyToDataDirectory(Path sourcePath) throws DataException { private Path copyToDataDirectory(Path sourcePath, byte[] signature) throws DataException {
if (this.hash58 == null || this.filePath == null) { if (this.hash58 == null || this.filePath == null) {
return null; return null;
} }
Path outputFilePath = getOutputFilePath(this.hash58, true); Path outputFilePath = getOutputFilePath(this.hash58, signature, true);
sourcePath = sourcePath.toAbsolutePath(); sourcePath = sourcePath.toAbsolutePath();
Path destPath = outputFilePath.toAbsolutePath(); Path destPath = outputFilePath.toAbsolutePath();
try { try {
@ -165,13 +168,25 @@ public class ArbitraryDataFile {
} }
} }
public static Path getOutputFilePath(String hash58, boolean createDirectories) throws DataException { public static Path getOutputFilePath(String hash58, byte[] signature, boolean createDirectories) throws DataException {
Path directory;
if (hash58 == null) { if (hash58 == null) {
return null; return null;
} }
if (signature != null) {
// Key by signature
String signature58 = Base58.encode(signature);
String sig58First2Chars = signature58.substring(0, 2).toLowerCase();
String sig58Next2Chars = signature58.substring(2, 4).toLowerCase();
directory = Paths.get(Settings.getInstance().getDataPath(), sig58First2Chars, sig58Next2Chars, signature58);
}
else {
// Put files without signatures in a "_misc" directory, and the files will be relocated later
String hash58First2Chars = hash58.substring(0, 2).toLowerCase(); String hash58First2Chars = hash58.substring(0, 2).toLowerCase();
String hash58Next2Chars = hash58.substring(2, 4).toLowerCase(); String hash58Next2Chars = hash58.substring(2, 4).toLowerCase();
Path directory = Paths.get(Settings.getInstance().getDataPath(), hash58First2Chars, hash58Next2Chars); directory = Paths.get(Settings.getInstance().getDataPath(), "_misc", hash58First2Chars, hash58Next2Chars);
}
if (createDirectories) { if (createDirectories) {
try { try {
@ -217,7 +232,7 @@ public class ArbitraryDataFile {
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH]; byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH];
byteBuffer.get(chunkDigest); byteBuffer.get(chunkDigest);
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkDigest); ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkDigest, this.signature);
this.addChunk(chunk); this.addChunk(chunk);
} }
} }
@ -252,7 +267,7 @@ public class ArbitraryDataFile {
out.write(buffer, 0, numberOfBytes); out.write(buffer, 0, numberOfBytes);
out.flush(); out.flush();
ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray()); ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray(), this.signature);
ValidationResult validationResult = chunk.isValid(); ValidationResult validationResult = chunk.isValid();
if (validationResult == ValidationResult.OK) { if (validationResult == ValidationResult.OK) {
this.chunks.add(chunk); this.chunks.add(chunk);
@ -301,7 +316,7 @@ public class ArbitraryDataFile {
out.close(); out.close();
// Copy temporary file to data directory // Copy temporary file to data directory
this.filePath = this.copyToDataDirectory(outputPath); this.filePath = this.copyToDataDirectory(outputPath, this.signature);
if (FilesystemUtils.pathInsideDataOrTempPath(outputPath)) { if (FilesystemUtils.pathInsideDataOrTempPath(outputPath)) {
Files.delete(outputPath); Files.delete(outputPath);
} }
@ -425,7 +440,7 @@ public class ArbitraryDataFile {
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH]; byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH];
byteBuffer.get(chunkHash); byteBuffer.get(chunkHash);
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash); ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
if (!chunk.exists()) { if (!chunk.exists()) {
return false; return false;
} }
@ -441,7 +456,7 @@ public class ArbitraryDataFile {
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH]; byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH];
byteBuffer.get(chunkHash); byteBuffer.get(chunkHash);
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash); ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
if (chunk.exists()) { if (chunk.exists()) {
return true; return true;
} }

16
src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java
View File

@ -13,20 +13,20 @@ public class ArbitraryDataFileChunk extends ArbitraryDataFile {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class); private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class);
public ArbitraryDataFileChunk(String hash58) throws DataException { public ArbitraryDataFileChunk(String hash58, byte[] signature) throws DataException {
super(hash58); super(hash58, signature);
} }
public ArbitraryDataFileChunk(byte[] fileContent) throws DataException { public ArbitraryDataFileChunk(byte[] fileContent, byte[] signature) throws DataException {
super(fileContent); super(fileContent, signature);
} }
public static ArbitraryDataFileChunk fromHash58(String hash58) throws DataException { public static ArbitraryDataFileChunk fromHash58(String hash58, byte[] signature) throws DataException {
return new ArbitraryDataFileChunk(hash58); return new ArbitraryDataFileChunk(hash58, signature);
} }
public static ArbitraryDataFileChunk fromHash(byte[] hash) throws DataException { public static ArbitraryDataFileChunk fromHash(byte[] hash, byte[] signature) throws DataException {
return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash)); return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash), signature);
} }
@Override @Override

11
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -18,6 +18,7 @@ import org.qortal.repository.RepositoryManager;
import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.settings.Settings; import org.qortal.settings.Settings;
import org.qortal.transform.Transformer; import org.qortal.transform.Transformer;
import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
import org.qortal.utils.FilesystemUtils; import org.qortal.utils.FilesystemUtils;
import org.qortal.utils.ZipUtils; import org.qortal.utils.ZipUtils;
@ -75,7 +76,7 @@ public class ArbitraryDataReader {
this.identifier = identifier; this.identifier = identifier;
this.workingPath = this.buildWorkingPath(); this.workingPath = this.buildWorkingPath();
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); this.uncompressedPath = Paths.get(this.workingPath.toString(), "data");
// By default we can request missing files // By default we can request missing files
// Callers can use setCanRequestMissingFiles(false) to prevent it // Callers can use setCanRequestMissingFiles(false) to prevent it
@ -249,8 +250,8 @@ public class ArbitraryDataReader {
} }
private void fetchFromFileHash() throws DataException { private void fetchFromFileHash() throws DataException {
// Load data file directly from the hash // Load data file directly from the hash (without a signature)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId, null);
// Set filePath to the location of the ArbitraryDataFile // Set filePath to the location of the ArbitraryDataFile
this.filePath = arbitraryDataFile.getFilePath(); this.filePath = arbitraryDataFile.getFilePath();
} }
@ -303,6 +304,7 @@ public class ArbitraryDataReader {
// Load hashes // Load hashes
byte[] digest = transactionData.getData(); byte[] digest = transactionData.getData();
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
byte[] signature = transactionData.getSignature();
// Load secret // Load secret
byte[] secret = transactionData.getSecret(); byte[] secret = transactionData.getSecret();
@ -311,7 +313,8 @@ public class ArbitraryDataReader {
} }
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
if (!arbitraryDataFile.exists()) { if (!arbitraryDataFile.exists()) {
if (!arbitraryDataFile.allChunksExist(chunkHashes) || chunkHashes == null) { if (!arbitraryDataFile.allChunksExist(chunkHashes) || chunkHashes == null) {
if (ArbitraryDataStorageManager.getInstance().isNameInBlacklist(transactionData.getName())) { if (ArbitraryDataStorageManager.getInstance().isNameInBlacklist(transactionData.getName())) {

5
src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java
View File

@ -56,7 +56,7 @@ public class ArbitraryDataWriter {
this.compression = compression; this.compression = compression;
} }
public void save() throws DataException, IOException, DataException, InterruptedException, MissingDataException { public void save() throws IOException, DataException, InterruptedException, MissingDataException {
try { try {
this.preExecute(); this.preExecute();
this.validateService(); this.validateService();
@ -251,7 +251,8 @@ public class ArbitraryDataWriter {
} }
private void split() throws IOException, DataException { private void split() throws IOException, DataException {
this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath); // We don't have a signature yet, so use null to put the file in a generic folder
this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath, null);
if (this.arbitraryDataFile == null) { if (this.arbitraryDataFile == null) {
throw new IOException("No file available when trying to split"); throw new IOException("No file available when trying to split");
} }

21
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -493,11 +493,11 @@ public class ArbitraryDataManager extends Thread {
// Fetch data files by hash // Fetch data files by hash
private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, byte[] hash) { private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, byte[] signature, byte[] hash) {
String hash58 = Base58.encode(hash); String hash58 = Base58.encode(hash);
LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer));
arbitraryDataFileRequests.put(hash58, NTP.getTime()); arbitraryDataFileRequests.put(hash58, NTP.getTime());
Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(hash); Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash);
Message message = null; Message message = null;
try { try {
@ -624,7 +624,7 @@ public class ArbitraryDataManager extends Thread {
List<byte[]> hashes) throws DataException { List<byte[]> hashes) throws DataException {
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData()); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature);
arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes());
// If hashes are null, we will treat this to mean all data hashes associated with this file // If hashes are null, we will treat this to mean all data hashes associated with this file
@ -646,13 +646,13 @@ public class ArbitraryDataManager extends Thread {
if (!arbitraryDataFile.chunkExists(hash)) { if (!arbitraryDataFile.chunkExists(hash)) {
// Only request the file if we aren't already requesting it from someone else // Only request the file if we aren't already requesting it from someone else
if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) { if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, hash); ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, signature, hash);
if (receivedArbitraryDataFile != null) { if (receivedArbitraryDataFile != null) {
LOGGER.info("Received data file {} from peer {}", receivedArbitraryDataFile, peer); LOGGER.info("Received data file {} from peer {}", receivedArbitraryDataFile, peer);
receivedAtLeastOneFile = true; receivedAtLeastOneFile = true;
} }
else { else {
LOGGER.info("Peer {} didn't respond with data file {}", peer, hash); LOGGER.info("Peer {} didn't respond with data file {}", peer, Base58.encode(hash));
} }
} }
else { else {
@ -769,7 +769,7 @@ public class ArbitraryDataManager extends Thread {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData()); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature);
arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes());
// Check all hashes exist // Check all hashes exist
@ -809,13 +809,14 @@ public class ArbitraryDataManager extends Thread {
public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) { public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) {
GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message; GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message;
byte[] hash = getArbitraryDataFileMessage.getHash(); byte[] hash = getArbitraryDataFileMessage.getHash();
byte[] signature = getArbitraryDataFileMessage.getSignature();
Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet(); Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet();
try { try {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
if (arbitraryDataFile.exists()) { if (arbitraryDataFile.exists()) {
ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(arbitraryDataFile); ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile);
arbitraryDataFileMessage.setId(message.getId()); arbitraryDataFileMessage.setId(message.getId());
if (!peer.sendMessage(arbitraryDataFileMessage)) { if (!peer.sendMessage(arbitraryDataFileMessage)) {
LOGGER.info("Couldn't sent file"); LOGGER.info("Couldn't sent file");
@ -829,7 +830,7 @@ public class ArbitraryDataManager extends Thread {
Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement(); Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement();
// Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout
LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile)); LOGGER.debug(String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile));
// We'll send empty block summaries message as it's very short // We'll send empty block summaries message as it's very short
// TODO: use a different message type here // TODO: use a different message type here
@ -869,7 +870,7 @@ public class ArbitraryDataManager extends Thread {
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
// Load file(s) and add any that exist to the list of hashes // Load file(s) and add any that exist to the list of hashes
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) { for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) {

21
src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java
View File

@ -3,6 +3,7 @@ package org.qortal.network.message;
import com.google.common.primitives.Ints; import com.google.common.primitives.Ints;
import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.transform.Transformer;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
@ -11,17 +12,22 @@ import java.nio.ByteBuffer;
public class ArbitraryDataFileMessage extends Message { public class ArbitraryDataFileMessage extends Message {
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
private final byte[] signature;
private final ArbitraryDataFile arbitraryDataFile; private final ArbitraryDataFile arbitraryDataFile;
public ArbitraryDataFileMessage(ArbitraryDataFile arbitraryDataFile) { public ArbitraryDataFileMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) {
super(MessageType.ARBITRARY_DATA_FILE); super(MessageType.ARBITRARY_DATA_FILE);
this.signature = signature;
this.arbitraryDataFile = arbitraryDataFile; this.arbitraryDataFile = arbitraryDataFile;
} }
public ArbitraryDataFileMessage(int id, ArbitraryDataFile arbitraryDataFile) { public ArbitraryDataFileMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) {
super(id, MessageType.ARBITRARY_DATA_FILE); super(id, MessageType.ARBITRARY_DATA_FILE);
this.signature = signature;
this.arbitraryDataFile = arbitraryDataFile; this.arbitraryDataFile = arbitraryDataFile;
} }
@ -30,6 +36,9 @@ public class ArbitraryDataFileMessage extends Message {
} }
public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
byte[] signature = new byte[SIGNATURE_LENGTH];
byteBuffer.get(signature);
int dataLength = byteBuffer.getInt(); int dataLength = byteBuffer.getInt();
if (byteBuffer.remaining() != dataLength) if (byteBuffer.remaining() != dataLength)
@ -39,8 +48,8 @@ public class ArbitraryDataFileMessage extends Message {
byteBuffer.get(data); byteBuffer.get(data);
try { try {
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data); ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data, signature);
return new ArbitraryDataFileMessage(id, arbitraryDataFile); return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile);
} }
catch (DataException e) { catch (DataException e) {
return null; return null;
@ -61,6 +70,8 @@ public class ArbitraryDataFileMessage extends Message {
try { try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream(); ByteArrayOutputStream bytes = new ByteArrayOutputStream();
bytes.write(signature);
bytes.write(Ints.toByteArray(data.length)); bytes.write(Ints.toByteArray(data.length));
bytes.write(data); bytes.write(data);
@ -72,7 +83,7 @@ public class ArbitraryDataFileMessage extends Message {
} }
public ArbitraryDataFileMessage cloneWithNewId(int newId) { public ArbitraryDataFileMessage cloneWithNewId(int newId) {
ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.arbitraryDataFile); ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.signature, this.arbitraryDataFile);
clone.setId(newId); clone.setId(newId);
return clone; return clone;
} }

24
src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java
View File

@ -1,5 +1,6 @@
package org.qortal.network.message; package org.qortal.network.message;
import org.qortal.transform.Transformer;
import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.transform.transaction.TransactionTransformer;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
@ -9,33 +10,42 @@ import java.nio.ByteBuffer;
public class GetArbitraryDataFileMessage extends Message { public class GetArbitraryDataFileMessage extends Message {
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH;
private final byte[] signature;
private final byte[] hash; private final byte[] hash;
public GetArbitraryDataFileMessage(byte[] hash) { public GetArbitraryDataFileMessage(byte[] signature, byte[] hash) {
this(-1, hash); this(-1, signature, hash);
} }
private GetArbitraryDataFileMessage(int id, byte[] hash) { private GetArbitraryDataFileMessage(int id, byte[] signature, byte[] hash) {
super(id, MessageType.GET_ARBITRARY_DATA_FILE); super(id, MessageType.GET_ARBITRARY_DATA_FILE);
this.signature = signature;
this.hash = hash; this.hash = hash;
} }
public byte[] getSignature() {
return this.signature;
}
public byte[] getHash() { public byte[] getHash() {
return this.hash; return this.hash;
} }
public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
if (bytes.remaining() != HASH_LENGTH) if (bytes.remaining() != HASH_LENGTH + SIGNATURE_LENGTH)
return null; return null;
byte[] hash = new byte[HASH_LENGTH]; byte[] signature = new byte[SIGNATURE_LENGTH];
bytes.get(signature);
byte[] hash = new byte[HASH_LENGTH];
bytes.get(hash); bytes.get(hash);
return new GetArbitraryDataFileMessage(id, hash); return new GetArbitraryDataFileMessage(id, signature, hash);
} }
@Override @Override
@ -43,6 +53,8 @@ public class GetArbitraryDataFileMessage extends Message {
try { try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream(); ByteArrayOutputStream bytes = new ByteArrayOutputStream();
bytes.write(this.signature);
bytes.write(this.hash); bytes.write(this.hash);
return bytes.toByteArray(); return bytes.toByteArray();

25
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@ -12,6 +12,7 @@ import org.qortal.repository.ArbitraryRepository;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.transaction.Transaction.ApprovalStatus; import org.qortal.transaction.Transaction.ApprovalStatus;
import org.qortal.utils.ArbitraryTransactionUtils;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
@ -53,25 +54,24 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
} }
// Check if we already have the complete data file // Check if we already have the complete data file or all chunks
if (arbitraryDataFile.exists()) { if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(chunkHashes)) {
return true; return true;
} }
// If this transaction doesn't have any chunks, then we require the complete file // We may need to relocate files from the "misc_" folder to the signature folder
if (chunkHashes == null) { int relocatedCount = ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
return false; if (relocatedCount > 0) {
} // Files were relocated, so check again to see if they exist in the correct place
if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(chunkHashes)) {
// Alternatively, if we have all the chunks, then it's safe to assume the data is local
if (arbitraryDataFile.allChunksExist(chunkHashes)) {
return true; return true;
} }
}
return false; return false;
} }
@ -93,7 +93,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
} }
@ -143,7 +143,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); byte[] signature = arbitraryTransactionData.getSignature();
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
} }

102
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@ -3,6 +3,7 @@ package org.qortal.utils;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataFileChunk;
import org.qortal.arbitrary.misc.Service; import org.qortal.arbitrary.misc.Service;
import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.TransactionData;
@ -15,6 +16,9 @@ import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays; import java.util.Arrays;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
public class ArbitraryTransactionUtils { public class ArbitraryTransactionUtils {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransactionUtils.class); private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransactionUtils.class);
@ -83,9 +87,10 @@ public class ArbitraryTransactionUtils {
} }
byte[] digest = transactionData.getData(); byte[] digest = transactionData.getData();
byte[] signature = transactionData.getSignature();
// Load complete file // Load complete file
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
return arbitraryDataFile.exists(); return arbitraryDataFile.exists();
} }
@ -97,6 +102,7 @@ public class ArbitraryTransactionUtils {
byte[] digest = transactionData.getData(); byte[] digest = transactionData.getData();
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
byte[] signature = transactionData.getSignature();
if (chunkHashes == null) { if (chunkHashes == null) {
// This file doesn't have any chunks, which is the same as us having them all // This file doesn't have any chunks, which is the same as us having them all
@ -104,7 +110,7 @@ public class ArbitraryTransactionUtils {
} }
// Load complete file and chunks // Load complete file and chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
} }
@ -118,6 +124,7 @@ public class ArbitraryTransactionUtils {
byte[] digest = transactionData.getData(); byte[] digest = transactionData.getData();
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
byte[] signature = transactionData.getSignature();
if (chunkHashes == null) { if (chunkHashes == null) {
// This file doesn't have any chunks, which means none exist // This file doesn't have any chunks, which means none exist
@ -125,7 +132,7 @@ public class ArbitraryTransactionUtils {
} }
// Load complete file and chunks // Load complete file and chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
} }
@ -139,6 +146,7 @@ public class ArbitraryTransactionUtils {
byte[] digest = transactionData.getData(); byte[] digest = transactionData.getData();
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
byte[] signature = transactionData.getSignature();
if (chunkHashes == null) { if (chunkHashes == null) {
// This file doesn't have any chunks // This file doesn't have any chunks
@ -146,7 +154,7 @@ public class ArbitraryTransactionUtils {
} }
// Load complete file and chunks // Load complete file and chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
} }
@ -174,8 +182,8 @@ public class ArbitraryTransactionUtils {
return true; return true;
} }
public static boolean isFileHashRecent(byte[] hash, long now, long cleanupAfter) throws DataException { public static boolean isFileHashRecent(byte[] hash, byte[] signature, long now, long cleanupAfter) throws DataException {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
if (arbitraryDataFile == null || !arbitraryDataFile.exists()) { if (arbitraryDataFile == null || !arbitraryDataFile.exists()) {
// No hash, or file doesn't exist, so it's not recent // No hash, or file doesn't exist, so it's not recent
return false; return false;
@ -188,11 +196,12 @@ public class ArbitraryTransactionUtils {
public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
byte[] completeHash = arbitraryTransactionData.getData(); byte[] completeHash = arbitraryTransactionData.getData();
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
byte[] signature = arbitraryTransactionData.getSignature();
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature);
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, now, cleanupAfter)) { if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, signature, now, cleanupAfter)) {
LOGGER.info("Deleting file {} because it can be rebuilt from chunks " + LOGGER.info("Deleting file {} because it can be rebuilt from chunks " +
"if needed", Base58.encode(completeHash)); "if needed", Base58.encode(completeHash));
@ -203,8 +212,9 @@ public class ArbitraryTransactionUtils {
public static void deleteCompleteFileAndChunks(ArbitraryTransactionData arbitraryTransactionData) throws DataException { public static void deleteCompleteFileAndChunks(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
byte[] completeHash = arbitraryTransactionData.getData(); byte[] completeHash = arbitraryTransactionData.getData();
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
byte[] signature = arbitraryTransactionData.getSignature();
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature);
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
arbitraryDataFile.deleteAll(); arbitraryDataFile.deleteAll();
} }
@ -212,9 +222,10 @@ public class ArbitraryTransactionUtils {
public static void convertFileToChunks(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { public static void convertFileToChunks(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
byte[] completeHash = arbitraryTransactionData.getData(); byte[] completeHash = arbitraryTransactionData.getData();
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
byte[] signature = arbitraryTransactionData.getSignature();
// Split the file into chunks // Split the file into chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature);
int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE);
if (chunkCount > 1) { if (chunkCount > 1) {
LOGGER.info(String.format("Successfully split %s into %d chunk%s", LOGGER.info(String.format("Successfully split %s into %d chunk%s",
@ -226,7 +237,7 @@ public class ArbitraryTransactionUtils {
if (arbitraryDataFile.allChunksExist(chunkHashes)) { if (arbitraryDataFile.allChunksExist(chunkHashes)) {
// Now delete the original file if it's not recent // Now delete the original file if it's not recent
if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, now, cleanupAfter)) { if (!ArbitraryTransactionUtils.isFileHashRecent(completeHash, signature, now, cleanupAfter)) {
LOGGER.info("Deleting file {} because it can now be rebuilt from " + LOGGER.info("Deleting file {} because it can now be rebuilt from " +
"chunks if needed", Base58.encode(completeHash)); "chunks if needed", Base58.encode(completeHash));
@ -240,4 +251,73 @@ public class ArbitraryTransactionUtils {
} }
} }
/**
* When first uploaded, files go into a _misc folder as they are not yet associated with a
* transaction signature. Once the transaction is broadcast, they need to be moved to the
* correct location, keyed by the transaction signature.
* @param arbitraryTransactionData
* @return
* @throws DataException
*/
public static int checkAndRelocateMiscFiles(ArbitraryTransactionData arbitraryTransactionData) {
int filesRelocatedCount = 0;
try {
// Load hashes
byte[] digest = arbitraryTransactionData.getData();
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
// Load signature
byte[] signature = arbitraryTransactionData.getSignature();
// Check if any files for this transaction exist in the misc folder
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, null);
if (chunkHashes != null && chunkHashes.length > 0) {
arbitraryDataFile.addChunkHashes(chunkHashes);
}
if (arbitraryDataFile.anyChunksExist(chunkHashes)) {
// At least one chunk exists in the misc folder - move them
for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) {
if (chunk.exists()) {
// Determine the correct path by initializing a new ArbitraryDataFile instance with the signature
ArbitraryDataFile newChunk = ArbitraryDataFile.fromHash(chunk.getHash(), signature);
Path oldPath = chunk.getFilePath();
Path newPath = newChunk.getFilePath();
// Ensure parent directories exist, then copy the file
LOGGER.info("Relocating chunk from {} to {}...", oldPath, newPath);
Files.createDirectories(newPath.getParent());
Files.move(oldPath, newPath, REPLACE_EXISTING);
filesRelocatedCount++;
// Delete empty parent directories
FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath);
}
}
}
// Also move the complete file if it exists
if (arbitraryDataFile.exists()) {
// Determine the correct path by initializing a new ArbitraryDataFile instance with the signature
ArbitraryDataFile newCompleteFile = ArbitraryDataFile.fromHash(arbitraryDataFile.getHash(), signature);
Path oldPath = arbitraryDataFile.getFilePath();
Path newPath = newCompleteFile.getFilePath();
// Ensure parent directories exist, then copy the file
LOGGER.info("Relocating complete file from {} to {}...", oldPath, newPath);
Files.createDirectories(newPath.getParent());
Files.move(oldPath, newPath, REPLACE_EXISTING);
filesRelocatedCount++;
// Delete empty parent directories
FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath);
}
}
catch (DataException | IOException e) {
LOGGER.info("Unable to check and relocate all files for signature {}: {}",
Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage());
}
return filesRelocatedCount;
}
} }

4
src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java
View File

@ -20,7 +20,7 @@ public class ArbitraryDataFileTests extends Common {
@Test @Test
public void testSplitAndJoin() throws DataException { public void testSplitAndJoin() throws DataException {
String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes()); ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes(), null);
assertTrue(arbitraryDataFile.exists()); assertTrue(arbitraryDataFile.exists());
assertEquals(62, arbitraryDataFile.size()); assertEquals(62, arbitraryDataFile.size());
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58()); assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58());
@ -50,7 +50,7 @@ public class ArbitraryDataFileTests extends Common {
byte[] randomData = new byte[fileSize]; byte[] randomData = new byte[fileSize];
new Random().nextBytes(randomData); // No need for SecureRandom here new Random().nextBytes(randomData); // No need for SecureRandom here
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData); ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData, null);
assertTrue(arbitraryDataFile.exists()); assertTrue(arbitraryDataFile.exists());
assertEquals(fileSize, arbitraryDataFile.size()); assertEquals(fileSize, arbitraryDataFile.size());
String originalFileDigest = arbitraryDataFile.digest58(); String originalFileDigest = arbitraryDataFile.digest58();