Major upgrade of arbitrary data functionality, to support on-chain data for small payloads.

Max size for on-chain data is 239 bytes, due to 16-byte IV. Must be a single file resource, without .qortal folder.
This commit is contained in:
CalDescent 2023-03-26 16:54:07 +01:00
parent 7deb9328fa
commit a83e332c11
17 changed files with 495 additions and 146 deletions

View File

@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.crypto.Crypto; import org.qortal.crypto.Crypto;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.settings.Settings; import org.qortal.settings.Settings;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
@ -15,7 +16,6 @@ import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.util.*; import java.util.*;
import java.util.stream.Stream;
import static java.util.Arrays.stream; import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toMap;
@ -85,6 +85,7 @@ public class ArbitraryDataFile {
return; return;
} }
this.chunks = new ArrayList<>();
this.hash58 = Base58.encode(Crypto.digest(fileContent)); this.hash58 = Base58.encode(Crypto.digest(fileContent));
this.signature = signature; this.signature = signature;
LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length)); LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length));
@ -111,6 +112,41 @@ public class ArbitraryDataFile {
return ArbitraryDataFile.fromHash58(Base58.encode(hash), signature); return ArbitraryDataFile.fromHash58(Base58.encode(hash), signature);
} }
public static ArbitraryDataFile fromRawData(byte[] data, byte[] signature) throws DataException {
if (data == null) {
return null;
}
return new ArbitraryDataFile(data, signature);
}
public static ArbitraryDataFile fromTransactionData(ArbitraryTransactionData transactionData) throws DataException {
ArbitraryDataFile arbitraryDataFile = null;
byte[] signature = transactionData.getSignature();
byte[] data = transactionData.getData();
if (data == null) {
return null;
}
// Create data file
switch (transactionData.getDataType()) {
case DATA_HASH:
arbitraryDataFile = ArbitraryDataFile.fromHash(data, signature);
break;
case RAW_DATA:
arbitraryDataFile = ArbitraryDataFile.fromRawData(data, signature);
break;
}
// Set metadata hash
if (arbitraryDataFile != null) {
arbitraryDataFile.setMetadataHash(transactionData.getMetadataHash());
}
return arbitraryDataFile;
}
public static ArbitraryDataFile fromPath(Path path, byte[] signature) { public static ArbitraryDataFile fromPath(Path path, byte[] signature) {
if (path == null) { if (path == null) {
return null; return null;
@ -260,6 +296,11 @@ public class ArbitraryDataFile {
this.chunks = new ArrayList<>(); this.chunks = new ArrayList<>();
if (file != null) { if (file != null) {
if (file.exists() && file.length() <= chunkSize) {
// No need to split into chunks if we're already below the chunk size
return 0;
}
try (FileInputStream fileInputStream = new FileInputStream(file); try (FileInputStream fileInputStream = new FileInputStream(file);
BufferedInputStream bis = new BufferedInputStream(fileInputStream)) { BufferedInputStream bis = new BufferedInputStream(fileInputStream)) {

View File

@ -362,11 +362,6 @@ public class ArbitraryDataReader {
throw new DataException(String.format("Transaction data not found for signature %s", this.resourceId)); throw new DataException(String.format("Transaction data not found for signature %s", this.resourceId));
} }
// Load hashes
byte[] digest = transactionData.getData();
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
// Load secret // Load secret
byte[] secret = transactionData.getSecret(); byte[] secret = transactionData.getSecret();
if (secret != null) { if (secret != null) {
@ -374,16 +369,14 @@ public class ArbitraryDataReader {
} }
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData); ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
arbitraryDataFile.setMetadataHash(metadataHash);
if (!arbitraryDataFile.allFilesExist()) { if (!arbitraryDataFile.allFilesExist()) {
if (ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName())) { if (ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName())) {
throw new DataException( throw new DataException(
String.format("Unable to request missing data for file %s because the name is blocked", arbitraryDataFile)); String.format("Unable to request missing data for file %s because the name is blocked", arbitraryDataFile));
} } else {
else {
// Ask the arbitrary data manager to fetch data for this transaction // Ask the arbitrary data manager to fetch data for this transaction
String message; String message;
if (this.canRequestMissingFiles) { if (this.canRequestMissingFiles) {
@ -394,8 +387,7 @@ public class ArbitraryDataReader {
} else { } else {
message = String.format("Unable to reissue request for missing file %s for signature %s due to rate limit. Please try again later.", arbitraryDataFile, Base58.encode(transactionData.getSignature())); message = String.format("Unable to reissue request for missing file %s for signature %s due to rate limit. Please try again later.", arbitraryDataFile, Base58.encode(transactionData.getSignature()));
} }
} } else {
else {
message = String.format("Missing data for file %s", arbitraryDataFile); message = String.format("Missing data for file %s", arbitraryDataFile);
} }
@ -405,6 +397,8 @@ public class ArbitraryDataReader {
} }
} }
// Data hashes need some extra processing
if (transactionData.getDataType() == DataType.DATA_HASH) {
if (arbitraryDataFile.allChunksExist() && !arbitraryDataFile.exists()) { if (arbitraryDataFile.allChunksExist() && !arbitraryDataFile.exists()) {
// We have all the chunks but not the complete file, so join them // We have all the chunks but not the complete file, so join them
arbitraryDataFile.join(); arbitraryDataFile.join();
@ -415,11 +409,13 @@ public class ArbitraryDataReader {
throw new IOException(String.format("File doesn't exist: %s", arbitraryDataFile)); throw new IOException(String.format("File doesn't exist: %s", arbitraryDataFile));
} }
// Ensure the complete hash matches the joined chunks // Ensure the complete hash matches the joined chunks
if (!Arrays.equals(arbitraryDataFile.digest(), digest)) { if (!Arrays.equals(arbitraryDataFile.digest(), transactionData.getData())) {
// Delete the invalid file // Delete the invalid file
arbitraryDataFile.delete(); arbitraryDataFile.delete();
throw new DataException("Unable to validate complete file hash"); throw new DataException("Unable to validate complete file hash");
} }
}
// Ensure the file's size matches the size reported by the transaction (throws a DataException if not) // Ensure the file's size matches the size reported by the transaction (throws a DataException if not)
arbitraryDataFile.validateFileSize(transactionData.getSize()); arbitraryDataFile.validateFileSize(transactionData.getSize());

View File

@ -150,11 +150,7 @@ public class ArbitraryDataResource {
List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions); List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
for (ArbitraryTransactionData transactionData : transactionDataList) { for (ArbitraryTransactionData transactionData : transactionDataList) {
byte[] hash = transactionData.getData(); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
// Delete any chunks or complete files from each transaction // Delete any chunks or complete files from each transaction
arbitraryDataFile.deleteAll(deleteMetadata); arbitraryDataFile.deleteAll(deleteMetadata);

View File

@ -9,6 +9,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Category;
import org.qortal.arbitrary.misc.Service; import org.qortal.arbitrary.misc.Service;
import org.qortal.crypto.AES;
import org.qortal.crypto.Crypto; import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData; import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData;
@ -181,6 +182,7 @@ public class ArbitraryDataTransactionBuilder {
for (ModifiedPath path : metadata.getModifiedPaths()) { for (ModifiedPath path : metadata.getModifiedPaths()) {
if (path.getDiffType() != DiffType.COMPLETE_FILE) { if (path.getDiffType() != DiffType.COMPLETE_FILE) {
atLeastOnePatch = true; atLeastOnePatch = true;
break;
} }
} }
} }
@ -229,10 +231,12 @@ public class ArbitraryDataTransactionBuilder {
random.nextBytes(lastReference); random.nextBytes(lastReference);
} }
Compression compression = Compression.ZIP; // Single file resources are handled differently, especially for very small data payloads, as these go on chain
final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(path, false);
final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE);
// FUTURE? Use zip compression for directories, or no compression for single files // Use zip compression if data isn't going on chain
// Compression compression = (path.toFile().isDirectory()) ? Compression.ZIP : Compression.NONE; Compression compression = shouldUseOnChainData ? Compression.NONE : Compression.ZIP;
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method,
compression, title, description, tags, category); compression, title, description, tags, category);
@ -250,16 +254,21 @@ public class ArbitraryDataTransactionBuilder {
throw new DataException("Arbitrary data file is null"); throw new DataException("Arbitrary data file is null");
} }
// Get chunks metadata file // Get metadata file
ArbitraryDataFile metadataFile = arbitraryDataFile.getMetadataFile(); ArbitraryDataFile metadataFile = arbitraryDataFile.getMetadataFile();
if (metadataFile == null && arbitraryDataFile.chunkCount() > 1) { if (metadataFile == null && arbitraryDataFile.chunkCount() > 1) {
throw new DataException(String.format("Chunks metadata data file is null but there are %d chunks", arbitraryDataFile.chunkCount())); throw new DataException(String.format("Chunks metadata data file is null but there are %d chunks", arbitraryDataFile.chunkCount()));
} }
String digest58 = arbitraryDataFile.digest58(); // Default to using a data hash, with data held off-chain
if (digest58 == null) { ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
LOGGER.error("Unable to calculate file digest"); byte[] data = arbitraryDataFile.digest();
throw new DataException("Unable to calculate file digest");
// For small, single-chunk resources, we can store the data directly on chain
if (shouldUseOnChainData && arbitraryDataFile.getBytes().length <= ArbitraryTransaction.MAX_DATA_SIZE && arbitraryDataFile.chunkCount() == 0) {
// Within allowed on-chain data size
dataType = DataType.RAW_DATA;
data = arbitraryDataFile.getBytes();
} }
final BaseTransactionData baseTransactionData = new BaseTransactionData(now, Group.NO_GROUP, final BaseTransactionData baseTransactionData = new BaseTransactionData(now, Group.NO_GROUP,
@ -268,22 +277,21 @@ public class ArbitraryDataTransactionBuilder {
final int version = 5; final int version = 5;
final int nonce = 0; final int nonce = 0;
byte[] secret = arbitraryDataFile.getSecret(); byte[] secret = arbitraryDataFile.getSecret();
final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
final byte[] digest = arbitraryDataFile.digest();
final byte[] metadataHash = (metadataFile != null) ? metadataFile.getHash() : null; final byte[] metadataHash = (metadataFile != null) ? metadataFile.getHash() : null;
final List<PaymentData> payments = new ArrayList<>(); final List<PaymentData> payments = new ArrayList<>();
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
version, service.value, nonce, size, name, identifier, method, version, service.value, nonce, size, name, identifier, method,
secret, compression, digest, dataType, metadataHash, payments); secret, compression, data, dataType, metadataHash, payments);
this.arbitraryTransactionData = transactionData; this.arbitraryTransactionData = transactionData;
} catch (DataException e) { } catch (DataException | IOException e) {
if (arbitraryDataFile != null) { if (arbitraryDataFile != null) {
arbitraryDataFile.deleteAll(true); arbitraryDataFile.deleteAll(true);
} }
throw(e); throw new DataException(e);
} }
} }

View File

@ -107,10 +107,9 @@ public class ArbitraryDataWriter {
private void preExecute() throws DataException { private void preExecute() throws DataException {
this.checkEnabled(); this.checkEnabled();
// Enforce compression when uploading a directory // Enforce compression when uploading multiple files
File file = new File(this.filePath.toString()); if (!FilesystemUtils.isSingleFileResource(this.filePath, false) && compression == Compression.NONE) {
if (file.isDirectory() && compression == Compression.NONE) { throw new DataException("Unable to publish multiple files without compression");
throw new DataException("Unable to upload a directory without compression");
} }
// Create temporary working directory // Create temporary working directory
@ -168,6 +167,9 @@ public class ArbitraryDataWriter {
if (this.files.size() == 1) { if (this.files.size() == 1) {
singleFilePath = Paths.get(this.filePath.toString(), this.files.get(0)); singleFilePath = Paths.get(this.filePath.toString(), this.files.get(0));
// Update filePath to point to the single file (instead of the directory containing the file)
this.filePath = singleFilePath;
} }
} }
} }
@ -314,9 +316,6 @@ public class ArbitraryDataWriter {
if (chunkCount > 0) { if (chunkCount > 0) {
LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s")));
} }
else {
throw new DataException("Unable to split file into chunks");
}
} }
private void createMetadataFile() throws IOException, DataException { private void createMetadataFile() throws IOException, DataException {

View File

@ -258,8 +258,6 @@ public class ArbitraryDataFileListManager {
// Lookup file lists by signature (and optionally hashes) // Lookup file lists by signature (and optionally hashes)
public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) { public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) {
byte[] digest = arbitraryTransactionData.getData();
byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
byte[] signature = arbitraryTransactionData.getSignature(); byte[] signature = arbitraryTransactionData.getSignature();
String signature58 = Base58.encode(signature); String signature58 = Base58.encode(signature);
@ -286,8 +284,7 @@ public class ArbitraryDataFileListManager {
// Find hashes that we are missing // Find hashes that we are missing
try { try {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
arbitraryDataFile.setMetadataHash(metadataHash);
missingHashes = arbitraryDataFile.missingHashes(); missingHashes = arbitraryDataFile.missingHashes();
} catch (DataException e) { } catch (DataException e) {
// Leave missingHashes as null, so that all hashes are requested // Leave missingHashes as null, so that all hashes are requested
@ -460,10 +457,9 @@ public class ArbitraryDataFileListManager {
arbitraryTransactionData = (ArbitraryTransactionData) transactionData; arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
// Load data file(s) // // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature); // ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
arbitraryDataFile.setMetadataHash(arbitraryTransactionData.getMetadataHash()); //
// // Check all hashes exist // // Check all hashes exist
// for (byte[] hash : hashes) { // for (byte[] hash : hashes) {
// //LOGGER.debug("Received hash {}", Base58.encode(hash)); // //LOGGER.debug("Received hash {}", Base58.encode(hash));
@ -594,12 +590,8 @@ public class ArbitraryDataFileListManager {
// Check if we're even allowed to serve data for this transaction // Check if we're even allowed to serve data for this transaction
if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
byte[] hash = transactionData.getData();
byte[] metadataHash = transactionData.getMetadataHash();
// Load file(s) and add any that exist to the list of hashes // Load file(s) and add any that exist to the list of hashes
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
arbitraryDataFile.setMetadataHash(metadataHash);
// If the peer didn't supply a hash list, we need to return all hashes for this transaction // If the peer didn't supply a hash list, we need to return all hashes for this transaction
if (requestedHashes == null || requestedHashes.isEmpty()) { if (requestedHashes == null || requestedHashes.isEmpty()) {

View File

@ -132,9 +132,7 @@ public class ArbitraryDataFileManager extends Thread {
List<byte[]> hashes) throws DataException { List<byte[]> hashes) throws DataException {
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
arbitraryDataFile.setMetadataHash(metadataHash);
boolean receivedAtLeastOneFile = false; boolean receivedAtLeastOneFile = false;
// Now fetch actual data from this peer // Now fetch actual data from this peer

View File

@ -202,4 +202,12 @@ public class AES {
.decode(cipherText))); .decode(cipherText)));
} }
public static long getEncryptedFileSize(long inFileSize) {
// To calculate the resulting file size, add 16 (for the IV), then round up to the nearest multiple of 16
final int ivSize = 16;
final int chunkSize = 16;
final int expectedSize = Math.round((inFileSize + ivSize) / chunkSize) * chunkSize + chunkSize;
return expectedSize;
}
} }

View File

@ -5,9 +5,7 @@ import org.apache.logging.log4j.Logger;
import org.bouncycastle.util.Longs; import org.bouncycastle.util.Longs;
import org.qortal.arbitrary.misc.Service; import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo; import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.crypto.Crypto;
import org.qortal.data.arbitrary.ArbitraryResourceNameInfo; import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
import org.qortal.data.network.ArbitraryPeerData;
import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.BaseTransactionData;
@ -15,6 +13,7 @@ import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.ArbitraryRepository;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction.ApprovalStatus; import org.qortal.transaction.Transaction.ApprovalStatus;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
@ -27,8 +26,6 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
private static final Logger LOGGER = LogManager.getLogger(HSQLDBArbitraryRepository.class); private static final Logger LOGGER = LogManager.getLogger(HSQLDBArbitraryRepository.class);
private static final int MAX_RAW_DATA_SIZE = 255; // size of VARBINARY
protected HSQLDBRepository repository; protected HSQLDBRepository repository;
public HSQLDBArbitraryRepository(HSQLDBRepository repository) { public HSQLDBArbitraryRepository(HSQLDBRepository repository) {
@ -55,13 +52,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
return true; return true;
} }
// Load hashes
byte[] hash = transactionData.getData();
byte[] metadataHash = transactionData.getMetadataHash();
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
arbitraryDataFile.setMetadataHash(metadataHash);
// Check if we already have the complete data file or all chunks // Check if we already have the complete data file or all chunks
if (arbitraryDataFile.allFilesExist()) { if (arbitraryDataFile.allFilesExist()) {
@ -84,13 +76,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
return transactionData.getData(); return transactionData.getData();
} }
// Load hashes
byte[] digest = transactionData.getData();
byte[] metadataHash = transactionData.getMetadataHash();
// Load data file(s) // Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
arbitraryDataFile.setMetadataHash(metadataHash);
// If we have the complete data file, return it // If we have the complete data file, return it
if (arbitraryDataFile.exists()) { if (arbitraryDataFile.exists()) {
@ -105,6 +92,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
arbitraryDataFile.join(); arbitraryDataFile.join();
// Verify that the combined hash matches the expected hash // Verify that the combined hash matches the expected hash
byte[] digest = transactionData.getData();
if (!digest.equals(arbitraryDataFile.digest())) { if (!digest.equals(arbitraryDataFile.digest())) {
LOGGER.info(String.format("Hash mismatch for transaction: %s", Base58.encode(signature))); LOGGER.info(String.format("Hash mismatch for transaction: %s", Base58.encode(signature)));
return null; return null;
@ -132,11 +120,11 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
} }
// Trivial-sized payloads can remain in raw form // Trivial-sized payloads can remain in raw form
if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE) { if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= ArbitraryTransaction.MAX_DATA_SIZE) {
return; return;
} }
throw new IllegalStateException(String.format("Supplied data is larger than maximum size (%d bytes). Please use ArbitraryDataWriter.", MAX_RAW_DATA_SIZE)); throw new IllegalStateException(String.format("Supplied data is larger than maximum size (%d bytes). Please use ArbitraryDataWriter.", ArbitraryTransaction.MAX_DATA_SIZE));
} }
@Override @Override
@ -146,14 +134,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
return; return;
} }
// Load hashes
byte[] hash = arbitraryTransactionData.getData();
byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
// Load data file(s) // Load data file(s)
byte[] signature = arbitraryTransactionData.getSignature(); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
// Delete file, chunks, and metadata // Delete file, chunks, and metadata
arbitraryDataFile.deleteAll(true); arbitraryDataFile.deleteAll(true);

View File

@ -33,7 +33,7 @@ public class ArbitraryTransaction extends Transaction {
private ArbitraryTransactionData arbitraryTransactionData; private ArbitraryTransactionData arbitraryTransactionData;
// Other useful constants // Other useful constants
public static final int MAX_DATA_SIZE = 4000; public static final int MAX_DATA_SIZE = 256;
public static final int MAX_METADATA_LENGTH = 32; public static final int MAX_METADATA_LENGTH = 32;
public static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; public static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH;
public static final int MAX_IDENTIFIER_LENGTH = 64; public static final int MAX_IDENTIFIER_LENGTH = 64;

View File

@ -110,13 +110,8 @@ public class ArbitraryTransactionUtils {
return false; return false;
} }
byte[] digest = transactionData.getData();
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
// Load complete file and chunks // Load complete file and chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
arbitraryDataFile.setMetadataHash(metadataHash);
return arbitraryDataFile.allChunksExist(); return arbitraryDataFile.allChunksExist();
} }
@ -126,18 +121,13 @@ public class ArbitraryTransactionUtils {
return false; return false;
} }
byte[] digest = transactionData.getData(); if (transactionData.getMetadataHash() == null) {
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
if (metadataHash == null) {
// This file doesn't have any metadata/chunks, which means none exist // This file doesn't have any metadata/chunks, which means none exist
return false; return false;
} }
// Load complete file and chunks // Load complete file and chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
arbitraryDataFile.setMetadataHash(metadataHash);
return arbitraryDataFile.anyChunksExist(); return arbitraryDataFile.anyChunksExist();
} }
@ -147,12 +137,7 @@ public class ArbitraryTransactionUtils {
return 0; return 0;
} }
byte[] digest = transactionData.getData(); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
// Find the folder containing the files // Find the folder containing the files
Path parentPath = arbitraryDataFile.getFilePath().getParent(); Path parentPath = arbitraryDataFile.getFilePath().getParent();
@ -180,18 +165,13 @@ public class ArbitraryTransactionUtils {
return 0; return 0;
} }
byte[] digest = transactionData.getData(); if (transactionData.getMetadataHash() == null) {
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
if (metadataHash == null) {
// This file doesn't have any metadata, therefore it has a single (complete) chunk // This file doesn't have any metadata, therefore it has a single (complete) chunk
return 1; return 1;
} }
// Load complete file and chunks // Load complete file and chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
arbitraryDataFile.setMetadataHash(metadataHash);
return arbitraryDataFile.fileCount(); return arbitraryDataFile.fileCount();
} }
@ -243,31 +223,24 @@ public class ArbitraryTransactionUtils {
} }
public static void deleteCompleteFileAndChunks(ArbitraryTransactionData arbitraryTransactionData) throws DataException { public static void deleteCompleteFileAndChunks(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
byte[] completeHash = arbitraryTransactionData.getData(); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
byte[] signature = arbitraryTransactionData.getSignature();
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
arbitraryDataFile.deleteAll(true); arbitraryDataFile.deleteAll(true);
} }
public static void convertFileToChunks(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException { public static void convertFileToChunks(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
byte[] completeHash = arbitraryTransactionData.getData();
byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
byte[] signature = arbitraryTransactionData.getSignature();
// Find the expected chunk hashes // Find the expected chunk hashes
ArbitraryDataFile expectedDataFile = ArbitraryDataFile.fromHash(completeHash, signature); ArbitraryDataFile expectedDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
expectedDataFile.setMetadataHash(metadataHash);
if (metadataHash == null || !expectedDataFile.getMetadataFile().exists()) { if (arbitraryTransactionData.getMetadataHash() == null || !expectedDataFile.getMetadataFile().exists()) {
// We don't have the metadata file, or this transaction doesn't have one - nothing to do // We don't have the metadata file, or this transaction doesn't have one - nothing to do
return; return;
} }
byte[] completeHash = arbitraryTransactionData.getData();
byte[] signature = arbitraryTransactionData.getSignature();
// Split the file into chunks // Split the file into chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(completeHash, signature); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE); int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE);
if (chunkCount > 1) { if (chunkCount > 1) {
LOGGER.info(String.format("Successfully split %s into %d chunk%s", LOGGER.info(String.format("Successfully split %s into %d chunk%s",

View File

@ -250,6 +250,39 @@ public class FilesystemUtils {
return data; return data;
} }
/**
* isSingleFileResource
* Returns true if the path points to a file, or a
* directory containing a single file only.
*
* @param path to file or directory
* @param excludeQortalDirectory - if true, a directory containing a single file and a .qortal directory is considered a single file resource
* @return
* @throws IOException
*/
public static boolean isSingleFileResource(Path path, boolean excludeQortalDirectory) {
// If the path is a file, read the contents directly
if (path.toFile().isFile()) {
return true;
}
// Or if it's a directory, only load file contents if there is a single file inside it
else if (path.toFile().isDirectory()) {
String[] files = path.toFile().list();
if (excludeQortalDirectory) {
files = ArrayUtils.removeElement(files, ".qortal");
}
if (files.length == 1) {
Path filePath = Paths.get(path.toString(), files[0]);
if (filePath.toFile().isFile()) {
return true;
}
}
}
return false;
}
public static byte[] readFromFile(String filePath, long position, int size) throws IOException { public static byte[] readFromFile(String filePath, long position, int size) throws IOException {
RandomAccessFile file = new RandomAccessFile(filePath, "r"); RandomAccessFile file = new RandomAccessFile(filePath, "r");
file.seek(position); file.seek(position);

View File

@ -111,8 +111,8 @@ public class ArbitraryDataTests extends Common {
fail("Creating transaction should fail due to nonexistent PUT transaction"); fail("Creating transaction should fail due to nonexistent PUT transaction");
} catch (DataException expectedException) { } catch (DataException expectedException) {
assertEquals(String.format("Couldn't find PUT transaction for " + assertTrue(expectedException.getMessage().contains(String.format("Couldn't find PUT transaction for " +
"name %s, service %s and identifier ", name, service), expectedException.getMessage()); "name %s, service %s and identifier ", name, service)));
} }
} }
@ -358,7 +358,7 @@ public class ArbitraryDataTests extends Common {
byte[] path1FileDigest = Crypto.digest(path1.toFile()); byte[] path1FileDigest = Crypto.digest(path1.toFile());
ArbitraryDataDigest path1DirectoryDigest = new ArbitraryDataDigest(path1.getParent()); ArbitraryDataDigest path1DirectoryDigest = new ArbitraryDataDigest(path1.getParent());
path1DirectoryDigest.compute(); path1DirectoryDigest.compute();
ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice);
// Now build the latest data state for this name // Now build the latest data state for this name
ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);

View File

@ -0,0 +1,135 @@
package org.qortal.test.arbitrary;
import org.junit.Before;
import org.junit.Test;
import org.qortal.arbitrary.ArbitraryDataDigest;
import org.qortal.crypto.AES;
import org.qortal.crypto.Crypto;
import org.qortal.repository.DataException;
import org.qortal.test.common.Common;
import org.qortal.utils.ZipUtils;
import javax.crypto.BadPaddingException;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Random;
import static org.junit.Assert.*;
public class ArbitraryEncryptionTests extends Common {
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
}
@Test
public void testEncryption() throws IOException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException {
String enclosingFolderName = "data";
Path inputFilePath = Files.createTempFile("inputFile", null);
Path outputDirectory = Files.createTempDirectory("outputDirectory");
Path outputFilePath = Paths.get(outputDirectory.toString(), enclosingFolderName);
inputFilePath.toFile().deleteOnExit();
outputDirectory.toFile().deleteOnExit();
// Write random data to the input file
byte[] data = new byte[10];
new Random().nextBytes(data);
Files.write(inputFilePath, data, StandardOpenOption.CREATE);
assertTrue(Files.exists(inputFilePath));
assertFalse(Files.exists(outputFilePath));
// Encrypt...
String algorithm = "AES/CBC/PKCS5Padding";
SecretKey aesKey = AES.generateKey(256);
AES.encryptFile(algorithm, aesKey, inputFilePath.toString(), outputFilePath.toString());
assertTrue(Files.exists(inputFilePath));
assertTrue(Files.exists(outputFilePath));
// Ensure encrypted file's hash differs from the original
assertFalse(Arrays.equals(Crypto.digest(inputFilePath.toFile()), Crypto.digest(outputFilePath.toFile())));
// Create paths for decrypting
Path decryptedDirectory = Files.createTempDirectory("decryptedDirectory");
Path decryptedFile = Paths.get(decryptedDirectory.toString(), enclosingFolderName, inputFilePath.getFileName().toString());
decryptedDirectory.toFile().deleteOnExit();
assertFalse(Files.exists(decryptedFile));
// Now decrypt...
AES.decryptFile(algorithm, aesKey, outputFilePath.toString(), decryptedFile.toString());
// Ensure resulting file exists
assertTrue(Files.exists(decryptedFile));
// And make sure it matches the original input file
assertTrue(Arrays.equals(Crypto.digest(inputFilePath.toFile()), Crypto.digest(decryptedFile.toFile())));
}
@Test
public void testEncryptionSizeOverhead() throws IOException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, NoSuchPaddingException, IllegalBlockSizeException, BadPaddingException, InvalidKeyException {
for (int size = 1; size < 256; size++) {
String enclosingFolderName = "data";
Path inputFilePath = Files.createTempFile("inputFile", null);
Path outputDirectory = Files.createTempDirectory("outputDirectory");
Path outputFilePath = Paths.get(outputDirectory.toString(), enclosingFolderName);
inputFilePath.toFile().deleteOnExit();
outputDirectory.toFile().deleteOnExit();
// Write random data to the input file
byte[] data = new byte[size];
new Random().nextBytes(data);
Files.write(inputFilePath, data, StandardOpenOption.CREATE);
assertTrue(Files.exists(inputFilePath));
assertFalse(Files.exists(outputFilePath));
// Ensure input file is the same size as the data
assertEquals(size, inputFilePath.toFile().length());
// Encrypt...
String algorithm = "AES/CBC/PKCS5Padding";
SecretKey aesKey = AES.generateKey(256);
AES.encryptFile(algorithm, aesKey, inputFilePath.toString(), outputFilePath.toString());
assertTrue(Files.exists(inputFilePath));
assertTrue(Files.exists(outputFilePath));
final long expectedSize = AES.getEncryptedFileSize(inputFilePath.toFile().length());
System.out.println(String.format("Plaintext size: %d bytes, Ciphertext size: %d bytes", inputFilePath.toFile().length(), outputFilePath.toFile().length()));
// Ensure encryption added a fixed amount of space to the output file
assertEquals(expectedSize, outputFilePath.toFile().length());
// Ensure encrypted file's hash differs from the original
assertFalse(Arrays.equals(Crypto.digest(inputFilePath.toFile()), Crypto.digest(outputFilePath.toFile())));
// Create paths for decrypting
Path decryptedDirectory = Files.createTempDirectory("decryptedDirectory");
Path decryptedFile = Paths.get(decryptedDirectory.toString(), enclosingFolderName, inputFilePath.getFileName().toString());
decryptedDirectory.toFile().deleteOnExit();
assertFalse(Files.exists(decryptedFile));
// Now decrypt...
AES.decryptFile(algorithm, aesKey, outputFilePath.toString(), decryptedFile.toString());
// Ensure resulting file exists
assertTrue(Files.exists(decryptedFile));
// And make sure it matches the original input file
assertTrue(Arrays.equals(Crypto.digest(inputFilePath.toFile()), Crypto.digest(decryptedFile.toFile())));
}
}
}

View File

@ -91,8 +91,8 @@ public class ArbitraryTransactionMetadataTests extends Common {
String name = "TEST"; // Can be anything for this test String name = "TEST"; // Can be anything for this test
String identifier = null; // Not used for this test String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA; Service service = Service.ARBITRARY_DATA;
int chunkSize = 1000; int chunkSize = 10000;
int dataLength = 10; // Actual data length will be longer due to encryption int dataLength = 1000; // Actual data length will be longer due to encryption
String title = "Test title"; String title = "Test title";
String description = "Test description"; String description = "Test description";
@ -142,8 +142,8 @@ public class ArbitraryTransactionMetadataTests extends Common {
String name = "TEST"; // Can be anything for this test String name = "TEST"; // Can be anything for this test
String identifier = null; // Not used for this test String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA; Service service = Service.ARBITRARY_DATA;
int chunkSize = 1000; int chunkSize = 10000;
int dataLength = 10; // Actual data length will be longer due to encryption int dataLength = 1000; // Actual data length will be longer due to encryption
String title = "Test title"; String title = "Test title";
String description = "Test description"; String description = "Test description";
@ -265,7 +265,7 @@ public class ArbitraryTransactionMetadataTests extends Common {
Category category = Category.CRYPTOCURRENCY; Category category = Category.CRYPTOCURRENCY;
String expectedTitle = "title Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent feugiat "; // 80 chars String expectedTitle = "title Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent feugiat "; // 80 chars
String expectedDescription = "description Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent feugiat pretium massa, non pulvinar mi pretium id. Ut gravida sapien vitae dui posuere tincidunt. Quisque in nibh est. Curabitur at blandit nunc, id aliquet neque. Nulla condimentum eget dolor a egestas. Vestibulum vel tincidunt ex. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Cras congue lacus in risus mattis suscipit. Quisque nisl eros, facilisis a lorem quis, vehicula biben"; // 500 chars String expectedDescription = "description Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent feugiat pretium massa, non pulvinar mi pretium id. Ut gravida sapien vitae dui posuere tincidunt. Quisque in nibh est. Curabitur at blandit nunc, id aliquet neque"; // 240 chars
List<String> expectedTags = Arrays.asList("tag 1", "tag 2", "tag 4", "tag 5", "tag 6"); List<String> expectedTags = Arrays.asList("tag 1", "tag 2", "tag 4", "tag 5", "tag 6");
// Register the name to Alice // Register the name to Alice

View File

@ -5,9 +5,13 @@ import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.qortal.account.PrivateKeyAccount; import org.qortal.account.PrivateKeyAccount;
import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataTransactionBuilder; import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Category;
import org.qortal.arbitrary.misc.Service; import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData; import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.BaseTransactionData;
@ -24,12 +28,16 @@ import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.RegisterNameTransaction; import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction;
import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
import org.qortal.utils.NTP; import org.qortal.utils.NTP;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Random; import java.util.Random;
@ -467,4 +475,173 @@ public class ArbitraryTransactionTests extends Common {
secret, compression, digest, dataType, metadataHash, payments); secret, compression, digest, dataType, metadataHash, payments);
assertNull(transactionData.getService()); assertNull(transactionData.getService());
} }
@Test
public void testOnChainData() throws DataException, IOException, MissingDataException, IllegalAccessException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Set difficulty to 1 to speed up the tests
FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true);
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
String name = "TEST"; // Can be anything for this test
String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA;
int chunkSize = 1000;
int dataLength = 239; // Max possible size. Becomes 256 bytes after encryption.
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength, true);
ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize, 0L, true,
null, null, null, null);
byte[] signature = arbitraryDataFile.getSignature();
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(signature);
// Check that the data is published on chain
assertEquals(ArbitraryTransactionData.DataType.RAW_DATA, arbitraryTransactionData.getDataType());
assertEquals(arbitraryDataFile.getBytes().length, arbitraryTransactionData.getData().length);
assertArrayEquals(arbitraryDataFile.getBytes(), arbitraryTransactionData.getData());
// Check that we have no chunks because the complete file is already less than the chunk size
assertEquals(0, arbitraryDataFile.chunkCount());
// Check that we have one file total - just the complete file (no chunks or metadata)
assertEquals(1, arbitraryDataFile.fileCount());
// Check the metadata isn't present
assertNull(arbitraryDataFile.getMetadata());
// Now build the latest data state for this name
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
arbitraryDataReader.loadSynchronously(true);
// Filename will be "data" because it's been held as raw bytes in the transaction,
// so there is nowhere to store the original filename
File outputFile = Paths.get(arbitraryDataReader.getFilePath().toString(), "data").toFile();
assertArrayEquals(Crypto.digest(outputFile), Crypto.digest(path1.toFile()));
}
}
@Test
public void testOnChainDataWithMetadata() throws DataException, IOException, MissingDataException, IllegalAccessException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Set difficulty to 1 to speed up the tests
FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true);
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
String name = "TEST"; // Can be anything for this test
String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA;
int chunkSize = 1000;
int dataLength = 239; // Max possible size. Becomes 256 bytes after encryption.
String title = "Test title";
String description = "Test description";
List<String> tags = Arrays.asList("Test", "tag", "another tag");
Category category = Category.QORTAL;
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength, true);
ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize, 0L, true,
title, description, tags, category);
byte[] signature = arbitraryDataFile.getSignature();
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(signature);
// Check that the data is published on chain
assertEquals(ArbitraryTransactionData.DataType.RAW_DATA, arbitraryTransactionData.getDataType());
assertEquals(arbitraryDataFile.getBytes().length, arbitraryTransactionData.getData().length);
assertArrayEquals(arbitraryDataFile.getBytes(), arbitraryTransactionData.getData());
// Check that we have no chunks because the complete file is already less than the chunk size
assertEquals(0, arbitraryDataFile.chunkCount());
// Check that we have two files total - one for the complete file, and the other for the metadata
assertEquals(2, arbitraryDataFile.fileCount());
// Check the metadata is correct
assertEquals(title, arbitraryDataFile.getMetadata().getTitle());
assertEquals(description, arbitraryDataFile.getMetadata().getDescription());
assertEquals(tags, arbitraryDataFile.getMetadata().getTags());
assertEquals(category, arbitraryDataFile.getMetadata().getCategory());
assertEquals("text/plain", arbitraryDataFile.getMetadata().getMimeType());
// Now build the latest data state for this name
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
arbitraryDataReader.loadSynchronously(true);
// Filename will be "data" because it's been held as raw bytes in the transaction,
// so there is nowhere to store the original filename
File outputFile = Paths.get(arbitraryDataReader.getFilePath().toString(), "data").toFile();
assertArrayEquals(Crypto.digest(outputFile), Crypto.digest(path1.toFile()));
}
}
@Test
public void testOffChainData() throws DataException, IOException, MissingDataException, IllegalAccessException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Set difficulty to 1 to speed up the tests
FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true);
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
String name = "TEST"; // Can be anything for this test
String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA;
int chunkSize = 1000;
int dataLength = 240; // Min possible size. Becomes 257 bytes after encryption.
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength, true);
ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize, 0L, true,
null, null, null, null);
byte[] signature = arbitraryDataFile.getSignature();
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(signature);
// Check that the data is published on chain
assertEquals(ArbitraryTransactionData.DataType.DATA_HASH, arbitraryTransactionData.getDataType());
assertEquals(TransactionTransformer.SHA256_LENGTH, arbitraryTransactionData.getData().length);
assertFalse(Arrays.equals(arbitraryDataFile.getBytes(), arbitraryTransactionData.getData()));
// Check that we have no chunks because the complete file is already less than the chunk size
assertEquals(0, arbitraryDataFile.chunkCount());
// Check that we have one file total - just the complete file (no chunks or metadata)
assertEquals(1, arbitraryDataFile.fileCount());
// Check the metadata isn't present
assertNull(arbitraryDataFile.getMetadata());
// Now build the latest data state for this name
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
arbitraryDataReader.loadSynchronously(true);
// File content should match original file
File outputFile = Paths.get(arbitraryDataReader.getFilePath().toString(), "file.txt").toFile();
assertArrayEquals(Crypto.digest(outputFile), Crypto.digest(path1.toFile()));
}
}
} }

View File

@ -51,12 +51,8 @@ public class ArbitraryUtils {
BlockUtils.mintBlock(repository); BlockUtils.mintBlock(repository);
// We need a new ArbitraryDataFile instance because the files will have been moved to the signature's folder // We need a new ArbitraryDataFile instance because the files will have been moved to the signature's folder
byte[] hash = txnBuilder.getArbitraryDataFile().getHash(); // Or, it may now be using RAW_DATA instead of a hash
byte[] signature = transactionData.getSignature(); return ArbitraryDataFile.fromTransactionData(transactionData);
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
arbitraryDataFile.setMetadataHash(transactionData.getMetadataHash());
return arbitraryDataFile;
} }
public static ArbitraryDataFile createAndMintTxn(Repository repository, String publicKey58, Path path, String name, String identifier, public static ArbitraryDataFile createAndMintTxn(Repository repository, String publicKey58, Path path, String name, String identifier,
@ -68,6 +64,17 @@ public class ArbitraryUtils {
} }
public static Path generateRandomDataPath(int length) throws IOException { public static Path generateRandomDataPath(int length) throws IOException {
return generateRandomDataPath(length, false);
}
/**
* Generate random data, held in a single file within a directory
* @param length - size of file to create
* @param returnFilePath - if true, the file's path is returned. If false, the outer directory's path is returned.
* @return - path to file or directory, depending on the "returnFilePath" boolean
* @throws IOException
*/
public static Path generateRandomDataPath(int length, boolean returnFilePath) throws IOException {
// Create a file in a random temp directory // Create a file in a random temp directory
Path tempDir = Files.createTempDirectory("generateRandomDataPath"); Path tempDir = Files.createTempDirectory("generateRandomDataPath");
File file = new File(Paths.get(tempDir.toString(), "file.txt").toString()); File file = new File(Paths.get(tempDir.toString(), "file.txt").toString());
@ -84,6 +91,10 @@ public class ArbitraryUtils {
file1Writer.newLine(); file1Writer.newLine();
file1Writer.close(); file1Writer.close();
if (returnFilePath) {
return file.toPath();
}
return tempDir; return tempDir;
} }