Significant refactor of DataFile and DataFileChunk

This introduces the hash58 property, which stores the base58 hash of the file passed in at initialization. It leaves digest() and digest58() for when we need to compute a new hash from the file itself.
This commit is contained in:
CalDescent 2021-07-05 07:26:20 +01:00
parent 10dc19652e
commit 0086c6373b
5 changed files with 159 additions and 154 deletions

View File

@ -28,7 +28,6 @@ import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.block.BlockChain; import org.qortal.block.BlockChain;
import org.qortal.crypto.Crypto; import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData; import org.qortal.data.PaymentData;
import org.qortal.data.account.AccountData;
import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.TransactionData;
@ -271,7 +270,7 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
} }
DataFile dataFile = new DataFile(path); DataFile dataFile = DataFile.fromPath(path);
if (dataFile == null) { if (dataFile == null) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
@ -283,7 +282,7 @@ public class ArbitraryResource {
LOGGER.error("Invalid file: {}", validationResult); LOGGER.error("Invalid file: {}", validationResult);
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); LOGGER.info("Whole file digest: {}", dataFile.digest58());
int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); int chunkCount = dataFile.split(DataFile.CHUNK_SIZE);
if (chunkCount == 0) { if (chunkCount == 0) {
@ -292,8 +291,8 @@ public class ArbitraryResource {
} }
LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s")));
String base58Digest = dataFile.base58Digest(); String digest58 = dataFile.digest58();
if (base58Digest == null) { if (digest58 == null) {
LOGGER.error("Unable to calculate digest"); LOGGER.error("Unable to calculate digest");
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
@ -364,10 +363,10 @@ public class ArbitraryResource {
) )
} }
) )
public String deleteFile(String base58Digest) { public String deleteFile(String hash58) {
Security.checkApiCallAllowed(request); Security.checkApiCallAllowed(request);
DataFile dataFile = DataFile.fromBase58Digest(base58Digest); DataFile dataFile = DataFile.fromHash58(hash58);
if (dataFile.delete()) { if (dataFile.delete()) {
return "true"; return "true";
} }
@ -377,7 +376,7 @@ public class ArbitraryResource {
@GET @GET
@Path("/file/{hash}/frompeer/{peer}") @Path("/file/{hash}/frompeer/{peer}")
@Operation( @Operation(
summary = "Request file from a given peer, using supplied base58 encoded SHA256 digest string", summary = "Request file from a given peer, using supplied base58 encoded SHA256 hash",
responses = { responses = {
@ApiResponse( @ApiResponse(
description = "true if retrieved, false if not", description = "true if retrieved, false if not",
@ -391,10 +390,10 @@ public class ArbitraryResource {
} }
) )
@ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY}) @ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.FILE_NOT_FOUND, ApiError.NO_REPLY})
public Response getFileFromPeer(@PathParam("hash") String base58Digest, public Response getFileFromPeer(@PathParam("hash") String hash58,
@PathParam("peer") String targetPeerAddress) { @PathParam("peer") String targetPeerAddress) {
try { try {
if (base58Digest == null) { if (hash58 == null) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
} }
if (targetPeerAddress == null) { if (targetPeerAddress == null) {
@ -412,7 +411,7 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
} }
boolean success = this.requestFile(base58Digest, targetPeer); boolean success = this.requestFile(hash58, targetPeer);
if (success) { if (success) {
return Response.ok("true").build(); return Response.ok("true").build();
} }
@ -426,7 +425,7 @@ public class ArbitraryResource {
@POST @POST
@Path("/files/frompeer/{peer}") @Path("/files/frompeer/{peer}")
@Operation( @Operation(
summary = "Request multiple files from a given peer, using supplied comma separated base58 encoded SHA256 digest strings", summary = "Request multiple files from a given peer, using supplied comma separated base58 encoded SHA256 hashes",
requestBody = @RequestBody( requestBody = @RequestBody(
required = true, required = true,
content = @Content( content = @Content(
@ -470,12 +469,12 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
} }
String base58DigestList[] = files.split(","); String hash58List[] = files.split(",");
for (String base58Digest : base58DigestList) { for (String hash58 : hash58List) {
if (base58Digest != null) { if (hash58 != null) {
boolean success = this.requestFile(base58Digest, targetPeer); boolean success = this.requestFile(hash58, targetPeer);
if (!success) { if (!success) {
LOGGER.info("Failed to request file {} from peer {}", base58Digest, targetPeerAddress); LOGGER.info("Failed to request file {} from peer {}", hash58, targetPeerAddress);
} }
} }
} }
@ -487,17 +486,17 @@ public class ArbitraryResource {
} }
private boolean requestFile(String base58Digest, Peer targetPeer) { private boolean requestFile(String hash58, Peer targetPeer) {
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
DataFile dataFile = DataFile.fromBase58Digest(base58Digest); DataFile dataFile = DataFile.fromHash58(hash58);
if (dataFile.exists()) { if (dataFile.exists()) {
LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile); LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile);
} }
byte[] digest = null; byte[] digest = null;
try { try {
digest = Base58.decode(base58Digest); digest = Base58.decode(hash58);
} catch (NumberFormatException e) { } catch (NumberFormatException e) {
LOGGER.info("Invalid base58 encoded string"); LOGGER.info("Invalid base58 encoded string");
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
@ -558,22 +557,22 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
} }
DataFile dataFile = DataFile.fromBase58Digest(combinedHash); DataFile dataFile = DataFile.fromHash58(combinedHash);
if (dataFile.exists()) { if (dataFile.exists()) {
LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash); LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash);
} }
String base58DigestList[] = files.split(","); String hash58List[] = files.split(",");
for (String base58Digest : base58DigestList) { for (String hash58 : hash58List) {
if (base58Digest != null) { if (hash58 != null) {
DataFileChunk chunk = DataFileChunk.fromBase58Digest(base58Digest); DataFileChunk chunk = DataFileChunk.fromHash58(hash58);
dataFile.addChunk(chunk); dataFile.addChunk(chunk);
} }
} }
boolean success = dataFile.join(); boolean success = dataFile.join();
if (success) { if (success) {
if (combinedHash.equals(dataFile.base58Digest())) { if (combinedHash.equals(dataFile.digest58())) {
LOGGER.info("Valid hash {} after joining {} files", dataFile.base58Digest(), dataFile.chunkCount()); LOGGER.info("Valid hash {} after joining {} files", dataFile.digest58(), dataFile.chunkCount());
return Response.ok("true").build(); return Response.ok("true").build();
} }
} }

View File

@ -30,7 +30,6 @@ import org.qortal.api.Security;
import org.qortal.block.BlockChain; import org.qortal.block.BlockChain;
import org.qortal.crypto.Crypto; import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData; import org.qortal.data.PaymentData;
import org.qortal.data.account.AccountData;
import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.group.Group; import org.qortal.group.Group;
@ -101,8 +100,8 @@ public class WebsiteResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
String base58Digest = dataFile.base58Digest(); String digest58 = dataFile.digest58();
if (base58Digest == null) { if (digest58 == null) {
LOGGER.error("Unable to calculate digest"); LOGGER.error("Unable to calculate digest");
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
@ -179,9 +178,9 @@ public class WebsiteResource {
DataFile dataFile = this.hostWebsite(directoryPath); DataFile dataFile = this.hostWebsite(directoryPath);
if (dataFile != null) { if (dataFile != null) {
String base58Digest = dataFile.base58Digest(); String digest58 = dataFile.digest58();
if (base58Digest != null) { if (digest58 != null) {
return "http://localhost:12393/site/" + base58Digest; return "http://localhost:12393/site/" + digest58;
} }
} }
return "Unable to generate preview URL"; return "Unable to generate preview URL";
@ -215,13 +214,13 @@ public class WebsiteResource {
} }
try { try {
DataFile dataFile = new DataFile(outputFilePath); DataFile dataFile = DataFile.fromPath(outputFilePath);
DataFile.ValidationResult validationResult = dataFile.isValid(); DataFile.ValidationResult validationResult = dataFile.isValid();
if (validationResult != DataFile.ValidationResult.OK) { if (validationResult != DataFile.ValidationResult.OK) {
LOGGER.error("Invalid file: {}", validationResult); LOGGER.error("Invalid file: {}", validationResult);
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
LOGGER.info("Whole file digest: {}", dataFile.base58Digest()); LOGGER.info("Whole file digest: {}", dataFile.digest58());
int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); int chunkCount = dataFile.split(DataFile.CHUNK_SIZE);
if (chunkCount > 0) { if (chunkCount > 0) {
@ -265,13 +264,13 @@ public class WebsiteResource {
if (!Files.exists(Paths.get(unzippedPath))) { if (!Files.exists(Paths.get(unzippedPath))) {
// Load file // Load file
DataFile dataFile = DataFile.fromBase58Digest(resourceId); DataFile dataFile = DataFile.fromHash58(resourceId);
if (dataFile == null || !dataFile.exists()) { if (dataFile == null || !dataFile.exists()) {
LOGGER.info("Unable to validate complete file hash"); LOGGER.info("Unable to validate complete file hash");
return this.get404Response(); return this.get404Response();
} }
if (!dataFile.base58Digest().equals(resourceId)) { if (!dataFile.digest58().equals(resourceId)) {
LOGGER.info("Unable to validate complete file hash"); LOGGER.info("Unable to validate complete file hash");
return this.get404Response(); return this.get404Response();
} }

View File

@ -13,9 +13,7 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.util.ArrayList; import java.util.*;
import java.util.Iterator;
import java.util.Map;
import java.util.stream.Stream; import java.util.stream.Stream;
import static java.util.Arrays.stream; import static java.util.Arrays.stream;
@ -50,28 +48,17 @@ public class DataFile {
public static int SHORT_DIGEST_LENGTH = 8; public static int SHORT_DIGEST_LENGTH = 8;
protected String filePath; protected String filePath;
protected String hash58;
private ArrayList<DataFileChunk> chunks; private ArrayList<DataFileChunk> chunks;
public DataFile() { public DataFile() {
} }
public DataFile(String filePath) { public DataFile(String hash58) {
this.createDataDirectory(); this.createDataDirectory();
this.filePath = filePath; this.filePath = DataFile.getOutputFilePath(hash58, false);
this.chunks = new ArrayList<>(); this.chunks = new ArrayList<>();
this.hash58 = hash58;
if (!this.isInBaseDirectory(filePath)) {
// Copy file to base directory
LOGGER.debug("Copying file to data directory...");
this.filePath = this.copyToDataDirectory();
if (this.filePath == null) {
throw new IllegalStateException("Invalid file path after copy");
}
}
}
public DataFile(File file) {
this(file.getPath());
} }
public DataFile(byte[] fileContent) { public DataFile(byte[] fileContent) {
@ -80,17 +67,17 @@ public class DataFile {
return; return;
} }
String base58Digest = Base58.encode(Crypto.digest(fileContent)); this.hash58 = Base58.encode(Crypto.digest(fileContent));
LOGGER.debug(String.format("File digest: %s, size: %d bytes", base58Digest, fileContent.length)); LOGGER.debug(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length));
String outputFilePath = this.getOutputFilePath(base58Digest, true); String outputFilePath = getOutputFilePath(this.hash58, true);
File outputFile = new File(outputFilePath); File outputFile = new File(outputFilePath);
try (FileOutputStream outputStream = new FileOutputStream(outputFile)) { try (FileOutputStream outputStream = new FileOutputStream(outputFile)) {
outputStream.write(fileContent); outputStream.write(fileContent);
this.filePath = outputFilePath; this.filePath = outputFilePath;
// Verify hash // Verify hash
if (!base58Digest.equals(this.base58Digest())) { if (!this.hash58.equals(this.digest58())) {
LOGGER.error("Digest {} does not match file digest {}", base58Digest, this.base58Digest()); LOGGER.error("Hash {} does not match file digest {}", this.hash58, this.digest58());
this.delete(); this.delete();
throw new IllegalStateException("Data file digest validation failed"); throw new IllegalStateException("Data file digest validation failed");
} }
@ -99,13 +86,38 @@ public class DataFile {
} }
} }
public static DataFile fromBase58Digest(String base58Digest) { public static DataFile fromHash58(String hash58) {
String filePath = DataFile.getOutputFilePath(base58Digest, false); return new DataFile(hash58);
return new DataFile(filePath);
} }
public static DataFile fromDigest(byte[] digest) { public static DataFile fromDigest(byte[] digest) {
return DataFile.fromBase58Digest(Base58.encode(digest)); return DataFile.fromHash58(Base58.encode(digest));
}
public static DataFile fromPath(String path) {
File file = new File(path);
if (file.exists()) {
try {
byte[] fileContent = Files.readAllBytes(file.toPath());
byte[] digest = Crypto.digest(fileContent);
DataFile dataFile = DataFile.fromDigest(digest);
// Copy file to base directory if needed
Path filePath = Paths.get(path);
if (Files.exists(filePath) && !dataFile.isInBaseDirectory(path)) {
dataFile.copyToDataDirectory(filePath);
}
return dataFile;
} catch (IOException e) {
LOGGER.error("Couldn't compute digest for DataFile");
}
}
return null;
}
public static DataFile fromFile(File file) {
return DataFile.fromPath(file.getPath());
} }
private boolean createDataDirectory() { private boolean createDataDirectory() {
@ -121,21 +133,27 @@ public class DataFile {
return true; return true;
} }
private String copyToDataDirectory() { private String copyToDataDirectory(Path sourcePath) {
String outputFilePath = this.getOutputFilePath(this.base58Digest(), true); if (this.hash58 == null || this.filePath == null) {
Path source = Paths.get(this.filePath).toAbsolutePath(); return null;
Path dest = Paths.get(outputFilePath).toAbsolutePath(); }
String outputFilePath = getOutputFilePath(this.hash58, true);
sourcePath = sourcePath.toAbsolutePath();
Path destPath = Paths.get(outputFilePath).toAbsolutePath();
try { try {
return Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING).toString(); return Files.copy(sourcePath, destPath, StandardCopyOption.REPLACE_EXISTING).toString();
} catch (IOException e) { } catch (IOException e) {
throw new IllegalStateException("Unable to copy file to data directory"); throw new IllegalStateException("Unable to copy file to data directory");
} }
} }
public static String getOutputFilePath(String base58Digest, boolean createDirectories) { public static String getOutputFilePath(String hash58, boolean createDirectories) {
String base58DigestFirst2Chars = base58Digest.substring(0, Math.min(base58Digest.length(), 2)); if (hash58 == null) {
String base58DigestNext2Chars = base58Digest.substring(2, Math.min(base58Digest.length(), 4)); return null;
String outputDirectory = Settings.getInstance().getDataPath() + File.separator + base58DigestFirst2Chars + File.separator + base58DigestNext2Chars; }
String hash58First2Chars = hash58.substring(0, 2);
String hash58Next2Chars = hash58.substring(2, 4);
String outputDirectory = Settings.getInstance().getDataPath() + File.separator + hash58First2Chars + File.separator + hash58Next2Chars;
Path outputDirectoryPath = Paths.get(outputDirectory); Path outputDirectoryPath = Paths.get(outputDirectory);
if (createDirectories) { if (createDirectories) {
@ -145,7 +163,7 @@ public class DataFile {
throw new IllegalStateException("Unable to create data subdirectory"); throw new IllegalStateException("Unable to create data subdirectory");
} }
} }
return outputDirectory + File.separator + base58Digest; return outputDirectory + File.separator + hash58;
} }
public ValidationResult isValid() { public ValidationResult isValid() {
@ -177,16 +195,11 @@ public class DataFile {
public void addChunkHashes(byte[] chunks) { public void addChunkHashes(byte[] chunks) {
ByteBuffer byteBuffer = ByteBuffer.wrap(chunks); ByteBuffer byteBuffer = ByteBuffer.wrap(chunks);
while (byteBuffer.remaining() > 0) { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
byte[] chunkData = new byte[TransactionTransformer.SHA256_LENGTH]; byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH];
byteBuffer.get(chunkData); byteBuffer.get(chunkDigest);
if (chunkData.length == TransactionTransformer.SHA256_LENGTH) { DataFileChunk chunk = DataFileChunk.fromHash(chunkDigest);
DataFileChunk chunk = new DataFileChunk(chunkData); this.addChunk(chunk);
this.addChunk(chunk);
}
else {
throw new IllegalStateException(String.format("Invalid chunk hash length: %d", chunkData.length));
}
} }
} }
@ -232,7 +245,7 @@ public class DataFile {
// Create temporary path for joined file // Create temporary path for joined file
Path tempPath; Path tempPath;
try { try {
tempPath = Files.createTempFile(this.chunks.get(0).base58Digest(), ".tmp"); tempPath = Files.createTempFile(this.chunks.get(0).digest58(), ".tmp");
} catch (IOException e) { } catch (IOException e) {
return false; return false;
} }
@ -245,7 +258,7 @@ public class DataFile {
File sourceFile = new File(chunk.filePath); File sourceFile = new File(chunk.filePath);
BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile));
byte[] buffer = new byte[2048]; byte[] buffer = new byte[2048];
int inSize = -1; int inSize;
while ((inSize = in.read(buffer)) != -1) { while ((inSize = in.read(buffer)) != -1) {
out.write(buffer, 0, inSize); out.write(buffer, 0, inSize);
} }
@ -254,7 +267,7 @@ public class DataFile {
out.close(); out.close();
// Copy temporary file to data directory // Copy temporary file to data directory
this.filePath = this.copyToDataDirectory(); this.filePath = this.copyToDataDirectory(tempPath);
Files.delete(tempPath); Files.delete(tempPath);
return true; return true;
@ -328,8 +341,7 @@ public class DataFile {
public byte[] getBytes() { public byte[] getBytes() {
Path path = Paths.get(this.filePath); Path path = Paths.get(this.filePath);
try { try {
byte[] bytes = Files.readAllBytes(path); return Files.readAllBytes(path);
return bytes;
} catch (IOException e) { } catch (IOException e) {
LOGGER.error("Unable to read bytes for file"); LOGGER.error("Unable to read bytes for file");
return null; return null;
@ -343,10 +355,7 @@ public class DataFile {
Path path = Paths.get(filePath).toAbsolutePath(); Path path = Paths.get(filePath).toAbsolutePath();
String dataPath = Settings.getInstance().getDataPath(); String dataPath = Settings.getInstance().getDataPath();
String basePath = Paths.get(dataPath).toAbsolutePath().toString(); String basePath = Paths.get(dataPath).toAbsolutePath().toString();
if (path.startsWith(basePath)) { return path.startsWith(basePath);
return true;
}
return false;
} }
public boolean exists() { public boolean exists() {
@ -354,9 +363,9 @@ public class DataFile {
return file.exists(); return file.exists();
} }
public boolean chunkExists(byte[] digest) { public boolean chunkExists(byte[] hash) {
for (DataFileChunk chunk : this.chunks) { for (DataFileChunk chunk : this.chunks) {
if (digest.equals(chunk.digest())) { // TODO: this is too heavy on the filesystem. We need a cache if (Arrays.equals(hash, chunk.getHash())) {
return chunk.exists(); return chunk.exists();
} }
} }
@ -366,17 +375,12 @@ public class DataFile {
public boolean allChunksExist(byte[] chunks) { public boolean allChunksExist(byte[] chunks) {
ByteBuffer byteBuffer = ByteBuffer.wrap(chunks); ByteBuffer byteBuffer = ByteBuffer.wrap(chunks);
while (byteBuffer.remaining() > 0) { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH]; byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH];
byteBuffer.get(chunkDigest); byteBuffer.get(chunkHash);
if (chunkDigest.length == TransactionTransformer.SHA256_LENGTH) { DataFileChunk chunk = DataFileChunk.fromHash(chunkHash);
DataFileChunk chunk = DataFileChunk.fromDigest(chunkDigest); if (!chunk.exists()) {
if (chunk.exists() == false) { return false;
return false;
}
}
else {
throw new IllegalStateException(String.format("Invalid chunk hash length: %d", chunkDigest.length));
} }
} }
return true; return true;
@ -395,6 +399,35 @@ public class DataFile {
return this.chunks.size(); return this.chunks.size();
} }
public List<DataFileChunk> getChunks() {
return this.chunks;
}
public byte[] chunkHashes() {
if (this.chunks != null && this.chunks.size() > 0) {
// Return null if we only have one chunk, with the same hash as the parent
if (Arrays.equals(this.digest(), this.chunks.get(0).digest())) {
return null;
}
try {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
for (DataFileChunk chunk : this.chunks) {
byte[] chunkHash = chunk.digest();
if (chunkHash.length != 32) {
LOGGER.info("Invalid chunk hash length: {}", chunkHash.length);
throw new IllegalStateException("Invalid chunk hash length");
}
outputStream.write(chunk.digest());
}
return outputStream.toByteArray();
} catch (IOException e) {
return null;
}
}
return null;
}
private File getFile() { private File getFile() {
File file = new File(this.filePath); File file = new File(this.filePath);
if (file.exists()) { if (file.exists()) {
@ -421,32 +454,7 @@ public class DataFile {
return null; return null;
} }
public byte[] chunkHashes() { public String digest58() {
if (this.chunks != null && this.chunks.size() > 0) {
// Return null if we only have one chunk, with the same hash as the parent
if (this.digest().equals(this.chunks.get(0).digest())) {
return null;
}
try {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
for (DataFileChunk chunk : this.chunks) {
byte[] chunkHash = chunk.digest();
if (chunkHash.length != 32) {
LOGGER.info("Invalid chunk hash length: {}", chunkHash.length);
throw new IllegalStateException("Invalid chunk hash length");
}
outputStream.write(chunk.digest());
}
return outputStream.toByteArray();
} catch (IOException e) {
return null;
}
}
return null;
}
public String base58Digest() {
if (this.digest() != null) { if (this.digest() != null) {
return Base58.encode(this.digest()); return Base58.encode(this.digest());
} }
@ -454,10 +462,18 @@ public class DataFile {
} }
public String shortDigest() { public String shortDigest() {
if (this.base58Digest() == null) { if (this.digest58() == null) {
return null; return null;
} }
return this.base58Digest().substring(0, Math.min(this.base58Digest().length(), SHORT_DIGEST_LENGTH)); return this.digest58().substring(0, Math.min(this.digest58().length(), SHORT_DIGEST_LENGTH));
}
public String getHash58() {
return this.hash58;
}
public byte[] getHash() {
return Base58.decode(this.hash58);
} }
public String printChunks() { public String printChunks() {
@ -467,7 +483,7 @@ public class DataFile {
if (outputString.length() > 0) { if (outputString.length() > 0) {
outputString = outputString.concat(","); outputString = outputString.concat(",");
} }
outputString = outputString.concat(chunk.base58Digest()); outputString = outputString.concat(chunk.digest58());
} }
} }
return outputString; return outputString;

View File

@ -4,7 +4,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@ -15,28 +14,20 @@ public class DataFileChunk extends DataFile {
private static final Logger LOGGER = LogManager.getLogger(DataFileChunk.class); private static final Logger LOGGER = LogManager.getLogger(DataFileChunk.class);
public DataFileChunk() { public DataFileChunk(String hash58) {
} super(hash58);
public DataFileChunk(String filePath) {
super(filePath);
}
public DataFileChunk(File file) {
super(file);
} }
public DataFileChunk(byte[] fileContent) { public DataFileChunk(byte[] fileContent) {
super(fileContent); super(fileContent);
} }
public static DataFileChunk fromBase58Digest(String base58Digest) { public static DataFileChunk fromHash58(String hash58) {
String filePath = DataFile.getOutputFilePath(base58Digest, false); return new DataFileChunk(hash58);
return new DataFileChunk(filePath);
} }
public static DataFileChunk fromDigest(byte[] digest) { public static DataFileChunk fromHash(byte[] hash) {
return DataFileChunk.fromBase58Digest(Base58.encode(digest)); return DataFileChunk.fromHash58(Base58.encode(hash));
} }
@Override @Override

View File

@ -23,7 +23,7 @@ public class DataTests extends Common {
DataFile dataFile = new DataFile(dummyDataString.getBytes()); DataFile dataFile = new DataFile(dummyDataString.getBytes());
assertTrue(dataFile.exists()); assertTrue(dataFile.exists());
assertEquals(62, dataFile.size()); assertEquals(62, dataFile.size());
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.base58Digest()); assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58());
// Split into 7 chunks, each 10 bytes long // Split into 7 chunks, each 10 bytes long
dataFile.split(10); dataFile.split(10);
@ -41,7 +41,7 @@ public class DataTests extends Common {
// Validate that the original file is intact // Validate that the original file is intact
assertTrue(dataFile.exists()); assertTrue(dataFile.exists());
assertEquals(62, dataFile.size()); assertEquals(62, dataFile.size());
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.base58Digest()); assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58());
} }
@Test @Test
@ -53,7 +53,7 @@ public class DataTests extends Common {
DataFile dataFile = new DataFile(randomData); DataFile dataFile = new DataFile(randomData);
assertTrue(dataFile.exists()); assertTrue(dataFile.exists());
assertEquals(fileSize, dataFile.size()); assertEquals(fileSize, dataFile.size());
String originalFileDigest = dataFile.base58Digest(); String originalFileDigest = dataFile.digest58();
// Split into chunks using 1MiB chunk size // Split into chunks using 1MiB chunk size
dataFile.split(1 * 1024 * 1024); dataFile.split(1 * 1024 * 1024);
@ -71,7 +71,7 @@ public class DataTests extends Common {
// Validate that the original file is intact // Validate that the original file is intact
assertTrue(dataFile.exists()); assertTrue(dataFile.exists());
assertEquals(fileSize, dataFile.size()); assertEquals(fileSize, dataFile.size());
assertEquals(originalFileDigest, dataFile.base58Digest()); assertEquals(originalFileDigest, dataFile.digest58());
} }
} }