diff --git a/.gitignore b/.gitignore index 8f2de896..225b48bf 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ /*.7z /tmp /data* +/src/test/resources/arbitrary/*/.qortal/cache diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index fcf9df1e..a70eeba2 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -145,7 +145,7 @@ public class ArbitraryDataBuilder { } } - private void buildLatestState() throws IOException { + private void buildLatestState() throws IOException, DataException { if (this.paths.size() == 1) { // No patching needed this.finalPath = this.paths.get(0); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java index 8366e7b4..73b670d6 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java @@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; +import org.qortal.repository.DataException; import org.qortal.utils.Base58; import org.qortal.utils.FilesystemUtils; @@ -31,7 +32,7 @@ public class ArbitraryDataCombiner { this.signatureBefore = signatureBefore; } - public void combine() throws IOException { + public void combine() throws IOException, DataException { try { this.preExecute(); this.readMetadata(); @@ -125,9 +126,8 @@ public class ArbitraryDataCombiner { } } - private void process() throws IOException { - String patchType = metadata.getPatchType(); - ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter, patchType); + private void process() throws IOException, DataException { + ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter); merge.compute(); this.finalPath = merge.getMergePath(); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 0b874d47..1eebb395 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -1,26 +1,64 @@ package org.qortal.arbitrary; -import com.github.difflib.DiffUtils; -import com.github.difflib.UnifiedDiffUtils; -import com.github.difflib.patch.Patch; -import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.json.JSONObject; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; +import org.qortal.arbitrary.patch.UnifiedDiffPatch; import org.qortal.crypto.Crypto; import org.qortal.settings.Settings; import java.io.*; -import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; +import java.util.*; + public class ArbitraryDataDiff { + /** Only create a patch if both the before and after file sizes are within defined limit **/ + private static long MAX_DIFF_FILE_SIZE = 100 * 1024L; // 100kiB + + + public enum DiffType { + COMPLETE_FILE, + UNIFIED_DIFF + } + + public static class ModifiedPath { + private Path path; + private DiffType diffType; + + public ModifiedPath(Path path, DiffType diffType) { + this.path = path; + this.diffType = diffType; + } + + public ModifiedPath(JSONObject jsonObject) { + String pathString = jsonObject.getString("path"); + if (pathString != null) { + this.path = Paths.get(pathString); + } + + String diffTypeString = jsonObject.getString("type"); + if (diffTypeString != null) { + this.diffType = DiffType.valueOf(diffTypeString); + } + } + + public Path getPath() { + return this.path; + } + + public DiffType getDiffType() { + return this.diffType; + } + + public String toString() { + return this.path.toString(); + } + } + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataDiff.class); private Path pathBefore; @@ -31,7 +69,7 @@ public class ArbitraryDataDiff { private String identifier; private List addedPaths; - private List modifiedPaths; + private List modifiedPaths; private List removedPaths; public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) { @@ -91,7 +129,7 @@ public class ArbitraryDataDiff { this.previousHash = digest.getHash(); } - private void findAddedOrModifiedFiles() { + private void findAddedOrModifiedFiles() throws IOException { try { final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); @@ -107,11 +145,11 @@ public class ArbitraryDataDiff { } @Override - public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException { - Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath()); - Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter); + public FileVisitResult visitFile(Path afterPathAbsolute, BasicFileAttributes attrs) throws IOException { + Path afterPathRelative = pathAfterAbsolute.relativize(afterPathAbsolute.toAbsolutePath()); + Path beforePathAbsolute = pathBeforeAbsolute.resolve(afterPathRelative); - if (filePathAfter.startsWith(".qortal")) { + if (afterPathRelative.startsWith(".qortal")) { // Ignore the .qortal metadata folder return FileVisitResult.CONTINUE; } @@ -119,31 +157,27 @@ public class ArbitraryDataDiff { boolean wasAdded = false; boolean wasModified = false; - if (!Files.exists(filePathBefore)) { - LOGGER.info("File was added: {}", filePathAfter.toString()); - diff.addedPaths.add(filePathAfter); + if (!Files.exists(beforePathAbsolute)) { + LOGGER.info("File was added: {}", afterPathRelative.toString()); + diff.addedPaths.add(afterPathRelative); wasAdded = true; } - else if (Files.size(after) != Files.size(filePathBefore)) { + else if (Files.size(afterPathAbsolute) != Files.size(beforePathAbsolute)) { // Check file size first because it's quicker - LOGGER.info("File size was modified: {}", filePathAfter.toString()); - diff.modifiedPaths.add(filePathAfter); + LOGGER.info("File size was modified: {}", afterPathRelative.toString()); wasModified = true; } - else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(after), ArbitraryDataDiff.digestFromPath(filePathBefore))) { + else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(afterPathAbsolute), ArbitraryDataDiff.digestFromPath(beforePathAbsolute))) { // Check hashes as a last resort - LOGGER.info("File contents were modified: {}", filePathAfter.toString()); - diff.modifiedPaths.add(filePathAfter); + LOGGER.info("File contents were modified: {}", afterPathRelative.toString()); wasModified = true; } if (wasAdded) { - ArbitraryDataDiff.copyFilePathToBaseDir(after, diffPathAbsolute, filePathAfter); + diff.copyFilePathToBaseDir(afterPathAbsolute, diffPathAbsolute, afterPathRelative); } if (wasModified) { - // Create patch using java-diff-utils - Path destination = Paths.get(diffPathAbsolute.toString(), filePathAfter.toString()); - ArbitraryDataDiff.createAndCopyDiffUtilsPatch(filePathBefore, after, destination); + diff.pathModified(beforePathAbsolute, afterPathAbsolute, afterPathRelative, diffPathAbsolute); } return FileVisitResult.CONTINUE; @@ -163,8 +197,8 @@ public class ArbitraryDataDiff { }); } catch (IOException e) { - // TODO: throw exception? LOGGER.info("IOException when walking through file tree: {}", e.getMessage()); + throw(e); } } @@ -240,7 +274,6 @@ public class ArbitraryDataDiff { private void writeMetadata() throws IOException { ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.diffPath); - metadata.setPatchType("unified-diff"); metadata.setAddedPaths(this.addedPaths); metadata.setModifiedPaths(this.modifiedPaths); metadata.setRemovedPaths(this.removedPaths); @@ -250,15 +283,38 @@ public class ArbitraryDataDiff { } - private static byte[] digestFromPath(Path path) { - try { - return Crypto.digest(Files.readAllBytes(path)); - } catch (IOException e) { - return null; + private void pathModified(Path beforePathAbsolute, Path afterPathAbsolute, Path afterPathRelative, + Path destinationBasePathAbsolute) throws IOException { + + Path destination = Paths.get(destinationBasePathAbsolute.toString(), afterPathRelative.toString()); + long beforeSize = Files.size(beforePathAbsolute); + long afterSize = Files.size(afterPathAbsolute); + DiffType diffType; + + if (beforeSize > MAX_DIFF_FILE_SIZE || afterSize > MAX_DIFF_FILE_SIZE) { + // Files are large, so don't attempt a diff + this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative); + diffType = DiffType.COMPLETE_FILE; } + else { + // Attempt to create patch using java-diff-utils + UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(beforePathAbsolute, afterPathAbsolute, destination); + unifiedDiffPatch.create(); + if (unifiedDiffPatch.isValid()) { + diffType = DiffType.UNIFIED_DIFF; + } + else { + // Diff failed validation, so copy the whole file instead + this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative); + diffType = DiffType.COMPLETE_FILE; + } + } + + ModifiedPath modifiedPath = new ModifiedPath(afterPathRelative, diffType); + this.modifiedPaths.add(modifiedPath); } - private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException { + private void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException { if (!Files.exists(source)) { throw new IOException(String.format("File not found: %s", source.toString())); } @@ -274,54 +330,21 @@ public class ArbitraryDataDiff { LOGGER.trace("Copying {} to {}", source, dest); Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); } - - private static void createAndCopyDiffUtilsPatch(Path before, Path after, Path destination) throws IOException { - if (!Files.exists(before)) { - throw new IOException(String.format("File not found (before): %s", before.toString())); - } - if (!Files.exists(after)) { - throw new IOException(String.format("File not found (after): %s", after.toString())); - } - - // Ensure parent folders exist in the destination - File file = new File(destination.toString()); - File parent = file.getParentFile(); - if (parent != null) { - parent.mkdirs(); - } - - // Delete an existing file if it exists - File destFile = destination.toFile(); - if (destFile.exists() && destFile.isFile()) { - Files.delete(destination); - } - - // Load the two files into memory - List original = FileUtils.readLines(before.toFile(), StandardCharsets.UTF_8); - List revised = FileUtils.readLines(after.toFile(), StandardCharsets.UTF_8); - - // Generate diff information - Patch diff = DiffUtils.diff(original, revised); - - // Generate unified diff format - String originalFileName = before.getFileName().toString(); - String revisedFileName = after.getFileName().toString(); - List unifiedDiff = UnifiedDiffUtils.generateUnifiedDiff(originalFileName, revisedFileName, original, diff, 0); - - // Write the diff to the destination directory - FileWriter fileWriter = new FileWriter(destination.toString(), true); - BufferedWriter writer = new BufferedWriter(fileWriter); - for (String line : unifiedDiff) { - writer.append(line); - writer.newLine(); - } - writer.flush(); - writer.close(); - } public Path getDiffPath() { return this.diffPath; } + + // Utils + + private static byte[] digestFromPath(Path path) { + try { + return Crypto.digest(Files.readAllBytes(path)); + } catch (IOException e) { + return null; + } + } + } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java index 98ac124f..8b5f949d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java @@ -1,24 +1,19 @@ package org.qortal.arbitrary; -import com.github.difflib.DiffUtils; -import com.github.difflib.UnifiedDiffUtils; -import com.github.difflib.patch.Patch; -import com.github.difflib.patch.PatchFailedException; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataDiff.*; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch; +import org.qortal.arbitrary.patch.UnifiedDiffPatch; +import org.qortal.repository.DataException; import org.qortal.settings.Settings; import org.qortal.utils.FilesystemUtils; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.util.List; -import java.util.Objects; import java.util.UUID; public class ArbitraryDataMerge { @@ -27,18 +22,16 @@ public class ArbitraryDataMerge { private Path pathBefore; private Path pathAfter; - private String patchType; private Path mergePath; private String identifier; private ArbitraryDataMetadataPatch metadata; - public ArbitraryDataMerge(Path pathBefore, Path pathAfter, String patchType) { + public ArbitraryDataMerge(Path pathBefore, Path pathAfter) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; - this.patchType = patchType; } - public void compute() throws IOException { + public void compute() throws IOException, DataException { try { this.preExecute(); this.copyPreviousStateToMergePath(); @@ -85,7 +78,7 @@ public class ArbitraryDataMerge { this.metadata.read(); } - private void applyDifferences() throws IOException { + private void applyDifferences() throws IOException, DataException { List addedPaths = this.metadata.getAddedPaths(); for (Path path : addedPaths) { @@ -94,10 +87,10 @@ public class ArbitraryDataMerge { ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path); } - List modifiedPaths = this.metadata.getModifiedPaths(); - for (Path path : modifiedPaths) { - LOGGER.info("File was modified: {}", path.toString()); - this.applyPatch(path); + List modifiedPaths = this.metadata.getModifiedPaths(); + for (ModifiedPath modifiedPath : modifiedPaths) { + LOGGER.info("File was modified: {}", modifiedPath.toString()); + this.applyPatch(modifiedPath); } List removedPaths = this.metadata.getRemovedPaths(); @@ -107,55 +100,19 @@ public class ArbitraryDataMerge { } } - private void applyPatch(Path path) throws IOException { - if (Objects.equals(this.patchType, "unified-diff")) { + private void applyPatch(ModifiedPath modifiedPath) throws IOException, DataException { + if (modifiedPath.getDiffType() == DiffType.UNIFIED_DIFF) { // Create destination file from patch - this.applyUnifiedDiffPatch(path); + UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(pathBefore, pathAfter, mergePath); + unifiedDiffPatch.apply(modifiedPath.getPath()); + } + else if (modifiedPath.getDiffType() == DiffType.COMPLETE_FILE) { + // Copy complete file + Path filePath = Paths.get(this.pathAfter.toString(), modifiedPath.getPath().toString()); + ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, modifiedPath.getPath()); } else { - // Copy complete file - Path filePath = Paths.get(this.pathAfter.toString(), path.toString()); - ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path); - } - } - - private void applyUnifiedDiffPatch(Path path) throws IOException { - Path originalPath = Paths.get(this.pathBefore.toString(), path.toString()); - Path patchPath = Paths.get(this.pathAfter.toString(), path.toString()); - Path mergePath = Paths.get(this.mergePath.toString(), path.toString()); - - if (!patchPath.toFile().exists()) { - throw new IllegalStateException("Patch file doesn't exist, but its path was included in modifiedPaths"); - } - - // Delete an existing file, as we are starting from a duplicate of pathBefore - File destFile = mergePath.toFile(); - if (destFile.exists() && destFile.isFile()) { - Files.delete(mergePath); - } - - List originalContents = FileUtils.readLines(originalPath.toFile(), StandardCharsets.UTF_8); - List patchContents = FileUtils.readLines(patchPath.toFile(), StandardCharsets.UTF_8); - - // At first, parse the unified diff file and get the patch - Patch patch = UnifiedDiffUtils.parseUnifiedDiff(patchContents); - - // Then apply the computed patch to the given text - try { - List patchedContents = DiffUtils.patch(originalContents, patch); - - // Write the patched file to the merge directory - FileWriter fileWriter = new FileWriter(mergePath.toString(), true); - BufferedWriter writer = new BufferedWriter(fileWriter); - for (String line : patchedContents) { - writer.append(line); - writer.newLine(); - } - writer.flush(); - writer.close(); - - } catch (PatchFailedException e) { - throw new IllegalStateException(String.format("Failed to apply patch for path %s: %s", path, e.getMessage())); + throw new DataException(String.format("Unrecognized patch diff type: %s", modifiedPath.getDiffType())); } } diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java index 0f7b6b56..e4d75f84 100644 --- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; +import org.qortal.arbitrary.ArbitraryDataDiff.*; import org.qortal.utils.Base58; import java.lang.reflect.Field; @@ -17,9 +18,8 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadataPatch.class); - private String patchType; private List addedPaths; - private List modifiedPaths; + private List modifiedPaths; private List removedPaths; private byte[] previousSignature; private byte[] previousHash; @@ -44,12 +44,6 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { } JSONObject patch = new JSONObject(this.jsonString); - if (patch.has("patchType")) { - String patchType = patch.getString("patchType"); - if (patchType != null) { - this.patchType = patchType; - } - } if (patch.has("prevSig")) { String prevSig = patch.getString("prevSig"); if (prevSig != null) { @@ -75,8 +69,9 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { JSONArray modified = (JSONArray) patch.get("modified"); if (modified != null) { for (int i=0; i addedPaths) { this.addedPaths = addedPaths; } @@ -131,11 +125,11 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { return this.addedPaths; } - public void setModifiedPaths(List modifiedPaths) { + public void setModifiedPaths(List modifiedPaths) { this.modifiedPaths = modifiedPaths; } - public List getModifiedPaths() { + public List getModifiedPaths() { return this.modifiedPaths; } diff --git a/src/main/java/org/qortal/arbitrary/patch/UnifiedDiffPatch.java b/src/main/java/org/qortal/arbitrary/patch/UnifiedDiffPatch.java new file mode 100644 index 00000000..ae24f03b --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/patch/UnifiedDiffPatch.java @@ -0,0 +1,208 @@ +package org.qortal.arbitrary.patch; + +import com.github.difflib.DiffUtils; +import com.github.difflib.UnifiedDiffUtils; +import com.github.difflib.patch.Patch; +import com.github.difflib.patch.PatchFailedException; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.crypto.Crypto; +import org.qortal.settings.Settings; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +public class UnifiedDiffPatch { + + private static final Logger LOGGER = LogManager.getLogger(UnifiedDiffPatch.class); + + private Path before; + private Path after; + private Path destination; + + private String identifier; + private Path validationPath; + + public UnifiedDiffPatch(Path before, Path after, Path destination) { + this.before = before; + this.after = after; + this.destination = destination; + } + + /** + * Create a patch based on the differences in path "after" + * compared with base path "before", outputting the patch + * to the "destination" path. + * + * @throws IOException + */ + public void create() throws IOException { + if (!Files.exists(before)) { + throw new IOException(String.format("File not found (before): %s", before.toString())); + } + if (!Files.exists(after)) { + throw new IOException(String.format("File not found (after): %s", after.toString())); + } + + // Ensure parent folders exist in the destination + File file = new File(destination.toString()); + File parent = file.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + + // Delete an existing file if it exists + File destFile = destination.toFile(); + if (destFile.exists() && destFile.isFile()) { + Files.delete(destination); + } + + // Load the two files into memory + List original = FileUtils.readLines(before.toFile(), StandardCharsets.UTF_8); + List revised = FileUtils.readLines(after.toFile(), StandardCharsets.UTF_8); + + // Generate diff information + Patch diff = DiffUtils.diff(original, revised); + + // Generate unified diff format + String originalFileName = before.getFileName().toString(); + String revisedFileName = after.getFileName().toString(); + List unifiedDiff = UnifiedDiffUtils.generateUnifiedDiff(originalFileName, revisedFileName, original, diff, 0); + + // Write the diff to the destination directory + FileWriter fileWriter = new FileWriter(destination.toString(), true); + BufferedWriter writer = new BufferedWriter(fileWriter); + for (String line : unifiedDiff) { + writer.append(line); + writer.newLine(); + } + writer.flush(); + writer.close(); + } + + /** + * Validate the patch to ensure it works correctly + * + * @return true if valid, false if invalid + * @throws IOException + */ + public boolean isValid() { + this.createRandomIdentifier(); + this.createTempValidationDirectory(); + + // Merge the patch with the original path + Path tempPath = Paths.get(this.validationPath.toString(), this.identifier); + + try { + UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(before, destination, tempPath); + unifiedDiffPatch.apply(null); + + byte[] inputDigest = Crypto.digest(after.toFile()); + byte[] outputDigest = Crypto.digest(tempPath.toFile()); + if (Arrays.equals(inputDigest, outputDigest)) { + // Patch is valid + return true; + } + else { + LOGGER.info("Checksum mismatch when verifying patch for file {}", destination.toString()); + return false; + } + + } + catch (IOException e) { + LOGGER.info("Failed to compute merge for file {}: {}", destination.toString(), e.getMessage()); + } + finally { + try { + Files.delete(tempPath); + } catch (IOException e) { + // Not important - will be cleaned up later + } + } + + return false; + } + + /** + * Apply a patch at path "after" on top of base path "before", + * outputting the combined results to the "destination" path. + * If before and after are directories, a relative path suffix + * can be used to specify the file within these folder structures. + * + * @param pathSuffix - a file path to append to the base paths, or null if the base paths are already files + * @throws IOException + */ + public void apply(Path pathSuffix) throws IOException { + Path originalPath = this.before; + Path patchPath = this.after; + Path mergePath = this.destination; + + // If a path has been supplied, we need to append it to the base paths + if (pathSuffix != null) { + originalPath = Paths.get(this.before.toString(), pathSuffix.toString()); + patchPath = Paths.get(this.after.toString(), pathSuffix.toString()); + mergePath = Paths.get(this.destination.toString(), pathSuffix.toString()); + } + + if (!patchPath.toFile().exists()) { + throw new IllegalStateException("Patch file doesn't exist, but its path was included in modifiedPaths"); + } + + // Delete an existing file, as we are starting from a duplicate of pathBefore + File destFile = mergePath.toFile(); + if (destFile.exists() && destFile.isFile()) { + Files.delete(mergePath); + } + + List originalContents = FileUtils.readLines(originalPath.toFile(), StandardCharsets.UTF_8); + List patchContents = FileUtils.readLines(patchPath.toFile(), StandardCharsets.UTF_8); + + // At first, parse the unified diff file and get the patch + Patch patch = UnifiedDiffUtils.parseUnifiedDiff(patchContents); + + // Then apply the computed patch to the given text + try { + List patchedContents = DiffUtils.patch(originalContents, patch); + + // Write the patched file to the merge directory + FileWriter fileWriter = new FileWriter(mergePath.toString(), true); + BufferedWriter writer = new BufferedWriter(fileWriter); + for (String line : patchedContents) { + writer.append(line); + writer.newLine(); + } + writer.flush(); + writer.close(); + + } catch (PatchFailedException e) { + throw new IllegalStateException(String.format("Failed to apply patch for path %s: %s", pathSuffix, e.getMessage())); + } + } + + private void createRandomIdentifier() { + this.identifier = UUID.randomUUID().toString(); + } + + private void createTempValidationDirectory() { + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + Path tempDir = Paths.get(baseDir, "diff", "validate"); + try { + Files.createDirectories(tempDir); + } catch (IOException e) { + throw new IllegalStateException("Unable to create temp directory"); + } + this.validationPath = tempDir; + } + +} diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataMergeTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataMergeTests.java index e873112e..7a607e21 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataMergeTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataMergeTests.java @@ -9,11 +9,17 @@ import org.qortal.crypto.Crypto; import org.qortal.repository.DataException; import org.qortal.test.common.Common; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.Objects; +import java.util.Random; import static org.junit.Assert.*; @@ -81,7 +87,7 @@ public class ArbitraryDataMergeTests extends Common { )); // Now merge the patch with the original path - ArbitraryDataMerge merge = new ArbitraryDataMerge(path1, patchPath, "unified-diff"); + ArbitraryDataMerge merge = new ArbitraryDataMerge(path1, patchPath); merge.compute(); Path finalPath = merge.getMergePath(); @@ -117,7 +123,7 @@ public class ArbitraryDataMergeTests extends Common { // Also check that the directory digests match ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(path2); path2Digest.compute(); - ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(path2); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); finalPathDigest.compute(); assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); } @@ -140,4 +146,276 @@ public class ArbitraryDataMergeTests extends Common { } + @Test + public void testMergeBinaryFiles() throws IOException, DataException { + // Create two files in random temp directories + Path tempDir1 = Files.createTempDirectory("testMergeBinaryFiles1"); + Path tempDir2 = Files.createTempDirectory("testMergeBinaryFiles2"); + File file1 = new File(Paths.get(tempDir1.toString(), "file.bin").toString()); + File file2 = new File(Paths.get(tempDir2.toString(), "file.bin").toString()); + file1.deleteOnExit(); + file2.deleteOnExit(); + + // Write random data to the first file + byte[] initialData = new byte[1024]; + new Random().nextBytes(initialData); + Files.write(file1.toPath(), initialData); + byte[] file1Digest = Crypto.digest(file1); + + // Write slightly modified data to the second file (bytes 100-116 are zeroed out) + byte[] updatedData = Arrays.copyOf(initialData, initialData.length); + final ByteBuffer byteBuffer = ByteBuffer.wrap(updatedData); + byteBuffer.position(100); + byteBuffer.put(new byte[16]); + updatedData = byteBuffer.array(); + Files.write(file2.toPath(), updatedData); + byte[] file2Digest = Crypto.digest(file2); + + // Make sure the two arrays are different + assertFalse(Arrays.equals(initialData, updatedData)); + + // And double check that they are both 1024 bytes long + assertEquals(1024, initialData.length); + assertEquals(1024, updatedData.length); + + // Ensure both files exist + assertTrue(Files.exists(file1.toPath())); + assertTrue(Files.exists(file2.toPath())); + + // Create a patch from the two paths + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, new byte[16]); + patch.create(); + Path patchPath = patch.getFinalPath(); + assertTrue(Files.exists(patchPath)); + + // Check that the patch file exists + Path patchFilePath = Paths.get(patchPath.toString(), "file.bin"); + assertTrue(Files.exists(patchFilePath)); + byte[] patchDigest = Crypto.digest(patchFilePath.toFile()); + + // Ensure that the patch file matches file2 exactly + // This is because binary files cannot currently be patched, and so the complete file + // is included instead + assertArrayEquals(patchDigest, file2Digest); + + // Make sure that the patch file is different from file1 + assertFalse(Arrays.equals(patchDigest, file1Digest)); + + // Now merge the patch with the original path + ArbitraryDataMerge merge = new ArbitraryDataMerge(tempDir1, patchPath); + merge.compute(); + Path finalPath = merge.getMergePath(); + + // Check that the directory digests match + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + } + + @Test + public void testMergeRandomStrings() throws IOException, DataException { + // Create two files in random temp directories + Path tempDir1 = Files.createTempDirectory("testMergeRandomStrings"); + Path tempDir2 = Files.createTempDirectory("testMergeRandomStrings"); + File file1 = new File(Paths.get(tempDir1.toString(), "file.txt").toString()); + File file2 = new File(Paths.get(tempDir2.toString(), "file.txt").toString()); + file1.deleteOnExit(); + file2.deleteOnExit(); + + // Write a random string to the first file + BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file1)); + String initialString = this.generateRandomString(1024); + file1Writer.write(initialString); + file1Writer.newLine(); + file1Writer.close(); + byte[] file1Digest = Crypto.digest(file1); + + // Write a slightly modified string to the second file + BufferedWriter file2Writer = new BufferedWriter(new FileWriter(file2)); + String updatedString = initialString.concat("-edit"); + file2Writer.write(updatedString); + file2Writer.newLine(); + file2Writer.close(); + byte[] file2Digest = Crypto.digest(file2); + + // Make sure the two strings are different + assertFalse(Objects.equals(initialString, updatedString)); + + // Ensure both files exist + assertTrue(Files.exists(file1.toPath())); + assertTrue(Files.exists(file2.toPath())); + + // Create a patch from the two paths + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, new byte[16]); + patch.create(); + Path patchPath = patch.getFinalPath(); + assertTrue(Files.exists(patchPath)); + + // Check that the patch file exists + Path patchFilePath = Paths.get(patchPath.toString(), "file.txt"); + assertTrue(Files.exists(patchFilePath)); + byte[] patchDigest = Crypto.digest(patchFilePath.toFile()); + + // Make sure that the patch file is different from file1 and file2 + assertFalse(Arrays.equals(patchDigest, file1Digest)); + assertFalse(Arrays.equals(patchDigest, file2Digest)); + + // Now merge the patch with the original path + ArbitraryDataMerge merge = new ArbitraryDataMerge(tempDir1, patchPath); + merge.compute(); + Path finalPath = merge.getMergePath(); + + // Check that the directory digests match + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + + } + + @Test + public void testMergeRandomStringsWithoutTrailingNewlines() throws IOException, DataException { + // Create two files in random temp directories + Path tempDir1 = Files.createTempDirectory("testMergeRandomStrings"); + Path tempDir2 = Files.createTempDirectory("testMergeRandomStrings"); + File file1 = new File(Paths.get(tempDir1.toString(), "file.txt").toString()); + File file2 = new File(Paths.get(tempDir2.toString(), "file.txt").toString()); + file1.deleteOnExit(); + file2.deleteOnExit(); + + // Write a random string to the first file + BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file1)); + String initialString = this.generateRandomString(1024); + file1Writer.write(initialString); + // No newline + file1Writer.close(); + byte[] file1Digest = Crypto.digest(file1); + + // Write a slightly modified string to the second file + BufferedWriter file2Writer = new BufferedWriter(new FileWriter(file2)); + String updatedString = initialString.concat("-edit"); + file2Writer.write(updatedString); + // No newline + file2Writer.close(); + byte[] file2Digest = Crypto.digest(file2); + + // Make sure the two strings are different + assertFalse(Objects.equals(initialString, updatedString)); + + // Ensure both files exist + assertTrue(Files.exists(file1.toPath())); + assertTrue(Files.exists(file2.toPath())); + + // Create a patch from the two paths + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, new byte[16]); + patch.create(); + Path patchPath = patch.getFinalPath(); + assertTrue(Files.exists(patchPath)); + + // Check that the patch file exists + Path patchFilePath = Paths.get(patchPath.toString(), "file.txt"); + assertTrue(Files.exists(patchFilePath)); + byte[] patchDigest = Crypto.digest(patchFilePath.toFile()); + + // The patch file should be identical to file2, because we don't currently + // support arbitrary diff patches on files without trailing newlines + assertArrayEquals(patchDigest, file2Digest); + + // Make sure that the patch file is different from file1 + assertFalse(Arrays.equals(patchDigest, file1Digest)); + + // Now merge the patch with the original path + ArbitraryDataMerge merge = new ArbitraryDataMerge(tempDir1, patchPath); + merge.compute(); + Path finalPath = merge.getMergePath(); + + // Check that the directory digests match + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + + } + + @Test + public void testMergeRandomLargeStrings() throws IOException, DataException { + // Create two files in random temp directories + Path tempDir1 = Files.createTempDirectory("testMergeRandomStrings"); + Path tempDir2 = Files.createTempDirectory("testMergeRandomStrings"); + File file1 = new File(Paths.get(tempDir1.toString(), "file.txt").toString()); + File file2 = new File(Paths.get(tempDir2.toString(), "file.txt").toString()); + file1.deleteOnExit(); + file2.deleteOnExit(); + + // Write a random string to the first file + BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file1)); + String initialString = this.generateRandomString(110 * 1024); + file1Writer.write(initialString); + file1Writer.newLine(); + file1Writer.close(); + byte[] file1Digest = Crypto.digest(file1); + + // Write a slightly modified string to the second file + BufferedWriter file2Writer = new BufferedWriter(new FileWriter(file2)); + String updatedString = initialString.concat("-edit"); + file2Writer.write(updatedString); + file2Writer.newLine(); + file2Writer.close(); + byte[] file2Digest = Crypto.digest(file2); + + // Make sure the two strings are different + assertFalse(Objects.equals(initialString, updatedString)); + + // Ensure both files exist + assertTrue(Files.exists(file1.toPath())); + assertTrue(Files.exists(file2.toPath())); + + // Create a patch from the two paths + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, new byte[16]); + patch.create(); + Path patchPath = patch.getFinalPath(); + assertTrue(Files.exists(patchPath)); + + // Check that the patch file exists + Path patchFilePath = Paths.get(patchPath.toString(), "file.txt"); + assertTrue(Files.exists(patchFilePath)); + byte[] patchDigest = Crypto.digest(patchFilePath.toFile()); + + // The patch file should be identical to file2 because the source files + // were over the maximum size limit for creating patches + assertArrayEquals(patchDigest, file2Digest); + + // Make sure that the patch file is different from file1 + assertFalse(Arrays.equals(patchDigest, file1Digest)); + + // Now merge the patch with the original path + ArbitraryDataMerge merge = new ArbitraryDataMerge(tempDir1, patchPath); + merge.compute(); + Path finalPath = merge.getMergePath(); + + // Check that the directory digests match + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2); + path2Digest.compute(); + ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath); + finalPathDigest.compute(); + assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58()); + + } + + private String generateRandomString(int length) { + int leftLimit = 48; // numeral '0' + int rightLimit = 122; // letter 'z' + Random random = new Random(); + + return random.ints(leftLimit, rightLimit + 1) + .filter(i -> (i <= 57 || i >= 65) && (i <= 90 || i >= 97)) + .limit(length) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + } + } diff --git a/src/test/resources/arbitrary/demo1/.qortal/cache b/src/test/resources/arbitrary/demo1/.qortal/cache deleted file mode 100644 index 90f0f8dc..00000000 --- a/src/test/resources/arbitrary/demo1/.qortal/cache +++ /dev/null @@ -1 +0,0 @@ -db2d9ab2-a97e-43bf-a259-ebbc1a1b0c59 \ No newline at end of file