From 055775b13dbae9485a0610b5f0836271659217b9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Oct 2022 18:54:38 +0000 Subject: [PATCH] Include a list of files in the QDN metadata. --- .../qortal/arbitrary/ArbitraryDataWriter.java | 33 +++++++-- .../ArbitraryDataTransactionMetadata.java | 31 ++++++++ .../ArbitraryTransactionMetadataTests.java | 72 +++++++++++++++++++ 3 files changed, 129 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 33802d4f..8b1d00c3 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -23,16 +23,13 @@ import javax.crypto.NoSuchPaddingException; import javax.crypto.SecretKey; import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class ArbitraryDataWriter { @@ -50,6 +47,7 @@ public class ArbitraryDataWriter { private final String description; private final List tags; private final Category category; + private List files; private int chunkSize = ArbitraryDataFile.CHUNK_SIZE; @@ -80,12 +78,14 @@ public class ArbitraryDataWriter { this.description = ArbitraryDataTransactionMetadata.limitDescription(description); this.tags = ArbitraryDataTransactionMetadata.limitTags(tags); this.category = category; + this.files = new ArrayList<>(); // Populated in buildFileList() } public void save() throws IOException, DataException, InterruptedException, MissingDataException { try { this.preExecute(); this.validateService(); + this.buildFileList(); this.process(); this.compress(); this.encrypt(); @@ -143,6 +143,24 @@ public class ArbitraryDataWriter { } } + private void buildFileList() throws IOException { + // Single file resources consist of a single element in the file list + boolean isSingleFile = this.filePath.toFile().isFile(); + if (isSingleFile) { + this.files.add(this.filePath.getFileName().toString()); + return; + } + + // Multi file resources require a walk through the directory tree + try (Stream stream = Files.walk(this.filePath)) { + this.files = stream + .filter(Files::isRegularFile) + .map(p -> this.filePath.relativize(p).toString()) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + } + private void process() throws DataException, IOException, MissingDataException { switch (this.method) { @@ -285,6 +303,7 @@ public class ArbitraryDataWriter { metadata.setTags(this.tags); metadata.setCategory(this.category); metadata.setChunks(this.arbitraryDataFile.chunkHashList()); + metadata.setFiles(this.files); metadata.write(); // Create an ArbitraryDataFile from the JSON file (we don't have a signature yet) diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java index 0f8b676b..33da343c 100644 --- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java @@ -19,6 +19,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata { private String description; private List tags; private Category category; + private List files; private static int MAX_TITLE_LENGTH = 80; private static int MAX_DESCRIPTION_LENGTH = 500; @@ -77,6 +78,20 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata { } this.chunks = chunksList; } + + List filesList = new ArrayList<>(); + if (metadata.has("files")) { + JSONArray files = metadata.getJSONArray("files"); + if (files != null) { + for (int i=0; i files) { + this.files = files; + } + + public List getFiles() { + return this.files; + } + public boolean containsChunk(byte[] chunk) { for (byte[] c : this.chunks) { if (Arrays.equals(c, chunk)) { diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java index 357046fe..d8071777 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java @@ -25,9 +25,13 @@ import org.qortal.transaction.RegisterNameTransaction; import org.qortal.utils.Base58; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.List; +import java.util.Random; import static org.junit.Assert.*; @@ -279,6 +283,74 @@ public class ArbitraryTransactionMetadataTests extends Common { } } + @Test + public void testSingleFileList() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + int chunkSize = 100; + int dataLength = 900; // Actual data length will be longer due to encryption + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Add a few files at multiple levels + byte[] data = new byte[1024]; + new Random().nextBytes(data); + Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength); + Path file1 = Paths.get(path1.toString(), "file.txt"); + + // Create PUT transaction + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, file1, name, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); + + // Check the file list metadata is correct + assertEquals(1, arbitraryDataFile.getMetadata().getFiles().size()); + assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("file.txt")); + } + } + + @Test + public void testMultipleFileList() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + int chunkSize = 100; + int dataLength = 900; // Actual data length will be longer due to encryption + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Add a few files at multiple levels + byte[] data = new byte[1024]; + new Random().nextBytes(data); + Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength); + Files.write(Paths.get(path1.toString(), "image1.jpg"), data, StandardOpenOption.CREATE); + + Path subdirectory = Paths.get(path1.toString(), "subdirectory"); + Files.createDirectories(subdirectory); + Files.write(Paths.get(subdirectory.toString(), "config.json"), data, StandardOpenOption.CREATE); + + // Create PUT transaction + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); + + // Check the file list metadata is correct + assertEquals(3, arbitraryDataFile.getMetadata().getFiles().size()); + assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("file.txt")); + assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("image1.jpg")); + assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("subdirectory/config.json")); + } + } + @Test public void testExistingCategories() { // Matching categories should be correctly located