From c59869982b78202091540c43502b7722faa1d6a4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 16 Apr 2022 11:25:44 +0100 Subject: [PATCH] Fix for system-wide QDN issues occuring when the metadata file has an empty chunks array. It is quite likely that existing resources with both metadata and an empty chunks array will need to be republished, because this bug may have led to incorrect file deletions. --- .../qortal/arbitrary/ArbitraryDataFile.java | 8 ++ .../ArbitraryTransactionMetadataTests.java | 112 ++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 3e0f0ab6..b974298b 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -478,6 +478,14 @@ public class ArbitraryDataFile { // Read the metadata List chunks = metadata.getChunks(); + + // If the chunks array is empty, then this resource has no chunks, + // so we must return false to avoid confusing the caller. + if (chunks.isEmpty()) { + return false; + } + + // Otherwise, we need to check each chunk individually for (byte[] chunkHash : chunks) { ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature); if (!chunk.exists()) { diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java index 77cb22b0..357046fe 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java @@ -78,6 +78,118 @@ public class ArbitraryTransactionMetadataTests extends Common { } } + @Test + public void testSingleChunkWithMetadata() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + int chunkSize = 1000; + int dataLength = 10; // Actual data length will be longer due to encryption + + String title = "Test title"; + String description = "Test description"; + List tags = Arrays.asList("Test", "tag", "another tag"); + Category category = Category.QORTAL; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength); + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, + identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize, + title, description, tags, category); + + // Check the chunk count is correct + assertEquals(0, arbitraryDataFile.chunkCount()); + + // Check the metadata is correct + assertEquals(title, arbitraryDataFile.getMetadata().getTitle()); + assertEquals(description, arbitraryDataFile.getMetadata().getDescription()); + assertEquals(tags, arbitraryDataFile.getMetadata().getTags()); + assertEquals(category, arbitraryDataFile.getMetadata().getCategory()); + + // Now build the latest data state for this name + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader.loadSynchronously(true); + Path initialLayerPath = arbitraryDataReader.getFilePath(); + ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath); + initialLayerDigest.compute(); + + // Its directory hash should match the original directory hash + ArbitraryDataDigest path1Digest = new ArbitraryDataDigest(path1); + path1Digest.compute(); + assertEquals(path1Digest.getHash58(), initialLayerDigest.getHash58()); + } + } + + @Test + public void testSingleNonLocalChunkWithMetadata() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + int chunkSize = 1000; + int dataLength = 10; // Actual data length will be longer due to encryption + + String title = "Test title"; + String description = "Test description"; + List tags = Arrays.asList("Test", "tag", "another tag"); + Category category = Category.QORTAL; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength); + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, + identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize, + title, description, tags, category); + + // Check the chunk count is correct + assertEquals(0, arbitraryDataFile.chunkCount()); + + // Check the metadata is correct + assertEquals(title, arbitraryDataFile.getMetadata().getTitle()); + assertEquals(description, arbitraryDataFile.getMetadata().getDescription()); + assertEquals(tags, arbitraryDataFile.getMetadata().getTags()); + assertEquals(category, arbitraryDataFile.getMetadata().getCategory()); + + // Delete the file, to simulate that it hasn't been fetched from the network yet + arbitraryDataFile.delete(); + + boolean missingDataExceptionCaught = false; + boolean ioExceptionCaught = false; + + // Now build the latest data state for this name + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + try { + arbitraryDataReader.loadSynchronously(true); + } + catch (MissingDataException e) { + missingDataExceptionCaught = true; + } + catch (IOException e) { + ioExceptionCaught = true; + } + + // We expect a MissingDataException, not an IOException. + // This is because MissingDataException means that the core has correctly identified a file is missing, + // whereas an IOException would be due to trying to build without first having everything that is needed. + assertTrue(missingDataExceptionCaught); + assertFalse(ioExceptionCaught); + } + } + @Test public void testDescriptiveMetadata() throws DataException, IOException, MissingDataException { try (final Repository repository = RepositoryManager.getRepository()) {