From 4b1a5a5e14509706d2d7e48ec10af2e084b7c17e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 11 Nov 2021 09:12:54 +0000 Subject: [PATCH] Connected the rest of the system up to the recently added "identifier" feature. --- .../api/resource/ArbitraryResource.java | 93 +++++++++++++------ .../qortal/api/resource/WebsiteResource.java | 4 +- .../ArbitraryDataBuildQueueItem.java | 8 +- .../arbitrary/ArbitraryDataBuilder.java | 27 ++++-- .../qortal/arbitrary/ArbitraryDataCache.java | 6 +- .../qortal/arbitrary/ArbitraryDataReader.java | 23 +++-- .../ArbitraryDataTransactionBuilder.java | 4 +- .../qortal/arbitrary/ArbitraryDataWriter.java | 6 +- .../data/arbitrary/ArbitraryResourceInfo.java | 1 + .../repository/ArbitraryRepository.java | 4 +- .../hsqldb/HSQLDBArbitraryRepository.java | 22 +++-- .../utils/ArbitraryTransactionUtils.java | 3 +- .../test/arbitrary/ArbitraryDataTests.java | 76 ++++++++++++++- 13 files changed, 202 insertions(+), 75 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 104a71d6..2ef4d553 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -290,39 +290,34 @@ public class ArbitraryResource { @QueryParam("rebuild") boolean rebuild) { Security.checkApiCallAllowed(request); - if (filepath == null) { - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing filepath"); - } + return this.download(serviceString, name, null, filepath, rebuild); + } - Service service = Service.valueOf(serviceString); - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service); - try { - - // Loop until we have data - while (!Controller.isStopping()) { - try { - arbitraryDataReader.loadSynchronously(rebuild); - break; - } catch (MissingDataException e) { - continue; - } + @GET + @Path("/{service}/{name}/{identifier}") + @Operation( + summary = "Fetch raw data from file with supplied service, name, identifier, and relative path", + description = "An optional rebuild boolean can be supplied. If true, any existing cached data will be invalidated.", + responses = { + @ApiResponse( + description = "Path to file structure containing requested data", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) } + ) + public HttpServletResponse get(@PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @QueryParam("filepath") String filepath, + @QueryParam("rebuild") boolean rebuild) { + Security.checkApiCallAllowed(request); - // TODO: limit file size that can be read into memory - java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath); - if (!Files.exists(path)) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - } - byte[] data = Files.readAllBytes(path); - response.setContentType(context.getMimeType(path.toString())); - response.setContentLength(data.length); - response.getOutputStream().write(data); - - return response; - } catch (Exception e) { - LOGGER.info(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage()); - } + return this.download(serviceString, name, identifier, filepath, rebuild); } @POST @@ -432,6 +427,7 @@ public class ArbitraryResource { return this.upload(Method.PATCH, Service.valueOf(serviceString), name, identifier, path); } + private String upload(Method method, Service service, String name, String identifier, String path) { // It's too dangerous to allow user-supplied file paths in weaker security contexts if (Settings.getInstance().isApiRestricted()) { @@ -470,6 +466,43 @@ public class ArbitraryResource { } } + private HttpServletResponse download(String serviceString, String name, String identifier, String filepath, boolean rebuild) { + + if (filepath == null) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing filepath"); + } + + Service service = Service.valueOf(serviceString); + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); + try { + + // Loop until we have data + while (!Controller.isStopping()) { + try { + arbitraryDataReader.loadSynchronously(rebuild); + break; + } catch (MissingDataException e) { + continue; + } + } + + // TODO: limit file size that can be read into memory + java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath); + if (!Files.exists(path)) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + } + byte[] data = Files.readAllBytes(path); + response.setContentType(context.getMimeType(path.toString())); + response.setContentLength(data.length); + response.getOutputStream().write(data); + + return response; + } catch (Exception e) { + LOGGER.info(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage()); + } + } + @DELETE @Path("/file") diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index 09f6241d..af294f7c 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -93,7 +93,7 @@ public class WebsiteResource { Method method = Method.PUT; Compression compression = Compression.ZIP; - ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, method, compression); + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, null, method, compression); try { arbitraryDataWriter.save(); } catch (IOException | DataException | InterruptedException | MissingDataException e) { @@ -178,7 +178,7 @@ public class WebsiteResource { } Service service = Service.WEBSITE; - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service); + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, null); arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only try { if (!arbitraryDataReader.isCachedDataAvailable()) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index fb3e534c..35f0318f 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -13,6 +13,7 @@ public class ArbitraryDataBuildQueueItem { private String resourceId; private ResourceIdType resourceIdType; private Service service; + private String identifier; private Long creationTimestamp = null; private Long buildStartTimestamp = null; private Long buildEndTimestamp = null; @@ -24,10 +25,11 @@ public class ArbitraryDataBuildQueueItem { /* The amount of time to remember that a build has failed, to avoid retries */ public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes - public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) { + public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) { this.resourceId = resourceId.toLowerCase(); this.resourceIdType = resourceIdType; this.service = service; + this.identifier = identifier; this.creationTimestamp = NTP.getTime(); } @@ -39,7 +41,7 @@ public class ArbitraryDataBuildQueueItem { this.buildStartTimestamp = now; ArbitraryDataReader arbitraryDataReader = - new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service); + new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier); try { arbitraryDataReader.loadSynchronously(true); @@ -86,7 +88,7 @@ public class ArbitraryDataBuildQueueItem { @Override public String toString() { - return String.format("%s %s", this.service, this.resourceId); + return String.format("%s %s %s", this.service, this.resourceId, this.identifier); } } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index b9dd2652..d2da2cff 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -28,6 +28,7 @@ public class ArbitraryDataBuilder { private String name; private Service service; + private String identifier; private List transactions; private ArbitraryTransactionData latestPutTransaction; @@ -35,9 +36,10 @@ public class ArbitraryDataBuilder { private byte[] latestSignature; private Path finalPath; - public ArbitraryDataBuilder(String name, Service service) { + public ArbitraryDataBuilder(String name, Service service, String identifier) { this.name = name; this.service = service; + this.identifier = identifier; this.paths = new ArrayList<>(); } @@ -56,16 +58,17 @@ public class ArbitraryDataBuilder { // Get the most recent PUT ArbitraryTransactionData latestPut = repository.getArbitraryRepository() - .getLatestTransaction(this.name, this.service, Method.PUT); + .getLatestTransaction(this.name, this.service, Method.PUT, this.identifier); if (latestPut == null) { - throw new IllegalStateException(String.format( - "Couldn't find PUT transaction for name %s and service %s", this.name, this.service)); + String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s", + this.name, this.service, this.identifierString()); + throw new IllegalStateException(message); } this.latestPutTransaction = latestPut; // Load all transactions since the latest PUT List transactionDataList = repository.getArbitraryRepository() - .getArbitraryTransactions(this.name, this.service, latestPut.getTimestamp()); + .getArbitraryTransactions(this.name, this.service, this.identifier, latestPut.getTimestamp()); this.transactions = transactionDataList; } } @@ -81,8 +84,8 @@ public class ArbitraryDataBuilder { throw new IllegalStateException("Expected PUT but received PATCH"); } if (transactionDataList.size() == 0) { - throw new IllegalStateException(String.format("No transactions found for name %s, service %s, since %d", - name, service, latestPut.getTimestamp())); + throw new IllegalStateException(String.format("No transactions found for name %s, service %s, " + + "identifier: %s, since %d", name, service, this.identifierString(), latestPut.getTimestamp())); } // Verify that the signature of the first transaction matches the latest PUT @@ -115,7 +118,8 @@ public class ArbitraryDataBuilder { // Build the data file, overwriting anything that was previously there String sig58 = Base58.encode(transactionData.getSignature()); - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service); + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, + this.service, this.identifier); arbitraryDataReader.setTransactionData(transactionData); boolean hasMissingData = false; try { @@ -179,7 +183,8 @@ public class ArbitraryDataBuilder { // Loop from the second path onwards for (int i=1; i getArbitraryTransactions(String name, Service service, long since) throws DataException; + public List getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException; - public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException; + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException; public List getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index c90af07c..c348c352 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -153,17 +153,18 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } @Override - public List getArbitraryTransactions(String name, Service service, long since) throws DataException { + public List getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException { String sql = "SELECT type, reference, signature, creator, created_when, fee, " + "tx_group_id, block_height, approval_status, approval_height, " + "version, nonce, service, size, is_data_raw, data, chunk_hashes, " + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + "JOIN Transactions USING (signature) " + - "WHERE lower(name) = ? AND service = ? AND created_when >= ? " + - "ORDER BY created_when ASC"; + "WHERE lower(name) = ? AND service = ?" + + "AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))" + + "AND created_when >= ? ORDER BY created_when ASC"; List arbitraryTransactionData = new ArrayList<>(); - try (ResultSet resultSet = this.repository.checkedExecute(sql, name.toLowerCase(), service.value, since)) { + try (ResultSet resultSet = this.repository.checkedExecute(sql, name.toLowerCase(), service.value, identifier, identifier, since)) { if (resultSet == null) return null; @@ -221,7 +222,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { } @Override - public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException { + public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException { StringBuilder sql = new StringBuilder(1024); sql.append("SELECT type, reference, signature, creator, created_when, fee, " + @@ -229,7 +230,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { "version, nonce, service, size, is_data_raw, data, chunk_hashes, " + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + "JOIN Transactions USING (signature) " + - "WHERE lower(name) = ? AND service = ?"); + "WHERE lower(name) = ? AND service = ? " + + "AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))"); if (method != null) { sql.append(" AND update_method = "); @@ -238,7 +240,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { sql.append("ORDER BY created_when DESC LIMIT 1"); - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value)) { + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) { if (resultSet == null) return null; @@ -295,14 +297,14 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { public List getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException { StringBuilder sql = new StringBuilder(512); - sql.append("SELECT name, service FROM ArbitraryTransactions"); + sql.append("SELECT name, service, identifier FROM ArbitraryTransactions"); if (service != null) { sql.append(" WHERE service = "); sql.append(service.value); } - sql.append(" GROUP BY name, service ORDER BY name"); + sql.append(" GROUP BY name, service, identifier ORDER BY name"); if (reverse != null && reverse) { sql.append(" DESC"); @@ -319,6 +321,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { do { String name = resultSet.getString(1); Service serviceResult = Service.valueOf(resultSet.getInt(2)); + String identifier = resultSet.getString(3); // We should filter out resources without names if (name == null) { @@ -328,6 +331,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); arbitraryResourceInfo.name = name; arbitraryResourceInfo.service = serviceResult; + arbitraryResourceInfo.identifier = identifier; arbitraryResources.add(arbitraryResourceInfo); } while (resultSet.next()); diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index 79fbf60c..1a0a671d 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -39,6 +39,7 @@ public class ArbitraryTransactionUtils { String name = arbitraryTransactionData.getName(); ArbitraryTransactionData.Service service = arbitraryTransactionData.getService(); + String identifier = arbitraryTransactionData.getIdentifier(); if (name == null || service == null) { return null; @@ -48,7 +49,7 @@ public class ArbitraryTransactionUtils { ArbitraryTransactionData latestPut; try { latestPut = repository.getArbitraryRepository() - .getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT); + .getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT, identifier); } catch (DataException e) { return null; } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java index 4d4d04ff..6659000e 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java @@ -63,7 +63,7 @@ public class ArbitraryDataTests extends Common { this.createAndMintTxn(repository, publicKey58, path3, name, identifier, Method.PATCH, service, alice); // Now build the latest data state for this name - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service); + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); arbitraryDataReader.loadSynchronously(true); Path finalPath = arbitraryDataReader.getFilePath(); @@ -102,7 +102,7 @@ public class ArbitraryDataTests extends Common { } catch (DataException expectedException) { assertEquals(String.format("Couldn't find PUT transaction for " + - "name %s and service %s", name, service), expectedException.getMessage()); + "name %s, service %s and identifier ", name, service), expectedException.getMessage()); } } @@ -181,7 +181,7 @@ public class ArbitraryDataTests extends Common { this.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); // Now build the latest data state for this name - ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service); + ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); arbitraryDataReader1.loadSynchronously(true); Path initialLayerPath = arbitraryDataReader1.getFilePath(); ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath); @@ -192,7 +192,75 @@ public class ArbitraryDataTests extends Common { this.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice); // Rebuild the latest state - ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service); + ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader2.loadSynchronously(false); + Path secondLayerPath = arbitraryDataReader2.getFilePath(); + ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath); + secondLayerDigest.compute(); + + // Ensure that the second state is different to the first state + assertFalse(Arrays.equals(initialLayerDigest.getHash(), secondLayerDigest.getHash())); + + // Its directory hash should match the hash of demo2 + ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(path2); + path2Digest.compute(); + assertEquals(path2Digest.getHash58(), secondLayerDigest.getHash58()); + } + } + + @Test + public void testIdentifier() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = "test_identifier"; + Service service = Service.WEBSITE; // Can be anything for this test + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1"); + this.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Build the latest data state for this name, with a null identifier, ensuring that it fails + ArbitraryDataReader arbitraryDataReader1a = new ArbitraryDataReader(name, ResourceIdType.NAME, service, null); + try { + arbitraryDataReader1a.loadSynchronously(true); + fail("Loading data with null identifier should fail due to nonexistent PUT transaction"); + + } catch (IllegalStateException expectedException) { + assertEquals(String.format("Couldn't find PUT transaction for name %s, service %s " + + "and identifier ", name.toLowerCase(), service), expectedException.getMessage()); + } + + // Build the latest data state for this name, with a different identifier, ensuring that it fails + String differentIdentifier = "different_identifier"; + ArbitraryDataReader arbitraryDataReader1b = new ArbitraryDataReader(name, ResourceIdType.NAME, service, differentIdentifier); + try { + arbitraryDataReader1b.loadSynchronously(true); + fail("Loading data with incorrect identifier should fail due to nonexistent PUT transaction"); + + } catch (IllegalStateException expectedException) { + assertEquals(String.format("Couldn't find PUT transaction for name %s, service %s " + + "and identifier %s", name.toLowerCase(), service, differentIdentifier), expectedException.getMessage()); + } + + // Now build the latest data state for this name, with the correct identifier + ArbitraryDataReader arbitraryDataReader1c = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); + arbitraryDataReader1c.loadSynchronously(true); + Path initialLayerPath = arbitraryDataReader1c.getFilePath(); + ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath); + initialLayerDigest.compute(); + + // Create PATCH transaction + Path path2 = Paths.get("src/test/resources/arbitrary/demo2"); + this.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice); + + // Rebuild the latest state + ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); arbitraryDataReader2.loadSynchronously(false); Path secondLayerPath = arbitraryDataReader2.getFilePath(); ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath);