From c069c39ce12e6f1490ffd96edbd575a9f934485a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 13 Nov 2021 09:56:13 +0000 Subject: [PATCH] Implemented automatic PUT/PATCH detection When using POST /arbitrary/{service}/{name}... it will now automatically decide which method to use (PUT/PATCH) based on a few factors: - If there are already 10 or more layers, use PUT to reset back to a single layer - If the next layer's patch is more than 20% of the total resource file size, use PUT - If the next layer modifies more than 50% of the total file count, use PUT - Otherwise, use PATCH The PUT method causes a new base layer to be created and all previous update history for that resource becomes obsolete. The PATCH method adds a small delta layer on top of the existing layer(s). The idea is to wipe the slate clean with a new base layer once the patches start to get demanding for the network to apply. Nodes which view the content will ultimately have build timeouts to prevent someone from deploying a resource with hundreds of complex layers for example, so this approach is there to maximize the chances of the resource being buildable. The constants above (10 layers, 20% total size, 50% file count) will most likely need tweaking once we have some real world data. --- .../api/resource/ArbitraryResource.java | 11 +-- .../arbitrary/ArbitraryDataBuilder.java | 7 ++ .../arbitrary/ArbitraryDataCreatePatch.java | 14 +++ .../qortal/arbitrary/ArbitraryDataDiff.java | 13 +++ .../qortal/arbitrary/ArbitraryDataReader.java | 16 ++++ .../ArbitraryDataTransactionBuilder.java | 91 ++++++++++++++++++- .../org/qortal/utils/FilesystemUtils.java | 10 ++ .../test/arbitrary/ArbitraryDataTests.java | 3 +- tools/qdata | 4 +- 9 files changed, 158 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 2bfd8452..0b3afc68 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -352,8 +352,7 @@ public class ArbitraryResource { String path) { Security.checkApiCallAllowed(request); - // TODO: automatic PUT/PATCH - return this.upload(Method.PUT, Service.valueOf(serviceString), name, null, path); + return this.upload(null, Service.valueOf(serviceString), name, null, path); } @PUT @@ -458,8 +457,7 @@ public class ArbitraryResource { String path) { Security.checkApiCallAllowed(request); - // TODO: automatic PUT/PATCH - return this.upload(Method.PUT, Service.valueOf(serviceString), name, identifier, path); + return this.upload(null, Service.valueOf(serviceString), name, identifier, path); } @PUT @@ -560,10 +558,11 @@ public class ArbitraryResource { publicKey58, Paths.get(path), name, method, service, identifier ); - ArbitraryTransactionData transactionData = transactionBuilder.build(); + transactionBuilder.build(); + ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData(); return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData)); - } catch (DataException | TransformationException e) { + } catch (DataException | TransformationException | IllegalStateException e) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index d2da2cff..0b76ce1d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -35,6 +35,7 @@ public class ArbitraryDataBuilder { private List paths; private byte[] latestSignature; private Path finalPath; + private int layerCount; public ArbitraryDataBuilder(String name, Service service, String identifier) { this.name = name; @@ -69,7 +70,9 @@ public class ArbitraryDataBuilder { // Load all transactions since the latest PUT List transactionDataList = repository.getArbitraryRepository() .getArbitraryTransactions(this.name, this.service, this.identifier, latestPut.getTimestamp()); + this.transactions = transactionDataList; + this.layerCount = transactionDataList.size(); } } @@ -228,4 +231,8 @@ public class ArbitraryDataBuilder { return this.latestSignature; } + public int getLayerCount() { + return this.layerCount; + } + } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java index 2e2233fe..69bda4cf 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java @@ -19,7 +19,10 @@ public class ArbitraryDataCreatePatch { private Path pathBefore; private Path pathAfter; private byte[] previousSignature; + private Path finalPath; + private int totalFileCount; + private int fileDifferencesCount; private Path workingPath; private String identifier; @@ -116,10 +119,21 @@ public class ArbitraryDataCreatePatch { ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter, this.previousSignature); this.finalPath = diff.getDiffPath(); diff.compute(); + + this.totalFileCount = diff.getTotalFileCount(); + this.fileDifferencesCount = diff.getFileDifferencesCount(); } public Path getFinalPath() { return this.finalPath; } + public int getTotalFileCount() { + return this.totalFileCount; + } + + public int getFileDifferencesCount() { + return this.fileDifferencesCount; + } + } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index e44fbf8e..1699878e 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -73,6 +73,8 @@ public class ArbitraryDataDiff { private List modifiedPaths; private List removedPaths; + private int totalFileCount; + public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; @@ -182,6 +184,9 @@ public class ArbitraryDataDiff { diff.pathModified(beforePathAbsolute, afterPathAbsolute, afterPathRelative, diffPathAbsolute); } + // Keep a tally of the total number of files to help with decision making + diff.totalFileCount++; + return FileVisitResult.CONTINUE; } @@ -345,6 +350,14 @@ public class ArbitraryDataDiff { return this.diffPath; } + public int getTotalFileCount() { + return this.totalFileCount; + } + + public int getFileDifferencesCount() { + return this.addedPaths.size() + this.modifiedPaths.size() + this.removedPaths.size(); + } + // Utils diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 38b5ce26..142a8016 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -52,6 +52,10 @@ public class ArbitraryDataReader { private Path uncompressedPath; private Path unencryptedPath; + // Stats (available for synchronous builds only) + private int layerCount; + private byte[] latestSignature; + public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) { // Ensure names are always lowercase if (resourceIdType == ResourceIdType.NAME) { @@ -256,6 +260,10 @@ public class ArbitraryDataReader { throw new IllegalStateException("Unable to build path"); } + // Update stats + this.layerCount = builder.getLayerCount(); + this.latestSignature = builder.getLatestSignature(); + // Set filePath to the builtPath this.filePath = builtPath; @@ -453,4 +461,12 @@ public class ArbitraryDataReader { return this.filePath; } + public int getLayerCount() { + return this.layerCount; + } + + public byte[] getLatestSignature() { + return this.latestSignature; + } + } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java index 280fc5c9..584bfab5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java @@ -3,6 +3,7 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; import org.qortal.block.BlockChain; import org.qortal.crypto.Crypto; import org.qortal.data.PaymentData; @@ -17,6 +18,7 @@ import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.Transformer; import org.qortal.utils.Base58; +import org.qortal.utils.FilesystemUtils; import org.qortal.utils.NTP; import java.io.IOException; @@ -29,6 +31,13 @@ public class ArbitraryDataTransactionBuilder { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataTransactionBuilder.class); + // Maximum number of PATCH layers allowed + private static final int MAX_LAYERS = 10; + // Maximum size difference (out of 1) allowed for PATCH transactions + private static final double MAX_SIZE_DIFF = 0.2f; + // Maximum proportion of files modified relative to total + private static final double MAX_FILE_DIFF = 0.5f; + private String publicKey58; private Path path; private String name; @@ -36,6 +45,8 @@ public class ArbitraryDataTransactionBuilder { private Service service; private String identifier; + private ArbitraryTransactionData arbitraryTransactionData; + public ArbitraryDataTransactionBuilder(String publicKey58, Path path, String name, Method method, Service service, String identifier) { this.publicKey58 = publicKey58; @@ -46,7 +57,79 @@ public class ArbitraryDataTransactionBuilder { this.identifier = identifier; } - public ArbitraryTransactionData build() throws DataException { + public void build() throws DataException { + try { + this.preExecute(); + this.checkMethod(); + this.createTransaction(); + } + finally { + this.postExecute(); + } + } + + private void preExecute() { + + } + + private void postExecute() { + + } + + private void checkMethod() throws DataException { + if (this.method == null) { + // We need to automatically determine the method + this.method = this.determineMethodAutomatically(); + } + } + + private Method determineMethodAutomatically() throws DataException { + ArbitraryDataReader reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier); + try { + reader.loadSynchronously(true); + + // Check layer count + int layerCount = reader.getLayerCount(); + if (layerCount >= MAX_LAYERS) { + LOGGER.info("Reached maximum layer count ({} / {}) - using PUT", layerCount, MAX_LAYERS); + return Method.PUT; + } + + // Check size of differences between this layer and previous layer + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(reader.getFilePath(), this.path, reader.getLatestSignature()); + patch.create(); + long diffSize = FilesystemUtils.getDirectorySize(patch.getFinalPath()); + long existingStateSize = FilesystemUtils.getDirectorySize(reader.getFilePath()); + double difference = (double) diffSize / (double) existingStateSize; + if (difference > MAX_SIZE_DIFF) { + LOGGER.info("Reached maximum difference ({} / {}) - using PUT", difference, MAX_SIZE_DIFF); + return Method.PUT; + } + + // Check number of modified files + int totalFileCount = patch.getTotalFileCount(); + int differencesCount = patch.getFileDifferencesCount(); + difference = (double) differencesCount / (double) totalFileCount; + if (difference > MAX_FILE_DIFF) { + LOGGER.info("Reached maximum file differences ({} / {}) - using PUT", difference, MAX_FILE_DIFF); + return Method.PUT; + } + + // State is appropriate for a PATCH transaction + return Method.PATCH; + } + catch (IOException | DataException | MissingDataException | IllegalStateException e) { + // Handle matching states separately, as it's best to block transactions with duplicate states + if (e.getMessage().equals("Current state matches previous state. Nothing to do.")) { + throw new DataException(e); + } + LOGGER.info("Caught exception: {}", e.getMessage()); + LOGGER.info("Unable to load existing resource - using PUT to overwrite it."); + return Method.PUT; + } + } + + private void createTransaction() throws DataException { ArbitraryDataFile arbitraryDataFile = null; try (final Repository repository = RepositoryManager.getRepository()) { @@ -115,7 +198,7 @@ public class ArbitraryDataTransactionBuilder { } LOGGER.info("Transaction is valid"); - return transactionData; + this.arbitraryTransactionData = transactionData; } catch (DataException e) { if (arbitraryDataFile != null) { @@ -126,4 +209,8 @@ public class ArbitraryDataTransactionBuilder { } + public ArbitraryTransactionData getArbitraryTransactionData() { + return this.arbitraryTransactionData; + } + } diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index bbf54e74..9c193ca6 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -180,4 +180,14 @@ public class FilesystemUtils { return false; } + public static long getDirectorySize(Path path) throws IOException { + if (path == null || !Files.exists(path)) { + return 0L; + } + return Files.walk(path) + .filter(p -> p.toFile().isFile()) + .mapToLong(p -> p.toFile().length()) + .sum(); + } + } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java index c1bc07b4..f16f585c 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java @@ -370,7 +370,8 @@ public class ArbitraryDataTests extends Common { ArbitraryDataTransactionBuilder txnBuilder = new ArbitraryDataTransactionBuilder( publicKey58, path, name, method, service, identifier); - ArbitraryTransactionData transactionData = txnBuilder.build(); + txnBuilder.build(); + ArbitraryTransactionData transactionData = txnBuilder.getArbitraryTransactionData(); Transaction.ValidationResult result = TransactionUtils.signAndImport(repository, transactionData, account); assertEquals(Transaction.ValidationResult.OK, result); BlockUtils.mintBlock(repository); diff --git a/tools/qdata b/tools/qdata index de8dd1d7..04450852 100755 --- a/tools/qdata +++ b/tools/qdata @@ -8,7 +8,7 @@ if [ -z "$*" ]; then echo "Usage:" echo echo "Host/update data:" - echo "qdata [PUT/PATCH] [service] [name] [dirpath] " + echo "qdata [POST/PUT/PATCH] [service] [name] [dirpath] " echo echo "Fetch data:" echo "qdata GET [service] [name] " @@ -36,7 +36,7 @@ if [ -z "${name}" ]; then fi -if [[ "${method}" == "PUT" || "${method}" == "PATCH" ]]; then +if [[ "${method}" == "POST" || "${method}" == "PUT" || "${method}" == "PATCH" ]]; then directory=$4 identifier=$5