Implemented automatic PUT/PATCH detection

When using POST /arbitrary/{service}/{name}... it will now automatically decide which method to use (PUT/PATCH) based on a few factors:

- If there are already 10 or more layers, use PUT to reset back to a single layer
- If the next layer's patch is more than 20% of the total resource file size, use PUT
- If the next layer modifies more than 50% of the total file count, use PUT
- Otherwise, use PATCH

The PUT method causes a new base layer to be created and all previous update history for that resource becomes obsolete. The PATCH method adds a small delta layer on top of the existing layer(s).

The idea is to wipe the slate clean with a new base layer once the patches start to get demanding for the network to apply. Nodes which view the content will ultimately have build timeouts to prevent someone from deploying a resource with hundreds of complex layers for example, so this approach is there to maximize the chances of the resource being buildable.

The constants above (10 layers, 20% total size, 50% file count) will most likely need tweaking once we have some real world data.

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -352,8 +352,7 @@ public class ArbitraryResource {
 					   String path) {
 		Security.checkApiCallAllowed(request);
 
-		// TODO: automatic PUT/PATCH
-		return this.upload(Method.PUT, Service.valueOf(serviceString), name, null, path);
+		return this.upload(null, Service.valueOf(serviceString), name, null, path);
 	}
 
 	@PUT
@@ -458,8 +457,7 @@ public class ArbitraryResource {
 					   String path) {
 		Security.checkApiCallAllowed(request);
 
-		// TODO: automatic PUT/PATCH
-		return this.upload(Method.PUT, Service.valueOf(serviceString), name, identifier, path);
+		return this.upload(null, Service.valueOf(serviceString), name, identifier, path);
 	}
 
 	@PUT
@@ -560,10 +558,11 @@ public class ArbitraryResource {
 						publicKey58, Paths.get(path), name, method, service, identifier
 				);
 
-				ArbitraryTransactionData transactionData = transactionBuilder.build();
+				transactionBuilder.build();
+				ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData();
 				return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData));
 
-			} catch (DataException | TransformationException e) {
+			} catch (DataException | TransformationException | IllegalStateException e) {
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage());
 			}
 

src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java

@@ -35,6 +35,7 @@ public class ArbitraryDataBuilder {
     private List<Path> paths;
     private byte[] latestSignature;
     private Path finalPath;
+    private int layerCount;
 
     public ArbitraryDataBuilder(String name, Service service, String identifier) {
         this.name = name;
@@ -69,7 +70,9 @@ public class ArbitraryDataBuilder {
             // Load all transactions since the latest PUT
             List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
                     .getArbitraryTransactions(this.name, this.service, this.identifier, latestPut.getTimestamp());
+
             this.transactions = transactionDataList;
+            this.layerCount = transactionDataList.size();
         }
     }
 
@@ -228,4 +231,8 @@ public class ArbitraryDataBuilder {
         return this.latestSignature;
     }
 
+    public int getLayerCount() {
+        return this.layerCount;
+    }
+
 }

src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java

@@ -19,7 +19,10 @@ public class ArbitraryDataCreatePatch {
     private Path pathBefore;
     private Path pathAfter;
     private byte[] previousSignature;
+
     private Path finalPath;
+    private int totalFileCount;
+    private int fileDifferencesCount;
 
     private Path workingPath;
     private String identifier;
@@ -116,10 +119,21 @@ public class ArbitraryDataCreatePatch {
         ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter, this.previousSignature);
         this.finalPath = diff.getDiffPath();
         diff.compute();
+
+        this.totalFileCount = diff.getTotalFileCount();
+        this.fileDifferencesCount = diff.getFileDifferencesCount();
     }
 
     public Path getFinalPath() {
         return this.finalPath;
     }
 
+    public int getTotalFileCount() {
+        return this.totalFileCount;
+    }
+
+    public int getFileDifferencesCount() {
+        return this.fileDifferencesCount;
+    }
+
 }

src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java

@@ -73,6 +73,8 @@ public class ArbitraryDataDiff {
     private List<ModifiedPath> modifiedPaths;
     private List<Path> removedPaths;
 
+    private int totalFileCount;
+
     public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) {
         this.pathBefore = pathBefore;
         this.pathAfter = pathAfter;
@@ -182,6 +184,9 @@ public class ArbitraryDataDiff {
                         diff.pathModified(beforePathAbsolute, afterPathAbsolute, afterPathRelative, diffPathAbsolute);
                     }
 
+                    // Keep a tally of the total number of files to help with decision making
+                    diff.totalFileCount++;
+
                     return FileVisitResult.CONTINUE;
                 }
 
@@ -345,6 +350,14 @@ public class ArbitraryDataDiff {
         return this.diffPath;
     }
 
+    public int getTotalFileCount() {
+        return this.totalFileCount;
+    }
+
+    public int getFileDifferencesCount() {
+        return this.addedPaths.size() + this.modifiedPaths.size() + this.removedPaths.size();
+    }
+
 
     // Utils
 

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -52,6 +52,10 @@ public class ArbitraryDataReader {
     private Path uncompressedPath;
     private Path unencryptedPath;
 
+    // Stats (available for synchronous builds only)
+    private int layerCount;
+    private byte[] latestSignature;
+
     public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
         // Ensure names are always lowercase
         if (resourceIdType == ResourceIdType.NAME) {
@@ -256,6 +260,10 @@ public class ArbitraryDataReader {
                 throw new IllegalStateException("Unable to build path");
             }
 
+            // Update stats
+            this.layerCount = builder.getLayerCount();
+            this.latestSignature = builder.getLatestSignature();
+
             // Set filePath to the builtPath
             this.filePath = builtPath;
 
@@ -453,4 +461,12 @@ public class ArbitraryDataReader {
         return this.filePath;
     }
 
+    public int getLayerCount() {
+        return this.layerCount;
+    }
+
+    public byte[] getLatestSignature() {
+        return this.latestSignature;
+    }
+
 }

src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java

@@ -3,6 +3,7 @@ package org.qortal.arbitrary;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
 import org.qortal.block.BlockChain;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.PaymentData;
@@ -17,6 +18,7 @@ import org.qortal.transaction.ArbitraryTransaction;
 import org.qortal.transaction.Transaction;
 import org.qortal.transform.Transformer;
 import org.qortal.utils.Base58;
+import org.qortal.utils.FilesystemUtils;
 import org.qortal.utils.NTP;
 
 import java.io.IOException;
@@ -29,6 +31,13 @@ public class ArbitraryDataTransactionBuilder {
 
     private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataTransactionBuilder.class);
 
+    // Maximum number of PATCH layers allowed
+    private static final int MAX_LAYERS = 10;
+    // Maximum size difference (out of 1) allowed for PATCH transactions
+    private static final double MAX_SIZE_DIFF = 0.2f;
+    // Maximum proportion of files modified relative to total
+    private static final double MAX_FILE_DIFF = 0.5f;
+
     private String publicKey58;
     private Path path;
     private String name;
@@ -36,6 +45,8 @@ public class ArbitraryDataTransactionBuilder {
     private Service service;
     private String identifier;
 
+    private ArbitraryTransactionData arbitraryTransactionData;
+
     public ArbitraryDataTransactionBuilder(String publicKey58, Path path, String name,
                                            Method method, Service service, String identifier) {
         this.publicKey58 = publicKey58;
@@ -46,7 +57,79 @@ public class ArbitraryDataTransactionBuilder {
         this.identifier = identifier;
     }
 
-    public ArbitraryTransactionData build() throws DataException {
+    public void build() throws DataException {
+        try {
+            this.preExecute();
+            this.checkMethod();
+            this.createTransaction();
+        }
+        finally {
+            this.postExecute();
+        }
+    }
+
+    private void preExecute() {
+
+    }
+
+    private void postExecute() {
+
+    }
+
+    private void checkMethod() throws DataException {
+        if (this.method == null) {
+            // We need to automatically determine the method
+            this.method = this.determineMethodAutomatically();
+        }
+    }
+
+    private Method determineMethodAutomatically() throws DataException {
+        ArbitraryDataReader reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier);
+        try {
+            reader.loadSynchronously(true);
+
+            // Check layer count
+            int layerCount = reader.getLayerCount();
+            if (layerCount >= MAX_LAYERS) {
+                LOGGER.info("Reached maximum layer count ({} / {}) - using PUT", layerCount, MAX_LAYERS);
+                return Method.PUT;
+            }
+
+            // Check size of differences between this layer and previous layer
+            ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(reader.getFilePath(), this.path, reader.getLatestSignature());
+            patch.create();
+            long diffSize = FilesystemUtils.getDirectorySize(patch.getFinalPath());
+            long existingStateSize = FilesystemUtils.getDirectorySize(reader.getFilePath());
+            double difference = (double) diffSize / (double) existingStateSize;
+            if (difference > MAX_SIZE_DIFF) {
+                LOGGER.info("Reached maximum difference ({} / {}) - using PUT", difference, MAX_SIZE_DIFF);
+                return Method.PUT;
+            }
+
+            // Check number of modified files
+            int totalFileCount = patch.getTotalFileCount();
+            int differencesCount = patch.getFileDifferencesCount();
+            difference = (double) differencesCount / (double) totalFileCount;
+            if (difference > MAX_FILE_DIFF) {
+                LOGGER.info("Reached maximum file differences ({} / {}) - using PUT", difference, MAX_FILE_DIFF);
+                return Method.PUT;
+            }
+
+            // State is appropriate for a PATCH transaction
+            return Method.PATCH;
+        }
+        catch (IOException | DataException | MissingDataException | IllegalStateException e) {
+            // Handle matching states separately, as it's best to block transactions with duplicate states
+            if (e.getMessage().equals("Current state matches previous state. Nothing to do.")) {
+                throw new DataException(e);
+            }
+            LOGGER.info("Caught exception: {}", e.getMessage());
+            LOGGER.info("Unable to load existing resource - using PUT to overwrite it.");
+            return Method.PUT;
+        }
+    }
+
+    private void createTransaction() throws DataException {
         ArbitraryDataFile arbitraryDataFile = null;
         try (final Repository repository = RepositoryManager.getRepository()) {
 
@@ -115,7 +198,7 @@ public class ArbitraryDataTransactionBuilder {
             }
             LOGGER.info("Transaction is valid");
 
-            return transactionData;
+            this.arbitraryTransactionData = transactionData;
 
         } catch (DataException e) {
             if (arbitraryDataFile != null) {
@@ -126,4 +209,8 @@ public class ArbitraryDataTransactionBuilder {
 
     }
 
+    public ArbitraryTransactionData getArbitraryTransactionData() {
+        return this.arbitraryTransactionData;
+    }
+
 }

src/main/java/org/qortal/utils/FilesystemUtils.java

@@ -180,4 +180,14 @@ public class FilesystemUtils {
         return false;
     }
 
+    public static long getDirectorySize(Path path) throws IOException {
+        if (path == null || !Files.exists(path)) {
+            return 0L;
+        }
+        return Files.walk(path)
+                .filter(p -> p.toFile().isFile())
+                .mapToLong(p -> p.toFile().length())
+                .sum();
+    }
+
 }

src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java

@@ -370,7 +370,8 @@ public class ArbitraryDataTests extends Common {
         ArbitraryDataTransactionBuilder txnBuilder = new ArbitraryDataTransactionBuilder(
                 publicKey58, path, name, method, service, identifier);
 
-        ArbitraryTransactionData transactionData = txnBuilder.build();
+        txnBuilder.build();
+        ArbitraryTransactionData transactionData = txnBuilder.getArbitraryTransactionData();
         Transaction.ValidationResult result = TransactionUtils.signAndImport(repository, transactionData, account);
         assertEquals(Transaction.ValidationResult.OK, result);
         BlockUtils.mintBlock(repository);

tools/qdata

@@ -8,7 +8,7 @@ if [ -z "$*" ]; then
 	echo "Usage:"
 	echo
 	echo "Host/update data:"
-	echo "qdata [PUT/PATCH] [service] [name] [dirpath] <identifier>"
+	echo "qdata [POST/PUT/PATCH] [service] [name] [dirpath] <identifier>"
 	echo
 	echo "Fetch data:"
 	echo "qdata GET [service] [name] <identifier-or-default> <filepath-or-default> <rebuild>"
@@ -36,7 +36,7 @@ if [ -z "${name}" ]; then
 fi
 
 
-if [[ "${method}" == "PUT" || "${method}" == "PATCH" ]]; then
+if [[ "${method}" == "POST" || "${method}" == "PUT" || "${method}" == "PATCH" ]]; then
   directory=$4
   identifier=$5