From e34fd855a9c8bd7032367ed3c0f28bad60d9672d Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 3 Dec 2021 21:11:56 +0000
Subject: [PATCH] Added GET /arbitrary/hosted/transactions API to list all
 arbitrary transactions that the node is hosting (at least partial) data for.

This API call could get quite heavy when large amounts of files are hosted, but it's preferable to maintaining a list in the database. Ideally we need to keep the database generic so that it can be bootstrapped without interfering with the state. We can always add caching and rate limiting if needed.
---
 .../api/resource/ArbitraryResource.java       | 26 ++++++++++
 .../ArbitraryDataStorageManager.java          | 49 +++++++++++++++++++
 2 files changed, 75 insertions(+)

diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
index 186f8825..c289ac3e 100644
--- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java
+++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
@@ -39,6 +39,7 @@ import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
 import org.qortal.arbitrary.exception.MissingDataException;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
+import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
 import org.qortal.data.account.AccountData;
 import org.qortal.data.arbitrary.ArbitraryResourceInfo;
 import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
@@ -588,6 +589,31 @@ public class ArbitraryResource {
 		return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64);
 	}
 
+	@GET
+	@Path("/hosted/transactions")
+	@Operation(
+			summary = "List arbitrary transactions hosted by this node",
+			responses = {
+					@ApiResponse(
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryTransactionData.class))
+					)
+			}
+	)
+	@ApiErrors({ApiError.REPOSITORY_ISSUE})
+	public List<ArbitraryTransactionData> getHostedTransactions() {
+		Security.checkApiCallAllowed(request);
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+
+			List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedData(repository);
+
+			return hostedTransactions;
+
+		} catch (DataException | IOException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
+	}
+
 	@POST
 	@Path("/compute")
 	@Operation(
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
index e37d120a..eb47c9fc 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
@@ -4,8 +4,13 @@ import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
 import org.qortal.list.ResourceListManager;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
 import org.qortal.settings.Settings;
+import org.qortal.transaction.Transaction;
+import org.qortal.utils.Base58;
 import org.qortal.utils.FilesystemUtils;
 import org.qortal.utils.NTP;
 
@@ -13,6 +18,9 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
 
 public class ArbitraryDataStorageManager extends Thread {
 
@@ -208,6 +216,47 @@ public class ArbitraryDataStorageManager extends Thread {
     }
 
 
+    // Hosted data
+    public List<ArbitraryTransactionData> listAllHostedData(Repository repository) throws IOException {
+        List<ArbitraryTransactionData> arbitraryTransactionDataList = new ArrayList<>();
+
+        Path dataPath = Paths.get(Settings.getInstance().getDataPath());
+        Path tempPath = Paths.get(Settings.getInstance().getTempDataPath());
+
+        // Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length
+        // Also exclude the _temp and _misc paths if present
+        List<Path> allPaths = Files.walk(dataPath, 3)
+                .filter(Files::isDirectory)
+                .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString()))
+                .filter(path -> !path.toString().contains("_misc"))
+                .filter(path -> path.getFileName().toString().length() > 32)
+                .sorted().collect(Collectors.toList());
+
+        // Loop through each path and attempt to match it to a signature
+        for (Path path : allPaths) {
+            try {
+                if (path.toFile().list().length == 0) {
+                    // Ignore empty directories
+                    continue;
+                }
+
+                String signature58 = path.getFileName().toString();
+                byte[] signature = Base58.decode(signature58);
+                TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+                if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) {
+                    continue;
+                }
+                arbitraryTransactionDataList.add((ArbitraryTransactionData) transactionData);
+
+            } catch (DataException e) {
+                continue;
+            }
+        }
+
+        return arbitraryTransactionDataList;
+    }
+
+
     // Size limits
 
     /**