From ab4730cef02753e0f2c4906c766b81cdfaa2d86d Mon Sep 17 00:00:00 2001 From: kennycud Date: Wed, 12 Mar 2025 11:21:57 -0700 Subject: [PATCH] initial implementation of arbitrary data indexing support --- .../api/resource/ArbitraryResource.java | 64 +++++ .../org/qortal/controller/Controller.java | 6 + .../data/arbitrary/ArbitraryDataIndex.java | 37 +++ .../arbitrary/ArbitraryDataIndexDetail.java | 43 ++++ .../arbitrary/ArbitraryDataIndexScoreKey.java | 45 ++++ .../ArbitraryDataIndexScorecard.java | 38 +++ .../org/qortal/data/arbitrary/IndexCache.java | 20 ++ .../java/org/qortal/settings/Settings.java | 12 + .../org/qortal/utils/ArbitraryIndexUtils.java | 219 ++++++++++++++++++ 9 files changed, 484 insertions(+) create mode 100644 src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java create mode 100644 src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java create mode 100644 src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java create mode 100644 src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java create mode 100644 src/main/java/org/qortal/data/arbitrary/IndexCache.java create mode 100644 src/main/java/org/qortal/utils/ArbitraryIndexUtils.java diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 2aa7b07d..68c50786 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -33,9 +33,13 @@ import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; import org.qortal.controller.arbitrary.ArbitraryMetadataManager; import org.qortal.data.account.AccountData; import org.qortal.data.arbitrary.ArbitraryCategoryInfo; +import org.qortal.data.arbitrary.ArbitraryDataIndexDetail; +import org.qortal.data.arbitrary.ArbitraryDataIndexScoreKey; +import org.qortal.data.arbitrary.ArbitraryDataIndexScorecard; import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.arbitrary.ArbitraryResourceStatus; +import org.qortal.data.arbitrary.IndexCache; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; @@ -69,8 +73,11 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; @Path("/arbitrary") @Tag(name = "Arbitrary") @@ -1186,6 +1193,63 @@ public class ArbitraryResource { } } + @GET + @Path("/indices") + @Operation( + summary = "Find matching arbitrary resource indices", + description = "", + responses = { + @ApiResponse( + description = "indices", + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = ArbitraryDataIndexScorecard.class + ) + ) + ) + ) + } + ) + public List searchIndices(@QueryParam("terms") String[] terms) { + + List indices = new ArrayList<>(); + + // get index details for each term + for( String term : terms ) { + List details = IndexCache.getInstance().getIndicesByTerm().get(term); + + if( details != null ) { + indices.addAll(details); + } + } + + // sum up the scores for each index with identical attributes + Map scoreForKey + = indices.stream() + .collect( + Collectors.groupingBy( + index -> new ArbitraryDataIndexScoreKey(index.name, index.service, index.link), + Collectors.summingDouble(detail -> 1.0 / detail.rank) + ) + ); + + // create scorecards for each index group and put them in descending order by score + List scorecards + = scoreForKey.entrySet().stream().map( + entry + -> + new ArbitraryDataIndexScorecard( + entry.getValue(), + entry.getKey().name, + entry.getKey().service, + entry.getKey().link) + ) + .sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed()) + .collect(Collectors.toList()); + + return scorecards; + } // Shared methods diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 180ef4d1..14153e97 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -423,6 +423,12 @@ public class Controller extends Thread { LOGGER.info("Db Cache Disabled"); } + LOGGER.info("Arbitrary Indexing Starting ..."); + ArbitraryIndexUtils.startCaching( + Settings.getInstance().getArbitraryIndexingPriority(), + Settings.getInstance().getArbitraryIndexingFrequency() + ); + if( Settings.getInstance().isBalanceRecorderEnabled() ) { Optional recorder = HSQLDBBalanceRecorder.getInstance(); diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java new file mode 100644 index 00000000..22e8758d --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java @@ -0,0 +1,37 @@ +package org.qortal.data.arbitrary; + +import org.qortal.arbitrary.misc.Service; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryDataIndex { + + public String term; + public String name; + public Service service; + public String identifier; + public String link; + + public ArbitraryDataIndex() {} + + public ArbitraryDataIndex(String term, String name, Service service, String identifier, String link) { + this.term = term; + this.name = name; + this.service = service; + this.identifier = identifier; + this.link = link; + } + + @Override + public String toString() { + return "ArbitraryDataIndex{" + + "term='" + term + '\'' + + ", name='" + name + '\'' + + ", service=" + service + + ", identifier='" + identifier + '\'' + + ", link='" + link + '\'' + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java new file mode 100644 index 00000000..30a4186a --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java @@ -0,0 +1,43 @@ +package org.qortal.data.arbitrary; + +import org.qortal.arbitrary.misc.Service; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryDataIndexDetail { + + public String issuer; + public int rank; + public String term; + public String name; + public Service service; + public String identifier; + public String link; + + public ArbitraryDataIndexDetail() {} + + public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index) { + this.issuer = issuer; + this.rank = rank; + this.term = index.term; + this.name = index.name; + this.service = index.service; + this.identifier = index.identifier; + this.link = index.link; + } + + @Override + public String toString() { + return "ArbitraryDataIndexDetail{" + + "issuer='" + issuer + '\'' + + ", rank=" + rank + + ", term='" + term + '\'' + + ", name='" + name + '\'' + + ", service=" + service + + ", identifier='" + identifier + '\'' + + ", link='" + link + '\'' + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java new file mode 100644 index 00000000..1fc892a7 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java @@ -0,0 +1,45 @@ +package org.qortal.data.arbitrary; + +import org.qortal.arbitrary.misc.Service; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryDataIndexScoreKey { + + public String name; + public Service service; + public String link; + + public ArbitraryDataIndexScoreKey() {} + + public ArbitraryDataIndexScoreKey(String name, Service service, String link) { + this.name = name; + this.service = service; + this.link = link; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o; + return Objects.equals(name, that.name) && service == that.service && Objects.equals(link, that.link); + } + + @Override + public int hashCode() { + return Objects.hash(name, service, link); + } + + @Override + public String toString() { + return "ArbitraryDataIndexScoreKey{" + + "name='" + name + '\'' + + ", service=" + service + + ", link='" + link + '\'' + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java new file mode 100644 index 00000000..62e85d26 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java @@ -0,0 +1,38 @@ +package org.qortal.data.arbitrary; + +import org.qortal.arbitrary.misc.Service; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryDataIndexScorecard { + + public double score; + public String name; + public Service service; + public String link; + + public ArbitraryDataIndexScorecard() {} + + public ArbitraryDataIndexScorecard(double score, String name, Service service, String link) { + this.score = score; + this.name = name; + this.service = service; + this.link = link; + } + + public double getScore() { + return score; + } + + @Override + public String toString() { + return "ArbitraryDataIndexScorecard{" + + "score=" + score + + ", name='" + name + '\'' + + ", service=" + service + + ", link='" + link + '\'' + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/arbitrary/IndexCache.java b/src/main/java/org/qortal/data/arbitrary/IndexCache.java new file mode 100644 index 00000000..d688af76 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/IndexCache.java @@ -0,0 +1,20 @@ +package org.qortal.data.arbitrary; + +import org.qortal.list.ResourceList; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +public class IndexCache { + + public static final IndexCache SINGLETON = new IndexCache(); + private ConcurrentHashMap> indicesByTerm = new ConcurrentHashMap<>(); + + public static IndexCache getInstance() { + return SINGLETON; + } + + public ConcurrentHashMap> getIndicesByTerm() { + return indicesByTerm; + } +} diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 78e1f73e..831dd881 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -510,6 +510,10 @@ public class Settings { private int buildArbitraryResourcesBatchSize = 200; + private int arbitraryIndexingPriority = 5; + + private int arbitraryIndexingFrequency = 1; + // Domain mapping public static class ThreadLimit { private String messageType; @@ -1339,4 +1343,12 @@ public class Settings { public int getBuildArbitraryResourcesBatchSize() { return buildArbitraryResourcesBatchSize; } + + public int getArbitraryIndexingPriority() { + return arbitraryIndexingPriority; + } + + public int getArbitraryIndexingFrequency() { + return arbitraryIndexingFrequency; + } } diff --git a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java new file mode 100644 index 00000000..5dc97324 --- /dev/null +++ b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java @@ -0,0 +1,219 @@ +package org.qortal.utils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.api.SearchMode; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataReader; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryDataIndex; +import org.qortal.data.arbitrary.ArbitraryDataIndexDetail; +import org.qortal.data.arbitrary.ArbitraryResourceData; +import org.qortal.data.arbitrary.IndexCache; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ArbitraryIndexUtils { + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final Logger LOGGER = LogManager.getLogger(ArbitraryIndexUtils.class); + + public static final String INDEX_CACHE_TIMER = "Arbitrary Index Cache Timer"; + public static final String INDEX_CACHE_TIMER_TASK = "Arbitrary Index Cache Timer Task"; + + public static void startCaching(int priorityRequested, int frequency) { + + Timer timer = buildTimer(INDEX_CACHE_TIMER, priorityRequested); + + TimerTask task = new TimerTask() { + @Override + public void run() { + + Thread.currentThread().setName(INDEX_CACHE_TIMER_TASK); + + try { + fillCache(IndexCache.getInstance()); + } catch (IOException | DataException e) { + LOGGER.error(e.getMessage(), e); + } + } + }; + + // delay 1 second + timer.scheduleAtFixedRate(task, 1_000, frequency * 60_000); + } + + private static void fillCache(IndexCache instance) throws DataException, IOException { + + try (final Repository repository = RepositoryManager.getRepository()) { + + List indexResources + = repository.getArbitraryRepository().searchArbitraryResources( + Service.JSON, + null, + "idx-", + null, + null, + null, + null, + true, + null, + false, + SearchMode.ALL, + 0, + null, + null, + null, + null, + 0L, + 0L, + 0, + 0, + true); + + List indexDetails = new ArrayList<>(); + + LOGGER.debug("processing index resource data: count = " + indexResources.size()); + + // process all index resources + for( ArbitraryResourceData indexResource : indexResources ) { + + LOGGER.debug("processing index resource: name = " + indexResource.name + ", identifier = " + indexResource.identifier); + String json = ArbitraryIndexUtils.getJson(indexResource.name, indexResource.identifier); + + // map the JSON string to a list of Java objects + List indices = OBJECT_MAPPER.readValue(json, new TypeReference>() {}); + + LOGGER.debug("processed indices = " + indices); + + // rank and create index detail for each index in this index resource + for( int rank = 1; rank <= indices.size(); rank++ ) { + + indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1) )); + } + } + + LOGGER.debug("processing indices by term ..."); + Map> indicesByTerm + = indexDetails.stream().collect( + Collectors.toMap( + detail -> detail.term, // map by term + detail -> List.of(detail), // create list for term + (list1, list2) // merge lists for same term + -> Stream.of(list1, list2) + .flatMap(List::stream) + .collect(Collectors.toList()) + ) + ); + + LOGGER.info("processed indices by term = " + indicesByTerm); + + // lock, clear old, load new + synchronized( IndexCache.getInstance().getIndicesByTerm() ) { + IndexCache.getInstance().getIndicesByTerm().clear(); + IndexCache.getInstance().getIndicesByTerm().putAll(indicesByTerm); + } + + LOGGER.info("loaded indices by term"); + } + } + + private static Timer buildTimer( final String name, int priorityRequested) { + // ensure priority is in between 1-10 + final int priority = Math.max(0, Math.min(10, priorityRequested)); + + // Create a custom Timer with updated priority threads + Timer timer = new Timer(true) { // 'true' to make the Timer daemon + @Override + public void schedule(TimerTask task, long delay) { + Thread thread = new Thread(task, name) { + @Override + public void run() { + this.setPriority(priority); + super.run(); + } + }; + thread.setPriority(priority); + thread.start(); + } + }; + return timer; + } + + + public static String getJsonWithExceptionHandling( String name, String identifier ) { + try { + return getJson(name, identifier); + } + catch( Exception e ) { + LOGGER.error(e.getMessage(), e); + return e.getMessage(); + } + } + + public static String getJson(String name, String identifier) throws IOException { + + try { + ArbitraryDataReader arbitraryDataReader + = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, Service.JSON, identifier); + + int attempts = 0; + Integer maxAttempts = 5; + + while (!Controller.isStopping()) { + attempts++; + if (!arbitraryDataReader.isBuilding()) { + try { + arbitraryDataReader.loadSynchronously(false); + break; + } catch (MissingDataException e) { + if (attempts > maxAttempts) { + // Give up after 5 attempts + throw new IOException("Data unavailable. Please try again later."); + } + } + } + Thread.sleep(3000L); + } + + java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); + if (outputPath == null) { + // Assume the resource doesn't exist + throw new IOException( "File not found"); + } + + // No file path supplied - so check if this is a single file resource + String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); + String filepath = files[0]; + + java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); + if (!Files.exists(path)) { + String message = String.format("No file exists at filepath: %s", filepath); + throw new IOException( message ); + } + + String data = Files.readString(path); + + return data; + } catch (Exception e) { + throw new IOException(String.format("Unable to load %s %s: %s", Service.JSON, name, e.getMessage())); + } + } +} \ No newline at end of file