Qortal/qortal
1
0
Fork 0
mirror of https://github.com/Qortal/qortal.git synced 2025-04-14 23:35:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

initial implementation of arbitrary data indexing support

Browse Source
This commit is contained in:
kennycud 2025-03-12 11:21:57 -07:00
parent 7f3c1d553f
commit ab4730cef0
9 changed files with 484 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -33,9 +33,13 @@ import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.controller.arbitrary.ArbitraryMetadataManager;
import org.qortal.data.account.AccountData;
import org.qortal.data.arbitrary.ArbitraryCategoryInfo;
import org.qortal.data.arbitrary.ArbitraryDataIndexDetail;
import org.qortal.data.arbitrary.ArbitraryDataIndexScoreKey;
import org.qortal.data.arbitrary.ArbitraryDataIndexScorecard;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.arbitrary.IndexCache;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
@ -69,8 +73,11 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@Path("/arbitrary")
@Tag(name = "Arbitrary")
@ -1186,6 +1193,63 @@ public class ArbitraryResource {
}
}
@GET
@Path("/indices")
@Operation(
summary = "Find matching arbitrary resource indices",
description = "",
responses = {
@ApiResponse(
description = "indices",
content = @Content(
array = @ArraySchema(
schema = @Schema(
implementation = ArbitraryDataIndexScorecard.class
)
)
)
)
}
)
public List<ArbitraryDataIndexScorecard> searchIndices(@QueryParam("terms") String[] terms) {
List<ArbitraryDataIndexDetail> indices = new ArrayList<>();
// get index details for each term
for( String term : terms ) {
List<ArbitraryDataIndexDetail> details = IndexCache.getInstance().getIndicesByTerm().get(term);
if( details != null ) {
indices.addAll(details);
}
}
// sum up the scores for each index with identical attributes
Map<ArbitraryDataIndexScoreKey, Double> scoreForKey
= indices.stream()
.collect(
Collectors.groupingBy(
index -> new ArbitraryDataIndexScoreKey(index.name, index.service, index.link),
Collectors.summingDouble(detail -> 1.0 / detail.rank)
)
);
// create scorecards for each index group and put them in descending order by score
List<ArbitraryDataIndexScorecard> scorecards
= scoreForKey.entrySet().stream().map(
entry
->
new ArbitraryDataIndexScorecard(
entry.getValue(),
entry.getKey().name,
entry.getKey().service,
entry.getKey().link)
)
.sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed())
.collect(Collectors.toList());
return scorecards;
}
// Shared methods

6
src/main/java/org/qortal/controller/Controller.java
View File

@ -423,6 +423,12 @@ public class Controller extends Thread {
LOGGER.info("Db Cache Disabled");
}
LOGGER.info("Arbitrary Indexing Starting ...");
ArbitraryIndexUtils.startCaching(
Settings.getInstance().getArbitraryIndexingPriority(),
Settings.getInstance().getArbitraryIndexingFrequency()
);
if( Settings.getInstance().isBalanceRecorderEnabled() ) {
Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();

37
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java Normal file
View File

@ -0,0 +1,37 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndex {
public String term;
public String name;
public Service service;
public String identifier;
public String link;
public ArbitraryDataIndex() {}
public ArbitraryDataIndex(String term, String name, Service service, String identifier, String link) {
this.term = term;
this.name = name;
this.service = service;
this.identifier = identifier;
this.link = link;
}
@Override
public String toString() {
return "ArbitraryDataIndex{" +
"term='" + term + '\'' +
", name='" + name + '\'' +
", service=" + service +
", identifier='" + identifier + '\'' +
", link='" + link + '\'' +
'}';
}
}

43
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java Normal file
View File

@ -0,0 +1,43 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndexDetail {
public String issuer;
public int rank;
public String term;
public String name;
public Service service;
public String identifier;
public String link;
public ArbitraryDataIndexDetail() {}
public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index) {
this.issuer = issuer;
this.rank = rank;
this.term = index.term;
this.name = index.name;
this.service = index.service;
this.identifier = index.identifier;
this.link = index.link;
}
@Override
public String toString() {
return "ArbitraryDataIndexDetail{" +
"issuer='" + issuer + '\'' +
", rank=" + rank +
", term='" + term + '\'' +
", name='" + name + '\'' +
", service=" + service +
", identifier='" + identifier + '\'' +
", link='" + link + '\'' +
'}';
}
}

45
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java Normal file
View File

@ -0,0 +1,45 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.Objects;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndexScoreKey {
public String name;
public Service service;
public String link;
public ArbitraryDataIndexScoreKey() {}
public ArbitraryDataIndexScoreKey(String name, Service service, String link) {
this.name = name;
this.service = service;
this.link = link;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o;
return Objects.equals(name, that.name) && service == that.service && Objects.equals(link, that.link);
}
@Override
public int hashCode() {
return Objects.hash(name, service, link);
}
@Override
public String toString() {
return "ArbitraryDataIndexScoreKey{" +
"name='" + name + '\'' +
", service=" + service +
", link='" + link + '\'' +
'}';
}
}

38
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java Normal file
View File

@ -0,0 +1,38 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndexScorecard {
public double score;
public String name;
public Service service;
public String link;
public ArbitraryDataIndexScorecard() {}
public ArbitraryDataIndexScorecard(double score, String name, Service service, String link) {
this.score = score;
this.name = name;
this.service = service;
this.link = link;
}
public double getScore() {
return score;
}
@Override
public String toString() {
return "ArbitraryDataIndexScorecard{" +
"score=" + score +
", name='" + name + '\'' +
", service=" + service +
", link='" + link + '\'' +
'}';
}
}

20
src/main/java/org/qortal/data/arbitrary/IndexCache.java Normal file
View File

@ -0,0 +1,20 @@
package org.qortal.data.arbitrary;
import org.qortal.list.ResourceList;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
public class IndexCache {
public static final IndexCache SINGLETON = new IndexCache();
private ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> indicesByTerm = new ConcurrentHashMap<>();
public static IndexCache getInstance() {
return SINGLETON;
}
public ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> getIndicesByTerm() {
return indicesByTerm;
}
}

12
src/main/java/org/qortal/settings/Settings.java
View File

@ -510,6 +510,10 @@ public class Settings {
private int buildArbitraryResourcesBatchSize = 200;
private int arbitraryIndexingPriority = 5;
private int arbitraryIndexingFrequency = 1;
// Domain mapping
public static class ThreadLimit {
private String messageType;
@ -1339,4 +1343,12 @@ public class Settings {
public int getBuildArbitraryResourcesBatchSize() {
return buildArbitraryResourcesBatchSize;
}
public int getArbitraryIndexingPriority() {
return arbitraryIndexingPriority;
}
public int getArbitraryIndexingFrequency() {
return arbitraryIndexingFrequency;
}
}

219
src/main/java/org/qortal/utils/ArbitraryIndexUtils.java Normal file
View File

@ -0,0 +1,219 @@
package org.qortal.utils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.SearchMode;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryDataIndex;
import org.qortal.data.arbitrary.ArbitraryDataIndexDetail;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.IndexCache;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class ArbitraryIndexUtils {
public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final Logger LOGGER = LogManager.getLogger(ArbitraryIndexUtils.class);
public static final String INDEX_CACHE_TIMER = "Arbitrary Index Cache Timer";
public static final String INDEX_CACHE_TIMER_TASK = "Arbitrary Index Cache Timer Task";
public static void startCaching(int priorityRequested, int frequency) {
Timer timer = buildTimer(INDEX_CACHE_TIMER, priorityRequested);
TimerTask task = new TimerTask() {
@Override
public void run() {
Thread.currentThread().setName(INDEX_CACHE_TIMER_TASK);
try {
fillCache(IndexCache.getInstance());
} catch (IOException | DataException e) {
LOGGER.error(e.getMessage(), e);
}
}
};
// delay 1 second
timer.scheduleAtFixedRate(task, 1_000, frequency * 60_000);
}
private static void fillCache(IndexCache instance) throws DataException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
List<ArbitraryResourceData> indexResources
= repository.getArbitraryRepository().searchArbitraryResources(
Service.JSON,
null,
"idx-",
null,
null,
null,
null,
true,
null,
false,
SearchMode.ALL,
0,
null,
null,
null,
null,
0L,
0L,
0,
0,
true);
List<ArbitraryDataIndexDetail> indexDetails = new ArrayList<>();
LOGGER.debug("processing index resource data: count = " + indexResources.size());
// process all index resources
for( ArbitraryResourceData indexResource : indexResources ) {
LOGGER.debug("processing index resource: name = " + indexResource.name + ", identifier = " + indexResource.identifier);
String json = ArbitraryIndexUtils.getJson(indexResource.name, indexResource.identifier);
// map the JSON string to a list of Java objects
List<ArbitraryDataIndex> indices = OBJECT_MAPPER.readValue(json, new TypeReference<List<ArbitraryDataIndex>>() {});
LOGGER.debug("processed indices = " + indices);
// rank and create index detail for each index in this index resource
for( int rank = 1; rank <= indices.size(); rank++ ) {
indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1) ));
}
}
LOGGER.debug("processing indices by term ...");
Map<String, List<ArbitraryDataIndexDetail>> indicesByTerm
= indexDetails.stream().collect(
Collectors.toMap(
detail -> detail.term, // map by term
detail -> List.of(detail), // create list for term
(list1, list2) // merge lists for same term
-> Stream.of(list1, list2)
.flatMap(List::stream)
.collect(Collectors.toList())
)
);
LOGGER.info("processed indices by term = " + indicesByTerm);
// lock, clear old, load new
synchronized( IndexCache.getInstance().getIndicesByTerm() ) {
IndexCache.getInstance().getIndicesByTerm().clear();
IndexCache.getInstance().getIndicesByTerm().putAll(indicesByTerm);
}
LOGGER.info("loaded indices by term");
}
}
private static Timer buildTimer( final String name, int priorityRequested) {
// ensure priority is in between 1-10
final int priority = Math.max(0, Math.min(10, priorityRequested));
// Create a custom Timer with updated priority threads
Timer timer = new Timer(true) { // 'true' to make the Timer daemon
@Override
public void schedule(TimerTask task, long delay) {
Thread thread = new Thread(task, name) {
@Override
public void run() {
this.setPriority(priority);
super.run();
}
};
thread.setPriority(priority);
thread.start();
}
};
return timer;
}
public static String getJsonWithExceptionHandling( String name, String identifier ) {
try {
return getJson(name, identifier);
}
catch( Exception e ) {
LOGGER.error(e.getMessage(), e);
return e.getMessage();
}
}
public static String getJson(String name, String identifier) throws IOException {
try {
ArbitraryDataReader arbitraryDataReader
= new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, Service.JSON, identifier);
int attempts = 0;
Integer maxAttempts = 5;
while (!Controller.isStopping()) {
attempts++;
if (!arbitraryDataReader.isBuilding()) {
try {
arbitraryDataReader.loadSynchronously(false);
break;
} catch (MissingDataException e) {
if (attempts > maxAttempts) {
// Give up after 5 attempts
throw new IOException("Data unavailable. Please try again later.");
}
}
}
Thread.sleep(3000L);
}
java.nio.file.Path outputPath = arbitraryDataReader.getFilePath();
if (outputPath == null) {
// Assume the resource doesn't exist
throw new IOException( "File not found");
}
// No file path supplied - so check if this is a single file resource
String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal");
String filepath = files[0];
java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
if (!Files.exists(path)) {
String message = String.format("No file exists at filepath: %s", filepath);
throw new IOException( message );
}
String data = Files.readString(path);
return data;
} catch (Exception e) {
throw new IOException(String.format("Unable to load %s %s: %s", Service.JSON, name, e.getMessage()));
}
}
}