From a6309e925b27a35cb763b65632113875299a4dbb Mon Sep 17 00:00:00 2001 From: kennycud Date: Thu, 3 Apr 2025 10:18:45 -0700 Subject: [PATCH] replaced index service attribute with a category attribute and reduced index attribute names to single characters to reduce memory footprint, t is for term, n is for name, c is for category, l if for link changed default indexing frequency from 1 minute to 10 minutes to reduce memory use added arbitrary resource endpoint for index search by issuer name and index prefix added some additional error handling concerning unrecognized properties in the indices --- .../api/resource/ArbitraryResource.java | 31 +++++++++++++++++-- .../data/arbitrary/ArbitraryDataIndex.java | 29 ++++++++--------- .../arbitrary/ArbitraryDataIndexDetail.java | 22 ++++++------- .../arbitrary/ArbitraryDataIndexScoreKey.java | 19 ++++-------- .../ArbitraryDataIndexScorecard.java | 8 ++--- .../java/org/qortal/settings/Settings.java | 2 +- .../org/qortal/utils/ArbitraryIndexUtils.java | 28 ++++++++++++++++- 7 files changed, 90 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 68c50786..a6f44373 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1229,7 +1229,7 @@ public class ArbitraryResource { = indices.stream() .collect( Collectors.groupingBy( - index -> new ArbitraryDataIndexScoreKey(index.name, index.service, index.link), + index -> new ArbitraryDataIndexScoreKey(index.name, index.category, index.link), Collectors.summingDouble(detail -> 1.0 / detail.rank) ) ); @@ -1242,7 +1242,7 @@ public class ArbitraryResource { new ArbitraryDataIndexScorecard( entry.getValue(), entry.getKey().name, - entry.getKey().service, + entry.getKey().category, entry.getKey().link) ) .sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed()) @@ -1251,6 +1251,33 @@ public class ArbitraryResource { return scorecards; } + @GET + @Path("/indices/{name}/{idPrefix}") + @Operation( + summary = "Find matching arbitrary resource indices for a registered name and identifier prefix", + description = "", + responses = { + @ApiResponse( + description = "indices", + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = ArbitraryDataIndexDetail.class + ) + ) + ) + ) + } + ) + public List searchIndicesByName(@PathParam("name") String name, @PathParam("idPrefix") String idPrefix) { + + return + IndexCache.getInstance().getIndicesByIssuer() + .getOrDefault(name, new ArrayList<>(0)).stream() + .filter( indexDetail -> indexDetail.indexIdentifer.startsWith(idPrefix)) + .collect(Collectors.toList()); + } + // Shared methods private String preview(String directoryPath, Service service) { diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java index 22e8758d..9b6bf415 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java @@ -8,30 +8,27 @@ import javax.xml.bind.annotation.XmlAccessorType; @XmlAccessorType(XmlAccessType.FIELD) public class ArbitraryDataIndex { - public String term; - public String name; - public Service service; - public String identifier; - public String link; + public String t; + public String n; + public int c; + public String l; public ArbitraryDataIndex() {} - public ArbitraryDataIndex(String term, String name, Service service, String identifier, String link) { - this.term = term; - this.name = name; - this.service = service; - this.identifier = identifier; - this.link = link; + public ArbitraryDataIndex(String t, String n, int c, String l) { + this.t = t; + this.n = n; + this.c = c; + this.l = l; } @Override public String toString() { return "ArbitraryDataIndex{" + - "term='" + term + '\'' + - ", name='" + name + '\'' + - ", service=" + service + - ", identifier='" + identifier + '\'' + - ", link='" + link + '\'' + + "t='" + t + '\'' + + ", n='" + n + '\'' + + ", c=" + c + + ", l='" + l + '\'' + '}'; } } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java index 30a4186a..d073c736 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java @@ -1,7 +1,5 @@ package org.qortal.data.arbitrary; -import org.qortal.arbitrary.misc.Service; - import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -12,20 +10,20 @@ public class ArbitraryDataIndexDetail { public int rank; public String term; public String name; - public Service service; - public String identifier; + public int category; public String link; + public String indexIdentifer; public ArbitraryDataIndexDetail() {} - public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index) { + public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index, String indexIdentifer) { this.issuer = issuer; this.rank = rank; - this.term = index.term; - this.name = index.name; - this.service = index.service; - this.identifier = index.identifier; - this.link = index.link; + this.term = index.t; + this.name = index.n; + this.category = index.c; + this.link = index.l; + this.indexIdentifer = indexIdentifer; } @Override @@ -35,9 +33,9 @@ public class ArbitraryDataIndexDetail { ", rank=" + rank + ", term='" + term + '\'' + ", name='" + name + '\'' + - ", service=" + service + - ", identifier='" + identifier + '\'' + + ", category=" + category + ", link='" + link + '\'' + + ", indexIdentifer='" + indexIdentifer + '\'' + '}'; } } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java index 1fc892a7..46a661e5 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java @@ -10,14 +10,14 @@ import java.util.Objects; public class ArbitraryDataIndexScoreKey { public String name; - public Service service; + public int category; public String link; public ArbitraryDataIndexScoreKey() {} - public ArbitraryDataIndexScoreKey(String name, Service service, String link) { + public ArbitraryDataIndexScoreKey(String name, int category, String link) { this.name = name; - this.service = service; + this.category = category; this.link = link; } @@ -26,20 +26,13 @@ public class ArbitraryDataIndexScoreKey { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o; - return Objects.equals(name, that.name) && service == that.service && Objects.equals(link, that.link); + return category == that.category && Objects.equals(name, that.name) && Objects.equals(link, that.link); } @Override public int hashCode() { - return Objects.hash(name, service, link); + return Objects.hash(name, category, link); } - @Override - public String toString() { - return "ArbitraryDataIndexScoreKey{" + - "name='" + name + '\'' + - ", service=" + service + - ", link='" + link + '\'' + - '}'; - } + } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java index 62e85d26..1888a4a5 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java @@ -10,15 +10,15 @@ public class ArbitraryDataIndexScorecard { public double score; public String name; - public Service service; + public int category; public String link; public ArbitraryDataIndexScorecard() {} - public ArbitraryDataIndexScorecard(double score, String name, Service service, String link) { + public ArbitraryDataIndexScorecard(double score, String name, int category, String link) { this.score = score; this.name = name; - this.service = service; + this.category = category; this.link = link; } @@ -31,7 +31,7 @@ public class ArbitraryDataIndexScorecard { return "ArbitraryDataIndexScorecard{" + "score=" + score + ", name='" + name + '\'' + - ", service=" + service + + ", category=" + category + ", link='" + link + '\'' + '}'; } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index c84eecb7..5ddcffc7 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -527,7 +527,7 @@ public class Settings { * * The frequency at which the arbitrary indices are cached. */ - private int arbitraryIndexingFrequency = 1; + private int arbitraryIndexingFrequency = 10; private boolean rebuildArbitraryResourceCacheTaskEnabled = false; diff --git a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java index 64042604..17c966fe 100644 --- a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java @@ -3,6 +3,7 @@ package org.qortal.utils; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.exc.InvalidFormatException; +import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -108,10 +109,12 @@ public class ArbitraryIndexUtils { // rank and create index detail for each index in this index resource for( int rank = 1; rank <= indices.size(); rank++ ) { - indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1) )); + indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1), indexResource.identifier )); } } catch (InvalidFormatException e) { LOGGER.debug("invalid format, skipping: " + indexResource); + } catch (UnrecognizedPropertyException e) { + LOGGER.debug("unrecognized property, skipping " + indexResource); } } @@ -137,6 +140,29 @@ public class ArbitraryIndexUtils { } LOGGER.info("loaded indices by term"); + + LOGGER.debug("processing indices by issuer ..."); + Map> indicesByIssuer + = indexDetails.stream().collect( + Collectors.toMap( + detail -> detail.issuer, // map by issuer + detail -> List.of(detail), // create list for issuer + (list1, list2) // merge lists for same issuer + -> Stream.of(list1, list2) + .flatMap(List::stream) + .collect(Collectors.toList()) + ) + ); + + LOGGER.info("processed indices by issuer: count = " + indicesByIssuer.size()); + + // lock, clear old, load new + synchronized( IndexCache.getInstance().getIndicesByIssuer() ) { + IndexCache.getInstance().getIndicesByIssuer().clear(); + IndexCache.getInstance().getIndicesByIssuer().putAll(indicesByIssuer); + } + + LOGGER.info("loaded indices by issuer"); } }