replaced index service attribute with a category attribute and reduced index attribute names to single characters to reduce memory footprint, t is for term, n is for name, c is for category, l if for link

changed default indexing frequency from 1 minute to 10 minutes to reduce memory use

added arbitrary resource endpoint for index search by issuer name and index prefix

added some additional error handling concerning unrecognized properties in the indices
This commit is contained in:
kennycud 2025-04-03 10:18:45 -07:00
parent 23de8a98bc
commit a6309e925b
7 changed files with 90 additions and 49 deletions

View File

@ -1229,7 +1229,7 @@ public class ArbitraryResource {
= indices.stream() = indices.stream()
.collect( .collect(
Collectors.groupingBy( Collectors.groupingBy(
index -> new ArbitraryDataIndexScoreKey(index.name, index.service, index.link), index -> new ArbitraryDataIndexScoreKey(index.name, index.category, index.link),
Collectors.summingDouble(detail -> 1.0 / detail.rank) Collectors.summingDouble(detail -> 1.0 / detail.rank)
) )
); );
@ -1242,7 +1242,7 @@ public class ArbitraryResource {
new ArbitraryDataIndexScorecard( new ArbitraryDataIndexScorecard(
entry.getValue(), entry.getValue(),
entry.getKey().name, entry.getKey().name,
entry.getKey().service, entry.getKey().category,
entry.getKey().link) entry.getKey().link)
) )
.sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed()) .sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed())
@ -1251,6 +1251,33 @@ public class ArbitraryResource {
return scorecards; return scorecards;
} }
@GET
@Path("/indices/{name}/{idPrefix}")
@Operation(
summary = "Find matching arbitrary resource indices for a registered name and identifier prefix",
description = "",
responses = {
@ApiResponse(
description = "indices",
content = @Content(
array = @ArraySchema(
schema = @Schema(
implementation = ArbitraryDataIndexDetail.class
)
)
)
)
}
)
public List<ArbitraryDataIndexDetail> searchIndicesByName(@PathParam("name") String name, @PathParam("idPrefix") String idPrefix) {
return
IndexCache.getInstance().getIndicesByIssuer()
.getOrDefault(name, new ArrayList<>(0)).stream()
.filter( indexDetail -> indexDetail.indexIdentifer.startsWith(idPrefix))
.collect(Collectors.toList());
}
// Shared methods // Shared methods
private String preview(String directoryPath, Service service) { private String preview(String directoryPath, Service service) {

View File

@ -8,30 +8,27 @@ import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD) @XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndex { public class ArbitraryDataIndex {
public String term; public String t;
public String name; public String n;
public Service service; public int c;
public String identifier; public String l;
public String link;
public ArbitraryDataIndex() {} public ArbitraryDataIndex() {}
public ArbitraryDataIndex(String term, String name, Service service, String identifier, String link) { public ArbitraryDataIndex(String t, String n, int c, String l) {
this.term = term; this.t = t;
this.name = name; this.n = n;
this.service = service; this.c = c;
this.identifier = identifier; this.l = l;
this.link = link;
} }
@Override @Override
public String toString() { public String toString() {
return "ArbitraryDataIndex{" + return "ArbitraryDataIndex{" +
"term='" + term + '\'' + "t='" + t + '\'' +
", name='" + name + '\'' + ", n='" + n + '\'' +
", service=" + service + ", c=" + c +
", identifier='" + identifier + '\'' + ", l='" + l + '\'' +
", link='" + link + '\'' +
'}'; '}';
} }
} }

View File

@ -1,7 +1,5 @@
package org.qortal.data.arbitrary; package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlAccessorType;
@ -12,20 +10,20 @@ public class ArbitraryDataIndexDetail {
public int rank; public int rank;
public String term; public String term;
public String name; public String name;
public Service service; public int category;
public String identifier;
public String link; public String link;
public String indexIdentifer;
public ArbitraryDataIndexDetail() {} public ArbitraryDataIndexDetail() {}
public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index) { public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index, String indexIdentifer) {
this.issuer = issuer; this.issuer = issuer;
this.rank = rank; this.rank = rank;
this.term = index.term; this.term = index.t;
this.name = index.name; this.name = index.n;
this.service = index.service; this.category = index.c;
this.identifier = index.identifier; this.link = index.l;
this.link = index.link; this.indexIdentifer = indexIdentifer;
} }
@Override @Override
@ -35,9 +33,9 @@ public class ArbitraryDataIndexDetail {
", rank=" + rank + ", rank=" + rank +
", term='" + term + '\'' + ", term='" + term + '\'' +
", name='" + name + '\'' + ", name='" + name + '\'' +
", service=" + service + ", category=" + category +
", identifier='" + identifier + '\'' +
", link='" + link + '\'' + ", link='" + link + '\'' +
", indexIdentifer='" + indexIdentifer + '\'' +
'}'; '}';
} }
} }

View File

@ -10,14 +10,14 @@ import java.util.Objects;
public class ArbitraryDataIndexScoreKey { public class ArbitraryDataIndexScoreKey {
public String name; public String name;
public Service service; public int category;
public String link; public String link;
public ArbitraryDataIndexScoreKey() {} public ArbitraryDataIndexScoreKey() {}
public ArbitraryDataIndexScoreKey(String name, Service service, String link) { public ArbitraryDataIndexScoreKey(String name, int category, String link) {
this.name = name; this.name = name;
this.service = service; this.category = category;
this.link = link; this.link = link;
} }
@ -26,20 +26,13 @@ public class ArbitraryDataIndexScoreKey {
if (this == o) return true; if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false; if (o == null || getClass() != o.getClass()) return false;
ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o; ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o;
return Objects.equals(name, that.name) && service == that.service && Objects.equals(link, that.link); return category == that.category && Objects.equals(name, that.name) && Objects.equals(link, that.link);
} }
@Override @Override
public int hashCode() { public int hashCode() {
return Objects.hash(name, service, link); return Objects.hash(name, category, link);
} }
@Override
public String toString() {
return "ArbitraryDataIndexScoreKey{" +
"name='" + name + '\'' +
", service=" + service +
", link='" + link + '\'' +
'}';
}
} }

View File

@ -10,15 +10,15 @@ public class ArbitraryDataIndexScorecard {
public double score; public double score;
public String name; public String name;
public Service service; public int category;
public String link; public String link;
public ArbitraryDataIndexScorecard() {} public ArbitraryDataIndexScorecard() {}
public ArbitraryDataIndexScorecard(double score, String name, Service service, String link) { public ArbitraryDataIndexScorecard(double score, String name, int category, String link) {
this.score = score; this.score = score;
this.name = name; this.name = name;
this.service = service; this.category = category;
this.link = link; this.link = link;
} }
@ -31,7 +31,7 @@ public class ArbitraryDataIndexScorecard {
return "ArbitraryDataIndexScorecard{" + return "ArbitraryDataIndexScorecard{" +
"score=" + score + "score=" + score +
", name='" + name + '\'' + ", name='" + name + '\'' +
", service=" + service + ", category=" + category +
", link='" + link + '\'' + ", link='" + link + '\'' +
'}'; '}';
} }

View File

@ -527,7 +527,7 @@ public class Settings {
* *
* The frequency at which the arbitrary indices are cached. * The frequency at which the arbitrary indices are cached.
*/ */
private int arbitraryIndexingFrequency = 1; private int arbitraryIndexingFrequency = 10;
private boolean rebuildArbitraryResourceCacheTaskEnabled = false; private boolean rebuildArbitraryResourceCacheTaskEnabled = false;

View File

@ -3,6 +3,7 @@ package org.qortal.utils;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.InvalidFormatException; import com.fasterxml.jackson.databind.exc.InvalidFormatException;
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
@ -108,10 +109,12 @@ public class ArbitraryIndexUtils {
// rank and create index detail for each index in this index resource // rank and create index detail for each index in this index resource
for( int rank = 1; rank <= indices.size(); rank++ ) { for( int rank = 1; rank <= indices.size(); rank++ ) {
indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1) )); indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1), indexResource.identifier ));
} }
} catch (InvalidFormatException e) { } catch (InvalidFormatException e) {
LOGGER.debug("invalid format, skipping: " + indexResource); LOGGER.debug("invalid format, skipping: " + indexResource);
} catch (UnrecognizedPropertyException e) {
LOGGER.debug("unrecognized property, skipping " + indexResource);
} }
} }
@ -137,6 +140,29 @@ public class ArbitraryIndexUtils {
} }
LOGGER.info("loaded indices by term"); LOGGER.info("loaded indices by term");
LOGGER.debug("processing indices by issuer ...");
Map<String, List<ArbitraryDataIndexDetail>> indicesByIssuer
= indexDetails.stream().collect(
Collectors.toMap(
detail -> detail.issuer, // map by issuer
detail -> List.of(detail), // create list for issuer
(list1, list2) // merge lists for same issuer
-> Stream.of(list1, list2)
.flatMap(List::stream)
.collect(Collectors.toList())
)
);
LOGGER.info("processed indices by issuer: count = " + indicesByIssuer.size());
// lock, clear old, load new
synchronized( IndexCache.getInstance().getIndicesByIssuer() ) {
IndexCache.getInstance().getIndicesByIssuer().clear();
IndexCache.getInstance().getIndicesByIssuer().putAll(indicesByIssuer);
}
LOGGER.info("loaded indices by issuer");
} }
} }