mirror of
https://github.com/Qortal/qortal.git
synced 2025-04-16 16:15:53 +00:00
replaced index service attribute with a category attribute and reduced index attribute names to single characters to reduce memory footprint, t is for term, n is for name, c is for category, l if for link
changed default indexing frequency from 1 minute to 10 minutes to reduce memory use added arbitrary resource endpoint for index search by issuer name and index prefix added some additional error handling concerning unrecognized properties in the indices
This commit is contained in:
kennycud
parent
23de8a98bc
commit
a6309e925b
@ -1229,7 +1229,7 @@ public class ArbitraryResource {
|
||||
= indices.stream()
|
||||
.collect(
|
||||
Collectors.groupingBy(
|
||||
index -> new ArbitraryDataIndexScoreKey(index.name, index.service, index.link),
|
||||
index -> new ArbitraryDataIndexScoreKey(index.name, index.category, index.link),
|
||||
Collectors.summingDouble(detail -> 1.0 / detail.rank)
|
||||
)
|
||||
);
|
||||
@ -1242,7 +1242,7 @@ public class ArbitraryResource {
|
||||
new ArbitraryDataIndexScorecard(
|
||||
entry.getValue(),
|
||||
entry.getKey().name,
|
||||
entry.getKey().service,
|
||||
entry.getKey().category,
|
||||
entry.getKey().link)
|
||||
)
|
||||
.sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed())
|
||||
@ -1251,6 +1251,33 @@ public class ArbitraryResource {
|
||||
return scorecards;
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path("/indices/{name}/{idPrefix}")
|
||||
@Operation(
|
||||
summary = "Find matching arbitrary resource indices for a registered name and identifier prefix",
|
||||
description = "",
|
||||
responses = {
|
||||
@ApiResponse(
|
||||
description = "indices",
|
||||
content = @Content(
|
||||
array = @ArraySchema(
|
||||
schema = @Schema(
|
||||
implementation = ArbitraryDataIndexDetail.class
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
}
|
||||
)
|
||||
public List<ArbitraryDataIndexDetail> searchIndicesByName(@PathParam("name") String name, @PathParam("idPrefix") String idPrefix) {
|
||||
|
||||
return
|
||||
IndexCache.getInstance().getIndicesByIssuer()
|
||||
.getOrDefault(name, new ArrayList<>(0)).stream()
|
||||
.filter( indexDetail -> indexDetail.indexIdentifer.startsWith(idPrefix))
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
// Shared methods
|
||||
|
||||
private String preview(String directoryPath, Service service) {
|
||||
|
||||
@ -8,30 +8,27 @@ import javax.xml.bind.annotation.XmlAccessorType;
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class ArbitraryDataIndex {
|
||||
|
||||
public String term;
|
||||
public String name;
|
||||
public Service service;
|
||||
public String identifier;
|
||||
public String link;
|
||||
public String t;
|
||||
public String n;
|
||||
public int c;
|
||||
public String l;
|
||||
|
||||
public ArbitraryDataIndex() {}
|
||||
|
||||
public ArbitraryDataIndex(String term, String name, Service service, String identifier, String link) {
|
||||
this.term = term;
|
||||
this.name = name;
|
||||
this.service = service;
|
||||
this.identifier = identifier;
|
||||
this.link = link;
|
||||
public ArbitraryDataIndex(String t, String n, int c, String l) {
|
||||
this.t = t;
|
||||
this.n = n;
|
||||
this.c = c;
|
||||
this.l = l;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ArbitraryDataIndex{" +
|
||||
"term='" + term + '\'' +
|
||||
", name='" + name + '\'' +
|
||||
", service=" + service +
|
||||
", identifier='" + identifier + '\'' +
|
||||
", link='" + link + '\'' +
|
||||
"t='" + t + '\'' +
|
||||
", n='" + n + '\'' +
|
||||
", c=" + c +
|
||||
", l='" + l + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,7 +1,5 @@
|
||||
package org.qortal.data.arbitrary;
|
||||
|
||||
import org.qortal.arbitrary.misc.Service;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
|
||||
@ -12,20 +10,20 @@ public class ArbitraryDataIndexDetail {
|
||||
public int rank;
|
||||
public String term;
|
||||
public String name;
|
||||
public Service service;
|
||||
public String identifier;
|
||||
public int category;
|
||||
public String link;
|
||||
public String indexIdentifer;
|
||||
|
||||
public ArbitraryDataIndexDetail() {}
|
||||
|
||||
public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index) {
|
||||
public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index, String indexIdentifer) {
|
||||
this.issuer = issuer;
|
||||
this.rank = rank;
|
||||
this.term = index.term;
|
||||
this.name = index.name;
|
||||
this.service = index.service;
|
||||
this.identifier = index.identifier;
|
||||
this.link = index.link;
|
||||
this.term = index.t;
|
||||
this.name = index.n;
|
||||
this.category = index.c;
|
||||
this.link = index.l;
|
||||
this.indexIdentifer = indexIdentifer;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -35,9 +33,9 @@ public class ArbitraryDataIndexDetail {
|
||||
", rank=" + rank +
|
||||
", term='" + term + '\'' +
|
||||
", name='" + name + '\'' +
|
||||
", service=" + service +
|
||||
", identifier='" + identifier + '\'' +
|
||||
", category=" + category +
|
||||
", link='" + link + '\'' +
|
||||
", indexIdentifer='" + indexIdentifer + '\'' +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@ -10,14 +10,14 @@ import java.util.Objects;
|
||||
public class ArbitraryDataIndexScoreKey {
|
||||
|
||||
public String name;
|
||||
public Service service;
|
||||
public int category;
|
||||
public String link;
|
||||
|
||||
public ArbitraryDataIndexScoreKey() {}
|
||||
|
||||
public ArbitraryDataIndexScoreKey(String name, Service service, String link) {
|
||||
public ArbitraryDataIndexScoreKey(String name, int category, String link) {
|
||||
this.name = name;
|
||||
this.service = service;
|
||||
this.category = category;
|
||||
this.link = link;
|
||||
}
|
||||
|
||||
@ -26,20 +26,13 @@ public class ArbitraryDataIndexScoreKey {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o;
|
||||
return Objects.equals(name, that.name) && service == that.service && Objects.equals(link, that.link);
|
||||
return category == that.category && Objects.equals(name, that.name) && Objects.equals(link, that.link);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, service, link);
|
||||
return Objects.hash(name, category, link);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "ArbitraryDataIndexScoreKey{" +
|
||||
"name='" + name + '\'' +
|
||||
", service=" + service +
|
||||
", link='" + link + '\'' +
|
||||
'}';
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -10,15 +10,15 @@ public class ArbitraryDataIndexScorecard {
|
||||
|
||||
public double score;
|
||||
public String name;
|
||||
public Service service;
|
||||
public int category;
|
||||
public String link;
|
||||
|
||||
public ArbitraryDataIndexScorecard() {}
|
||||
|
||||
public ArbitraryDataIndexScorecard(double score, String name, Service service, String link) {
|
||||
public ArbitraryDataIndexScorecard(double score, String name, int category, String link) {
|
||||
this.score = score;
|
||||
this.name = name;
|
||||
this.service = service;
|
||||
this.category = category;
|
||||
this.link = link;
|
||||
}
|
||||
|
||||
@ -31,7 +31,7 @@ public class ArbitraryDataIndexScorecard {
|
||||
return "ArbitraryDataIndexScorecard{" +
|
||||
"score=" + score +
|
||||
", name='" + name + '\'' +
|
||||
", service=" + service +
|
||||
", category=" + category +
|
||||
", link='" + link + '\'' +
|
||||
'}';
|
||||
}
|
||||
|
||||
@ -527,7 +527,7 @@ public class Settings {
|
||||
*
|
||||
* The frequency at which the arbitrary indices are cached.
|
||||
*/
|
||||
private int arbitraryIndexingFrequency = 1;
|
||||
private int arbitraryIndexingFrequency = 10;
|
||||
|
||||
private boolean rebuildArbitraryResourceCacheTaskEnabled = false;
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ package org.qortal.utils;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.exc.InvalidFormatException;
|
||||
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
|
||||
import org.apache.commons.lang3.ArrayUtils;
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
@ -108,10 +109,12 @@ public class ArbitraryIndexUtils {
|
||||
// rank and create index detail for each index in this index resource
|
||||
for( int rank = 1; rank <= indices.size(); rank++ ) {
|
||||
|
||||
indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1) ));
|
||||
indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1), indexResource.identifier ));
|
||||
}
|
||||
} catch (InvalidFormatException e) {
|
||||
LOGGER.debug("invalid format, skipping: " + indexResource);
|
||||
} catch (UnrecognizedPropertyException e) {
|
||||
LOGGER.debug("unrecognized property, skipping " + indexResource);
|
||||
}
|
||||
}
|
||||
|
||||
@ -137,6 +140,29 @@ public class ArbitraryIndexUtils {
|
||||
}
|
||||
|
||||
LOGGER.info("loaded indices by term");
|
||||
|
||||
LOGGER.debug("processing indices by issuer ...");
|
||||
Map<String, List<ArbitraryDataIndexDetail>> indicesByIssuer
|
||||
= indexDetails.stream().collect(
|
||||
Collectors.toMap(
|
||||
detail -> detail.issuer, // map by issuer
|
||||
detail -> List.of(detail), // create list for issuer
|
||||
(list1, list2) // merge lists for same issuer
|
||||
-> Stream.of(list1, list2)
|
||||
.flatMap(List::stream)
|
||||
.collect(Collectors.toList())
|
||||
)
|
||||
);
|
||||
|
||||
LOGGER.info("processed indices by issuer: count = " + indicesByIssuer.size());
|
||||
|
||||
// lock, clear old, load new
|
||||
synchronized( IndexCache.getInstance().getIndicesByIssuer() ) {
|
||||
IndexCache.getInstance().getIndicesByIssuer().clear();
|
||||
IndexCache.getInstance().getIndicesByIssuer().putAll(indicesByIssuer);
|
||||
}
|
||||
|
||||
LOGGER.info("loaded indices by issuer");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user