forked from Qortal/qortal
Connected the rest of the system up to the recently added "identifier" feature.
This commit is contained in:
parent
a364206159
commit
4b1a5a5e14
@ -290,39 +290,34 @@ public class ArbitraryResource {
|
|||||||
@QueryParam("rebuild") boolean rebuild) {
|
@QueryParam("rebuild") boolean rebuild) {
|
||||||
Security.checkApiCallAllowed(request);
|
Security.checkApiCallAllowed(request);
|
||||||
|
|
||||||
if (filepath == null) {
|
return this.download(serviceString, name, null, filepath, rebuild);
|
||||||
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing filepath");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Service service = Service.valueOf(serviceString);
|
@GET
|
||||||
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service);
|
@Path("/{service}/{name}/{identifier}")
|
||||||
try {
|
@Operation(
|
||||||
|
summary = "Fetch raw data from file with supplied service, name, identifier, and relative path",
|
||||||
|
description = "An optional rebuild boolean can be supplied. If true, any existing cached data will be invalidated.",
|
||||||
|
responses = {
|
||||||
|
@ApiResponse(
|
||||||
|
description = "Path to file structure containing requested data",
|
||||||
|
content = @Content(
|
||||||
|
mediaType = MediaType.TEXT_PLAIN,
|
||||||
|
schema = @Schema(
|
||||||
|
type = "string"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
public HttpServletResponse get(@PathParam("service") String serviceString,
|
||||||
|
@PathParam("name") String name,
|
||||||
|
@PathParam("identifier") String identifier,
|
||||||
|
@QueryParam("filepath") String filepath,
|
||||||
|
@QueryParam("rebuild") boolean rebuild) {
|
||||||
|
Security.checkApiCallAllowed(request);
|
||||||
|
|
||||||
// Loop until we have data
|
return this.download(serviceString, name, identifier, filepath, rebuild);
|
||||||
while (!Controller.isStopping()) {
|
|
||||||
try {
|
|
||||||
arbitraryDataReader.loadSynchronously(rebuild);
|
|
||||||
break;
|
|
||||||
} catch (MissingDataException e) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO: limit file size that can be read into memory
|
|
||||||
java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath);
|
|
||||||
if (!Files.exists(path)) {
|
|
||||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
|
|
||||||
}
|
|
||||||
byte[] data = Files.readAllBytes(path);
|
|
||||||
response.setContentType(context.getMimeType(path.toString()));
|
|
||||||
response.setContentLength(data.length);
|
|
||||||
response.getOutputStream().write(data);
|
|
||||||
|
|
||||||
return response;
|
|
||||||
} catch (Exception e) {
|
|
||||||
LOGGER.info(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
|
|
||||||
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@POST
|
@POST
|
||||||
@ -432,6 +427,7 @@ public class ArbitraryResource {
|
|||||||
return this.upload(Method.PATCH, Service.valueOf(serviceString), name, identifier, path);
|
return this.upload(Method.PATCH, Service.valueOf(serviceString), name, identifier, path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private String upload(Method method, Service service, String name, String identifier, String path) {
|
private String upload(Method method, Service service, String name, String identifier, String path) {
|
||||||
// It's too dangerous to allow user-supplied file paths in weaker security contexts
|
// It's too dangerous to allow user-supplied file paths in weaker security contexts
|
||||||
if (Settings.getInstance().isApiRestricted()) {
|
if (Settings.getInstance().isApiRestricted()) {
|
||||||
@ -470,6 +466,43 @@ public class ArbitraryResource {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private HttpServletResponse download(String serviceString, String name, String identifier, String filepath, boolean rebuild) {
|
||||||
|
|
||||||
|
if (filepath == null) {
|
||||||
|
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing filepath");
|
||||||
|
}
|
||||||
|
|
||||||
|
Service service = Service.valueOf(serviceString);
|
||||||
|
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
|
||||||
|
try {
|
||||||
|
|
||||||
|
// Loop until we have data
|
||||||
|
while (!Controller.isStopping()) {
|
||||||
|
try {
|
||||||
|
arbitraryDataReader.loadSynchronously(rebuild);
|
||||||
|
break;
|
||||||
|
} catch (MissingDataException e) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: limit file size that can be read into memory
|
||||||
|
java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath);
|
||||||
|
if (!Files.exists(path)) {
|
||||||
|
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
|
||||||
|
}
|
||||||
|
byte[] data = Files.readAllBytes(path);
|
||||||
|
response.setContentType(context.getMimeType(path.toString()));
|
||||||
|
response.setContentLength(data.length);
|
||||||
|
response.getOutputStream().write(data);
|
||||||
|
|
||||||
|
return response;
|
||||||
|
} catch (Exception e) {
|
||||||
|
LOGGER.info(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
|
||||||
|
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@DELETE
|
@DELETE
|
||||||
@Path("/file")
|
@Path("/file")
|
||||||
|
@ -93,7 +93,7 @@ public class WebsiteResource {
|
|||||||
Method method = Method.PUT;
|
Method method = Method.PUT;
|
||||||
Compression compression = Compression.ZIP;
|
Compression compression = Compression.ZIP;
|
||||||
|
|
||||||
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, method, compression);
|
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, null, method, compression);
|
||||||
try {
|
try {
|
||||||
arbitraryDataWriter.save();
|
arbitraryDataWriter.save();
|
||||||
} catch (IOException | DataException | InterruptedException | MissingDataException e) {
|
} catch (IOException | DataException | InterruptedException | MissingDataException e) {
|
||||||
@ -178,7 +178,7 @@ public class WebsiteResource {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Service service = Service.WEBSITE;
|
Service service = Service.WEBSITE;
|
||||||
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service);
|
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, null);
|
||||||
arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
|
arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
|
||||||
try {
|
try {
|
||||||
if (!arbitraryDataReader.isCachedDataAvailable()) {
|
if (!arbitraryDataReader.isCachedDataAvailable()) {
|
||||||
|
@ -13,6 +13,7 @@ public class ArbitraryDataBuildQueueItem {
|
|||||||
private String resourceId;
|
private String resourceId;
|
||||||
private ResourceIdType resourceIdType;
|
private ResourceIdType resourceIdType;
|
||||||
private Service service;
|
private Service service;
|
||||||
|
private String identifier;
|
||||||
private Long creationTimestamp = null;
|
private Long creationTimestamp = null;
|
||||||
private Long buildStartTimestamp = null;
|
private Long buildStartTimestamp = null;
|
||||||
private Long buildEndTimestamp = null;
|
private Long buildEndTimestamp = null;
|
||||||
@ -24,10 +25,11 @@ public class ArbitraryDataBuildQueueItem {
|
|||||||
/* The amount of time to remember that a build has failed, to avoid retries */
|
/* The amount of time to remember that a build has failed, to avoid retries */
|
||||||
public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes
|
public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes
|
||||||
|
|
||||||
public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) {
|
public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
|
||||||
this.resourceId = resourceId.toLowerCase();
|
this.resourceId = resourceId.toLowerCase();
|
||||||
this.resourceIdType = resourceIdType;
|
this.resourceIdType = resourceIdType;
|
||||||
this.service = service;
|
this.service = service;
|
||||||
|
this.identifier = identifier;
|
||||||
this.creationTimestamp = NTP.getTime();
|
this.creationTimestamp = NTP.getTime();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -39,7 +41,7 @@ public class ArbitraryDataBuildQueueItem {
|
|||||||
|
|
||||||
this.buildStartTimestamp = now;
|
this.buildStartTimestamp = now;
|
||||||
ArbitraryDataReader arbitraryDataReader =
|
ArbitraryDataReader arbitraryDataReader =
|
||||||
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service);
|
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
arbitraryDataReader.loadSynchronously(true);
|
arbitraryDataReader.loadSynchronously(true);
|
||||||
@ -86,7 +88,7 @@ public class ArbitraryDataBuildQueueItem {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public String toString() {
|
||||||
return String.format("%s %s", this.service, this.resourceId);
|
return String.format("%s %s %s", this.service, this.resourceId, this.identifier);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -28,6 +28,7 @@ public class ArbitraryDataBuilder {
|
|||||||
|
|
||||||
private String name;
|
private String name;
|
||||||
private Service service;
|
private Service service;
|
||||||
|
private String identifier;
|
||||||
|
|
||||||
private List<ArbitraryTransactionData> transactions;
|
private List<ArbitraryTransactionData> transactions;
|
||||||
private ArbitraryTransactionData latestPutTransaction;
|
private ArbitraryTransactionData latestPutTransaction;
|
||||||
@ -35,9 +36,10 @@ public class ArbitraryDataBuilder {
|
|||||||
private byte[] latestSignature;
|
private byte[] latestSignature;
|
||||||
private Path finalPath;
|
private Path finalPath;
|
||||||
|
|
||||||
public ArbitraryDataBuilder(String name, Service service) {
|
public ArbitraryDataBuilder(String name, Service service, String identifier) {
|
||||||
this.name = name;
|
this.name = name;
|
||||||
this.service = service;
|
this.service = service;
|
||||||
|
this.identifier = identifier;
|
||||||
this.paths = new ArrayList<>();
|
this.paths = new ArrayList<>();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -56,16 +58,17 @@ public class ArbitraryDataBuilder {
|
|||||||
|
|
||||||
// Get the most recent PUT
|
// Get the most recent PUT
|
||||||
ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
|
ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
|
||||||
.getLatestTransaction(this.name, this.service, Method.PUT);
|
.getLatestTransaction(this.name, this.service, Method.PUT, this.identifier);
|
||||||
if (latestPut == null) {
|
if (latestPut == null) {
|
||||||
throw new IllegalStateException(String.format(
|
String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s",
|
||||||
"Couldn't find PUT transaction for name %s and service %s", this.name, this.service));
|
this.name, this.service, this.identifierString());
|
||||||
|
throw new IllegalStateException(message);
|
||||||
}
|
}
|
||||||
this.latestPutTransaction = latestPut;
|
this.latestPutTransaction = latestPut;
|
||||||
|
|
||||||
// Load all transactions since the latest PUT
|
// Load all transactions since the latest PUT
|
||||||
List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
|
List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
|
||||||
.getArbitraryTransactions(this.name, this.service, latestPut.getTimestamp());
|
.getArbitraryTransactions(this.name, this.service, this.identifier, latestPut.getTimestamp());
|
||||||
this.transactions = transactionDataList;
|
this.transactions = transactionDataList;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -81,8 +84,8 @@ public class ArbitraryDataBuilder {
|
|||||||
throw new IllegalStateException("Expected PUT but received PATCH");
|
throw new IllegalStateException("Expected PUT but received PATCH");
|
||||||
}
|
}
|
||||||
if (transactionDataList.size() == 0) {
|
if (transactionDataList.size() == 0) {
|
||||||
throw new IllegalStateException(String.format("No transactions found for name %s, service %s, since %d",
|
throw new IllegalStateException(String.format("No transactions found for name %s, service %s, " +
|
||||||
name, service, latestPut.getTimestamp()));
|
"identifier: %s, since %d", name, service, this.identifierString(), latestPut.getTimestamp()));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify that the signature of the first transaction matches the latest PUT
|
// Verify that the signature of the first transaction matches the latest PUT
|
||||||
@ -115,7 +118,8 @@ public class ArbitraryDataBuilder {
|
|||||||
|
|
||||||
// Build the data file, overwriting anything that was previously there
|
// Build the data file, overwriting anything that was previously there
|
||||||
String sig58 = Base58.encode(transactionData.getSignature());
|
String sig58 = Base58.encode(transactionData.getSignature());
|
||||||
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service);
|
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA,
|
||||||
|
this.service, this.identifier);
|
||||||
arbitraryDataReader.setTransactionData(transactionData);
|
arbitraryDataReader.setTransactionData(transactionData);
|
||||||
boolean hasMissingData = false;
|
boolean hasMissingData = false;
|
||||||
try {
|
try {
|
||||||
@ -179,7 +183,8 @@ public class ArbitraryDataBuilder {
|
|||||||
|
|
||||||
// Loop from the second path onwards
|
// Loop from the second path onwards
|
||||||
for (int i=1; i<paths.size(); i++) {
|
for (int i=1; i<paths.size(); i++) {
|
||||||
LOGGER.info(String.format("[%s][%s] Applying layer %d...", this.service, this.name, i));
|
String identifierPrefix = this.identifier != null ? String.format("[%s]", this.identifier) : "";
|
||||||
|
LOGGER.info(String.format("[%s][%s]%s Applying layer %d...", this.service, this.name, identifierPrefix, i));
|
||||||
|
|
||||||
// Create an instance of ArbitraryDataCombiner
|
// Create an instance of ArbitraryDataCombiner
|
||||||
Path pathAfter = this.paths.get(i);
|
Path pathAfter = this.paths.get(i);
|
||||||
@ -211,6 +216,10 @@ public class ArbitraryDataBuilder {
|
|||||||
cache.write();
|
cache.write();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private String identifierString() {
|
||||||
|
return identifier != null ? identifier : "";
|
||||||
|
}
|
||||||
|
|
||||||
public Path getFinalPath() {
|
public Path getFinalPath() {
|
||||||
return this.finalPath;
|
return this.finalPath;
|
||||||
}
|
}
|
||||||
|
@ -22,14 +22,16 @@ public class ArbitraryDataCache {
|
|||||||
private String resourceId;
|
private String resourceId;
|
||||||
private ResourceIdType resourceIdType;
|
private ResourceIdType resourceIdType;
|
||||||
private Service service;
|
private Service service;
|
||||||
|
private String identifier;
|
||||||
|
|
||||||
public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId,
|
public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId,
|
||||||
ResourceIdType resourceIdType, Service service) {
|
ResourceIdType resourceIdType, Service service, String identifier) {
|
||||||
this.filePath = filePath;
|
this.filePath = filePath;
|
||||||
this.overwrite = overwrite;
|
this.overwrite = overwrite;
|
||||||
this.resourceId = resourceId;
|
this.resourceId = resourceId;
|
||||||
this.resourceIdType = resourceIdType;
|
this.resourceIdType = resourceIdType;
|
||||||
this.service = service;
|
this.service = service;
|
||||||
|
this.identifier = identifier;
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isCachedDataAvailable() {
|
public boolean isCachedDataAvailable() {
|
||||||
@ -134,7 +136,7 @@ public class ArbitraryDataCache {
|
|||||||
|
|
||||||
// Find latest transaction for name and service, with any method
|
// Find latest transaction for name and service, with any method
|
||||||
ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
|
ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
|
||||||
.getLatestTransaction(this.resourceId, this.service, null);
|
.getLatestTransaction(this.resourceId, this.service, null, this.identifier);
|
||||||
|
|
||||||
if (latestTransaction != null) {
|
if (latestTransaction != null) {
|
||||||
return latestTransaction.getSignature();
|
return latestTransaction.getSignature();
|
||||||
|
@ -42,6 +42,7 @@ public class ArbitraryDataReader {
|
|||||||
private String resourceId;
|
private String resourceId;
|
||||||
private ResourceIdType resourceIdType;
|
private ResourceIdType resourceIdType;
|
||||||
private Service service;
|
private Service service;
|
||||||
|
private String identifier;
|
||||||
private ArbitraryTransactionData transactionData;
|
private ArbitraryTransactionData transactionData;
|
||||||
private String secret58;
|
private String secret58;
|
||||||
private Path filePath;
|
private Path filePath;
|
||||||
@ -51,7 +52,7 @@ public class ArbitraryDataReader {
|
|||||||
private Path uncompressedPath;
|
private Path uncompressedPath;
|
||||||
private Path unencryptedPath;
|
private Path unencryptedPath;
|
||||||
|
|
||||||
public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service) {
|
public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
|
||||||
// Ensure names are always lowercase
|
// Ensure names are always lowercase
|
||||||
if (resourceIdType == ResourceIdType.NAME) {
|
if (resourceIdType == ResourceIdType.NAME) {
|
||||||
resourceId = resourceId.toLowerCase();
|
resourceId = resourceId.toLowerCase();
|
||||||
@ -60,25 +61,31 @@ public class ArbitraryDataReader {
|
|||||||
this.resourceId = resourceId;
|
this.resourceId = resourceId;
|
||||||
this.resourceIdType = resourceIdType;
|
this.resourceIdType = resourceIdType;
|
||||||
this.service = service;
|
this.service = service;
|
||||||
|
this.identifier = identifier;
|
||||||
|
|
||||||
|
this.workingPath = this.buildWorkingPath();
|
||||||
|
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
|
||||||
|
}
|
||||||
|
|
||||||
|
private Path buildWorkingPath() {
|
||||||
// Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
|
// Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
|
||||||
String baseDir = Settings.getInstance().getTempDataPath();
|
String baseDir = Settings.getInstance().getTempDataPath();
|
||||||
this.workingPath = Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString());
|
String identifier = this.identifier != null ? this.identifier : "default";
|
||||||
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
|
return Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
|
||||||
}
|
}
|
||||||
|
|
||||||
public boolean isCachedDataAvailable() {
|
public boolean isCachedDataAvailable() {
|
||||||
// If this resource is in the build queue then we shouldn't attempt to serve
|
// If this resource is in the build queue then we shouldn't attempt to serve
|
||||||
// cached data, as it may not be fully built
|
// cached data, as it may not be fully built
|
||||||
ArbitraryDataBuildQueueItem queueItem =
|
ArbitraryDataBuildQueueItem queueItem =
|
||||||
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
|
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service, this.identifier);
|
||||||
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
|
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Not in the build queue - so check the cache itself
|
// Not in the build queue - so check the cache itself
|
||||||
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false,
|
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false,
|
||||||
this.resourceId, this.resourceIdType, this.service);
|
this.resourceId, this.resourceIdType, this.service, this.identifier);
|
||||||
if (cache.isCachedDataAvailable()) {
|
if (cache.isCachedDataAvailable()) {
|
||||||
this.filePath = this.uncompressedPath;
|
this.filePath = this.uncompressedPath;
|
||||||
return true;
|
return true;
|
||||||
@ -98,7 +105,7 @@ public class ArbitraryDataReader {
|
|||||||
*/
|
*/
|
||||||
public boolean loadAsynchronously() {
|
public boolean loadAsynchronously() {
|
||||||
ArbitraryDataBuildQueueItem queueItem =
|
ArbitraryDataBuildQueueItem queueItem =
|
||||||
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
|
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service, this.identifier);
|
||||||
return ArbitraryDataBuildManager.getInstance().addToBuildQueue(queueItem);
|
return ArbitraryDataBuildManager.getInstance().addToBuildQueue(queueItem);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -117,7 +124,7 @@ public class ArbitraryDataReader {
|
|||||||
public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException, MissingDataException {
|
public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException, MissingDataException {
|
||||||
try {
|
try {
|
||||||
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
|
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
|
||||||
this.resourceId, this.resourceIdType, this.service);
|
this.resourceId, this.resourceIdType, this.service, this.identifier);
|
||||||
if (cache.isCachedDataAvailable()) {
|
if (cache.isCachedDataAvailable()) {
|
||||||
// Use cached data
|
// Use cached data
|
||||||
this.filePath = this.uncompressedPath;
|
this.filePath = this.uncompressedPath;
|
||||||
@ -233,7 +240,7 @@ public class ArbitraryDataReader {
|
|||||||
try {
|
try {
|
||||||
|
|
||||||
// Build the existing state using past transactions
|
// Build the existing state using past transactions
|
||||||
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service);
|
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service, this.identifier);
|
||||||
builder.build();
|
builder.build();
|
||||||
Path builtPath = builder.getFinalPath();
|
Path builtPath = builder.getFinalPath();
|
||||||
if (builtPath == null) {
|
if (builtPath == null) {
|
||||||
|
@ -15,9 +15,7 @@ import org.qortal.repository.Repository;
|
|||||||
import org.qortal.repository.RepositoryManager;
|
import org.qortal.repository.RepositoryManager;
|
||||||
import org.qortal.transaction.ArbitraryTransaction;
|
import org.qortal.transaction.ArbitraryTransaction;
|
||||||
import org.qortal.transaction.Transaction;
|
import org.qortal.transaction.Transaction;
|
||||||
import org.qortal.transform.TransformationException;
|
|
||||||
import org.qortal.transform.Transformer;
|
import org.qortal.transform.Transformer;
|
||||||
import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
|
|
||||||
import org.qortal.utils.Base58;
|
import org.qortal.utils.Base58;
|
||||||
import org.qortal.utils.NTP;
|
import org.qortal.utils.NTP;
|
||||||
|
|
||||||
@ -69,7 +67,7 @@ public class ArbitraryDataTransactionBuilder {
|
|||||||
|
|
||||||
ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP;
|
ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP;
|
||||||
|
|
||||||
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, method, compression);
|
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, compression);
|
||||||
try {
|
try {
|
||||||
arbitraryDataWriter.save();
|
arbitraryDataWriter.save();
|
||||||
} catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) {
|
} catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) {
|
||||||
|
@ -34,6 +34,7 @@ public class ArbitraryDataWriter {
|
|||||||
private Path filePath;
|
private Path filePath;
|
||||||
private String name;
|
private String name;
|
||||||
private Service service;
|
private Service service;
|
||||||
|
private String identifier;
|
||||||
private Method method;
|
private Method method;
|
||||||
private Compression compression;
|
private Compression compression;
|
||||||
|
|
||||||
@ -45,10 +46,11 @@ public class ArbitraryDataWriter {
|
|||||||
private Path compressedPath;
|
private Path compressedPath;
|
||||||
private Path encryptedPath;
|
private Path encryptedPath;
|
||||||
|
|
||||||
public ArbitraryDataWriter(Path filePath, String name, Service service, Method method, Compression compression) {
|
public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression) {
|
||||||
this.filePath = filePath;
|
this.filePath = filePath;
|
||||||
this.name = name;
|
this.name = name;
|
||||||
this.service = service;
|
this.service = service;
|
||||||
|
this.identifier = identifier;
|
||||||
this.method = method;
|
this.method = method;
|
||||||
this.compression = compression;
|
this.compression = compression;
|
||||||
}
|
}
|
||||||
@ -114,7 +116,7 @@ public class ArbitraryDataWriter {
|
|||||||
private void processPatch() throws DataException, IOException, MissingDataException {
|
private void processPatch() throws DataException, IOException, MissingDataException {
|
||||||
|
|
||||||
// Build the existing state using past transactions
|
// Build the existing state using past transactions
|
||||||
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service);
|
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service, this.identifier);
|
||||||
builder.build();
|
builder.build();
|
||||||
Path builtPath = builder.getFinalPath();
|
Path builtPath = builder.getFinalPath();
|
||||||
|
|
||||||
|
@ -10,6 +10,7 @@ public class ArbitraryResourceInfo {
|
|||||||
|
|
||||||
public String name;
|
public String name;
|
||||||
public ArbitraryTransactionData.Service service;
|
public ArbitraryTransactionData.Service service;
|
||||||
|
public String identifier;
|
||||||
|
|
||||||
public ArbitraryResourceInfo() {
|
public ArbitraryResourceInfo() {
|
||||||
}
|
}
|
||||||
|
@ -17,9 +17,9 @@ public interface ArbitraryRepository {
|
|||||||
|
|
||||||
public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
|
public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
|
||||||
|
|
||||||
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, long since) throws DataException;
|
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
|
||||||
|
|
||||||
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException;
|
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
|
||||||
|
|
||||||
|
|
||||||
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException;
|
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException;
|
||||||
|
@ -153,17 +153,18 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, long since) throws DataException {
|
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException {
|
||||||
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
|
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
|
||||||
"tx_group_id, block_height, approval_status, approval_height, " +
|
"tx_group_id, block_height, approval_status, approval_height, " +
|
||||||
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
|
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
|
||||||
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
|
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
|
||||||
"JOIN Transactions USING (signature) " +
|
"JOIN Transactions USING (signature) " +
|
||||||
"WHERE lower(name) = ? AND service = ? AND created_when >= ? " +
|
"WHERE lower(name) = ? AND service = ?" +
|
||||||
"ORDER BY created_when ASC";
|
"AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))" +
|
||||||
|
"AND created_when >= ? ORDER BY created_when ASC";
|
||||||
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
|
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
|
||||||
|
|
||||||
try (ResultSet resultSet = this.repository.checkedExecute(sql, name.toLowerCase(), service.value, since)) {
|
try (ResultSet resultSet = this.repository.checkedExecute(sql, name.toLowerCase(), service.value, identifier, identifier, since)) {
|
||||||
if (resultSet == null)
|
if (resultSet == null)
|
||||||
return null;
|
return null;
|
||||||
|
|
||||||
@ -221,7 +222,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException {
|
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
|
||||||
StringBuilder sql = new StringBuilder(1024);
|
StringBuilder sql = new StringBuilder(1024);
|
||||||
|
|
||||||
sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
|
sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
|
||||||
@ -229,7 +230,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
|||||||
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
|
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
|
||||||
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
|
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
|
||||||
"JOIN Transactions USING (signature) " +
|
"JOIN Transactions USING (signature) " +
|
||||||
"WHERE lower(name) = ? AND service = ?");
|
"WHERE lower(name) = ? AND service = ? " +
|
||||||
|
"AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))");
|
||||||
|
|
||||||
if (method != null) {
|
if (method != null) {
|
||||||
sql.append(" AND update_method = ");
|
sql.append(" AND update_method = ");
|
||||||
@ -238,7 +240,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
|||||||
|
|
||||||
sql.append("ORDER BY created_when DESC LIMIT 1");
|
sql.append("ORDER BY created_when DESC LIMIT 1");
|
||||||
|
|
||||||
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value)) {
|
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) {
|
||||||
if (resultSet == null)
|
if (resultSet == null)
|
||||||
return null;
|
return null;
|
||||||
|
|
||||||
@ -295,14 +297,14 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
|||||||
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException {
|
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException {
|
||||||
StringBuilder sql = new StringBuilder(512);
|
StringBuilder sql = new StringBuilder(512);
|
||||||
|
|
||||||
sql.append("SELECT name, service FROM ArbitraryTransactions");
|
sql.append("SELECT name, service, identifier FROM ArbitraryTransactions");
|
||||||
|
|
||||||
if (service != null) {
|
if (service != null) {
|
||||||
sql.append(" WHERE service = ");
|
sql.append(" WHERE service = ");
|
||||||
sql.append(service.value);
|
sql.append(service.value);
|
||||||
}
|
}
|
||||||
|
|
||||||
sql.append(" GROUP BY name, service ORDER BY name");
|
sql.append(" GROUP BY name, service, identifier ORDER BY name");
|
||||||
|
|
||||||
if (reverse != null && reverse) {
|
if (reverse != null && reverse) {
|
||||||
sql.append(" DESC");
|
sql.append(" DESC");
|
||||||
@ -319,6 +321,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
|||||||
do {
|
do {
|
||||||
String name = resultSet.getString(1);
|
String name = resultSet.getString(1);
|
||||||
Service serviceResult = Service.valueOf(resultSet.getInt(2));
|
Service serviceResult = Service.valueOf(resultSet.getInt(2));
|
||||||
|
String identifier = resultSet.getString(3);
|
||||||
|
|
||||||
// We should filter out resources without names
|
// We should filter out resources without names
|
||||||
if (name == null) {
|
if (name == null) {
|
||||||
@ -328,6 +331,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
|||||||
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
|
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
|
||||||
arbitraryResourceInfo.name = name;
|
arbitraryResourceInfo.name = name;
|
||||||
arbitraryResourceInfo.service = serviceResult;
|
arbitraryResourceInfo.service = serviceResult;
|
||||||
|
arbitraryResourceInfo.identifier = identifier;
|
||||||
|
|
||||||
arbitraryResources.add(arbitraryResourceInfo);
|
arbitraryResources.add(arbitraryResourceInfo);
|
||||||
} while (resultSet.next());
|
} while (resultSet.next());
|
||||||
|
@ -39,6 +39,7 @@ public class ArbitraryTransactionUtils {
|
|||||||
|
|
||||||
String name = arbitraryTransactionData.getName();
|
String name = arbitraryTransactionData.getName();
|
||||||
ArbitraryTransactionData.Service service = arbitraryTransactionData.getService();
|
ArbitraryTransactionData.Service service = arbitraryTransactionData.getService();
|
||||||
|
String identifier = arbitraryTransactionData.getIdentifier();
|
||||||
|
|
||||||
if (name == null || service == null) {
|
if (name == null || service == null) {
|
||||||
return null;
|
return null;
|
||||||
@ -48,7 +49,7 @@ public class ArbitraryTransactionUtils {
|
|||||||
ArbitraryTransactionData latestPut;
|
ArbitraryTransactionData latestPut;
|
||||||
try {
|
try {
|
||||||
latestPut = repository.getArbitraryRepository()
|
latestPut = repository.getArbitraryRepository()
|
||||||
.getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT);
|
.getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT, identifier);
|
||||||
} catch (DataException e) {
|
} catch (DataException e) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
@ -63,7 +63,7 @@ public class ArbitraryDataTests extends Common {
|
|||||||
this.createAndMintTxn(repository, publicKey58, path3, name, identifier, Method.PATCH, service, alice);
|
this.createAndMintTxn(repository, publicKey58, path3, name, identifier, Method.PATCH, service, alice);
|
||||||
|
|
||||||
// Now build the latest data state for this name
|
// Now build the latest data state for this name
|
||||||
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service);
|
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
|
||||||
arbitraryDataReader.loadSynchronously(true);
|
arbitraryDataReader.loadSynchronously(true);
|
||||||
Path finalPath = arbitraryDataReader.getFilePath();
|
Path finalPath = arbitraryDataReader.getFilePath();
|
||||||
|
|
||||||
@ -102,7 +102,7 @@ public class ArbitraryDataTests extends Common {
|
|||||||
|
|
||||||
} catch (DataException expectedException) {
|
} catch (DataException expectedException) {
|
||||||
assertEquals(String.format("Couldn't find PUT transaction for " +
|
assertEquals(String.format("Couldn't find PUT transaction for " +
|
||||||
"name %s and service %s", name, service), expectedException.getMessage());
|
"name %s, service %s and identifier ", name, service), expectedException.getMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -181,7 +181,7 @@ public class ArbitraryDataTests extends Common {
|
|||||||
this.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice);
|
this.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice);
|
||||||
|
|
||||||
// Now build the latest data state for this name
|
// Now build the latest data state for this name
|
||||||
ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service);
|
ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
|
||||||
arbitraryDataReader1.loadSynchronously(true);
|
arbitraryDataReader1.loadSynchronously(true);
|
||||||
Path initialLayerPath = arbitraryDataReader1.getFilePath();
|
Path initialLayerPath = arbitraryDataReader1.getFilePath();
|
||||||
ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath);
|
ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath);
|
||||||
@ -192,7 +192,75 @@ public class ArbitraryDataTests extends Common {
|
|||||||
this.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice);
|
this.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice);
|
||||||
|
|
||||||
// Rebuild the latest state
|
// Rebuild the latest state
|
||||||
ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service);
|
ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
|
||||||
|
arbitraryDataReader2.loadSynchronously(false);
|
||||||
|
Path secondLayerPath = arbitraryDataReader2.getFilePath();
|
||||||
|
ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath);
|
||||||
|
secondLayerDigest.compute();
|
||||||
|
|
||||||
|
// Ensure that the second state is different to the first state
|
||||||
|
assertFalse(Arrays.equals(initialLayerDigest.getHash(), secondLayerDigest.getHash()));
|
||||||
|
|
||||||
|
// Its directory hash should match the hash of demo2
|
||||||
|
ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(path2);
|
||||||
|
path2Digest.compute();
|
||||||
|
assertEquals(path2Digest.getHash58(), secondLayerDigest.getHash58());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testIdentifier() throws DataException, IOException, MissingDataException {
|
||||||
|
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||||
|
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
|
||||||
|
String publicKey58 = Base58.encode(alice.getPublicKey());
|
||||||
|
String name = "TEST"; // Can be anything for this test
|
||||||
|
String identifier = "test_identifier";
|
||||||
|
Service service = Service.WEBSITE; // Can be anything for this test
|
||||||
|
|
||||||
|
// Register the name to Alice
|
||||||
|
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
|
||||||
|
TransactionUtils.signAndMint(repository, transactionData, alice);
|
||||||
|
|
||||||
|
// Create PUT transaction
|
||||||
|
Path path1 = Paths.get("src/test/resources/arbitrary/demo1");
|
||||||
|
this.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice);
|
||||||
|
|
||||||
|
// Build the latest data state for this name, with a null identifier, ensuring that it fails
|
||||||
|
ArbitraryDataReader arbitraryDataReader1a = new ArbitraryDataReader(name, ResourceIdType.NAME, service, null);
|
||||||
|
try {
|
||||||
|
arbitraryDataReader1a.loadSynchronously(true);
|
||||||
|
fail("Loading data with null identifier should fail due to nonexistent PUT transaction");
|
||||||
|
|
||||||
|
} catch (IllegalStateException expectedException) {
|
||||||
|
assertEquals(String.format("Couldn't find PUT transaction for name %s, service %s "
|
||||||
|
+ "and identifier ", name.toLowerCase(), service), expectedException.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the latest data state for this name, with a different identifier, ensuring that it fails
|
||||||
|
String differentIdentifier = "different_identifier";
|
||||||
|
ArbitraryDataReader arbitraryDataReader1b = new ArbitraryDataReader(name, ResourceIdType.NAME, service, differentIdentifier);
|
||||||
|
try {
|
||||||
|
arbitraryDataReader1b.loadSynchronously(true);
|
||||||
|
fail("Loading data with incorrect identifier should fail due to nonexistent PUT transaction");
|
||||||
|
|
||||||
|
} catch (IllegalStateException expectedException) {
|
||||||
|
assertEquals(String.format("Couldn't find PUT transaction for name %s, service %s "
|
||||||
|
+ "and identifier %s", name.toLowerCase(), service, differentIdentifier), expectedException.getMessage());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now build the latest data state for this name, with the correct identifier
|
||||||
|
ArbitraryDataReader arbitraryDataReader1c = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
|
||||||
|
arbitraryDataReader1c.loadSynchronously(true);
|
||||||
|
Path initialLayerPath = arbitraryDataReader1c.getFilePath();
|
||||||
|
ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath);
|
||||||
|
initialLayerDigest.compute();
|
||||||
|
|
||||||
|
// Create PATCH transaction
|
||||||
|
Path path2 = Paths.get("src/test/resources/arbitrary/demo2");
|
||||||
|
this.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice);
|
||||||
|
|
||||||
|
// Rebuild the latest state
|
||||||
|
ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
|
||||||
arbitraryDataReader2.loadSynchronously(false);
|
arbitraryDataReader2.loadSynchronously(false);
|
||||||
Path secondLayerPath = arbitraryDataReader2.getFilePath();
|
Path secondLayerPath = arbitraryDataReader2.getFilePath();
|
||||||
ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath);
|
ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath);
|
||||||
|
Loading…
x
Reference in New Issue
Block a user