Connected the rest of the system up to the recently added "identifier" feature.

This commit is contained in:
CalDescent 2021-11-11 09:12:54 +00:00
parent a364206159
commit 4b1a5a5e14
13 changed files with 202 additions and 75 deletions

View File

@ -290,39 +290,34 @@ public class ArbitraryResource {
@QueryParam("rebuild") boolean rebuild) { @QueryParam("rebuild") boolean rebuild) {
Security.checkApiCallAllowed(request); Security.checkApiCallAllowed(request);
if (filepath == null) { return this.download(serviceString, name, null, filepath, rebuild);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing filepath");
} }
Service service = Service.valueOf(serviceString); @GET
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service); @Path("/{service}/{name}/{identifier}")
try { @Operation(
summary = "Fetch raw data from file with supplied service, name, identifier, and relative path",
description = "An optional rebuild boolean can be supplied. If true, any existing cached data will be invalidated.",
responses = {
@ApiResponse(
description = "Path to file structure containing requested data",
content = @Content(
mediaType = MediaType.TEXT_PLAIN,
schema = @Schema(
type = "string"
)
)
)
}
)
public HttpServletResponse get(@PathParam("service") String serviceString,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("filepath") String filepath,
@QueryParam("rebuild") boolean rebuild) {
Security.checkApiCallAllowed(request);
// Loop until we have data return this.download(serviceString, name, identifier, filepath, rebuild);
while (!Controller.isStopping()) {
try {
arbitraryDataReader.loadSynchronously(rebuild);
break;
} catch (MissingDataException e) {
continue;
}
}
// TODO: limit file size that can be read into memory
java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath);
if (!Files.exists(path)) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
}
byte[] data = Files.readAllBytes(path);
response.setContentType(context.getMimeType(path.toString()));
response.setContentLength(data.length);
response.getOutputStream().write(data);
return response;
} catch (Exception e) {
LOGGER.info(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
}
} }
@POST @POST
@ -432,6 +427,7 @@ public class ArbitraryResource {
return this.upload(Method.PATCH, Service.valueOf(serviceString), name, identifier, path); return this.upload(Method.PATCH, Service.valueOf(serviceString), name, identifier, path);
} }
private String upload(Method method, Service service, String name, String identifier, String path) { private String upload(Method method, Service service, String name, String identifier, String path) {
// It's too dangerous to allow user-supplied file paths in weaker security contexts // It's too dangerous to allow user-supplied file paths in weaker security contexts
if (Settings.getInstance().isApiRestricted()) { if (Settings.getInstance().isApiRestricted()) {
@ -470,6 +466,43 @@ public class ArbitraryResource {
} }
} }
private HttpServletResponse download(String serviceString, String name, String identifier, String filepath, boolean rebuild) {
if (filepath == null) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing filepath");
}
Service service = Service.valueOf(serviceString);
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
try {
// Loop until we have data
while (!Controller.isStopping()) {
try {
arbitraryDataReader.loadSynchronously(rebuild);
break;
} catch (MissingDataException e) {
continue;
}
}
// TODO: limit file size that can be read into memory
java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath);
if (!Files.exists(path)) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
}
byte[] data = Files.readAllBytes(path);
response.setContentType(context.getMimeType(path.toString()));
response.setContentLength(data.length);
response.getOutputStream().write(data);
return response;
} catch (Exception e) {
LOGGER.info(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
}
}
@DELETE @DELETE
@Path("/file") @Path("/file")

View File

@ -93,7 +93,7 @@ public class WebsiteResource {
Method method = Method.PUT; Method method = Method.PUT;
Compression compression = Compression.ZIP; Compression compression = Compression.ZIP;
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, method, compression); ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, null, method, compression);
try { try {
arbitraryDataWriter.save(); arbitraryDataWriter.save();
} catch (IOException | DataException | InterruptedException | MissingDataException e) { } catch (IOException | DataException | InterruptedException | MissingDataException e) {
@ -178,7 +178,7 @@ public class WebsiteResource {
} }
Service service = Service.WEBSITE; Service service = Service.WEBSITE;
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service); ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, null);
arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
try { try {
if (!arbitraryDataReader.isCachedDataAvailable()) { if (!arbitraryDataReader.isCachedDataAvailable()) {

View File

@ -13,6 +13,7 @@ public class ArbitraryDataBuildQueueItem {
private String resourceId; private String resourceId;
private ResourceIdType resourceIdType; private ResourceIdType resourceIdType;
private Service service; private Service service;
private String identifier;
private Long creationTimestamp = null; private Long creationTimestamp = null;
private Long buildStartTimestamp = null; private Long buildStartTimestamp = null;
private Long buildEndTimestamp = null; private Long buildEndTimestamp = null;
@ -24,10 +25,11 @@ public class ArbitraryDataBuildQueueItem {
/* The amount of time to remember that a build has failed, to avoid retries */ /* The amount of time to remember that a build has failed, to avoid retries */
public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes
public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) { public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
this.resourceId = resourceId.toLowerCase(); this.resourceId = resourceId.toLowerCase();
this.resourceIdType = resourceIdType; this.resourceIdType = resourceIdType;
this.service = service; this.service = service;
this.identifier = identifier;
this.creationTimestamp = NTP.getTime(); this.creationTimestamp = NTP.getTime();
} }
@ -39,7 +41,7 @@ public class ArbitraryDataBuildQueueItem {
this.buildStartTimestamp = now; this.buildStartTimestamp = now;
ArbitraryDataReader arbitraryDataReader = ArbitraryDataReader arbitraryDataReader =
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service); new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);
try { try {
arbitraryDataReader.loadSynchronously(true); arbitraryDataReader.loadSynchronously(true);
@ -86,7 +88,7 @@ public class ArbitraryDataBuildQueueItem {
@Override @Override
public String toString() { public String toString() {
return String.format("%s %s", this.service, this.resourceId); return String.format("%s %s %s", this.service, this.resourceId, this.identifier);
} }
} }

View File

@ -28,6 +28,7 @@ public class ArbitraryDataBuilder {
private String name; private String name;
private Service service; private Service service;
private String identifier;
private List<ArbitraryTransactionData> transactions; private List<ArbitraryTransactionData> transactions;
private ArbitraryTransactionData latestPutTransaction; private ArbitraryTransactionData latestPutTransaction;
@ -35,9 +36,10 @@ public class ArbitraryDataBuilder {
private byte[] latestSignature; private byte[] latestSignature;
private Path finalPath; private Path finalPath;
public ArbitraryDataBuilder(String name, Service service) { public ArbitraryDataBuilder(String name, Service service, String identifier) {
this.name = name; this.name = name;
this.service = service; this.service = service;
this.identifier = identifier;
this.paths = new ArrayList<>(); this.paths = new ArrayList<>();
} }
@ -56,16 +58,17 @@ public class ArbitraryDataBuilder {
// Get the most recent PUT // Get the most recent PUT
ArbitraryTransactionData latestPut = repository.getArbitraryRepository() ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
.getLatestTransaction(this.name, this.service, Method.PUT); .getLatestTransaction(this.name, this.service, Method.PUT, this.identifier);
if (latestPut == null) { if (latestPut == null) {
throw new IllegalStateException(String.format( String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s",
"Couldn't find PUT transaction for name %s and service %s", this.name, this.service)); this.name, this.service, this.identifierString());
throw new IllegalStateException(message);
} }
this.latestPutTransaction = latestPut; this.latestPutTransaction = latestPut;
// Load all transactions since the latest PUT // Load all transactions since the latest PUT
List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository() List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
.getArbitraryTransactions(this.name, this.service, latestPut.getTimestamp()); .getArbitraryTransactions(this.name, this.service, this.identifier, latestPut.getTimestamp());
this.transactions = transactionDataList; this.transactions = transactionDataList;
} }
} }
@ -81,8 +84,8 @@ public class ArbitraryDataBuilder {
throw new IllegalStateException("Expected PUT but received PATCH"); throw new IllegalStateException("Expected PUT but received PATCH");
} }
if (transactionDataList.size() == 0) { if (transactionDataList.size() == 0) {
throw new IllegalStateException(String.format("No transactions found for name %s, service %s, since %d", throw new IllegalStateException(String.format("No transactions found for name %s, service %s, " +
name, service, latestPut.getTimestamp())); "identifier: %s, since %d", name, service, this.identifierString(), latestPut.getTimestamp()));
} }
// Verify that the signature of the first transaction matches the latest PUT // Verify that the signature of the first transaction matches the latest PUT
@ -115,7 +118,8 @@ public class ArbitraryDataBuilder {
// Build the data file, overwriting anything that was previously there // Build the data file, overwriting anything that was previously there
String sig58 = Base58.encode(transactionData.getSignature()); String sig58 = Base58.encode(transactionData.getSignature());
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service); ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA,
this.service, this.identifier);
arbitraryDataReader.setTransactionData(transactionData); arbitraryDataReader.setTransactionData(transactionData);
boolean hasMissingData = false; boolean hasMissingData = false;
try { try {
@ -179,7 +183,8 @@ public class ArbitraryDataBuilder {
// Loop from the second path onwards // Loop from the second path onwards
for (int i=1; i<paths.size(); i++) { for (int i=1; i<paths.size(); i++) {
LOGGER.info(String.format("[%s][%s] Applying layer %d...", this.service, this.name, i)); String identifierPrefix = this.identifier != null ? String.format("[%s]", this.identifier) : "";
LOGGER.info(String.format("[%s][%s]%s Applying layer %d...", this.service, this.name, identifierPrefix, i));
// Create an instance of ArbitraryDataCombiner // Create an instance of ArbitraryDataCombiner
Path pathAfter = this.paths.get(i); Path pathAfter = this.paths.get(i);
@ -211,6 +216,10 @@ public class ArbitraryDataBuilder {
cache.write(); cache.write();
} }
private String identifierString() {
return identifier != null ? identifier : "";
}
public Path getFinalPath() { public Path getFinalPath() {
return this.finalPath; return this.finalPath;
} }

View File

@ -22,14 +22,16 @@ public class ArbitraryDataCache {
private String resourceId; private String resourceId;
private ResourceIdType resourceIdType; private ResourceIdType resourceIdType;
private Service service; private Service service;
private String identifier;
public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId, public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId,
ResourceIdType resourceIdType, Service service) { ResourceIdType resourceIdType, Service service, String identifier) {
this.filePath = filePath; this.filePath = filePath;
this.overwrite = overwrite; this.overwrite = overwrite;
this.resourceId = resourceId; this.resourceId = resourceId;
this.resourceIdType = resourceIdType; this.resourceIdType = resourceIdType;
this.service = service; this.service = service;
this.identifier = identifier;
} }
public boolean isCachedDataAvailable() { public boolean isCachedDataAvailable() {
@ -134,7 +136,7 @@ public class ArbitraryDataCache {
// Find latest transaction for name and service, with any method // Find latest transaction for name and service, with any method
ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository() ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
.getLatestTransaction(this.resourceId, this.service, null); .getLatestTransaction(this.resourceId, this.service, null, this.identifier);
if (latestTransaction != null) { if (latestTransaction != null) {
return latestTransaction.getSignature(); return latestTransaction.getSignature();

View File

@ -42,6 +42,7 @@ public class ArbitraryDataReader {
private String resourceId; private String resourceId;
private ResourceIdType resourceIdType; private ResourceIdType resourceIdType;
private Service service; private Service service;
private String identifier;
private ArbitraryTransactionData transactionData; private ArbitraryTransactionData transactionData;
private String secret58; private String secret58;
private Path filePath; private Path filePath;
@ -51,7 +52,7 @@ public class ArbitraryDataReader {
private Path uncompressedPath; private Path uncompressedPath;
private Path unencryptedPath; private Path unencryptedPath;
public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service) { public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
// Ensure names are always lowercase // Ensure names are always lowercase
if (resourceIdType == ResourceIdType.NAME) { if (resourceIdType == ResourceIdType.NAME) {
resourceId = resourceId.toLowerCase(); resourceId = resourceId.toLowerCase();
@ -60,25 +61,31 @@ public class ArbitraryDataReader {
this.resourceId = resourceId; this.resourceId = resourceId;
this.resourceIdType = resourceIdType; this.resourceIdType = resourceIdType;
this.service = service; this.service = service;
this.identifier = identifier;
this.workingPath = this.buildWorkingPath();
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
}
private Path buildWorkingPath() {
// Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
String baseDir = Settings.getInstance().getTempDataPath(); String baseDir = Settings.getInstance().getTempDataPath();
this.workingPath = Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString()); String identifier = this.identifier != null ? this.identifier : "default";
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); return Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
} }
public boolean isCachedDataAvailable() { public boolean isCachedDataAvailable() {
// If this resource is in the build queue then we shouldn't attempt to serve // If this resource is in the build queue then we shouldn't attempt to serve
// cached data, as it may not be fully built // cached data, as it may not be fully built
ArbitraryDataBuildQueueItem queueItem = ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service); new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service, this.identifier);
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
return false; return false;
} }
// Not in the build queue - so check the cache itself // Not in the build queue - so check the cache itself
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false, ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false,
this.resourceId, this.resourceIdType, this.service); this.resourceId, this.resourceIdType, this.service, this.identifier);
if (cache.isCachedDataAvailable()) { if (cache.isCachedDataAvailable()) {
this.filePath = this.uncompressedPath; this.filePath = this.uncompressedPath;
return true; return true;
@ -98,7 +105,7 @@ public class ArbitraryDataReader {
*/ */
public boolean loadAsynchronously() { public boolean loadAsynchronously() {
ArbitraryDataBuildQueueItem queueItem = ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service); new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service, this.identifier);
return ArbitraryDataBuildManager.getInstance().addToBuildQueue(queueItem); return ArbitraryDataBuildManager.getInstance().addToBuildQueue(queueItem);
} }
@ -117,7 +124,7 @@ public class ArbitraryDataReader {
public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException, MissingDataException { public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException, MissingDataException {
try { try {
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
this.resourceId, this.resourceIdType, this.service); this.resourceId, this.resourceIdType, this.service, this.identifier);
if (cache.isCachedDataAvailable()) { if (cache.isCachedDataAvailable()) {
// Use cached data // Use cached data
this.filePath = this.uncompressedPath; this.filePath = this.uncompressedPath;
@ -233,7 +240,7 @@ public class ArbitraryDataReader {
try { try {
// Build the existing state using past transactions // Build the existing state using past transactions
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service); ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service, this.identifier);
builder.build(); builder.build();
Path builtPath = builder.getFinalPath(); Path builtPath = builder.getFinalPath();
if (builtPath == null) { if (builtPath == null) {

View File

@ -15,9 +15,7 @@ import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager; import org.qortal.repository.RepositoryManager;
import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction;
import org.qortal.transform.TransformationException;
import org.qortal.transform.Transformer; import org.qortal.transform.Transformer;
import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
import org.qortal.utils.NTP; import org.qortal.utils.NTP;
@ -69,7 +67,7 @@ public class ArbitraryDataTransactionBuilder {
ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP; ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP;
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, method, compression); ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, compression);
try { try {
arbitraryDataWriter.save(); arbitraryDataWriter.save();
} catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) { } catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) {

View File

@ -34,6 +34,7 @@ public class ArbitraryDataWriter {
private Path filePath; private Path filePath;
private String name; private String name;
private Service service; private Service service;
private String identifier;
private Method method; private Method method;
private Compression compression; private Compression compression;
@ -45,10 +46,11 @@ public class ArbitraryDataWriter {
private Path compressedPath; private Path compressedPath;
private Path encryptedPath; private Path encryptedPath;
public ArbitraryDataWriter(Path filePath, String name, Service service, Method method, Compression compression) { public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression) {
this.filePath = filePath; this.filePath = filePath;
this.name = name; this.name = name;
this.service = service; this.service = service;
this.identifier = identifier;
this.method = method; this.method = method;
this.compression = compression; this.compression = compression;
} }
@ -114,7 +116,7 @@ public class ArbitraryDataWriter {
private void processPatch() throws DataException, IOException, MissingDataException { private void processPatch() throws DataException, IOException, MissingDataException {
// Build the existing state using past transactions // Build the existing state using past transactions
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service); ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service, this.identifier);
builder.build(); builder.build();
Path builtPath = builder.getFinalPath(); Path builtPath = builder.getFinalPath();

View File

@ -10,6 +10,7 @@ public class ArbitraryResourceInfo {
public String name; public String name;
public ArbitraryTransactionData.Service service; public ArbitraryTransactionData.Service service;
public String identifier;
public ArbitraryResourceInfo() { public ArbitraryResourceInfo() {
} }

View File

@ -17,9 +17,9 @@ public interface ArbitraryRepository {
public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException; public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, long since) throws DataException; public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException; public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException; public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException;

View File

@ -153,17 +153,18 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
} }
@Override @Override
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, long since) throws DataException { public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " + String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " + "tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " + "version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " + "JOIN Transactions USING (signature) " +
"WHERE lower(name) = ? AND service = ? AND created_when >= ? " + "WHERE lower(name) = ? AND service = ?" +
"ORDER BY created_when ASC"; "AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))" +
"AND created_when >= ? ORDER BY created_when ASC";
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>(); List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql, name.toLowerCase(), service.value, since)) { try (ResultSet resultSet = this.repository.checkedExecute(sql, name.toLowerCase(), service.value, identifier, identifier, since)) {
if (resultSet == null) if (resultSet == null)
return null; return null;
@ -221,7 +222,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
} }
@Override @Override
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException { public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
StringBuilder sql = new StringBuilder(1024); StringBuilder sql = new StringBuilder(1024);
sql.append("SELECT type, reference, signature, creator, created_when, fee, " + sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
@ -229,7 +230,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " + "version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " + "name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " + "JOIN Transactions USING (signature) " +
"WHERE lower(name) = ? AND service = ?"); "WHERE lower(name) = ? AND service = ? " +
"AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))");
if (method != null) { if (method != null) {
sql.append(" AND update_method = "); sql.append(" AND update_method = ");
@ -238,7 +240,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
sql.append("ORDER BY created_when DESC LIMIT 1"); sql.append("ORDER BY created_when DESC LIMIT 1");
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value)) { try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) {
if (resultSet == null) if (resultSet == null)
return null; return null;
@ -295,14 +297,14 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException { public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512); StringBuilder sql = new StringBuilder(512);
sql.append("SELECT name, service FROM ArbitraryTransactions"); sql.append("SELECT name, service, identifier FROM ArbitraryTransactions");
if (service != null) { if (service != null) {
sql.append(" WHERE service = "); sql.append(" WHERE service = ");
sql.append(service.value); sql.append(service.value);
} }
sql.append(" GROUP BY name, service ORDER BY name"); sql.append(" GROUP BY name, service, identifier ORDER BY name");
if (reverse != null && reverse) { if (reverse != null && reverse) {
sql.append(" DESC"); sql.append(" DESC");
@ -319,6 +321,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
do { do {
String name = resultSet.getString(1); String name = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2)); Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifier = resultSet.getString(3);
// We should filter out resources without names // We should filter out resources without names
if (name == null) { if (name == null) {
@ -328,6 +331,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
arbitraryResourceInfo.name = name; arbitraryResourceInfo.name = name;
arbitraryResourceInfo.service = serviceResult; arbitraryResourceInfo.service = serviceResult;
arbitraryResourceInfo.identifier = identifier;
arbitraryResources.add(arbitraryResourceInfo); arbitraryResources.add(arbitraryResourceInfo);
} while (resultSet.next()); } while (resultSet.next());

View File

@ -39,6 +39,7 @@ public class ArbitraryTransactionUtils {
String name = arbitraryTransactionData.getName(); String name = arbitraryTransactionData.getName();
ArbitraryTransactionData.Service service = arbitraryTransactionData.getService(); ArbitraryTransactionData.Service service = arbitraryTransactionData.getService();
String identifier = arbitraryTransactionData.getIdentifier();
if (name == null || service == null) { if (name == null || service == null) {
return null; return null;
@ -48,7 +49,7 @@ public class ArbitraryTransactionUtils {
ArbitraryTransactionData latestPut; ArbitraryTransactionData latestPut;
try { try {
latestPut = repository.getArbitraryRepository() latestPut = repository.getArbitraryRepository()
.getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT); .getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT, identifier);
} catch (DataException e) { } catch (DataException e) {
return null; return null;
} }

View File

@ -63,7 +63,7 @@ public class ArbitraryDataTests extends Common {
this.createAndMintTxn(repository, publicKey58, path3, name, identifier, Method.PATCH, service, alice); this.createAndMintTxn(repository, publicKey58, path3, name, identifier, Method.PATCH, service, alice);
// Now build the latest data state for this name // Now build the latest data state for this name
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service); ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
arbitraryDataReader.loadSynchronously(true); arbitraryDataReader.loadSynchronously(true);
Path finalPath = arbitraryDataReader.getFilePath(); Path finalPath = arbitraryDataReader.getFilePath();
@ -102,7 +102,7 @@ public class ArbitraryDataTests extends Common {
} catch (DataException expectedException) { } catch (DataException expectedException) {
assertEquals(String.format("Couldn't find PUT transaction for " + assertEquals(String.format("Couldn't find PUT transaction for " +
"name %s and service %s", name, service), expectedException.getMessage()); "name %s, service %s and identifier ", name, service), expectedException.getMessage());
} }
} }
@ -181,7 +181,7 @@ public class ArbitraryDataTests extends Common {
this.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); this.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice);
// Now build the latest data state for this name // Now build the latest data state for this name
ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service); ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
arbitraryDataReader1.loadSynchronously(true); arbitraryDataReader1.loadSynchronously(true);
Path initialLayerPath = arbitraryDataReader1.getFilePath(); Path initialLayerPath = arbitraryDataReader1.getFilePath();
ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath); ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath);
@ -192,7 +192,75 @@ public class ArbitraryDataTests extends Common {
this.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice); this.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice);
// Rebuild the latest state // Rebuild the latest state
ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service); ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
arbitraryDataReader2.loadSynchronously(false);
Path secondLayerPath = arbitraryDataReader2.getFilePath();
ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath);
secondLayerDigest.compute();
// Ensure that the second state is different to the first state
assertFalse(Arrays.equals(initialLayerDigest.getHash(), secondLayerDigest.getHash()));
// Its directory hash should match the hash of demo2
ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(path2);
path2Digest.compute();
assertEquals(path2Digest.getHash58(), secondLayerDigest.getHash58());
}
}
@Test
public void testIdentifier() throws DataException, IOException, MissingDataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
String name = "TEST"; // Can be anything for this test
String identifier = "test_identifier";
Service service = Service.WEBSITE; // Can be anything for this test
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
Path path1 = Paths.get("src/test/resources/arbitrary/demo1");
this.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice);
// Build the latest data state for this name, with a null identifier, ensuring that it fails
ArbitraryDataReader arbitraryDataReader1a = new ArbitraryDataReader(name, ResourceIdType.NAME, service, null);
try {
arbitraryDataReader1a.loadSynchronously(true);
fail("Loading data with null identifier should fail due to nonexistent PUT transaction");
} catch (IllegalStateException expectedException) {
assertEquals(String.format("Couldn't find PUT transaction for name %s, service %s "
+ "and identifier ", name.toLowerCase(), service), expectedException.getMessage());
}
// Build the latest data state for this name, with a different identifier, ensuring that it fails
String differentIdentifier = "different_identifier";
ArbitraryDataReader arbitraryDataReader1b = new ArbitraryDataReader(name, ResourceIdType.NAME, service, differentIdentifier);
try {
arbitraryDataReader1b.loadSynchronously(true);
fail("Loading data with incorrect identifier should fail due to nonexistent PUT transaction");
} catch (IllegalStateException expectedException) {
assertEquals(String.format("Couldn't find PUT transaction for name %s, service %s "
+ "and identifier %s", name.toLowerCase(), service, differentIdentifier), expectedException.getMessage());
}
// Now build the latest data state for this name, with the correct identifier
ArbitraryDataReader arbitraryDataReader1c = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
arbitraryDataReader1c.loadSynchronously(true);
Path initialLayerPath = arbitraryDataReader1c.getFilePath();
ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath);
initialLayerDigest.compute();
// Create PATCH transaction
Path path2 = Paths.get("src/test/resources/arbitrary/demo2");
this.createAndMintTxn(repository, publicKey58, path2, name, identifier, Method.PATCH, service, alice);
// Rebuild the latest state
ArbitraryDataReader arbitraryDataReader2 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
arbitraryDataReader2.loadSynchronously(false); arbitraryDataReader2.loadSynchronously(false);
Path secondLayerPath = arbitraryDataReader2.getFilePath(); Path secondLayerPath = arbitraryDataReader2.getFilePath();
ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath); ArbitraryDataDigest secondLayerDigest = new ArbitraryDataDigest(secondLayerPath);