Rework of "Service" types to allow for validation

Each service supports basic validation params, plus has the option for an entirely custom validation function.

Initial validation settings:
- IMAGE must be less than 10MiB
- THUMBNAIL must be less than 500KiB
- METADATA must be less than 10KiB and must contain JSON keys "title", "description", and "tags"
This commit is contained in:
CalDescent 2021-11-16 19:28:25 +00:00
parent 9c952785e6
commit fb09d77cdc
25 changed files with 266 additions and 51 deletions

View File

@ -30,6 +30,7 @@ import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.account.AccountData;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;

View File

@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger;
import org.qortal.api.ApiError;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.arbitrary.misc.Service;
import org.qortal.arbitrary.*;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.data.transaction.ArbitraryTransactionData.*;

View File

@ -1,8 +1,8 @@
package org.qortal.arbitrary;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.arbitrary.misc.Service;
import org.qortal.repository.DataException;
import org.qortal.utils.NTP;

View File

@ -4,9 +4,9 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.Method;
import org.qortal.data.transaction.ArbitraryTransactionData.Service;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;

View File

@ -2,9 +2,9 @@ package org.qortal.arbitrary;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;

View File

@ -1,10 +1,12 @@
package org.qortal.arbitrary;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
@ -26,6 +28,7 @@ import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import javax.xml.crypto.Data;
import java.io.File;
import java.io.IOException;
import java.io.InvalidObjectException;
@ -146,6 +149,7 @@ public class ArbitraryDataReader {
this.fetch();
this.decrypt();
this.uncompress();
this.validate();
} finally {
this.postExecute();
@ -425,6 +429,20 @@ public class ArbitraryDataReader {
this.filePath = this.uncompressedPath;
}
private void validate() throws IOException, DataException {
if (this.service.isValidationRequired()) {
byte[] data = FilesystemUtils.getSingleFileContents(this.filePath);
long size = FilesystemUtils.getDirectorySize(this.filePath);
Service.ValidationResult result = this.service.validate(data, size);
if (result != Service.ValidationResult.OK) {
throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
}
}
}
private void moveFilePathToFinalDestination() throws IOException {
if (this.filePath.compareTo(this.uncompressedPath) != 0) {
File source = new File(this.filePath.toString());

View File

@ -6,8 +6,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.HTMLParser;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.arbitrary.misc.Service;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
@ -60,7 +59,7 @@ public class ArbitraryDataRenderer {
inPath = File.separator + inPath;
}
ArbitraryTransactionData.Service service = Service.WEBSITE;
Service service = Service.WEBSITE;
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, null);
arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
try {

View File

@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
import org.qortal.arbitrary.misc.Service;
import org.qortal.block.BlockChain;
import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData;

View File

@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.crypto.Crypto;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.crypto.AES;
@ -58,6 +59,7 @@ public class ArbitraryDataWriter {
public void save() throws IllegalStateException, IOException, DataException, InterruptedException, MissingDataException {
try {
this.preExecute();
this.validateService();
this.process();
this.compress();
this.encrypt();
@ -97,6 +99,19 @@ public class ArbitraryDataWriter {
this.workingPath = tempDir;
}
private void validateService() throws IOException, DataException {
if (this.service.isValidationRequired()) {
byte[] data = FilesystemUtils.getSingleFileContents(this.filePath);
long size = FilesystemUtils.getDirectorySize(this.filePath);
Service.ValidationResult result = this.service.validate(data, size);
if (result != Service.ValidationResult.OK) {
throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
}
}
}
private void process() throws DataException, IOException, MissingDataException {
switch (this.method) {

View File

@ -0,0 +1,105 @@
package org.qortal.arbitrary.misc;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import org.qortal.transaction.Transaction;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
public enum Service {
AUTO_UPDATE(1, false, null, null),
ARBITRARY_DATA(100, false, null, null),
WEBSITE(200, false, null, null),
GIT_REPOSITORY(300, false, null, null),
IMAGE(400, true, 10*1024*1024L, null),
THUMBNAIL(410, true, 500*1024L, null),
VIDEO(500, false, null, null),
AUDIO(600, false, null, null),
BLOG(700, false, null, null),
BLOG_POST(777, false, null, null),
BLOG_COMMENT(778, false, null, null),
DOCUMENT(800, false, null, null),
PLAYLIST(900, true, null, null),
APP(1000, false, null, null),
METADATA(1100, true, 10*1024L, Arrays.asList("title", "description", "tags"));
public final int value;
private final boolean requiresValidation;
private final Long maxSize;
private final List<String> requiredKeys;
private static final Logger LOGGER = LogManager.getLogger(Service.class);
private static final Map<Integer, Service> map = stream(Service.values())
.collect(toMap(service -> service.value, service -> service));
Service(int value, boolean requiresValidation, Long maxSize, List<String> requiredKeys) {
this.value = value;
this.requiresValidation = requiresValidation;
this.maxSize = maxSize;
this.requiredKeys = requiredKeys;
}
public ValidationResult validate(byte[] data, long size) {
if (!this.isValidationRequired()) {
return ValidationResult.OK;
}
// Validate max size if needed
if (this.maxSize != null) {
if (size > this.maxSize || data.length > this.maxSize) {
return ValidationResult.EXCEEDS_SIZE_LIMIT;
}
}
// Validate required keys if needed
if (this.requiredKeys != null) {
JSONObject json = Service.toJsonObject(data);
for (String key : this.requiredKeys) {
if (!json.has(key)) {
return ValidationResult.MISSING_KEYS;
}
}
}
// Validation passed
return ValidationResult.OK;
}
public boolean isValidationRequired() {
return this.requiresValidation;
}
public static Service valueOf(int value) {
return map.get(value);
}
public static JSONObject toJsonObject(byte[] data) {
String dataString = new String(data);
return new JSONObject(dataString);
}
public enum ValidationResult {
OK(1),
MISSING_KEYS(2),
EXCEEDS_SIZE_LIMIT(3);
public final int value;
private static final Map<Integer, Transaction.ValidationResult> map = stream(Transaction.ValidationResult.values()).collect(toMap(result -> result.value, result -> result));
ValidationResult(int value) {
this.value = value;
}
public static Transaction.ValidationResult valueOf(int value) {
return map.get(value);
}
}
}

View File

@ -1,6 +1,6 @@
package org.qortal.data.arbitrary;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ -9,7 +9,7 @@ import javax.xml.bind.annotation.XmlAccessorType;
public class ArbitraryResourceInfo {
public String name;
public ArbitraryTransactionData.Service service;
public Service service;
public String identifier;
public ArbitraryResourceInfo() {

View File

@ -8,6 +8,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.eclipse.persistence.oxm.annotations.XmlDiscriminatorValue;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.PaymentData;
import org.qortal.transaction.Transaction.TransactionType;
@ -29,38 +30,6 @@ public class ArbitraryTransactionData extends TransactionData {
DATA_HASH;
}
// Service types
public enum Service {
AUTO_UPDATE(1),
ARBITRARY_DATA(100),
WEBSITE(200),
GIT_REPOSITORY(300),
IMAGE(400),
THUMBNAIL(410),
VIDEO(500),
AUDIO(600),
BLOG(700),
BLOG_POST(777),
BLOG_COMMENT(778),
DOCUMENT(800),
PLAYLIST(900),
APP(1000),
METADATA(1100);
public final int value;
private static final Map<Integer, Service> map = stream(Service.values())
.collect(toMap(service -> service.value, service -> service));
Service(int value) {
this.value = value;
}
public static Service valueOf(int value) {
return map.get(value);
}
}
// Methods
public enum Method {
PUT(0), // A complete replacement of a resource

View File

@ -1,5 +1,6 @@
package org.qortal.repository;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.data.network.ArbitraryPeerData;
import org.qortal.data.transaction.ArbitraryTransactionData;

View File

@ -5,8 +5,8 @@ import java.util.List;
import java.util.Map;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.group.GroupApprovalData;
import org.qortal.data.transaction.ArbitraryTransactionData.Service;
import org.qortal.data.transaction.GroupApprovalTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.data.transaction.TransferAssetTransactionData;

View File

@ -1,8 +1,8 @@
package org.qortal.repository.hsqldb;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData;
import org.qortal.data.network.ArbitraryPeerData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;

View File

@ -4,6 +4,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.BaseTransactionData;
@ -30,7 +31,7 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
int version = resultSet.getInt(1);
int nonce = resultSet.getInt(2);
ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.valueOf(resultSet.getInt(3));
Service service = Service.valueOf(resultSet.getInt(3));
int size = resultSet.getInt(4);
boolean isDataRaw = resultSet.getBoolean(5); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;

View File

@ -16,13 +16,13 @@ import java.util.Map;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.PaymentData;
import org.qortal.data.group.GroupApprovalData;
import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.GroupApprovalTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.data.transaction.TransferAssetTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.Service;
import org.qortal.repository.DataException;
import org.qortal.repository.TransactionRepository;
import org.qortal.repository.hsqldb.HSQLDBRepository;

View File

@ -7,6 +7,7 @@ import java.util.ArrayList;
import java.util.List;
import com.google.common.base.Utf8;
import org.qortal.arbitrary.misc.Service;
import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData;
@ -130,7 +131,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
payments.add(PaymentTransformer.fromByteBuffer(byteBuffer));
}
ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.valueOf(byteBuffer.getInt());
Service service = Service.valueOf(byteBuffer.getInt());
// We might be receiving hash of data instead of actual raw data
boolean isRaw = byteBuffer.get() != 0;

View File

@ -3,6 +3,7 @@ package org.qortal.utils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.DataException;
@ -38,7 +39,7 @@ public class ArbitraryTransactionUtils {
}
String name = arbitraryTransactionData.getName();
ArbitraryTransactionData.Service service = arbitraryTransactionData.getService();
Service service = arbitraryTransactionData.getService();
String identifier = arbitraryTransactionData.getIdentifier();
if (name == null || service == null) {

View File

@ -1,6 +1,7 @@
package org.qortal.utils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.qortal.settings.Settings;
import java.io.File;
@ -190,4 +191,36 @@ public class FilesystemUtils {
.sum();
}
/**
* getSingleFileContents
* Return the content of the file at given path.
* If the path is a directory, the contents will be returned
* only if it contains a single file.
*
* @param path
* @return
* @throws IOException
*/
public static byte[] getSingleFileContents(Path path) throws IOException {
byte[] data = null;
// TODO: limit the file size that can be loaded into memory
// If the path is a file, read the contents directly
if (path.toFile().isFile()) {
data = Files.readAllBytes(path);
}
// Or if it's a directory, only load file contents if there is a single file inside it
else if (path.toFile().isDirectory()) {
String[] files = ArrayUtils.removeElement(path.toFile().list(), ".qortal");
if (files.length == 1) {
Path filePath = Paths.get(path.toString(), files[0]);
data = Files.readAllBytes(filePath);
}
}
return data;
}
}

View File

@ -6,8 +6,7 @@ import org.junit.Before;
import org.junit.Test;
import org.qortal.api.resource.ArbitraryResource;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.Service;
import org.qortal.arbitrary.misc.Service;
import org.qortal.test.common.ApiCommon;
public class ArbitraryApiTests extends ApiCommon {
@ -24,7 +23,7 @@ public class ArbitraryApiTests extends ApiCommon {
Integer[] startingBlocks = new Integer[] { null, 0, 1, 999999999 };
Integer[] blockLimits = new Integer[] { null, 0, 1, 999999999 };
Integer[] txGroupIds = new Integer[] { null, 0, 1, 999999999 };
ArbitraryTransactionData.Service[] services = new Service[] { Service.WEBSITE, Service.GIT_REPOSITORY, Service.BLOG_COMMENT };
Service[] services = new Service[] { Service.WEBSITE, Service.GIT_REPOSITORY, Service.BLOG_COMMENT };
String[] names = new String[] { null, "Test" };
String[] addresses = new String[] { null, this.aliceAddress };
ConfirmationStatus[] confirmationStatuses = new ConfirmationStatus[] { ConfirmationStatus.UNCONFIRMED, ConfirmationStatus.CONFIRMED, ConfirmationStatus.BOTH };

View File

@ -9,6 +9,7 @@ import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
import org.qortal.arbitrary.misc.Service;
import org.qortal.crypto.Crypto;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;

View File

@ -0,0 +1,67 @@
package org.qortal.test.arbitrary;
import org.junit.Before;
import org.junit.Test;
import org.qortal.arbitrary.misc.Service;
import org.qortal.arbitrary.misc.Service.ValidationResult;
import org.qortal.repository.DataException;
import org.qortal.test.common.Common;
import java.util.Random;
import static org.junit.Assert.*;
public class ArbitraryServiceTests extends Common {
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
}
@Test
public void testDefaultValidation() {
// We don't validate websites yet, but we still want to test the default validation method
byte[] data = new byte[1024];
new Random().nextBytes(data);
Service service = Service.WEBSITE;
assertFalse(service.isValidationRequired());
// Test validation anyway to ensure that no exception is thrown
assertEquals(ValidationResult.OK, service.validate(data, data.length));
}
@Test
public void testValidMetadata() {
// Metadata is to describe an arbitrary resource (title, description, tags, etc)
String dataString = "{\"title\":\"Test Title\", \"description\":\"Test description\", \"tags\":[\"test\"]}";
byte[] data = dataString.getBytes();
Service service = Service.METADATA;
assertTrue(service.isValidationRequired());
assertEquals(ValidationResult.OK, service.validate(data, data.length));
}
@Test
public void testMetadataMissingKeys() {
// Metadata is to describe an arbitrary resource (title, description, tags, etc)
String dataString = "{\"description\":\"Test description\", \"tags\":[\"test\"]}";
byte[] data = dataString.getBytes();
Service service = Service.METADATA;
assertTrue(service.isValidationRequired());
assertEquals(ValidationResult.MISSING_KEYS, service.validate(data, data.length));
}
@Test
public void testMetadataTooLarge() {
// Metadata is to describe an arbitrary resource (title, description, tags, etc)
String dataString = "{\"title\":\"Test Title\", \"description\":\"Test description\", \"tags\":[\"test\"]}";
byte[] data = dataString.getBytes();
long totalResourceSize = 11*1024L; // Larger than allowed 10kiB
Service service = Service.METADATA;
assertTrue(service.isValidationRequired());
assertEquals(ValidationResult.EXCEEDS_SIZE_LIMIT, service.validate(data, totalResourceSize));
}
}

View File

@ -3,6 +3,7 @@ package org.qortal.test.arbitrary;
import org.junit.Before;
import org.junit.Test;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.repository.DataException;
@ -36,7 +37,7 @@ public class ArbitraryTransactionTests extends Common {
TestAccount alice = Common.getTestAccount(repository, "alice");
ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.ARBITRARY_DATA;
Service service = Service.ARBITRARY_DATA;
ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT;
ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.NONE;
List<PaymentData> payments = new ArrayList<>();

View File

@ -4,6 +4,7 @@ import java.util.ArrayList;
import java.util.List;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.arbitrary.misc.Service;
import org.qortal.asset.Asset;
import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData;
@ -17,7 +18,7 @@ public class ArbitraryTestTransaction extends TestTransaction {
public static TransactionData randomTransaction(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException {
final int version = 5;
final ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.ARBITRARY_DATA;
final Service service = Service.ARBITRARY_DATA;
final int nonce = 0;
final int size = 4 * 1024 * 1024;
final String name = "TEST";