Work on auto-update

Arbitrary transactions now [de]serialize data-type (raw/hash) for v4+ transactions.
Data type also stored in repository. Very small (<=255 byte) data payloads are also stored directly in HSQLDB.

Added ArbitraryDataManager which looks for hash-only data payloads and possibly requests raw data over network
depending on 'policy' (which currently is "fetch everything").

Added networking support for finding, and transferring, arbitrary data payloads.

Minor optimization to message ID generation in Peer.

Minor optimization in Serialization.serializeSizedString()

src/main/java/org/qora/controller/ArbitraryDataManager.java

@@ -0,0 +1,91 @@
+package org.qora.controller;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qora.api.resource.TransactionsResource.ConfirmationStatus;
+import org.qora.data.transaction.ArbitraryTransactionData;
+import org.qora.data.transaction.TransactionData;
+import org.qora.repository.DataException;
+import org.qora.repository.Repository;
+import org.qora.repository.RepositoryManager;
+import org.qora.transaction.ArbitraryTransaction;
+import org.qora.transaction.Transaction.TransactionType;
+
+public class ArbitraryDataManager extends Thread {
+
+	private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class);
+	private static final List<TransactionType> ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY);
+
+	private static ArbitraryDataManager instance;
+
+	private volatile boolean isStopping = false;
+
+	private ArbitraryDataManager() {
+	}
+
+	public static ArbitraryDataManager getInstance() {
+		if (instance == null)
+			instance = new ArbitraryDataManager();
+
+		return instance;
+	}
+
+	@Override
+	public void run() {
+		Thread.currentThread().setName("Arbitrary Data Manager");
+
+		try {
+			while (!isStopping) {
+				Thread.sleep(2000);
+
+				// Any arbitrary transactions we want to fetch data for?
+				try (final Repository repository = RepositoryManager.getRepository()) {
+					List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true);
+					if (signatures == null || signatures.isEmpty())
+						continue;
+
+					// Filter out those that already have local data
+					signatures.removeIf(signature -> hasLocalData(repository, signature));
+
+					if (signatures.isEmpty())
+						continue;
+
+					// Pick one at random
+					final int index = new Random().nextInt(signatures.size());
+					byte[] signature = signatures.get(index);
+
+					Controller.getInstance().fetchArbitraryData(signature);
+				} catch (DataException e) {
+					LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
+				}
+			}
+		} catch (InterruptedException e) {
+			return;
+		}
+	}
+
+	public void shutdown() {
+		isStopping = true;
+		this.interrupt();
+	}
+
+	private boolean hasLocalData(final Repository repository, final byte[] signature) {
+		try {
+			TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+			if (transactionData == null || !(transactionData instanceof ArbitraryTransactionData))
+				return true;
+
+			ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
+
+			return arbitraryTransaction.isDataLocal();
+		} catch (DataException e) {
+			LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e);
+			return true;
+		}
+	}
+
+}

src/main/java/org/qora/controller/AutoUpdate.java

@@ -51,7 +51,7 @@ public class AutoUpdate extends Thread {
 
 	private static AutoUpdate instance;
 
-	private boolean isStopping = false;
+	private volatile boolean isStopping = false;
 
 	private AutoUpdate() {
 	}

src/main/java/org/qora/controller/Controller.java

@@ -9,8 +9,12 @@ import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
+import java.util.Random;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Predicate;
 
@@ -23,15 +27,20 @@ import org.qora.block.Block;
 import org.qora.block.BlockChain;
 import org.qora.block.BlockGenerator;
 import org.qora.controller.Synchronizer.SynchronizationResult;
+import org.qora.crypto.Crypto;
 import org.qora.data.block.BlockData;
 import org.qora.data.network.BlockSummaryData;
 import org.qora.data.network.PeerData;
+import org.qora.data.transaction.ArbitraryTransactionData;
+import org.qora.data.transaction.ArbitraryTransactionData.DataType;
 import org.qora.data.transaction.TransactionData;
 import org.qora.gui.Gui;
 import org.qora.network.Network;
 import org.qora.network.Peer;
+import org.qora.network.message.ArbitraryDataMessage;
 import org.qora.network.message.BlockMessage;
 import org.qora.network.message.BlockSummariesMessage;
+import org.qora.network.message.GetArbitraryDataMessage;
 import org.qora.network.message.GetBlockMessage;
 import org.qora.network.message.GetBlockSummariesMessage;
 import org.qora.network.message.GetPeersMessage;
@@ -51,11 +60,14 @@ import org.qora.repository.RepositoryFactory;
 import org.qora.repository.RepositoryManager;
 import org.qora.repository.hsqldb.HSQLDBRepositoryFactory;
 import org.qora.settings.Settings;
+import org.qora.transaction.ArbitraryTransaction;
 import org.qora.transaction.Transaction;
+import org.qora.transaction.Transaction.TransactionType;
 import org.qora.transaction.Transaction.ValidationResult;
 import org.qora.ui.UiService;
 import org.qora.utils.Base58;
 import org.qora.utils.NTP;
+import org.qora.utils.Triple;
 
 public class Controller extends Thread {
 
@@ -72,14 +84,35 @@ public class Controller extends Thread {
 	private static final int MAX_BLOCKCHAIN_TIP_AGE = 5; // blocks
 	private static final Object shutdownLock = new Object();
 	private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s/blockchain;create=true";
+	private static final long ARBITRARY_REQUEST_TIMEOUT = 5 * 1000; // ms
 
 	private static volatile boolean isStopping = false;
 	private static BlockGenerator blockGenerator = null;
 	private static volatile boolean requestSync = false;
 	private static Controller instance;
+
 	private final String buildVersion;
 	private final long buildTimestamp; // seconds
 
+	/**
+	 * Map of recent requests for ARBITRARY transaction data payloads.
+	 * <p>
+	 * Key is original request's message ID<br>
+	 * Value is Triple&lt;transaction signature in base58, first requesting peer, first request's timestamp&gt;
+	 * <p>
+	 * If peer is null then either:<br>
+	 * <ul>
+	 * <li>we are the original requesting peer</li>
+	 * <li>we have already sent data payload to original requesting peer.</li>
+	 * </ul>
+	 * If signature is null then we have already received the data payload and either:<br>
+	 * <ul>
+	 * <li>we are the original requesting peer and have saved it locally</li>
+	 * <li>we have forwarded the data payload (and maybe also saved it locally)</li>
+	 * </ul>
+	 */
+	private Map<Integer, Triple<String, Peer, Long>> arbitraryDataRequests = Collections.synchronizedMap(new HashMap<>());
+
 	/** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly generated block. */
 	private final ReentrantLock blockchainLock = new ReentrantLock();
 
@@ -223,6 +256,10 @@ public class Controller extends Thread {
 		LOGGER.info("Starting controller");
 		Controller.getInstance().start();
 
+		// Arbitrary transaction data manager
+		LOGGER.info("Starting arbitrary-transaction data manager");
+		ArbitraryDataManager.getInstance().start();
+
 		// Auto-update service
 		LOGGER.info("Starting auto-update");
 		AutoUpdate.getInstance().start();
@@ -263,6 +300,10 @@ public class Controller extends Thread {
 				requestSync = false;
 				potentiallySynchronize();
 			}
+
+			// Clean up arbitrary data request cache
+			final long requestMinimumTimestamp = NTP.getTime() - ARBITRARY_REQUEST_TIMEOUT;
+			arbitraryDataRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp);
 		}
 	}
 
@@ -351,6 +392,10 @@ public class Controller extends Thread {
 				LOGGER.info("Shutting down auto-update");
 				AutoUpdate.getInstance().shutdown();
 
+				// Arbitrary transaction data manager
+				LOGGER.info("Shutting down arbitrary-transaction data manager");
+				ArbitraryDataManager.getInstance().shutdown();
+
 				LOGGER.info("Shutting down controller");
 				this.interrupt();
 				try {
@@ -780,6 +825,107 @@ public class Controller extends Thread {
 				break;
 			}
 
+			case GET_ARBITRARY_DATA: {
+				GetArbitraryDataMessage getArbitraryDataMessage = (GetArbitraryDataMessage) message;
+
+				byte[] signature = getArbitraryDataMessage.getSignature();
+				String signature58 = Base58.encode(signature);
+				Long timestamp = NTP.getTime();
+				Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peer, timestamp);
+
+				// If we've seen this request recently, then ignore
+				if (arbitraryDataRequests.putIfAbsent(message.getId(), newEntry) != null)
+					break;
+
+				// Do we even have this transaction?
+				try (final Repository repository = RepositoryManager.getRepository()) {
+					TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+					if (transactionData == null || transactionData.getType() != TransactionType.ARBITRARY)
+						break;
+
+					ArbitraryTransaction transaction = new ArbitraryTransaction(repository, transactionData);
+
+					// If we have the data then send it
+					if (transaction.isDataLocal()) {
+						byte[] data = transaction.fetchData();
+						if (data == null)
+							break;
+
+						// Update requests map to reflect that we've sent it
+						newEntry = new Triple<>(signature58, null, timestamp);
+						arbitraryDataRequests.put(message.getId(), newEntry);
+
+						Message arbitraryDataMessage = new ArbitraryDataMessage(signature, data);
+						arbitraryDataMessage.setId(message.getId());
+						if (!peer.sendMessage(arbitraryDataMessage))
+							peer.disconnect("failed to send arbitrary data");
+
+						break;
+					}
+
+					// Ask our other peers if they have it
+					Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : message);
+				} catch (DataException e) {
+					LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e);
+				}
+
+				break;
+			}
+
+			case ARBITRARY_DATA: {
+				ArbitraryDataMessage arbitraryDataMessage = (ArbitraryDataMessage) message;
+
+				// Do we have a pending request for this data?
+				Triple<String, Peer, Long> request = arbitraryDataRequests.get(message.getId());
+				if (request == null || request.getA() == null)
+					break;
+
+				// Does this message's signature match what we're expecting?
+				byte[] signature = arbitraryDataMessage.getSignature();
+				String signature58 = Base58.encode(signature);
+				if (!request.getA().equals(signature58))
+					break;
+
+				byte[] data = arbitraryDataMessage.getData();
+
+				// Check transaction exists and payload hash is correct
+				try (final Repository repository = RepositoryManager.getRepository()) {
+					TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+					if (transactionData == null || !(transactionData instanceof ArbitraryTransactionData))
+						break;
+
+					ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+
+					byte[] actualHash = Crypto.digest(data);
+
+					// "data" from repository will always be hash of actual raw data
+					if (!Arrays.equals(arbitraryTransactionData.getData(), actualHash))
+						break;
+
+					// Update requests map to reflect that we've received it
+					Triple<String, Peer, Long> newEntry = new Triple<>(null, null, request.getC());
+					arbitraryDataRequests.put(message.getId(), newEntry);
+
+					// Save payload locally
+					// TODO: storage policy
+					arbitraryTransactionData.setDataType(DataType.RAW_DATA);
+					arbitraryTransactionData.setData(data);
+					repository.getArbitraryRepository().save(arbitraryTransactionData);
+					repository.saveChanges();
+				} catch (DataException e) {
+					LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e);
+				}
+
+				Peer requestingPeer = request.getB();
+				if (requestingPeer != null) {
+					// Forward to requesting peer;
+					if (!requestingPeer.sendMessage(arbitraryDataMessage))
+						requestingPeer.disconnect("failed to forward arbitrary data");
+				}
+
+				break;
+			}
+
 			default:
 				LOGGER.debug(String.format("Unhandled %s message [ID %d] from peer %s", message.getType().name(), message.getId(), peer));
 				break;
@@ -788,6 +934,51 @@ public class Controller extends Thread {
 
 	// Utilities
 
+	public byte[] fetchArbitraryData(byte[] signature) throws InterruptedException {
+		// Build request
+		Message getArbitraryDataMessage = new GetArbitraryDataMessage(signature);
+
+		// Save our request into requests map
+		String signature58 = Base58.encode(signature);
+		Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+		// Assign random ID to this message
+		int id;
+		do {
+			id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+			// Put queue into map (keyed by message ID) so we can poll for a response
+			// If putIfAbsent() doesn't return null, then this ID is already taken
+		} while (arbitraryDataRequests.put(id, requestEntry) != null);
+		getArbitraryDataMessage.setId(id);
+
+		// Broadcast request
+		Network.getInstance().broadcast(peer -> peer.getVersion() < 2 ? null : getArbitraryDataMessage);
+
+		// Poll to see if data has arrived
+		final long singleWait = 100;
+		long totalWait = 0;
+		while (totalWait < ARBITRARY_REQUEST_TIMEOUT) {
+			Thread.sleep(singleWait);
+
+			requestEntry = arbitraryDataRequests.get(id);
+			if (requestEntry == null)
+				return null;
+
+			if (requestEntry.getA() == null)
+				break;
+
+			totalWait += singleWait;
+		}
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			return repository.getArbitraryRepository().fetchData(signature);
+		} catch (DataException e) {
+			LOGGER.error(String.format("Repository issue while fetching arbitrary transaction data"), e);
+			return null;
+		}
+	}
+
 	public static final Predicate<Peer> hasPeerMisbehaved = peer -> {
 		Long lastMisbehaved = peer.getPeerData().getLastMisbehaved();
 		return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF;

src/main/java/org/qora/data/transaction/ArbitraryTransactionData.java

@@ -11,7 +11,6 @@ import org.qora.data.PaymentData;
 import org.qora.transaction.Transaction.TransactionType;
 
 import io.swagger.v3.oas.annotations.media.Schema;
-import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
 
 // All properties to be converted to JSON via JAXB
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -21,7 +20,6 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
 public class ArbitraryTransactionData extends TransactionData {
 
 	// "data" field types
-	@Schema(accessMode = AccessMode.READ_ONLY)
 	public enum DataType {
 		RAW_DATA,
 		DATA_HASH;

src/main/java/org/qora/network/Peer.java

@@ -357,12 +357,12 @@ public class Peer implements Runnable {
 		// Assign random ID to this message
 		int id;
 		do {
-			id = new SecureRandom().nextInt(Integer.MAX_VALUE - 1) + 1;
+			id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
-			message.setId(id);
 
 			// Put queue into map (keyed by message ID) so we can poll for a response
 			// If putIfAbsent() doesn't return null, then this ID is already taken
 		} while (this.replyQueues.putIfAbsent(id, blockingQueue) != null);
+		message.setId(id);
 
 		// Try to send message
 		if (!this.sendMessage(message)) {

src/main/java/org/qora/network/message/ArbitraryDataMessage.java

@@ -0,0 +1,73 @@
+package org.qora.network.message;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+import org.qora.transform.Transformer;
+
+import com.google.common.primitives.Ints;
+
+public class ArbitraryDataMessage extends Message {
+
+	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
+
+	private byte[] signature;
+	private byte[] data;
+
+	public ArbitraryDataMessage(byte[] signature, byte[] data) {
+		this(-1, signature, data);
+	}
+
+	private ArbitraryDataMessage(int id, byte[] signature, byte[] data) {
+		super(id, MessageType.ARBITRARY_DATA);
+
+		this.signature = signature;
+		this.data = data;
+	}
+
+	public byte[] getSignature() {
+		return this.signature;
+	}
+
+	public byte[] getData() {
+		return this.data;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
+		byte[] signature = new byte[SIGNATURE_LENGTH];
+		byteBuffer.get(signature);
+
+		int dataLength = byteBuffer.getInt();
+
+		if (byteBuffer.remaining() != dataLength)
+			return null;
+
+		byte[] data = new byte[dataLength];
+		byteBuffer.get(data);
+
+		return new ArbitraryDataMessage(id, signature, data);
+	}
+
+	@Override
+	protected byte[] toData() {
+		if (this.data == null)
+			return null;
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+			bytes.write(this.signature);
+
+			bytes.write(Ints.toByteArray(this.data.length));
+
+			bytes.write(this.data);
+
+			return bytes.toByteArray();
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+}

src/main/java/org/qora/network/message/GetArbitraryDataMessage.java

@@ -0,0 +1,54 @@
+package org.qora.network.message;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+import org.qora.transform.Transformer;
+
+public class GetArbitraryDataMessage extends Message {
+
+	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
+
+	private byte[] signature;
+
+	public GetArbitraryDataMessage(byte[] signature) {
+		this(-1, signature);
+	}
+
+	private GetArbitraryDataMessage(int id, byte[] signature) {
+		super(id, MessageType.GET_ARBITRARY_DATA);
+
+		this.signature = signature;
+	}
+
+	public byte[] getSignature() {
+		return this.signature;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
+		if (bytes.remaining() != SIGNATURE_LENGTH)
+			return null;
+
+		byte[] signature = new byte[SIGNATURE_LENGTH];
+
+		bytes.get(signature);
+
+		return new GetArbitraryDataMessage(id, signature);
+	}
+
+	@Override
+	protected byte[] toData() {
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+			bytes.write(this.signature);
+
+			return bytes.toByteArray();
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+}

src/main/java/org/qora/network/message/Message.java

@@ -68,7 +68,9 @@ public abstract class Message {
 		HEIGHT_V2(19),
 		GET_TRANSACTION(20),
 		GET_UNCONFIRMED_TRANSACTIONS(21),
-		TRANSACTION_SIGNATURES(22);
+		TRANSACTION_SIGNATURES(22),
+		GET_ARBITRARY_DATA(23),
+		ARBITRARY_DATA(24);
 
 		public final int value;
 		public final Method fromByteBuffer;

src/main/java/org/qora/repository/hsqldb/HSQLDBArbitraryRepository.java

@@ -19,6 +19,8 @@ import org.qora.utils.Base58;
 
 public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
+	private static final int MAX_RAW_DATA_SIZE = 255; // size of VARBINARY
+
 	protected HSQLDBRepository repository;
 
 	public HSQLDBArbitraryRepository(HSQLDBRepository repository) {
@@ -51,30 +53,42 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		return stringBuilder.toString();
 	}
 
-	private String buildPathname(byte[] signature) throws DataException {
+	private ArbitraryTransactionData getTransactionData(byte[] signature) throws DataException {
 		TransactionData transactionData = this.repository.getTransactionRepository().fromSignature(signature);
 		if (transactionData == null)
 			return null;
 
-		return buildPathname((ArbitraryTransactionData) transactionData);
+		return (ArbitraryTransactionData) transactionData;
 	}
 
 	@Override
 	public boolean isDataLocal(byte[] signature) throws DataException {
-		String dataPathname = buildPathname(signature);
+		ArbitraryTransactionData transactionData = getTransactionData(signature);
-		if (dataPathname == null)
+		if (transactionData == null)
 			return false;
 
+		// Raw data is always available
+		if (transactionData.getDataType() == DataType.RAW_DATA)
+			return true;
+
+		String dataPathname = buildPathname(transactionData);
+
 		Path dataPath = Paths.get(dataPathname);
 		return Files.exists(dataPath);
 	}
 
 	@Override
 	public byte[] fetchData(byte[] signature) throws DataException {
-		String dataPathname = buildPathname(signature);
+		ArbitraryTransactionData transactionData = getTransactionData(signature);
-		if (dataPathname == null)
+		if (transactionData == null)
 			return null;
 
+		// Raw data is always available
+		if (transactionData.getDataType() == DataType.RAW_DATA)
+			return transactionData.getData();
+
+		String dataPathname = buildPathname(transactionData);
+
 		Path dataPath = Paths.get(dataPathname);
 		try {
 			return Files.readAllBytes(dataPath);
@@ -85,37 +99,47 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
 	@Override
 	public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
-		// Refuse to store raw data in the repository - it needs to be saved elsewhere!
+		// Already hashed? Nothing to do
-		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) {
+		if (arbitraryTransactionData.getDataType() == DataType.DATA_HASH)
-			byte[] rawData = arbitraryTransactionData.getData();
+			return;
 
-			// Calculate hash of data and update our transaction to use that
+		// Trivial-sized payloads can remain in raw form
-			byte[] dataHash = Crypto.digest(rawData);
+		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE)
-			arbitraryTransactionData.setData(dataHash);
+			return;
-			arbitraryTransactionData.setDataType(DataType.DATA_HASH);
 
-			String dataPathname = buildPathname(arbitraryTransactionData);
+		// Store non-trivial payloads in filesystem and convert transaction's data to hash form
+		byte[] rawData = arbitraryTransactionData.getData();
 
-			Path dataPath = Paths.get(dataPathname);
+		// Calculate hash of data and update our transaction to use that
+		byte[] dataHash = Crypto.digest(rawData);
+		arbitraryTransactionData.setData(dataHash);
+		arbitraryTransactionData.setDataType(DataType.DATA_HASH);
 
-			// Make sure directory structure exists
+		String dataPathname = buildPathname(arbitraryTransactionData);
-			try {
-				Files.createDirectories(dataPath.getParent());
-			} catch (IOException e) {
-				throw new DataException("Unable to create arbitrary transaction directory", e);
-			}
 
-			// Output actual transaction data
+		Path dataPath = Paths.get(dataPathname);
-			try (OutputStream dataOut = Files.newOutputStream(dataPath)) {
+
-				dataOut.write(rawData);
+		// Make sure directory structure exists
-			} catch (IOException e) {
+		try {
-				throw new DataException("Unable to store arbitrary transaction data", e);
+			Files.createDirectories(dataPath.getParent());
-			}
+		} catch (IOException e) {
+			throw new DataException("Unable to create arbitrary transaction directory", e);
+		}
+
+		// Output actual transaction data
+		try (OutputStream dataOut = Files.newOutputStream(dataPath)) {
+			dataOut.write(rawData);
+		} catch (IOException e) {
+			throw new DataException("Unable to store arbitrary transaction data", e);
 		}
 	}
 
 	@Override
 	public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
+		// No need to do anything if we still only have raw data, and hence nothing saved in filesystem
+		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA)
+			return;
+
 		String dataPathname = buildPathname(arbitraryTransactionData);
 		Path dataPath = Paths.get(dataPathname);
 		try {

src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -758,6 +758,14 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("CREATE INDEX TransactionApprovalHeightIndex on Transactions (approval_height)");
 					break;
 
+				case 52:
+					// Arbitrary transactions changes to allow storage of very small payloads locally
+					stmt.execute("CREATE TYPE ArbitraryData AS VARBINARY(255)");
+					stmt.execute("ALTER TABLE ArbitraryTransactions ADD COLUMN is_data_raw BOOLEAN NOT NULL");
+					stmt.execute("ALTER TABLE ArbitraryTransactions ALTER COLUMN data_hash ArbitraryData");
+					stmt.execute("ALTER TABLE ArbitraryTransactions ALTER COLUMN data_hash RENAME TO data");
+					break;
+
 				default:
 					// nothing to do
 					return false;

src/main/java/org/qora/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java

@@ -20,7 +20,7 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
 	}
 
 	TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException {
-		final String sql = "SELECT version, service, data_hash from ArbitraryTransactions WHERE signature = ?";
+		final String sql = "SELECT version, service, is_data_raw, data from ArbitraryTransactions WHERE signature = ?";
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) {
 			if (resultSet == null)
@@ -28,11 +28,13 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
 
 			int version = resultSet.getInt(1);
 			int service = resultSet.getInt(2);
-			byte[] dataHash = resultSet.getBytes(3);
+			boolean isDataRaw = resultSet.getBoolean(3); // NOT NULL, so no null to false
+			DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
+			byte[] data = resultSet.getBytes(4);
 
 			List<PaymentData> payments = this.getPaymentsFromSignature(baseTransactionData.getSignature());
 
-			return new ArbitraryTransactionData(baseTransactionData, version, service, dataHash, DataType.DATA_HASH, payments);
+			return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments);
 		} catch (SQLException e) {
 			throw new DataException("Unable to fetch arbitrary transaction from repository", e);
 		}
@@ -42,15 +44,15 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
 	public void save(TransactionData transactionData) throws DataException {
 		ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
 
-		// Refuse to store raw data in the repository - it needs to be saved elsewhere!
+		// For V4+, we might not store raw data in the repository but elsewhere
-		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA)
+		if (arbitraryTransactionData.getVersion() >= 4)
 			this.repository.getArbitraryRepository().save(arbitraryTransactionData);
 
 		HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryTransactions");
 
 		saveHelper.bind("signature", arbitraryTransactionData.getSignature()).bind("sender", arbitraryTransactionData.getSenderPublicKey())
 				.bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService())
-				.bind("data_hash", arbitraryTransactionData.getData());
+				.bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData());
 
 		try {
 			saveHelper.execute(this.repository);

src/main/java/org/qora/transaction/ArbitraryTransaction.java

@@ -148,6 +148,7 @@ public class ArbitraryTransaction extends Transaction {
 		return this.repository.getArbitraryRepository().isDataLocal(this.transactionData.getSignature());
 	}
 
+	/** Returns arbitrary data payload, fetching from network if needed. Can block for a while! */
 	public byte[] fetchData() throws DataException {
 		// If local, read from file
 		if (isDataLocal())

src/main/java/org/qora/transform/transaction/ArbitraryTransactionTransformer.java

@@ -27,7 +27,9 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 	// Property lengths
 	private static final int SERVICE_LENGTH = INT_LENGTH;
+	private static final int DATA_TYPE_LENGTH = BYTE_LENGTH;
 	private static final int DATA_SIZE_LENGTH = INT_LENGTH;
+	private static final int NUMBER_PAYMENTS_LENGTH = INT_LENGTH;
 
 	private static final int EXTRAS_LENGTH = SERVICE_LENGTH + DATA_SIZE_LENGTH;
 
@@ -47,6 +49,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 		layout.add("* payment amount", TransformationType.AMOUNT);
 
 		layout.add("service ID", TransformationType.INT);
+		layout.add("is data raw?", TransformationType.BOOLEAN);
 		layout.add("data length", TransformationType.INT);
 		layout.add("data", TransformationType.DATA);
 		layout.add("fee", TransformationType.AMOUNT);
@@ -78,6 +81,15 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 		int service = byteBuffer.getInt();
 
+		// With V4+ we might be receiving hash of data instead of actual raw data
+		DataType dataType = DataType.RAW_DATA;
+		if (version >= 4) {
+			boolean isRaw = byteBuffer.get() != 0;
+
+			if (!isRaw)
+				dataType = DataType.DATA_HASH;
+		}
+
 		int dataSize = byteBuffer.getInt();
 		// Don't allow invalid dataSize here to avoid run-time issues
 		if (dataSize > ArbitraryTransaction.MAX_DATA_SIZE)
@@ -93,17 +105,21 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 		BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature);
 
-		return new ArbitraryTransactionData(baseTransactionData, version, service, data, DataType.RAW_DATA, payments);
+		return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments);
 	}
 
 	public static int getDataLength(TransactionData transactionData) throws TransformationException {
 		ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
 
-		int length = getBaseLength(transactionData) + EXTRAS_LENGTH;
+		int length = getBaseLength(transactionData) + EXTRAS_LENGTH + arbitraryTransactionData.getData().length;
+
+		// V4+ transactions have data type
+		if (arbitraryTransactionData.getVersion() >= 4)
+			length += DATA_TYPE_LENGTH;
 
 		// V3+ transactions have optional payments
 		if (arbitraryTransactionData.getVersion() >= 3)
-			length += arbitraryTransactionData.getData().length + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength();
+			length += NUMBER_PAYMENTS_LENGTH + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength();
 
 		return length;
 	}
@@ -126,6 +142,10 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 			bytes.write(Ints.toByteArray(arbitraryTransactionData.getService()));
 
+			// V4+ also has data type
+			if (arbitraryTransactionData.getVersion() >= 4)
+				bytes.write((byte) (arbitraryTransactionData.getDataType() == DataType.RAW_DATA ? 1 : 0));
+
 			bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length));
 			bytes.write(arbitraryTransactionData.getData());
 
@@ -188,6 +208,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 			bytes.write(Ints.toByteArray(arbitraryTransactionData.getService()));
 
+			bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length));
 			switch (arbitraryTransactionData.getDataType()) {
 				case DATA_HASH:
 					bytes.write(arbitraryTransactionData.getData());

src/main/java/org/qora/utils/Serialization.java

@@ -94,7 +94,7 @@ public class Serialization {
 	public static void serializeSizedString(ByteArrayOutputStream bytes, String string) throws UnsupportedEncodingException, IOException {
 		byte[] stringBytes = string.getBytes("UTF-8");
 		bytes.write(Ints.toByteArray(stringBytes.length));
-		bytes.write(string.getBytes("UTF-8"));
+		bytes.write(stringBytes);
 	}
 
 	public static String deserializeSizedString(ByteBuffer byteBuffer, int maxSize) throws TransformationException {