Initial auto-update support, API improvements, arbitrary tx improvements

Removed all @Produces from API resources as response content type is
sorted by Swagger.

Added API /admin/info for generic node info.

Added API /arbitrary/ endpoints.

Moved arbitrary data storage from ArbitraryTransaction to ArbitraryRepository.

V4 arbitrary transaction signature is based on data's hash.

Original commit was d02f282, and commit message was:
Initial auto-update support, network MAGIC change, arbitrary tx improvements

src/main/java/org/qora/AutoUpdate.java

@@ -0,0 +1,189 @@
+package org.qora;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.security.Security;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAnyElement;
+
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
+import org.qora.api.ApiRequest;
+import org.qora.api.model.NodeInfo;
+import org.qora.data.transaction.ArbitraryTransactionData;
+import org.qora.data.transaction.TransactionData;
+import org.qora.settings.Settings;
+import org.qora.utils.Base58;
+
+import com.google.common.hash.HashCode;
+
+public class AutoUpdate {
+
+	private static final String JAR_FILENAME = "qora-core.jar";
+	private static final String NODE_EXE = "qora-core.exe";
+
+	private static final long CHECK_INTERVAL = 1 * 1000;
+	private static final int MAX_ATTEMPTS = 10;
+
+	private static final Map<String, String> ARBITRARY_PARAMS = new HashMap<>();
+	static {
+		ARBITRARY_PARAMS.put("txGroupID", "1"); // dev group
+		ARBITRARY_PARAMS.put("service", "1"); // "update" service
+		ARBITRARY_PARAMS.put("confirmationStatus", "CONFIRMED");
+		ARBITRARY_PARAMS.put("limit", "1");
+		ARBITRARY_PARAMS.put("reverse", "true");
+	}
+
+	static {
+		// This must go before any calls to LogManager/Logger
+		System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
+	}
+
+	@XmlAccessorType(XmlAccessType.FIELD)
+	public static class Transactions {
+		@XmlAnyElement(lax = true)
+		public List<TransactionData> transactions;
+
+		public Transactions() {
+		}
+	}
+
+	public static void main(String[] args) {
+		Security.insertProviderAt(new BouncyCastleProvider(), 0);
+		Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
+
+		// Load/check settings, which potentially sets up blockchain config, etc.
+		Settings.getInstance();
+
+		final String BASE_URI = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
+
+		Long buildTimestamp = null; // ms
+		
+		while (true) {
+			try {
+				Thread.sleep(CHECK_INTERVAL);
+			} catch (InterruptedException e) {
+				return;
+			}
+
+			// If we don't know current node's version then grab that
+			if (buildTimestamp == null) {
+				// Grab node version and timestamp
+				Object response = ApiRequest.perform(BASE_URI + "admin/info", NodeInfo.class, null);
+				if (response == null || !(response instanceof NodeInfo))
+					continue;
+
+				NodeInfo nodeInfo = (NodeInfo) response;
+				buildTimestamp = nodeInfo.buildTimestamp * 1000L;
+			}
+
+			// Look for "update" tx which is arbitrary tx with service 1 and timestamp later than buildTimestamp
+			// http://localhost:9085/arbitrary/search?txGroupId=1&service=1&confirmationStatus=CONFIRMED&limit=1&reverse=true
+			Object response = ApiRequest.perform(BASE_URI + "arbitrary/search", TransactionData.class, ARBITRARY_PARAMS);
+			if (response == null || !(response instanceof List<?>))
+				continue;
+
+			List<?> listResponse = (List<?>) response;
+			if (listResponse.isEmpty() || !(listResponse.get(0) instanceof TransactionData))
+				continue;
+
+			@SuppressWarnings("unchecked")
+			TransactionData transactionData = ((List<TransactionData>) listResponse).get(0);
+
+			if (transactionData.getTimestamp() <= buildTimestamp)
+				continue;
+
+			ArbitraryTransactionData arbitraryTxData = (ArbitraryTransactionData) transactionData;
+
+			// Arbitrary transaction's data contains git commit hash needed to grab JAR:
+			// https://github.com/catbref/qora-core/blob/cf86b5f3ce828f75cb18db1b685f2d9e29630d77/qora-core.jar
+			InputStream in = ApiRequest.fetchStream(BASE_URI + "arbitrary/raw/" + Base58.encode(arbitraryTxData.getSignature()));
+			if (in == null)
+				continue;
+
+			byte[] commitHash = new byte[20];
+			try {
+				in.read(commitHash);
+			} catch (IOException e) {
+				continue;
+			}
+
+			String[] autoUpdateRepos = Settings.getInstance().getAutoUpdateRepos();
+			for (String repo : autoUpdateRepos)
+				if (attemptUpdate(commitHash, repo, BASE_URI))
+					break;
+
+			// Reset cached node info in case we've updated
+			buildTimestamp = null;
+		}
+	}
+
+	private static boolean attemptUpdate(byte[] commitHash, String repoBaseUri, String BASE_URI) {
+		Path realJar = Paths.get(System.getProperty("user.dir"), JAR_FILENAME);
+
+		Path tmpJar = null;
+		InputStream in = ApiRequest.fetchStream(repoBaseUri + "/raw/" + HashCode.fromBytes(commitHash).toString() + "/" + JAR_FILENAME);
+		if (in == null)
+			return false;
+
+		try {
+			// Save input stream into temporary file
+			tmpJar = Files.createTempFile(JAR_FILENAME + "-", null);
+			Files.copy(in, tmpJar, StandardCopyOption.REPLACE_EXISTING);
+
+			// Keep trying to shutdown node
+			for (int i = 0; i < MAX_ATTEMPTS; ++i) {
+				String response = ApiRequest.perform(BASE_URI + "admin/stop", null);
+				if (response == null || !response.equals("true"))
+					break;
+
+				try {
+					Thread.sleep(5000);
+				} catch (InterruptedException e) {
+					// We still need to restart the node!
+					break;
+				}
+			}
+
+			try {
+				Files.move(tmpJar, realJar, StandardCopyOption.REPLACE_EXISTING);
+			} catch (IOException e) {
+				// Failed to replace but we still need to restart node
+			}
+
+			// Restart node!
+			restartNode();
+
+			return true;
+		} catch (IOException e) {
+			// Couldn't close input stream - fail?
+			return false;
+		} finally {
+			if (tmpJar != null)
+				try {
+					Files.deleteIfExists(tmpJar);
+				} catch (IOException e) {
+					// we tried...
+				}
+
+		}
+	}
+
+	private static void restartNode() {
+		try {
+			Path execPath = Paths.get(System.getProperty("user.dir"), NODE_EXE);
+			new ProcessBuilder(execPath.toString()).start();
+		} catch (IOException e) {
+		}
+	}
+
+}

src/main/java/org/qora/api/ApiRequest.java

@@ -0,0 +1,178 @@
+package org.qora.api;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLEncoder;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Scanner;
+
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SNIHostName;
+import javax.net.ssl.SNIServerName;
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.SSLSocket;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.UnmarshalException;
+import javax.xml.bind.Unmarshaller;
+import javax.xml.transform.stream.StreamSource;
+
+import org.bouncycastle.jsse.util.CustomSSLSocketFactory;
+import org.eclipse.persistence.exceptions.XMLMarshalException;
+import org.eclipse.persistence.jaxb.JAXBContextFactory;
+import org.eclipse.persistence.jaxb.UnmarshallerProperties;
+
+public class ApiRequest {
+
+	public static String perform(String uri, Map<String, String> params) {
+		if (params != null && !params.isEmpty())
+			uri += "?" + getParamsString(params);
+
+		InputStream in = fetchStream(uri);
+		if (in == null)
+			return null;
+
+		try (Scanner scanner = new Scanner(in, "UTF8")) {
+			scanner.useDelimiter("\\A");
+			return scanner.hasNext() ? scanner.next() : "";
+		} finally {
+			try {
+				in.close();
+			} catch (IOException e) {
+				// We tried...
+			}
+		}
+	}
+
+	public static Object perform(String uri, Class<?> responseClass, Map<String, String> params) {
+		Unmarshaller unmarshaller = createUnmarshaller(responseClass);
+
+		if (params != null && !params.isEmpty())
+			uri += "?" + getParamsString(params);
+
+		InputStream in = fetchStream(uri);
+		if (in == null)
+			return null;
+
+		try {
+			StreamSource json = new StreamSource(in);
+
+			// Attempt to unmarshal JSON stream to Settings
+			return unmarshaller.unmarshal(json, responseClass).getValue();
+		} catch (UnmarshalException e) {
+			Throwable linkedException = e.getLinkedException();
+			if (linkedException instanceof XMLMarshalException) {
+				String message = ((XMLMarshalException) linkedException).getInternalException().getLocalizedMessage();
+				throw new RuntimeException(message);
+			}
+
+			throw new RuntimeException("Unable to unmarshall API response", e);
+		} catch (JAXBException e) {
+			throw new RuntimeException("Unable to unmarshall API response", e);
+		}
+	}
+
+	private static Unmarshaller createUnmarshaller(Class<?> responseClass) {
+		try {
+			// Create JAXB context aware of Settings
+			JAXBContext jc = JAXBContextFactory.createContext(new Class[] { responseClass }, null);
+
+			// Create unmarshaller
+			Unmarshaller unmarshaller = jc.createUnmarshaller();
+
+			// Set the unmarshaller media type to JSON
+			unmarshaller.setProperty(UnmarshallerProperties.MEDIA_TYPE, "application/json");
+
+			// Tell unmarshaller that there's no JSON root element in the JSON input
+			unmarshaller.setProperty(UnmarshallerProperties.JSON_INCLUDE_ROOT, false);
+
+			return unmarshaller;
+		} catch (JAXBException e) {
+			throw new RuntimeException("Unable to create API unmarshaller", e);
+		}
+	}
+
+	public static String getParamsString(Map<String, String> params) {
+		StringBuilder result = new StringBuilder();
+
+		try {
+			for (Map.Entry<String, String> entry : params.entrySet()) {
+				result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
+				result.append("=");
+				result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
+				result.append("&");
+			}
+		} catch (UnsupportedEncodingException e) {
+			throw new RuntimeException("Cannot encode API request params", e);
+		}
+
+		String resultString = result.toString();
+		return resultString.length() > 0 ? resultString.substring(0, resultString.length() - 1) : resultString;
+	}
+
+	public static InputStream fetchStream(String uri) {
+		try {
+			URL url = new URL(uri);
+			HttpURLConnection con = (HttpURLConnection) url.openConnection();
+
+			try {
+				con.setRequestMethod("GET");
+				con.setConnectTimeout(5000);
+				con.setReadTimeout(5000);
+				ApiRequest.setConnectionSSL(con);
+
+				try {
+					int status = con.getResponseCode();
+
+					if (status != 200)
+						return null;
+				} catch (IOException e) {
+					return null;
+				}
+
+				return con.getInputStream();
+			} catch (IOException e) {
+				return null;
+			}
+		} catch (MalformedURLException e) {
+			throw new RuntimeException("Malformed API request", e);
+		} catch (IOException e) {
+			// Temporary fail
+			return null;
+		}
+	}
+
+	public static void setConnectionSSL(HttpURLConnection con) {
+		if (!(con instanceof HttpsURLConnection))
+			return;
+
+		HttpsURLConnection httpsCon = (HttpsURLConnection) con;
+		URL url = con.getURL();
+
+		httpsCon.setSSLSocketFactory(new org.bouncycastle.jsse.util.CustomSSLSocketFactory(httpsCon.getSSLSocketFactory()) {
+			@Override
+			protected Socket configureSocket(Socket s) {
+				if (s instanceof SSLSocket) {
+					SSLSocket ssl = (SSLSocket) s;
+
+					SNIHostName sniHostName = new SNIHostName(url.getHost());
+					if (null != sniHostName) {
+						SSLParameters sslParameters = new SSLParameters();
+
+						sslParameters.setServerNames(Collections.<SNIServerName>singletonList(sniHostName));
+						ssl.setSSLParameters(sslParameters);
+					}
+				}
+
+				return s;
+			}
+		});
+	}
+
+}

src/main/java/org/qora/api/model/NodeInfo.java

@@ -0,0 +1,16 @@
+package org.qora.api.model;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NodeInfo {
+
+	public long uptime;
+	public String buildVersion;
+	public long buildTimestamp;
+
+	public NodeInfo() {
+	}
+
+}

src/main/java/org/qora/api/resource/AddressesResource.java

@@ -17,7 +17,6 @@ import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
@@ -46,7 +45,6 @@ import org.qora.transform.transaction.ProxyForgingTransactionTransformer;
 import org.qora.utils.Base58;
 
 @Path("/addresses")
-@Produces({MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN})
 @Tag(name = "Addresses")
 public class AddressesResource {
 
@@ -334,14 +332,20 @@ public class AddressesResource {
 		}
 	)
 	public String calculateProxyKey(@PathParam("generatorprivatekey") String generatorKey58, @PathParam("recipientpublickey") String recipientKey58) {
-		PrivateKeyAccount generator = new PrivateKeyAccount(null, Base58.decode(generatorKey58));
-		byte[] recipientKey = Base58.decode(recipientKey58);
+		try {
+			byte[] generatorKey = Base58.decode(generatorKey58);
+			byte[] recipientKey = Base58.decode(recipientKey58);
+			if (generatorKey.length != Transformer.PRIVATE_KEY_LENGTH || recipientKey.length != Transformer.PRIVATE_KEY_LENGTH)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
+			PrivateKeyAccount generator = new PrivateKeyAccount(null, generatorKey);
+			byte[] sharedSecret = generator.getSharedSecret(recipientKey);
 
-		byte[] sharedSecret = generator.getSharedSecret(recipientKey);
+			byte[] proxySeed = Crypto.digest(sharedSecret);
 
-		byte[] proxySeed = Crypto.digest(sharedSecret);
-
-		return Base58.encode(proxySeed);
+			return Base58.encode(proxySeed);
+		} catch (NumberFormatException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e);
+		}
 	}
 
 	@POST

src/main/java/org/qora/api/resource/AdminResource.java

@@ -25,7 +25,6 @@ import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
@@ -39,6 +38,7 @@ import org.qora.api.ApiErrors;
 import org.qora.api.ApiExceptionFactory;
 import org.qora.api.Security;
 import org.qora.api.model.ActivitySummary;
+import org.qora.api.model.NodeInfo;
 import org.qora.controller.Controller;
 import org.qora.repository.DataException;
 import org.qora.repository.Repository;
@@ -50,7 +50,6 @@ import org.qora.utils.Base58;
 import com.google.common.collect.Lists;
 
 @Path("/admin")
-@Produces({MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN})
 @Tag(name = "Admin")
 public class AdminResource {
 
@@ -88,6 +87,26 @@ public class AdminResource {
 		return System.currentTimeMillis() - Controller.startTime;
 	}
 
+	@GET
+	@Path("/info")
+	@Operation(
+		summary = "Fetch generic node info",
+		responses = {
+			@ApiResponse(
+				content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = NodeInfo.class))
+			)
+		}
+	)
+	public NodeInfo info() {
+		NodeInfo nodeInfo = new NodeInfo();
+
+		nodeInfo.uptime = System.currentTimeMillis() - Controller.startTime;
+		nodeInfo.buildVersion = Controller.getInstance().getVersionString();
+		nodeInfo.buildTimestamp = Controller.getInstance().getBuildTimestamp();
+
+		return nodeInfo;
+	}
+
 	@GET
 	@Path("/stop")
 	@Operation(

src/main/java/org/qora/api/resource/ApiDefinition.java

@@ -11,6 +11,7 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 		tags = {
 			@Tag(name = "Addresses"),
 			@Tag(name = "Admin"),
+			@Tag(name = "Arbitrary"),
 			@Tag(name = "Assets"),
 			@Tag(name = "Blocks"),
 			@Tag(name = "Groups"),

src/main/java/org/qora/api/resource/ArbitraryResource.java

@@ -0,0 +1,210 @@
+package org.qora.api.resource;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.ArraySchema;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.tags.Tag;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+
+import org.qora.api.ApiError;
+import org.qora.api.ApiErrors;
+import org.qora.api.ApiException;
+import org.qora.api.ApiExceptionFactory;
+import org.qora.api.resource.TransactionsResource.ConfirmationStatus;
+import org.qora.data.transaction.ArbitraryTransactionData;
+import org.qora.data.transaction.TransactionData;
+import org.qora.data.transaction.ArbitraryTransactionData.DataType;
+import org.qora.repository.DataException;
+import org.qora.repository.Repository;
+import org.qora.repository.RepositoryManager;
+import org.qora.settings.Settings;
+import org.qora.transaction.ArbitraryTransaction;
+import org.qora.transaction.Transaction;
+import org.qora.transaction.Transaction.TransactionType;
+import org.qora.transaction.Transaction.ValidationResult;
+import org.qora.transform.TransformationException;
+import org.qora.transform.transaction.ArbitraryTransactionTransformer;
+import org.qora.utils.Base58;
+
+@Path("/arbitrary")
+@Tag(name = "Arbitrary")
+public class ArbitraryResource {
+
+	@Context
+	HttpServletRequest request;
+	
+	@GET
+	@Path("/search")
+	@Operation(
+		summary = "Find matching arbitrary transactions",
+		description = "Returns transactions that match criteria. At least either service or address or limit <= 20 must be provided. Block height ranges allowed when searching CONFIRMED transactions ONLY.",
+		responses = {
+			@ApiResponse(
+				description = "transactions",
+				content = @Content(
+					array = @ArraySchema(
+						schema = @Schema(
+							implementation = TransactionData.class
+						)
+					)
+				)
+			)
+		}
+	)
+	@ApiErrors({
+		ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE
+	})
+	public List<TransactionData> searchTransactions(@QueryParam("startBlock") Integer startBlock, @QueryParam("blockLimit") Integer blockLimit,
+			@QueryParam("txGroupId") Integer txGroupId,
+			@QueryParam("service") Integer service, @QueryParam("address") String address, @Parameter(
+				description = "whether to include confirmed, unconfirmed or both",
+				required = true
+			) @QueryParam("confirmationStatus") ConfirmationStatus confirmationStatus, @Parameter(
+				ref = "limit"
+			) @QueryParam("limit") Integer limit, @Parameter(
+				ref = "offset"
+			) @QueryParam("offset") Integer offset, @Parameter(
+				ref = "reverse"
+			) @QueryParam("reverse") Boolean reverse) {
+		// Must have at least one of txType / address / limit <= 20
+		if (service == null && (address == null || address.isEmpty()) && (limit == null || limit > 20))
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		// You can't ask for unconfirmed and impose a block height range
+		if (confirmationStatus != ConfirmationStatus.CONFIRMED && (startBlock != null || blockLimit != null))
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		List<TransactionType> txTypes = new ArrayList<>();
+		txTypes.add(TransactionType.ARBITRARY);
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txGroupId, txTypes, 
+					service, address, confirmationStatus, limit, offset, reverse);
+
+			// Expand signatures to transactions
+			List<TransactionData> transactions = new ArrayList<TransactionData>(signatures.size());
+			for (byte[] signature : signatures)
+				transactions.add(repository.getTransactionRepository().fromSignature(signature));
+
+			return transactions;
+		} catch (ApiException e) {
+			throw e;
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
+	}
+
+	@GET
+	@Path("/raw/{signature}")
+	@Operation(
+		summary = "Fetch raw data associated with passed transaction signature",
+		responses = {
+			@ApiResponse(
+				description = "raw data",
+				content = @Content(
+					schema = @Schema(type = "string", format = "byte"),
+					mediaType = MediaType.APPLICATION_OCTET_STREAM
+				)
+			)
+		}
+	)
+	@ApiErrors({
+		ApiError.INVALID_SIGNATURE, ApiError.REPOSITORY_ISSUE, ApiError.TRANSACTION_INVALID
+	})
+	public byte[] fetchRawData(@PathParam("signature") String signature58) {
+		// Decode signature
+		byte[] signature;
+		try {
+			signature = Base58.decode(signature58);
+		} catch (NumberFormatException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_SIGNATURE, e);
+		}
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+
+			if (transactionData == null || transactionData.getType() != TransactionType.ARBITRARY) 
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_SIGNATURE);
+
+			ArbitraryTransactionData arbitraryTxData = (ArbitraryTransactionData) transactionData;
+
+			// We're really expecting to only fetch the data's hash from repository
+			if (arbitraryTxData.getDataType() != DataType.DATA_HASH)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID);
+
+			ArbitraryTransaction arbitraryTx = new ArbitraryTransaction(repository, arbitraryTxData);
+
+			// For now, we only allow locally stored data
+			if (!arbitraryTx.isDataLocal())
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID);
+
+			return arbitraryTx.fetchData();
+		} catch (ApiException e) {
+			throw e;
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
+	}
+
+	@POST
+	@Path("/")
+	@Operation(
+		summary = "Build raw, unsigned, ARBITRARY transaction",
+		requestBody = @RequestBody(
+			required = true,
+			content = @Content(
+				mediaType = MediaType.APPLICATION_JSON,
+				schema = @Schema(
+					implementation = ArbitraryTransactionData.class
+				)
+			)
+		),
+		responses = {
+			@ApiResponse(
+				description = "raw, unsigned, ARBITRARY transaction encoded in Base58",
+				content = @Content(
+					mediaType = MediaType.TEXT_PLAIN,
+					schema = @Schema(
+						type = "string"
+					)
+				)
+			)
+		}
+	)
+	@ApiErrors({ApiError.NON_PRODUCTION, ApiError.TRANSACTION_INVALID, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
+	public String createArbitrary(ArbitraryTransactionData transactionData) {
+		if (Settings.getInstance().isApiRestricted())
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION);
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			Transaction transaction = Transaction.fromData(repository, transactionData);
+
+			ValidationResult result = transaction.isValidUnconfirmed();
+			if (result != ValidationResult.OK)
+				throw TransactionsResource.createTransactionInvalidException(request, result);
+
+			byte[] bytes = ArbitraryTransactionTransformer.toBytes(transactionData);
+			return Base58.encode(bytes);
+		} catch (TransformationException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
+	}
+
+}

src/main/java/org/qora/api/resource/AssetsResource.java

@@ -20,7 +20,6 @@ import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
@@ -62,12 +61,7 @@ import org.qora.transform.transaction.UpdateAssetTransactionTransformer;
 import org.qora.utils.Base58;
 
 @Path("/assets")
-@Produces({
-	MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN
-})
-@Tag(
-	name = "Assets"
-)
+@Tag(name = "Assets")
 public class AssetsResource {
 
 	@Context

src/main/java/org/qora/api/resource/BlocksResource.java

@@ -19,7 +19,6 @@ import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
@@ -46,12 +45,7 @@ import org.qora.transform.transaction.EnableForgingTransactionTransformer;
 import org.qora.utils.Base58;
 
 @Path("/blocks")
-@Produces({
-	MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN
-})
-@Tag(
-	name = "Blocks"
-)
+@Tag(name = "Blocks")
 public class BlocksResource {
 
 	@Context

src/main/java/org/qora/api/resource/GroupsResource.java

@@ -18,7 +18,6 @@ import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
@@ -71,7 +70,6 @@ import org.qora.transform.transaction.UpdateGroupTransactionTransformer;
 import org.qora.utils.Base58;
 
 @Path("/groups")
-@Produces({MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN})
 @Tag(name = "Groups")
 public class GroupsResource {
 

src/main/java/org/qora/api/resource/NamesResource.java

@@ -17,7 +17,6 @@ import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
@@ -48,7 +47,6 @@ import org.qora.transform.transaction.UpdateNameTransactionTransformer;
 import org.qora.utils.Base58;
 
 @Path("/names")
-@Produces({MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN})
 @Tag(name = "Names")
 public class NamesResource {
 

src/main/java/org/qora/api/resource/PaymentsResource.java

@@ -10,7 +10,6 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -29,7 +28,6 @@ import org.qora.transform.transaction.PaymentTransactionTransformer;
 import org.qora.utils.Base58;
 
 @Path("/payments")
-@Produces({MediaType.TEXT_PLAIN})
 @Tag(name = "Payments")
 public class PaymentsResource {
 

src/main/java/org/qora/api/resource/PeersResource.java

@@ -16,7 +16,6 @@ import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -34,12 +33,7 @@ import org.qora.repository.Repository;
 import org.qora.repository.RepositoryManager;
 
 @Path("/peers")
-@Produces({
-	MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN
-})
-@Tag(
-	name = "Peers"
-)
+@Tag(name = "Peers")
 public class PeersResource {
 
 	@Context

src/main/java/org/qora/api/resource/TransactionsResource.java

@@ -17,7 +17,6 @@ import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
@@ -45,12 +44,7 @@ import org.qora.utils.Base58;
 import com.google.common.primitives.Bytes;
 
 @Path("/transactions")
-@Produces({
-	MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN
-})
-@Tag(
-	name = "Transactions"
-)
+@Tag(name = "Transactions")
 public class TransactionsResource {
 
 	@Context
@@ -291,6 +285,7 @@ public class TransactionsResource {
 		ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE
 	})
 	public List<TransactionData> searchTransactions(@QueryParam("startBlock") Integer startBlock, @QueryParam("blockLimit") Integer blockLimit,
+			@QueryParam("txGroupId") Integer txGroupId,
 			@QueryParam("txType") List<TransactionType> txTypes, @QueryParam("address") String address, @Parameter(
 				description = "whether to include confirmed, unconfirmed or both",
 				required = true
@@ -310,8 +305,8 @@ public class TransactionsResource {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txTypes, address,
-					confirmationStatus, limit, offset, reverse);
+			List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txGroupId,
+					txTypes, null, address, confirmationStatus, limit, offset, reverse);
 
 			// Expand signatures to transactions
 			List<TransactionData> transactions = new ArrayList<TransactionData>(signatures.size());

src/main/java/org/qora/api/resource/UtilsResource.java

@@ -19,7 +19,6 @@ import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
@@ -31,6 +30,7 @@ import org.qora.api.ApiExceptionFactory;
 import org.qora.crypto.Crypto;
 import org.qora.settings.Settings;
 import org.qora.transaction.Transaction.TransactionType;
+import org.qora.transform.Transformer;
 import org.qora.transform.transaction.TransactionTransformer;
 import org.qora.transform.transaction.TransactionTransformer.Transformation;
 import org.qora.utils.BIP39;
@@ -42,12 +42,7 @@ import com.google.common.primitives.Bytes;
 import com.google.common.primitives.Longs;
 
 @Path("/utils")
-@Produces({
-	MediaType.TEXT_PLAIN, MediaType.APPLICATION_JSON
-})
-@Tag(
-	name = "Utilities"
-)
+@Tag(name = "Utilities")
 public class UtilsResource {
 
 	@Context
@@ -382,7 +377,7 @@ public class UtilsResource {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
 		}
 
-		if (privateKey.length != 32)
+		if (privateKey.length != Transformer.PRIVATE_KEY_LENGTH)
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
 
 		try {

src/main/java/org/qora/data/transaction/ArbitraryTransactionData.java

@@ -7,17 +7,22 @@ import javax.xml.bind.Unmarshaller;
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 
+import org.eclipse.persistence.oxm.annotations.XmlDiscriminatorValue;
 import org.qora.data.PaymentData;
 import org.qora.transaction.Transaction.TransactionType;
 
 import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
 
 // All properties to be converted to JSON via JAXB
 @XmlAccessorType(XmlAccessType.FIELD)
 @Schema(allOf = { TransactionData.class })
+//JAXB: use this subclass if XmlDiscriminatorNode matches XmlDiscriminatorValue below:
+@XmlDiscriminatorValue("ARBITRARY")
 public class ArbitraryTransactionData extends TransactionData {
 
 	// "data" field types
+	@Schema(accessMode = AccessMode.READ_ONLY)
 	public enum DataType {
 		RAW_DATA,
 		DATA_HASH;

src/main/java/org/qora/repository/ArbitraryRepository.java

@@ -0,0 +1,15 @@
+package org.qora.repository;
+
+import org.qora.data.transaction.ArbitraryTransactionData;
+
+public interface ArbitraryRepository {
+
+	public boolean isDataLocal(byte[] signature) throws DataException;
+
+	public byte[] fetchData(byte[] signature) throws DataException;
+
+	public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
+
+	public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
+
+}

src/main/java/org/qora/repository/Repository.java

@@ -6,6 +6,8 @@ public interface Repository extends AutoCloseable {
 
 	public AccountRepository getAccountRepository();
 
+	public ArbitraryRepository getArbitraryRepository();
+
 	public AssetRepository getAssetRepository();
 
 	public BlockRepository getBlockRepository();

src/main/java/org/qora/repository/TransactionRepository.java

@@ -46,7 +46,8 @@ public interface TransactionRepository {
 	 */
 	public Map<TransactionType, Integer> getTransactionSummary(int startHeight, int endHeight) throws DataException;
 
-	public List<byte[]> getSignaturesMatchingCriteria(Integer startBlock, Integer blockLimit, List<TransactionType> txTypes, String address,
+	public List<byte[]> getSignaturesMatchingCriteria(Integer startBlock, Integer blockLimit, Integer txGroupId,
+			List<TransactionType> txTypes, Integer service, String address,
 			ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse) throws DataException;
 
 	/**

src/main/java/org/qora/repository/hsqldb/HSQLDBArbitraryRepository.java

@@ -0,0 +1,138 @@
+package org.qora.repository.hsqldb;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.DirectoryNotEmptyException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.qora.crypto.Crypto;
+import org.qora.data.transaction.ArbitraryTransactionData;
+import org.qora.data.transaction.ArbitraryTransactionData.DataType;
+import org.qora.data.transaction.TransactionData;
+import org.qora.repository.ArbitraryRepository;
+import org.qora.repository.DataException;
+import org.qora.settings.Settings;
+import org.qora.utils.Base58;
+
+public class HSQLDBArbitraryRepository implements ArbitraryRepository {
+
+	protected HSQLDBRepository repository;
+
+	public HSQLDBArbitraryRepository(HSQLDBRepository repository) {
+		this.repository = repository;
+	}
+
+	/**
+	 * Returns pathname for saving arbitrary transaction data payloads.
+	 * <p>
+	 * Format: <tt>arbitrary/<sender>/<service><tx-sig>.raw</tt>
+	 * 
+	 * @param arbitraryTransactionData
+	 * @return
+	 */
+	public static String buildPathname(ArbitraryTransactionData arbitraryTransactionData) {
+		String senderAddress = Crypto.toAddress(arbitraryTransactionData.getSenderPublicKey());
+
+		StringBuilder stringBuilder = new StringBuilder(1024);
+
+		stringBuilder.append(Settings.getInstance().getUserPath());
+		stringBuilder.append("arbitrary");
+		stringBuilder.append(File.separator);
+		stringBuilder.append(senderAddress);
+		stringBuilder.append(File.separator);
+		stringBuilder.append(arbitraryTransactionData.getService());
+		stringBuilder.append(File.separator);
+		stringBuilder.append(Base58.encode(arbitraryTransactionData.getSignature()));
+		stringBuilder.append(".raw");
+
+		return stringBuilder.toString();
+	}
+
+	private String buildPathname(byte[] signature) throws DataException {
+		TransactionData transactionData = this.repository.getTransactionRepository().fromSignature(signature);
+		if (transactionData == null)
+			return null;
+
+		return buildPathname((ArbitraryTransactionData) transactionData);
+	}
+
+	@Override
+	public boolean isDataLocal(byte[] signature) throws DataException {
+		String dataPathname = buildPathname(signature);
+		if (dataPathname == null)
+			return false;
+
+		Path dataPath = Paths.get(dataPathname);
+		return Files.exists(dataPath);
+	}
+
+	@Override
+	public byte[] fetchData(byte[] signature) throws DataException {
+		String dataPathname = buildPathname(signature);
+		if (dataPathname == null)
+			return null;
+
+		Path dataPath = Paths.get(dataPathname);
+		try {
+			return Files.readAllBytes(dataPath);
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+	@Override
+	public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
+		// Refuse to store raw data in the repository - it needs to be saved elsewhere!
+		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) {
+			byte[] rawData = arbitraryTransactionData.getData();
+
+			// Calculate hash of data and update our transaction to use that
+			byte[] dataHash = Crypto.digest(rawData);
+			arbitraryTransactionData.setData(dataHash);
+			arbitraryTransactionData.setDataType(DataType.DATA_HASH);
+
+			String dataPathname = buildPathname(arbitraryTransactionData);
+
+			Path dataPath = Paths.get(dataPathname);
+
+			// Make sure directory structure exists
+			try {
+				Files.createDirectories(dataPath.getParent());
+			} catch (IOException e) {
+				throw new DataException("Unable to create arbitrary transaction directory", e);
+			}
+
+			// Output actual transaction data
+			try (OutputStream dataOut = Files.newOutputStream(dataPath)) {
+				dataOut.write(rawData);
+			} catch (IOException e) {
+				throw new DataException("Unable to store arbitrary transaction data", e);
+			}
+		}
+	}
+
+	@Override
+	public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
+		String dataPathname = buildPathname(arbitraryTransactionData);
+		Path dataPath = Paths.get(dataPathname);
+		try {
+			Files.deleteIfExists(dataPath);
+
+			// Also attempt to delete parent <service> directory if empty
+			Path servicePath = dataPath.getParent();
+			Files.deleteIfExists(servicePath);
+
+			// Also attempt to delete parent <sender's address> directory if empty
+			Path senderpath = servicePath.getParent();
+			Files.deleteIfExists(senderpath);
+		} catch (DirectoryNotEmptyException e) {
+			// One of the parent service/sender directories still has data from other transactions - this is OK
+		} catch (IOException e) {
+			throw new DataException("Unable to delete arbitrary transaction data", e);
+		}
+	}
+
+}

src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java

@@ -21,6 +21,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qora.repository.ATRepository;
 import org.qora.repository.AccountRepository;
+import org.qora.repository.ArbitraryRepository;
 import org.qora.repository.AssetRepository;
 import org.qora.repository.BlockRepository;
 import org.qora.repository.GroupRepository;
@@ -83,6 +84,11 @@ public class HSQLDBRepository implements Repository {
 		return new HSQLDBAccountRepository(this);
 	}
 
+	@Override
+	public ArbitraryRepository getArbitraryRepository() {
+		return new HSQLDBArbitraryRepository(this);
+	}
+
 	@Override
 	public AssetRepository getAssetRepository() {
 		return new HSQLDBAssetRepository(this);

src/main/java/org/qora/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java

@@ -43,8 +43,8 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
 		ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
 
 		// Refuse to store raw data in the repository - it needs to be saved elsewhere!
-		if (arbitraryTransactionData.getDataType() != DataType.DATA_HASH)
-			throw new DataException("Refusing to save arbitrary transaction data into repository");
+		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA)
+			this.repository.getArbitraryRepository().save(arbitraryTransactionData);
 
 		HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryTransactions");
 
@@ -63,4 +63,11 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
 			this.savePayments(transactionData.getSignature(), arbitraryTransactionData.getPayments());
 	}
 
+	public void delete(TransactionData transactionData) throws DataException {
+		ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+
+		// Potentially delete raw data stored locally too
+		this.repository.getArbitraryRepository().delete(arbitraryTransactionData);
+	}
+
 }

src/main/java/org/qora/repository/hsqldb/transaction/HSQLDBTransactionRepository.java

@@ -42,6 +42,7 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 		public Constructor<?> constructor;
 		public Method fromBaseMethod;
 		public Method saveMethod;
+		public Method deleteMethod;
 	}
 
 	private static final RepositorySubclassInfo[] subclassInfos;
@@ -79,6 +80,15 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 				LOGGER.debug(String.format("HSQLDBTransactionRepository subclass's \"save\" method not found for transaction type \"%s\"", txType.name()));
 			}
 
+			try {
+				subclassInfo.deleteMethod = subclassInfo.clazz.getDeclaredMethod("delete", TransactionData.class);
+			} catch (NoSuchMethodException e) {
+				// Subclass has no "delete" method - this is OK
+				subclassInfo.deleteMethod = null;
+			} catch (IllegalArgumentException | SecurityException e) {
+				LOGGER.debug(String.format("HSQLDBTransactionRepository subclass's \"save\" method not found for transaction type \"%s\"", txType.name()));
+			}
+
 			subclassInfos[txType.value] = subclassInfo;
 		}
 
@@ -355,7 +365,8 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 	}
 
 	@Override
-	public List<byte[]> getSignaturesMatchingCriteria(Integer startBlock, Integer blockLimit, List<TransactionType> txTypes, String address,
+	public List<byte[]> getSignaturesMatchingCriteria(Integer startBlock, Integer blockLimit, Integer txGroupId,
+			List<TransactionType> txTypes, Integer service, String address,
 			ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse) throws DataException {
 		List<byte[]> signatures = new ArrayList<byte[]>();
 
@@ -399,6 +410,11 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 			signatureColumn = "TransactionParticipants.signature";
 		}
 
+		if (service != null) {
+			// This is for ARBITRARY transactions
+			tables += " LEFT OUTER JOIN ArbitraryTransactions ON ArbitraryTransactions.signature = Transactions.signature";
+		}
+
 		// WHERE clauses next
 		if (hasHeightRange) {
 			whereClauses.add("Blocks.height >= " + startBlock);
@@ -407,11 +423,21 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 				whereClauses.add("Blocks.height < " + (startBlock + blockLimit));
 		}
 
+		if (txGroupId != null) {
+			whereClauses.add("Transactions.tx_group_id = ?");
+			bindParams.add(txGroupId);
+		}
+
 		if (hasTxTypes) {
 			whereClauses.add("Transactions.type IN (" + HSQLDBRepository.nPlaceholders(txTypes.size()) + ")");
 			bindParams.addAll(txTypes.stream().map(txType -> txType.value).collect(Collectors.toList()));
 		}
 
+		if (service != null) {
+			whereClauses.add("ArbitraryTransactions.service = ?");
+			bindParams.add(service);
+		}
+
 		if (hasAddress) {
 			whereClauses.add("TransactionParticipants.participant = ?");
 			bindParams.add(address);
@@ -816,6 +842,23 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 		} catch (SQLException e) {
 			throw new DataException("Unable to remove transaction from unconfirmed transactions repository", e);
 		}
+
+		// If transaction subclass has a "delete" method - call that now
+		TransactionType type = transactionData.getType();
+		if (subclassInfos[type.value].deleteMethod != null) {
+			HSQLDBTransactionRepository txRepository = repositoryByTxType[type.value];
+
+			try {
+				subclassInfos[type.value].deleteMethod.invoke(txRepository, transactionData);
+			} catch (InvocationTargetException e) {
+				if (e.getCause() instanceof DataException)
+					throw (DataException) e.getCause();
+
+				throw new DataException("Exception during delete of transaction type [" + type.name() + "] from HSQLDB repository");
+			} catch (IllegalAccessException | IllegalArgumentException e) {
+				throw new DataException("Unsupported transaction type [" + type.name() + "] during delete from HSQLDB repository");
+			}
+		}
 	}
 
 }

src/main/java/org/qora/settings/Settings.java

@@ -68,6 +68,11 @@ public class Settings {
 	/** Repository storage path. */
 	private String repositoryPath = null;
 
+	// Auto-update sources
+	private String[] autoUpdateRepos = new String[] {
+		"https://github.com/catbref/qora-core"
+	};
+
 	// Constructors
 
 	private Settings() {
@@ -242,4 +247,8 @@ public class Settings {
 		return this.repositoryPath;
 	}
 
+	public String[] getAutoUpdateRepos() {
+		return this.autoUpdateRepos;
+	}
+
 }

src/main/java/org/qora/transaction/ArbitraryTransaction.java

@@ -1,43 +1,26 @@
 package org.qora.transaction;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
 import java.math.BigDecimal;
-import java.nio.file.DirectoryNotEmptyException;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.qora.account.Account;
 import org.qora.account.PublicKeyAccount;
 import org.qora.asset.Asset;
 import org.qora.block.BlockChain;
-import org.qora.crypto.Crypto;
 import org.qora.data.PaymentData;
 import org.qora.data.transaction.ArbitraryTransactionData;
 import org.qora.data.transaction.TransactionData;
-import org.qora.data.transaction.ArbitraryTransactionData.DataType;
 import org.qora.payment.Payment;
 import org.qora.repository.DataException;
 import org.qora.repository.Repository;
-import org.qora.settings.Settings;
-import org.qora.utils.Base58;
 
 public class ArbitraryTransaction extends Transaction {
 
 	// Properties
 	private ArbitraryTransactionData arbitraryTransactionData;
 
-	// Other properties
-	private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransaction.class);
-
 	// Other useful constants
 	public static final int MAX_DATA_SIZE = 4000;
 
@@ -133,47 +116,12 @@ public class ArbitraryTransaction extends Transaction {
 	@Override
 	public void process() throws DataException {
 		/*
-		 * We might have either raw data or only a hash of data, depending on content filtering.
+		 * Save the transaction.
 		 * 
-		 * If we have raw data then we need to save it somewhere and store the hash in the repository.
+		 * We might have either raw data or only a hash of data, depending on content filtering.
+		 * If we have raw data then the repository save will store the raw data somewhere and save the data's hash in the repository.
+		 * This also modifies the passed transactionData.
 		 */
-		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) {
-			byte[] rawData = arbitraryTransactionData.getData();
-
-			// Calculate hash of data and update our transaction to use that
-			byte[] dataHash = Crypto.digest(rawData);
-			arbitraryTransactionData.setData(dataHash);
-			arbitraryTransactionData.setDataType(DataType.DATA_HASH);
-
-			// Now store actual data somewhere, e.g. <userpath>/arbitrary/<sender address>/<block height>/<tx-sig>-<service>.raw
-			Account sender = this.getSender();
-			int blockHeight = this.repository.getBlockRepository().getBlockchainHeight();
-
-			String senderPathname = Settings.getInstance().getUserPath() + "arbitrary" + File.separator + sender.getAddress();
-			String blockPathname = senderPathname + File.separator + blockHeight;
-			String dataPathname = blockPathname + File.separator + Base58.encode(arbitraryTransactionData.getSignature()) + "-"
-					+ arbitraryTransactionData.getService() + ".raw";
-
-			Path dataPath = Paths.get(dataPathname);
-
-			// Make sure directory structure exists
-			try {
-				Files.createDirectories(dataPath.getParent());
-			} catch (IOException e) {
-				LOGGER.error("Unable to create arbitrary transaction directory", e);
-				throw new DataException("Unable to create arbitrary transaction directory", e);
-			}
-
-			// Output actual transaction data
-			try (OutputStream dataOut = Files.newOutputStream(dataPath)) {
-				dataOut.write(rawData);
-			} catch (IOException e) {
-				LOGGER.error("Unable to store arbitrary transaction data", e);
-				throw new DataException("Unable to store arbitrary transaction data", e);
-			}
-		}
-
-		// Save this transaction itself
 		this.repository.getTransactionRepository().save(this.transactionData);
 
 		// Wrap and delegate payment processing to Payment class. Always update recipients' last references regardless of asset.
@@ -183,33 +131,11 @@ public class ArbitraryTransaction extends Transaction {
 
 	@Override
 	public void orphan() throws DataException {
-		// Delete corresponding data file (if any - storing raw data is optional)
-		Account sender = this.getSender();
-		int blockHeight = this.repository.getBlockRepository().getBlockchainHeight();
-
-		String senderPathname = Settings.getInstance().getUserPath() + "arbitrary" + File.separator + sender.getAddress();
-		String blockPathname = senderPathname + File.separator + blockHeight;
-		String dataPathname = blockPathname + File.separator + Base58.encode(arbitraryTransactionData.getSignature()) + "-"
-				+ arbitraryTransactionData.getService() + ".raw";
-
-		try {
-			// Delete the actual arbitrary data
-			Files.delete(Paths.get(dataPathname));
-
-			// If block-directory now empty, delete that too
-			Files.delete(Paths.get(blockPathname));
-
-			// If sender-directory now empty, delete that too
-			Files.delete(Paths.get(senderPathname));
-		} catch (NoSuchFileException e) {
-			LOGGER.warn("Unable to remove old arbitrary transaction data at " + dataPathname);
-		} catch (DirectoryNotEmptyException e) {
-			// This happens when block-directory or sender-directory is not empty but is OK
-		} catch (IOException e) {
-			LOGGER.warn("IOException when trying to remove old arbitrary transaction data", e);
-		}
-
-		// Delete this transaction itself
+		/*
+		 * Delete the transaction.
+		 * 
+		 * The repository will also remove the stored raw data, if present.
+		 */
 		this.repository.getTransactionRepository().delete(this.transactionData);
 
 		// Wrap and delegate payment processing to Payment class. Always revert recipients' last references regardless of asset.
@@ -217,4 +143,19 @@ public class ArbitraryTransaction extends Transaction {
 				arbitraryTransactionData.getFee(), arbitraryTransactionData.getSignature(), arbitraryTransactionData.getReference(), true);
 	}
 
+	// Data access
+
+	public boolean isDataLocal() throws DataException {
+		return this.repository.getArbitraryRepository().isDataLocal(this.transactionData.getSignature());
+	}
+
+	public byte[] fetchData() throws DataException {
+		// If local, read from file
+		if (isDataLocal())
+			return this.repository.getArbitraryRepository().fetchData(this.transactionData.getSignature());
+
+		// TODO If not local, attempt to fetch via network?
+		return null;
+	}
+
 }

src/main/java/org/qora/transform/Transformer.java

@@ -12,6 +12,7 @@ public abstract class Transformer {
 	public static final int ADDRESS_LENGTH = 25;
 
 	public static final int PUBLIC_KEY_LENGTH = 32;
+	public static final int PRIVATE_KEY_LENGTH = 32;
 	public static final int SIGNATURE_LENGTH = 64;
 	public static final int TIMESTAMP_LENGTH = LONG_LENGTH;
 

src/main/java/org/qora/transform/transaction/ArbitraryTransactionTransformer.java

@@ -9,6 +9,7 @@ import java.util.Arrays;
 import java.util.List;
 
 import org.qora.block.BlockChain;
+import org.qora.crypto.Crypto;
 import org.qora.data.PaymentData;
 import org.qora.data.transaction.ArbitraryTransactionData;
 import org.qora.data.transaction.TransactionData;
@@ -146,8 +147,12 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 	 */
 	public static byte[] toBytesForSigningImpl(TransactionData transactionData) throws TransformationException {
 		ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
-		byte[] bytes = TransactionTransformer.toBytesForSigningImpl(transactionData);
 
+		// For v4, signature uses hash of data, not raw data itself
+		if (arbitraryTransactionData.getVersion() == 4)
+			return toBytesForSigningImplV4(arbitraryTransactionData);
+
+		byte[] bytes = TransactionTransformer.toBytesForSigningImpl(transactionData);
 		if (arbitraryTransactionData.getVersion() == 1 || transactionData.getTimestamp() >= BlockChain.getInstance().getQoraV2Timestamp())
 			return bytes;
 
@@ -165,4 +170,41 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 		return Arrays.copyOfRange(bytes, v1Start, bytes.length);
 	}
 
+	private static byte[] toBytesForSigningImplV4(ArbitraryTransactionData arbitraryTransactionData) throws TransformationException {
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+			transformCommonBytes(arbitraryTransactionData, bytes);
+
+			if (arbitraryTransactionData.getVersion() != 1) {
+				List<PaymentData> payments = arbitraryTransactionData.getPayments();
+				bytes.write(Ints.toByteArray(payments.size()));
+
+				for (PaymentData paymentData : payments)
+					bytes.write(PaymentTransformer.toBytes(paymentData));
+			}
+
+			bytes.write(Ints.toByteArray(arbitraryTransactionData.getService()));
+
+			switch (arbitraryTransactionData.getDataType()) {
+				case DATA_HASH:
+					bytes.write(arbitraryTransactionData.getData());
+					break;
+
+				case RAW_DATA:
+					bytes.write(Crypto.digest(arbitraryTransactionData.getData()));
+					break;
+			}
+
+			Serialization.serializeBigDecimal(bytes, arbitraryTransactionData.getFee());
+
+			// Never append signature
+
+			return bytes.toByteArray();
+		} catch (IOException | ClassCastException e) {
+			throw new TransformationException(e);
+		}
+
+	}
+
 }