diff --git a/pom.xml b/pom.xml index abc6b85c..b7bff221 100644 --- a/pom.xml +++ b/pom.xml @@ -796,5 +796,10 @@ jaxb-runtime ${jaxb-runtime.version} + +org.apache.tika +tika-core +3.1.0 + diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index 722e70da..f741b166 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -2,12 +2,14 @@ package org.qortal.account; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.api.resource.TransactionsResource; import org.qortal.block.BlockChain; import org.qortal.controller.LiteNode; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.naming.NameData; +import org.qortal.data.transaction.TransactionData; import org.qortal.repository.DataException; import org.qortal.repository.GroupRepository; import org.qortal.repository.NameRepository; @@ -19,7 +21,11 @@ import org.qortal.utils.Groups; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import static org.qortal.utils.Amounts.prettyAmount; @@ -361,6 +367,142 @@ public class Account { return accountData.getLevel(); } + /** + * Get Primary Name + * + * @return the primary name for this address if present, otherwise empty + * + * @throws DataException + */ + public Optional getPrimaryName() throws DataException { + + return this.repository.getNameRepository().getPrimaryName(this.address); + } + + /** + * Remove Primary Name + * + * @throws DataException + */ + public void removePrimaryName() throws DataException { + this.repository.getNameRepository().removePrimaryName(this.address); + } + + /** + * Reset Primary Name + * + * Set primary name based on the names (and their history) this account owns. + * + * @param confirmationStatus the status of the transactions for the determining the primary name + * + * @return the primary name, empty if their isn't one + * + * @throws DataException + */ + public Optional resetPrimaryName(TransactionsResource.ConfirmationStatus confirmationStatus) throws DataException { + Optional primaryName = determinePrimaryName(confirmationStatus); + + if(primaryName.isPresent()) { + return setPrimaryName(primaryName.get()); + } + else { + return primaryName; + } + } + + /** + * Determine Primary Name + * + * Determine primary name based on a list of registered names. + * + * @param confirmationStatus the status of the transactions for this determination + * + * @return the primary name, empty if there is no primary name + * + * @throws DataException + */ + public Optional determinePrimaryName(TransactionsResource.ConfirmationStatus confirmationStatus) throws DataException { + + // all registered names for the owner + List names = this.repository.getNameRepository().getNamesByOwner(this.address); + + Optional primaryName; + + // if no registered names, the no primary name possible + if (names.isEmpty()) { + primaryName = Optional.empty(); + } + // if names + else { + // if one name, then that is the primary name + if (names.size() == 1) { + primaryName = Optional.of( names.get(0).getName() ); + } + // if more than one name, then seek the earliest name acquisition that was never released + else { + Map txByName = new HashMap<>(names.size()); + + // for each name, get the latest transaction + for (NameData nameData : names) { + + // since the name is currently registered to the owner, + // we assume the latest transaction involving this name was the transaction that the acquired + // name through registration, purchase or update + Optional latestTransaction + = this.repository + .getTransactionRepository() + .getTransactionsInvolvingName( + nameData.getName(), + confirmationStatus + ) + .stream() + .sorted(Comparator.comparing( + TransactionData::getTimestamp).reversed() + ) + .findFirst(); // first is the last, since it was reversed + + // if there is a latest transaction, expected for all registered names + if (latestTransaction.isPresent()) { + txByName.put(nameData.getName(), latestTransaction.get()); + } + // if there is no latest transaction, then + else { + LOGGER.warn("No matching transaction for name: " + nameData.getName()); + } + } + + // get the first name aqcuistion for this address + Optional> firstNameEntry + = txByName.entrySet().stream().sorted(Comparator.comparing(entry -> entry.getValue().getTimestamp())).findFirst(); + + // if their is a name acquisition, then the first one is the primary name + if (firstNameEntry.isPresent()) { + primaryName = Optional.of( firstNameEntry.get().getKey() ); + } + // if there is no nameacquistion, then there is no primary name + else { + primaryName = Optional.empty(); + } + } + } + return primaryName; + } + + /** + * Set Primary Name + * + * @param primaryName the primary to set to this address + * + * @return the primary name if successful, empty if unsuccessful + * + * @throws DataException + */ + public Optional setPrimaryName( String primaryName ) throws DataException { + int changed = this.repository.getNameRepository().setPrimaryName(this.address, primaryName); + + return changed > 0 ? Optional.of(primaryName) : Optional.empty(); + } + /** * Returns reward-share minting address, or unknown if reward-share does not exist. * diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java index 2cebe8e5..1cfab1da 100644 --- a/src/main/java/org/qortal/api/ApiService.java +++ b/src/main/java/org/qortal/api/ApiService.java @@ -46,6 +46,7 @@ public class ApiService { private ApiService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.restricted.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); @@ -197,6 +198,7 @@ public class ApiService { context.addServlet(DataMonitorSocket.class, "/websockets/datamonitor"); context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*"); context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages"); + context.addServlet(UnsignedFeesSocket.class, "/websockets/crosschain/unsignedfees"); context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers"); context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot"); context.addServlet(TradePresenceWebSocket.class, "/websockets/crosschain/tradepresence"); diff --git a/src/main/java/org/qortal/api/DevProxyService.java b/src/main/java/org/qortal/api/DevProxyService.java index e0bf02db..c0c4e224 100644 --- a/src/main/java/org/qortal/api/DevProxyService.java +++ b/src/main/java/org/qortal/api/DevProxyService.java @@ -40,6 +40,7 @@ public class DevProxyService { private DevProxyService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.proxy.resource", "org.qortal.api.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/DomainMapService.java b/src/main/java/org/qortal/api/DomainMapService.java index 8b791121..5b4a6bbe 100644 --- a/src/main/java/org/qortal/api/DomainMapService.java +++ b/src/main/java/org/qortal/api/DomainMapService.java @@ -39,6 +39,7 @@ public class DomainMapService { private DomainMapService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.domainmap.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/GatewayService.java b/src/main/java/org/qortal/api/GatewayService.java index 24a7b7c9..3ac77799 100644 --- a/src/main/java/org/qortal/api/GatewayService.java +++ b/src/main/java/org/qortal/api/GatewayService.java @@ -39,6 +39,7 @@ public class GatewayService { private GatewayService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.gateway.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java index 4887cf84..f5af7138 100644 --- a/src/main/java/org/qortal/api/HTMLParser.java +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -1,14 +1,13 @@ package org.qortal.api; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; import org.qortal.arbitrary.misc.Service; - import java.util.Objects; - public class HTMLParser { private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class); @@ -22,10 +21,11 @@ public class HTMLParser { private String identifier; private String path; private String theme; + private String lang; private boolean usingCustomRouting; public HTMLParser(String resourceId, String inPath, String prefix, boolean includeResourceIdInPrefix, byte[] data, - String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting) { + String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting, String lang) { String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : String.format("/%s",inPath); this.qdnBase = includeResourceIdInPrefix ? String.format("%s/%s", prefix, resourceId) : prefix; this.qdnBaseWithPath = includeResourceIdInPrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : String.format("%s%s", prefix, inPathWithoutFilename); @@ -36,6 +36,7 @@ public class HTMLParser { this.identifier = identifier; this.path = inPath; this.theme = theme; + this.lang = lang; this.usingCustomRouting = usingCustomRouting; } @@ -61,9 +62,13 @@ public class HTMLParser { String identifier = this.identifier != null ? this.identifier.replace("\\", "").replace("\"","\\\"") : ""; String path = this.path != null ? this.path.replace("\\", "").replace("\"","\\\"") : ""; String theme = this.theme != null ? this.theme.replace("\\", "").replace("\"","\\\"") : ""; + String lang = this.lang != null ? this.lang.replace("\\", "").replace("\"", "\\\"") : ""; String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\\", "").replace("\"","\\\"") : ""; String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\\", "").replace("\"","\\\"") : ""; - String qdnContextVar = String.format("", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath); + String qdnContextVar = String.format( + "", + qdnContext, theme, lang, service, name, identifier, path, qdnBase, qdnBaseWithPath + ); head.get(0).prepend(qdnContextVar); // Add base href tag diff --git a/src/main/java/org/qortal/api/model/crosschain/BitcoinyTBDRequest.java b/src/main/java/org/qortal/api/model/crosschain/BitcoinyTBDRequest.java index 3a531413..1ffef4fa 100644 --- a/src/main/java/org/qortal/api/model/crosschain/BitcoinyTBDRequest.java +++ b/src/main/java/org/qortal/api/model/crosschain/BitcoinyTBDRequest.java @@ -304,11 +304,11 @@ public class BitcoinyTBDRequest { private String networkName; /** - * Fee Ceiling + * Fee Required * - * web search, LTC fee ceiling = 1000L + * web search, LTC fee required = 1000L */ - private long feeCeiling; + private long feeRequired; /** * Extended Public Key @@ -570,8 +570,8 @@ public class BitcoinyTBDRequest { return this.networkName; } - public long getFeeCeiling() { - return this.feeCeiling; + public long getFeeRequired() { + return this.feeRequired; } public String getExtendedPublicKey() { @@ -671,7 +671,7 @@ public class BitcoinyTBDRequest { ", minimumOrderAmount=" + minimumOrderAmount + ", feePerKb=" + feePerKb + ", networkName='" + networkName + '\'' + - ", feeCeiling=" + feeCeiling + + ", feeRequired=" + feeRequired + ", extendedPublicKey='" + extendedPublicKey + '\'' + ", sendAmount=" + sendAmount + ", sendingFeePerByte=" + sendingFeePerByte + diff --git a/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java b/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java index 7972c551..bdaf1ced 100644 --- a/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java +++ b/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java @@ -142,10 +142,20 @@ public class DevProxyServerResource { } } + String lang = request.getParameter("lang"); + if (lang == null || lang.isBlank()) { + lang = "en"; // fallback + } + + String theme = request.getParameter("theme"); + if (theme == null || theme.isBlank()) { + theme = "light"; + } + // Parse and modify output if needed if (HTMLParser.isHtmlFile(filename)) { // HTML file - needs to be parsed - HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true); + HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, theme , true, lang); htmlParser.addAdditionalHeaderTags(); response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;"); response.setContentType(con.getContentType()); diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index a6f44373..bf69fda0 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -3,6 +3,7 @@ package org.qortal.api.resource; import com.google.common.primitives.Bytes; import com.j256.simplemagic.ContentInfo; import com.j256.simplemagic.ContentInfoUtil; + import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -12,6 +13,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; @@ -63,14 +65,19 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; + import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.FileNameMap; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -78,6 +85,16 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; + +import org.apache.tika.Tika; +import org.apache.tika.mime.MimeTypeException; +import org.apache.tika.mime.MimeTypes; + +import javax.ws.rs.core.Response; + +import org.glassfish.jersey.media.multipart.FormDataParam; +import static org.qortal.api.ApiError.REPOSITORY_ISSUE; @Path("/arbitrary") @Tag(name = "Arbitrary") @@ -686,20 +703,20 @@ public class ArbitraryResource { ) } ) - public HttpServletResponse get(@PathParam("service") Service service, + public void get(@PathParam("service") Service service, @PathParam("name") String name, @QueryParam("filepath") String filepath, @QueryParam("encoding") String encoding, @QueryParam("rebuild") boolean rebuild, @QueryParam("async") boolean async, - @QueryParam("attempts") Integer attempts) { + @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) { // Authentication can be bypassed in the settings, for those running public QDN nodes if (!Settings.getInstance().isQDNAuthBypassEnabled()) { Security.checkApiCallAllowed(request); } - return this.download(service, name, null, filepath, encoding, rebuild, async, attempts); + this.download(service, name, null, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename); } @GET @@ -719,21 +736,21 @@ public class ArbitraryResource { ) } ) - public HttpServletResponse get(@PathParam("service") Service service, + public void get(@PathParam("service") Service service, @PathParam("name") String name, @PathParam("identifier") String identifier, @QueryParam("filepath") String filepath, @QueryParam("encoding") String encoding, @QueryParam("rebuild") boolean rebuild, @QueryParam("async") boolean async, - @QueryParam("attempts") Integer attempts) { + @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) { // Authentication can be bypassed in the settings, for those running public QDN nodes if (!Settings.getInstance().isQDNAuthBypassEnabled()) { Security.checkApiCallAllowed(request, null); } - return this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts); + this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename); } @@ -878,6 +895,464 @@ public class ArbitraryResource { } + @GET + @Path("/check/tmp") + @Produces(MediaType.TEXT_PLAIN) + @Operation( + summary = "Check if the disk has enough disk space for an upcoming upload", + responses = { + @ApiResponse(description = "OK if sufficient space", responseCode = "200"), + @ApiResponse(description = "Insufficient space", responseCode = "507") // 507 = Insufficient Storage + } + ) + @SecurityRequirement(name = "apiKey") + public Response checkUploadSpace(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @QueryParam("totalSize") Long totalSize) { + Security.checkApiCallAllowed(request); + + if (totalSize == null || totalSize <= 0) { + return Response.status(Response.Status.BAD_REQUEST) + .entity("Missing or invalid totalSize parameter").build(); + } + + File uploadDir = new File("uploads-temp"); + if (!uploadDir.exists()) { + uploadDir.mkdirs(); // ensure the folder exists + } + + long usableSpace = uploadDir.getUsableSpace(); + long requiredSpace = (long)(((double)totalSize) * 2.2); // estimate for chunks + merge + + if (usableSpace < requiredSpace) { + return Response.status(507).entity("Insufficient disk space").build(); + } + + return Response.ok("Sufficient disk space").build(); + } + + @POST +@Path("/{service}/{name}/chunk") +@Consumes(MediaType.MULTIPART_FORM_DATA) +@Operation( + summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource (no identifier)", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.MULTIPART_FORM_DATA, + schema = @Schema( + implementation = Object.class + ) + ) + ), + responses = { + @ApiResponse( + description = "Chunk uploaded successfully", + responseCode = "200" + ), + @ApiResponse( + description = "Error writing chunk", + responseCode = "500" + ) + } +) +@SecurityRequirement(name = "apiKey") +public Response uploadChunkNoIdentifier(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @FormDataParam("chunk") InputStream chunkStream, + @FormDataParam("index") int index) { + Security.checkApiCallAllowed(request); + + try { + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + + java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName); + Files.createDirectories(tempDir); + + + + java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); + Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING); + + return Response.ok("Chunk " + index + " received").build(); + } catch (IOException e) { + LOGGER.error("Failed to write chunk {} for service '{}' and name '{}'", index, serviceString, name, e); + return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); + } +} + +@POST +@Path("/{service}/{name}/finalize") +@Produces(MediaType.TEXT_PLAIN) +@Operation( + summary = "Finalize a chunked upload (no identifier) and build a raw, unsigned, ARBITRARY transaction", + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content(mediaType = MediaType.TEXT_PLAIN) + ) + } +) +@SecurityRequirement(name = "apiKey") +public String finalizeUploadNoIdentifier( + @HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") List tags, + @QueryParam("category") Category category, + @QueryParam("filename") String filename, + @QueryParam("fee") Long fee, + @QueryParam("preview") Boolean preview, + @QueryParam("isZip") Boolean isZip +) { + Security.checkApiCallAllowed(request); + java.nio.file.Path tempFile = null; + java.nio.file.Path tempDir = null; + java.nio.file.Path chunkDir = null; + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + + + + try { + chunkDir = Paths.get("uploads-temp", safeService, safeName); + + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); + } + + String safeFilename = (filename == null || filename.isBlank()) ? "qortal-" + NTP.getTime() : filename; + tempDir = Files.createTempDirectory("qortal-"); + String sanitizedFilename = Paths.get(safeFilename).getFileName().toString(); + tempFile = tempDir.resolve(sanitizedFilename); + + try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + byte[] buffer = new byte[65536]; + for (java.nio.file.Path chunk : Files.list(chunkDir) + .filter(path -> path.getFileName().toString().startsWith("chunk_")) + .sorted(Comparator.comparingInt(path -> { + String name2 = path.getFileName().toString(); + String numberPart = name2.substring("chunk_".length()); + return Integer.parseInt(numberPart); + })).collect(Collectors.toList())) { + try (InputStream in = Files.newInputStream(chunk)) { + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } + } + + String detectedExtension = ""; + String uploadFilename = null; + boolean extensionIsValid = false; + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + if (lastDot > 0 && lastDot < filename.length() - 1) { + extensionIsValid = true; + uploadFilename = filename; + } + } + + if (!extensionIsValid) { + Tika tika = new Tika(); + String mimeType = tika.detect(tempFile.toFile()); + try { + MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); + org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType); + detectedExtension = mime.getExtension(); + } catch (MimeTypeException e) { + LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e); + } + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename; + uploadFilename = baseName + (detectedExtension != null ? detectedExtension : ""); + } else { + uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : ""); + } + } + + Boolean isZipBoolean = false; + + if (isZip != null && isZip) { + isZipBoolean = true; + } + + + // ✅ Call upload with `null` as identifier + return this.upload( + Service.valueOf(serviceString), + name, + null, // no identifier + tempFile.toString(), + null, + null, + isZipBoolean, + fee, + uploadFilename, + title, + description, + tags, + category, + preview + ); + + } catch (IOException e) { + LOGGER.error("Failed to merge chunks for service='{}', name='{}'", serviceString, name, e); + + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); + } finally { + if (tempDir != null) { + try { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete temp directory: {}", tempDir, e); + } + } + + try { + Files.walk(chunkDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e); + } + } +} + + + + @POST +@Path("/{service}/{name}/{identifier}/chunk") +@Consumes(MediaType.MULTIPART_FORM_DATA) +@Operation( + summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.MULTIPART_FORM_DATA, + schema = @Schema( + implementation = Object.class + ) + ) + ), + responses = { + @ApiResponse( + description = "Chunk uploaded successfully", + responseCode = "200" + ), + @ApiResponse( + description = "Error writing chunk", + responseCode = "500" + ) + } +) +@SecurityRequirement(name = "apiKey") +public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @FormDataParam("chunk") InputStream chunkStream, + @FormDataParam("index") int index) { + Security.checkApiCallAllowed(request); + + try { + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + String safeIdentifier = Paths.get(identifier).getFileName().toString(); + + java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName, safeIdentifier); + + Files.createDirectories(tempDir); + + java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); + Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING); + + return Response.ok("Chunk " + index + " received").build(); + } catch (IOException e) { + LOGGER.error("Failed to write chunk {} for service='{}', name='{}', identifier='{}'", index, serviceString, name, identifier, e); + return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); + } +} + +@POST +@Path("/{service}/{name}/{identifier}/finalize") +@Produces(MediaType.TEXT_PLAIN) +@Operation( + summary = "Finalize a chunked upload and build a raw, unsigned, ARBITRARY transaction", + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content(mediaType = MediaType.TEXT_PLAIN) + ) + } +) +@SecurityRequirement(name = "apiKey") +public String finalizeUpload( + @HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") List tags, + @QueryParam("category") Category category, + @QueryParam("filename") String filename, + @QueryParam("fee") Long fee, + @QueryParam("preview") Boolean preview, + @QueryParam("isZip") Boolean isZip +) { + Security.checkApiCallAllowed(request); + java.nio.file.Path tempFile = null; + java.nio.file.Path tempDir = null; + java.nio.file.Path chunkDir = null; + + + + + + try { + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + String safeIdentifier = Paths.get(identifier).getFileName().toString(); + java.nio.file.Path baseUploadsDir = Paths.get("uploads-temp"); // relative to Qortal working dir + chunkDir = baseUploadsDir.resolve(safeService).resolve(safeName).resolve(safeIdentifier); + + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); + } + + // Step 1: Determine a safe filename for disk temp file (regardless of extension correctness) + String safeFilename = filename; + if (filename == null || filename.isBlank()) { + safeFilename = "qortal-" + NTP.getTime(); + } + + tempDir = Files.createTempDirectory("qortal-"); + String sanitizedFilename = Paths.get(safeFilename).getFileName().toString(); + tempFile = tempDir.resolve(sanitizedFilename); + + + // Step 2: Merge chunks + + try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + byte[] buffer = new byte[65536]; + for (java.nio.file.Path chunk : Files.list(chunkDir) + .filter(path -> path.getFileName().toString().startsWith("chunk_")) + .sorted(Comparator.comparingInt(path -> { + String name2 = path.getFileName().toString(); + String numberPart = name2.substring("chunk_".length()); + return Integer.parseInt(numberPart); + })).collect(Collectors.toList())) { + try (InputStream in = Files.newInputStream(chunk)) { + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } + } + + + // Step 3: Determine correct extension + String detectedExtension = ""; + String uploadFilename = null; + boolean extensionIsValid = false; + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + if (lastDot > 0 && lastDot < filename.length() - 1) { + extensionIsValid = true; + uploadFilename = filename; + } + } + + if (!extensionIsValid) { + Tika tika = new Tika(); + String mimeType = tika.detect(tempFile.toFile()); + try { + MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); + org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType); + detectedExtension = mime.getExtension(); + } catch (MimeTypeException e) { + LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e); + } + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename; + uploadFilename = baseName + (detectedExtension != null ? detectedExtension : ""); + } else { + uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : ""); + } + } + + + Boolean isZipBoolean = false; + + if (isZip != null && isZip) { + isZipBoolean = true; + } + + + return this.upload( + Service.valueOf(serviceString), + name, + identifier, + tempFile.toString(), + null, + null, + isZipBoolean, + fee, + uploadFilename, + title, + description, + tags, + category, + preview + ); + + } catch (IOException e) { + LOGGER.error("Unexpected error in finalizeUpload for service='{}', name='{}', name='{}'", serviceString, name, identifier, e); + + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); + } finally { + if (tempDir != null) { + try { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete temp directory: {}", tempDir, e); + } + } + + try { + Files.walk(chunkDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e); + } + } +} + + + + // Upload base64-encoded data @@ -1343,7 +1818,7 @@ public class ArbitraryResource { if (path == null) { // See if we have a string instead if (string != null) { - if (filename == null) { + if (filename == null || filename.isBlank()) { // Use current time as filename filename = String.format("qortal-%d", NTP.getTime()); } @@ -1358,7 +1833,7 @@ public class ArbitraryResource { } // ... or base64 encoded raw data else if (base64 != null) { - if (filename == null) { + if (filename == null || filename.isBlank()) { // Use current time as filename filename = String.format("qortal-%d", NTP.getTime()); } @@ -1409,6 +1884,7 @@ public class ArbitraryResource { ); transactionBuilder.build(); + // Don't compute nonce - this is done by the client (or via POST /arbitrary/compute) ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData(); return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData)); @@ -1424,22 +1900,20 @@ public class ArbitraryResource { } } - private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { - + private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts, boolean attachment, String attachmentFilename) { try { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); - + int attempts = 0; if (maxAttempts == null) { maxAttempts = 5; } - + // Loop until we have data if (async) { // Asynchronous arbitraryDataReader.loadAsynchronously(false, 1); - } - else { + } else { // Synchronous while (!Controller.isStopping()) { attempts++; @@ -1449,88 +1923,189 @@ public class ArbitraryResource { break; } catch (MissingDataException e) { if (attempts > maxAttempts) { - // Give up after 5 attempts throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); } } } - Thread.sleep(3000L); } } - + java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); if (outputPath == null) { // Assume the resource doesn't exist throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); } - + if (filepath == null || filepath.isEmpty()) { // No file path supplied - so check if this is a single file resource String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); if (files != null && files.length == 1) { // This is a single file resource filepath = files[0]; - } - else { - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, - "filepath is required for resources containing more than one file"); + } else { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "filepath is required for resources containing more than one file"); } } - + java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); if (!Files.exists(path)) { - String message = String.format("No file exists at filepath: %s", filepath); - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No file exists at filepath: " + filepath); } - - byte[] data; - int fileSize = (int)path.toFile().length(); - int length = fileSize; - - // Parse "Range" header - Integer rangeStart = null; - Integer rangeEnd = null; + + if (attachment) { + String rawFilename; + + if (attachmentFilename != null && !attachmentFilename.isEmpty()) { + // 1. Sanitize first + String safeAttachmentFilename = attachmentFilename.replaceAll("[\\\\/:*?\"<>|]", "_"); + + // 2. Check for a valid extension (3–5 alphanumeric chars) + if (!safeAttachmentFilename.matches(".*\\.[a-zA-Z0-9]{2,5}$")) { + safeAttachmentFilename += ".bin"; + } + + rawFilename = safeAttachmentFilename; + } else { + // Fallback if no filename is provided + String baseFilename = (identifier != null && !identifier.isEmpty()) + ? name + "-" + identifier + : name; + rawFilename = baseFilename.replaceAll("[\\\\/:*?\"<>|]", "_") + ".bin"; + } + + // Optional: trim length + rawFilename = rawFilename.length() > 100 ? rawFilename.substring(0, 100) : rawFilename; + + // 3. Set Content-Disposition header + response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\""); + } + + // Determine the total size of the requested file + long fileSize = Files.size(path); + String mimeType = context.getMimeType(path.toString()); + + // Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads) String range = request.getHeader("Range"); - if (range != null) { - range = range.replace("bytes=", ""); - String[] parts = range.split("-"); - rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null; - rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize; + + long rangeStart = 0; + long rangeEnd = fileSize - 1; + boolean isPartial = false; + + // If a Range header is present and no base64 encoding is requested, parse the range values + if (range != null && encoding == null) { + range = range.replace("bytes=", ""); // Remove the "bytes=" prefix + String[] parts = range.split("-"); // Split the range into start and end + + // Parse range start + if (parts.length > 0 && !parts[0].isEmpty()) { + rangeStart = Long.parseLong(parts[0]); + } + + // Parse range end, if present + if (parts.length > 1 && !parts[1].isEmpty()) { + rangeEnd = Long.parseLong(parts[1]); + } + + isPartial = true; // Indicate that this is a partial content request } - - if (rangeStart != null && rangeEnd != null) { - // We have a range, so update the requested length - length = rangeEnd - rangeStart; + + // Calculate how many bytes should be sent in the response + long contentLength = rangeEnd - rangeStart + 1; + + // Inform the client that byte ranges are supported + response.setHeader("Accept-Ranges", "bytes"); + + if (isPartial) { + // If partial content was requested, return 206 Partial Content with appropriate headers + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); + } else { + // Otherwise, return the entire file with status 200 OK + response.setStatus(HttpServletResponse.SC_OK); } - - if (length < fileSize && encoding == null) { - // Partial content requested, and not encoding the data - response.setStatus(206); - response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize)); - data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length); + + // Initialize output streams for writing the file to the response + OutputStream rawOut = null; + OutputStream base64Out = null; + OutputStream gzipOut = null; + + try { + rawOut = response.getOutputStream(); + + if (encoding != null && "base64".equalsIgnoreCase(encoding)) { + // If base64 encoding is requested, override content type + response.setContentType("text/plain"); + + // Check if the client accepts gzip encoding + String acceptEncoding = request.getHeader("Accept-Encoding"); + boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip"); + + if (wantsGzip) { + // Wrap output in GZIP and Base64 streams if gzip is accepted + response.setHeader("Content-Encoding", "gzip"); + gzipOut = new GZIPOutputStream(rawOut); + base64Out = java.util.Base64.getEncoder().wrap(gzipOut); + } else { + // Wrap output in Base64 only + base64Out = java.util.Base64.getEncoder().wrap(rawOut); + } + + rawOut = base64Out; // Use the wrapped stream for writing + } else { + // For raw binary output, set the content type and length + response.setContentType(mimeType != null ? mimeType : "application/octet-stream"); + response.setContentLength((int) contentLength); + } + + // Stream file content + try (InputStream inputStream = Files.newInputStream(path)) { + if (rangeStart > 0) { + inputStream.skip(rangeStart); + } + + byte[] buffer = new byte[65536]; + long bytesRemaining = contentLength; + int bytesRead; + + while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) { + rawOut.write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } + } + + // Stream finished + if (base64Out != null) { + base64Out.close(); // Also flushes and closes the wrapped gzipOut + } else if (gzipOut != null) { + gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out + } else { + rawOut.flush(); // Flush only the base output stream if nothing was wrapped + } + + if (!response.isCommitted()) { + response.setStatus(HttpServletResponse.SC_OK); + response.getWriter().write(" "); + } + + } catch (IOException e) { + // Streaming errors should not rethrow — just log + LOGGER.warn(String.format("Streaming error for %s %s: %s", service, name, e.getMessage())); } - else { - // Full content requested (or encoded data) - response.setStatus(200); - data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory + + } catch (IOException | ApiException | DataException e) { + LOGGER.warn(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); + if (!response.isCommitted()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); } - - // Encode the data if requested - if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) { - data = Base64.encode(data); + } catch (NumberFormatException e) { + LOGGER.warn(String.format("Invalid range for %s %s: %s", service, name, e.getMessage())); + if (!response.isCommitted()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); } - - response.addHeader("Accept-Ranges", "bytes"); - response.setContentType(context.getMimeType(path.toString())); - response.setContentLength(data.length); - response.getOutputStream().write(data); - - return response; - } catch (Exception e) { - LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); } } + + private FileProperties getFileProperties(Service service, String name, String identifier) { try { diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java index 3720a0b5..fb070178 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -502,10 +502,10 @@ public class CrossChainBitcoinResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Bitcoin fee per Kb.", - description = "Returns Bitcoin fee per Kb.", + summary = "The total fee required for unlocking BTC to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -516,17 +516,17 @@ public class CrossChainBitcoinResource { ) } ) - public String getBitcoinFeeCeiling() { + public String getBitcoinFeeRequired() { Bitcoin bitcoin = Bitcoin.getInstance(); - return String.valueOf(bitcoin.getFeeCeiling()); + return String.valueOf(bitcoin.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Bitcoin fee ceiling.", - description = "Sets Bitcoin fee ceiling.", + summary = "The total fee required for unlocking BTC to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -545,13 +545,13 @@ public class CrossChainBitcoinResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setBitcoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setBitcoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Bitcoin bitcoin = Bitcoin.getInstance(); try { - return CrossChainUtils.setFeeCeiling(bitcoin, fee); + return CrossChainUtils.setFeeRequired(bitcoin, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java b/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java index 72b10096..1bf707db 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java @@ -502,10 +502,10 @@ public class CrossChainDigibyteResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Digibyte fee per Kb.", - description = "Returns Digibyte fee per Kb.", + summary = "The total fee required for unlocking DGB to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -516,17 +516,17 @@ public class CrossChainDigibyteResource { ) } ) - public String getDigibyteFeeCeiling() { + public String getDigibyteFeeRequired() { Digibyte digibyte = Digibyte.getInstance(); - return String.valueOf(digibyte.getFeeCeiling()); + return String.valueOf(digibyte.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Digibyte fee ceiling.", - description = "Sets Digibyte fee ceiling.", + summary = "The total fee required for unlocking DGB to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -545,13 +545,13 @@ public class CrossChainDigibyteResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setDigibyteFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setDigibyteFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Digibyte digibyte = Digibyte.getInstance(); try { - return CrossChainUtils.setFeeCeiling(digibyte, fee); + return CrossChainUtils.setFeeRequired(digibyte, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java index 4aa82e2b..5de39fb1 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java @@ -502,10 +502,10 @@ public class CrossChainDogecoinResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Dogecoin fee per Kb.", - description = "Returns Dogecoin fee per Kb.", + summary = "The total fee required for unlocking DOGE to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -516,17 +516,17 @@ public class CrossChainDogecoinResource { ) } ) - public String getDogecoinFeeCeiling() { + public String getDogecoinFeeRequired() { Dogecoin dogecoin = Dogecoin.getInstance(); - return String.valueOf(dogecoin.getFeeCeiling()); + return String.valueOf(dogecoin.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Dogecoin fee ceiling.", - description = "Sets Dogecoin fee ceiling.", + summary = "The total fee required for unlocking DOGE to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -545,13 +545,13 @@ public class CrossChainDogecoinResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setDogecoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setDogecoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Dogecoin dogecoin = Dogecoin.getInstance(); try { - return CrossChainUtils.setFeeCeiling(dogecoin, fee); + return CrossChainUtils.setFeeRequired(dogecoin, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java index 5b9e29d2..1564ff61 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -540,10 +540,10 @@ public class CrossChainLitecoinResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Litecoin fee per Kb.", - description = "Returns Litecoin fee per Kb.", + summary = "The total fee required for unlocking LTC to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -554,17 +554,17 @@ public class CrossChainLitecoinResource { ) } ) - public String getLitecoinFeeCeiling() { + public String getLitecoinFeeRequired() { Litecoin litecoin = Litecoin.getInstance(); - return String.valueOf(litecoin.getFeeCeiling()); + return String.valueOf(litecoin.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Litecoin fee ceiling.", - description = "Sets Litecoin fee ceiling.", + summary = "The total fee required for unlocking LTC to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -583,13 +583,13 @@ public class CrossChainLitecoinResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setLitecoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setLitecoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Litecoin litecoin = Litecoin.getInstance(); try { - return CrossChainUtils.setFeeCeiling(litecoin, fee); + return CrossChainUtils.setFeeRequired(litecoin, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainPirateChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainPirateChainResource.java index c6378f0b..c5e8ac82 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainPirateChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainPirateChainResource.java @@ -587,10 +587,10 @@ public class CrossChainPirateChainResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns PirateChain fee per Kb.", - description = "Returns PirateChain fee per Kb.", + summary = "The total fee required for unlocking ARRR to the trade offer creator.", + description = "The total fee required for unlocking ARRR to the trade offer creator.", responses = { @ApiResponse( content = @Content( @@ -601,17 +601,17 @@ public class CrossChainPirateChainResource { ) } ) - public String getPirateChainFeeCeiling() { + public String getPirateChainFeeRequired() { PirateChain pirateChain = PirateChain.getInstance(); - return String.valueOf(pirateChain.getFeeCeiling()); + return String.valueOf(pirateChain.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets PirateChain fee ceiling.", - description = "Sets PirateChain fee ceiling.", + summary = "The total fee required for unlocking ARRR to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -630,13 +630,13 @@ public class CrossChainPirateChainResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setPirateChainFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setPirateChainFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); PirateChain pirateChain = PirateChain.getInstance(); try { - return CrossChainUtils.setFeeCeiling(pirateChain, fee); + return CrossChainUtils.setFeeRequired(pirateChain, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java index c06122a9..72ff57f9 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java @@ -502,10 +502,10 @@ public class CrossChainRavencoinResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Ravencoin fee per Kb.", - description = "Returns Ravencoin fee per Kb.", + summary = "The total fee required for unlocking RVN to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -516,17 +516,17 @@ public class CrossChainRavencoinResource { ) } ) - public String getRavencoinFeeCeiling() { + public String getRavencoinFeeRequired() { Ravencoin ravencoin = Ravencoin.getInstance(); - return String.valueOf(ravencoin.getFeeCeiling()); + return String.valueOf(ravencoin.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Ravencoin fee ceiling.", - description = "Sets Ravencoin fee ceiling.", + summary = "The total fee required for unlocking RVN to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -545,13 +545,13 @@ public class CrossChainRavencoinResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setRavencoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setRavencoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Ravencoin ravencoin = Ravencoin.getInstance(); try { - return CrossChainUtils.setFeeCeiling(ravencoin, fee); + return CrossChainUtils.setFeeRequired(ravencoin, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java index 3f7acf68..7ee7878c 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainResource.java @@ -10,6 +10,8 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.glassfish.jersey.media.multipart.ContentDisposition; import org.qortal.api.ApiError; import org.qortal.api.ApiErrors; @@ -18,6 +20,7 @@ import org.qortal.api.Security; import org.qortal.api.model.CrossChainCancelRequest; import org.qortal.api.model.CrossChainTradeLedgerEntry; import org.qortal.api.model.CrossChainTradeSummary; +import org.qortal.controller.ForeignFeesManager; import org.qortal.controller.tradebot.TradeBot; import org.qortal.crosschain.ACCT; import org.qortal.crosschain.AcctMode; @@ -29,6 +32,8 @@ import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.data.crosschain.CrossChainTradeData; import org.qortal.data.crosschain.TransactionSummary; +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.data.crosschain.ForeignFeeEncodedData; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.MessageTransactionData; import org.qortal.data.transaction.TransactionData; @@ -64,6 +69,8 @@ import java.util.stream.Collectors; @Tag(name = "Cross-Chain") public class CrossChainResource { + private static final Logger LOGGER = LogManager.getLogger(CrossChainResource.class); + @Context HttpServletRequest request; @@ -360,6 +367,101 @@ public class CrossChainResource { } } + @POST + @Path("/signedfees") + @Operation( + summary = "", + description = "", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + array = @ArraySchema( + schema = @Schema( + implementation = ForeignFeeEncodedData.class + ) + ) + ) + ), + responses = { + @ApiResponse( + description = "true on success", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "boolean" + ) + ) + ) + } + ) + public String postSignedForeignFees(List signedFees) { + + LOGGER.info("signedFees = " + signedFees); + + try { + ForeignFeesManager.getInstance().addSignedFees(signedFees); + + return "true"; + } + catch( Exception e ) { + + LOGGER.error(e.getMessage(), e); + + return "false"; + } + } + + @GET + @Path("/unsignedfees/{address}") + @Operation( + summary = "", + description = "", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = ForeignFeeEncodedData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public List getUnsignedFees(@PathParam("address") String address) { + + List unsignedFeesForAddress = ForeignFeesManager.getInstance().getUnsignedFeesForAddress(address); + + LOGGER.info("address = " + address); + LOGGER.info("returning unsigned = " + unsignedFeesForAddress); + return unsignedFeesForAddress; + } + + @GET + @Path("/signedfees") + @Operation( + summary = "", + description = "", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = ForeignFeeDecodedData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public List getSignedFees() { + + return ForeignFeesManager.getInstance().getSignedFees(); + } + /** * Decode Public Key * diff --git a/src/main/java/org/qortal/api/resource/CrossChainUtils.java b/src/main/java/org/qortal/api/resource/CrossChainUtils.java index ddd1d2d6..eec784e7 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainUtils.java +++ b/src/main/java/org/qortal/api/resource/CrossChainUtils.java @@ -12,10 +12,15 @@ import org.bouncycastle.util.Strings; import org.json.simple.JSONObject; import org.qortal.api.model.CrossChainTradeLedgerEntry; import org.qortal.api.model.crosschain.BitcoinyTBDRequest; +import org.qortal.asset.Asset; import org.qortal.crosschain.*; +import org.qortal.data.account.AccountBalanceData; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.data.crosschain.*; +import org.qortal.event.EventBus; +import org.qortal.event.LockingFeeUpdateEvent; +import org.qortal.event.RequiredFeeUpdateEvent; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.utils.Amounts; @@ -23,15 +28,11 @@ import org.qortal.utils.BitTwiddling; import java.io.BufferedWriter; import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; import java.io.Writer; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; @@ -103,11 +104,13 @@ public class CrossChainUtils { bitcoiny.setFeePerKb(Coin.valueOf(satoshis) ); + EventBus.INSTANCE.notify(new LockingFeeUpdateEvent()); + return String.valueOf(bitcoiny.getFeePerKb().value); } /** - * Set Fee Ceiling + * Set Fee Required * * @param bitcoiny the blockchain support * @param fee the fee in satoshis @@ -116,14 +119,16 @@ public class CrossChainUtils { * * @throws IllegalArgumentException if invalid */ - public static String setFeeCeiling(Bitcoiny bitcoiny, String fee) throws IllegalArgumentException{ + public static String setFeeRequired(Bitcoiny bitcoiny, String fee) throws IllegalArgumentException{ long satoshis = Long.parseLong(fee); if( satoshis < 0 ) throw new IllegalArgumentException("can't set fee to negative number"); - bitcoiny.setFeeCeiling( Long.parseLong(fee)); + bitcoiny.setFeeRequired( Long.parseLong(fee)); - return String.valueOf(bitcoiny.getFeeCeiling()); + EventBus.INSTANCE.notify(new RequiredFeeUpdateEvent(bitcoiny)); + + return String.valueOf(bitcoiny.getFeeRequired()); } /** @@ -232,6 +237,9 @@ public class CrossChainUtils { return bitcoiny.getBlockchainProvider().removeServer(server); } + public static ChainableServer getCurrentServer( Bitcoiny bitcoiny ) { + return bitcoiny.getBlockchainProvider().getCurrentServer(); + } /** * Set Current Server * @@ -771,4 +779,46 @@ public class CrossChainUtils { entries.add(ledgerEntry); } } + + public static List populateTradeDataList(Repository repository, ACCT acct, List atDataList) throws DataException { + + if(atDataList.isEmpty()) return new ArrayList<>(0); + + List latestATStates + = repository.getATRepository() + .getLatestATStates( + atDataList.stream() + .map(ATData::getATAddress) + .collect(Collectors.toList()) + ); + + Map atStateDataByAtAddress + = latestATStates.stream().collect(Collectors.toMap(ATStateData::getATAddress, Function.identity())); + + Map atDataByAtAddress + = atDataList.stream().collect(Collectors.toMap(ATData::getATAddress, Function.identity())); + + Map balanceByAtAddress + = repository + .getAccountRepository() + .getBalances(new ArrayList<>(atDataByAtAddress.keySet()), Asset.QORT) + .stream().collect(Collectors.toMap(AccountBalanceData::getAddress, AccountBalanceData::getBalance)); + + List crossChainTradeDataList = new ArrayList<>(latestATStates.size()); + + for( ATStateData atStateData : latestATStates ) { + ATData atData = atDataByAtAddress.get(atStateData.getATAddress()); + crossChainTradeDataList.add( + acct.populateTradeData( + repository, + atData.getCreatorPublicKey(), + atData.getCreation(), + atStateData, + OptionalLong.of(balanceByAtAddress.get(atStateData.getATAddress())) + ) + ); + } + + return crossChainTradeDataList; + } } \ No newline at end of file diff --git a/src/main/java/org/qortal/api/resource/NamesResource.java b/src/main/java/org/qortal/api/resource/NamesResource.java index c7d4a425..a28615fd 100644 --- a/src/main/java/org/qortal/api/resource/NamesResource.java +++ b/src/main/java/org/qortal/api/resource/NamesResource.java @@ -33,6 +33,7 @@ import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; @Path("/names") @@ -104,6 +105,45 @@ public class NamesResource { } } + @GET + @Path("/primary/{address}") + @Operation( + summary = "primary name owned by address", + responses = { + @ApiResponse( + description = "registered primary name info", + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema(implementation = NameSummary.class) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE, ApiError.UNAUTHORIZED}) + public NameSummary getPrimaryNameByAddress(@PathParam("address") String address) { + if (!Crypto.isValidAddress(address)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + + if (Settings.getInstance().isLite()) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED); + } + else { + Optional primaryName = repository.getNameRepository().getPrimaryName(address); + + if(primaryName.isPresent()) { + return new NameSummary(new NameData(primaryName.get(), address)); + } + else { + return new NameSummary((new NameData(null, address))); + } + } + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + @GET @Path("/{name}") @Operation( diff --git a/src/main/java/org/qortal/api/restricted/resource/AdminResource.java b/src/main/java/org/qortal/api/restricted/resource/AdminResource.java index 439904eb..8c075d7e 100644 --- a/src/main/java/org/qortal/api/restricted/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/restricted/resource/AdminResource.java @@ -1092,25 +1092,4 @@ public class AdminResource { return info; } - - @GET - @Path("/dbstates") - @Operation( - summary = "Get DB States", - description = "Get DB States", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = DbConnectionInfo.class))) - ) - } - ) - public List getDbConnectionsStates() { - - try { - return Controller.REPOSITORY_FACTORY.getDbConnectionsStates(); - } catch (Exception e) { - LOGGER.error(e.getMessage(), e); - return new ArrayList<>(0); - } - } } \ No newline at end of file diff --git a/src/main/java/org/qortal/api/restricted/resource/RenderResource.java b/src/main/java/org/qortal/api/restricted/resource/RenderResource.java index 5eb169a4..3bab7bf3 100644 --- a/src/main/java/org/qortal/api/restricted/resource/RenderResource.java +++ b/src/main/java/org/qortal/api/restricted/resource/RenderResource.java @@ -71,33 +71,33 @@ public class RenderResource { @Path("/signature/{signature}") @SecurityRequirement(name = "apiKey") public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null); - return this.get(signature, ResourceIdType.SIGNATURE, null, null, "/", null, "/render/signature", true, true, theme); + return this.get(signature, ResourceIdType.SIGNATURE, null, null, "/", null, "/render/signature", true, true, theme, lang); } @GET @Path("/signature/{signature}/{path:.*}") @SecurityRequirement(name = "apiKey") public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null); - return this.get(signature, ResourceIdType.SIGNATURE, null, null, inPath,null, "/render/signature", true, true, theme); + return this.get(signature, ResourceIdType.SIGNATURE, null, null, inPath,null, "/render/signature", true, true, theme, lang); } @GET @Path("/hash/{hash}") @SecurityRequirement(name = "apiKey") public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null); - return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, "/", secret58, "/render/hash", true, false, theme); + return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, "/", secret58, "/render/hash", true, false, theme, lang); } @GET @@ -105,11 +105,11 @@ public class RenderResource { @SecurityRequirement(name = "apiKey") public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath, @QueryParam("secret") String secret58, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null); - return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, inPath, secret58, "/render/hash", true, false, theme); + return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, inPath, secret58, "/render/hash", true, false, theme, lang); } @GET @@ -119,12 +119,12 @@ public class RenderResource { @PathParam("name") String name, @PathParam("path") String inPath, @QueryParam("identifier") String identifier, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, name, service, null); String prefix = String.format("/render/%s", service); - return this.get(name, ResourceIdType.NAME, service, identifier, inPath, null, prefix, true, true, theme); + return this.get(name, ResourceIdType.NAME, service, identifier, inPath, null, prefix, true, true, theme, lang); } @GET @@ -133,18 +133,18 @@ public class RenderResource { public HttpServletResponse getIndexByName(@PathParam("service") Service service, @PathParam("name") String name, @QueryParam("identifier") String identifier, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, name, service, null); String prefix = String.format("/render/%s", service); - return this.get(name, ResourceIdType.NAME, service, identifier, "/", null, prefix, true, true, theme); + return this.get(name, ResourceIdType.NAME, service, identifier, "/", null, prefix, true, true, theme, lang); } private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String identifier, - String inPath, String secret58, String prefix, boolean includeResourceIdInPrefix, boolean async, String theme) { + String inPath, String secret58, String prefix, boolean includeResourceIdInPrefix, boolean async, String theme, String lang) { ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, identifier, inPath, secret58, prefix, includeResourceIdInPrefix, async, "render", request, response, context); @@ -152,6 +152,9 @@ public class RenderResource { if (theme != null) { renderer.setTheme(theme); } + if (lang != null) { + renderer.setLang(lang); + } return renderer.render(); } diff --git a/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java b/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java new file mode 100644 index 00000000..c19c2227 --- /dev/null +++ b/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java @@ -0,0 +1,83 @@ +package org.qortal.api.websocket; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketException; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.qortal.data.crosschain.UnsignedFeeEvent; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.FeeWaitingEvent; +import org.qortal.event.Listener; + +import java.io.IOException; +import java.io.StringWriter; + +@WebSocket +@SuppressWarnings("serial") +public class UnsignedFeesSocket extends ApiWebSocket implements Listener { + + private static final Logger LOGGER = LogManager.getLogger(UnsignedFeesSocket.class); + + @Override + public void configure(WebSocketServletFactory factory) { + LOGGER.info("configure"); + + factory.register(UnsignedFeesSocket.class); + + EventBus.INSTANCE.addListener(this); + } + + @Override + public void listen(Event event) { + if (!(event instanceof FeeWaitingEvent)) + return; + + for (Session session : getSessions()) { + FeeWaitingEvent feeWaitingEvent = (FeeWaitingEvent) event; + sendUnsignedFeeEvent(session, new UnsignedFeeEvent(feeWaitingEvent.isPositive(), feeWaitingEvent.getAddress())); + } + } + + + @OnWebSocketConnect + @Override + public void onWebSocketConnect(Session session) { + super.onWebSocketConnect(session); + } + + @OnWebSocketClose + @Override + public void onWebSocketClose(Session session, int statusCode, String reason) { + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* We ignore errors for now, but method here to silence log spam */ + } + + @OnWebSocketMessage + public void onWebSocketMessage(Session session, String message) { + LOGGER.info("onWebSocketMessage: message = " + message); + } + + private void sendUnsignedFeeEvent(Session session, UnsignedFeeEvent unsignedFeeEvent) { + StringWriter stringWriter = new StringWriter(); + + try { + marshall(stringWriter, unsignedFeeEvent); + + session.getRemote().sendStringByFuture(stringWriter.toString()); + } catch (IOException | WebSocketException e) { + // No output this time + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java index b07fd396..004bda6e 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java @@ -4,6 +4,7 @@ import org.qortal.repository.DataException; import org.qortal.utils.Base58; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; @@ -25,47 +26,53 @@ public class ArbitraryDataDigest { } public void compute() throws IOException, DataException { - List allPaths = Files.walk(path).filter(Files::isRegularFile).sorted().collect(Collectors.toList()); + List allPaths = Files.walk(path) + .filter(Files::isRegularFile) + .sorted() + .collect(Collectors.toList()); + Path basePathAbsolute = this.path.toAbsolutePath(); - + MessageDigest sha256; try { sha256 = MessageDigest.getInstance("SHA-256"); } catch (NoSuchAlgorithmException e) { throw new DataException("SHA-256 hashing algorithm unavailable"); } - + for (Path path : allPaths) { // We need to work with paths relative to the base path, to ensure the same hash // is generated on different systems Path relativePath = basePathAbsolute.relativize(path.toAbsolutePath()); - + // Exclude Qortal folder since it can be different each time // We only care about hashing the actual user data if (relativePath.startsWith(".qortal/")) { continue; } - + // Account for \ VS / : Linux VS Windows String pathString = relativePath.toString(); - - if(relativePath.getFileSystem().toString().contains("Windows")) { - pathString = pathString.replace("\\","/"); + if (relativePath.getFileSystem().toString().contains("Windows")) { + pathString = pathString.replace("\\", "/"); } - + // Hash path byte[] filePathBytes = pathString.getBytes(StandardCharsets.UTF_8); - System.out.printf("Path: %s \n", pathString); - System.out.printf("Path Byte Array: %s \n", Arrays.toString(filePathBytes)); sha256.update(filePathBytes); - - // Hash contents - byte[] fileContent = Files.readAllBytes(path); - System.out.printf("File Content: %s \n", Arrays.toString(fileContent)); - sha256.update(fileContent); + + try (InputStream in = Files.newInputStream(path)) { + byte[] buffer = new byte[65536]; // 64 KB + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + sha256.update(buffer, 0, bytesRead); + } + } } + this.hash = sha256.digest(); } + public boolean isHashValid(byte[] hash) { return Arrays.equals(hash, this.hash); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 6e1ca0b9..96bdcdb5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -52,7 +52,7 @@ public class ArbitraryDataFile { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class); - public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB + public static final long MAX_FILE_SIZE = 2L * 1024 * 1024 * 1024; // 2 GiB protected static final int MAX_CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static final int CHUNK_SIZE = 512 * 1024; // 0.5MiB public static int SHORT_DIGEST_LENGTH = 8; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java index eb51e8a4..e1794edf 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java @@ -37,6 +37,7 @@ public class ArbitraryDataRenderer { private final Service service; private final String identifier; private String theme = "light"; + private String lang = "en"; private String inPath; private final String secret58; private final String prefix; @@ -166,9 +167,9 @@ public class ArbitraryDataRenderer { if (HTMLParser.isHtmlFile(filename)) { // HTML file - needs to be parsed byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory - HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, includeResourceIdInPrefix, data, qdnContext, service, identifier, theme, usingCustomRouting); + HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, includeResourceIdInPrefix, data, qdnContext, service, identifier, theme, usingCustomRouting, lang); htmlParser.addAdditionalHeaderTags(); - response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' wss:;"); + response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' wss: blob:;"); response.setContentType(context.getMimeType(filename)); response.setContentLength(htmlParser.getData().length); response.getOutputStream().write(htmlParser.getData()); @@ -256,5 +257,8 @@ public class ArbitraryDataRenderer { public void setTheme(String theme) { this.theme = theme; } + public void setLang(String lang) { + this.lang = lang; + } } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java index a77442ec..b40f72c7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java @@ -29,6 +29,7 @@ import org.qortal.utils.FilesystemUtils; import org.qortal.utils.NTP; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -197,7 +198,7 @@ public class ArbitraryDataTransactionBuilder { // We can't use PATCH for on-chain data because this requires the .qortal directory, which can't be put on chain final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(this.path, false); - final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE); + final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE); if (shouldUseOnChainData) { LOGGER.info("Data size is small enough to go on chain - using PUT"); return Method.PUT; @@ -245,7 +246,7 @@ public class ArbitraryDataTransactionBuilder { // Single file resources are handled differently, especially for very small data payloads, as these go on chain final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(path, false); - final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE); + final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE); // Use zip compression if data isn't going on chain Compression compression = shouldUseOnChainData ? Compression.NONE : Compression.ZIP; diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java index e154f001..21c027c4 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Service.java +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -62,7 +62,17 @@ public enum Service { // Custom validation function to require an index HTML file in the root directory List fileNames = ArbitraryDataRenderer.indexFiles(); - String[] files = path.toFile().list(); + List files; + + // single files are paackaged differently + if( path.toFile().isFile() ) { + files = new ArrayList<>(1); + files.add(path.getFileName().toString()); + } + else { + files = new ArrayList<>(Arrays.asList(path.toFile().list())); + } + if (files != null) { for (String file : files) { Path fileName = Paths.get(file).getFileName(); diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 67e6dd43..1e07f2e7 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1640,6 +1640,8 @@ public class Block { SelfSponsorshipAlgoV2Block.processAccountPenalties(this); } else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) { SelfSponsorshipAlgoV3Block.processAccountPenalties(this); + } else if (this.blockData.getHeight() == BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + PrimaryNamesBlock.processNames(this.repository); } } } @@ -1721,11 +1723,19 @@ public class Block { accountData.setBlocksMinted(accountData.getBlocksMinted() + 1); LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); - final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment() + accountData.getBlocksMintedPenalty(); + int blocksMintedAdjustment + = + (this.blockData.getHeight() > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight()) + ? + 0 + : + accountData.getBlocksMintedAdjustment(); + + final int effectiveBlocksMinted = accountData.getBlocksMinted() + blocksMintedAdjustment + accountData.getBlocksMintedPenalty(); for (int newLevel = maximumLevel; newLevel >= 0; --newLevel) if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) { - if (newLevel > accountData.getLevel()) { + if (newLevel != accountData.getLevel()) { // Account has increased in level! accountData.setLevel(newLevel); bumpedAccounts.put(accountData.getAddress(), newLevel); @@ -1952,6 +1962,8 @@ public class Block { SelfSponsorshipAlgoV2Block.orphanAccountPenalties(this); } else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) { SelfSponsorshipAlgoV3Block.orphanAccountPenalties(this); + } else if (this.blockData.getHeight() == BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + PrimaryNamesBlock.orphanNames( this.repository ); } } @@ -2127,11 +2139,19 @@ public class Block { accountData.setBlocksMinted(accountData.getBlocksMinted() - 1); LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); - final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment() + accountData.getBlocksMintedPenalty(); + int blocksMintedAdjustment + = + (this.blockData.getHeight() -1 > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight()) + ? + 0 + : + accountData.getBlocksMintedAdjustment(); + + final int effectiveBlocksMinted = accountData.getBlocksMinted() + blocksMintedAdjustment + accountData.getBlocksMintedPenalty(); for (int newLevel = maximumLevel; newLevel >= 0; --newLevel) if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) { - if (newLevel < accountData.getLevel()) { + if (newLevel != accountData.getLevel()) { // Account has decreased in level! accountData.setLevel(newLevel); repository.getAccountRepository().setLevel(accountData); diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index bce09aed..1349383d 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -92,7 +92,9 @@ public class BlockChain { adminsReplaceFoundersHeight, nullGroupMembershipHeight, ignoreLevelForRewardShareHeight, - adminQueryFixHeight + adminQueryFixHeight, + multipleNamesPerAccountHeight, + mintedBlocksAdjustmentRemovalHeight } // Custom transaction fees @@ -112,7 +114,8 @@ public class BlockChain { /** Whether to use legacy, broken RIPEMD160 implementation when converting public keys to addresses. */ private boolean useBrokenMD160ForAddresses = false; - /** Whether only one registered name is allowed per account. */ + /** This should get ignored and overwritten in the oneNamePerAccount(int blockchainHeight) method, + * because it is based on block height, not based on the genesis block.*/ private boolean oneNamePerAccount = false; /** Checkpoints */ @@ -474,8 +477,9 @@ public class BlockChain { return this.useBrokenMD160ForAddresses; } - public boolean oneNamePerAccount() { - return this.oneNamePerAccount; + public boolean oneNamePerAccount(int blockchainHeight) { + // this is not set on a simple blockchain setting, it is based on a feature trigger height + return blockchainHeight < this.getMultipleNamesPerAccountHeight(); } public List getCheckpoints() { @@ -688,6 +692,14 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.adminQueryFixHeight.name()).intValue(); } + public int getMultipleNamesPerAccountHeight() { + return this.featureTriggers.get(FeatureTrigger.multipleNamesPerAccountHeight.name()).intValue(); + } + + public int getMintedBlocksAdjustmentRemovalHeight() { + return this.featureTriggers.get(FeatureTrigger.mintedBlocksAdjustmentRemovalHeight.name()).intValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/java/org/qortal/block/PrimaryNamesBlock.java b/src/main/java/org/qortal/block/PrimaryNamesBlock.java new file mode 100644 index 00000000..3dd21344 --- /dev/null +++ b/src/main/java/org/qortal/block/PrimaryNamesBlock.java @@ -0,0 +1,47 @@ +package org.qortal.block; + +import org.qortal.account.Account; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.data.naming.NameData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class PrimaryNamesBlock + */ +public class PrimaryNamesBlock { + + /** + * Process Primary Names + * + * @param repository + * @throws DataException + */ + public static void processNames(Repository repository) throws DataException { + + Set addressesWithNames + = repository.getNameRepository().getAllNames().stream() + .map(NameData::getOwner).collect(Collectors.toSet()); + + // for each address with a name, set primary name to the address + for( String address : addressesWithNames ) { + + Account account = new Account(repository, address); + account.resetPrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED); + } + } + + /** + * Orphan the Primary Names Block + * + * @param repository + * @throws DataException + */ + public static void orphanNames(Repository repository) throws DataException { + + repository.getNameRepository().clearPrimaryNames(); + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 22f7e2d2..31337b42 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -46,6 +46,7 @@ import org.qortal.utils.*; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; + import java.awt.TrayIcon.MessageType; import java.io.File; import java.io.FileNotFoundException; @@ -53,6 +54,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.security.SecureRandom; @@ -70,11 +72,10 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; public class Controller extends Thread { - public static HSQLDBRepositoryFactory REPOSITORY_FACTORY; - static { // This must go before any calls to LogManager/Logger System.setProperty("log4j2.formatMsgNoLookups", "true"); @@ -396,6 +397,9 @@ public class Controller extends Thread { Controller.newInstance(args); + + cleanChunkUploadTempDir(); // cleanup leftover chunks from streaming to disk + LOGGER.info("Starting NTP"); Long ntpOffset = Settings.getInstance().getTestNtpOffset(); if (ntpOffset != null) @@ -405,8 +409,8 @@ public class Controller extends Thread { LOGGER.info("Starting repository"); try { - REPOSITORY_FACTORY = new HSQLDBRepositoryFactory(getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(REPOSITORY_FACTORY); + HSQLDBRepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); try (final Repository repository = RepositoryManager.getRepository()) { @@ -560,6 +564,9 @@ public class Controller extends Thread { LOGGER.info("Starting online accounts manager"); OnlineAccountsManager.getInstance().start(); + LOGGER.info("Starting foreign fees manager"); + ForeignFeesManager.getInstance().start(); + LOGGER.info("Starting transaction importer"); TransactionImporter.getInstance().start(); @@ -1130,6 +1137,9 @@ public class Controller extends Thread { LOGGER.info("Shutting down online accounts manager"); OnlineAccountsManager.getInstance().shutdown(); + LOGGER.info("Shutting down foreign fees manager"); + ForeignFeesManager.getInstance().shutdown(); + LOGGER.info("Shutting down transaction importer"); TransactionImporter.getInstance().shutdown(); @@ -1474,6 +1484,14 @@ public class Controller extends Thread { OnlineAccountsManager.getInstance().onNetworkOnlineAccountsV3Message(peer, message); break; + case GET_FOREIGN_FEES: + ForeignFeesManager.getInstance().onNetworkGetForeignFeesMessage(peer, message); + break; + + case FOREIGN_FEES: + ForeignFeesManager.getInstance().onNetworkForeignFeesMessage(peer, message); + break; + case GET_ARBITRARY_DATA: // Not currently supported break; @@ -2160,6 +2178,24 @@ public class Controller extends Thread { return now - offset; } + private static void cleanChunkUploadTempDir() { + Path uploadsTemp = Paths.get("uploads-temp"); + if (!Files.exists(uploadsTemp)) { + return; + } + + try (Stream paths = Files.walk(uploadsTemp)) { + paths.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + + LOGGER.info("Cleaned up all temporary uploads in {}", uploadsTemp); + } catch (IOException e) { + LOGGER.warn("Failed to clean up uploads-temp directory", e); + } + } + + public StatsSnapshot getStatsSnapshot() { return this.stats; } diff --git a/src/main/java/org/qortal/controller/ForeignFeesManager.java b/src/main/java/org/qortal/controller/ForeignFeesManager.java new file mode 100644 index 00000000..3763b7b3 --- /dev/null +++ b/src/main/java/org/qortal/controller/ForeignFeesManager.java @@ -0,0 +1,1202 @@ +package org.qortal.controller; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.Coin; +import org.json.JSONArray; +import org.json.JSONObject; +import org.qortal.controller.tradebot.TradeBot; +import org.qortal.controller.tradebot.TradeStates; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.ForeignBlockchain; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.ForeignFeeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.data.crosschain.ForeignFeeEncodedData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.FeeWaitingEvent; +import org.qortal.event.LockingFeeUpdateEvent; +import org.qortal.event.RequiredFeeUpdateEvent; +import org.qortal.event.Listener; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.ForeignFeesMessage; +import org.qortal.network.message.GetForeignFeesMessage; +import org.qortal.network.message.Message; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBImportExport; +import org.qortal.settings.Settings; +import org.qortal.utils.Base58; +import org.qortal.utils.ForeignFeesMessageUtils; +import org.qortal.utils.NTP; +import org.qortal.utils.NamedThreadFactory; +import org.qortal.utils.Triple; + +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class ForeignFeesManager implements Listener { + + private static final Logger LOGGER = LogManager.getLogger(ForeignFeesManager.class); + + public static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + public static final String SIGNED_FOREIGN_FEES_TYPE = "signedForeignFees"; + public static final String SIGNED_FOREIGN_FEES_FILE_NAME = "SignedForeignFees.json"; + public static final String CURRENT_DATASET_LABEL = "current"; + private static final String REQUIRED_FOREIGN_FEES_TYPE = "requiredForeignFees"; + private static final String LOCKING_FOREIGN_FEES_TYPE = "lockingForeignFees"; + public static final String REQUIRED_FOREIGN_FEES_FILE_NAME = "RequiredForeignFees.json"; + public static final String LOCKING_FOREIGN_FEES_FILE_NAME = "LockingForeignFees.json"; + + private final ScheduledExecutorService executor + = Executors.newScheduledThreadPool(4, new NamedThreadFactory("Foreign Fee Manager", Thread.NORM_PRIORITY)); + + private volatile boolean isStopping = false; + + private final Set foreignFeesImportQueue = ConcurrentHashMap.newKeySet(); + + /** + * Cache of signed foreign fees, keyed by AT address + */ + private final ConcurrentHashMap> signedByAT = new ConcurrentHashMap<>(); + + /** + * Cache of unsigned foreign fees on this node, key by AT address + */ + private final ConcurrentHashMap unsignedByAT = new ConcurrentHashMap<>(); + + /** + * Cache of trade offers, keyed by creator address + */ + private final ConcurrentHashMap> offersByAddress = new ConcurrentHashMap<>(); + + /** + * Need to Backup Locking Foreign Fees? + * + * Set when the locking foreign fees need to be backed up to a file. + */ + private AtomicBoolean needToBackupLockingForeignFees = new AtomicBoolean(true); + + /** + * Need to Backup Required Foreign Fees? + * + * Set when the required foreign fees need to be backed up to a file. + */ + private AtomicBoolean needToBackupRequiredForeignFees = new AtomicBoolean(true); + + /** + * Need to Backup Signed Foreign Fees? + * + * Set when the signed foreign fees for this node need to be backed up to a file. + */ + private AtomicBoolean needToBackupSignedForeignFees = new AtomicBoolean(true); + + private ForeignFeesManager() { + + EventBus.INSTANCE.addListener(this); + } + + /** + * Import Data + * + * Import signed transaction data for this node and the required fees for unlocking foreign trade funds + * for this node. + */ + private void importData() { + + try { + String exportPath = Settings.getInstance().getExportPath(); + + // import signed foreign fees + try { + Path importSignedForeignFeesPath = Paths.get(exportPath, SIGNED_FOREIGN_FEES_FILE_NAME); + importDataFromFile(importSignedForeignFeesPath.toString()); + } + catch (FileNotFoundException e) { + LOGGER.warn(e.getMessage()); + } + + // import required foreign fees + try { + Path importRequiredForeignFeespath = Paths.get(exportPath, REQUIRED_FOREIGN_FEES_FILE_NAME); + importDataFromFile(importRequiredForeignFeespath.toString()); + } + catch (FileNotFoundException e) { + LOGGER.warn(e.getMessage()); + } + + // import locking foreign fees + try { + Path importLockingForeignFeespath = Paths.get(exportPath, LOCKING_FOREIGN_FEES_FILE_NAME); + importDataFromFile(importLockingForeignFeespath.toString()); + } + catch (FileNotFoundException e) { + LOGGER.warn(e.getMessage()); + } + + } + catch (DataException | IOException e) { + LOGGER.debug("Unable to import data into foreign fees manager: {}", e.getMessage()); + } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Get Unsigned Fees For Address + * + * @param address the address + * + * @return the unsigned fee data + */ + public List getUnsignedFeesForAddress(String address) { + + // the trade offers for this address on this node + List atAddressesForOffers + = this.offersByAddress.getOrDefault(address, new ArrayList<>(0)).stream() + .map( data -> data.qortalAtAddress ) + .collect(Collectors.toList()); + + // the unsigned fee data for the address's trade offers + return this.unsignedByAT.entrySet().stream() + .filter( entry -> atAddressesForOffers.contains(entry.getKey())) + .map( entry -> entry.getValue()) + .collect(Collectors.toList()); + } + + /** + * Get Signed Fees + * + * @return the signed fee data on this node + */ + public List getSignedFees() { + + return this.signedByAT.values().stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + /** + * Add Signed Fees + * + * Add signed fees to the import queue. + * + * @param signedFees the signed fees + */ + public void addSignedFees(List signedFees) { + + LOGGER.debug("adding signed fees: count = " + signedFees.size()); + + // for each encoided fee, decode and add to import queue + for( ForeignFeeEncodedData signedFeeEncoded : signedFees ) { + + LOGGER.debug("adding to import queue: " + signedFeeEncoded); + + // decode the fee data and add to the queue + this.foreignFeesImportQueue.add( + new ForeignFeeDecodedData( + signedFeeEncoded.getTimestamp(), + Base58.decode(signedFeeEncoded.getData()), + signedFeeEncoded.getAtAddress(), + signedFeeEncoded.getFee() + ) + ); + LOGGER.debug("added"); + } + + LOGGER.debug("done adding to queue: count = " + this.foreignFeesImportQueue.size()); + + // process the fees immediately (not waiting for the fee process timer task already in place) + processForeignFeesImportQueue(); + } + + @Override + public void listen(Event event) { + + // locking fee update, then flag locking fee backup + if( event instanceof LockingFeeUpdateEvent) { + this.needToBackupLockingForeignFees.compareAndSet(false, true); + } + // if required fee update, then flag required fee update and process fees for the coin updated + else if( event instanceof RequiredFeeUpdateEvent) { + + this.needToBackupRequiredForeignFees.compareAndSet(false, true); + + for( String address : processLocalForeignFeesForCoin(((RequiredFeeUpdateEvent) event).getBitcoiny()) ) { + EventBus.INSTANCE.notify(new FeeWaitingEvent(true, address)); + } + } + // + else if( event instanceof TradeBot.StateChangeEvent ) { + + TradeBotData data = ((TradeBot.StateChangeEvent) event).getTradeBotData(); + + // if offer is waiting and the time now is determined, + // then process the trade data to be signed later + if( data.getStateValue() == TradeStates.State.BOB_WAITING_FOR_MESSAGE.value ) { + Optional nowDetermined = determineNow(); + if (nowDetermined.isPresent()) { + + long now = nowDetermined.get(); + try (final Repository repository = RepositoryManager.getRepository()) { + LOGGER.debug("processing trade offer in waiting event"); + + Optional offerOptional = getTradeOfferData(repository, data.getAtAddress()); + + if( offerOptional.isPresent() ) { + CrossChainTradeData offer = offerOptional.get(); + this.offersByAddress.computeIfAbsent( offer.qortalCreator, x -> new ArrayList<>()).add(offer); + + if( processTradeOfferInWaiting(now, data) ) { + EventBus.INSTANCE.notify(new FeeWaitingEvent(true, data.getCreatorAddress())); + } + } + else { + LOGGER.warn("offer not present for new trade bot offer = " + data); + } + } catch (IOException | DataException e) { + LOGGER.error(e.getMessage(), e); + } + } + } + } + } + + private static class SingletonContainer { + private static final ForeignFeesManager INSTANCE = new ForeignFeesManager(); + } + + public static ForeignFeesManager getInstance() { + return SingletonContainer.INSTANCE; + } + + /** + * Get Signed Foreign Fee Data By AT Address + * + * @return st address -> signed foreign fee data + */ + public ConcurrentHashMap> getSignedByAT() { + return signedByAT; + } + + /** + * Start Manager + */ + public void start() { + + // import data after a 1 minute delay + // this will get locking fees, required fees and signed data + // there will be nothing to import the first time running this manager + executor.schedule(this::importData, 1, TimeUnit.MINUTES); + + // process local foreign fees for all coins once after a 2 minute delay + // this will get the unsigned fee data + executor.schedule(this::processLocalForeignFeesForAll, 2, TimeUnit.MINUTES); + + // maintain AT's every 5 minutes + executor.scheduleAtFixedRate(this::maintainCrossChainOffers, 3, 5, TimeUnit.MINUTES); + + // request foreign fees from peers every 5 minutes + executor.scheduleAtFixedRate(this::requestRemoteForeignFees, 4, 5, TimeUnit.MINUTES); + + // process imported foreign fees every 5 minutes + executor.scheduleWithFixedDelay(this::processForeignFeesImportQueue, 5, 5, TimeUnit.MINUTES); + + // backup data every 5 minutes + executor.scheduleAtFixedRate(this::backup, 6, 5, TimeUnit.MINUTES); + } + + /** + * Backup + * + * Backup member data used by this manager. + */ + private void backup() { + + try { + if( this.needToBackupLockingForeignFees.compareAndSet( true, false )) { + LOGGER.debug("backing up locking foreign fees"); + backupForeignFeeData( bitcoiny -> bitcoiny.getFeePerKb().value, LOCKING_FOREIGN_FEES_FILE_NAME, LOCKING_FOREIGN_FEES_TYPE); + } + if( this.needToBackupRequiredForeignFees.compareAndSet(true, false) ) { + LOGGER.debug("backing up required foreign fees"); + backupForeignFeeData(Bitcoiny::getFeeRequired, REQUIRED_FOREIGN_FEES_FILE_NAME, REQUIRED_FOREIGN_FEES_TYPE); + } + + if( this.needToBackupSignedForeignFees.compareAndSet( true, false ) ) { + LOGGER.debug("backing up signed foreign fees"); + backupSignedForeignFeeData(); + } + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Shutdown Manager + */ + public void shutdown() { + isStopping = true; + executor.shutdownNow(); + } + + /** + * Process Import Queue + */ + private void processForeignFeesImportQueue() { + + LOGGER.debug("processing foreign fees import queue ..."); + + if (this.foreignFeesImportQueue.isEmpty()) { + + LOGGER.debug("foreign fees import queue is empty"); + return; + } + + LOGGER.debug("Processing foreign fee import queue (size: {})", this.foreignFeesImportQueue.size()); + + Set foreignFeesToRemove = new HashSet<>(this.foreignFeesImportQueue.size()); + + try (final Repository repository = RepositoryManager.getRepository()) { + + // for each signed foreign fee in the queue, + // compare timestamps to prior imports, verify signature and possibly import in + for (ForeignFeeDecodedData foreignFeeToImport : this.foreignFeesImportQueue) { + if (isStopping) + return; + + // need to get the AT address for mapping key identification purposes + String atAddress = foreignFeeToImport.getAtAddress(); + + LOGGER.debug("foreign fee import, timestamp = " + getFormattedDateTime(foreignFeeToImport.getTimestamp())); + + Optional validatedForeignFeeData + = this.signedByAT.getOrDefault( atAddress, Optional.empty() ); + + // if there is no established, validated foreign fee for this AT address or + // if the import foreign fee is after the validated foreign fee, + // then verify the signature and map it to the AT address + if (validatedForeignFeeData.isEmpty() || validatedForeignFeeData.get().getTimestamp() < foreignFeeToImport.getTimestamp()) { + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + + LOGGER.debug("verify signer for atAddress = " + atAddress); + + // determine if the creator authorized the foreign fee + byte[] publicKey = atData.getCreatorPublicKey(); + byte[] signature = foreignFeeToImport.getData(); + byte[] message + = ForeignFeesMessageUtils.buildForeignFeesDataMessage( + foreignFeeToImport.getTimestamp(), + atAddress, + foreignFeeToImport.getFee() + ); + + // if trade offer creator authorized the imported fee, + // then finish the import and clear it from the unsigned mapping + if( Crypto.verify(publicKey, signature, message) ) { + LOGGER.debug("signer verified"); + this.signedByAT.put(atAddress, Optional.of(foreignFeeToImport)); + this.needToBackupSignedForeignFees.compareAndSet(false, true); + this.unsignedByAT.remove(atAddress); + + String tradeOfferCreatorAddress = Crypto.toAddress(publicKey); + boolean allSignedForCreatorAddress + = this.offersByAddress + .getOrDefault(tradeOfferCreatorAddress, new ArrayList<>(0)).stream() + .map(data -> data.qortalAtAddress) + .filter(qortalAtAddress -> this.unsignedByAT.contains(qortalAtAddress)) + .findAny() + .isEmpty(); + + LOGGER.debug("tradeOfferCreatorAddress = " + tradeOfferCreatorAddress); + LOGGER.debug("allSignedForCreatorAddress = " + allSignedForCreatorAddress); + + if(allSignedForCreatorAddress) { + EventBus.INSTANCE.notify(new FeeWaitingEvent(false, tradeOfferCreatorAddress)); + } + } + // otherwise this fee will get discarded + else { + LOGGER.debug("invalid signature"); + } + } + else { + LOGGER.debug( + "skipping imported fee since the timestamp is not updated: atAddress = {}, timestamp = {}", + atAddress, + foreignFeeToImport.getTimestamp() + ); + } + + // now that this fee has been processed, remove it from the process queue + foreignFeesToRemove.add(foreignFeeToImport); + } + } catch (Exception e) { + LOGGER.error("Repository issue while verifying foreign fees", e); + } finally { + LOGGER.debug("removing foreign fees from import queue: count = " + foreignFeesToRemove.size()); + this.foreignFeesImportQueue.removeAll(foreignFeesToRemove); + } + } + + /** + * Get Formatted Date Time + * + * For logging purposes only. + * + * @param timestamp utc time in milliseconds + * + * @return the formatted string + */ + private static String getFormattedDateTime( long timestamp ) { + + Instant instant = Instant.ofEpochMilli(timestamp); + ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.systemDefault()); + String formattedDateTime = zdt.format(TIMESTAMP_FORMATTER); + + return formattedDateTime; + } + + /** + * Maintain AT's + * + * This removes fee data for AT addreses that are no longer offered on the trade portal. + */ + private void maintainCrossChainOffers() { + + LOGGER.debug("maintaining ATs ..."); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List crossChainTradeOffers = resetOffersByAddress(repository); + + // remove failed trades, then collect AT addresses for trade offers + Set atAddresses + = TradeBot.getInstance() + .removeFailedTrades(repository, crossChainTradeOffers).stream() + .map( data -> data.qortalAtAddress ) + .collect(Collectors.toSet()); + + LOGGER.debug("foreign fees before AT removal: count = " + this.signedByAT.size() ); + + // retain the fees for the current sell offers, remove all others + if( retainFeeByAT(this.signedByAT, atAddresses) ) { + this.needToBackupSignedForeignFees.compareAndSet(false, true); + } + + LOGGER.debug("foreign fees after AT removal: count = " + this.signedByAT.size() ); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Reset Offers By Address + * + * @param repository the data repository + * + * @return address -> cross chain trades + * + * @throws DataException + */ + private List resetOffersByAddress(Repository repository) throws DataException { + List crossChainTradeOffers = new ArrayList<>(); + + // lockdown map while reseting offers by address + synchronized( this.offersByAddress) { + + // for each supported foreign blockchaine, get AT data for trade offers + for ( SupportedBlockchain blockchain : SupportedBlockchain.values()) { + crossChainTradeOffers.addAll( getCrossTradeOffers(repository, blockchain) ); + } + + // group all trade offers by trade offer creator, then reset map + Map> groupedOffersByAddress + = crossChainTradeOffers.stream().collect(Collectors.groupingBy(data -> data.qortalCreator)); + + this.offersByAddress.clear(); + this.offersByAddress.putAll(groupedOffersByAddress); + } + + return crossChainTradeOffers; + } + + /** + * Retain Fee By AT + * + * Retain fees for a list of AT addresses; remove all others. + * + * @param feeByAT the foreign fee data for each trade offer AT address + * @param atAddresses the AT addresses to retain fees for + * + * @return true if any removals, otherwise false + */ + private static boolean retainFeeByAT(ConcurrentHashMap> feeByAT, Set atAddresses) { + + // return value, false until there is a removal + boolean anyRemovals = false; + + // prepate iterator for all AT -> fee mappings + Iterator>> iterator = feeByAT.entrySet().iterator(); + + // iterate over all AT's mapped under management + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + + // if the at address do not contain this entry in the iteration, + // then remove it + if (!atAddresses.contains(entry.getKey())) { + + iterator.remove(); + anyRemovals = true; + } + } + + return anyRemovals; + } + + /** + * Get Cross Trade Offers + * + * @param repository the data repository + * @param blockchain the foreign blockchain supporting the trade + * + * @return the trade offers + * + * @throws DataException + */ + private static List getCrossTradeOffers(Repository repository, SupportedBlockchain blockchain) throws DataException { + + // get ACCT for the foreign blockchain + ACCT acct = blockchain.getLatestAcct(); + + // get AT's for foreign blockchain + List ats + = repository.getATRepository() + .getATsByFunctionality(acct.getCodeBytesHash(), true, null, null, null); + + // prepare the return list of cross chain data + List crossChainTradeOffers = new ArrayList<>(ats.size()); + + // for each AT, get cross chain data and look for trade offer to add + for (ATData at : ats) { + + CrossChainTradeData crossChainTrade = acct.populateTradeData(repository, at); + + // if the trade is in offering mode, then add it to return list + if (crossChainTrade.mode == AcctMode.OFFERING) { + crossChainTradeOffers.add(crossChainTrade); + } + } + + return crossChainTradeOffers; + } + + private static Optional getTradeOfferData(Repository repository, String atAddress) throws DataException { + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + CrossChainTradeData crossChainTrade = acct.populateTradeData(repository, atData); + + // if the trade is in offering mode, then add it to return list + if (crossChainTrade.mode == AcctMode.OFFERING) { + return Optional.of(crossChainTrade); + } + else { + return Optional.empty(); + } + } + + /** + * Request data from other peers + */ + private void requestRemoteForeignFees() { + + LOGGER.debug("requesting remote foreign fees ..."); + + if (!isUpToDate()) return; + + LOGGER.debug("Requesting foreign fees via broadcast..."); + + Message message + = new GetForeignFeesMessage( + this.signedByAT.values().stream() + .filter(Optional::isPresent).map(Optional::get) + .collect(Collectors.toList()) + ); + + Network.getInstance().broadcast(peer -> message); + + LOGGER.debug("Requested foreign fees via broadcast..."); + } + + /** + * Is Up To Date? + * + * @return true if up to date, otherwise false + */ + private static boolean isUpToDate() { + final Long now = NTP.getTime(); + + if (now == null) { + + LOGGER.warn("time is null, aborting"); + return false; + } + + if (!Controller.getInstance().isUpToDate()) { + + LOGGER.debug("not up to date, aborting"); + return false; + } + return true; + } + + /** + * Process foreign fees for all coins + * + * Collect foreign fees for trades waiting locally and store to this manager. + * + * @return if any fee signatures are needed after this process + */ + private void processLocalForeignFeesForAll() { + + Set addressesThatNeedSignatures = new HashSet<>(); + + List names + = Arrays.stream(SupportedBlockchain.values()) + .map( value -> value.getLatestAcct().getClass().getSimpleName()) + .collect(Collectors.toList()); + + for( String name : names ) { + ForeignBlockchain blockchain = SupportedBlockchain.getAcctByName(name).getBlockchain(); + + if( blockchain instanceof Bitcoiny ) { + addressesThatNeedSignatures.addAll( processLocalForeignFeesForCoin((Bitcoiny) blockchain) ); + } + } + + for( String addressThatNeedsSignature : addressesThatNeedSignatures ) { + EventBus.INSTANCE.notify(new FeeWaitingEvent(true, addressThatNeedsSignature)); + } + } + + /** + * Process foreign fees for coin + * + * Collect foreign fees for trades waiting locally and store to this manager. + * + * @param bitcoiny the coin + * + * @return addresses that need fee signatures + */ + private Set processLocalForeignFeesForCoin(final Bitcoiny bitcoiny) { + + Set addressesThatNeedSignatures = new HashSet<>(); + + LOGGER.debug("processing local foreign fees ..."); + + Optional nowDetermined = determineNow(); + if (nowDetermined.isEmpty()){ + return new HashSet<>(0); + } + + long now = nowDetermined.get(); + + try (final Repository repository = RepositoryManager.getRepository()) { + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + + // collect all local trade offers waiting + List tradeOffersWaiting + = allTradeBotData.stream() + .filter(d -> d.getStateValue() == TradeStates.State.BOB_WAITING_FOR_MESSAGE.value) + .filter(d -> SupportedBlockchain.getAcctByName( d.getAcctName() ).getBlockchain().equals( bitcoiny )) + .collect(Collectors.toList()); + + LOGGER.debug("trade offers waiting: count = " + tradeOffersWaiting.size()); + + // process each local trade offer waiting (listed) + for (TradeBotData tradeOfferWaiting : tradeOffersWaiting) { + + // process trade offer first, + // then reset the fee signatures needed status next relative to prior status + if(processTradeOfferInWaiting(now, tradeOfferWaiting) ) { + addressesThatNeedSignatures.add(tradeOfferWaiting.getCreatorAddress()); + } + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + + return addressesThatNeedSignatures; + } + + /** + * Determine Now + * + * @return now if in synce, otherwise empty + */ + private static Optional determineNow() { + // if time is not available, then abort + Long now = NTP.getTime(); + if (now == null) { + + LOGGER.warn("current time is not available, abort sending foreign fees"); + return Optional.empty(); + } + + // if 2 hours behind w/o recovery mode, then abort + final Long minLatestBlockTimestamp = now - (2 * 60 * 60 * 1000L); + if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp) && !Synchronizer.getInstance().getRecoveryMode()) { + + LOGGER.warn("out of sync, abort sending foreign fees"); + return Optional.empty(); + } + return Optional.of(now); + } + + /** + * Process Trade Offer In Waiting + * + * @param now the time in millis for now + * @param tradeOfferWaiting the trade offer in waiting + * + * @return true if the fee for this offer needs to be signed, otherwise false + * + * @throws IOException + */ + private boolean processTradeOfferInWaiting(Long now, TradeBotData tradeOfferWaiting) throws IOException { + + boolean isFeeWaiting = false; + + // derive the supported blockchain for the trade offer waiting + String foreignBlockchain = tradeOfferWaiting.getForeignBlockchain(); + SupportedBlockchain supportedBlockchain = SupportedBlockchain.fromString(foreignBlockchain); + + LOGGER.debug("trade offer waiting: blockchain = " + foreignBlockchain); + + // if the supported blockchain is a Bitcoiny blockchain, then the fee will be available + if (supportedBlockchain.getInstance() instanceof Bitcoiny) { + + // get the foreign blockcahin, the AT address and the foreign fee set to this node + Bitcoiny bitcoiny = (Bitcoiny) supportedBlockchain.getInstance(); + String atAddress = tradeOfferWaiting.getAtAddress(); + int fee = Math.toIntExact(bitcoiny.getFeeRequired()); + + LOGGER.debug("atAddress = {}, fee = {}", atAddress, fee); + + // get the signed foreign fee, if it exists + Optional foreignFeeDecodedData = this.signedByAT.get(atAddress); + + // if the foreign fee has been signed + if (foreignFeeDecodedData != null && foreignFeeDecodedData.isPresent()) { + + LOGGER.debug("signed available"); + + // if the local fee is different than the fee stored in this manager, + // then empty the fee in the manager and set the updated fee to unsigned data + if (!foreignFeeDecodedData.get().getFee().equals(fee)) { + + LOGGER.debug("fee updated"); + this.signedByAT.remove(atAddress); + + this.needToBackupSignedForeignFees.compareAndSet(false, true); + setUnsignedData(now, atAddress, fee); + isFeeWaiting = true; + } + else { + LOGGER.debug("fee not updated"); + } + } + // if the foreign fee has not been signed, then set the fee to unsigned data + else { + LOGGER.debug("fee not signed"); + setUnsignedData(now, atAddress, fee); + isFeeWaiting = true; + } + } + // if the supported blockchain is not a Bitcoiny blockchain, then the fee is not available + else { + LOGGER.warn("Blockchain fee not available: blockchain = " + foreignBlockchain); + } + + return isFeeWaiting; + } + + /** + * Set Unisgned Data + * + * @param timestamp + * @param atAddress + * @param fee + * + * @throws IOException + */ + private void setUnsignedData(Long timestamp, String atAddress, int fee) throws IOException { + ForeignFeeEncodedData feeData + = new ForeignFeeEncodedData( + timestamp, + Base58.encode(ForeignFeesMessageUtils.buildForeignFeesDataMessage(timestamp, atAddress, fee)), + atAddress, + fee + ); + + LOGGER.debug("updating unsigned"); + this.unsignedByAT.put(atAddress, feeData); + LOGGER.debug("updated unsigned = " + this.unsignedByAT); + } + + // Network handlers + + /** + * Handle GetForeignFeesMessage + * + * @param peer + * @param message + */ + public void onNetworkGetForeignFeesMessage(Peer peer, Message message) { + GetForeignFeesMessage getForeignFeesMessage = (GetForeignFeesMessage) message; + + // map the fees the peer already has + Map inMessageDataByAT + = getForeignFeesMessage.getForeignFeeData().stream() + .collect(Collectors.toMap( ForeignFeeDecodedData::getAtAddress, Function.identity())); + + // start collecting fees to send to the peer + List outgoingForeignFees = new ArrayList<>(); + + // for all the signed fees locally stored, compare to what the peer currently has and send out what they need + for(Map.Entry> entry : this.signedByAT.entrySet() ) { + + Optional signedForeignFeeData = entry.getValue(); + + // if the fee has been signed, then evaluate it for sending + if (signedForeignFeeData.isPresent()) { + + String atAddress = entry.getKey(); + + LOGGER.debug("comparing signed foreign fee for get foreign fee message: atAddress = " + atAddress); + + // if message contains AT address, then check timestamps + if (inMessageDataByAT.containsKey(atAddress) ) { + + LOGGER.debug("message does contain: atAddress = " + atAddress); + + // get data from message for AT address + ForeignFeeDecodedData feeData = inMessageDataByAT.get(atAddress); + + // if message data is earlier than what is here locally, then send local out to the peer + if( feeData != null && signedForeignFeeData.get().getTimestamp() > feeData.getTimestamp()) { + outgoingForeignFees.add(signedForeignFeeData.get()); + } + } + // if the message does not contain data for this AT, then send the data out to the peer + else { + LOGGER.debug("message does not contain: atAddress = " + atAddress); + + outgoingForeignFees.add(signedForeignFeeData.get()); + } + } + // if value is empty, then do nothing + else { + LOGGER.debug("unavailable signed foreign fee for get foreign fee message: atAddress = " + entry.getKey()); + } + } + + LOGGER.debug("Sending {} foreign fees to {}", outgoingForeignFees.size(), peer); + + // send out to peer + peer.sendMessage(new ForeignFeesMessage(outgoingForeignFees)); + + LOGGER.debug("Sent {} foreign fees to {}", outgoingForeignFees.size(), peer); + } + + /** + * Handle ForeignFeesMessage + * + * @param peer + * @param message + */ + public void onNetworkForeignFeesMessage(Peer peer, Message message) { + ForeignFeesMessage onlineAccountsMessage = (ForeignFeesMessage) message; + + List peersForeignFees = onlineAccountsMessage.getForeignFees(); + LOGGER.debug("Received {} foreign fees from {}", peersForeignFees.size(), peer); + + int importCount = 0; + + // add any foreign fees to the queue that aren't already present + for (ForeignFeeDecodedData peerForeignFee : peersForeignFees) { + + if( foreignFeesImportQueue.add(peerForeignFee) ) + importCount++; + } + + if (importCount > 0) + LOGGER.debug("Added {} foreign to queue", importCount); + } + + /** + * Backup Signed Foreign Fee Data + * + * @throws DataException + */ + private void backupSignedForeignFeeData() throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // get all signed foreigh fee data on this node + List signedForeignFees + = this.signedByAT.values().stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + + // get the JSON for the signed foreign fees + JSONArray currentSignedForeignFeeDataJson = new JSONArray(); + for (ForeignFeeDecodedData signedForeignFee : signedForeignFees) { + JSONObject foreignFeeSignatureJson = signedForeignFee.toJson(); + currentSignedForeignFeeDataJson.put(foreignFeeSignatureJson); + } + + // put data into a JSON + JSONObject currentSignedForeignFeeDataJsonWrapper = new JSONObject(); + currentSignedForeignFeeDataJsonWrapper.put("type", SIGNED_FOREIGN_FEES_TYPE); + currentSignedForeignFeeDataJsonWrapper.put("dataset", CURRENT_DATASET_LABEL); + currentSignedForeignFeeDataJsonWrapper.put("data", currentSignedForeignFeeDataJson); + + // write signed fee data to backup file + String fileName = Paths.get(backupDirectory.toString(), SIGNED_FOREIGN_FEES_FILE_NAME).toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(currentSignedForeignFeeDataJsonWrapper.toString(2)); + writer.close(); + + } + catch (DataException e) { + throw new DataException("Unable to export foreign fee signatures."); + } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Backup Foreign Fee Data + * + * @param feeGetter the fee from the Bitcoiny instance + * @param filename the backup file name + * @param type the type of fee label + * + * @throws DataException + */ + private void backupForeignFeeData(Function feeGetter, String filename, String type) throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // get the names of the supported blockchains + List names + = Arrays.stream(SupportedBlockchain.values()) + .map( value -> value.getLatestAcct().getClass().getSimpleName()) + .collect(Collectors.toList()); + + // start list of required foreign fees + List foreignFees = new ArrayList<>(names.size()); + + // for each blockchain name, get the blockchain and collect the foreign fee for this node + for( String name : names) { + ForeignBlockchain blockchain = SupportedBlockchain.getAcctByName(name).getBlockchain(); + + // if the blockchain supports fees, add the data to the list + if( blockchain instanceof Bitcoiny ) { + foreignFees.add( new ForeignFeeData(name, feeGetter.apply((Bitcoiny) blockchain)) ); + } + } + + // put the list of fee data into a JSON array + JSONArray currentForeignFeesJson = new JSONArray(); + for (ForeignFeeData foreignFee : foreignFees) { + JSONObject requiredForeignFeeJson = foreignFee.toJson(); + currentForeignFeesJson.put(requiredForeignFeeJson); + } + + // put the JSON array and some metadata into a JSON object + JSONObject currentForeignFeeDataJsonWrapper = new JSONObject(); + currentForeignFeeDataJsonWrapper.put("type", type); + currentForeignFeeDataJsonWrapper.put("dataset", CURRENT_DATASET_LABEL); + currentForeignFeeDataJsonWrapper.put("data", currentForeignFeesJson); + + // write the JSON to the backup file + String fileName = Paths.get(backupDirectory.toString(), filename).toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(currentForeignFeeDataJsonWrapper.toString(2)); + writer.close(); + + } + catch (DataException e) { + throw new DataException("Unable to export required foreign fees."); + } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Import Data From File + * + * @param filename the file name + * + * @throws DataException + * @throws IOException + */ + public void importDataFromFile(String filename) throws DataException, IOException { + + // file path and check for existance + Path path = Paths.get(filename); + if (!path.toFile().exists()) { + throw new FileNotFoundException(String.format("File doesn't exist: %s", filename)); + } + + // read in the file + byte[] fileContents = Files.readAllBytes(path); + if (fileContents == null) { + throw new FileNotFoundException(String.format("Unable to read file contents: %s", filename)); + } + + LOGGER.debug(String.format("Importing %s into foreign fees manager ...", filename)); + + String jsonString = new String(fileContents); + + // get the data type and data from the JSON + Triple parsedJSON = HSQLDBImportExport.parseJSONString(jsonString); + if (parsedJSON.getA() == null || parsedJSON.getC() == null) { + throw new DataException(String.format("Missing data when importing %s into foreign fees manager", filename)); + } + String type = parsedJSON.getA(); + JSONArray data = parsedJSON.getC(); + + Iterator iterator = data.iterator(); + while(iterator.hasNext()) { + JSONObject dataJsonObject = (JSONObject)iterator.next(); + + if (type.equals(SIGNED_FOREIGN_FEES_TYPE)) { + importSignedForeignFeeDataJSON(dataJsonObject); + } + else if( type.equals(REQUIRED_FOREIGN_FEES_TYPE)) { + importRequiredForeignFeeDataJSON(dataJsonObject); + } + else if( type.equals(LOCKING_FOREIGN_FEES_TYPE)) { + importLockingForeignFeeDataJSON(dataJsonObject); + } + else { + throw new DataException(String.format("Unrecognized data type when importing %s", filename)); + } + } + + LOGGER.debug(String.format("Imported %s into foreign fees manager from %s", type, filename)); + } + + /** + * Import Signed Foreign Fee Data JSON + * + * @param signedForeignFeeDataJson the JSON object + * + * @throws DataException + */ + private void importSignedForeignFeeDataJSON(JSONObject signedForeignFeeDataJson) throws DataException { + + ForeignFeeDecodedData signedForeignFeeData = ForeignFeeDecodedData.fromJson(signedForeignFeeDataJson); + + this.signedByAT.put(signedForeignFeeData.getAtAddress(), Optional.of(signedForeignFeeData)); + } + + /** + * Import Required Foreign Fee Data JSON + * + * @param requiredForeignFeeDataJson the JSON object + * + * @throws DataException + */ + private static void importRequiredForeignFeeDataJSON(JSONObject requiredForeignFeeDataJson) throws DataException { + + // the data + ForeignFeeData requiredForeignFeeData = ForeignFeeData.fromJson( requiredForeignFeeDataJson ); + + // the blockchain + ForeignBlockchain blockchain + = SupportedBlockchain + .getAcctByName(requiredForeignFeeData.getBlockchain()) + .getBlockchain(); + + // if the blockchain is Bitcoiny, then get the required fee and set it to blockchain + if( blockchain != null && blockchain instanceof Bitcoiny ) { + ((Bitcoiny) blockchain).setFeeRequired( requiredForeignFeeData.getFee() ); + } + else { + LOGGER.warn("no support for required fee import: blockchain = " + requiredForeignFeeData.getBlockchain()); + } + } + + /** + * Import Locking Foreign Fee Data JSON + * + * @param lockingForeignFeeDataJson the JSON object + * + * @throws DataException + */ + private static void importLockingForeignFeeDataJSON(JSONObject lockingForeignFeeDataJson) throws DataException { + + // get the data + ForeignFeeData lockingForeignFeeData = ForeignFeeData.fromJson(lockingForeignFeeDataJson); + + // get the blockchain + ForeignBlockchain blockchain + = SupportedBlockchain + .getAcctByName(lockingForeignFeeData.getBlockchain()) + .getBlockchain(); + + // if the blockchain is Bitcoiny, then set the locking fee to it + if( blockchain != null && blockchain instanceof Bitcoiny ) { + ((Bitcoiny) blockchain).setFeePerKb(Coin.valueOf(lockingForeignFeeData.getFee())); + } + else { + LOGGER.warn("no support for locking fee import: blockchain = " + lockingForeignFeeData.getBlockchain()); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/controller/TransactionImporter.java b/src/main/java/org/qortal/controller/TransactionImporter.java index 6c846f3b..c1ace5fb 100644 --- a/src/main/java/org/qortal/controller/TransactionImporter.java +++ b/src/main/java/org/qortal/controller/TransactionImporter.java @@ -2,6 +2,7 @@ package org.qortal.controller; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.arbitrary.PeerMessage; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; @@ -20,7 +21,11 @@ import org.qortal.utils.Base58; import org.qortal.utils.NTP; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; import java.util.stream.Collectors; public class TransactionImporter extends Thread { @@ -50,6 +55,10 @@ public class TransactionImporter extends Thread { /** Cached list of unconfirmed transactions, used when counting per creator. This is replaced regularly */ public static List unconfirmedTransactionsCache = null; + public TransactionImporter() { + signatureMessageScheduler.scheduleAtFixedRate(this::processNetworkTransactionSignaturesMessage, 60, 1, TimeUnit.SECONDS); + getTransactionMessageScheduler.scheduleAtFixedRate(this::processNetworkGetTransactionMessages, 60, 1, TimeUnit.SECONDS); + } public static synchronized TransactionImporter getInstance() { if (instance == null) { @@ -371,36 +380,104 @@ public class TransactionImporter extends Thread { } } + // List to collect messages + private final List getTransactionMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object getTransactionMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService getTransactionMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkGetTransactionMessage(Peer peer, Message message) { - GetTransactionMessage getTransactionMessage = (GetTransactionMessage) message; - byte[] signature = getTransactionMessage.getSignature(); - try (final Repository repository = RepositoryManager.getRepository()) { + synchronized (getTransactionMessageLock) { + getTransactionMessageList.add(new PeerMessage(peer, message)); + } + } + + private void processNetworkGetTransactionMessages() { + + try { + List messagesToProcess; + synchronized (getTransactionMessageLock) { + messagesToProcess = new ArrayList<>(getTransactionMessageList); + getTransactionMessageList.clear(); + } + + if( messagesToProcess.isEmpty() ) return; + + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); + + for( PeerMessage peerMessage : messagesToProcess ) { + GetTransactionMessage getTransactionMessage = (GetTransactionMessage) peerMessage.getMessage(); + byte[] signature = getTransactionMessage.getSignature(); + + peerMessageBySignature58.put(Base58.encode(signature), peerMessage); + } + // Firstly check the sig-valid transactions that are currently queued for import - TransactionData transactionData = this.getCachedSigValidTransactions().stream() - .filter(t -> Arrays.equals(signature, t.getSignature())) - .findFirst().orElse(null); + Map transactionsCachedBySignature58 + = this.getCachedSigValidTransactions().stream() + .collect(Collectors.toMap(t -> Base58.encode(t.getSignature()), Function.identity())); - if (transactionData == null) { + Map>> transactionsCachedBySignature58Partition + = peerMessageBySignature58.entrySet().stream() + .collect(Collectors.partitioningBy(entry -> transactionsCachedBySignature58.containsKey(entry.getKey()))); + + List signaturesNeeded + = transactionsCachedBySignature58Partition.get(false).stream() + .map(Map.Entry::getValue) + .map(PeerMessage::getMessage) + .map(message -> (GetTransactionMessage) message) + .map(GetTransactionMessage::getSignature) + .collect(Collectors.toList()); + + // transaction found in the import queue + Map transactionsToSendBySignature58 = new HashMap<>(messagesToProcess.size()); + for( Map.Entry entry : transactionsCachedBySignature58Partition.get(true)) { + transactionsToSendBySignature58.put(entry.getKey(), transactionsCachedBySignature58.get(entry.getKey())); + } + + if( !signaturesNeeded.isEmpty() ) { // Not found in import queue, so try the database - transactionData = repository.getTransactionRepository().fromSignature(signature); + try (final Repository repository = RepositoryManager.getRepository()) { + transactionsToSendBySignature58.putAll( + repository.getTransactionRepository().fromSignatures(signaturesNeeded).stream() + .collect(Collectors.toMap(transactionData -> Base58.encode(transactionData.getSignature()), Function.identity())) + ); + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + } } - if (transactionData == null) { - // Still not found - so we don't have this transaction - LOGGER.debug(() -> String.format("Ignoring GET_TRANSACTION request from peer %s for unknown transaction %s", peer, Base58.encode(signature))); - // Send no response at all??? - return; - } + for( final Map.Entry entry : transactionsToSendBySignature58.entrySet() ) { - Message transactionMessage = new TransactionMessage(transactionData); + PeerMessage peerMessage = peerMessageBySignature58.get(entry.getKey()); + final Message message = peerMessage.getMessage(); + final Peer peer = peerMessage.getPeer(); + + Runnable sendTransactionMessageRunner = () -> sendTransactionMessage(entry.getKey(), entry.getValue(), message, peer); + Thread sendTransactionMessageThread = new Thread(sendTransactionMessageRunner); + sendTransactionMessageThread.start(); + } + } catch (Exception e) { + LOGGER.error(e.getMessage(),e); + } + } + + private static void sendTransactionMessage(String signature58, TransactionData data, Message message, Peer peer) { + try { + Message transactionMessage = new TransactionMessage(data); transactionMessage.setId(message.getId()); + if (!peer.sendMessage(transactionMessage)) peer.disconnect("failed to send transaction"); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); - } catch (TransformationException e) { - LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); + } + catch (TransformationException e) { + LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", signature58, peer), e); + } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } @@ -421,44 +498,86 @@ public class TransactionImporter extends Thread { } } + // List to collect messages + private final List signatureMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object signatureMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService signatureMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkTransactionSignaturesMessage(Peer peer, Message message) { - TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) message; - List signatures = transactionSignaturesMessage.getSignatures(); + synchronized (signatureMessageLock) { + signatureMessageList.add(new PeerMessage(peer, message)); + } + } - try (final Repository repository = RepositoryManager.getRepository()) { - for (byte[] signature : signatures) { - String signature58 = Base58.encode(signature); - if (invalidUnconfirmedTransactions.containsKey(signature58)) { - // Previously invalid transaction - don't keep requesting it - // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks - continue; - } + public void processNetworkTransactionSignaturesMessage() { - // Ignore if this transaction is in the queue - if (incomingTransactionQueueContains(signature)) { - LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer)); - continue; - } + try { + List messagesToProcess; + synchronized (signatureMessageLock) { + messagesToProcess = new ArrayList<>(signatureMessageList); + signatureMessageList.clear(); + } - // Do we have it already? (Before requesting transaction data itself) - if (repository.getTransactionRepository().exists(signature)) { - LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer)); - continue; - } + Map signatureBySignature58 = new HashMap<>(messagesToProcess.size() * 10); + Map peerBySignature58 = new HashMap<>( messagesToProcess.size() * 10 ); - // Check isInterrupted() here and exit fast - if (Thread.currentThread().isInterrupted()) - return; + for( PeerMessage peerMessage : messagesToProcess ) { - // Fetch actual transaction data from peer - Message getTransactionMessage = new GetTransactionMessage(signature); - if (!peer.sendMessage(getTransactionMessage)) { - peer.disconnect("failed to request transaction"); - return; + TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) peerMessage.getMessage(); + List signatures = transactionSignaturesMessage.getSignatures(); + + for (byte[] signature : signatures) { + String signature58 = Base58.encode(signature); + if (invalidUnconfirmedTransactions.containsKey(signature58)) { + // Previously invalid transaction - don't keep requesting it + // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks + continue; + } + + // Ignore if this transaction is in the queue + if (incomingTransactionQueueContains(signature)) { + LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peerMessage.getPeer())); + continue; + } + + signatureBySignature58.put(signature58, signature); + peerBySignature58.put(signature58, peerMessage.getPeer()); } } - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer %s", peer), e); + + if( !signatureBySignature58.isEmpty() ) { + try (final Repository repository = RepositoryManager.getRepository()) { + + // remove signatures in db already + repository.getTransactionRepository() + .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream() + .map(TransactionData::getSignature) + .map(signature -> Base58.encode(signature)) + .forEach(signature58 -> signatureBySignature58.remove(signature58)); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer"), e); + } + } + + // Check isInterrupted() here and exit fast + if (Thread.currentThread().isInterrupted()) + return; + + for (Map.Entry entry : signatureBySignature58.entrySet()) { + + Peer peer = peerBySignature58.get(entry.getKey()); + + // Fetch actual transaction data from peer + Message getTransactionMessage = new GetTransactionMessage(entry.getValue()); + if (peer != null && !peer.sendMessage(getTransactionMessage)) { + peer.disconnect("failed to request transaction"); + } + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 1d5e4149..ee37dbec 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -25,6 +25,10 @@ import org.qortal.utils.NTP; import org.qortal.utils.Triple; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.qortal.controller.arbitrary.ArbitraryDataFileManager.MAX_FILE_HASH_RESPONSES; @@ -73,6 +77,8 @@ public class ArbitraryDataFileListManager { private ArbitraryDataFileListManager() { + getArbitraryDataFileListMessageScheduler.scheduleAtFixedRate(this::processNetworkGetArbitraryDataFileListMessage, 60, 1, TimeUnit.SECONDS); + arbitraryDataFileListMessageScheduler.scheduleAtFixedRate(this::processNetworkArbitraryDataFileListMessage, 60, 1, TimeUnit.SECONDS); } public static ArbitraryDataFileListManager getInstance() { @@ -400,7 +406,7 @@ public class ArbitraryDataFileListManager { String signature58 = Base58.encode(signature); for (Iterator>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) { Map.Entry> entry = it.next(); - if (entry == null || entry.getKey() == null || entry.getValue() != null) { + if (entry == null || entry.getKey() == null || entry.getValue() == null) { continue; } if (Objects.equals(entry.getValue().getA(), signature58)) { @@ -413,70 +419,116 @@ public class ArbitraryDataFileListManager { // Network handlers + // List to collect messages + private final List arbitraryDataFileListMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object arbitraryDataFileListMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService arbitraryDataFileListMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) { // Don't process if QDN is disabled if (!Settings.getInstance().isQdnEnabled()) { return; } - ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; - LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); - - if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) { - long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime(); - LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}", - totalRequestTime, arbitraryDataFileListMessage.getRequestHops(), - arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + synchronized (arbitraryDataFileListMessageLock) { + arbitraryDataFileListMessageList.add(new PeerMessage(peer, message)); } + } - // Do we have a pending request for this data? - Triple request = arbitraryDataFileListRequests.get(message.getId()); - if (request == null || request.getA() == null) { - return; - } - boolean isRelayRequest = (request.getB() != null); + private void processNetworkArbitraryDataFileListMessage() { - // Does this message's signature match what we're expecting? - byte[] signature = arbitraryDataFileListMessage.getSignature(); - String signature58 = Base58.encode(signature); - if (!request.getA().equals(signature58)) { - return; - } + try { + List messagesToProcess; + synchronized (arbitraryDataFileListMessageLock) { + messagesToProcess = new ArrayList<>(arbitraryDataFileListMessageList); + arbitraryDataFileListMessageList.clear(); + } - List hashes = arbitraryDataFileListMessage.getHashes(); - if (hashes == null || hashes.isEmpty()) { - return; - } + if (messagesToProcess.isEmpty()) return; - ArbitraryTransactionData arbitraryTransactionData = null; + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); + Map signatureBySignature58 = new HashMap<>(messagesToProcess.size()); + Map isRelayRequestBySignature58 = new HashMap<>(messagesToProcess.size()); + Map> hashesBySignature58 = new HashMap<>(messagesToProcess.size()); + Map> requestBySignature58 = new HashMap<>(messagesToProcess.size()); - // Check transaction exists and hashes are correct - try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) + for (PeerMessage peerMessage : messagesToProcess) { + Peer peer = peerMessage.getPeer(); + Message message = peerMessage.getMessage(); + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; + LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); + + if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) { + long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime(); + LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}", + totalRequestTime, arbitraryDataFileListMessage.getRequestHops(), + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + } + + // Do we have a pending request for this data? + Triple request = arbitraryDataFileListRequests.get(message.getId()); + if (request == null || request.getA() == null) { + continue; + } + boolean isRelayRequest = (request.getB() != null); + + // Does this message's signature match what we're expecting? + byte[] signature = arbitraryDataFileListMessage.getSignature(); + String signature58 = Base58.encode(signature); + if (!request.getA().equals(signature58)) { + continue; + } + + List hashes = arbitraryDataFileListMessage.getHashes(); + if (hashes == null || hashes.isEmpty()) { + continue; + } + + peerMessageBySignature58.put(signature58, peerMessage); + signatureBySignature58.put(signature58, signature); + isRelayRequestBySignature58.put(signature58, isRelayRequest); + hashesBySignature58.put(signature58, hashes); + requestBySignature58.put(signature58, request); + } + + if (signatureBySignature58.isEmpty()) return; + + List arbitraryTransactionDataList; + + // Check transaction exists and hashes are correct + try (final Repository repository = RepositoryManager.getRepository()) { + arbitraryTransactionDataList + = repository.getTransactionRepository() + .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream() + .filter(data -> data instanceof ArbitraryTransactionData) + .map(data -> (ArbitraryTransactionData) data) + .collect(Collectors.toList()); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list"), e); return; + } - arbitraryTransactionData = (ArbitraryTransactionData) transactionData; + for (ArbitraryTransactionData arbitraryTransactionData : arbitraryTransactionDataList) { -// // Load data file(s) -// ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData); -// -// // Check all hashes exist -// for (byte[] hash : hashes) { -// //LOGGER.debug("Received hash {}", Base58.encode(hash)); -// if (!arbitraryDataFile.containsChunk(hash)) { -// // Check the hash against the complete file -// if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) { -// LOGGER.info("Received non-matching chunk hash {} for signature {}. This could happen if we haven't obtained the metadata file yet.", Base58.encode(hash), signature58); -// return; -// } -// } -// } + byte[] signature = arbitraryTransactionData.getSignature(); + String signature58 = Base58.encode(signature); - if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { - Long now = NTP.getTime(); + List hashes = hashesBySignature58.get(signature58); + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + Peer peer = peerMessage.getPeer(); + Message message = peerMessage.getMessage(); + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; + + Boolean isRelayRequest = isRelayRequestBySignature58.get(signature58); + if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { + Long now = NTP.getTime(); - if (ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.size() < MAX_FILE_HASH_RESPONSES) { // Keep track of the hashes this peer reports to have access to for (byte[] hash : hashes) { String hash58 = Base58.encode(hash); @@ -487,233 +539,303 @@ public class ArbitraryDataFileListManager { ArbitraryFileListResponseInfo responseInfo = new ArbitraryFileListResponseInfo(hash58, signature58, peer, now, arbitraryDataFileListMessage.getRequestTime(), requestHops); - ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.add(responseInfo); + ArbitraryDataFileManager.getInstance().addResponse(responseInfo); + } + + // Keep track of the source peer, for direct connections + if (arbitraryDataFileListMessage.getPeerAddress() != null) { + ArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique( + new ArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now)); } } - // Keep track of the source peer, for direct connections - if (arbitraryDataFileListMessage.getPeerAddress() != null) { - ArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique( - new ArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now)); - } - } + // Forwarding + if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); - } + boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); + if (!isBlocked) { + Triple request = requestBySignature58.get(signature58); + Peer requestingPeer = request.getB(); + if (requestingPeer != null) { + Long requestTime = arbitraryDataFileListMessage.getRequestTime(); + Integer requestHops = arbitraryDataFileListMessage.getRequestHops(); - // Forwarding - if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); - if (!isBlocked) { - Peer requestingPeer = request.getB(); - if (requestingPeer != null) { - Long requestTime = arbitraryDataFileListMessage.getRequestTime(); - Integer requestHops = arbitraryDataFileListMessage.getRequestHops(); + // Add each hash to our local mapping so we know who to ask later + Long now = NTP.getTime(); + for (byte[] hash : hashes) { + String hash58 = Base58.encode(hash); + ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); + ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo); + } - // Add each hash to our local mapping so we know who to ask later - Long now = NTP.getTime(); - for (byte[] hash : hashes) { - String hash58 = Base58.encode(hash); - ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); - ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo); - } + // Bump requestHops if it exists + if (requestHops != null) { + requestHops++; + } - // Bump requestHops if it exists - if (requestHops != null) { - requestHops++; - } + ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage; - ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage; + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + } + forwardArbitraryDataFileListMessage.setId(message.getId()); - // Remove optional parameters if the requesting peer doesn't support it yet - // A message with less statistical data is better than no message at all - if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); - } else { - forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, - arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); - } - forwardArbitraryDataFileListMessage.setId(message.getId()); - - // Forward to requesting peer - LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); - if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) { - requestingPeer.disconnect("failed to forward arbitrary data file list"); + // Forward to requesting peer + LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); + if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) { + requestingPeer.disconnect("failed to forward arbitrary data file list"); + } + } } } } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } + // List to collect messages + private final List getArbitraryDataFileListMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object getArbitraryDataFileListMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService getArbitraryDataFileListMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) { // Don't respond if QDN is disabled if (!Settings.getInstance().isQdnEnabled()) { return; } - Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); - - GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; - byte[] signature = getArbitraryDataFileListMessage.getSignature(); - String signature58 = Base58.encode(signature); - Long now = NTP.getTime(); - Triple newEntry = new Triple<>(signature58, peer, now); - - // If we've seen this request recently, then ignore - if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) { - LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58); - return; + synchronized (getArbitraryDataFileListMessageLock) { + getArbitraryDataFileListMessageList.add(new PeerMessage(peer, message)); } + } - List requestedHashes = getArbitraryDataFileListMessage.getHashes(); - int hashCount = requestedHashes != null ? requestedHashes.size() : 0; - String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer(); + private void processNetworkGetArbitraryDataFileListMessage() { - if (requestingPeer != null) { - LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58); - } - else { - LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58); - } + try { + List messagesToProcess; + synchronized (getArbitraryDataFileListMessageLock) { + messagesToProcess = new ArrayList<>(getArbitraryDataFileListMessageList); + getArbitraryDataFileListMessageList.clear(); + } - List hashes = new ArrayList<>(); - ArbitraryTransactionData transactionData = null; - boolean allChunksExist = false; - boolean hasMetadata = false; + if (messagesToProcess.isEmpty()) return; - try (final Repository repository = RepositoryManager.getRepository()) { + Map signatureBySignature58 = new HashMap<>(messagesToProcess.size()); + Map> requestedHashesBySignature58 = new HashMap<>(messagesToProcess.size()); + Map requestingPeerBySignature58 = new HashMap<>(messagesToProcess.size()); + Map nowBySignature58 = new HashMap<>((messagesToProcess.size())); + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); - // Firstly we need to lookup this file on chain to get a list of its hashes - transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); - if (transactionData instanceof ArbitraryTransactionData) { + for (PeerMessage messagePeer : messagesToProcess) { + Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); + + Message message = messagePeer.message; + Peer peer = messagePeer.peer; + + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; + byte[] signature = getArbitraryDataFileListMessage.getSignature(); + String signature58 = Base58.encode(signature); + Long now = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peer, now); + + // If we've seen this request recently, then ignore + if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) { + LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58); + continue; + } + + List requestedHashes = getArbitraryDataFileListMessage.getHashes(); + int hashCount = requestedHashes != null ? requestedHashes.size() : 0; + String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer(); + + if (requestingPeer != null) { + LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58); + } else { + LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58); + } + + signatureBySignature58.put(signature58, signature); + requestedHashesBySignature58.put(signature58, requestedHashes); + requestingPeerBySignature58.put(signature58, requestingPeer); + nowBySignature58.put(signature58, now); + peerMessageBySignature58.put(signature58, messagePeer); + } + + if (signatureBySignature58.isEmpty()) { + return; + } + + List hashes = new ArrayList<>(); + boolean allChunksExist = false; + boolean hasMetadata = false; + + List transactionDataList; + try (final Repository repository = RepositoryManager.getRepository()) { + + // Firstly we need to lookup this file on chain to get a list of its hashes + transactionDataList + = repository.getTransactionRepository() + .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream() + .filter(data -> data instanceof ArbitraryTransactionData) + .map(data -> (ArbitraryTransactionData) data) + .collect(Collectors.toList()); + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer"), e); + return; + } + + for (ArbitraryTransactionData transactionData : transactionDataList) { + byte[] signature = transactionData.getSignature(); + String signature58 = Base58.encode(signature); + List requestedHashes = requestedHashesBySignature58.get(signature58); // Check if we're even allowed to serve data for this transaction if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { - // Load file(s) and add any that exist to the list of hashes - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData); + try { + // Load file(s) and add any that exist to the list of hashes + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData); - // If the peer didn't supply a hash list, we need to return all hashes for this transaction - if (requestedHashes == null || requestedHashes.isEmpty()) { - requestedHashes = new ArrayList<>(); + // If the peer didn't supply a hash list, we need to return all hashes for this transaction + if (requestedHashes == null || requestedHashes.isEmpty()) { + requestedHashes = new ArrayList<>(); - // Add the metadata file - if (arbitraryDataFile.getMetadataHash() != null) { - requestedHashes.add(arbitraryDataFile.getMetadataHash()); - hasMetadata = true; + // Add the metadata file + if (arbitraryDataFile.getMetadataHash() != null) { + requestedHashes.add(arbitraryDataFile.getMetadataHash()); + hasMetadata = true; + } + + // Add the chunk hashes + if (!arbitraryDataFile.getChunkHashes().isEmpty()) { + requestedHashes.addAll(arbitraryDataFile.getChunkHashes()); + } + // Add complete file if there are no hashes + else { + requestedHashes.add(arbitraryDataFile.getHash()); + } } - // Add the chunk hashes - if (!arbitraryDataFile.getChunkHashes().isEmpty()) { - requestedHashes.addAll(arbitraryDataFile.getChunkHashes()); - } - // Add complete file if there are no hashes - else { - requestedHashes.add(arbitraryDataFile.getHash()); + + // Assume all chunks exists, unless one can't be found below + allChunksExist = true; + + for (byte[] requestedHash : requestedHashes) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature); + if (chunk.exists()) { + hashes.add(chunk.getHash()); + //LOGGER.trace("Added hash {}", chunk.getHash58()); + } else { + LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); + allChunksExist = false; + } } + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + } + } + + // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that, + // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam. + if (hasMetadata && hashes.size() == 1) { + hashes.clear(); + } + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + Peer peer = peerMessage.getPeer(); + Message message = peerMessage.getMessage(); + + Long now = nowBySignature58.get(signature58); + + // We should only respond if we have at least one hash + String requestingPeer = requestingPeerBySignature58.get(signature58); + if (!hashes.isEmpty()) { + + // Firstly we should keep track of the requesting peer, to allow for potential direct connections later + ArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer); + + // We have all the chunks, so update requests map to reflect that we've sent it + // There is no need to keep track of the request, as we can serve all the chunks + if (allChunksExist) { + Triple newEntry = new Triple<>(null, null, now); + arbitraryDataFileListRequests.put(message.getId(), newEntry); } - // Assume all chunks exists, unless one can't be found below - allChunksExist = true; + String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort(); + ArbitraryDataFileListMessage arbitraryDataFileListMessage; + + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, + hashes, NTP.getTime(), 0, ourAddress, true); + } + + arbitraryDataFileListMessage.setId(message.getId()); + + if (!peer.sendMessage(arbitraryDataFileListMessage)) { + LOGGER.debug("Couldn't send list of hashes"); + peer.disconnect("failed to send list of hashes"); + continue; + } + + if (allChunksExist) { + // Nothing left to do, so return to prevent any unnecessary forwarding from occurring + LOGGER.debug("No need for any forwarding because file list request is fully served"); + continue; + } + + } + + // We may need to forward this request on + boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName())); + if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { + // In relay mode - so ask our other peers if they have it + + + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; + + long requestTime = getArbitraryDataFileListMessage.getRequestTime(); + int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1; + long totalRequestTime = now - requestTime; + + if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { + // Relay request hasn't timed out yet, so can potentially be rebroadcast + if (requestHops < RELAY_REQUEST_MAX_HOPS) { + // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + + Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer); + relayGetArbitraryDataFileListMessage.setId(message.getId()); + + LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); + Network.getInstance().broadcast( + broadcastPeer -> + !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : + broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage + ); - for (byte[] requestedHash : requestedHashes) { - ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature); - if (chunk.exists()) { - hashes.add(chunk.getHash()); - //LOGGER.trace("Added hash {}", chunk.getHash58()); } else { - LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); - allChunksExist = false; + // This relay request has reached the maximum number of allowed hops } + } else { + // This relay request has timed out } } } - - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e); - } - - // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that, - // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam. - if (hasMetadata && hashes.size() == 1) { - hashes.clear(); - } - - // We should only respond if we have at least one hash - if (!hashes.isEmpty()) { - - // Firstly we should keep track of the requesting peer, to allow for potential direct connections later - ArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer); - - // We have all the chunks, so update requests map to reflect that we've sent it - // There is no need to keep track of the request, as we can serve all the chunks - if (allChunksExist) { - newEntry = new Triple<>(null, null, now); - arbitraryDataFileListRequests.put(message.getId(), newEntry); - } - - String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort(); - ArbitraryDataFileListMessage arbitraryDataFileListMessage; - - // Remove optional parameters if the requesting peer doesn't support it yet - // A message with less statistical data is better than no message at all - if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); - } else { - arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, - hashes, NTP.getTime(), 0, ourAddress, true); - } - - arbitraryDataFileListMessage.setId(message.getId()); - - if (!peer.sendMessage(arbitraryDataFileListMessage)) { - LOGGER.debug("Couldn't send list of hashes"); - peer.disconnect("failed to send list of hashes"); - return; - } - LOGGER.debug("Sent list of hashes (count: {})", hashes.size()); - - if (allChunksExist) { - // Nothing left to do, so return to prevent any unnecessary forwarding from occurring - LOGGER.debug("No need for any forwarding because file list request is fully served"); - return; - } - - } - - // We may need to forward this request on - boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName())); - if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { - // In relay mode - so ask our other peers if they have it - - long requestTime = getArbitraryDataFileListMessage.getRequestTime(); - int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1; - long totalRequestTime = now - requestTime; - - if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { - // Relay request hasn't timed out yet, so can potentially be rebroadcast - if (requestHops < RELAY_REQUEST_MAX_HOPS) { - // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast - - Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer); - relayGetArbitraryDataFileListMessage.setId(message.getId()); - - LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); - Network.getInstance().broadcast( - broadcastPeer -> - !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : - broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage - ); - - } - else { - // This relay request has reached the maximum number of allowed hops - } - } - else { - // This relay request has timed out - } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 5836dcd8..a4034596 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -25,6 +25,8 @@ import java.security.SecureRandom; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class ArbitraryDataFileManager extends Thread { @@ -48,7 +50,7 @@ public class ArbitraryDataFileManager extends Thread { /** * List to keep track of any arbitrary data file hash responses */ - public final List arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>()); + private final List arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>()); /** * List to keep track of peers potentially available for direct connections, based on recent requests @@ -67,6 +69,7 @@ public class ArbitraryDataFileManager extends Thread { private ArbitraryDataFileManager() { + this.arbitraryDataFileHashResponseScheduler.scheduleAtFixedRate( this::processResponses, 60, 1, TimeUnit.SECONDS); } public static ArbitraryDataFileManager getInstance() { @@ -81,13 +84,6 @@ public class ArbitraryDataFileManager extends Thread { Thread.currentThread().setName("Arbitrary Data File Manager"); try { - // Use a fixed thread pool to execute the arbitrary data file requests - int threadCount = 5; - ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount); - for (int i = 0; i < threadCount; i++) { - arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread()); - } - while (!isStopping) { // Nothing to do yet Thread.sleep(1000); @@ -112,7 +108,6 @@ public class ArbitraryDataFileManager extends Thread { final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT; arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp); - arbitraryDataFileHashResponses.removeIf(entry -> entry.getTimestamp() < relayMinimumTimestamp); final long directConnectionInfoMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_DIRECT_CONNECTION_INFO_TIMEOUT; directConnectionInfo.removeIf(entry -> entry.getTimestamp() < directConnectionInfoMinimumTimestamp); @@ -125,8 +120,7 @@ public class ArbitraryDataFileManager extends Thread { // Fetch data files by hash - public boolean fetchArbitraryDataFiles(Repository repository, - Peer peer, + public boolean fetchArbitraryDataFiles(Peer peer, byte[] signature, ArbitraryTransactionData arbitraryTransactionData, List hashes) throws DataException { @@ -151,16 +145,10 @@ public class ArbitraryDataFileManager extends Thread { if (receivedArbitraryDataFile != null) { LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFile.getHash58(), peer, (endTime-startTime)); receivedAtLeastOneFile = true; - - // Remove this hash from arbitraryDataFileHashResponses now that we have received it - arbitraryDataFileHashResponses.remove(hash58); } else { LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime)); - // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it - arbitraryDataFileHashResponses.remove(hash58); - // Stop asking for files from this peer break; } @@ -169,10 +157,6 @@ public class ArbitraryDataFileManager extends Thread { LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer); } } - else { - // Remove this hash from arbitraryDataFileHashResponses because we have a local copy - arbitraryDataFileHashResponses.remove(hash58); - } } if (receivedAtLeastOneFile) { @@ -191,6 +175,38 @@ public class ArbitraryDataFileManager extends Thread { return receivedAtLeastOneFile; } + // Lock to synchronize access to the list + private final Object arbitraryDataFileHashResponseLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService arbitraryDataFileHashResponseScheduler = Executors.newScheduledThreadPool(1); + + + public void addResponse( ArbitraryFileListResponseInfo responseInfo ) { + + synchronized (arbitraryDataFileHashResponseLock) { + this.arbitraryDataFileHashResponses.add(responseInfo); + } + } + + private void processResponses() { + try { + List responsesToProcess; + synchronized (arbitraryDataFileHashResponseLock) { + responsesToProcess = new ArrayList<>(arbitraryDataFileHashResponses); + arbitraryDataFileHashResponses.clear(); + } + + if (responsesToProcess.isEmpty()) return; + + Long now = NTP.getTime(); + + ArbitraryDataFileRequestThread.getInstance().processFileHashes(now, responsesToProcess, this); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash, Message originalMessage) throws DataException { ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature); boolean fileAlreadyExists = existingFile.exists(); @@ -212,8 +228,7 @@ public class ArbitraryDataFileManager extends Thread { arbitraryDataFileRequests.remove(hash58); LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); - // We may need to remove the file list request, if we have all the files for this transaction - this.handleFileListRequests(signature); + if (response == null) { LOGGER.debug("Received null response from peer {}", peer); @@ -258,6 +273,9 @@ public class ArbitraryDataFileManager extends Thread { } } + // We may need to remove the file list request, if we have all the files for this transaction + this.handleFileListRequests(signature); + return arbitraryDataFile; } @@ -270,10 +288,12 @@ public class ArbitraryDataFileManager extends Thread { return; } - boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData); + boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData); - if (allChunksExist) { - // Update requests map to reflect that we've received all chunks + if (completeFileExists) { + String signature58 = Base58.encode(arbitraryTransactionData.getSignature()); + LOGGER.info("All chunks or complete file exist for transaction {}", signature58); + ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index b8285052..b27eef26 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -4,127 +4,179 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo; +import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.event.DataMonitorEvent; -import org.qortal.event.EventBus; import org.qortal.network.Peer; +import org.qortal.network.message.MessageType; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; import org.qortal.utils.NTP; +import java.net.http.HttpResponse; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import static java.lang.Thread.NORM_PRIORITY; -public class ArbitraryDataFileRequestThread implements Runnable { +public class ArbitraryDataFileRequestThread { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class); - public ArbitraryDataFileRequestThread() { + private ConcurrentHashMap executorByPeer = new ConcurrentHashMap<>(); + private ArbitraryDataFileRequestThread() { + cleanupExecutorByPeerScheduler.scheduleAtFixedRate(this::cleanupExecutorsByPeer, 1, 1, TimeUnit.MINUTES); } - @Override - public void run() { - Thread.currentThread().setName("Arbitrary Data File Request Thread"); - Thread.currentThread().setPriority(NORM_PRIORITY); + private static ArbitraryDataFileRequestThread instance = null; + + public static ArbitraryDataFileRequestThread getInstance() { + + if( instance == null ) { + instance = new ArbitraryDataFileRequestThread(); + } + + return instance; + } + + private final ScheduledExecutorService cleanupExecutorByPeerScheduler = Executors.newScheduledThreadPool(1); + + private void cleanupExecutorsByPeer() { try { - while (!Controller.isStopping()) { - Long now = NTP.getTime(); - this.processFileHashes(now); - } - } catch (InterruptedException e) { - // Fall-through to exit thread... + this.executorByPeer.forEach((key, value) -> { + if (value instanceof ThreadPoolExecutor) { + ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) value; + if (threadPoolExecutor.getActiveCount() == 0) { + if (this.executorByPeer.computeIfPresent(key, (k, v) -> null) == null) { + LOGGER.info("removed executor: peer = " + key); + } + } + } else { + LOGGER.warn("casting issue in cleanup"); + } + }); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } - private void processFileHashes(Long now) throws InterruptedException { + public void processFileHashes(Long now, List responseInfos, ArbitraryDataFileManager arbitraryDataFileManager) { if (Controller.isStopping()) { return; } - ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); - String signature58 = null; - String hash58 = null; - Peer peer = null; - boolean shouldProcess = false; + Map signatureBySignature58 = new HashMap<>(responseInfos.size()); + Map> responseInfoBySignature58 = new HashMap<>(); - synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) { - if (!arbitraryDataFileManager.arbitraryDataFileHashResponses.isEmpty()) { + for( ArbitraryFileListResponseInfo responseInfo : responseInfos) { - // Sort by lowest number of node hops first - Comparator lowestHopsFirstComparator = - Comparator.comparingInt(ArbitraryFileListResponseInfo::getRequestHops); - arbitraryDataFileManager.arbitraryDataFileHashResponses.sort(lowestHopsFirstComparator); + if( responseInfo == null ) continue; - Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.iterator(); - while (iterator.hasNext()) { - if (Controller.isStopping()) { - return; - } - - ArbitraryFileListResponseInfo responseInfo = (ArbitraryFileListResponseInfo) iterator.next(); - if (responseInfo == null) { - iterator.remove(); - continue; - } - - hash58 = responseInfo.getHash58(); - peer = responseInfo.getPeer(); - signature58 = responseInfo.getSignature58(); - Long timestamp = responseInfo.getTimestamp(); - - if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { - // Ignore - to be deleted - iterator.remove(); - continue; - } - - // Skip if already requesting, but don't remove, as we might want to retry later - if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) { - // Already requesting - leave this attempt for later - continue; - } - - // We want to process this file - shouldProcess = true; - iterator.remove(); - break; - } + if (Controller.isStopping()) { + return; } + + Peer peer = responseInfo.getPeer(); + + // if relay timeout, then move on + if (now - responseInfo.getTimestamp() >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || responseInfo.getSignature58() == null || peer == null) { + continue; + } + + // Skip if already requesting, but don't remove, as we might want to retry later + if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(responseInfo.getHash58())) { + // Already requesting - leave this attempt for later + arbitraryDataFileManager.addResponse(responseInfo); // don't remove -> adding back, beacause it was removed already above + continue; + } + + + byte[] hash = Base58.decode(responseInfo.getHash58()); + byte[] signature = Base58.decode(responseInfo.getSignature58()); + + // check for null + if (signature == null || hash == null || peer == null) { + continue; + } + + // We want to process this file, store and map data to process later + signatureBySignature58.put(responseInfo.getSignature58(), signature); + responseInfoBySignature58 + .computeIfAbsent(responseInfo.getSignature58(), signature58 -> new ArrayList<>()) + .add(responseInfo); } - if (!shouldProcess) { - // Nothing to do - Thread.sleep(1000L); - return; - } + // if there are no signatures, then there is nothing to process and nothing query the database + if( signatureBySignature58.isEmpty() ) return; - byte[] hash = Base58.decode(hash58); - byte[] signature = Base58.decode(signature58); + List arbitraryTransactionDataList = new ArrayList<>(); // Fetch the transaction data try (final Repository repository = RepositoryManager.getRepository()) { - ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); - if (arbitraryTransactionData == null) { - return; - } - - if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) { - return; - } - - LOGGER.trace("Fetching file {} from peer {} via request thread...", hash58, peer); - arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); - + arbitraryTransactionDataList.addAll( + ArbitraryTransactionUtils.fetchTransactionDataList(repository, new ArrayList<>(signatureBySignature58.values()))); } catch (DataException e) { - LOGGER.debug("Unable to process file hashes: {}", e.getMessage()); + LOGGER.warn("Unable to fetch transaction data: {}", e.getMessage()); + } + + if( !arbitraryTransactionDataList.isEmpty() ) { + long start = System.currentTimeMillis(); + + for(ArbitraryTransactionData data : arbitraryTransactionDataList ) { + String signature58 = Base58.encode(data.getSignature()); + for( ArbitraryFileListResponseInfo responseInfo : responseInfoBySignature58.get(signature58)) { + Runnable fetcher = () -> arbitraryDataFileFetcher(arbitraryDataFileManager, responseInfo, data); + this.executorByPeer + .computeIfAbsent( + responseInfo.getPeer().toString(), + peer -> Executors.newFixedThreadPool( + Settings.getInstance().getMaxThreadsForMessageType(MessageType.GET_ARBITRARY_DATA_FILE)) + ) + .execute(fetcher); + } + } + long timeLapse = System.currentTimeMillis() - start; } } -} + + private void arbitraryDataFileFetcher(ArbitraryDataFileManager arbitraryDataFileManager, ArbitraryFileListResponseInfo responseInfo, ArbitraryTransactionData arbitraryTransactionData) { + try { + Long now = NTP.getTime(); + + if (now - responseInfo.getTimestamp() >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT ) { + + Peer peer = responseInfo.getPeer(); + String hash58 = responseInfo.getHash58(); + String signature58 = responseInfo.getSignature58(); + LOGGER.debug("Peer {} version {} didn't fetch data file {} for signature {} due to relay timeout.", peer, peer.getPeersVersionString(), hash58, signature58); + return; + } + + arbitraryDataFileManager.fetchArbitraryDataFiles( + responseInfo.getPeer(), + arbitraryTransactionData.getSignature(), + arbitraryTransactionData, + Arrays.asList(Base58.decode(responseInfo.getHash58())) + ); + } catch (DataException e) { + LOGGER.warn("Unable to process file hashes: {}", e.getMessage()); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index c54a1e12..ab16605f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -47,15 +47,15 @@ public class ArbitraryDataStorageManager extends Thread { private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes - /** Treat storage as full at 90% usage, to reduce risk of going over the limit. + /** Treat storage as full at 80% usage, to reduce risk of going over the limit. * This is necessary because we don't calculate total storage values before every write. * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. * This must be lower than DELETION_THRESHOLD. */ - private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90% + private static final double STORAGE_FULL_THRESHOLD = 0.8f; // 80% - /** Start deleting files once we reach 98% usage. + /** Start deleting files once we reach 90% usage. * This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */ - public static final double DELETION_THRESHOLD = 0.98f; // 98% + public static final double DELETION_THRESHOLD = 0.9f; // 90% private static final long PER_NAME_STORAGE_MULTIPLIER = 4L; diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index 993a2b72..d38d329f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -24,6 +24,11 @@ import org.qortal.utils.Triple; import java.io.IOException; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.qortal.controller.arbitrary.ArbitraryDataFileListManager.*; @@ -61,6 +66,7 @@ public class ArbitraryMetadataManager { private ArbitraryMetadataManager() { + scheduler.scheduleAtFixedRate(this::processNetworkGetArbitraryMetadataMessage, 60, 1, TimeUnit.SECONDS); } public static ArbitraryMetadataManager getInstance() { @@ -371,107 +377,160 @@ public class ArbitraryMetadataManager { } } + // List to collect messages + private final List messageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object lock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + public void onNetworkGetArbitraryMetadataMessage(Peer peer, Message message) { + // Don't respond if QDN is disabled if (!Settings.getInstance().isQdnEnabled()) { return; } - Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet(); - - GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) message; - byte[] signature = getArbitraryMetadataMessage.getSignature(); - String signature58 = Base58.encode(signature); - Long now = NTP.getTime(); - Triple newEntry = new Triple<>(signature58, peer, now); - - // If we've seen this request recently, then ignore - if (arbitraryMetadataRequests.putIfAbsent(message.getId(), newEntry) != null) { - LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peer, signature58); - return; - } - - LOGGER.debug("Received metadata request from peer {} for signature {}", peer, signature58); - - ArbitraryTransactionData transactionData = null; - ArbitraryDataFile metadataFile = null; - - try (final Repository repository = RepositoryManager.getRepository()) { - - // Firstly we need to lookup this file on chain to get its metadata hash - transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); - if (transactionData instanceof ArbitraryTransactionData) { - - // Check if we're even allowed to serve metadata for this transaction - if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { - - byte[] metadataHash = transactionData.getMetadataHash(); - if (metadataHash != null) { - - // Load metadata file - metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature); - } - } - } - - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while fetching arbitrary metadata for peer %s", peer), e); - } - - // We should only respond if we have the metadata file - if (metadataFile != null && metadataFile.exists()) { - - // We have the metadata file, so update requests map to reflect that we've sent it - newEntry = new Triple<>(null, null, now); - arbitraryMetadataRequests.put(message.getId(), newEntry); - - ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, metadataFile); - arbitraryMetadataMessage.setId(message.getId()); - if (!peer.sendMessage(arbitraryMetadataMessage)) { - LOGGER.debug("Couldn't send metadata"); - peer.disconnect("failed to send metadata"); - return; - } - LOGGER.debug("Sent metadata"); - - // Nothing left to do, so return to prevent any unnecessary forwarding from occurring - LOGGER.debug("No need for any forwarding because metadata request is fully served"); - return; - - } - - // We may need to forward this request on - boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName())); - if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { - // In relay mode - so ask our other peers if they have it - - long requestTime = getArbitraryMetadataMessage.getRequestTime(); - int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1; - long totalRequestTime = now - requestTime; - - if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { - // Relay request hasn't timed out yet, so can potentially be rebroadcast - if (requestHops < RELAY_REQUEST_MAX_HOPS) { - // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast - - Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops); - relayGetArbitraryMetadataMessage.setId(message.getId()); - - LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); - Network.getInstance().broadcast( - broadcastPeer -> - !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : - broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage); - - } - else { - // This relay request has reached the maximum number of allowed hops - } - } - else { - // This relay request has timed out - } + synchronized (lock) { + messageList.add(new PeerMessage(peer, message)); } } + private void processNetworkGetArbitraryMetadataMessage() { + + try { + List messagesToProcess; + synchronized (lock) { + messagesToProcess = new ArrayList<>(messageList); + messageList.clear(); + } + + Map signatureBySignature58 = new HashMap<>((messagesToProcess.size())); + Map nowBySignature58 = new HashMap<>(messagesToProcess.size()); + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); + + for( PeerMessage peerMessage : messagesToProcess) { + Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet(); + + GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) peerMessage.message; + byte[] signature = getArbitraryMetadataMessage.getSignature(); + String signature58 = Base58.encode(signature); + Long now = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peerMessage.peer, now); + + // If we've seen this request recently, then ignore + if (arbitraryMetadataRequests.putIfAbsent(peerMessage.message.getId(), newEntry) != null) { + LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peerMessage.peer, signature58); + continue; + } + + LOGGER.debug("Received metadata request from peer {} for signature {}", peerMessage.peer, signature58); + + signatureBySignature58.put(signature58, signature); + nowBySignature58.put(signature58, now); + peerMessageBySignature58.put(signature58, peerMessage); + } + + if( signatureBySignature58.isEmpty() ) return; + + List transactionDataList; + try (final Repository repository = RepositoryManager.getRepository()) { + + // Firstly we need to lookup this file on chain to get its metadata hash + transactionDataList = repository.getTransactionRepository().fromSignatures(new ArrayList(signatureBySignature58.values())); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary transactions"), e); + return; + } + + Map dataBySignature58 + = transactionDataList.stream() + .filter(data -> data instanceof ArbitraryTransactionData) + .map(ArbitraryTransactionData.class::cast) + .collect(Collectors.toMap(data -> Base58.encode(data.getSignature()), Function.identity())); + + for(Map.Entry entry : dataBySignature58.entrySet()) { + String signature58 = entry.getKey(); + ArbitraryTransactionData transactionData = entry.getValue(); + + try { + + // Check if we're even allowed to serve metadata for this transaction + if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { + + byte[] metadataHash = transactionData.getMetadataHash(); + if (metadataHash != null) { + + // Load metadata file + ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, transactionData.getSignature()); + // We should only respond if we have the metadata file + if (metadataFile != null && metadataFile.exists()) { + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + Message message = peerMessage.message; + Peer peer = peerMessage.peer; + + // We have the metadata file, so update requests map to reflect that we've sent it + Triple newEntry = new Triple<>(null, null, nowBySignature58.get(signature58)); + arbitraryMetadataRequests.put(message.getId(), newEntry); + + ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(entry.getValue().getSignature(), metadataFile); + arbitraryMetadataMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryMetadataMessage)) { + LOGGER.debug("Couldn't send metadata"); + peer.disconnect("failed to send metadata"); + continue; + } + LOGGER.debug("Sent metadata"); + + // Nothing left to do, so return to prevent any unnecessary forwarding from occurring + LOGGER.debug("No need for any forwarding because metadata request is fully served"); + } + + } + } + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary metadata"), e); + } + + // We may need to forward this request on + boolean isBlocked = (transactionDataList == null || ListUtils.isNameBlocked(transactionData.getName())); + if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { + // In relay mode - so ask our other peers if they have it + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) peerMessage.message; + long requestTime = getArbitraryMetadataMessage.getRequestTime(); + int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1; + long totalRequestTime = nowBySignature58.get(signature58) - requestTime; + + if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { + // Relay request hasn't timed out yet, so can potentially be rebroadcast + if (requestHops < RELAY_REQUEST_MAX_HOPS) { + // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + + byte[] signature = signatureBySignature58.get(signature58); + Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops); + relayGetArbitraryMetadataMessage.setId(getArbitraryMetadataMessage.getId()); + + Peer peer = peerMessage.peer; + LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); + Network.getInstance().broadcast( + broadcastPeer -> + !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : + broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage); + + } else { + // This relay request has reached the maximum number of allowed hops + } + } else { + // This relay request has timed out + } + } + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } } diff --git a/src/main/java/org/qortal/controller/arbitrary/PeerMessage.java b/src/main/java/org/qortal/controller/arbitrary/PeerMessage.java new file mode 100644 index 00000000..e77eca4b --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/PeerMessage.java @@ -0,0 +1,22 @@ +package org.qortal.controller.arbitrary; + +import org.qortal.network.Peer; +import org.qortal.network.message.Message; + +public class PeerMessage { + Peer peer; + Message message; + + public PeerMessage(Peer peer, Message message) { + this.peer = peer; + this.message = message; + } + + public Peer getPeer() { + return peer; + } + + public Message getMessage() { + return message; + } +} diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 654513f2..c17e5758 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -8,6 +8,7 @@ import org.qortal.account.PrivateKeyAccount; import org.qortal.api.model.crosschain.TradeBotCreateRequest; import org.qortal.controller.Controller; import org.qortal.controller.Synchronizer; +import org.qortal.controller.arbitrary.PeerMessage; import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult; import org.qortal.crosschain.*; import org.qortal.crypto.Crypto; @@ -37,7 +38,12 @@ import org.qortal.utils.NTP; import java.awt.TrayIcon.MessageType; import java.security.SecureRandom; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Performing cross-chain trading steps on behalf of user. @@ -118,6 +124,9 @@ public class TradeBot implements Listener { private Map validTrades = new HashMap<>(); private TradeBot() { + + tradePresenceMessageScheduler.scheduleAtFixedRate( this::processTradePresencesMessages, 60, 1, TimeUnit.SECONDS); + EventBus.INSTANCE.addListener(event -> TradeBot.getInstance().listen(event)); } @@ -551,77 +560,139 @@ public class TradeBot implements Listener { } } + // List to collect messages + private final List tradePresenceMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object tradePresenceMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService tradePresenceMessageScheduler = Executors.newScheduledThreadPool(1); + public void onTradePresencesMessage(Peer peer, Message message) { - TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) message; - List peersTradePresences = tradePresencesMessage.getTradePresences(); + synchronized (tradePresenceMessageLock) { + tradePresenceMessageList.add(new PeerMessage(peer, message)); + } + } - long now = NTP.getTime(); - // Timestamps before this are too far into the past - long pastThreshold = now; - // Timestamps after this are too far into the future - long futureThreshold = now + PRESENCE_LIFETIME; + public void processTradePresencesMessages() { - Map> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap(); + try { + List messagesToProcess; + synchronized (tradePresenceMessageLock) { + messagesToProcess = new ArrayList<>(tradePresenceMessageList); + tradePresenceMessageList.clear(); + } - int newCount = 0; + if( messagesToProcess.isEmpty() ) return; - try (final Repository repository = RepositoryManager.getRepository()) { - for (TradePresenceData peersTradePresence : peersTradePresences) { - long timestamp = peersTradePresence.getTimestamp(); + Map> tradePresencesByPeer = new HashMap<>(messagesToProcess.size()); - // Ignore if timestamp is out of bounds - if (timestamp < pastThreshold || timestamp > futureThreshold) { - if (timestamp < pastThreshold) - LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}", - peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold - ); - else - LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}", - peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold + // map all trade presences from the messages to their peer + for( PeerMessage peerMessage : messagesToProcess ) { + TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) peerMessage.getMessage(); + + List peersTradePresences = tradePresencesMessage.getTradePresences(); + + tradePresencesByPeer.put(peerMessage.getPeer(), peersTradePresences); + } + + long now = NTP.getTime(); + // Timestamps before this are too far into the past + long pastThreshold = now; + // Timestamps after this are too far into the future + long futureThreshold = now + PRESENCE_LIFETIME; + + Map> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap(); + + int newCount = 0; + + Map> peersByAtAddress = new HashMap<>(tradePresencesByPeer.size()); + Map tradePresenceByAtAddress = new HashMap<>(tradePresencesByPeer.size()); + + // for each batch of trade presence data from a peer, validate and populate the maps declared above + for ( Map.Entry> entry: tradePresencesByPeer.entrySet()) { + + Peer peer = entry.getKey(); + + for( TradePresenceData peersTradePresence : entry.getValue() ) { + // TradePresenceData peersTradePresence + long timestamp = peersTradePresence.getTimestamp(); + + // Ignore if timestamp is out of bounds + if (timestamp < pastThreshold || timestamp > futureThreshold) { + if (timestamp < pastThreshold) + LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}", + peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold + ); + else + LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}", + peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold + ); + + continue; + } + + ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey()); + + // Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older + TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray); + if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) { + if (timestamp == existingTradeData.getTimestamp()) + LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before", + peersTradePresence.getAtAddress(), peer, timestamp + ); + else + LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}", + peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp() + ); + + continue; + } + + // Check timestamp signature + byte[] timestampSignature = peersTradePresence.getSignature(); + byte[] timestampBytes = Longs.toByteArray(timestamp); + byte[] publicKey = peersTradePresence.getPublicKey(); + if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) { + LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify", + peersTradePresence.getAtAddress(), peer ); - continue; + continue; + } + + peersByAtAddress.computeIfAbsent(peersTradePresence.getAtAddress(), address -> new ArrayList<>()).add(peer); + tradePresenceByAtAddress.put(peersTradePresence.getAtAddress(), peersTradePresence); } + } - ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey()); + if( tradePresenceByAtAddress.isEmpty() ) return; - // Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older - TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray); - if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) { - if (timestamp == existingTradeData.getTimestamp()) - LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before", - peersTradePresence.getAtAddress(), peer, timestamp - ); - else - LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}", - peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp() - ); + List atDataList; + try (final Repository repository = RepositoryManager.getRepository()) { + atDataList = repository.getATRepository().fromATAddresses( new ArrayList<>(tradePresenceByAtAddress.keySet()) ); + } catch (DataException e) { + LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e); + return; + } - continue; - } + Map> supplierByAtAddress = new HashMap<>(atDataList.size()); - // Check timestamp signature - byte[] timestampSignature = peersTradePresence.getSignature(); - byte[] timestampBytes = Longs.toByteArray(timestamp); - byte[] publicKey = peersTradePresence.getPublicKey(); - if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) { - LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify", - peersTradePresence.getAtAddress(), peer - ); + List validatedAtDataList = new ArrayList<>(atDataList.size()); - continue; - } + // for each trade + for( ATData atData : atDataList ) { - ATData atData = repository.getATRepository().fromATAddress(peersTradePresence.getAtAddress()); + TradePresenceData peersTradePresence = tradePresenceByAtAddress.get(atData.getATAddress()); if (atData == null || atData.getIsFrozen() || atData.getIsFinished()) { if (atData == null) - LOGGER.trace("Ignoring trade presence {} from peer {} as AT doesn't exist", - peersTradePresence.getAtAddress(), peer + LOGGER.trace("Ignoring trade presence {} from peer as AT doesn't exist", + peersTradePresence.getAtAddress() ); else - LOGGER.trace("Ignoring trade presence {} from peer {} as AT is frozen or finished", - peersTradePresence.getAtAddress(), peer + LOGGER.trace("Ignoring trade presence {} from peer as AT is frozen or finished", + peersTradePresence.getAtAddress() ); continue; @@ -630,51 +701,87 @@ public class TradeBot implements Listener { ByteArray atCodeHash = ByteArray.wrap(atData.getCodeHash()); Supplier acctSupplier = acctSuppliersByCodeHash.get(atCodeHash); if (acctSupplier == null) { - LOGGER.trace("Ignoring trade presence {} from peer {} as AT isn't a known ACCT?", - peersTradePresence.getAtAddress(), peer + LOGGER.trace("Ignoring trade presence {} from peer as AT isn't a known ACCT?", + peersTradePresence.getAtAddress() ); continue; } - - CrossChainTradeData tradeData = acctSupplier.get().populateTradeData(repository, atData); - if (tradeData == null) { - LOGGER.trace("Ignoring trade presence {} from peer {} as trade data not found?", - peersTradePresence.getAtAddress(), peer - ); - - continue; - } - - // Convert signer's public key to address form - String signerAddress = peersTradePresence.getTradeAddress(); - - // Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form) - if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) { - LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?", - peersTradePresence.getAtAddress(), peer - ); - - continue; - } - - // This is new to us - this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence); - ++newCount; - - LOGGER.trace("Added trade presence {} from peer {} with timestamp {}", - peersTradePresence.getAtAddress(), peer, timestamp - ); - - EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence)); + validatedAtDataList.add(atData); } - } catch (DataException e) { - LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e); - } - if (newCount > 0) { - LOGGER.debug("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size()); - rebuildSafeAllTradePresences(); + // populated data for each trade + List crossChainTradeDataList; + + // validated trade data grouped by code (cross chain coin) + Map> atDataByCodeHash + = validatedAtDataList.stream().collect( + Collectors.groupingBy(data -> ByteArray.wrap(data.getCodeHash()))); + + try (final Repository repository = RepositoryManager.getRepository()) { + + crossChainTradeDataList = new ArrayList<>(); + + // for each code (cross chain coin), get each trade, then populate trade data + for( Map.Entry> entry : atDataByCodeHash.entrySet() ) { + + Supplier acctSupplier = acctSuppliersByCodeHash.get(entry.getKey()); + + crossChainTradeDataList.addAll( + acctSupplier.get().populateTradeDataList( + repository, + entry.getValue() + ) + .stream().filter( data -> data != null ) + .collect(Collectors.toList()) + ); + } + } catch (DataException e) { + LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e); + return; + } + + // for each populated trade data, validate and fire event + for( CrossChainTradeData tradeData : crossChainTradeDataList ) { + + List peers = peersByAtAddress.get(tradeData.qortalAtAddress); + + for( Peer peer : peers ) { + + TradePresenceData peersTradePresence = tradePresenceByAtAddress.get(tradeData.qortalAtAddress); + + // Convert signer's public key to address form + String signerAddress = peersTradePresence.getTradeAddress(); + + // Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form) + if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) { + LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?", + peersTradePresence.getAtAddress(), peer + ); + + continue; + } + + ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey()); + + // This is new to us + this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence); + ++newCount; + + LOGGER.trace("Added trade presence {} from peer {} with timestamp {}", + peersTradePresence.getAtAddress(), peer, tradeData.creationTimestamp + ); + + EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence)); + } + } + + if (newCount > 0) { + LOGGER.info("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size()); + rebuildSafeAllTradePresences(); + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } diff --git a/src/main/java/org/qortal/crosschain/ACCT.java b/src/main/java/org/qortal/crosschain/ACCT.java index de28cfce..83b453e6 100644 --- a/src/main/java/org/qortal/crosschain/ACCT.java +++ b/src/main/java/org/qortal/crosschain/ACCT.java @@ -6,6 +6,9 @@ import org.qortal.data.crosschain.CrossChainTradeData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import java.util.List; +import java.util.OptionalLong; + public interface ACCT { public byte[] getCodeBytesHash(); @@ -16,8 +19,12 @@ public interface ACCT { public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException; + public List populateTradeDataList(Repository respository, List atDataList) throws DataException; + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException; + CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException; + public byte[] buildCancelMessage(String creatorQortalAddress); public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException; diff --git a/src/main/java/org/qortal/crosschain/Bitcoin.java b/src/main/java/org/qortal/crosschain/Bitcoin.java index 79ab38af..9bda03d9 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoin.java +++ b/src/main/java/org/qortal/crosschain/Bitcoin.java @@ -1,5 +1,6 @@ package org.qortal.crosschain; +import org.bitcoinj.core.Coin; import org.bitcoinj.core.Context; import org.bitcoinj.core.NetworkParameters; import org.bitcoinj.core.Transaction; @@ -14,15 +15,21 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Bitcoin extends Bitcoiny { public static final String CURRENCY_CODE = "BTC"; - private static final long MINIMUM_ORDER_AMOUNT = 100000; // 0.001 BTC minimum order, due to high fees + // Locking fee to lock in a QORT for BTC. This is the default value that the user should reset to + // a value inline with the BTC fee market. This is 5 sats per kB. + private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(5_000); // 0.00005 BTC per 1000 bytes - // Temporary values until a dynamic fee system is written. - private static final long NEW_FEE_AMOUNT = 6_000L; + private static final long MINIMUM_ORDER_AMOUNT = 100_000; // 0.001 BTC minimum order, due to high fees + + // Default value until user resets fee to compete with the current market. This is a total value for a + // p2sh transaction, size 300 kB, 5 sats per kB + private static final long NEW_FEE_AMOUNT = 1_500L; private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST @@ -111,7 +118,7 @@ public class Bitcoin extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -173,14 +180,14 @@ public class Bitcoin extends Bitcoiny { } }; - private long feeCeiling = NEW_FEE_AMOUNT; + private AtomicLong feeRequired = new AtomicLong(NEW_FEE_AMOUNT); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -196,7 +203,7 @@ public class Bitcoin extends Bitcoiny { // Constructors and instance private Bitcoin(BitcoinNet bitcoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { - super(blockchain, bitcoinjContext, currencyCode, bitcoinjContext.getFeePerKb()); + super(blockchain, bitcoinjContext, currencyCode, DEFAULT_FEE_PER_KB); this.bitcoinNet = bitcoinNet; LOGGER.info(() -> String.format("Starting Bitcoin support using %s", this.bitcoinNet.name())); @@ -242,14 +249,14 @@ public class Bitcoin extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.bitcoinNet.getFeeCeiling(); + public long getFeeRequired() { + return this.bitcoinNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.bitcoinNet.setFeeCeiling( fee ); + this.bitcoinNet.setFeeRequired( fee ); } /** * Returns bitcoinj transaction sending amount to recipient using 20 sat/byte fee. diff --git a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java index cb855466..d7b8bb77 100644 --- a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java +++ b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -608,7 +610,14 @@ public class BitcoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -617,13 +626,14 @@ public class BitcoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -636,8 +646,13 @@ public class BitcoinACCTv1 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java b/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java index ecf768ed..efef0959 100644 --- a/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class BitcoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class BitcoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class BitcoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index d93fa65f..5cd6df35 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -840,9 +840,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { } while (true); } - public abstract long getFeeCeiling(); + public abstract long getFeeRequired(); - public abstract void setFeeCeiling(long fee); + public abstract void setFeeRequired(long fee); // UTXOProvider support diff --git a/src/main/java/org/qortal/crosschain/BitcoinyTBD.java b/src/main/java/org/qortal/crosschain/BitcoinyTBD.java index c25d2094..4b2f8173 100644 --- a/src/main/java/org/qortal/crosschain/BitcoinyTBD.java +++ b/src/main/java/org/qortal/crosschain/BitcoinyTBD.java @@ -89,7 +89,7 @@ public class BitcoinyTBD extends Bitcoiny { NetTBD netTBD = new NetTBD( bitcoinyTBDRequest.getNetworkName(), - bitcoinyTBDRequest.getFeeCeiling(), + bitcoinyTBDRequest.getFeeRequired(), networkParams, Collections.emptyList(), bitcoinyTBDRequest.getExpectedGenesisHash() @@ -134,18 +134,18 @@ public class BitcoinyTBD extends Bitcoiny { @Override public long getP2shFee(Long timestamp) throws ForeignBlockchainException { - return this.netTBD.getFeeCeiling(); + return this.netTBD.getFeeRequired(); } @Override - public long getFeeCeiling() { + public long getFeeRequired() { - return this.netTBD.getFeeCeiling(); + return this.netTBD.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.netTBD.setFeeCeiling( fee ); + this.netTBD.setFeeRequired( fee ); } } \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/Digibyte.java b/src/main/java/org/qortal/crosschain/Digibyte.java index f0a31087..9ee1f06a 100644 --- a/src/main/java/org/qortal/crosschain/Digibyte.java +++ b/src/main/java/org/qortal/crosschain/Digibyte.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Digibyte extends Bitcoiny { @@ -59,7 +60,7 @@ public class Digibyte extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -109,14 +110,14 @@ public class Digibyte extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -178,13 +179,13 @@ public class Digibyte extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.digibyteNet.getFeeCeiling(); + public long getFeeRequired() { + return this.digibyteNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.digibyteNet.setFeeCeiling( fee ); + this.digibyteNet.setFeeRequired( fee ); } } diff --git a/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java b/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java index 9fa67592..5dec80f0 100644 --- a/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java +++ b/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class DigibyteACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class DigibyteACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class DigibyteACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/Dogecoin.java b/src/main/java/org/qortal/crosschain/Dogecoin.java index dff98b1c..03898b93 100644 --- a/src/main/java/org/qortal/crosschain/Dogecoin.java +++ b/src/main/java/org/qortal/crosschain/Dogecoin.java @@ -13,6 +13,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Dogecoin extends Bitcoiny { @@ -60,7 +61,7 @@ public class Dogecoin extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -110,14 +111,14 @@ public class Dogecoin extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -179,13 +180,13 @@ public class Dogecoin extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.dogecoinNet.getFeeCeiling(); + public long getFeeRequired() { + return this.dogecoinNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.dogecoinNet.setFeeCeiling( fee ); + this.dogecoinNet.setFeeRequired( fee ); } } diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java index a5ec6f1f..a0caeb80 100644 --- a/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java +++ b/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -566,7 +568,14 @@ public class DogecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -575,13 +584,14 @@ public class DogecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -594,8 +604,13 @@ public class DogecoinACCTv1 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java index 06b04705..18581b2c 100644 --- a/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class DogecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class DogecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class DogecoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java index f13c1043..7c311b71 100644 --- a/src/main/java/org/qortal/crosschain/Litecoin.java +++ b/src/main/java/org/qortal/crosschain/Litecoin.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Litecoin extends Bitcoiny { @@ -63,7 +64,7 @@ public class Litecoin extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -116,14 +117,14 @@ public class Litecoin extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -185,13 +186,13 @@ public class Litecoin extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.litecoinNet.getFeeCeiling(); + public long getFeeRequired() { + return this.litecoinNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.litecoinNet.setFeeCeiling( fee ); + this.litecoinNet.setFeeRequired( fee ); } } diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java index 6a828981..b91e8e65 100644 --- a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -559,7 +561,14 @@ public class LitecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -568,13 +577,14 @@ public class LitecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -587,8 +597,13 @@ public class LitecoinACCTv1 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java index 4a533b4b..97222317 100644 --- a/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -562,7 +564,14 @@ public class LitecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -571,13 +580,14 @@ public class LitecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -590,8 +600,13 @@ public class LitecoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/NetTBD.java b/src/main/java/org/qortal/crosschain/NetTBD.java index c52449b4..21e0f106 100644 --- a/src/main/java/org/qortal/crosschain/NetTBD.java +++ b/src/main/java/org/qortal/crosschain/NetTBD.java @@ -3,18 +3,19 @@ package org.qortal.crosschain; import org.bitcoinj.core.NetworkParameters; import java.util.Collection; +import java.util.concurrent.atomic.AtomicLong; public class NetTBD { private String name; - private long feeCeiling; + private AtomicLong feeRequired; private NetworkParameters params; private Collection servers; private String genesisHash; - public NetTBD(String name, long feeCeiling, NetworkParameters params, Collection servers, String genesisHash) { + public NetTBD(String name, long feeRequired, NetworkParameters params, Collection servers, String genesisHash) { this.name = name; - this.feeCeiling = feeCeiling; + this.feeRequired = new AtomicLong(feeRequired); this.params = params; this.servers = servers; this.genesisHash = genesisHash; @@ -25,14 +26,14 @@ public class NetTBD { return this.name; } - public long getFeeCeiling() { + public long getFeeRequired() { - return feeCeiling; + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { + public void setFeeRequired(long feeRequired) { - this.feeCeiling = feeCeiling; + this.feeRequired.set(feeRequired); } public NetworkParameters getParams() { diff --git a/src/main/java/org/qortal/crosschain/PirateChain.java b/src/main/java/org/qortal/crosschain/PirateChain.java index 5475c929..53a6fa9c 100644 --- a/src/main/java/org/qortal/crosschain/PirateChain.java +++ b/src/main/java/org/qortal/crosschain/PirateChain.java @@ -21,6 +21,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.*; +import java.util.concurrent.atomic.AtomicLong; public class PirateChain extends Bitcoiny { @@ -51,12 +52,7 @@ public class PirateChain extends Bitcoiny { public Collection getServers() { return Arrays.asList( // Servers chosen on NO BASIS WHATSOEVER from various sources! - new Server("lightd.pirate.black", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr1.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr2.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr3.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr4.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr5.qortal.online", Server.ConnectionType.SSL, 443) + new Server("lightd.pirate.black", Server.ConnectionType.SSL, 443) ); } @@ -67,7 +63,7 @@ public class PirateChain extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -117,14 +113,14 @@ public class PirateChain extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -186,14 +182,14 @@ public class PirateChain extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.pirateChainNet.getFeeCeiling(); + public long getFeeRequired() { + return this.pirateChainNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.pirateChainNet.setFeeCeiling( fee ); + this.pirateChainNet.setFeeRequired( fee ); } /** * Returns confirmed balance, based on passed payment script. diff --git a/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java b/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java index 8873eeab..4c2893ea 100644 --- a/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java +++ b/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -580,7 +582,14 @@ public class PirateChainACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -589,13 +598,14 @@ public class PirateChainACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -608,8 +618,13 @@ public class PirateChainACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/PirateWallet.java b/src/main/java/org/qortal/crosschain/PirateWallet.java index d8fdc351..9ecfc2c0 100644 --- a/src/main/java/org/qortal/crosschain/PirateWallet.java +++ b/src/main/java/org/qortal/crosschain/PirateWallet.java @@ -8,6 +8,7 @@ import org.bouncycastle.util.encoders.DecoderException; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.controller.PirateChainWalletController; import org.qortal.crypto.Crypto; import org.qortal.settings.Settings; @@ -67,8 +68,8 @@ public class PirateWallet { } // Pick a random server - PirateLightClient.Server server = this.getRandomServer(); - String serverUri = String.format("https://%s:%d/", server.hostname, server.port); + ChainableServer server = PirateChain.getInstance().blockchainProvider.getCurrentServer(); + String serverUri = String.format("https://%s:%d/", server.getHostName(), server.getPort()); // Pirate library uses base64 encoding String entropy64 = Base64.toBase64String(this.entropyBytes); diff --git a/src/main/java/org/qortal/crosschain/Ravencoin.java b/src/main/java/org/qortal/crosschain/Ravencoin.java index cd98fb69..08b9be32 100644 --- a/src/main/java/org/qortal/crosschain/Ravencoin.java +++ b/src/main/java/org/qortal/crosschain/Ravencoin.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Ravencoin extends Bitcoiny { @@ -61,7 +62,7 @@ public class Ravencoin extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -111,14 +112,14 @@ public class Ravencoin extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong( MAINNET_FEE ); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -180,13 +181,13 @@ public class Ravencoin extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.ravencoinNet.getFeeCeiling(); + public long getFeeRequired() { + return this.ravencoinNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.ravencoinNet.setFeeCeiling( fee ); + this.ravencoinNet.setFeeRequired( fee ); } } diff --git a/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java b/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java index f027e9ca..b880f831 100644 --- a/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class RavencoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class RavencoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class RavencoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crypto/AES.java b/src/main/java/org/qortal/crypto/AES.java index d42e22f9..9bfa172a 100644 --- a/src/main/java/org/qortal/crypto/AES.java +++ b/src/main/java/org/qortal/crypto/AES.java @@ -100,7 +100,7 @@ public class AES { // Prepend the output stream with the 16 byte initialization vector outputStream.write(iv.getIV()); - byte[] buffer = new byte[1024]; + byte[] buffer = new byte[65536]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { byte[] output = cipher.update(buffer, 0, bytesRead); @@ -138,7 +138,7 @@ public class AES { Cipher cipher = Cipher.getInstance(algorithm); cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv)); - byte[] buffer = new byte[64]; + byte[] buffer = new byte[65536]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { byte[] output = cipher.update(buffer, 0, bytesRead); diff --git a/src/main/java/org/qortal/data/crosschain/ForeignFeeData.java b/src/main/java/org/qortal/data/crosschain/ForeignFeeData.java new file mode 100644 index 00000000..1b0e1199 --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/ForeignFeeData.java @@ -0,0 +1,57 @@ +package org.qortal.data.crosschain; + +import org.json.JSONObject; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class ForeignFeeData { + + private String blockchain; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long fee; + + protected ForeignFeeData() { + /* JAXB */ + } + + public ForeignFeeData(String blockchain, + long fee) { + this.blockchain = blockchain; + this.fee = fee; + } + + public String getBlockchain() { + return this.blockchain; + } + + public long getFee() { + return this.fee; + } + + public JSONObject toJson() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("blockchain", this.getBlockchain()); + jsonObject.put("fee", this.getFee()); + return jsonObject; + } + + public static ForeignFeeData fromJson(JSONObject json) { + return new ForeignFeeData( + json.isNull("blockchain") ? null : json.getString("blockchain"), + json.isNull("fee") ? null : json.getLong("fee") + ); + } + + @Override + public String toString() { + return "ForeignFeeData{" + + "blockchain='" + blockchain + '\'' + + ", fee=" + fee + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/crosschain/ForeignFeeDecodedData.java b/src/main/java/org/qortal/data/crosschain/ForeignFeeDecodedData.java new file mode 100644 index 00000000..de5bddc9 --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/ForeignFeeDecodedData.java @@ -0,0 +1,90 @@ +package org.qortal.data.crosschain; + +import org.json.JSONObject; +import org.qortal.data.account.MintingAccountData; +import org.qortal.utils.Base58; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class ForeignFeeDecodedData { + + protected long timestamp; + protected byte[] data; + protected String atAddress; + protected Integer fee; + + // Constructors + + // necessary for JAXB serialization + protected ForeignFeeDecodedData() { + } + + public ForeignFeeDecodedData(long timestamp, byte[] data, String atAddress, Integer fee) { + this.timestamp = timestamp; + this.data = data; + this.atAddress = atAddress; + this.fee = fee; + } + + public long getTimestamp() { + return this.timestamp; + } + + public byte[] getData() { + return this.data; + } + + public String getAtAddress() { + return atAddress; + } + + public Integer getFee() { + return this.fee; + } + + // Comparison + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ForeignFeeDecodedData that = (ForeignFeeDecodedData) o; + return timestamp == that.timestamp && Objects.equals(atAddress, that.atAddress) && Objects.equals(fee, that.fee); + } + + @Override + public int hashCode() { + return Objects.hash(timestamp, atAddress, fee); + } + + @Override + public String toString() { + return "ForeignFeeDecodedData{" + + "timestamp=" + timestamp + + ", atAddress='" + atAddress + '\'' + + ", fee=" + fee + + '}'; + } + + public JSONObject toJson() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("data", Base58.encode(this.data)); + jsonObject.put("atAddress", this.atAddress); + jsonObject.put("timestamp", this.timestamp); + jsonObject.put("fee", this.fee); + return jsonObject; + } + + public static ForeignFeeDecodedData fromJson(JSONObject json) { + return new ForeignFeeDecodedData( + json.isNull("timestamp") ? null : json.getLong("timestamp"), + json.isNull("data") ? null : Base58.decode(json.getString("data")), + json.isNull("atAddress") ? null : json.getString("atAddress"), + json.isNull("fee") ? null : json.getInt("fee")); + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/data/crosschain/ForeignFeeEncodedData.java b/src/main/java/org/qortal/data/crosschain/ForeignFeeEncodedData.java new file mode 100644 index 00000000..f0a3db2d --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/ForeignFeeEncodedData.java @@ -0,0 +1,69 @@ +package org.qortal.data.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class ForeignFeeEncodedData { + + protected long timestamp; + protected String data; + protected String atAddress; + protected Integer fee; + + // Constructors + + // necessary for JAXB serialization + protected ForeignFeeEncodedData() { + } + + public ForeignFeeEncodedData(long timestamp, String data, String atAddress, Integer fee) { + this.timestamp = timestamp; + this.data = data; + this.atAddress = atAddress; + this.fee = fee; + } + + public long getTimestamp() { + return this.timestamp; + } + + public String getData() { + return this.data; + } + + public String getAtAddress() { + return atAddress; + } + + public Integer getFee() { + return this.fee; + } + + // Comparison + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ForeignFeeEncodedData that = (ForeignFeeEncodedData) o; + return timestamp == that.timestamp && Objects.equals(atAddress, that.atAddress) && Objects.equals(fee, that.fee); + } + + @Override + public int hashCode() { + return Objects.hash(timestamp, atAddress, fee); + } + + @Override + public String toString() { + return "ForeignFeeDecodedData{" + + "timestamp=" + timestamp + + ", atAddress='" + atAddress + '\'' + + ", fee=" + fee + + '}'; + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java b/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java new file mode 100644 index 00000000..55c3589b --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java @@ -0,0 +1,29 @@ +package org.qortal.data.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class UnsignedFeeEvent { + + private boolean positive; + + private String address; + + public UnsignedFeeEvent() { + } + + public UnsignedFeeEvent(boolean positive, String address) { + + this.positive = positive; + this.address = address; + } + + public boolean isPositive() { + return positive; + } + + public String getAddress() { + return address; + } +} diff --git a/src/main/java/org/qortal/data/naming/NameData.java b/src/main/java/org/qortal/data/naming/NameData.java index 16e490a2..c76b7d48 100644 --- a/src/main/java/org/qortal/data/naming/NameData.java +++ b/src/main/java/org/qortal/data/naming/NameData.java @@ -67,6 +67,11 @@ public class NameData { this(name, reducedName, owner, data, registered, null, false, null, reference, creationGroupId); } + // Typically used for name summsry + public NameData(String name, String owner) { + this(name, null, owner, null, 0L, null, false, null, null, 0); + } + // Getters / setters public String getName() { diff --git a/src/main/java/org/qortal/event/FeeWaitingEvent.java b/src/main/java/org/qortal/event/FeeWaitingEvent.java new file mode 100644 index 00000000..bb5bc0aa --- /dev/null +++ b/src/main/java/org/qortal/event/FeeWaitingEvent.java @@ -0,0 +1,30 @@ +package org.qortal.event; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class FeeWaitingEvent implements Event{ + + private boolean positive; + + private String address; + + public FeeWaitingEvent() { + } + + public FeeWaitingEvent(boolean positive, String address) { + + this.positive = positive; + this.address = address; + + } + + public boolean isPositive() { + return positive; + } + + public String getAddress() { + return address; + } +} diff --git a/src/main/java/org/qortal/event/LockingFeeUpdateEvent.java b/src/main/java/org/qortal/event/LockingFeeUpdateEvent.java new file mode 100644 index 00000000..783b442e --- /dev/null +++ b/src/main/java/org/qortal/event/LockingFeeUpdateEvent.java @@ -0,0 +1,4 @@ +package org.qortal.event; + +public class LockingFeeUpdateEvent implements Event{ +} diff --git a/src/main/java/org/qortal/event/RequiredFeeUpdateEvent.java b/src/main/java/org/qortal/event/RequiredFeeUpdateEvent.java new file mode 100644 index 00000000..a0526196 --- /dev/null +++ b/src/main/java/org/qortal/event/RequiredFeeUpdateEvent.java @@ -0,0 +1,15 @@ +package org.qortal.event; + +import org.qortal.crosschain.Bitcoiny; + +public class RequiredFeeUpdateEvent implements Event{ + private final Bitcoiny bitcoiny; + + public RequiredFeeUpdateEvent(Bitcoiny bitcoiny) { + this.bitcoiny = bitcoiny; + } + + public Bitcoiny getBitcoiny() { + return bitcoiny; + } +} diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index f500b2e8..d6d662dc 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -714,6 +714,7 @@ public class Network { // We can't block here so use tryRepository(). We don't NEED to connect a new peer. try (Repository repository = RepositoryManager.tryRepository()) { if (repository == null) { + LOGGER.warn("Unable to get repository connection : Network.getConnectablePeer()"); return null; } @@ -982,7 +983,7 @@ public class Network { if (maxThreadsForMessageType != null) { Integer threadCount = threadsPerMessageType.get(message.getType()); if (threadCount != null && threadCount >= maxThreadsForMessageType) { - LOGGER.trace("Discarding {} message as there are already {} active threads", message.getType().name(), threadCount); + LOGGER.warn("Discarding {} message as there are already {} active threads", message.getType().name(), threadCount); return; } } @@ -1499,6 +1500,7 @@ public class Network { // Pruning peers isn't critical so no need to block for a repository instance. try (Repository repository = RepositoryManager.tryRepository()) { if (repository == null) { + LOGGER.warn("Unable to get repository connection : Network.prunePeers()"); return; } @@ -1567,6 +1569,7 @@ public class Network { // Merging peers isn't critical so don't block for a repository instance. try (Repository repository = RepositoryManager.tryRepository()) { if (repository == null) { + LOGGER.warn("Unable to get repository connection : Network.opportunisticMergePeers()"); return; } diff --git a/src/main/java/org/qortal/network/message/ForeignFeesMessage.java b/src/main/java/org/qortal/network/message/ForeignFeesMessage.java new file mode 100644 index 00000000..0b7d66d6 --- /dev/null +++ b/src/main/java/org/qortal/network/message/ForeignFeesMessage.java @@ -0,0 +1,43 @@ +package org.qortal.network.message; + +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.utils.ForeignFeesMessageUtils; + +import java.nio.ByteBuffer; +import java.util.List; + +/** + * For sending online accounts info to remote peer. + * + * Same format as V2, but with added support for a mempow nonce. + */ +public class ForeignFeesMessage extends Message { + + public static final long MIN_PEER_VERSION = 0x300060000L; // 3.6.0 + + private List foreignFees; + + public ForeignFeesMessage(List foreignFeeDecodedData) { + super(MessageType.FOREIGN_FEES); + + this.dataBytes = ForeignFeesMessageUtils.fromDataToSendBytes(foreignFeeDecodedData); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private ForeignFeesMessage(int id, List foreignFees) { + super(id, MessageType.FOREIGN_FEES); + + this.foreignFees = foreignFees; + } + + public List getForeignFees() { + return this.foreignFees; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + List foreignFeeDecodedData = ForeignFeesMessageUtils.fromSendBytesToData(bytes); + + return new ForeignFeesMessage(id, foreignFeeDecodedData); + } + +} diff --git a/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java b/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java new file mode 100644 index 00000000..b746cc65 --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java @@ -0,0 +1,46 @@ +package org.qortal.network.message; + +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.utils.ForeignFeesMessageUtils; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class GetForeignFeesMessage extends Message { + + private static final Map> EMPTY_ONLINE_ACCOUNTS = Collections.emptyMap(); + private final List foreignFeeDecodedData; + + public GetForeignFeesMessage(List foreignFeeDecodedData) { + super(MessageType.GET_FOREIGN_FEES); + + this.foreignFeeDecodedData = foreignFeeDecodedData; + + // If we don't have ANY online accounts then it's an easier construction... + if (foreignFeeDecodedData.isEmpty()) { + this.dataBytes = EMPTY_DATA_BYTES; + return; + } + + this.dataBytes = ForeignFeesMessageUtils.fromDataToGetBytes(foreignFeeDecodedData); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private GetForeignFeesMessage(int id, List foreignFeeDecodedData) { + super(id, MessageType.GET_FOREIGN_FEES); + + this.foreignFeeDecodedData = foreignFeeDecodedData; + } + + public List getForeignFeeData() { + return foreignFeeDecodedData; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + + return new GetForeignFeesMessage(id, ForeignFeesMessageUtils.fromGetBytesToData(bytes)); + } + +} diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java index 6b420e2d..ca35ece1 100644 --- a/src/main/java/org/qortal/network/message/MessageType.java +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -79,7 +79,10 @@ public enum MessageType { GET_NAME(182, GetNameMessage::fromByteBuffer), TRANSACTIONS(190, TransactionsMessage::fromByteBuffer), - GET_ACCOUNT_TRANSACTIONS(191, GetAccountTransactionsMessage::fromByteBuffer); + GET_ACCOUNT_TRANSACTIONS(191, GetAccountTransactionsMessage::fromByteBuffer), + + FOREIGN_FEES( 200, ForeignFeesMessage::fromByteBuffer), + GET_FOREIGN_FEES( 201, GetForeignFeesMessage::fromByteBuffer); public final int value; public final MessageProducer fromByteBufferMethod; diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 2b653ab5..7e6c16ff 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -14,6 +14,8 @@ public interface ATRepository { /** Returns ATData using AT's address or null if none found */ public ATData fromATAddress(String atAddress) throws DataException; + public List fromATAddresses(List atAddresses) throws DataException; + /** Returns where AT with passed address exists in repository */ public boolean exists(String atAddress) throws DataException; @@ -62,6 +64,8 @@ public interface ATRepository { */ public ATStateData getLatestATState(String atAddress) throws DataException; + public List getLatestATStates(List collect) throws DataException; + /** * Returns final ATStateData for ATs matching codeHash (required) * and specific data segment value (optional). diff --git a/src/main/java/org/qortal/repository/AccountRepository.java b/src/main/java/org/qortal/repository/AccountRepository.java index f68fe8eb..daac1e02 100644 --- a/src/main/java/org/qortal/repository/AccountRepository.java +++ b/src/main/java/org/qortal/repository/AccountRepository.java @@ -130,6 +130,8 @@ public interface AccountRepository { */ public AccountBalanceData getBalance(String address, long assetId) throws DataException; + public List getBalances(List addresses, long assetId) throws DataException; + /** Returns all account balances for given assetID, optionally excluding zero balances. */ public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException; diff --git a/src/main/java/org/qortal/repository/NameRepository.java b/src/main/java/org/qortal/repository/NameRepository.java index c49d5d18..c61bb96f 100644 --- a/src/main/java/org/qortal/repository/NameRepository.java +++ b/src/main/java/org/qortal/repository/NameRepository.java @@ -3,6 +3,7 @@ package org.qortal.repository; import org.qortal.data.naming.NameData; import java.util.List; +import java.util.Optional; public interface NameRepository { @@ -34,10 +35,17 @@ public interface NameRepository { return getNamesByOwner(address, null, null, null); } + public int setPrimaryName(String address, String primaryName) throws DataException; + + public void removePrimaryName(String address) throws DataException; + + public Optional getPrimaryName(String address) throws DataException; + + public int clearPrimaryNames() throws DataException; + public List getRecentNames(long startTimestamp) throws DataException; public void save(NameData nameData) throws DataException; public void delete(String name) throws DataException; - } diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index c0bdb0d9..a361ee95 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -1,6 +1,7 @@ package org.qortal.repository; import java.io.IOException; +import java.sql.Connection; import java.util.concurrent.TimeoutException; public interface Repository extends AutoCloseable { @@ -62,4 +63,5 @@ public interface Repository extends AutoCloseable { public static void attemptRecovery(String connectionUrl, String name) throws DataException {} + public Connection getConnection(); } diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index d4517485..79d55c9a 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -18,6 +18,8 @@ public interface TransactionRepository { public TransactionData fromSignature(byte[] signature) throws DataException; + public List fromSignatures(List signatures) throws DataException; + public TransactionData fromReference(byte[] reference) throws DataException; public TransactionData fromHeightAndSequence(int height, int sequence) throws DataException; @@ -351,4 +353,5 @@ public interface TransactionRepository { public void delete(TransactionData transactionData) throws DataException; + } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 6310ec02..c941e15a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -15,8 +15,12 @@ import org.qortal.utils.ByteArray; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.qortal.data.account.AccountData; @@ -76,6 +80,63 @@ public class HSQLDBATRepository implements ATRepository { } } + @Override + public List fromATAddresses(List atAddresses) throws DataException { + String sql = "SELECT creator, created_when, version, asset_id, code_bytes, code_hash, " + + "is_sleeping, sleep_until_height, is_finished, had_fatal_error, " + + "is_frozen, frozen_balance, sleep_until_message_timestamp, AT_address " + + "FROM ATs " + + "WHERE AT_address IN (" + + String.join(", ", Collections.nCopies(atAddresses.size(), "?")) + + ")" + ; + + List list; + try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddresses.toArray(new String[atAddresses.size()]))) { + if (resultSet == null) { + return new ArrayList<>(0); + } + + list = new ArrayList<>(atAddresses.size()); + + do { + byte[] creatorPublicKey = resultSet.getBytes(1); + long created = resultSet.getLong(2); + int version = resultSet.getInt(3); + long assetId = resultSet.getLong(4); + byte[] codeBytes = resultSet.getBytes(5); // Actually BLOB + byte[] codeHash = resultSet.getBytes(6); + boolean isSleeping = resultSet.getBoolean(7); + + Integer sleepUntilHeight = resultSet.getInt(8); + if (sleepUntilHeight == 0 && resultSet.wasNull()) + sleepUntilHeight = null; + + boolean isFinished = resultSet.getBoolean(9); + boolean hadFatalError = resultSet.getBoolean(10); + boolean isFrozen = resultSet.getBoolean(11); + + Long frozenBalance = resultSet.getLong(12); + if (frozenBalance == 0 && resultSet.wasNull()) + frozenBalance = null; + + Long sleepUntilMessageTimestamp = resultSet.getLong(13); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + + String atAddress = resultSet.getString(14); + + list.add(new ATData(atAddress, creatorPublicKey, created, version, assetId, codeBytes, codeHash, + isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance, + sleepUntilMessageTimestamp)); + } while ( resultSet.next()); + + return list; + } catch (SQLException e) { + throw new DataException("Unable to fetch AT from repository", e); + } + } + @Override public boolean exists(String atAddress) throws DataException { try { @@ -403,6 +464,56 @@ public class HSQLDBATRepository implements ATRepository { } } + @Override + public List getLatestATStates(List atAddresses) throws DataException{ + String sql = "SELECT height, state_data, state_hash, fees, is_initial, sleep_until_message_timestamp, AT_address " + + "FROM ATStates " + + "JOIN ATStatesData USING (AT_address, height) " + + "WHERE ATStates.AT_address IN (" + + String.join(", ", Collections.nCopies(atAddresses.size(), "?")) + + ")"; + + List stateDataList; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddresses.toArray(new String[atAddresses.size()]))) { + if (resultSet == null) + return new ArrayList<>(0); + + stateDataList = new ArrayList<>(); + + do { + int height = resultSet.getInt(1); + byte[] stateData = resultSet.getBytes(2); // Actually BLOB + byte[] stateHash = resultSet.getBytes(3); + long fees = resultSet.getLong(4); + boolean isInitial = resultSet.getBoolean(5); + + Long sleepUntilMessageTimestamp = resultSet.getLong(6); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + + String atAddress = resultSet.getString(7); + stateDataList.add(new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial, sleepUntilMessageTimestamp)); + } while( resultSet.next()); + } catch (SQLException e) { + throw new DataException("Unable to fetch latest AT state from repository", e); + } + + Map> stateDataByAtAddress + = stateDataList.stream() + .collect(Collectors.groupingBy(ATStateData::getATAddress)); + + List latestForEachAtAddress + = stateDataByAtAddress.values().stream() + .map(list -> list.stream() + .max(Comparator.comparing(ATStateData::getHeight)) + .orElse(null)) + .filter(obj -> obj != null) + .collect(Collectors.toList()); + + return latestForEachAtAddress; + } + @Override public List getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished, Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java index 9cec85b2..7a0e486c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java @@ -407,6 +407,39 @@ public class HSQLDBAccountRepository implements AccountRepository { } } + @Override + public List getBalances(List addresses, long assetId) throws DataException { + + StringBuffer sql = new StringBuffer(); + sql.append("SELECT balance, account, asset_id FROM AccountBalances "); + sql.append("WHERE account IN ("); + sql.append(String.join(", ", Collections.nCopies(addresses.size(), "?"))); + sql.append(")"); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), addresses.toArray(new String[addresses.size()]))) { + if (resultSet == null) + return new ArrayList<>(0); + + List balances = new ArrayList<>(addresses.size()); + do { + long balance = resultSet.getLong(1); + String address = resultSet.getString(2); + Long assetIdResult = resultSet.getLong(3); + + if( assetIdResult != assetId ) { + LOGGER.warn("assetIdResult = " + assetIdResult); + continue; + } + + balances.add(new AccountBalanceData(address, assetId, balance) ); + } while( resultSet.next()); + + return balances; + } catch (SQLException e) { + throw new DataException("Unable to fetch account balance from repository", e); + } + } + @Override public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException { StringBuilder sql = new StringBuilder(1024); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index 46cd7cab..bee629b8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -468,7 +468,7 @@ public class HSQLDBCacheUtils { Thread.currentThread().setName(DB_CACHE_TIMER_TASK); - try (final HSQLDBRepository respository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) { + try (final Repository respository = RepositoryManager.getRepository()) { fillCache(ArbitraryResourceCache.getInstance(), respository); } catch( DataException e ) { @@ -611,7 +611,7 @@ public class HSQLDBCacheUtils { private static int recordCurrentBalances(ConcurrentHashMap> balancesByHeight) { int currentHeight; - try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) { + try (final Repository repository = RepositoryManager.getRepository()) { // get current balances List accountBalances = getAccountBalances(repository); @@ -675,7 +675,7 @@ public class HSQLDBCacheUtils { * @param cache the cache to fill * @param repository the data source to fill the cache with */ - public static void fillCache(ArbitraryResourceCache cache, HSQLDBRepository repository) { + public static void fillCache(ArbitraryResourceCache cache, Repository repository) { try { // ensure all data is committed in, before we query it @@ -713,7 +713,7 @@ public class HSQLDBCacheUtils { * * @throws SQLException */ - private static void fillNamepMap(ConcurrentHashMap levelByName, HSQLDBRepository repository ) throws SQLException { + private static void fillNamepMap(ConcurrentHashMap levelByName, Repository repository ) throws SQLException { StringBuilder sql = new StringBuilder(512); @@ -721,7 +721,7 @@ public class HSQLDBCacheUtils { sql.append("FROM NAMES "); sql.append("INNER JOIN ACCOUNTS on owner = account "); - Statement statement = repository.connection.createStatement(); + Statement statement = repository.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql.toString()); @@ -744,7 +744,7 @@ public class HSQLDBCacheUtils { * @return the resources * @throws SQLException */ - private static List getResources( HSQLDBRepository repository) throws SQLException { + private static List getResources( Repository repository) throws SQLException { List resources = new ArrayList<>(); @@ -756,7 +756,7 @@ public class HSQLDBCacheUtils { sql.append("LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) WHERE name IS NOT NULL"); List arbitraryResources = new ArrayList<>(); - Statement statement = repository.connection.createStatement(); + Statement statement = repository.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql.toString()); @@ -822,7 +822,7 @@ public class HSQLDBCacheUtils { return resources; } - public static List getAccountBalances(HSQLDBRepository repository) { + public static List getAccountBalances(Repository repository) { StringBuilder sql = new StringBuilder(); @@ -836,7 +836,7 @@ public class HSQLDBCacheUtils { LOGGER.info( "Getting account balances ..."); try { - Statement statement = repository.connection.createStatement(); + Statement statement = repository.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql.toString()); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java index 80865739..535c3ed6 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java @@ -1,5 +1,8 @@ package org.qortal.repository.hsqldb; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.BlockChain; import org.qortal.data.chat.ActiveChats; import org.qortal.data.chat.ActiveChats.DirectChat; import org.qortal.data.chat.ActiveChats.GroupChat; @@ -18,6 +21,8 @@ import static org.qortal.data.chat.ChatMessage.Encoding; public class HSQLDBChatRepository implements ChatRepository { + private static final Logger LOGGER = LogManager.getLogger(HSQLDBChatRepository.class); + protected HSQLDBRepository repository; public HSQLDBChatRepository(HSQLDBRepository repository) { @@ -35,13 +40,25 @@ public class HSQLDBChatRepository implements ChatRepository { StringBuilder sql = new StringBuilder(1024); + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + sql.append("SELECT created_when, tx_group_id, Transactions.reference, creator, " + "sender, SenderNames.name, recipient, RecipientNames.name, " + "chat_reference, data, is_text, is_encrypted, signature " + "FROM ChatTransactions " + "JOIN Transactions USING (signature) " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " - + "LEFT OUTER JOIN Names AS RecipientNames ON RecipientNames.owner = recipient "); + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS RecipientNames ON RecipientNames.owner = recipient "); // WHERE clauses @@ -142,10 +159,23 @@ public class HSQLDBChatRepository implements ChatRepository { @Override public ChatMessage toChatMessage(ChatTransactionData chatTransactionData, Encoding encoding) throws DataException { + + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + String sql = "SELECT SenderNames.name, RecipientNames.name " + "FROM ChatTransactions " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " - + "LEFT OUTER JOIN Names AS RecipientNames ON RecipientNames.owner = recipient " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS RecipientNames ON RecipientNames.owner = recipient " + "WHERE signature = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, chatTransactionData.getSignature())) { @@ -184,6 +214,18 @@ public class HSQLDBChatRepository implements ChatRepository { } private List getActiveGroupChats(String address, Encoding encoding, Boolean hasChatReference) throws DataException { + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + // Find groups where address is a member and potential latest message details String groupsSql = "SELECT group_id, group_name, latest_timestamp, sender, sender_name, signature, data " + "FROM GroupMembers " @@ -192,7 +234,7 @@ public class HSQLDBChatRepository implements ChatRepository { + "SELECT created_when AS latest_timestamp, sender, name AS sender_name, signature, data " + "FROM ChatTransactions " + "JOIN Transactions USING (signature) " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " // NOTE: We need to qualify "Groups.group_id" here to avoid "General error" bug in HSQLDB v2.5.0 + "WHERE tx_group_id = Groups.group_id AND type = " + TransactionType.CHAT.value + " "; @@ -236,7 +278,7 @@ public class HSQLDBChatRepository implements ChatRepository { String grouplessSql = "SELECT created_when, sender, SenderNames.name, signature, data " + "FROM ChatTransactions " + "JOIN Transactions USING (signature) " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + "WHERE tx_group_id = 0 " + "AND recipient IS NULL "; @@ -276,6 +318,18 @@ public class HSQLDBChatRepository implements ChatRepository { } private List getActiveDirectChats(String address, Boolean hasChatReference) throws DataException { + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + // Find chat messages involving address String directSql = "SELECT other_address, name, latest_timestamp, sender, sender_name " + "FROM (" @@ -289,7 +343,7 @@ public class HSQLDBChatRepository implements ChatRepository { + "SELECT created_when AS latest_timestamp, sender, name AS sender_name " + "FROM ChatTransactions " + "NATURAL JOIN Transactions " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + "WHERE (sender = other_address AND recipient = ?) " + "OR (sender = ? AND recipient = other_address) "; @@ -305,7 +359,7 @@ public class HSQLDBChatRepository implements ChatRepository { directSql += "ORDER BY created_when DESC " + "LIMIT 1" + ") AS LatestMessages " - + "LEFT OUTER JOIN Names ON owner = other_address"; + + "LEFT OUTER JOIN " + tableName + " ON owner = other_address"; Object[] bindParams = new Object[] { address, address, address, address }; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index ca55f3a8..69a06b6a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -1053,6 +1053,12 @@ public class HSQLDBDatabaseUpdates { stmt.execute("UPDATE Accounts SET blocks_minted_penalty = -5000000 WHERE blocks_minted_penalty < 0"); break; + case 50: + // Primary name for a Qortal Address, 0-1 for any address + stmt.execute("CREATE TABLE PrimaryNames (owner QortalAddress, name RegisteredName, " + + "PRIMARY KEY (owner), FOREIGN KEY (name) REFERENCES Names (name) ON DELETE CASCADE)"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java index 06e41663..fba1e83d 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java @@ -8,6 +8,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; public class HSQLDBNameRepository implements NameRepository { @@ -267,7 +268,7 @@ public class HSQLDBNameRepository implements NameRepository { StringBuilder sql = new StringBuilder(512); sql.append("SELECT name, reduced_name, data, registered_when, updated_when, " - + "is_for_sale, sale_price, reference, creation_group_id FROM Names WHERE owner = ? ORDER BY name"); + + "is_for_sale, sale_price, reference, creation_group_id FROM Names WHERE owner = ? ORDER BY registered_when"); if (reverse != null && reverse) sql.append(" DESC"); @@ -333,6 +334,55 @@ public class HSQLDBNameRepository implements NameRepository { } } + @Override + public void removePrimaryName(String address) throws DataException { + try { + this.repository.delete("PrimaryNames", "owner = ?", address); + } catch (SQLException e) { + throw new DataException("Unable to delete primary name from repository", e); + } + } + + @Override + public Optional getPrimaryName(String address) throws DataException { + String sql = "SELECT name FROM PrimaryNames WHERE owner = ?"; + + List names = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql, address)) { + if (resultSet == null) + return Optional.empty(); + + String name = resultSet.getString(1); + + return Optional.of(name); + } catch (SQLException e) { + throw new DataException("Unable to fetch recent names from repository", e); + } + } + + @Override + public int setPrimaryName(String address, String primaryName) throws DataException { + + String sql = "INSERT INTO PrimaryNames (owner, name) VALUES (?, ?) ON DUPLICATE KEY UPDATE name = ?"; + + try{ + return this.repository.executeCheckedUpdate(sql, address, primaryName, primaryName); + } catch (SQLException e) { + throw new DataException("Unable to set primary name", e); + } + } + + @Override + public int clearPrimaryNames() throws DataException { + + try { + return this.repository.delete("PrimaryNames"); + } catch (SQLException e) { + throw new DataException("Unable to clear primary names from repository", e); + } + } + @Override public void save(NameData nameData) throws DataException { HSQLDBSaver saveHelper = new HSQLDBSaver("Names"); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 4a41ed68..2bf88657 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -174,6 +174,11 @@ public class HSQLDBRepository implements Repository { // Transaction COMMIT / ROLLBACK / savepoints + @Override + public Connection getConnection() { + return this.connection; + } + @Override public void saveChanges() throws DataException { long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis(); diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index fe0b4d0b..cd646cb9 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -155,6 +155,58 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } } + @Override + public List fromSignatures(List signatures) throws DataException { + StringBuffer sql = new StringBuffer(); + + sql.append("SELECT type, reference, creator, created_when, fee, tx_group_id, block_height, approval_status, approval_height, signature "); + sql.append("FROM Transactions WHERE signature IN ("); + sql.append(String.join(", ", Collections.nCopies(signatures.size(), "?"))); + sql.append(")"); + + List list; + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), signatures.toArray(new byte[0][]))) { + if (resultSet == null) { + return new ArrayList<>(0); + } + + list = new ArrayList<>(signatures.size()); + + do { + TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] creatorPublicKey = resultSet.getBytes(3); + long timestamp = resultSet.getLong(4); + + Long fee = resultSet.getLong(5); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(6); + + Integer blockHeight = resultSet.getInt(7); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(8)); + Integer approvalHeight = resultSet.getInt(9); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + byte[] signature = resultSet.getBytes(10); + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + list.add( fromBase(type, baseTransactionData) ); + } while( resultSet.next()); + + return list; + } catch (SQLException e) { + throw new DataException("Unable to fetch transactions from repository", e); + } + } + @Override public TransactionData fromReference(byte[] reference) throws DataException { String sql = "SELECT type, signature, creator, created_when, fee, tx_group_id, block_height, approval_status, approval_height " diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 3e82655b..e5efc7b4 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -759,14 +759,14 @@ public class Settings { maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA", 5)); maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE_LIST", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE_LIST", 5)); + maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE_LIST", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE_LIST", 50)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_SIGNATURES", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_METADATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_TRANSACTION", 10)); - maxThreadsPerMessageType.add(new ThreadLimit("TRANSACTION_SIGNATURES", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("TRADE_PRESENCES", 5)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 100)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_TRANSACTION", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("TRANSACTION_SIGNATURES", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("TRADE_PRESENCES", 50)); } // Getters / setters diff --git a/src/main/java/org/qortal/transaction/BuyNameTransaction.java b/src/main/java/org/qortal/transaction/BuyNameTransaction.java index 72c15f69..fc5bf6fc 100644 --- a/src/main/java/org/qortal/transaction/BuyNameTransaction.java +++ b/src/main/java/org/qortal/transaction/BuyNameTransaction.java @@ -16,6 +16,7 @@ import org.qortal.utils.Unicode; import java.util.Collections; import java.util.List; +import java.util.Optional; public class BuyNameTransaction extends Transaction { @@ -48,6 +49,15 @@ public class BuyNameTransaction extends Transaction { @Override public ValidationResult isValid() throws DataException { + Optional buyerPrimaryName = this.getBuyer().getPrimaryName(); + if( buyerPrimaryName.isPresent() ) { + + NameData nameData = repository.getNameRepository().fromName(buyerPrimaryName.get()); + if (nameData.isForSale()) { + return ValidationResult.NOT_SUPPORTED; + } + } + String name = this.buyNameTransactionData.getName(); // Check seller address is valid @@ -79,7 +89,7 @@ public class BuyNameTransaction extends Transaction { return ValidationResult.BUYER_ALREADY_OWNER; // If accounts are only allowed one registered name then check for this - if (BlockChain.getInstance().oneNamePerAccount() + if (BlockChain.getInstance().oneNamePerAccount(this.repository.getBlockRepository().getBlockchainHeight()) && !this.repository.getNameRepository().getNamesByOwner(buyer.getAddress()).isEmpty()) return ValidationResult.MULTIPLE_NAMES_FORBIDDEN; @@ -92,7 +102,7 @@ public class BuyNameTransaction extends Transaction { return ValidationResult.INVALID_AMOUNT; // Check buyer has enough funds - if (buyer.getConfirmedBalance(Asset.QORT) < this.buyNameTransactionData.getFee()) + if (buyer.getConfirmedBalance(Asset.QORT) < this.buyNameTransactionData.getFee() + this.buyNameTransactionData.getAmount()) return ValidationResult.NO_BALANCE; return ValidationResult.OK; @@ -117,6 +127,25 @@ public class BuyNameTransaction extends Transaction { // Save transaction with updated "name reference" pointing to previous transaction that changed name this.repository.getTransactionRepository().save(this.buyNameTransactionData); + + // if multiple names feature is activated, then check the buyer and seller's primary name status + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account seller = new Account(this.repository, this.buyNameTransactionData.getSeller()); + Optional sellerPrimaryName = seller.getPrimaryName(); + + // if the seller sold their primary name, then remove their primary name + if (sellerPrimaryName.isPresent() && sellerPrimaryName.get().equals(buyNameTransactionData.getName())) { + seller.removePrimaryName(); + } + + Account buyer = new Account(this.repository, this.getBuyer().getAddress()); + + // if the buyer had no primary name, then set the primary name to the name bought + if( buyer.getPrimaryName().isEmpty() ) { + buyer.setPrimaryName(this.buyNameTransactionData.getName()); + } + } } @Override @@ -127,6 +156,24 @@ public class BuyNameTransaction extends Transaction { // Save this transaction, with previous "name reference" this.repository.getTransactionRepository().save(this.buyNameTransactionData); - } + // if multiple names feature is activated, then check the buyer and seller's primary name status + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account seller = new Account(this.repository, this.buyNameTransactionData.getSeller()); + + // if the seller lost their primary name, then set their primary name back + if (seller.getPrimaryName().isEmpty()) { + seller.setPrimaryName(this.buyNameTransactionData.getName()); + } + + Account buyer = new Account(this.repository, this.getBuyer().getAddress()); + Optional buyerPrimaryName = buyer.getPrimaryName(); + + // if the buyer bought their primary, then remove it + if( buyerPrimaryName.isPresent() && this.buyNameTransactionData.getName().equals(buyerPrimaryName.get()) ) { + buyer.removePrimaryName(); + } + } + } } diff --git a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java index a89e60c0..8e8e2fcc 100644 --- a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java +++ b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java @@ -2,10 +2,12 @@ package org.qortal.transaction; import com.google.common.base.Utf8; import org.qortal.account.Account; +import org.qortal.api.resource.TransactionsResource; import org.qortal.asset.Asset; import org.qortal.block.BlockChain; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; +import org.qortal.data.naming.NameData; import org.qortal.data.transaction.RegisterNameTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.naming.Name; @@ -15,6 +17,7 @@ import org.qortal.utils.Unicode; import java.util.Collections; import java.util.List; +import java.util.Optional; public class RegisterNameTransaction extends Transaction { @@ -54,6 +57,15 @@ public class RegisterNameTransaction extends Transaction { Account registrant = getRegistrant(); String name = this.registerNameTransactionData.getName(); + Optional registrantPrimaryName = registrant.getPrimaryName(); + if( registrantPrimaryName.isPresent() ) { + + NameData nameData = repository.getNameRepository().fromName(registrantPrimaryName.get()); + if (nameData.isForSale()) { + return ValidationResult.NOT_SUPPORTED; + } + } + int blockchainHeight = this.repository.getBlockRepository().getBlockchainHeight(); final int start = BlockChain.getInstance().getSelfSponsorshipAlgoV2Height() - 1180; final int end = BlockChain.getInstance().getSelfSponsorshipAlgoV3Height(); @@ -94,7 +106,7 @@ public class RegisterNameTransaction extends Transaction { return ValidationResult.NAME_ALREADY_REGISTERED; // If accounts are only allowed one registered name then check for this - if (BlockChain.getInstance().oneNamePerAccount() + if (BlockChain.getInstance().oneNamePerAccount(this.repository.getBlockRepository().getBlockchainHeight()) && !this.repository.getNameRepository().getNamesByOwner(getRegistrant().getAddress()).isEmpty()) return ValidationResult.MULTIPLE_NAMES_FORBIDDEN; @@ -117,6 +129,16 @@ public class RegisterNameTransaction extends Transaction { // Register Name Name name = new Name(this.repository, this.registerNameTransactionData); name.register(); + + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account account = new Account(this.repository, this.getCreator().getAddress()); + + // if there is no primary name established, then the new registered name is the primary name + if (account.getPrimaryName().isEmpty()) { + account.setPrimaryName(this.registerNameTransactionData.getName()); + } + } } @Override diff --git a/src/main/java/org/qortal/transaction/SellNameTransaction.java b/src/main/java/org/qortal/transaction/SellNameTransaction.java index 3b6bf5df..8dcf9bd2 100644 --- a/src/main/java/org/qortal/transaction/SellNameTransaction.java +++ b/src/main/java/org/qortal/transaction/SellNameTransaction.java @@ -45,6 +45,12 @@ public class SellNameTransaction extends Transaction { public ValidationResult isValid() throws DataException { String name = this.sellNameTransactionData.getName(); + // if the account has more than one name, then they cannot sell their primary name + if( this.repository.getNameRepository().getNamesByOwner(this.getOwner().getAddress()).size() > 1 && + this.getOwner().getPrimaryName().get().equals(name) ) { + return ValidationResult.NOT_SUPPORTED; + } + // Check name size bounds int nameLength = Utf8.encodedLength(name); if (nameLength < 1 || nameLength > Name.MAX_NAME_SIZE) diff --git a/src/main/java/org/qortal/transaction/UpdateNameTransaction.java b/src/main/java/org/qortal/transaction/UpdateNameTransaction.java index 8d42207c..bf0d12dc 100644 --- a/src/main/java/org/qortal/transaction/UpdateNameTransaction.java +++ b/src/main/java/org/qortal/transaction/UpdateNameTransaction.java @@ -3,6 +3,7 @@ package org.qortal.transaction; import com.google.common.base.Utf8; import org.qortal.account.Account; import org.qortal.asset.Asset; +import org.qortal.block.BlockChain; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; import org.qortal.data.naming.NameData; @@ -49,6 +50,12 @@ public class UpdateNameTransaction extends Transaction { public ValidationResult isValid() throws DataException { String name = this.updateNameTransactionData.getName(); + // if the account has more than one name, then they cannot update their primary name + if( this.repository.getNameRepository().getNamesByOwner(this.getOwner().getAddress()).size() > 1 && + this.getOwner().getPrimaryName().get().equals(name) ) { + return ValidationResult.NOT_SUPPORTED; + } + // Check name size bounds int nameLength = Utf8.encodedLength(name); if (nameLength < Name.MIN_NAME_SIZE || nameLength > Name.MAX_NAME_SIZE) @@ -152,6 +159,16 @@ public class UpdateNameTransaction extends Transaction { // Save this transaction, now with updated "name reference" to previous transaction that changed name this.repository.getTransactionRepository().save(this.updateNameTransactionData); + + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account account = new Account(this.repository, this.getCreator().getAddress()); + + // if updating the primary name, then set primary name to new name + if( account.getPrimaryName().isEmpty() || account.getPrimaryName().get().equals(this.updateNameTransactionData.getName())) { + account.setPrimaryName(this.updateNameTransactionData.getNewName()); + } + } } @Override @@ -167,6 +184,16 @@ public class UpdateNameTransaction extends Transaction { // Save this transaction, with previous "name reference" this.repository.getTransactionRepository().save(this.updateNameTransactionData); + + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account account = new Account(this.repository, this.getCreator().getAddress()); + + // if the primary name is the new updated name, then it needs to be set back to the previous name + if (account.getPrimaryName().isPresent() && account.getPrimaryName().get().equals(this.updateNameTransactionData.getNewName())) { + account.setPrimaryName(this.updateNameTransactionData.getName()); + } + } } } diff --git a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java index 17c966fe..156948a9 100644 --- a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java @@ -53,7 +53,7 @@ public class ArbitraryIndexUtils { try { fillCache(IndexCache.getInstance()); } catch (IOException | DataException e) { - LOGGER.error(e.getMessage(), e); + LOGGER.warn(e.getMessage()); } } }; @@ -111,6 +111,8 @@ public class ArbitraryIndexUtils { indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1), indexResource.identifier )); } + } catch (MissingDataException e) { + LOGGER.warn( e.getMessage() ); } catch (InvalidFormatException e) { LOGGER.debug("invalid format, skipping: " + indexResource); } catch (UnrecognizedPropertyException e) { @@ -131,16 +133,12 @@ public class ArbitraryIndexUtils { ) ); - LOGGER.info("processed indices by term: count = " + indicesByTerm.size()); - // lock, clear old, load new synchronized( IndexCache.getInstance().getIndicesByTerm() ) { IndexCache.getInstance().getIndicesByTerm().clear(); IndexCache.getInstance().getIndicesByTerm().putAll(indicesByTerm); } - LOGGER.info("loaded indices by term"); - LOGGER.debug("processing indices by issuer ..."); Map> indicesByIssuer = indexDetails.stream().collect( @@ -154,15 +152,11 @@ public class ArbitraryIndexUtils { ) ); - LOGGER.info("processed indices by issuer: count = " + indicesByIssuer.size()); - // lock, clear old, load new synchronized( IndexCache.getInstance().getIndicesByIssuer() ) { IndexCache.getInstance().getIndicesByIssuer().clear(); IndexCache.getInstance().getIndicesByIssuer().putAll(indicesByIssuer); } - - LOGGER.info("loaded indices by issuer"); } } @@ -199,7 +193,7 @@ public class ArbitraryIndexUtils { } } - public static String getJson(String name, String identifier) throws IOException { + public static String getJson(String name, String identifier) throws IOException, MissingDataException { try { ArbitraryDataReader arbitraryDataReader @@ -217,11 +211,10 @@ public class ArbitraryIndexUtils { } catch (MissingDataException e) { if (attempts > maxAttempts) { // Give up after 5 attempts - throw new IOException("Data unavailable. Please try again later."); + throw e; } } } - Thread.sleep(3000L); } java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index c860a034..1d4cab18 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -48,6 +48,24 @@ public class ArbitraryTransactionUtils { } } + public static List fetchTransactionDataList(final Repository repository, final List signature) { + try { + List transactions = repository.getTransactionRepository().fromSignatures(signature); + + List list + = transactions.stream() + .filter( transaction -> transaction instanceof ArbitraryTransactionData ) + .map( transactionData -> (ArbitraryTransactionData) transactionData) + .collect(Collectors.toList()); + + return list; + + } catch (DataException e) { + LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + return null; + } + } + public static ArbitraryTransactionData fetchLatestPut(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { if (arbitraryTransactionData == null) { return null; diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index e9921561..5a0f676c 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -6,6 +6,7 @@ import org.qortal.settings.Settings; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.RandomAccessFile; import java.nio.charset.StandardCharsets; import java.nio.file.*; @@ -232,31 +233,37 @@ public class FilesystemUtils { } public static byte[] getSingleFileContents(Path path, Integer maxLength) throws IOException { - byte[] data = null; - // TODO: limit the file size that can be loaded into memory - - // If the path is a file, read the contents directly - if (path.toFile().isFile()) { - int fileSize = (int)path.toFile().length(); - maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize; - data = FilesystemUtils.readFromFile(path.toString(), 0, maxLength); - } - - // Or if it's a directory, only load file contents if there is a single file inside it - else if (path.toFile().isDirectory()) { + Path filePath = null; + + if (Files.isRegularFile(path)) { + filePath = path; + } else if (Files.isDirectory(path)) { String[] files = ArrayUtils.removeElement(path.toFile().list(), ".qortal"); if (files.length == 1) { - Path filePath = Paths.get(path.toString(), files[0]); - if (filePath.toFile().isFile()) { - int fileSize = (int)filePath.toFile().length(); - maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize; - data = FilesystemUtils.readFromFile(filePath.toString(), 0, maxLength); - } + filePath = path.resolve(files[0]); } } - - return data; + + if (filePath == null || !Files.exists(filePath)) { + return null; + } + + long fileSize = Files.size(filePath); + int length = (maxLength != null) ? Math.min(maxLength, (int) Math.min(fileSize, Integer.MAX_VALUE)) : (int) Math.min(fileSize, Integer.MAX_VALUE); + + try (InputStream in = Files.newInputStream(filePath)) { + byte[] buffer = new byte[length]; + int bytesRead = in.read(buffer); + if (bytesRead < length) { + // Resize buffer to actual read size + byte[] trimmed = new byte[bytesRead]; + System.arraycopy(buffer, 0, trimmed, 0, bytesRead); + return trimmed; + } + return buffer; + } } + /** * isSingleFileResource diff --git a/src/main/java/org/qortal/utils/ForeignFeesMessageUtils.java b/src/main/java/org/qortal/utils/ForeignFeesMessageUtils.java new file mode 100644 index 00000000..67e78439 --- /dev/null +++ b/src/main/java/org/qortal/utils/ForeignFeesMessageUtils.java @@ -0,0 +1,187 @@ +package org.qortal.utils; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.qortal.transform.Transformer.ADDRESS_LENGTH; + +/** + * Class ForeignFeesMessageUtils + */ +public class ForeignFeesMessageUtils { + + private static final Logger LOGGER = LogManager.getLogger(ForeignFeesMessageUtils.class); + + /** + * From Data To Send Bytes + * + * Convert foreign fee data into bytes for send messages. + * + * @param foreignFees the data + * + * @return the bytes + */ + public static byte[] fromDataToSendBytes(List foreignFees) { + + return fromDataToBytes(foreignFees, true); + } + + /** + * From Data To Bytes + * + * @param foreignFees + * @param includeSignature + * @return + */ + private static byte[] fromDataToBytes(List foreignFees, boolean includeSignature) { + try { + if (foreignFees.isEmpty()) { + return new byte[0]; + } + else { + // allocate size for each data item for timestamp, AT address, fee and signature + int byteSize + = foreignFees.size() + * + (Transformer.TIMESTAMP_LENGTH + Transformer.ADDRESS_LENGTH + Transformer.INT_LENGTH + Transformer.SIGNATURE_LENGTH); + + if( includeSignature ) byteSize += foreignFees.size() * Transformer.SIGNATURE_LENGTH; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + // for each foreign fee data item, convert to bytes and fill the array + for( ForeignFeeDecodedData feeData : foreignFees) { + bytes.write(Longs.toByteArray(feeData.getTimestamp())); + bytes.write(Base58.decode(feeData.getAtAddress())); + bytes.write(Ints.toByteArray(feeData.getFee())); + if( includeSignature ) bytes.write(feeData.getData()); + } + return bytes.toByteArray(); + } + } catch (Exception e) { + LOGGER.warn(e.getMessage()); + + return new byte[0]; + } + } + + /** + * From Send Bytes to Data + * + * @param bytes the bytes to convert to data + * + * @return the data + */ + public static List fromSendBytesToData(ByteBuffer bytes) { + + return fromBytesToData(bytes, true); + } + + /** + * From Bytes To Data + * + * @param bytes the bytes + * @param includeSignature true if the bytes include signatures + * + * @return the foreign fee data with signatures (data member) + */ + private static List fromBytesToData(ByteBuffer bytes, boolean includeSignature) { + if( !bytes.hasRemaining() ) return new ArrayList<>(0); + + List foreignFees = new ArrayList<>(); + + try { + while (bytes.hasRemaining()) { + // read in the timestamp as a long + long timestamp = bytes.getLong(); + + // read in the address as a byte array with a predetermined length + byte[] atAddressBytes = new byte[ADDRESS_LENGTH]; + bytes.get(atAddressBytes); + String atAddress = Base58.encode(atAddressBytes); + + // rwad in the fee as an integer + int fee = bytes.getInt(); + + byte[] signature; + + if( includeSignature ) { + signature = new byte[Transformer.SIGNATURE_LENGTH]; + bytes.get(signature); + } + else { + signature = null; + } + + foreignFees.add(new ForeignFeeDecodedData(timestamp, signature, atAddress, fee)); + + } + } + // if there are any exception, log the error as a warning and clear the list before returning it + catch (Exception e) { + LOGGER.warn(e.getMessage()); + foreignFees.clear(); + } + + return foreignFees; + } + + /** + * From Data To Get Bytes + * + * Convert foreign fees data objects into get foreign fees messages. Get messages do not include signatures. + * + * @param foreignFees the foreign fees objects + * + * @return the messages + */ + public static byte[] fromDataToGetBytes(List foreignFees) { + return fromDataToBytes(foreignFees, false); + } + + /** + * From Get Bytes to Data + * + * Convert bytes from get foreign fees messages to foreign fees objects. Get messages do not include signatures. + * + * @param bytes the bytes to convert + * + * @return the foreign fees data objects + */ + public static List fromGetBytesToData(ByteBuffer bytes) { + return fromBytesToData(bytes, false); + } + + /** + * Build Foreign Fees Data Message + * + * Build the unsigned message for the foreign fees data objects. + * + * @param timestamp the timestamp in milliseconds + * @param atAddress the AT address + * @param fee the fee + * @return + * @throws IOException + */ + public static byte[] buildForeignFeesDataMessage(Long timestamp, String atAddress, int fee) throws IOException { + int byteSize = Transformer.TIMESTAMP_LENGTH + Transformer.ADDRESS_LENGTH + Transformer.INT_LENGTH; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + bytes.write(Longs.toByteArray(timestamp)); + bytes.write(Base58.decode(atAddress)); + bytes.write(Ints.toByteArray(fee)); + + return bytes.toByteArray(); + } +} diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index c61723e7..e747e175 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -27,6 +27,8 @@ package org.qortal.utils; +import java.io.BufferedOutputStream; + import org.qortal.controller.Controller; import java.io.File; @@ -44,11 +46,15 @@ public class ZipUtils { File sourceFile = new File(sourcePath); boolean isSingleFile = Paths.get(sourcePath).toFile().isFile(); FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); - ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); + + // 🔧 Use best speed compression level + ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile); + zipOutputStream.close(); fileOutputStream.close(); } + public static void zip(final File fileToZip, final String enclosingFolderName, final ZipOutputStream zipOut, boolean isSingleFile) throws IOException, InterruptedException { if (Controller.isStopping()) { @@ -82,7 +88,7 @@ public class ZipUtils { final FileInputStream fis = new FileInputStream(fileToZip); final ZipEntry zipEntry = new ZipEntry(enclosingFolderName); zipOut.putNextEntry(zipEntry); - final byte[] bytes = new byte[1024]; + final byte[] bytes = new byte[65536]; int length; while ((length = fis.read(bytes)) >= 0) { zipOut.write(bytes, 0, length); @@ -92,33 +98,34 @@ public class ZipUtils { public static void unzip(String sourcePath, String destPath) throws IOException { final File destDir = new File(destPath); - final byte[] buffer = new byte[1024]; - final ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath)); - ZipEntry zipEntry = zis.getNextEntry(); - while (zipEntry != null) { - final File newFile = ZipUtils.newFile(destDir, zipEntry); - if (zipEntry.isDirectory()) { - if (!newFile.isDirectory() && !newFile.mkdirs()) { - throw new IOException("Failed to create directory " + newFile); + final byte[] buffer = new byte[65536]; + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath))) { + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + final File newFile = ZipUtils.newFile(destDir, zipEntry); + if (zipEntry.isDirectory()) { + if (!newFile.isDirectory() && !newFile.mkdirs()) { + throw new IOException("Failed to create directory " + newFile); + } + } else { + File parent = newFile.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("Failed to create directory " + parent); + } + + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(newFile), buffer.length)) { + int len; + while ((len = zis.read(buffer)) > 0) { + bos.write(buffer, 0, len); + } + } } - } else { - File parent = newFile.getParentFile(); - if (!parent.isDirectory() && !parent.mkdirs()) { - throw new IOException("Failed to create directory " + parent); - } - - final FileOutputStream fos = new FileOutputStream(newFile); - int len; - while ((len = zis.read(buffer)) > 0) { - fos.write(buffer, 0, len); - } - fos.close(); + zipEntry = zis.getNextEntry(); } - zipEntry = zis.getNextEntry(); + zis.closeEntry(); } - zis.closeEntry(); - zis.close(); } + /** * See: https://snyk.io/research/zip-slip-vulnerability diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 3264b670..2f347d18 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -119,7 +119,9 @@ "adminsReplaceFoundersHeight": 2012800, "nullGroupMembershipHeight": 2012800, "ignoreLevelForRewardShareHeight": 2012800, - "adminQueryFixHeight": 2012800 + "adminQueryFixHeight": 2012800, + "multipleNamesPerAccountHeight": 9999999, + "mintedBlocksAdjustmentRemovalHeight": 9999999 }, "checkpoints": [ { "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" } diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js index d7222750..486c6543 100644 --- a/src/main/resources/q-apps/q-apps.js +++ b/src/main/resources/q-apps/q-apps.js @@ -45,6 +45,7 @@ function parseUrl(url) { // Remove theme, identifier, and time queries if they exist parsedUrl.searchParams.delete("theme"); + parsedUrl.searchParams.delete("lang"); parsedUrl.searchParams.delete("identifier"); parsedUrl.searchParams.delete("time"); parsedUrl.searchParams.delete("isManualNavigation"); @@ -213,8 +214,11 @@ function buildResourceUrl(service, name, identifier, path, isLink) { if (path != null) url = url.concat((path.startsWith("/") ? "" : "/") + path); } - if (isLink) url = url.concat((url.includes("?") ? "" : "?") + "&theme=" + _qdnTheme); - + if (isLink) { + const hasQuery = url.includes("?"); + const queryPrefix = hasQuery ? "&" : "?"; + url += queryPrefix + "theme=" + _qdnTheme + "&lang=" + _qdnLang; + } return url; } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java index b4c10fac..cd7d3b7a 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java @@ -98,6 +98,45 @@ public class ArbitraryServiceTests extends Common { assertEquals(ValidationResult.MISSING_INDEX_FILE, service.validate(path)); } + @Test + public void testValidateWebsiteWithIndexFile() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateWebsiteWithoutIndexFile"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "index.html"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data1.html"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data2"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data3"), data, StandardOpenOption.CREATE); + + Service service = Service.WEBSITE; + assertTrue(service.isValidationRequired()); + + // There is no index file in the root + assertEquals(ValidationResult.OK, service.validate(path)); + } + + @Test + public void testValidateWebsiteWithIndexFileOnly() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateWebsiteWithoutIndexFile"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "index.html"), data, StandardOpenOption.CREATE); + + Service service = Service.WEBSITE; + assertTrue(service.isValidationRequired()); + + // There is no index file in the root + assertEquals(ValidationResult.OK, service.validate(path)); + } + @Test public void testValidateWebsiteWithoutIndexFileInRoot() throws IOException { // Generate some random data diff --git a/src/test/java/org/qortal/test/group/DevGroupAdminTests.java b/src/test/java/org/qortal/test/group/DevGroupAdminTests.java index 38ad0c53..925e2f3e 100644 --- a/src/test/java/org/qortal/test/group/DevGroupAdminTests.java +++ b/src/test/java/org/qortal/test/group/DevGroupAdminTests.java @@ -1,6 +1,7 @@ package org.qortal.test.group; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; @@ -369,6 +370,105 @@ public class DevGroupAdminTests extends Common { } } + @Test + public void testOrphanSecondInviteApproval() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + + Block block = BlockUtils.mintBlocks(repository, NULL_GROUP_MEMBERSHIP_HEIGHT); + assertEquals(NULL_GROUP_MEMBERSHIP_HEIGHT + 1, block.getBlockData().getHeight().intValue()); + + // establish accounts + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); + PrivateKeyAccount chloe = Common.getTestAccount(repository, CHLOE); + PrivateKeyAccount dilbert = Common.getTestAccount(repository, DILBERT); + + // assert admin statuses + assertEquals(2, repository.getGroupRepository().countGroupAdmins(DEV_GROUP_ID).intValue()); + assertTrue(isAdmin(repository, Group.NULL_OWNER_ADDRESS, DEV_GROUP_ID)); + assertTrue(isAdmin(repository, alice.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, chloe.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, dilbert.getAddress(), DEV_GROUP_ID)); + + // confirm Bob is not a member + assertFalse(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + + // alice invites bob, alice signs which is 50% approval while 40% is needed + TransactionData createInviteTransactionData = createGroupInviteForGroupApproval(repository, alice, DEV_GROUP_ID, bob.getAddress(), 3600); + Transaction.ApprovalStatus bobsInviteStatus = signForGroupApproval(repository, createInviteTransactionData, List.of(alice)); + + // assert approval + assertEquals(Transaction.ApprovalStatus.APPROVED, bobsInviteStatus); + + // bob joins + joinGroup(repository, bob, DEV_GROUP_ID); + + // confirm Bob is a member now, but still not an admin + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + + // bob creates transaction to add himself as an admin + TransactionData addGroupAdminTransactionData1 = addGroupAdmin(repository, bob, DEV_GROUP_ID, bob.getAddress()); + + // bob creates add admin transaction for himself, alice signs which is 50% approval while 40% is needed + signForGroupApproval(repository, addGroupAdminTransactionData1, List.of(alice)); + + // assert 3 admins in group and bob is an admin now + assertEquals(3, repository.getGroupRepository().countGroupAdmins(DEV_GROUP_ID).intValue()); + assertTrue(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + + // bob invites chloe, bob signs which is 33% approval while 40% is needed + TransactionData chloeInvite1 = createGroupInviteForGroupApproval(repository, bob, DEV_GROUP_ID, chloe.getAddress(), 3600); + Transaction.ApprovalStatus chloeInvite1Status = signForGroupApproval(repository, chloeInvite1, List.of(bob)); + + // assert invite 1 pending + assertEquals(Transaction.ApprovalStatus.PENDING, chloeInvite1Status); + + // bob invites chloe again, bob signs which is 33% approval while 40% is needed + // since chloe is not a member yet, this invite is valie + TransactionData chloeInvite2 = createGroupInviteForGroupApproval(repository, bob, DEV_GROUP_ID, chloe.getAddress(), 3600); + Transaction.ApprovalStatus chloeInvite2Status = signForGroupApproval(repository, chloeInvite2, List.of(bob)); + + // assert invite 2 is pending + assertEquals(Transaction.ApprovalStatus.PENDING, chloeInvite2Status); + + // alice signs which is 66% approval while 40% is needed + chloeInvite1Status = signForGroupApproval(repository, chloeInvite1, List.of(alice)); + + // assert invite 1 approval + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeInvite1Status); + + // chloe joins + joinGroup(repository, chloe, DEV_GROUP_ID); + + // assert chloe is in the group + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + + // alice signs invite 2 which is 66% approval while 40% is needed + chloeInvite2Status = signForGroupApproval(repository, chloeInvite2, List.of(alice)); + + // assert invite 2 approval + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeInvite2Status); + + boolean exceptionThrown = false; + + try { + // confront the bug by orphaning the block of the second approval, approval after the join + // prior to the fix, this would raise an exception + BlockUtils.orphanLastBlock(repository); + } catch (DataException e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + + // assert chloe is still a member + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + } + } + @Test public void testNullOwnershipMembership() throws DataException{ try (final Repository repository = RepositoryManager.getRepository()) { diff --git a/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java b/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java index 65a616b0..16ff354f 100644 --- a/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java +++ b/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java @@ -4,8 +4,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.block.BlockChain; import org.qortal.controller.BlockMinter; import org.qortal.controller.OnlineAccountsManager; +import org.qortal.data.account.AccountData; import org.qortal.data.account.RewardShareData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -15,8 +17,9 @@ import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TestAccount; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import java.util.List; + +import static org.junit.Assert.*; public class BlocksMintedCountTests extends Common { @@ -85,6 +88,121 @@ public class BlocksMintedCountTests extends Common { } } + @Test + public void testLevelSetting() { + + boolean exceptionThrown = false; + + try (final Repository repository = RepositoryManager.getRepository()) { + + // get the Alice's reward share account + PrivateKeyAccount aliceMintingAccount = Common.getTestAccount(repository, "alice-reward-share"); + + // give Alice an 8 blocks minted adjustment + int blocksMintedAdjustmentForAlice = 8; + adjustMintingData(repository, "alice", blocksMintedAdjustmentForAlice); + + // Confirm reward-share exists + RewardShareData aliceRewardShareData = repository.getAccountRepository().getRewardShare(aliceMintingAccount.getPublicKey()); + assertNotNull(aliceRewardShareData); + + // mint 40 blocks + for( int i = 0; i < 40; i++ ) { + // Create signed timestamps + OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(aliceMintingAccount); + + // Mint another block + BlockMinter.mintTestingBlockRetainingTimestamps(repository, aliceMintingAccount); + + // assert Alice's minting data after another block minted + assertMintingData(repository, "alice", blocksMintedAdjustmentForAlice); + + // orphan the block + BlockUtils.orphanLastBlock(repository); + + // assert the orphaning + assertMintingData(repository, "alice", blocksMintedAdjustmentForAlice); + + // mint another block to reverse the orpaning + BlockMinter.mintTestingBlockRetainingTimestamps(repository, aliceMintingAccount); + } + } + catch (DataException e) { + exceptionThrown = true; + } + + assertFalse(exceptionThrown); + } + + /** + * Assert Minting Data + * + * @param repository the data repository + * @param name the name of the minting account + * @param adjustment the blocks minted adjustment + * + * @throws DataException + */ + private static void assertMintingData(Repository repository, String name, int adjustment ) throws DataException { + + // get the test account data + TestAccount testAccount = Common.getTestAccount(repository, name); + AccountData testAccountData = repository.getAccountRepository().getAccount(testAccount.getAddress()); + + List blocksNeededByLevel = BlockChain.getInstance().getBlocksNeededByLevel(); + + // determine current height and adjustment ability + int height = repository.getBlockRepository().getBlockchainHeight(); + int adjustmentRemovalHeight = BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight(); + boolean isAdjustingEnabled = height <= adjustmentRemovalHeight; + + // initialize loop variables + int blocksLeft; + + if( isAdjustingEnabled ) + blocksLeft = testAccountData.getBlocksMinted() + adjustment; + else + blocksLeft = testAccountData.getBlocksMinted(); + + int index = 0; + int expectedLevel = 0; + + // update expected level based on the blocks needed by level list entries + while( blocksNeededByLevel.size() > index ) { + + Integer blocksNeededByThisLevel = blocksNeededByLevel.get(index); + if( blocksNeededByThisLevel <= blocksLeft ) { + expectedLevel++; + blocksLeft -= blocksNeededByThisLevel; + } + else { + break; + } + + index++; + } + + // print and assert the expected and derived numbers + System.out.println(String.format("height = %s,expectedLevel = %s, adjustment = %s, blocksMinted = %s", height, expectedLevel, adjustment, testAccountData.getBlocksMinted()) ); + assertEquals( expectedLevel, testAccountData.getLevel() ); + } + + /** + * Adjust Minting Data + * + * @param repository the data repository + * @param name the name of the account to adjust + * @param blocksMintedAdjustment the number of blocks to adjust + * + * @throws DataException + */ + private static void adjustMintingData(Repository repository, String name, int blocksMintedAdjustment) throws DataException { + TestAccount testAccount = Common.getTestAccount(repository, name); + AccountData testAccountData = repository.getAccountRepository().getAccount(testAccount.getAddress()); + testAccountData.setBlocksMintedAdjustment(blocksMintedAdjustment); + repository.getAccountRepository().setBlocksMintedAdjustment(testAccountData); + } + private void testRewardShare(Repository repository, PrivateKeyAccount testRewardShareAccount, int aliceDelta, int bobDelta) throws DataException { // Create signed timestamps OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(testRewardShareAccount); @@ -124,5 +242,4 @@ public class BlocksMintedCountTests extends Common { TestAccount testAccount = Common.getTestAccount(repository, name); return repository.getAccountRepository().getAccount(testAccount.getAddress()).getBlocksMinted(); } - } diff --git a/src/test/java/org/qortal/test/naming/BuySellTests.java b/src/test/java/org/qortal/test/naming/BuySellTests.java index a1c644fc..6f807265 100644 --- a/src/test/java/org/qortal/test/naming/BuySellTests.java +++ b/src/test/java/org/qortal/test/naming/BuySellTests.java @@ -4,6 +4,8 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.block.BlockChain; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.BuyNameTransactionData; import org.qortal.data.transaction.CancelSellNameTransactionData; @@ -17,8 +19,11 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.transaction.Transaction; import org.qortal.utils.Amounts; +import java.util.List; +import java.util.Optional; import java.util.Random; import static org.junit.Assert.*; @@ -84,15 +89,79 @@ public class BuySellTests extends Common { assertTrue(repository.getNameRepository().nameExists(name)); } + @Test + public void testRegisterNameMultiple() throws DataException { + // register name 1 + RegisterNameTransactionData transactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData1, alice); + + String name1 = transactionData1.getName(); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name1)); + + // register another name, second registered name should fail before the feature trigger + final String name2 = "another name"; + RegisterNameTransactionData transactionData2 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult resultBeforeFeatureTrigger = TransactionUtils.signAndImport(repository, transactionData2, alice); + + // check that that multiple names is forbidden + assertTrue(Transaction.ValidationResult.MULTIPLE_NAMES_FORBIDDEN.equals(resultBeforeFeatureTrigger)); + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // register again, now that we are passed the feature trigger + RegisterNameTransactionData transactionData3 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult resultAfterFeatureTrigger = TransactionUtils.signAndImport(repository, transactionData3, alice); + + // check that multiple names is ok + assertTrue(Transaction.ValidationResult.OK.equals(resultAfterFeatureTrigger)); + + // mint block, confirm transaction + BlockUtils.mintBlock(repository); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name2)); + + // check that there are 2 names for one account + List namesByOwner = repository.getNameRepository().getNamesByOwner(alice.getAddress(), 0, 0, false); + + assertEquals(2, namesByOwner.size() ); + + // check that the order is correct + assertEquals(name1, namesByOwner.get(0).getName()); + + SellNameTransactionData sellPrimaryNameData = new SellNameTransactionData(TestTransaction.generateBase(alice), name, price); + Transaction.ValidationResult sellPrimaryNameResult = TransactionUtils.signAndImport(repository, sellPrimaryNameData, alice); + + // check that selling primary name is not supported while owning multiple addresses + assertTrue(Transaction.ValidationResult.NOT_SUPPORTED.equals(sellPrimaryNameResult)); + } + @Test public void testSellName() throws DataException { + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Register-name testRegisterName(); + // assert primary name for alice + Optional alicePrimaryName1 = alice.getPrimaryName(); + assertTrue(alicePrimaryName1.isPresent()); + assertTrue(alicePrimaryName1.get().equals(name)); + // Sell-name SellNameTransactionData transactionData = new SellNameTransactionData(TestTransaction.generateBase(alice), name, price); TransactionUtils.signAndMint(repository, transactionData, alice); + // assert primary name for alice + Optional alicePrimaryName2 = alice.getPrimaryName(); + assertTrue(alicePrimaryName2.isPresent()); + assertTrue(alicePrimaryName2.get().equals(name)); + NameData nameData; // Check name is for sale @@ -100,6 +169,14 @@ public class BuySellTests extends Common { assertTrue(nameData.isForSale()); assertEquals("price incorrect", price, nameData.getSalePrice()); + // assert alice cannot register another name while primary name is for sale + final String name2 = "another name"; + RegisterNameTransactionData registerSecondNameData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult registrationResult = TransactionUtils.signAndImport(repository, registerSecondNameData, alice); + + // check that registering is not supported while primary name is for sale + assertTrue(Transaction.ValidationResult.NOT_SUPPORTED.equals(registrationResult)); + // Orphan sell-name BlockUtils.orphanLastBlock(repository); @@ -119,6 +196,10 @@ public class BuySellTests extends Common { // Orphan sell-name and register-name BlockUtils.orphanBlocks(repository, 2); + // assert primary name for alice + Optional alicePrimaryName3 = alice.getPrimaryName(); + assertTrue(alicePrimaryName3.isEmpty()); + // Check name no longer exists assertFalse(repository.getNameRepository().nameExists(name)); nameData = repository.getNameRepository().fromName(name); @@ -212,15 +293,36 @@ public class BuySellTests extends Common { @Test public void testBuyName() throws DataException { + // move passed primary initiation + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Register-name and sell-name testSellName(); String seller = alice.getAddress(); + // assert alice has the name as primary + Optional alicePrimaryName1 = alice.getPrimaryName(); + assertTrue(alicePrimaryName1.isPresent()); + assertEquals(name, alicePrimaryName1.get()); + + // assert bob does not have a primary name + Optional bobPrimaryName1 = bob.getPrimaryName(); + assertTrue(bobPrimaryName1.isEmpty()); + // Buy-name BuyNameTransactionData transactionData = new BuyNameTransactionData(TestTransaction.generateBase(bob), name, price, seller); TransactionUtils.signAndMint(repository, transactionData, bob); + // assert alice does not have a primary name anymore + Optional alicePrimaryName2 = alice.getPrimaryName(); + assertTrue(alicePrimaryName2.isEmpty()); + + // assert bob does have the name as primary + Optional bobPrimaryName2 = bob.getPrimaryName(); + assertTrue(bobPrimaryName2.isPresent()); + assertEquals(name, bobPrimaryName2.get()); + NameData nameData; // Check name is sold @@ -231,6 +333,15 @@ public class BuySellTests extends Common { // Orphan buy-name BlockUtils.orphanLastBlock(repository); + // assert alice has the name as primary + Optional alicePrimaryNameOrphaned = alice.getPrimaryName(); + assertTrue(alicePrimaryNameOrphaned.isPresent()); + assertEquals(name, alicePrimaryNameOrphaned.get()); + + // assert bob does not have a primary name + Optional bobPrimaryNameOrphaned = bob.getPrimaryName(); + assertTrue(bobPrimaryNameOrphaned.isEmpty()); + // Check name is for sale (not sold) nameData = repository.getNameRepository().fromName(name); assertTrue(nameData.isForSale()); @@ -265,6 +376,9 @@ public class BuySellTests extends Common { assertFalse(nameData.isForSale()); // Not concerned about price assertEquals(bob.getAddress(), nameData.getOwner()); + + assertEquals(alice.getPrimaryName(), alice.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); + assertEquals(bob.getPrimaryName(), bob.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); } @Test @@ -324,6 +438,65 @@ public class BuySellTests extends Common { assertTrue(nameData.isForSale()); assertEquals("price incorrect", newPrice, nameData.getSalePrice()); assertEquals(bob.getAddress(), nameData.getOwner()); + + assertEquals(alice.getPrimaryName(), alice.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); + assertEquals(bob.getPrimaryName(), bob.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); } + @Test + public void testBuyInvalidationDuringPrimaryNameSale() throws DataException { + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // Register-name + testRegisterName(); + + // assert primary name for alice + Optional alicePrimaryName1 = alice.getPrimaryName(); + assertTrue(alicePrimaryName1.isPresent()); + assertTrue(alicePrimaryName1.get().equals(name)); + + // Sell-name + SellNameTransactionData transactionData = new SellNameTransactionData(TestTransaction.generateBase(alice), name, price); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // assert primary name for alice + Optional alicePrimaryName2 = alice.getPrimaryName(); + assertTrue(alicePrimaryName2.isPresent()); + assertTrue(alicePrimaryName2.get().equals(name)); + + NameData nameData; + + // Check name is for sale + nameData = repository.getNameRepository().fromName(name); + assertTrue(nameData.isForSale()); + assertEquals("price incorrect", price, nameData.getSalePrice()); + + // assert alice cannot register another name while primary name is for sale + final String name2 = "another name"; + RegisterNameTransactionData registerSecondNameData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult registrationResult = TransactionUtils.signAndImport(repository, registerSecondNameData, alice); + + // check that registering is not supported while primary name is for sale + assertTrue(Transaction.ValidationResult.NOT_SUPPORTED.equals(registrationResult)); + + String bobName = "bob"; + RegisterNameTransactionData bobRegisterData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(bobRegisterData.getTimestamp())); + TransactionUtils.signAndMint(repository, bobRegisterData, bob); + + Optional bobPrimaryName = bob.getPrimaryName(); + + assertTrue(bobPrimaryName.isPresent()); + assertEquals(bobName, bobPrimaryName.get()); + + SellNameTransactionData bobSellData = new SellNameTransactionData(TestTransaction.generateBase(bob), bobName, price); + TransactionUtils.signAndMint(repository, bobSellData, bob); + + BuyNameTransactionData aliceBuyData = new BuyNameTransactionData(TestTransaction.generateBase(alice), bobName, price, bob.getAddress()); + Transaction.ValidationResult aliceBuyResult = TransactionUtils.signAndImport(repository, aliceBuyData, alice); + + // check that buying is not supported while primary name is for sale + assertTrue(Transaction.ValidationResult.NOT_SUPPORTED.equals(aliceBuyResult)); + } } diff --git a/src/test/java/org/qortal/test/naming/IntegrityTests.java b/src/test/java/org/qortal/test/naming/IntegrityTests.java index 767ea388..14a6891b 100644 --- a/src/test/java/org/qortal/test/naming/IntegrityTests.java +++ b/src/test/java/org/qortal/test/naming/IntegrityTests.java @@ -4,6 +4,8 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.block.BlockChain; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.*; @@ -13,6 +15,7 @@ import org.qortal.repository.RepositoryFactory; import org.qortal.repository.RepositoryManager; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; +import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; @@ -385,6 +388,8 @@ public class IntegrityTests extends Common { @Test public void testUpdateToMissingName() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String initialName = "test-name"; @@ -422,7 +427,12 @@ public class IntegrityTests extends Common { // Therefore the name that we are trying to rename TO already exists Transaction.ValidationResult result = transaction.importAsUnconfirmed(); assertTrue("Transaction should be invalid", Transaction.ValidationResult.OK != result); - assertTrue("Destination name should already exist", Transaction.ValidationResult.NAME_ALREADY_REGISTERED == result); + + // this assertion has been updated, because the primary name logic now comes into play and you cannot update a primary name when there + // is other names registered and if your try a NOT SUPPORTED result will be given + assertTrue("Destination name should already exist", Transaction.ValidationResult.NOT_SUPPORTED == result); + + assertEquals(alice.getPrimaryName(), alice.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); } } diff --git a/src/test/java/org/qortal/test/naming/MiscTests.java b/src/test/java/org/qortal/test/naming/MiscTests.java index 324cdf12..b85ca95c 100644 --- a/src/test/java/org/qortal/test/naming/MiscTests.java +++ b/src/test/java/org/qortal/test/naming/MiscTests.java @@ -1,6 +1,7 @@ package org.qortal.test.naming; import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; @@ -8,6 +9,7 @@ import org.qortal.api.AmountTypeAdapter; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.UnitFeesByTimestamp; import org.qortal.controller.BlockMinter; +import org.qortal.data.naming.NameData; import org.qortal.data.transaction.PaymentTransactionData; import org.qortal.data.transaction.RegisterNameTransactionData; import org.qortal.data.transaction.TransactionData; @@ -28,6 +30,7 @@ import org.qortal.utils.NTP; import java.util.Arrays; import java.util.List; +import java.util.Optional; import static org.junit.Assert.*; @@ -121,6 +124,8 @@ public class MiscTests extends Common { transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); TransactionUtils.signAndMint(repository, transactionData, alice); + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Register another name that we will later attempt to rename to first name (above) String otherName = "new-name"; String otherData = ""; @@ -335,6 +340,8 @@ public class MiscTests extends Common { public void testRegisterNameFeeIncrease() throws Exception { try (final Repository repository = RepositoryManager.getRepository()) { + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Add original fee to nameRegistrationUnitFees UnitFeesByTimestamp originalFee = new UnitFeesByTimestamp(); originalFee.timestamp = 0L; @@ -517,4 +524,168 @@ public class MiscTests extends Common { } } -} + @Test + public void testPrimaryNameEmpty() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + Optional primaryName = repository.getNameRepository().getPrimaryName(alice.getAddress()); + + Assert.assertNotNull(primaryName); + Assert.assertTrue(primaryName.isEmpty()); + } + } + + @Test + public void testPrimaryNameSingle() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + String name = "alice 1"; + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + + // register name 1 + RegisterNameTransactionData transactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData1, alice); + + String name1 = transactionData1.getName(); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name1)); + + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight() + 1); + + Optional primaryName = repository.getNameRepository().getPrimaryName(alice.getAddress()); + + Assert.assertNotNull(primaryName); + Assert.assertTrue(primaryName.isPresent()); + Assert.assertEquals(name, primaryName.get()); + } + } + + @Test + public void testPrimaryNameSingleAfterFeature() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + String name = "alice 1"; + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // register name 1 + RegisterNameTransactionData transactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData1, alice); + + String name1 = transactionData1.getName(); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name1)); + + + Optional primaryName = repository.getNameRepository().getPrimaryName(alice.getAddress()); + + Assert.assertNotNull(primaryName); + Assert.assertTrue(primaryName.isPresent()); + Assert.assertEquals(name, primaryName.get()); + + BlockUtils.orphanLastBlock(repository); + + Optional primaryNameOrpaned = repository.getNameRepository().getPrimaryName(alice.getAddress()); + + Assert.assertNotNull(primaryNameOrpaned); + Assert.assertTrue(primaryNameOrpaned.isEmpty()); + } + } + + @Test + public void testUpdateNameMultiple() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + String name = "alice 1"; + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + + // register name 1 + RegisterNameTransactionData transactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData1, alice); + + String name1 = transactionData1.getName(); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name1)); + + // register another name, second registered name should fail before the feature trigger + final String name2 = "another name"; + RegisterNameTransactionData transactionData2 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult resultBeforeFeatureTrigger = TransactionUtils.signAndImport(repository, transactionData2, alice); + + // check that that multiple names is forbidden + assertTrue(Transaction.ValidationResult.MULTIPLE_NAMES_FORBIDDEN.equals(resultBeforeFeatureTrigger)); + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // register again, now that we are passed the feature trigger + RegisterNameTransactionData transactionData3 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult resultAfterFeatureTrigger = TransactionUtils.signAndImport(repository, transactionData3, alice); + + // check that multiple names is ok + assertTrue(Transaction.ValidationResult.OK.equals(resultAfterFeatureTrigger)); + + // mint block, confirm transaction + BlockUtils.mintBlock(repository); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name2)); + + // check that there are 2 names for one account + List namesByOwner = repository.getNameRepository().getNamesByOwner(alice.getAddress(), 0, 0, false); + + assertEquals(2, namesByOwner.size()); + + // check that the order is correct + assertEquals(name1, namesByOwner.get(0).getName()); + + String newestName = "newest-name"; + String newestReducedName = "newest-name"; + String newestData = "newest-data"; + TransactionData newestTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name2, newestName, newestData); + TransactionUtils.signAndMint(repository, newestTransactionData, alice); + + // Check previous name no longer exists + assertFalse(repository.getNameRepository().nameExists(name2)); + + // Check newest name exists + assertTrue(repository.getNameRepository().nameExists(newestName)); + + Optional alicePrimaryName1 = alice.getPrimaryName(); + + assertTrue( alicePrimaryName1.isPresent() ); + assertEquals( name1, alicePrimaryName1.get() ); + + // orphan and recheck + BlockUtils.orphanLastBlock(repository); + + Optional alicePrimaryName2 = alice.getPrimaryName(); + + assertTrue( alicePrimaryName2.isPresent() ); + assertEquals( name1, alicePrimaryName2.get() ); + + // Check newest name no longer exists + assertFalse(repository.getNameRepository().nameExists(newestName)); + assertNull(repository.getNameRepository().fromReducedName(newestReducedName)); + + // Check previous name exists again + assertTrue(repository.getNameRepository().nameExists(name2)); + } + } +} \ No newline at end of file diff --git a/src/test/java/org/qortal/test/naming/UpdateTests.java b/src/test/java/org/qortal/test/naming/UpdateTests.java index 8e54eb96..01b9a2ee 100644 --- a/src/test/java/org/qortal/test/naming/UpdateTests.java +++ b/src/test/java/org/qortal/test/naming/UpdateTests.java @@ -3,8 +3,12 @@ package org.qortal.test.naming; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.block.BlockChain; import org.qortal.data.naming.NameData; +import org.qortal.data.transaction.BuyNameTransactionData; import org.qortal.data.transaction.RegisterNameTransactionData; +import org.qortal.data.transaction.SellNameTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.UpdateNameTransactionData; import org.qortal.repository.DataException; @@ -15,6 +19,9 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.transaction.Transaction; + +import java.util.Optional; import static org.junit.Assert.*; @@ -395,6 +402,13 @@ public class UpdateTests extends Common { assertTrue(repository.getNameRepository().nameExists(initialName)); assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + // move passed primary initiation + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // check primary name + assertTrue(alice.getPrimaryName().isPresent()); + assertEquals(initialName, alice.getPrimaryName().get()); + // Update data String middleName = "middle-name"; String middleReducedName = "midd1e-name"; @@ -402,6 +416,11 @@ public class UpdateTests extends Common { transactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, middleName, middleData); TransactionUtils.signAndMint(repository, transactionData, alice); + // check primary name + Optional alicePrimaryName1 = alice.getPrimaryName(); + assertTrue(alicePrimaryName1.isPresent()); + assertEquals(middleName, alicePrimaryName1.get()); + // Check data is correct assertEquals(middleData, repository.getNameRepository().fromName(middleName).getData()); @@ -414,6 +433,11 @@ public class UpdateTests extends Common { // Check data is correct assertEquals(newestData, repository.getNameRepository().fromName(newestName).getData()); + // check primary name + Optional alicePrimaryName2 = alice.getPrimaryName(); + assertTrue(alicePrimaryName2.isPresent()); + assertEquals(newestName, alicePrimaryName2.get()); + // Check initial name no longer exists assertFalse(repository.getNameRepository().nameExists(initialName)); assertNull(repository.getNameRepository().fromReducedName(initialReducedName)); @@ -516,4 +540,101 @@ public class UpdateTests extends Common { } } + @Test + public void testUpdatePrimaryName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + + // register name 1 + String initialName = "initial-name"; + RegisterNameTransactionData registerNameTransactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, "{}"); + registerNameTransactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(registerNameTransactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, registerNameTransactionData1, alice); + + // assert name 1 registration, assert primary name + assertTrue(repository.getNameRepository().nameExists(initialName)); + + Optional primaryNameOptional = alice.getPrimaryName(); + assertTrue(primaryNameOptional.isPresent()); + assertEquals(initialName, primaryNameOptional.get()); + + // register name 2 + String secondName = "second-name"; + RegisterNameTransactionData registerNameTransactionData2 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), secondName, "{}"); + registerNameTransactionData2.setFee(new RegisterNameTransaction(null, null).getUnitFee(registerNameTransactionData2.getTimestamp())); + TransactionUtils.signAndMint(repository, registerNameTransactionData2, alice); + + // assert name 2 registration, assert primary has not changed + assertTrue(repository.getNameRepository().nameExists(secondName)); + + // the name alice is trying to update to + String newName = "updated-name"; + + // update name, assert invalid + updateName(repository, initialName, newName, Transaction.ValidationResult.NOT_SUPPORTED, alice); + + // check primary name did not update + // check primary name update + Optional primaryNameNotUpdateOptional = alice.getPrimaryName(); + assertTrue(primaryNameNotUpdateOptional.isPresent()); + assertEquals(initialName, primaryNameNotUpdateOptional.get()); + + // sell name 2, assert valid + Long amount = 1000000L; + SellNameTransactionData transactionData = new SellNameTransactionData(TestTransaction.generateBase(alice), secondName, amount); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Check name is for sale + NameData nameData = repository.getNameRepository().fromName(secondName); + assertTrue(nameData.isForSale()); + assertEquals("price incorrect", amount, nameData.getSalePrice()); + + // bob buys name 2, assert + BuyNameTransactionData bobBuysName2Data = new BuyNameTransactionData(TestTransaction.generateBase(bob), secondName, amount, alice.getAddress()); + TransactionUtils.signAndMint(repository, bobBuysName2Data, bob); + + // update name, assert valid, assert primary name change + updateName(repository, initialName, newName, Transaction.ValidationResult.OK, alice); + + // check primary name update + Optional primaryNameUpdateOptional = alice.getPrimaryName(); + assertTrue(primaryNameUpdateOptional.isPresent()); + assertEquals(newName, primaryNameUpdateOptional.get()); + + assertEquals(alice.getPrimaryName(), alice.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); + assertEquals(bob.getPrimaryName(), bob.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); + } + } + + /** + * Update Name + * + * @param repository + * @param initialName the name before the update + * @param newName the name after the update + * @param expectedValidationResult the validation result expected from the update + * @param account the account for the update + * + * @throws DataException + */ + private static void updateName(Repository repository, String initialName, String newName, Transaction.ValidationResult expectedValidationResult, PrivateKeyAccount account) throws DataException { + TransactionData data = new UpdateNameTransactionData(TestTransaction.generateBase(account), initialName, newName, "{}"); + Transaction.ValidationResult result = TransactionUtils.signAndImport(repository,data, account); + + assertEquals("Transaction invalid", expectedValidationResult, result); + + BlockUtils.mintBlock(repository); + + if( Transaction.ValidationResult.OK.equals(expectedValidationResult) ) { + assertTrue(repository.getNameRepository().nameExists(newName)); + } + else { + // the new name should not exist, because the update was invalid + assertFalse(repository.getNameRepository().nameExists(newName)); + } + } } diff --git a/src/test/java/org/qortal/test/network/message/ForeignFeesMessageTests.java b/src/test/java/org/qortal/test/network/message/ForeignFeesMessageTests.java new file mode 100644 index 00000000..9a2b0e7f --- /dev/null +++ b/src/test/java/org/qortal/test/network/message/ForeignFeesMessageTests.java @@ -0,0 +1,334 @@ +package org.qortal.test.network.message; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.junit.Assert; +import org.junit.Test; +import org.qortal.crypto.Crypto; +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.test.utils.TestUtils; +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; +import org.qortal.utils.ForeignFeesMessageUtils; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.security.Security; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +/** + * Class ForeignFeesMessageTests + */ +public class ForeignFeesMessageTests { + + /** + * Random + * + * Random input generator for seeding keys/addresses. + */ + private static final Random RANDOM = new Random(); + + static { + // add the Bouncy Castle provider for keys/addresses + Security.addProvider(new BouncyCastleProvider()); + } + + @Test + public void testDataToSendBytesToDataEmpty() { + + byte[] bytes = ForeignFeesMessageUtils.fromDataToSendBytes(new ArrayList<>(0)); + + List list = ForeignFeesMessageUtils.fromSendBytesToData(ByteBuffer.wrap(bytes)); + + Assert.assertNotNull(list); + + Assert.assertEquals(0, list.size()); + } + + @Test + public void testDataToGetBytesToDataEmpty() { + + byte[] bytes = ForeignFeesMessageUtils.fromDataToGetBytes(new ArrayList<>(0)); + + List list = ForeignFeesMessageUtils.fromGetBytesToData(ByteBuffer.wrap(bytes)); + + Assert.assertNotNull(list); + + Assert.assertEquals(0, list.size()); + } + + @Test + public void testSignature() { + + boolean exceptionThrown = false; + + try { + KeyPair keyPair = TestUtils.generateKeyPair(); + + long timestamp = 1_000_000L; + String atAddress = generateAtAddress(); + int fee = 1; + + assertSignature(keyPair, timestamp, atAddress, fee); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + /** + * Assert Signature + * + * @param keyPair the key pair that is signing + * @param timestamp the timestamp for the data + * @param atAddress the AT address + * @param fee the fee + * + * @return the signature bytes + * + * @throws IOException + */ + private static byte[] assertSignature(KeyPair keyPair, long timestamp, String atAddress, int fee) throws IOException { + + // build the message and sign it + byte[] message = ForeignFeesMessageUtils.buildForeignFeesDataMessage(timestamp, atAddress, fee); + byte[] signature = Crypto.sign( keyPair.getPrivate().getEncoded(), message ); + + // assert signaute length + Assert.assertEquals(Transformer.SIGNATURE_LENGTH, signature.length); + + // assert verification + boolean verified = Crypto.verify(Crypto.toPublicKey(keyPair.getPrivate().getEncoded()), signature, message); + Assert.assertTrue(verified); + + return signature; + } + + @Test + public void testDataToSendBytesToDataSingle() { + + Long timestamp = 1_000_000L; + String atAddress = generateAtAddress(); + int fee = 1; + + boolean exceptionThrown = false; + + try { + // random key generation for signing data + KeyPair keyPair = TestUtils.generateKeyPair(); + + // data to send, a list of 1 foreign fee data + List sendData + = List.of( + new ForeignFeeDecodedData(timestamp, assertSignature(keyPair,timestamp,atAddress, fee), atAddress, fee) + ); + + // from data to bytes + byte[] sendBytes = ForeignFeesMessageUtils.fromDataToSendBytes(sendData); + + // from bytes to data + List returnData = ForeignFeesMessageUtils.fromSendBytesToData(ByteBuffer.wrap(sendBytes)); + + + assertListedForeignFees(sendData, returnData, true); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testDataToGetBytesToDataSingle() { + + Long timestamp = 1_000_000L; + String atAddress = generateAtAddress(); + int fee = 1; + + boolean exceptionThrown = false; + + try { + // random key generation for signing data + KeyPair keyPair = TestUtils.generateKeyPair(); + + // data to send, a list of 1 foreign fee data + List sendData + = List.of( + new ForeignFeeDecodedData(timestamp, assertSignature(keyPair,timestamp,atAddress, fee), atAddress, fee) + ); + + // from data to bytes + byte[] sendBytes = ForeignFeesMessageUtils.fromDataToGetBytes(sendData); + + // from bytes to data + List returnData = ForeignFeesMessageUtils.fromGetBytesToData(ByteBuffer.wrap(sendBytes)); + + + assertListedForeignFees(sendData, returnData, false); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testDataToSendBytesToDataTriple() { + + Long timestamp1 = 1_000_000L; + String atAddress1 = generateAtAddress(); + int fee1 = 1; + + Long timestamp2 = 2_000_000L; + String atAddress2 = generateAtAddress(); + int fee2 = 2; + + Long timestamp3 = 5_000_000L; + String atAddress3 = generateAtAddress(); + int fee3 = 3; + + boolean exceptionThrown = false; + + try { + // random key generation for signing data + KeyPair keyPair1 = TestUtils.generateKeyPair(); + KeyPair keyPair2 = TestUtils.generateKeyPair(); + + // data to send, a list of 3 foreign fee data + List sendData + = List.of( + new ForeignFeeDecodedData(timestamp1, assertSignature(keyPair1,timestamp1,atAddress1, fee1), atAddress1, fee1), + new ForeignFeeDecodedData(timestamp2, assertSignature(keyPair1,timestamp2,atAddress2, fee2), atAddress2, fee2), + new ForeignFeeDecodedData(timestamp3, assertSignature(keyPair2,timestamp3,atAddress3, fee3), atAddress3, fee3) + ); + + // from data to bytes + byte[] sendBytes = ForeignFeesMessageUtils.fromDataToSendBytes(sendData); + + // from bytes to data + List returnData = ForeignFeesMessageUtils.fromSendBytesToData(ByteBuffer.wrap(sendBytes)); + + + assertListedForeignFees(sendData, returnData, true); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testDataToGetBytesToDataTriple() { + + Long timestamp1 = 1_000_000L; + String atAddress1 = generateAtAddress(); + int fee1 = 1; + + Long timestamp2 = 2_000_000L; + String atAddress2 = generateAtAddress(); + int fee2 = 2; + + Long timestamp3 = 5_000_000L; + String atAddress3 = generateAtAddress(); + int fee3 = 3; + + boolean exceptionThrown = false; + + try { + // random key generation for signing data + KeyPair keyPair1 = TestUtils.generateKeyPair(); + KeyPair keyPair2 = TestUtils.generateKeyPair(); + + // data to send, a list of 3 foreign fee data + List sendData + = List.of( + new ForeignFeeDecodedData(timestamp1, assertSignature(keyPair1,timestamp1,atAddress1, fee1), atAddress1, fee1), + new ForeignFeeDecodedData(timestamp2, assertSignature(keyPair1,timestamp2,atAddress2, fee2), atAddress2, fee2), + new ForeignFeeDecodedData(timestamp3, assertSignature(keyPair2,timestamp3,atAddress3, fee3), atAddress3, fee3) + ); + + // from data to bytes + byte[] sendBytes = ForeignFeesMessageUtils.fromDataToGetBytes(sendData); + + // from bytes to data + List returnData = ForeignFeesMessageUtils.fromGetBytesToData(ByteBuffer.wrap(sendBytes)); + + + assertListedForeignFees(sendData, returnData, false); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + /** + * Assert Listed Foreign Fees + * + * @param expectedList + * @param actualList + * @param includeSignature + */ + private static void assertListedForeignFees(List expectedList, List actualList, boolean includeSignature) { + + int expectedSize = expectedList.size(); + + // basic assertions on return data + Assert.assertNotNull(actualList); + Assert.assertEquals(expectedSize, actualList.size()); + + for( int index = 0; index < expectedSize; index++ ) { + // expected and actual fee data + ForeignFeeDecodedData expected = expectedList.get(index); + ForeignFeeDecodedData actual = actualList.get(index); + + assertForeignFeeEquality(expected, actual, includeSignature); + } + } + + /** + * Assert Foreign Fee Equality + * + * @param expected the expected data, for comparison + * @param actual the actual data, the response to evaluate + * @param includeSignature + */ + private static void assertForeignFeeEquality(ForeignFeeDecodedData expected, ForeignFeeDecodedData actual, boolean includeSignature) { + // assert + Assert.assertEquals(expected, actual); + + if( includeSignature ) { + // get the data members of each, since the data members are not part of the object comparison above + byte[] expectedData = expected.getData(); + byte[] actualData = actual.getData(); + + // assert data members, must encode them to strings for comparisons + Assert.assertNotNull(actualData); + Assert.assertEquals(Base58.encode(expectedData), Base58.encode(actualData)); + } + } + + /** + * Generate AT Address + * + * Generate AT address using a random inpute seed. + * + * @return the AT address + */ + private static String generateAtAddress() { + + byte[] signature = new byte[64]; + RANDOM.nextBytes(signature); + String atAddress = Crypto.toATAddress(signature); + + return atAddress; + } +} \ No newline at end of file diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 5395116f..d7f7ea13 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -114,7 +114,9 @@ "adminsReplaceFoundersHeight": 9999999999999, "ignoreLevelForRewardShareHeight": 9999999999999, "nullGroupMembershipHeight": 20, - "adminQueryFixHeight": 9999999999999 + "adminQueryFixHeight": 9999999999999, + "multipleNamesPerAccountHeight": 10, + "mintedBlocksAdjustmentRemovalHeight": 27 }, "genesisInfo": { "version": 4,