From 144d6cc5c747194387d5e76228fedb6e08d44b7c Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 25 Apr 2025 17:51:01 -0700 Subject: [PATCH 01/42] foreign fees manager implementation, feeCeiling -> feeRequired name change, thread-safety measures for fee values, fee backup file implementation, unsigned fees socket implementation --- src/main/java/org/qortal/api/ApiService.java | 1 + .../model/crosschain/BitcoinyTBDRequest.java | 12 +- .../resource/CrossChainBitcoinResource.java | 20 +- .../resource/CrossChainDigibyteResource.java | 20 +- .../resource/CrossChainDogecoinResource.java | 20 +- .../resource/CrossChainLitecoinResource.java | 20 +- .../CrossChainPirateChainResource.java | 20 +- .../resource/CrossChainRavencoinResource.java | 20 +- .../api/resource/CrossChainResource.java | 98 ++ .../qortal/api/resource/CrossChainUtils.java | 20 +- .../api/websocket/UnsignedFeesSocket.java | 81 ++ .../org/qortal/controller/Controller.java | 14 + .../qortal/controller/ForeignFeesManager.java | 1182 +++++++++++++++++ .../java/org/qortal/crosschain/Bitcoin.java | 35 +- .../java/org/qortal/crosschain/Bitcoiny.java | 4 +- .../org/qortal/crosschain/BitcoinyTBD.java | 12 +- .../java/org/qortal/crosschain/Digibyte.java | 21 +- .../java/org/qortal/crosschain/Dogecoin.java | 21 +- .../java/org/qortal/crosschain/Litecoin.java | 21 +- .../java/org/qortal/crosschain/NetTBD.java | 15 +- .../org/qortal/crosschain/PirateChain.java | 21 +- .../java/org/qortal/crosschain/Ravencoin.java | 21 +- .../data/crosschain/ForeignFeeData.java | 57 + .../crosschain/ForeignFeeDecodedData.java | 90 ++ .../crosschain/ForeignFeeEncodedData.java | 69 + .../data/crosschain/UnsignedFeeEvent.java | 8 + .../org/qortal/event/FeeWaitingEvent.java | 26 + .../qortal/event/LockingFeeUpdateEvent.java | 4 + .../qortal/event/RequiredFeeUpdateEvent.java | 15 + .../network/message/ForeignFeesMessage.java | 43 + .../message/GetForeignFeesMessage.java | 46 + .../qortal/network/message/MessageType.java | 5 +- .../qortal/utils/ForeignFeesMessageUtils.java | 187 +++ .../message/ForeignFeesMessageTests.java | 334 +++++ 34 files changed, 2428 insertions(+), 155 deletions(-) create mode 100644 src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java create mode 100644 src/main/java/org/qortal/controller/ForeignFeesManager.java create mode 100644 src/main/java/org/qortal/data/crosschain/ForeignFeeData.java create mode 100644 src/main/java/org/qortal/data/crosschain/ForeignFeeDecodedData.java create mode 100644 src/main/java/org/qortal/data/crosschain/ForeignFeeEncodedData.java create mode 100644 src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java create mode 100644 src/main/java/org/qortal/event/FeeWaitingEvent.java create mode 100644 src/main/java/org/qortal/event/LockingFeeUpdateEvent.java create mode 100644 src/main/java/org/qortal/event/RequiredFeeUpdateEvent.java create mode 100644 src/main/java/org/qortal/network/message/ForeignFeesMessage.java create mode 100644 src/main/java/org/qortal/network/message/GetForeignFeesMessage.java create mode 100644 src/main/java/org/qortal/utils/ForeignFeesMessageUtils.java create mode 100644 src/test/java/org/qortal/test/network/message/ForeignFeesMessageTests.java diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java index 2cebe8e5..00ab29e0 100644 --- a/src/main/java/org/qortal/api/ApiService.java +++ b/src/main/java/org/qortal/api/ApiService.java @@ -197,6 +197,7 @@ public class ApiService { context.addServlet(DataMonitorSocket.class, "/websockets/datamonitor"); context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*"); context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages"); + context.addServlet(UnsignedFeesSocket.class, "/websockets/crosschain/unsignedfees"); context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers"); context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot"); context.addServlet(TradePresenceWebSocket.class, "/websockets/crosschain/tradepresence"); diff --git a/src/main/java/org/qortal/api/model/crosschain/BitcoinyTBDRequest.java b/src/main/java/org/qortal/api/model/crosschain/BitcoinyTBDRequest.java index 3a531413..1ffef4fa 100644 --- a/src/main/java/org/qortal/api/model/crosschain/BitcoinyTBDRequest.java +++ b/src/main/java/org/qortal/api/model/crosschain/BitcoinyTBDRequest.java @@ -304,11 +304,11 @@ public class BitcoinyTBDRequest { private String networkName; /** - * Fee Ceiling + * Fee Required * - * web search, LTC fee ceiling = 1000L + * web search, LTC fee required = 1000L */ - private long feeCeiling; + private long feeRequired; /** * Extended Public Key @@ -570,8 +570,8 @@ public class BitcoinyTBDRequest { return this.networkName; } - public long getFeeCeiling() { - return this.feeCeiling; + public long getFeeRequired() { + return this.feeRequired; } public String getExtendedPublicKey() { @@ -671,7 +671,7 @@ public class BitcoinyTBDRequest { ", minimumOrderAmount=" + minimumOrderAmount + ", feePerKb=" + feePerKb + ", networkName='" + networkName + '\'' + - ", feeCeiling=" + feeCeiling + + ", feeRequired=" + feeRequired + ", extendedPublicKey='" + extendedPublicKey + '\'' + ", sendAmount=" + sendAmount + ", sendingFeePerByte=" + sendingFeePerByte + diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java index 3720a0b5..fb070178 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -502,10 +502,10 @@ public class CrossChainBitcoinResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Bitcoin fee per Kb.", - description = "Returns Bitcoin fee per Kb.", + summary = "The total fee required for unlocking BTC to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -516,17 +516,17 @@ public class CrossChainBitcoinResource { ) } ) - public String getBitcoinFeeCeiling() { + public String getBitcoinFeeRequired() { Bitcoin bitcoin = Bitcoin.getInstance(); - return String.valueOf(bitcoin.getFeeCeiling()); + return String.valueOf(bitcoin.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Bitcoin fee ceiling.", - description = "Sets Bitcoin fee ceiling.", + summary = "The total fee required for unlocking BTC to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -545,13 +545,13 @@ public class CrossChainBitcoinResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setBitcoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setBitcoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Bitcoin bitcoin = Bitcoin.getInstance(); try { - return CrossChainUtils.setFeeCeiling(bitcoin, fee); + return CrossChainUtils.setFeeRequired(bitcoin, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java b/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java index 72b10096..1bf707db 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java @@ -502,10 +502,10 @@ public class CrossChainDigibyteResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Digibyte fee per Kb.", - description = "Returns Digibyte fee per Kb.", + summary = "The total fee required for unlocking DGB to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -516,17 +516,17 @@ public class CrossChainDigibyteResource { ) } ) - public String getDigibyteFeeCeiling() { + public String getDigibyteFeeRequired() { Digibyte digibyte = Digibyte.getInstance(); - return String.valueOf(digibyte.getFeeCeiling()); + return String.valueOf(digibyte.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Digibyte fee ceiling.", - description = "Sets Digibyte fee ceiling.", + summary = "The total fee required for unlocking DGB to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -545,13 +545,13 @@ public class CrossChainDigibyteResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setDigibyteFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setDigibyteFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Digibyte digibyte = Digibyte.getInstance(); try { - return CrossChainUtils.setFeeCeiling(digibyte, fee); + return CrossChainUtils.setFeeRequired(digibyte, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java index 4aa82e2b..5de39fb1 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java @@ -502,10 +502,10 @@ public class CrossChainDogecoinResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Dogecoin fee per Kb.", - description = "Returns Dogecoin fee per Kb.", + summary = "The total fee required for unlocking DOGE to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -516,17 +516,17 @@ public class CrossChainDogecoinResource { ) } ) - public String getDogecoinFeeCeiling() { + public String getDogecoinFeeRequired() { Dogecoin dogecoin = Dogecoin.getInstance(); - return String.valueOf(dogecoin.getFeeCeiling()); + return String.valueOf(dogecoin.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Dogecoin fee ceiling.", - description = "Sets Dogecoin fee ceiling.", + summary = "The total fee required for unlocking DOGE to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -545,13 +545,13 @@ public class CrossChainDogecoinResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setDogecoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setDogecoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Dogecoin dogecoin = Dogecoin.getInstance(); try { - return CrossChainUtils.setFeeCeiling(dogecoin, fee); + return CrossChainUtils.setFeeRequired(dogecoin, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java index 5b9e29d2..1564ff61 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -540,10 +540,10 @@ public class CrossChainLitecoinResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Litecoin fee per Kb.", - description = "Returns Litecoin fee per Kb.", + summary = "The total fee required for unlocking LTC to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -554,17 +554,17 @@ public class CrossChainLitecoinResource { ) } ) - public String getLitecoinFeeCeiling() { + public String getLitecoinFeeRequired() { Litecoin litecoin = Litecoin.getInstance(); - return String.valueOf(litecoin.getFeeCeiling()); + return String.valueOf(litecoin.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Litecoin fee ceiling.", - description = "Sets Litecoin fee ceiling.", + summary = "The total fee required for unlocking LTC to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -583,13 +583,13 @@ public class CrossChainLitecoinResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setLitecoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setLitecoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Litecoin litecoin = Litecoin.getInstance(); try { - return CrossChainUtils.setFeeCeiling(litecoin, fee); + return CrossChainUtils.setFeeRequired(litecoin, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainPirateChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainPirateChainResource.java index c6378f0b..c5e8ac82 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainPirateChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainPirateChainResource.java @@ -587,10 +587,10 @@ public class CrossChainPirateChainResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns PirateChain fee per Kb.", - description = "Returns PirateChain fee per Kb.", + summary = "The total fee required for unlocking ARRR to the trade offer creator.", + description = "The total fee required for unlocking ARRR to the trade offer creator.", responses = { @ApiResponse( content = @Content( @@ -601,17 +601,17 @@ public class CrossChainPirateChainResource { ) } ) - public String getPirateChainFeeCeiling() { + public String getPirateChainFeeRequired() { PirateChain pirateChain = PirateChain.getInstance(); - return String.valueOf(pirateChain.getFeeCeiling()); + return String.valueOf(pirateChain.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets PirateChain fee ceiling.", - description = "Sets PirateChain fee ceiling.", + summary = "The total fee required for unlocking ARRR to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -630,13 +630,13 @@ public class CrossChainPirateChainResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setPirateChainFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setPirateChainFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); PirateChain pirateChain = PirateChain.getInstance(); try { - return CrossChainUtils.setFeeCeiling(pirateChain, fee); + return CrossChainUtils.setFeeRequired(pirateChain, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java index c06122a9..72ff57f9 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java @@ -502,10 +502,10 @@ public class CrossChainRavencoinResource { } @GET - @Path("/feeceiling") + @Path("/feerequired") @Operation( - summary = "Returns Ravencoin fee per Kb.", - description = "Returns Ravencoin fee per Kb.", + summary = "The total fee required for unlocking RVN to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", responses = { @ApiResponse( content = @Content( @@ -516,17 +516,17 @@ public class CrossChainRavencoinResource { ) } ) - public String getRavencoinFeeCeiling() { + public String getRavencoinFeeRequired() { Ravencoin ravencoin = Ravencoin.getInstance(); - return String.valueOf(ravencoin.getFeeCeiling()); + return String.valueOf(ravencoin.getFeeRequired()); } @POST - @Path("/updatefeeceiling") + @Path("/updatefeerequired") @Operation( - summary = "Sets Ravencoin fee ceiling.", - description = "Sets Ravencoin fee ceiling.", + summary = "The total fee required for unlocking RVN to the trade offer creator.", + description = "This is in sats for a transaction that is approximately 300 kB in size.", requestBody = @RequestBody( required = true, content = @Content( @@ -545,13 +545,13 @@ public class CrossChainRavencoinResource { } ) @ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA}) - public String setRavencoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { + public String setRavencoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) { Security.checkApiCallAllowed(request); Ravencoin ravencoin = Ravencoin.getInstance(); try { - return CrossChainUtils.setFeeCeiling(ravencoin, fee); + return CrossChainUtils.setFeeRequired(ravencoin, fee); } catch (IllegalArgumentException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java index 3f7acf68..8104aabe 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainResource.java @@ -10,6 +10,8 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.glassfish.jersey.media.multipart.ContentDisposition; import org.qortal.api.ApiError; import org.qortal.api.ApiErrors; @@ -18,6 +20,7 @@ import org.qortal.api.Security; import org.qortal.api.model.CrossChainCancelRequest; import org.qortal.api.model.CrossChainTradeLedgerEntry; import org.qortal.api.model.CrossChainTradeSummary; +import org.qortal.controller.ForeignFeesManager; import org.qortal.controller.tradebot.TradeBot; import org.qortal.crosschain.ACCT; import org.qortal.crosschain.AcctMode; @@ -29,6 +32,8 @@ import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.data.crosschain.CrossChainTradeData; import org.qortal.data.crosschain.TransactionSummary; +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.data.crosschain.ForeignFeeEncodedData; import org.qortal.data.transaction.BaseTransactionData; import org.qortal.data.transaction.MessageTransactionData; import org.qortal.data.transaction.TransactionData; @@ -64,6 +69,8 @@ import java.util.stream.Collectors; @Tag(name = "Cross-Chain") public class CrossChainResource { + private static final Logger LOGGER = LogManager.getLogger(CrossChainResource.class); + @Context HttpServletRequest request; @@ -360,6 +367,97 @@ public class CrossChainResource { } } + @POST + @Path("/signedfees") + @Operation( + summary = "", + description = "", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + array = @ArraySchema( + schema = @Schema( + implementation = ForeignFeeEncodedData.class + ) + ) + ) + ), + responses = { + @ApiResponse( + description = "true on success", + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "boolean" + ) + ) + ) + } + ) + public String postSignedForeignFees(List signedFees) { + + LOGGER.info("signedFees = " + signedFees); + + try { + ForeignFeesManager.getInstance().addSignedFees(signedFees); + + return "true"; + } + catch( Exception e ) { + + LOGGER.error(e.getMessage(), e); + + return "false"; + } + } + + @GET + @Path("/unsignedfees/{address}") + @Operation( + summary = "", + description = "", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = ForeignFeeEncodedData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public List getUnsignedFees(@PathParam("address") String address) { + + return ForeignFeesManager.getInstance().getUnsignedFeesForAddress(address); + } + + @GET + @Path("/signedfees") + @Operation( + summary = "", + description = "", + responses = { + @ApiResponse( + content = @Content( + array = @ArraySchema( + schema = @Schema( + implementation = ForeignFeeDecodedData.class + ) + ) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) + public List getSignedFees() { + + return ForeignFeesManager.getInstance().getSignedFees(); + } + /** * Decode Public Key * diff --git a/src/main/java/org/qortal/api/resource/CrossChainUtils.java b/src/main/java/org/qortal/api/resource/CrossChainUtils.java index ddd1d2d6..c923850f 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainUtils.java +++ b/src/main/java/org/qortal/api/resource/CrossChainUtils.java @@ -16,6 +16,9 @@ import org.qortal.crosschain.*; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.data.crosschain.*; +import org.qortal.event.EventBus; +import org.qortal.event.LockingFeeUpdateEvent; +import org.qortal.event.RequiredFeeUpdateEvent; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.utils.Amounts; @@ -23,14 +26,9 @@ import org.qortal.utils.BitTwiddling; import java.io.BufferedWriter; import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; import java.io.Writer; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; @@ -103,11 +101,13 @@ public class CrossChainUtils { bitcoiny.setFeePerKb(Coin.valueOf(satoshis) ); + EventBus.INSTANCE.notify(new LockingFeeUpdateEvent()); + return String.valueOf(bitcoiny.getFeePerKb().value); } /** - * Set Fee Ceiling + * Set Fee Required * * @param bitcoiny the blockchain support * @param fee the fee in satoshis @@ -116,14 +116,16 @@ public class CrossChainUtils { * * @throws IllegalArgumentException if invalid */ - public static String setFeeCeiling(Bitcoiny bitcoiny, String fee) throws IllegalArgumentException{ + public static String setFeeRequired(Bitcoiny bitcoiny, String fee) throws IllegalArgumentException{ long satoshis = Long.parseLong(fee); if( satoshis < 0 ) throw new IllegalArgumentException("can't set fee to negative number"); - bitcoiny.setFeeCeiling( Long.parseLong(fee)); + bitcoiny.setFeeRequired( Long.parseLong(fee)); - return String.valueOf(bitcoiny.getFeeCeiling()); + EventBus.INSTANCE.notify(new RequiredFeeUpdateEvent(bitcoiny)); + + return String.valueOf(bitcoiny.getFeeRequired()); } /** diff --git a/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java b/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java new file mode 100644 index 00000000..678b12b6 --- /dev/null +++ b/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java @@ -0,0 +1,81 @@ +package org.qortal.api.websocket; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.jetty.websocket.api.Session; +import org.eclipse.jetty.websocket.api.WebSocketException; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError; +import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage; +import org.eclipse.jetty.websocket.api.annotations.WebSocket; +import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory; +import org.qortal.data.crosschain.UnsignedFeeEvent; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.FeeWaitingEvent; +import org.qortal.event.Listener; + +import java.io.IOException; +import java.io.StringWriter; + +@WebSocket +@SuppressWarnings("serial") +public class UnsignedFeesSocket extends ApiWebSocket implements Listener { + + private static final Logger LOGGER = LogManager.getLogger(UnsignedFeesSocket.class); + + @Override + public void configure(WebSocketServletFactory factory) { + LOGGER.info("configure"); + + factory.register(UnsignedFeesSocket.class); + + EventBus.INSTANCE.addListener(this); + } + + @Override + public void listen(Event event) { + if (!(event instanceof FeeWaitingEvent)) + return; + + for (Session session : getSessions()) + sendUnsignedFeeEvent(session, new UnsignedFeeEvent()); + } + + + @OnWebSocketConnect + @Override + public void onWebSocketConnect(Session session) { + super.onWebSocketConnect(session); + } + + @OnWebSocketClose + @Override + public void onWebSocketClose(Session session, int statusCode, String reason) { + super.onWebSocketClose(session, statusCode, reason); + } + + @OnWebSocketError + public void onWebSocketError(Session session, Throwable throwable) { + /* We ignore errors for now, but method here to silence log spam */ + } + + @OnWebSocketMessage + public void onWebSocketMessage(Session session, String message) { + LOGGER.info("onWebSocketMessage: message = " + message); + } + + private void sendUnsignedFeeEvent(Session session, UnsignedFeeEvent unsignedFeeEvent) { + StringWriter stringWriter = new StringWriter(); + + try { + marshall(stringWriter, unsignedFeeEvent); + + session.getRemote().sendStringByFuture(stringWriter.toString()); + } catch (IOException | WebSocketException e) { + // No output this time + } + } + +} \ No newline at end of file diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 22f7e2d2..33c49bde 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -560,6 +560,9 @@ public class Controller extends Thread { LOGGER.info("Starting online accounts manager"); OnlineAccountsManager.getInstance().start(); + LOGGER.info("Starting foreign fees manager"); + ForeignFeesManager.getInstance().start(); + LOGGER.info("Starting transaction importer"); TransactionImporter.getInstance().start(); @@ -1130,6 +1133,9 @@ public class Controller extends Thread { LOGGER.info("Shutting down online accounts manager"); OnlineAccountsManager.getInstance().shutdown(); + LOGGER.info("Shutting down foreign fees manager"); + ForeignFeesManager.getInstance().shutdown(); + LOGGER.info("Shutting down transaction importer"); TransactionImporter.getInstance().shutdown(); @@ -1474,6 +1480,14 @@ public class Controller extends Thread { OnlineAccountsManager.getInstance().onNetworkOnlineAccountsV3Message(peer, message); break; + case GET_FOREIGN_FEES: + ForeignFeesManager.getInstance().onNetworkGetForeignFeesMessage(peer, message); + break; + + case FOREIGN_FEES: + ForeignFeesManager.getInstance().onNetworkForeignFeesMessage(peer, message); + break; + case GET_ARBITRARY_DATA: // Not currently supported break; diff --git a/src/main/java/org/qortal/controller/ForeignFeesManager.java b/src/main/java/org/qortal/controller/ForeignFeesManager.java new file mode 100644 index 00000000..7bae7aa9 --- /dev/null +++ b/src/main/java/org/qortal/controller/ForeignFeesManager.java @@ -0,0 +1,1182 @@ +package org.qortal.controller; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bitcoinj.core.Coin; +import org.json.JSONArray; +import org.json.JSONObject; +import org.qortal.controller.tradebot.TradeBot; +import org.qortal.controller.tradebot.TradeStates; +import org.qortal.crosschain.ACCT; +import org.qortal.crosschain.AcctMode; +import org.qortal.crosschain.Bitcoiny; +import org.qortal.crosschain.ForeignBlockchain; +import org.qortal.crosschain.SupportedBlockchain; +import org.qortal.crypto.Crypto; +import org.qortal.data.at.ATData; +import org.qortal.data.crosschain.CrossChainTradeData; +import org.qortal.data.crosschain.ForeignFeeData; +import org.qortal.data.crosschain.TradeBotData; +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.data.crosschain.ForeignFeeEncodedData; +import org.qortal.event.Event; +import org.qortal.event.EventBus; +import org.qortal.event.FeeWaitingEvent; +import org.qortal.event.LockingFeeUpdateEvent; +import org.qortal.event.RequiredFeeUpdateEvent; +import org.qortal.event.Listener; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.ForeignFeesMessage; +import org.qortal.network.message.GetForeignFeesMessage; +import org.qortal.network.message.Message; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.repository.hsqldb.HSQLDBImportExport; +import org.qortal.settings.Settings; +import org.qortal.utils.Base58; +import org.qortal.utils.ForeignFeesMessageUtils; +import org.qortal.utils.NTP; +import org.qortal.utils.NamedThreadFactory; +import org.qortal.utils.Triple; + +import java.io.FileNotFoundException; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class ForeignFeesManager implements Listener { + + private static final Logger LOGGER = LogManager.getLogger(ForeignFeesManager.class); + + public static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + public static final String SIGNED_FOREIGN_FEES_TYPE = "signedForeignFees"; + public static final String SIGNED_FOREIGN_FEES_FILE_NAME = "SignedForeignFees.json"; + public static final String CURRENT_DATASET_LABEL = "current"; + private static final String REQUIRED_FOREIGN_FEES_TYPE = "requiredForeignFees"; + private static final String LOCKING_FOREIGN_FEES_TYPE = "lockingForeignFees"; + public static final String REQUIRED_FOREIGN_FEES_FILE_NAME = "RequiredForeignFees.json"; + public static final String LOCKING_FOREIGN_FEES_FILE_NAME = "LockingForeignFees.json"; + + private final ScheduledExecutorService executor + = Executors.newScheduledThreadPool(4, new NamedThreadFactory("Foreign Fee Manager", Thread.NORM_PRIORITY)); + + private volatile boolean isStopping = false; + + private final Set foreignFeesImportQueue = ConcurrentHashMap.newKeySet(); + + /** + * Cache of signed foreign fees, keyed by AT address + */ + private final ConcurrentHashMap> signedByAT = new ConcurrentHashMap<>(); + + /** + * Cache of unsigned foreign fees on this node, key by AT address + */ + private final ConcurrentHashMap unsignedByAT = new ConcurrentHashMap<>(); + + /** + * Cache of trade offers, keyed by creator address + */ + private final ConcurrentHashMap> offersByAddress = new ConcurrentHashMap<>(); + + /** + * Need to Backup Locking Foreign Fees? + * + * Set when the locking foreign fees need to be backed up to a file. + */ + private AtomicBoolean needToBackupLockingForeignFees = new AtomicBoolean(true); + + /** + * Need to Backup Required Foreign Fees? + * + * Set when the required foreign fees need to be backed up to a file. + */ + private AtomicBoolean needToBackupRequiredForeignFees = new AtomicBoolean(true); + + /** + * Need to Backup Signed Foreign Fees? + * + * Set when the signed foreign fees for this node need to be backed up to a file. + */ + private AtomicBoolean needToBackupSignedForeignFees = new AtomicBoolean(true); + + private ForeignFeesManager() { + + EventBus.INSTANCE.addListener(this); + } + + /** + * Import Data + * + * Import signed transaction data for this node and the required fees for unlocking foreign trade funds + * for this node. + */ + private void importData() { + + try { + String exportPath = Settings.getInstance().getExportPath(); + + // import signed foreign fees + try { + Path importSignedForeignFeesPath = Paths.get(exportPath, SIGNED_FOREIGN_FEES_FILE_NAME); + importDataFromFile(importSignedForeignFeesPath.toString()); + } + catch (FileNotFoundException e) { + LOGGER.warn(e.getMessage()); + } + + // import required foreign fees + try { + Path importRequiredForeignFeespath = Paths.get(exportPath, REQUIRED_FOREIGN_FEES_FILE_NAME); + importDataFromFile(importRequiredForeignFeespath.toString()); + } + catch (FileNotFoundException e) { + LOGGER.warn(e.getMessage()); + } + + // import locking foreign fees + try { + Path importLockingForeignFeespath = Paths.get(exportPath, LOCKING_FOREIGN_FEES_FILE_NAME); + importDataFromFile(importLockingForeignFeespath.toString()); + } + catch (FileNotFoundException e) { + LOGGER.warn(e.getMessage()); + } + + } + catch (DataException | IOException e) { + LOGGER.info("Unable to import data into foreign fees manager: {}", e.getMessage()); + } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Get Unsigned Fees For Address + * + * @param address the address + * + * @return the unsigned fee data + */ + public List getUnsignedFeesForAddress(String address) { + + // the trade offers for this address on this node + List atAddressesForOffers + = this.offersByAddress.getOrDefault(address, new ArrayList<>(0)).stream() + .map( data -> data.qortalAtAddress ) + .collect(Collectors.toList()); + + // the unsigned fee data for the address's trade offers + return this.unsignedByAT.entrySet().stream() + .filter( entry -> atAddressesForOffers.contains(entry.getKey())) + .map( entry -> entry.getValue()) + .collect(Collectors.toList()); + } + + /** + * Get Signed Fees + * + * @return the signed fee data on this node + */ + public List getSignedFees() { + + return this.signedByAT.values().stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + /** + * Add Signed Fees + * + * Add signed fees to the import queue. + * + * @param signedFees the signed fees + */ + public void addSignedFees(List signedFees) { + + LOGGER.info("adding signed fees: count = " + signedFees.size()); + + // for each encoided fee, decode and add to import queue + for( ForeignFeeEncodedData signedFeeEncoded : signedFees ) { + + LOGGER.info("adding to import queue: " + signedFeeEncoded); + + // decode the fee data and add to the queue + this.foreignFeesImportQueue.add( + new ForeignFeeDecodedData( + signedFeeEncoded.getTimestamp(), + Base58.decode(signedFeeEncoded.getData()), + signedFeeEncoded.getAtAddress(), + signedFeeEncoded.getFee() + ) + ); + LOGGER.info("added"); + } + + LOGGER.info("done adding to queue: count = " + this.foreignFeesImportQueue.size()); + + // process the fees immediately (not waiting for the fee process timer task already in place) + processForeignFeesImportQueue(); + } + + @Override + public void listen(Event event) { + + // locking fee update, then flag locking fee backup + if( event instanceof LockingFeeUpdateEvent) { + this.needToBackupLockingForeignFees.compareAndSet(false, true); + } + // if required fee update, then flag required fee update and process fees for the coin updated + else if( event instanceof RequiredFeeUpdateEvent) { + + this.needToBackupRequiredForeignFees.compareAndSet(false, true); + + if( processLocalForeignFeesForCoin(((RequiredFeeUpdateEvent) event).getBitcoiny()) ) { + EventBus.INSTANCE.notify(new FeeWaitingEvent()); + } + } + // + else if( event instanceof TradeBot.StateChangeEvent ) { + + TradeBotData data = ((TradeBot.StateChangeEvent) event).getTradeBotData(); + + // if offer is waiting and the time now is determined, + // then process the trade data to be signed later + if( data.getStateValue() == TradeStates.State.BOB_WAITING_FOR_MESSAGE.value ) { + Optional nowDetermined = determineNow(); + if (nowDetermined.isPresent()) { + + long now = nowDetermined.get(); + try (final Repository repository = RepositoryManager.getRepository()) { + LOGGER.info("processing trade offer in waiting event"); + + Optional offerOptional = getTradeOfferData(repository, data.getAtAddress()); + + if( offerOptional.isPresent() ) { + CrossChainTradeData offer = offerOptional.get(); + this.offersByAddress.computeIfAbsent( offer.qortalCreator, x -> new ArrayList<>()).add(offer); + + if( processTradeOfferInWaiting(now, data) ) { + EventBus.INSTANCE.notify(new FeeWaitingEvent()); + } + } + else { + LOGGER.warn("offer not present for new trade bot offer = " + data); + } + } catch (IOException | DataException e) { + LOGGER.error(e.getMessage(), e); + } + } + } + } + } + + private static class SingletonContainer { + private static final ForeignFeesManager INSTANCE = new ForeignFeesManager(); + } + + public static ForeignFeesManager getInstance() { + return SingletonContainer.INSTANCE; + } + + /** + * Get Signed Foreign Fee Data By AT Address + * + * @return st address -> signed foreign fee data + */ + public ConcurrentHashMap> getSignedByAT() { + return signedByAT; + } + + /** + * Start Manager + */ + public void start() { + + // import data after a 1 minute delay + // this will get locking fees, required fees and signed data + // there will be nothing to import the first time running this manager + executor.schedule(this::importData, 1, TimeUnit.MINUTES); + + // process local foreign fees for all coins once after a 2 minute delay + // this will get the unsigned fee data + executor.schedule(this::processLocalForeignFeesForAll, 2, TimeUnit.MINUTES); + + // maintain AT's every 5 minutes + executor.scheduleAtFixedRate(this::maintainCrossChainOffers, 3, 5, TimeUnit.MINUTES); + + // request foreign fees from peers every 5 minutes + executor.scheduleAtFixedRate(this::requestRemoteForeignFees, 4, 5, TimeUnit.MINUTES); + + // process imported foreign fees every 5 minutes + executor.scheduleWithFixedDelay(this::processForeignFeesImportQueue, 5, 5, TimeUnit.MINUTES); + + // backup data every 5 minutes + executor.scheduleAtFixedRate(this::backup, 6, 5, TimeUnit.MINUTES); + } + + /** + * Backup + * + * Backup member data used by this manager. + */ + private void backup() { + + try { + if( this.needToBackupLockingForeignFees.compareAndSet( true, false )) { + LOGGER.info("backing up locking foreign fees"); + backupForeignFeeData( bitcoiny -> bitcoiny.getFeePerKb().value, LOCKING_FOREIGN_FEES_FILE_NAME, LOCKING_FOREIGN_FEES_TYPE); + } + if( this.needToBackupRequiredForeignFees.compareAndSet(true, false) ) { + LOGGER.info("backing up required foreign fees"); + backupForeignFeeData(Bitcoiny::getFeeRequired, REQUIRED_FOREIGN_FEES_FILE_NAME, REQUIRED_FOREIGN_FEES_TYPE); + } + + if( this.needToBackupSignedForeignFees.compareAndSet( true, false ) ) { + LOGGER.info("backing up signed foreign fees"); + backupSignedForeignFeeData(); + } + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Shutdown Manager + */ + public void shutdown() { + isStopping = true; + executor.shutdownNow(); + } + + /** + * Process Import Queue + */ + private void processForeignFeesImportQueue() { + + LOGGER.info("processing foreign fees import queue ..."); + + if (this.foreignFeesImportQueue.isEmpty()) { + + LOGGER.info("foreign fees import queue is empty"); + return; + } + + LOGGER.info("Processing foreign fee import queue (size: {})", this.foreignFeesImportQueue.size()); + + Set foreignFeesToRemove = new HashSet<>(this.foreignFeesImportQueue.size()); + + try (final Repository repository = RepositoryManager.getRepository()) { + + // for each signed foreign fee in the queue, + // compare timestamps to prior imports, verify signature and possibly import in + for (ForeignFeeDecodedData foreignFeeToImport : this.foreignFeesImportQueue) { + if (isStopping) + return; + + // need to get the AT address for mapping key identification purposes + String atAddress = foreignFeeToImport.getAtAddress(); + + LOGGER.info("foreign fee import, timestamp = " + getFormattedDateTime(foreignFeeToImport.getTimestamp())); + + Optional validatedForeignFeeData + = this.signedByAT.getOrDefault( atAddress, Optional.empty() ); + + // if there is no established, validated foreign fee for this AT address or + // if the import foreign fee is after the validated foreign fee, + // then verify the signature and map it to the AT address + if (validatedForeignFeeData.isEmpty() || validatedForeignFeeData.get().getTimestamp() < foreignFeeToImport.getTimestamp()) { + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + + LOGGER.info("verify signer for atAddress = " + atAddress); + + // determine if the creator authorized the foreign fee + byte[] publicKey = atData.getCreatorPublicKey(); + byte[] signature = foreignFeeToImport.getData(); + byte[] message + = ForeignFeesMessageUtils.buildForeignFeesDataMessage( + foreignFeeToImport.getTimestamp(), + atAddress, + foreignFeeToImport.getFee() + ); + + // if trade offer creator authorized the imported fee, + // then finish the import and clear it from the unsigned mapping + if( Crypto.verify(publicKey, signature, message) ) { + LOGGER.info("signer verified"); + this.signedByAT.put(atAddress, Optional.of(foreignFeeToImport)); + this.needToBackupSignedForeignFees.compareAndSet(false, true); + this.unsignedByAT.remove(atAddress); + } + // otherwise this fee will get discarded + else { + LOGGER.info("invalid signature"); + } + } + else { + LOGGER.info( + "skipping imported fee since the timestamp is not updated: atAddress = {}, timestamp = {}", + atAddress, + foreignFeeToImport.getTimestamp() + ); + } + + // now that this fee has been processed, remove it from the process queue + foreignFeesToRemove.add(foreignFeeToImport); + } + } catch (Exception e) { + LOGGER.error("Repository issue while verifying foreign fees", e); + } finally { + LOGGER.info("removing foreign fees from import queue: count = " + foreignFeesToRemove.size()); + this.foreignFeesImportQueue.removeAll(foreignFeesToRemove); + } + } + + /** + * Get Formatted Date Time + * + * For logging purposes only. + * + * @param timestamp utc time in milliseconds + * + * @return the formatted string + */ + private static String getFormattedDateTime( long timestamp ) { + + Instant instant = Instant.ofEpochMilli(timestamp); + ZonedDateTime zdt = ZonedDateTime.ofInstant(instant, ZoneId.systemDefault()); + String formattedDateTime = zdt.format(TIMESTAMP_FORMATTER); + + return formattedDateTime; + } + + /** + * Maintain AT's + * + * This removes fee data for AT addreses that are no longer offered on the trade portal. + */ + private void maintainCrossChainOffers() { + + LOGGER.info("maintaining ATs ..."); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List crossChainTradeOffers = resetOffersByAddress(repository); + + // remove failed trades, then collect AT addresses for trade offers + Set atAddresses + = TradeBot.getInstance() + .removeFailedTrades(repository, crossChainTradeOffers).stream() + .map( data -> data.qortalAtAddress ) + .collect(Collectors.toSet()); + + LOGGER.info("foreign fees before AT removal: count = " + this.signedByAT.size() ); + + // retain the fees for the current sell offers, remove all others + if( retainFeeByAT(this.signedByAT, atAddresses) ) { + this.needToBackupSignedForeignFees.compareAndSet(false, true); + } + + LOGGER.info("foreign fees after AT removal: count = " + this.signedByAT.size() ); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Reset Offers By Address + * + * @param repository the data repository + * + * @return address -> cross chain trades + * + * @throws DataException + */ + private List resetOffersByAddress(Repository repository) throws DataException { + List crossChainTradeOffers = new ArrayList<>(); + + // lockdown map while reseting offers by address + synchronized( this.offersByAddress) { + + // for each supported foreign blockchaine, get AT data for trade offers + for ( SupportedBlockchain blockchain : SupportedBlockchain.values()) { + crossChainTradeOffers.addAll( getCrossTradeOffers(repository, blockchain) ); + } + + // group all trade offers by trade offer creator, then reset map + Map> groupedOffersByAddress + = crossChainTradeOffers.stream().collect(Collectors.groupingBy(data -> data.qortalCreator)); + + this.offersByAddress.clear(); + this.offersByAddress.putAll(groupedOffersByAddress); + } + + return crossChainTradeOffers; + } + + /** + * Retain Fee By AT + * + * Retain fees for a list of AT addresses; remove all others. + * + * @param feeByAT the foreign fee data for each trade offer AT address + * @param atAddresses the AT addresses to retain fees for + * + * @return true if any removals, otherwise false + */ + private static boolean retainFeeByAT(ConcurrentHashMap> feeByAT, Set atAddresses) { + + // return value, false until there is a removal + boolean anyRemovals = false; + + // prepate iterator for all AT -> fee mappings + Iterator>> iterator = feeByAT.entrySet().iterator(); + + // iterate over all AT's mapped under management + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + + // if the at address do not contain this entry in the iteration, + // then remove it + if (!atAddresses.contains(entry.getKey())) { + + iterator.remove(); + anyRemovals = true; + } + } + + return anyRemovals; + } + + /** + * Get Cross Trade Offers + * + * @param repository the data repository + * @param blockchain the foreign blockchain supporting the trade + * + * @return the trade offers + * + * @throws DataException + */ + private static List getCrossTradeOffers(Repository repository, SupportedBlockchain blockchain) throws DataException { + + // get ACCT for the foreign blockchain + ACCT acct = blockchain.getLatestAcct(); + + // get AT's for foreign blockchain + List ats + = repository.getATRepository() + .getATsByFunctionality(acct.getCodeBytesHash(), true, null, null, null); + + // prepare the return list of cross chain data + List crossChainTradeOffers = new ArrayList<>(ats.size()); + + // for each AT, get cross chain data and look for trade offer to add + for (ATData at : ats) { + + CrossChainTradeData crossChainTrade = acct.populateTradeData(repository, at); + + // if the trade is in offering mode, then add it to return list + if (crossChainTrade.mode == AcctMode.OFFERING) { + crossChainTradeOffers.add(crossChainTrade); + } + } + + return crossChainTradeOffers; + } + + private static Optional getTradeOfferData(Repository repository, String atAddress) throws DataException { + + ATData atData = repository.getATRepository().fromATAddress(atAddress); + + ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash()); + CrossChainTradeData crossChainTrade = acct.populateTradeData(repository, atData); + + // if the trade is in offering mode, then add it to return list + if (crossChainTrade.mode == AcctMode.OFFERING) { + return Optional.of(crossChainTrade); + } + else { + return Optional.empty(); + } + } + + /** + * Request data from other peers + */ + private void requestRemoteForeignFees() { + + LOGGER.info("requesting remote foreign fees ..."); + + if (!isUpToDate()) return; + + LOGGER.info("Requesting foreign fees via broadcast..."); + + Message message + = new GetForeignFeesMessage( + this.signedByAT.values().stream() + .filter(Optional::isPresent).map(Optional::get) + .collect(Collectors.toList()) + ); + + Network.getInstance().broadcast(peer -> message); + + LOGGER.info("Requested foreign fees via broadcast..."); + } + + /** + * Is Up To Date? + * + * @return true if up to date, otherwise false + */ + private static boolean isUpToDate() { + final Long now = NTP.getTime(); + + if (now == null) { + + LOGGER.warn("time is null, aborting"); + return false; + } + + if (!Controller.getInstance().isUpToDate()) { + + LOGGER.info("not up to date, aborting"); + return false; + } + return true; + } + + /** + * Process foreign fees for all coins + * + * Collect foreign fees for trades waiting locally and store to this manager. + * + * @return if any fee signatures are needed after this process + */ + private void processLocalForeignFeesForAll() { + + boolean feeSignaturesNeeded = false; + + List names + = Arrays.stream(SupportedBlockchain.values()) + .map( value -> value.getLatestAcct().getClass().getSimpleName()) + .collect(Collectors.toList()); + + for( String name : names ) { + ForeignBlockchain blockchain = SupportedBlockchain.getAcctByName(name).getBlockchain(); + + if( blockchain instanceof Bitcoiny ) { + feeSignaturesNeeded = processLocalForeignFeesForCoin((Bitcoiny) blockchain) || feeSignaturesNeeded; + } + } + + if( feeSignaturesNeeded ) { + EventBus.INSTANCE.notify(new FeeWaitingEvent()); + } + } + + /** + * Process foreign fees for coin + * + * Collect foreign fees for trades waiting locally and store to this manager. + * + * @param bitcoiny the coin + * + * @return if any fee signatures are needed after this process + */ + private boolean processLocalForeignFeesForCoin(final Bitcoiny bitcoiny) { + + boolean feeSignaturesNeeded = false; + + LOGGER.info("processing local foreign fees ..."); + + Optional nowDetermined = determineNow(); + if (nowDetermined.isEmpty()){ + return false; + } + + long now = nowDetermined.get(); + + try (final Repository repository = RepositoryManager.getRepository()) { + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + + // collect all local trade offers waiting + List tradeOffersWaiting + = allTradeBotData.stream() + .filter(d -> d.getStateValue() == TradeStates.State.BOB_WAITING_FOR_MESSAGE.value) + .filter(d -> SupportedBlockchain.getAcctByName( d.getAcctName() ).getBlockchain().equals( bitcoiny )) + .collect(Collectors.toList()); + + LOGGER.info("trade offers waiting: count = " + tradeOffersWaiting.size()); + + // process each local trade offer waiting (listed) + for (TradeBotData tradeOfferWaiting : tradeOffersWaiting) { + + // process trade offer first, + // then reset the fee signatures needed status next relative to prior status + feeSignaturesNeeded = processTradeOfferInWaiting(now, tradeOfferWaiting) || feeSignaturesNeeded; + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + + return feeSignaturesNeeded; + } + + /** + * Determine Now + * + * @return now if in synce, otherwise empty + */ + private static Optional determineNow() { + // if time is not available, then abort + Long now = NTP.getTime(); + if (now == null) { + + LOGGER.warn("current time is not available, abort sending foreign fees"); + return Optional.empty(); + } + + // if 2 hours behind w/o recovery mode, then abort + final Long minLatestBlockTimestamp = now - (2 * 60 * 60 * 1000L); + if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp) && !Synchronizer.getInstance().getRecoveryMode()) { + + LOGGER.warn("out of sync, abort sending foreign fees"); + return Optional.empty(); + } + return Optional.of(now); + } + + /** + * Process Trade Offer In Waiting + * + * @param now the time in millis for now + * @param tradeOfferWaiting the trade offer in waiting + * + * @return true if the fee for this offer needs to be signed, otherwise false + * + * @throws IOException + */ + private boolean processTradeOfferInWaiting(Long now, TradeBotData tradeOfferWaiting) throws IOException { + + boolean isFeeWaiting = false; + + // derive the supported blockchain for the trade offer waiting + String foreignBlockchain = tradeOfferWaiting.getForeignBlockchain(); + SupportedBlockchain supportedBlockchain = SupportedBlockchain.fromString(foreignBlockchain); + + LOGGER.info("trade offer waiting: blockchain = " + foreignBlockchain); + + // if the supported blockchain is a Bitcoiny blockchain, then the fee will be available + if (supportedBlockchain.getInstance() instanceof Bitcoiny) { + + // get the foreign blockcahin, the AT address and the foreign fee set to this node + Bitcoiny bitcoiny = (Bitcoiny) supportedBlockchain.getInstance(); + String atAddress = tradeOfferWaiting.getAtAddress(); + int fee = Math.toIntExact(bitcoiny.getFeeRequired()); + + LOGGER.info("atAddress = {}, fee = {}", atAddress, fee); + + // get the signed foreign fee, if it exists + Optional foreignFeeDecodedData = this.signedByAT.get(atAddress); + + // if the foreign fee has been signed + if (foreignFeeDecodedData != null && foreignFeeDecodedData.isPresent()) { + + LOGGER.info("signed available"); + + // if the local fee is different than the fee stored in this manager, + // then empty the fee in the manager and set the updated fee to unsigned data + if (!foreignFeeDecodedData.get().getFee().equals(fee)) { + + LOGGER.info("fee updated"); + this.signedByAT.remove(atAddress); + + this.needToBackupSignedForeignFees.compareAndSet(false, true); + setUnsignedData(now, atAddress, fee); + isFeeWaiting = true; + } + else { + LOGGER.info("fee not updated"); + } + } + // if the foreign fee has not been signed, then set the fee to unsigned data + else { + LOGGER.info("fee not signed"); + setUnsignedData(now, atAddress, fee); + isFeeWaiting = true; + } + } + // if the supported blockchain is not a Bitcoiny blockchain, then the fee is not available + else { + LOGGER.warn("Blockchain fee not available: blockchain = " + foreignBlockchain); + } + + return isFeeWaiting; + } + + /** + * Set Unisgned Data + * + * @param timestamp + * @param atAddress + * @param fee + * + * @throws IOException + */ + private void setUnsignedData(Long timestamp, String atAddress, int fee) throws IOException { + ForeignFeeEncodedData feeData + = new ForeignFeeEncodedData( + timestamp, + Base58.encode(ForeignFeesMessageUtils.buildForeignFeesDataMessage(timestamp, atAddress, fee)), + atAddress, + fee + ); + + this.unsignedByAT.put(atAddress, feeData); + } + + // Network handlers + + /** + * Handle GetForeignFeesMessage + * + * @param peer + * @param message + */ + public void onNetworkGetForeignFeesMessage(Peer peer, Message message) { + GetForeignFeesMessage getForeignFeesMessage = (GetForeignFeesMessage) message; + + // map the fees the peer already has + Map inMessageDataByAT + = getForeignFeesMessage.getForeignFeeData().stream() + .collect(Collectors.toMap( ForeignFeeDecodedData::getAtAddress, Function.identity())); + + // start collecting fees to send to the peer + List outgoingForeignFees = new ArrayList<>(); + + // for all the signed fees locally stored, compare to what the peer currently has and send out what they need + for(Map.Entry> entry : this.signedByAT.entrySet() ) { + + Optional signedForeignFeeData = entry.getValue(); + + // if the fee has been signed, then evaluate it for sending + if (signedForeignFeeData.isPresent()) { + + String atAddress = entry.getKey(); + + LOGGER.info("comparing signed foreign fee for get foreign fee message: atAddress = " + atAddress); + + // if message contains AT address, then check timestamps + if (inMessageDataByAT.containsKey(atAddress) ) { + + LOGGER.info("message does contain: atAddress = " + atAddress); + + // get data from message for AT address + ForeignFeeDecodedData feeData = inMessageDataByAT.get(atAddress); + + // if message data is earlier than what is here locally, then send local out to the peer + if( feeData != null && signedForeignFeeData.get().getTimestamp() > feeData.getTimestamp()) { + outgoingForeignFees.add(signedForeignFeeData.get()); + } + } + // if the message does not contain data for this AT, then send the data out to the peer + else { + LOGGER.info("message does not contain: atAddress = " + atAddress); + + outgoingForeignFees.add(signedForeignFeeData.get()); + } + } + // if value is empty, then do nothing + else { + LOGGER.info("unavailable signed foreign fee for get foreign fee message: atAddress = " + entry.getKey()); + } + } + + LOGGER.info("Sending {} foreign fees to {}", outgoingForeignFees.size(), peer); + + // send out to peer + peer.sendMessage(new ForeignFeesMessage(outgoingForeignFees)); + + LOGGER.info("Sent {} foreign fees to {}", outgoingForeignFees.size(), peer); + } + + /** + * Handle ForeignFeesMessage + * + * @param peer + * @param message + */ + public void onNetworkForeignFeesMessage(Peer peer, Message message) { + ForeignFeesMessage onlineAccountsMessage = (ForeignFeesMessage) message; + + List peersForeignFees = onlineAccountsMessage.getForeignFees(); + LOGGER.info("Received {} foreign fees from {}", peersForeignFees.size(), peer); + + int importCount = 0; + + // add any foreign fees to the queue that aren't already present + for (ForeignFeeDecodedData peerForeignFee : peersForeignFees) { + + if( foreignFeesImportQueue.add(peerForeignFee) ) + importCount++; + } + + if (importCount > 0) + LOGGER.info("Added {} foreign to queue", importCount); + } + + /** + * Backup Signed Foreign Fee Data + * + * @throws DataException + */ + private void backupSignedForeignFeeData() throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // get all signed foreigh fee data on this node + List signedForeignFees + = this.signedByAT.values().stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + + // get the JSON for the signed foreign fees + JSONArray currentSignedForeignFeeDataJson = new JSONArray(); + for (ForeignFeeDecodedData signedForeignFee : signedForeignFees) { + JSONObject foreignFeeSignatureJson = signedForeignFee.toJson(); + currentSignedForeignFeeDataJson.put(foreignFeeSignatureJson); + } + + // put data into a JSON + JSONObject currentSignedForeignFeeDataJsonWrapper = new JSONObject(); + currentSignedForeignFeeDataJsonWrapper.put("type", SIGNED_FOREIGN_FEES_TYPE); + currentSignedForeignFeeDataJsonWrapper.put("dataset", CURRENT_DATASET_LABEL); + currentSignedForeignFeeDataJsonWrapper.put("data", currentSignedForeignFeeDataJson); + + // write signed fee data to backup file + String fileName = Paths.get(backupDirectory.toString(), SIGNED_FOREIGN_FEES_FILE_NAME).toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(currentSignedForeignFeeDataJsonWrapper.toString(2)); + writer.close(); + + } + catch (DataException e) { + throw new DataException("Unable to export foreign fee signatures."); + } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Backup Foreign Fee Data + * + * @param feeGetter the fee from the Bitcoiny instance + * @param filename the backup file name + * @param type the type of fee label + * + * @throws DataException + */ + private void backupForeignFeeData(Function feeGetter, String filename, String type) throws DataException { + try { + Path backupDirectory = HSQLDBImportExport.getExportDirectory(true); + + // get the names of the supported blockchains + List names + = Arrays.stream(SupportedBlockchain.values()) + .map( value -> value.getLatestAcct().getClass().getSimpleName()) + .collect(Collectors.toList()); + + // start list of required foreign fees + List foreignFees = new ArrayList<>(names.size()); + + // for each blockchain name, get the blockchain and collect the foreign fee for this node + for( String name : names) { + ForeignBlockchain blockchain = SupportedBlockchain.getAcctByName(name).getBlockchain(); + + // if the blockchain supports fees, add the data to the list + if( blockchain instanceof Bitcoiny ) { + foreignFees.add( new ForeignFeeData(name, feeGetter.apply((Bitcoiny) blockchain)) ); + } + } + + // put the list of fee data into a JSON array + JSONArray currentForeignFeesJson = new JSONArray(); + for (ForeignFeeData foreignFee : foreignFees) { + JSONObject requiredForeignFeeJson = foreignFee.toJson(); + currentForeignFeesJson.put(requiredForeignFeeJson); + } + + // put the JSON array and some metadata into a JSON object + JSONObject currentForeignFeeDataJsonWrapper = new JSONObject(); + currentForeignFeeDataJsonWrapper.put("type", type); + currentForeignFeeDataJsonWrapper.put("dataset", CURRENT_DATASET_LABEL); + currentForeignFeeDataJsonWrapper.put("data", currentForeignFeesJson); + + // write the JSON to the backup file + String fileName = Paths.get(backupDirectory.toString(), filename).toString(); + FileWriter writer = new FileWriter(fileName); + writer.write(currentForeignFeeDataJsonWrapper.toString(2)); + writer.close(); + + } + catch (DataException e) { + throw new DataException("Unable to export required foreign fees."); + } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + /** + * Import Data From File + * + * @param filename the file name + * + * @throws DataException + * @throws IOException + */ + public void importDataFromFile(String filename) throws DataException, IOException { + + // file path and check for existance + Path path = Paths.get(filename); + if (!path.toFile().exists()) { + throw new FileNotFoundException(String.format("File doesn't exist: %s", filename)); + } + + // read in the file + byte[] fileContents = Files.readAllBytes(path); + if (fileContents == null) { + throw new FileNotFoundException(String.format("Unable to read file contents: %s", filename)); + } + + LOGGER.info(String.format("Importing %s into foreign fees manager ...", filename)); + + String jsonString = new String(fileContents); + + // get the data type and data from the JSON + Triple parsedJSON = HSQLDBImportExport.parseJSONString(jsonString); + if (parsedJSON.getA() == null || parsedJSON.getC() == null) { + throw new DataException(String.format("Missing data when importing %s into foreign fees manager", filename)); + } + String type = parsedJSON.getA(); + JSONArray data = parsedJSON.getC(); + + Iterator iterator = data.iterator(); + while(iterator.hasNext()) { + JSONObject dataJsonObject = (JSONObject)iterator.next(); + + if (type.equals(SIGNED_FOREIGN_FEES_TYPE)) { + importSignedForeignFeeDataJSON(dataJsonObject); + } + else if( type.equals(REQUIRED_FOREIGN_FEES_TYPE)) { + importRequiredForeignFeeDataJSON(dataJsonObject); + } + else if( type.equals(LOCKING_FOREIGN_FEES_TYPE)) { + importLockingForeignFeeDataJSON(dataJsonObject); + } + else { + throw new DataException(String.format("Unrecognized data type when importing %s", filename)); + } + } + + LOGGER.info(String.format("Imported %s into foreign fees manager from %s", type, filename)); + } + + /** + * Import Signed Foreign Fee Data JSON + * + * @param signedForeignFeeDataJson the JSON object + * + * @throws DataException + */ + private void importSignedForeignFeeDataJSON(JSONObject signedForeignFeeDataJson) throws DataException { + + ForeignFeeDecodedData signedForeignFeeData = ForeignFeeDecodedData.fromJson(signedForeignFeeDataJson); + + this.signedByAT.put(signedForeignFeeData.getAtAddress(), Optional.of(signedForeignFeeData)); + } + + /** + * Import Required Foreign Fee Data JSON + * + * @param requiredForeignFeeDataJson the JSON object + * + * @throws DataException + */ + private static void importRequiredForeignFeeDataJSON(JSONObject requiredForeignFeeDataJson) throws DataException { + + // the data + ForeignFeeData requiredForeignFeeData = ForeignFeeData.fromJson( requiredForeignFeeDataJson ); + + // the blockchain + ForeignBlockchain blockchain + = SupportedBlockchain + .getAcctByName(requiredForeignFeeData.getBlockchain()) + .getBlockchain(); + + // if the blockchain is Bitcoiny, then get the required fee and set it to blockchain + if( blockchain != null && blockchain instanceof Bitcoiny ) { + ((Bitcoiny) blockchain).setFeeRequired( requiredForeignFeeData.getFee() ); + } + else { + LOGGER.warn("no support for required fee import: blockchain = " + requiredForeignFeeData.getBlockchain()); + } + } + + /** + * Import Locking Foreign Fee Data JSON + * + * @param lockingForeignFeeDataJson the JSON object + * + * @throws DataException + */ + private static void importLockingForeignFeeDataJSON(JSONObject lockingForeignFeeDataJson) throws DataException { + + // get the data + ForeignFeeData lockingForeignFeeData = ForeignFeeData.fromJson(lockingForeignFeeDataJson); + + // get the blockchain + ForeignBlockchain blockchain + = SupportedBlockchain + .getAcctByName(lockingForeignFeeData.getBlockchain()) + .getBlockchain(); + + // if the blockchain is Bitcoiny, then set the locking fee to it + if( blockchain != null && blockchain instanceof Bitcoiny ) { + ((Bitcoiny) blockchain).setFeePerKb(Coin.valueOf(lockingForeignFeeData.getFee())); + } + else { + LOGGER.warn("no support for locking fee import: blockchain = " + lockingForeignFeeData.getBlockchain()); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/Bitcoin.java b/src/main/java/org/qortal/crosschain/Bitcoin.java index 79ab38af..9bda03d9 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoin.java +++ b/src/main/java/org/qortal/crosschain/Bitcoin.java @@ -1,5 +1,6 @@ package org.qortal.crosschain; +import org.bitcoinj.core.Coin; import org.bitcoinj.core.Context; import org.bitcoinj.core.NetworkParameters; import org.bitcoinj.core.Transaction; @@ -14,15 +15,21 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Bitcoin extends Bitcoiny { public static final String CURRENCY_CODE = "BTC"; - private static final long MINIMUM_ORDER_AMOUNT = 100000; // 0.001 BTC minimum order, due to high fees + // Locking fee to lock in a QORT for BTC. This is the default value that the user should reset to + // a value inline with the BTC fee market. This is 5 sats per kB. + private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(5_000); // 0.00005 BTC per 1000 bytes - // Temporary values until a dynamic fee system is written. - private static final long NEW_FEE_AMOUNT = 6_000L; + private static final long MINIMUM_ORDER_AMOUNT = 100_000; // 0.001 BTC minimum order, due to high fees + + // Default value until user resets fee to compete with the current market. This is a total value for a + // p2sh transaction, size 300 kB, 5 sats per kB + private static final long NEW_FEE_AMOUNT = 1_500L; private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST @@ -111,7 +118,7 @@ public class Bitcoin extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -173,14 +180,14 @@ public class Bitcoin extends Bitcoiny { } }; - private long feeCeiling = NEW_FEE_AMOUNT; + private AtomicLong feeRequired = new AtomicLong(NEW_FEE_AMOUNT); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -196,7 +203,7 @@ public class Bitcoin extends Bitcoiny { // Constructors and instance private Bitcoin(BitcoinNet bitcoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) { - super(blockchain, bitcoinjContext, currencyCode, bitcoinjContext.getFeePerKb()); + super(blockchain, bitcoinjContext, currencyCode, DEFAULT_FEE_PER_KB); this.bitcoinNet = bitcoinNet; LOGGER.info(() -> String.format("Starting Bitcoin support using %s", this.bitcoinNet.name())); @@ -242,14 +249,14 @@ public class Bitcoin extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.bitcoinNet.getFeeCeiling(); + public long getFeeRequired() { + return this.bitcoinNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.bitcoinNet.setFeeCeiling( fee ); + this.bitcoinNet.setFeeRequired( fee ); } /** * Returns bitcoinj transaction sending amount to recipient using 20 sat/byte fee. diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index d93fa65f..5cd6df35 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -840,9 +840,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { } while (true); } - public abstract long getFeeCeiling(); + public abstract long getFeeRequired(); - public abstract void setFeeCeiling(long fee); + public abstract void setFeeRequired(long fee); // UTXOProvider support diff --git a/src/main/java/org/qortal/crosschain/BitcoinyTBD.java b/src/main/java/org/qortal/crosschain/BitcoinyTBD.java index c25d2094..4b2f8173 100644 --- a/src/main/java/org/qortal/crosschain/BitcoinyTBD.java +++ b/src/main/java/org/qortal/crosschain/BitcoinyTBD.java @@ -89,7 +89,7 @@ public class BitcoinyTBD extends Bitcoiny { NetTBD netTBD = new NetTBD( bitcoinyTBDRequest.getNetworkName(), - bitcoinyTBDRequest.getFeeCeiling(), + bitcoinyTBDRequest.getFeeRequired(), networkParams, Collections.emptyList(), bitcoinyTBDRequest.getExpectedGenesisHash() @@ -134,18 +134,18 @@ public class BitcoinyTBD extends Bitcoiny { @Override public long getP2shFee(Long timestamp) throws ForeignBlockchainException { - return this.netTBD.getFeeCeiling(); + return this.netTBD.getFeeRequired(); } @Override - public long getFeeCeiling() { + public long getFeeRequired() { - return this.netTBD.getFeeCeiling(); + return this.netTBD.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.netTBD.setFeeCeiling( fee ); + this.netTBD.setFeeRequired( fee ); } } \ No newline at end of file diff --git a/src/main/java/org/qortal/crosschain/Digibyte.java b/src/main/java/org/qortal/crosschain/Digibyte.java index f0a31087..9ee1f06a 100644 --- a/src/main/java/org/qortal/crosschain/Digibyte.java +++ b/src/main/java/org/qortal/crosschain/Digibyte.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Digibyte extends Bitcoiny { @@ -59,7 +60,7 @@ public class Digibyte extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -109,14 +110,14 @@ public class Digibyte extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -178,13 +179,13 @@ public class Digibyte extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.digibyteNet.getFeeCeiling(); + public long getFeeRequired() { + return this.digibyteNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.digibyteNet.setFeeCeiling( fee ); + this.digibyteNet.setFeeRequired( fee ); } } diff --git a/src/main/java/org/qortal/crosschain/Dogecoin.java b/src/main/java/org/qortal/crosschain/Dogecoin.java index dff98b1c..03898b93 100644 --- a/src/main/java/org/qortal/crosschain/Dogecoin.java +++ b/src/main/java/org/qortal/crosschain/Dogecoin.java @@ -13,6 +13,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Dogecoin extends Bitcoiny { @@ -60,7 +61,7 @@ public class Dogecoin extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -110,14 +111,14 @@ public class Dogecoin extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -179,13 +180,13 @@ public class Dogecoin extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.dogecoinNet.getFeeCeiling(); + public long getFeeRequired() { + return this.dogecoinNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.dogecoinNet.setFeeCeiling( fee ); + this.dogecoinNet.setFeeRequired( fee ); } } diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java index f13c1043..7c311b71 100644 --- a/src/main/java/org/qortal/crosschain/Litecoin.java +++ b/src/main/java/org/qortal/crosschain/Litecoin.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Litecoin extends Bitcoiny { @@ -63,7 +64,7 @@ public class Litecoin extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -116,14 +117,14 @@ public class Litecoin extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -185,13 +186,13 @@ public class Litecoin extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.litecoinNet.getFeeCeiling(); + public long getFeeRequired() { + return this.litecoinNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.litecoinNet.setFeeCeiling( fee ); + this.litecoinNet.setFeeRequired( fee ); } } diff --git a/src/main/java/org/qortal/crosschain/NetTBD.java b/src/main/java/org/qortal/crosschain/NetTBD.java index c52449b4..21e0f106 100644 --- a/src/main/java/org/qortal/crosschain/NetTBD.java +++ b/src/main/java/org/qortal/crosschain/NetTBD.java @@ -3,18 +3,19 @@ package org.qortal.crosschain; import org.bitcoinj.core.NetworkParameters; import java.util.Collection; +import java.util.concurrent.atomic.AtomicLong; public class NetTBD { private String name; - private long feeCeiling; + private AtomicLong feeRequired; private NetworkParameters params; private Collection servers; private String genesisHash; - public NetTBD(String name, long feeCeiling, NetworkParameters params, Collection servers, String genesisHash) { + public NetTBD(String name, long feeRequired, NetworkParameters params, Collection servers, String genesisHash) { this.name = name; - this.feeCeiling = feeCeiling; + this.feeRequired = new AtomicLong(feeRequired); this.params = params; this.servers = servers; this.genesisHash = genesisHash; @@ -25,14 +26,14 @@ public class NetTBD { return this.name; } - public long getFeeCeiling() { + public long getFeeRequired() { - return feeCeiling; + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { + public void setFeeRequired(long feeRequired) { - this.feeCeiling = feeCeiling; + this.feeRequired.set(feeRequired); } public NetworkParameters getParams() { diff --git a/src/main/java/org/qortal/crosschain/PirateChain.java b/src/main/java/org/qortal/crosschain/PirateChain.java index 5475c929..48178f28 100644 --- a/src/main/java/org/qortal/crosschain/PirateChain.java +++ b/src/main/java/org/qortal/crosschain/PirateChain.java @@ -21,6 +21,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.*; +import java.util.concurrent.atomic.AtomicLong; public class PirateChain extends Bitcoiny { @@ -67,7 +68,7 @@ public class PirateChain extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -117,14 +118,14 @@ public class PirateChain extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -186,14 +187,14 @@ public class PirateChain extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.pirateChainNet.getFeeCeiling(); + public long getFeeRequired() { + return this.pirateChainNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.pirateChainNet.setFeeCeiling( fee ); + this.pirateChainNet.setFeeRequired( fee ); } /** * Returns confirmed balance, based on passed payment script. diff --git a/src/main/java/org/qortal/crosschain/Ravencoin.java b/src/main/java/org/qortal/crosschain/Ravencoin.java index cd98fb69..08b9be32 100644 --- a/src/main/java/org/qortal/crosschain/Ravencoin.java +++ b/src/main/java/org/qortal/crosschain/Ravencoin.java @@ -14,6 +14,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.EnumMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; public class Ravencoin extends Bitcoiny { @@ -61,7 +62,7 @@ public class Ravencoin extends Bitcoiny { @Override public long getP2shFee(Long timestamp) { - return this.getFeeCeiling(); + return this.getFeeRequired(); } }, TEST3 { @@ -111,14 +112,14 @@ public class Ravencoin extends Bitcoiny { } }; - private long feeCeiling = MAINNET_FEE; + private AtomicLong feeRequired = new AtomicLong( MAINNET_FEE ); - public long getFeeCeiling() { - return feeCeiling; + public long getFeeRequired() { + return feeRequired.get(); } - public void setFeeCeiling(long feeCeiling) { - this.feeCeiling = feeCeiling; + public void setFeeRequired(long feeRequired) { + this.feeRequired.set(feeRequired); } public abstract NetworkParameters getParams(); @@ -180,13 +181,13 @@ public class Ravencoin extends Bitcoiny { } @Override - public long getFeeCeiling() { - return this.ravencoinNet.getFeeCeiling(); + public long getFeeRequired() { + return this.ravencoinNet.getFeeRequired(); } @Override - public void setFeeCeiling(long fee) { + public void setFeeRequired(long fee) { - this.ravencoinNet.setFeeCeiling( fee ); + this.ravencoinNet.setFeeRequired( fee ); } } diff --git a/src/main/java/org/qortal/data/crosschain/ForeignFeeData.java b/src/main/java/org/qortal/data/crosschain/ForeignFeeData.java new file mode 100644 index 00000000..1b0e1199 --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/ForeignFeeData.java @@ -0,0 +1,57 @@ +package org.qortal.data.crosschain; + +import org.json.JSONObject; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class ForeignFeeData { + + private String blockchain; + + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long fee; + + protected ForeignFeeData() { + /* JAXB */ + } + + public ForeignFeeData(String blockchain, + long fee) { + this.blockchain = blockchain; + this.fee = fee; + } + + public String getBlockchain() { + return this.blockchain; + } + + public long getFee() { + return this.fee; + } + + public JSONObject toJson() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("blockchain", this.getBlockchain()); + jsonObject.put("fee", this.getFee()); + return jsonObject; + } + + public static ForeignFeeData fromJson(JSONObject json) { + return new ForeignFeeData( + json.isNull("blockchain") ? null : json.getString("blockchain"), + json.isNull("fee") ? null : json.getLong("fee") + ); + } + + @Override + public String toString() { + return "ForeignFeeData{" + + "blockchain='" + blockchain + '\'' + + ", fee=" + fee + + '}'; + } +} diff --git a/src/main/java/org/qortal/data/crosschain/ForeignFeeDecodedData.java b/src/main/java/org/qortal/data/crosschain/ForeignFeeDecodedData.java new file mode 100644 index 00000000..de5bddc9 --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/ForeignFeeDecodedData.java @@ -0,0 +1,90 @@ +package org.qortal.data.crosschain; + +import org.json.JSONObject; +import org.qortal.data.account.MintingAccountData; +import org.qortal.utils.Base58; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class ForeignFeeDecodedData { + + protected long timestamp; + protected byte[] data; + protected String atAddress; + protected Integer fee; + + // Constructors + + // necessary for JAXB serialization + protected ForeignFeeDecodedData() { + } + + public ForeignFeeDecodedData(long timestamp, byte[] data, String atAddress, Integer fee) { + this.timestamp = timestamp; + this.data = data; + this.atAddress = atAddress; + this.fee = fee; + } + + public long getTimestamp() { + return this.timestamp; + } + + public byte[] getData() { + return this.data; + } + + public String getAtAddress() { + return atAddress; + } + + public Integer getFee() { + return this.fee; + } + + // Comparison + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ForeignFeeDecodedData that = (ForeignFeeDecodedData) o; + return timestamp == that.timestamp && Objects.equals(atAddress, that.atAddress) && Objects.equals(fee, that.fee); + } + + @Override + public int hashCode() { + return Objects.hash(timestamp, atAddress, fee); + } + + @Override + public String toString() { + return "ForeignFeeDecodedData{" + + "timestamp=" + timestamp + + ", atAddress='" + atAddress + '\'' + + ", fee=" + fee + + '}'; + } + + public JSONObject toJson() { + JSONObject jsonObject = new JSONObject(); + jsonObject.put("data", Base58.encode(this.data)); + jsonObject.put("atAddress", this.atAddress); + jsonObject.put("timestamp", this.timestamp); + jsonObject.put("fee", this.fee); + return jsonObject; + } + + public static ForeignFeeDecodedData fromJson(JSONObject json) { + return new ForeignFeeDecodedData( + json.isNull("timestamp") ? null : json.getLong("timestamp"), + json.isNull("data") ? null : Base58.decode(json.getString("data")), + json.isNull("atAddress") ? null : json.getString("atAddress"), + json.isNull("fee") ? null : json.getInt("fee")); + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/data/crosschain/ForeignFeeEncodedData.java b/src/main/java/org/qortal/data/crosschain/ForeignFeeEncodedData.java new file mode 100644 index 00000000..f0a3db2d --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/ForeignFeeEncodedData.java @@ -0,0 +1,69 @@ +package org.qortal.data.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Objects; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class ForeignFeeEncodedData { + + protected long timestamp; + protected String data; + protected String atAddress; + protected Integer fee; + + // Constructors + + // necessary for JAXB serialization + protected ForeignFeeEncodedData() { + } + + public ForeignFeeEncodedData(long timestamp, String data, String atAddress, Integer fee) { + this.timestamp = timestamp; + this.data = data; + this.atAddress = atAddress; + this.fee = fee; + } + + public long getTimestamp() { + return this.timestamp; + } + + public String getData() { + return this.data; + } + + public String getAtAddress() { + return atAddress; + } + + public Integer getFee() { + return this.fee; + } + + // Comparison + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ForeignFeeEncodedData that = (ForeignFeeEncodedData) o; + return timestamp == that.timestamp && Objects.equals(atAddress, that.atAddress) && Objects.equals(fee, that.fee); + } + + @Override + public int hashCode() { + return Objects.hash(timestamp, atAddress, fee); + } + + @Override + public String toString() { + return "ForeignFeeDecodedData{" + + "timestamp=" + timestamp + + ", atAddress='" + atAddress + '\'' + + ", fee=" + fee + + '}'; + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java b/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java new file mode 100644 index 00000000..354a7f36 --- /dev/null +++ b/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java @@ -0,0 +1,8 @@ +package org.qortal.data.crosschain; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class UnsignedFeeEvent { +} diff --git a/src/main/java/org/qortal/event/FeeWaitingEvent.java b/src/main/java/org/qortal/event/FeeWaitingEvent.java new file mode 100644 index 00000000..1706b93e --- /dev/null +++ b/src/main/java/org/qortal/event/FeeWaitingEvent.java @@ -0,0 +1,26 @@ +package org.qortal.event; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class FeeWaitingEvent implements Event{ + private long timestamp; + private String address; + + public FeeWaitingEvent() { + } + + public FeeWaitingEvent(long timestamp, String address) { + this.timestamp = timestamp; + this.address = address; + } + + public long getTimestamp() { + return timestamp; + } + + public String getAddress() { + return address; + } +} diff --git a/src/main/java/org/qortal/event/LockingFeeUpdateEvent.java b/src/main/java/org/qortal/event/LockingFeeUpdateEvent.java new file mode 100644 index 00000000..783b442e --- /dev/null +++ b/src/main/java/org/qortal/event/LockingFeeUpdateEvent.java @@ -0,0 +1,4 @@ +package org.qortal.event; + +public class LockingFeeUpdateEvent implements Event{ +} diff --git a/src/main/java/org/qortal/event/RequiredFeeUpdateEvent.java b/src/main/java/org/qortal/event/RequiredFeeUpdateEvent.java new file mode 100644 index 00000000..a0526196 --- /dev/null +++ b/src/main/java/org/qortal/event/RequiredFeeUpdateEvent.java @@ -0,0 +1,15 @@ +package org.qortal.event; + +import org.qortal.crosschain.Bitcoiny; + +public class RequiredFeeUpdateEvent implements Event{ + private final Bitcoiny bitcoiny; + + public RequiredFeeUpdateEvent(Bitcoiny bitcoiny) { + this.bitcoiny = bitcoiny; + } + + public Bitcoiny getBitcoiny() { + return bitcoiny; + } +} diff --git a/src/main/java/org/qortal/network/message/ForeignFeesMessage.java b/src/main/java/org/qortal/network/message/ForeignFeesMessage.java new file mode 100644 index 00000000..d7d0eabc --- /dev/null +++ b/src/main/java/org/qortal/network/message/ForeignFeesMessage.java @@ -0,0 +1,43 @@ +package org.qortal.network.message; + +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.utils.ForeignFeesMessageUtils; + +import java.nio.ByteBuffer; +import java.util.List; + +/** + * For sending online accounts info to remote peer. + * + * Same format as V2, but with added support for a mempow nonce. + */ +public class ForeignFeesMessage extends Message { + + public static final long MIN_PEER_VERSION = 0x300060000L; // 3.6.0 + + private List foreignFees; + + public ForeignFeesMessage(List foreignFeeDecodedData) { + super(MessageType.FOREIGN_FEES); + + this.dataBytes = ForeignFeesMessageUtils.fromDataToSendBytes(foreignFeeDecodedData); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private ForeignFeesMessage(int id, List foreignFees) { + super(id, MessageType.ONLINE_ACCOUNTS_V3); + + this.foreignFees = foreignFees; + } + + public List getForeignFees() { + return this.foreignFees; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + List foreignFeeDecodedData = ForeignFeesMessageUtils.fromSendBytesToData(bytes); + + return new ForeignFeesMessage(id, foreignFeeDecodedData); + } + +} diff --git a/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java b/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java new file mode 100644 index 00000000..1501097d --- /dev/null +++ b/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java @@ -0,0 +1,46 @@ +package org.qortal.network.message; + +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.utils.ForeignFeesMessageUtils; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class GetForeignFeesMessage extends Message { + + private static final Map> EMPTY_ONLINE_ACCOUNTS = Collections.emptyMap(); + private final List foreignFeeDecodedData; + + public GetForeignFeesMessage(List foreignFeeDecodedData) { + super(MessageType.GET_ONLINE_ACCOUNTS_V3); + + this.foreignFeeDecodedData = foreignFeeDecodedData; + + // If we don't have ANY online accounts then it's an easier construction... + if (foreignFeeDecodedData.isEmpty()) { + this.dataBytes = EMPTY_DATA_BYTES; + return; + } + + this.dataBytes = ForeignFeesMessageUtils.fromDataToGetBytes(foreignFeeDecodedData); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private GetForeignFeesMessage(int id, List foreignFeeDecodedData) { + super(id, MessageType.GET_FOREIGN_FEES); + + this.foreignFeeDecodedData = foreignFeeDecodedData; + } + + public List getForeignFeeData() { + return foreignFeeDecodedData; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + + return new GetForeignFeesMessage(id, ForeignFeesMessageUtils.fromGetBytesToData(bytes)); + } + +} diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java index 6b420e2d..ca35ece1 100644 --- a/src/main/java/org/qortal/network/message/MessageType.java +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -79,7 +79,10 @@ public enum MessageType { GET_NAME(182, GetNameMessage::fromByteBuffer), TRANSACTIONS(190, TransactionsMessage::fromByteBuffer), - GET_ACCOUNT_TRANSACTIONS(191, GetAccountTransactionsMessage::fromByteBuffer); + GET_ACCOUNT_TRANSACTIONS(191, GetAccountTransactionsMessage::fromByteBuffer), + + FOREIGN_FEES( 200, ForeignFeesMessage::fromByteBuffer), + GET_FOREIGN_FEES( 201, GetForeignFeesMessage::fromByteBuffer); public final int value; public final MessageProducer fromByteBufferMethod; diff --git a/src/main/java/org/qortal/utils/ForeignFeesMessageUtils.java b/src/main/java/org/qortal/utils/ForeignFeesMessageUtils.java new file mode 100644 index 00000000..67e78439 --- /dev/null +++ b/src/main/java/org/qortal/utils/ForeignFeesMessageUtils.java @@ -0,0 +1,187 @@ +package org.qortal.utils; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.transform.Transformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import static org.qortal.transform.Transformer.ADDRESS_LENGTH; + +/** + * Class ForeignFeesMessageUtils + */ +public class ForeignFeesMessageUtils { + + private static final Logger LOGGER = LogManager.getLogger(ForeignFeesMessageUtils.class); + + /** + * From Data To Send Bytes + * + * Convert foreign fee data into bytes for send messages. + * + * @param foreignFees the data + * + * @return the bytes + */ + public static byte[] fromDataToSendBytes(List foreignFees) { + + return fromDataToBytes(foreignFees, true); + } + + /** + * From Data To Bytes + * + * @param foreignFees + * @param includeSignature + * @return + */ + private static byte[] fromDataToBytes(List foreignFees, boolean includeSignature) { + try { + if (foreignFees.isEmpty()) { + return new byte[0]; + } + else { + // allocate size for each data item for timestamp, AT address, fee and signature + int byteSize + = foreignFees.size() + * + (Transformer.TIMESTAMP_LENGTH + Transformer.ADDRESS_LENGTH + Transformer.INT_LENGTH + Transformer.SIGNATURE_LENGTH); + + if( includeSignature ) byteSize += foreignFees.size() * Transformer.SIGNATURE_LENGTH; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + // for each foreign fee data item, convert to bytes and fill the array + for( ForeignFeeDecodedData feeData : foreignFees) { + bytes.write(Longs.toByteArray(feeData.getTimestamp())); + bytes.write(Base58.decode(feeData.getAtAddress())); + bytes.write(Ints.toByteArray(feeData.getFee())); + if( includeSignature ) bytes.write(feeData.getData()); + } + return bytes.toByteArray(); + } + } catch (Exception e) { + LOGGER.warn(e.getMessage()); + + return new byte[0]; + } + } + + /** + * From Send Bytes to Data + * + * @param bytes the bytes to convert to data + * + * @return the data + */ + public static List fromSendBytesToData(ByteBuffer bytes) { + + return fromBytesToData(bytes, true); + } + + /** + * From Bytes To Data + * + * @param bytes the bytes + * @param includeSignature true if the bytes include signatures + * + * @return the foreign fee data with signatures (data member) + */ + private static List fromBytesToData(ByteBuffer bytes, boolean includeSignature) { + if( !bytes.hasRemaining() ) return new ArrayList<>(0); + + List foreignFees = new ArrayList<>(); + + try { + while (bytes.hasRemaining()) { + // read in the timestamp as a long + long timestamp = bytes.getLong(); + + // read in the address as a byte array with a predetermined length + byte[] atAddressBytes = new byte[ADDRESS_LENGTH]; + bytes.get(atAddressBytes); + String atAddress = Base58.encode(atAddressBytes); + + // rwad in the fee as an integer + int fee = bytes.getInt(); + + byte[] signature; + + if( includeSignature ) { + signature = new byte[Transformer.SIGNATURE_LENGTH]; + bytes.get(signature); + } + else { + signature = null; + } + + foreignFees.add(new ForeignFeeDecodedData(timestamp, signature, atAddress, fee)); + + } + } + // if there are any exception, log the error as a warning and clear the list before returning it + catch (Exception e) { + LOGGER.warn(e.getMessage()); + foreignFees.clear(); + } + + return foreignFees; + } + + /** + * From Data To Get Bytes + * + * Convert foreign fees data objects into get foreign fees messages. Get messages do not include signatures. + * + * @param foreignFees the foreign fees objects + * + * @return the messages + */ + public static byte[] fromDataToGetBytes(List foreignFees) { + return fromDataToBytes(foreignFees, false); + } + + /** + * From Get Bytes to Data + * + * Convert bytes from get foreign fees messages to foreign fees objects. Get messages do not include signatures. + * + * @param bytes the bytes to convert + * + * @return the foreign fees data objects + */ + public static List fromGetBytesToData(ByteBuffer bytes) { + return fromBytesToData(bytes, false); + } + + /** + * Build Foreign Fees Data Message + * + * Build the unsigned message for the foreign fees data objects. + * + * @param timestamp the timestamp in milliseconds + * @param atAddress the AT address + * @param fee the fee + * @return + * @throws IOException + */ + public static byte[] buildForeignFeesDataMessage(Long timestamp, String atAddress, int fee) throws IOException { + int byteSize = Transformer.TIMESTAMP_LENGTH + Transformer.ADDRESS_LENGTH + Transformer.INT_LENGTH; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + bytes.write(Longs.toByteArray(timestamp)); + bytes.write(Base58.decode(atAddress)); + bytes.write(Ints.toByteArray(fee)); + + return bytes.toByteArray(); + } +} diff --git a/src/test/java/org/qortal/test/network/message/ForeignFeesMessageTests.java b/src/test/java/org/qortal/test/network/message/ForeignFeesMessageTests.java new file mode 100644 index 00000000..9a2b0e7f --- /dev/null +++ b/src/test/java/org/qortal/test/network/message/ForeignFeesMessageTests.java @@ -0,0 +1,334 @@ +package org.qortal.test.network.message; + +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.junit.Assert; +import org.junit.Test; +import org.qortal.crypto.Crypto; +import org.qortal.data.crosschain.ForeignFeeDecodedData; +import org.qortal.test.utils.TestUtils; +import org.qortal.transform.Transformer; +import org.qortal.utils.Base58; +import org.qortal.utils.ForeignFeesMessageUtils; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.KeyPair; +import java.security.Security; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +/** + * Class ForeignFeesMessageTests + */ +public class ForeignFeesMessageTests { + + /** + * Random + * + * Random input generator for seeding keys/addresses. + */ + private static final Random RANDOM = new Random(); + + static { + // add the Bouncy Castle provider for keys/addresses + Security.addProvider(new BouncyCastleProvider()); + } + + @Test + public void testDataToSendBytesToDataEmpty() { + + byte[] bytes = ForeignFeesMessageUtils.fromDataToSendBytes(new ArrayList<>(0)); + + List list = ForeignFeesMessageUtils.fromSendBytesToData(ByteBuffer.wrap(bytes)); + + Assert.assertNotNull(list); + + Assert.assertEquals(0, list.size()); + } + + @Test + public void testDataToGetBytesToDataEmpty() { + + byte[] bytes = ForeignFeesMessageUtils.fromDataToGetBytes(new ArrayList<>(0)); + + List list = ForeignFeesMessageUtils.fromGetBytesToData(ByteBuffer.wrap(bytes)); + + Assert.assertNotNull(list); + + Assert.assertEquals(0, list.size()); + } + + @Test + public void testSignature() { + + boolean exceptionThrown = false; + + try { + KeyPair keyPair = TestUtils.generateKeyPair(); + + long timestamp = 1_000_000L; + String atAddress = generateAtAddress(); + int fee = 1; + + assertSignature(keyPair, timestamp, atAddress, fee); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + /** + * Assert Signature + * + * @param keyPair the key pair that is signing + * @param timestamp the timestamp for the data + * @param atAddress the AT address + * @param fee the fee + * + * @return the signature bytes + * + * @throws IOException + */ + private static byte[] assertSignature(KeyPair keyPair, long timestamp, String atAddress, int fee) throws IOException { + + // build the message and sign it + byte[] message = ForeignFeesMessageUtils.buildForeignFeesDataMessage(timestamp, atAddress, fee); + byte[] signature = Crypto.sign( keyPair.getPrivate().getEncoded(), message ); + + // assert signaute length + Assert.assertEquals(Transformer.SIGNATURE_LENGTH, signature.length); + + // assert verification + boolean verified = Crypto.verify(Crypto.toPublicKey(keyPair.getPrivate().getEncoded()), signature, message); + Assert.assertTrue(verified); + + return signature; + } + + @Test + public void testDataToSendBytesToDataSingle() { + + Long timestamp = 1_000_000L; + String atAddress = generateAtAddress(); + int fee = 1; + + boolean exceptionThrown = false; + + try { + // random key generation for signing data + KeyPair keyPair = TestUtils.generateKeyPair(); + + // data to send, a list of 1 foreign fee data + List sendData + = List.of( + new ForeignFeeDecodedData(timestamp, assertSignature(keyPair,timestamp,atAddress, fee), atAddress, fee) + ); + + // from data to bytes + byte[] sendBytes = ForeignFeesMessageUtils.fromDataToSendBytes(sendData); + + // from bytes to data + List returnData = ForeignFeesMessageUtils.fromSendBytesToData(ByteBuffer.wrap(sendBytes)); + + + assertListedForeignFees(sendData, returnData, true); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testDataToGetBytesToDataSingle() { + + Long timestamp = 1_000_000L; + String atAddress = generateAtAddress(); + int fee = 1; + + boolean exceptionThrown = false; + + try { + // random key generation for signing data + KeyPair keyPair = TestUtils.generateKeyPair(); + + // data to send, a list of 1 foreign fee data + List sendData + = List.of( + new ForeignFeeDecodedData(timestamp, assertSignature(keyPair,timestamp,atAddress, fee), atAddress, fee) + ); + + // from data to bytes + byte[] sendBytes = ForeignFeesMessageUtils.fromDataToGetBytes(sendData); + + // from bytes to data + List returnData = ForeignFeesMessageUtils.fromGetBytesToData(ByteBuffer.wrap(sendBytes)); + + + assertListedForeignFees(sendData, returnData, false); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testDataToSendBytesToDataTriple() { + + Long timestamp1 = 1_000_000L; + String atAddress1 = generateAtAddress(); + int fee1 = 1; + + Long timestamp2 = 2_000_000L; + String atAddress2 = generateAtAddress(); + int fee2 = 2; + + Long timestamp3 = 5_000_000L; + String atAddress3 = generateAtAddress(); + int fee3 = 3; + + boolean exceptionThrown = false; + + try { + // random key generation for signing data + KeyPair keyPair1 = TestUtils.generateKeyPair(); + KeyPair keyPair2 = TestUtils.generateKeyPair(); + + // data to send, a list of 3 foreign fee data + List sendData + = List.of( + new ForeignFeeDecodedData(timestamp1, assertSignature(keyPair1,timestamp1,atAddress1, fee1), atAddress1, fee1), + new ForeignFeeDecodedData(timestamp2, assertSignature(keyPair1,timestamp2,atAddress2, fee2), atAddress2, fee2), + new ForeignFeeDecodedData(timestamp3, assertSignature(keyPair2,timestamp3,atAddress3, fee3), atAddress3, fee3) + ); + + // from data to bytes + byte[] sendBytes = ForeignFeesMessageUtils.fromDataToSendBytes(sendData); + + // from bytes to data + List returnData = ForeignFeesMessageUtils.fromSendBytesToData(ByteBuffer.wrap(sendBytes)); + + + assertListedForeignFees(sendData, returnData, true); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + @Test + public void testDataToGetBytesToDataTriple() { + + Long timestamp1 = 1_000_000L; + String atAddress1 = generateAtAddress(); + int fee1 = 1; + + Long timestamp2 = 2_000_000L; + String atAddress2 = generateAtAddress(); + int fee2 = 2; + + Long timestamp3 = 5_000_000L; + String atAddress3 = generateAtAddress(); + int fee3 = 3; + + boolean exceptionThrown = false; + + try { + // random key generation for signing data + KeyPair keyPair1 = TestUtils.generateKeyPair(); + KeyPair keyPair2 = TestUtils.generateKeyPair(); + + // data to send, a list of 3 foreign fee data + List sendData + = List.of( + new ForeignFeeDecodedData(timestamp1, assertSignature(keyPair1,timestamp1,atAddress1, fee1), atAddress1, fee1), + new ForeignFeeDecodedData(timestamp2, assertSignature(keyPair1,timestamp2,atAddress2, fee2), atAddress2, fee2), + new ForeignFeeDecodedData(timestamp3, assertSignature(keyPair2,timestamp3,atAddress3, fee3), atAddress3, fee3) + ); + + // from data to bytes + byte[] sendBytes = ForeignFeesMessageUtils.fromDataToGetBytes(sendData); + + // from bytes to data + List returnData = ForeignFeesMessageUtils.fromGetBytesToData(ByteBuffer.wrap(sendBytes)); + + + assertListedForeignFees(sendData, returnData, false); + + } catch (Exception e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + } + + /** + * Assert Listed Foreign Fees + * + * @param expectedList + * @param actualList + * @param includeSignature + */ + private static void assertListedForeignFees(List expectedList, List actualList, boolean includeSignature) { + + int expectedSize = expectedList.size(); + + // basic assertions on return data + Assert.assertNotNull(actualList); + Assert.assertEquals(expectedSize, actualList.size()); + + for( int index = 0; index < expectedSize; index++ ) { + // expected and actual fee data + ForeignFeeDecodedData expected = expectedList.get(index); + ForeignFeeDecodedData actual = actualList.get(index); + + assertForeignFeeEquality(expected, actual, includeSignature); + } + } + + /** + * Assert Foreign Fee Equality + * + * @param expected the expected data, for comparison + * @param actual the actual data, the response to evaluate + * @param includeSignature + */ + private static void assertForeignFeeEquality(ForeignFeeDecodedData expected, ForeignFeeDecodedData actual, boolean includeSignature) { + // assert + Assert.assertEquals(expected, actual); + + if( includeSignature ) { + // get the data members of each, since the data members are not part of the object comparison above + byte[] expectedData = expected.getData(); + byte[] actualData = actual.getData(); + + // assert data members, must encode them to strings for comparisons + Assert.assertNotNull(actualData); + Assert.assertEquals(Base58.encode(expectedData), Base58.encode(actualData)); + } + } + + /** + * Generate AT Address + * + * Generate AT address using a random inpute seed. + * + * @return the AT address + */ + private static String generateAtAddress() { + + byte[] signature = new byte[64]; + RANDOM.nextBytes(signature); + String atAddress = Crypto.toATAddress(signature); + + return atAddress; + } +} \ No newline at end of file From 1f6ee72fc5c225031221472882de0275ff394b71 Mon Sep 17 00:00:00 2001 From: kennycud Date: Sat, 26 Apr 2025 09:58:13 -0700 Subject: [PATCH 02/42] the message types were corrected --- .../java/org/qortal/network/message/ForeignFeesMessage.java | 2 +- .../java/org/qortal/network/message/GetForeignFeesMessage.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/network/message/ForeignFeesMessage.java b/src/main/java/org/qortal/network/message/ForeignFeesMessage.java index d7d0eabc..0b7d66d6 100644 --- a/src/main/java/org/qortal/network/message/ForeignFeesMessage.java +++ b/src/main/java/org/qortal/network/message/ForeignFeesMessage.java @@ -25,7 +25,7 @@ public class ForeignFeesMessage extends Message { } private ForeignFeesMessage(int id, List foreignFees) { - super(id, MessageType.ONLINE_ACCOUNTS_V3); + super(id, MessageType.FOREIGN_FEES); this.foreignFees = foreignFees; } diff --git a/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java b/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java index 1501097d..b746cc65 100644 --- a/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java +++ b/src/main/java/org/qortal/network/message/GetForeignFeesMessage.java @@ -14,7 +14,7 @@ public class GetForeignFeesMessage extends Message { private final List foreignFeeDecodedData; public GetForeignFeesMessage(List foreignFeeDecodedData) { - super(MessageType.GET_ONLINE_ACCOUNTS_V3); + super(MessageType.GET_FOREIGN_FEES); this.foreignFeeDecodedData = foreignFeeDecodedData; From 17b2bf384841dc1a5cecb5262a6493e8e98d5259 Mon Sep 17 00:00:00 2001 From: kennycud Date: Sat, 26 Apr 2025 17:53:41 -0700 Subject: [PATCH 03/42] added logging and added positive boolean to the fee waiting and unsigned fee events --- .../api/resource/CrossChainResource.java | 6 +++++- .../api/websocket/UnsignedFeesSocket.java | 6 ++++-- .../qortal/controller/ForeignFeesManager.java | 12 +++++++++--- .../data/crosschain/UnsignedFeeEvent.java | 13 +++++++++++++ .../org/qortal/event/FeeWaitingEvent.java | 19 ++++++++----------- 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java index 8104aabe..7ee7878c 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainResource.java @@ -432,7 +432,11 @@ public class CrossChainResource { @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) public List getUnsignedFees(@PathParam("address") String address) { - return ForeignFeesManager.getInstance().getUnsignedFeesForAddress(address); + List unsignedFeesForAddress = ForeignFeesManager.getInstance().getUnsignedFeesForAddress(address); + + LOGGER.info("address = " + address); + LOGGER.info("returning unsigned = " + unsignedFeesForAddress); + return unsignedFeesForAddress; } @GET diff --git a/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java b/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java index 678b12b6..8da0c217 100644 --- a/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java +++ b/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java @@ -39,8 +39,10 @@ public class UnsignedFeesSocket extends ApiWebSocket implements Listener { if (!(event instanceof FeeWaitingEvent)) return; - for (Session session : getSessions()) - sendUnsignedFeeEvent(session, new UnsignedFeeEvent()); + for (Session session : getSessions()) { + boolean positive = ((FeeWaitingEvent) event).isPositive(); + sendUnsignedFeeEvent(session, new UnsignedFeeEvent(positive)); + } } diff --git a/src/main/java/org/qortal/controller/ForeignFeesManager.java b/src/main/java/org/qortal/controller/ForeignFeesManager.java index 7bae7aa9..75cd21ed 100644 --- a/src/main/java/org/qortal/controller/ForeignFeesManager.java +++ b/src/main/java/org/qortal/controller/ForeignFeesManager.java @@ -258,7 +258,7 @@ public class ForeignFeesManager implements Listener { this.needToBackupRequiredForeignFees.compareAndSet(false, true); if( processLocalForeignFeesForCoin(((RequiredFeeUpdateEvent) event).getBitcoiny()) ) { - EventBus.INSTANCE.notify(new FeeWaitingEvent()); + EventBus.INSTANCE.notify(new FeeWaitingEvent(true)); } } // @@ -283,7 +283,7 @@ public class ForeignFeesManager implements Listener { this.offersByAddress.computeIfAbsent( offer.qortalCreator, x -> new ArrayList<>()).add(offer); if( processTradeOfferInWaiting(now, data) ) { - EventBus.INSTANCE.notify(new FeeWaitingEvent()); + EventBus.INSTANCE.notify(new FeeWaitingEvent(true)); } } else { @@ -451,6 +451,10 @@ public class ForeignFeesManager implements Listener { // now that this fee has been processed, remove it from the process queue foreignFeesToRemove.add(foreignFeeToImport); } + + if( this.unsignedByAT.isEmpty() ) { + EventBus.INSTANCE.notify(new FeeWaitingEvent(false)); + } } catch (Exception e) { LOGGER.error("Repository issue while verifying foreign fees", e); } finally { @@ -698,7 +702,7 @@ public class ForeignFeesManager implements Listener { } if( feeSignaturesNeeded ) { - EventBus.INSTANCE.notify(new FeeWaitingEvent()); + EventBus.INSTANCE.notify(new FeeWaitingEvent(true)); } } @@ -860,7 +864,9 @@ public class ForeignFeesManager implements Listener { fee ); + LOGGER.info("updating unsigned"); this.unsignedByAT.put(atAddress, feeData); + LOGGER.info("updated unsigned = " + this.unsignedByAT); } // Network handlers diff --git a/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java b/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java index 354a7f36..ea883515 100644 --- a/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java +++ b/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java @@ -5,4 +5,17 @@ import javax.xml.bind.annotation.XmlAccessorType; @XmlAccessorType(XmlAccessType.FIELD) public class UnsignedFeeEvent { + + private boolean positive; + + public UnsignedFeeEvent() { + } + + public UnsignedFeeEvent(boolean positive) { + this.positive = positive; + } + + public boolean isPositive() { + return positive; + } } diff --git a/src/main/java/org/qortal/event/FeeWaitingEvent.java b/src/main/java/org/qortal/event/FeeWaitingEvent.java index 1706b93e..4d4d86ab 100644 --- a/src/main/java/org/qortal/event/FeeWaitingEvent.java +++ b/src/main/java/org/qortal/event/FeeWaitingEvent.java @@ -5,22 +5,19 @@ import javax.xml.bind.annotation.XmlAccessorType; @XmlAccessorType(XmlAccessType.FIELD) public class FeeWaitingEvent implements Event{ - private long timestamp; - private String address; + + private boolean positive; public FeeWaitingEvent() { } - public FeeWaitingEvent(long timestamp, String address) { - this.timestamp = timestamp; - this.address = address; + public FeeWaitingEvent(boolean positive) { + + this.positive = positive; + } - public long getTimestamp() { - return timestamp; - } - - public String getAddress() { - return address; + public boolean isPositive() { + return positive; } } From 4b74bb37dc3acc7230402bd34d5b4ca254f463c3 Mon Sep 17 00:00:00 2001 From: kennycud Date: Sun, 27 Apr 2025 15:02:28 -0700 Subject: [PATCH 04/42] unsigned fee event handling now provides address --- .../api/websocket/UnsignedFeesSocket.java | 4 +- .../qortal/controller/ForeignFeesManager.java | 48 ++++++++++++------- .../data/crosschain/UnsignedFeeEvent.java | 10 +++- .../org/qortal/event/FeeWaitingEvent.java | 9 +++- 4 files changed, 50 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java b/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java index 8da0c217..c19c2227 100644 --- a/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java +++ b/src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java @@ -40,8 +40,8 @@ public class UnsignedFeesSocket extends ApiWebSocket implements Listener { return; for (Session session : getSessions()) { - boolean positive = ((FeeWaitingEvent) event).isPositive(); - sendUnsignedFeeEvent(session, new UnsignedFeeEvent(positive)); + FeeWaitingEvent feeWaitingEvent = (FeeWaitingEvent) event; + sendUnsignedFeeEvent(session, new UnsignedFeeEvent(feeWaitingEvent.isPositive(), feeWaitingEvent.getAddress())); } } diff --git a/src/main/java/org/qortal/controller/ForeignFeesManager.java b/src/main/java/org/qortal/controller/ForeignFeesManager.java index 75cd21ed..c2ffa70e 100644 --- a/src/main/java/org/qortal/controller/ForeignFeesManager.java +++ b/src/main/java/org/qortal/controller/ForeignFeesManager.java @@ -257,8 +257,8 @@ public class ForeignFeesManager implements Listener { this.needToBackupRequiredForeignFees.compareAndSet(false, true); - if( processLocalForeignFeesForCoin(((RequiredFeeUpdateEvent) event).getBitcoiny()) ) { - EventBus.INSTANCE.notify(new FeeWaitingEvent(true)); + for( String address : processLocalForeignFeesForCoin(((RequiredFeeUpdateEvent) event).getBitcoiny()) ) { + EventBus.INSTANCE.notify(new FeeWaitingEvent(true, address)); } } // @@ -283,7 +283,7 @@ public class ForeignFeesManager implements Listener { this.offersByAddress.computeIfAbsent( offer.qortalCreator, x -> new ArrayList<>()).add(offer); if( processTradeOfferInWaiting(now, data) ) { - EventBus.INSTANCE.notify(new FeeWaitingEvent(true)); + EventBus.INSTANCE.notify(new FeeWaitingEvent(true, data.getCreatorAddress())); } } else { @@ -434,6 +434,22 @@ public class ForeignFeesManager implements Listener { this.signedByAT.put(atAddress, Optional.of(foreignFeeToImport)); this.needToBackupSignedForeignFees.compareAndSet(false, true); this.unsignedByAT.remove(atAddress); + + String tradeOfferCreatorAddress = Crypto.toAddress(publicKey); + boolean allSignedForCreatorAddress + = this.offersByAddress + .get(tradeOfferCreatorAddress).stream() + .map(data -> data.qortalAtAddress) + .filter(qortalAtAddress -> this.unsignedByAT.contains(qortalAtAddress)) + .findAny() + .isEmpty(); + + LOGGER.info("tradeOfferCreatorAddress = " + tradeOfferCreatorAddress); + LOGGER.info("allSignedForCreatorAddress = " + allSignedForCreatorAddress); + + if(allSignedForCreatorAddress) { + EventBus.INSTANCE.notify(new FeeWaitingEvent(false, tradeOfferCreatorAddress)); + } } // otherwise this fee will get discarded else { @@ -451,10 +467,6 @@ public class ForeignFeesManager implements Listener { // now that this fee has been processed, remove it from the process queue foreignFeesToRemove.add(foreignFeeToImport); } - - if( this.unsignedByAT.isEmpty() ) { - EventBus.INSTANCE.notify(new FeeWaitingEvent(false)); - } } catch (Exception e) { LOGGER.error("Repository issue while verifying foreign fees", e); } finally { @@ -686,7 +698,7 @@ public class ForeignFeesManager implements Listener { */ private void processLocalForeignFeesForAll() { - boolean feeSignaturesNeeded = false; + Set addressesThatNeedSignatures = new HashSet<>(); List names = Arrays.stream(SupportedBlockchain.values()) @@ -697,12 +709,12 @@ public class ForeignFeesManager implements Listener { ForeignBlockchain blockchain = SupportedBlockchain.getAcctByName(name).getBlockchain(); if( blockchain instanceof Bitcoiny ) { - feeSignaturesNeeded = processLocalForeignFeesForCoin((Bitcoiny) blockchain) || feeSignaturesNeeded; + addressesThatNeedSignatures.addAll( processLocalForeignFeesForCoin((Bitcoiny) blockchain) ); } } - if( feeSignaturesNeeded ) { - EventBus.INSTANCE.notify(new FeeWaitingEvent(true)); + for( String addressThatNeedsSignature : addressesThatNeedSignatures ) { + EventBus.INSTANCE.notify(new FeeWaitingEvent(true, addressThatNeedsSignature)); } } @@ -713,17 +725,17 @@ public class ForeignFeesManager implements Listener { * * @param bitcoiny the coin * - * @return if any fee signatures are needed after this process + * @return addresses that need fee signatures */ - private boolean processLocalForeignFeesForCoin(final Bitcoiny bitcoiny) { + private Set processLocalForeignFeesForCoin(final Bitcoiny bitcoiny) { - boolean feeSignaturesNeeded = false; + Set addressesThatNeedSignatures = new HashSet<>(); LOGGER.info("processing local foreign fees ..."); Optional nowDetermined = determineNow(); if (nowDetermined.isEmpty()){ - return false; + return new HashSet<>(0); } long now = nowDetermined.get(); @@ -745,13 +757,15 @@ public class ForeignFeesManager implements Listener { // process trade offer first, // then reset the fee signatures needed status next relative to prior status - feeSignaturesNeeded = processTradeOfferInWaiting(now, tradeOfferWaiting) || feeSignaturesNeeded; + if(processTradeOfferInWaiting(now, tradeOfferWaiting) ) { + addressesThatNeedSignatures.add(tradeOfferWaiting.getCreatorAddress()); + } } } catch (Exception e) { LOGGER.error(e.getMessage(), e); } - return feeSignaturesNeeded; + return addressesThatNeedSignatures; } /** diff --git a/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java b/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java index ea883515..55c3589b 100644 --- a/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java +++ b/src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java @@ -8,14 +8,22 @@ public class UnsignedFeeEvent { private boolean positive; + private String address; + public UnsignedFeeEvent() { } - public UnsignedFeeEvent(boolean positive) { + public UnsignedFeeEvent(boolean positive, String address) { + this.positive = positive; + this.address = address; } public boolean isPositive() { return positive; } + + public String getAddress() { + return address; + } } diff --git a/src/main/java/org/qortal/event/FeeWaitingEvent.java b/src/main/java/org/qortal/event/FeeWaitingEvent.java index 4d4d86ab..bb5bc0aa 100644 --- a/src/main/java/org/qortal/event/FeeWaitingEvent.java +++ b/src/main/java/org/qortal/event/FeeWaitingEvent.java @@ -8,16 +8,23 @@ public class FeeWaitingEvent implements Event{ private boolean positive; + private String address; + public FeeWaitingEvent() { } - public FeeWaitingEvent(boolean positive) { + public FeeWaitingEvent(boolean positive, String address) { this.positive = positive; + this.address = address; } public boolean isPositive() { return positive; } + + public String getAddress() { + return address; + } } From 26a345a909bd01beda321898dfd81152d5a1d89b Mon Sep 17 00:00:00 2001 From: kennycud Date: Sun, 4 May 2025 11:52:09 -0700 Subject: [PATCH 05/42] introducing feature trigger that enables multiple registered names for single accounts --- .../java/org/qortal/block/BlockChain.java | 15 +++++-- .../transaction/BuyNameTransaction.java | 2 +- .../transaction/RegisterNameTransaction.java | 2 +- src/main/resources/blockchain.json | 3 +- .../org/qortal/test/naming/BuySellTests.java | 42 +++++++++++++++++++ src/test/resources/test-chain-v2.json | 3 +- 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index bce09aed..d7e405ed 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -92,7 +92,8 @@ public class BlockChain { adminsReplaceFoundersHeight, nullGroupMembershipHeight, ignoreLevelForRewardShareHeight, - adminQueryFixHeight + adminQueryFixHeight, + multipleNamesPerAccountHeight } // Custom transaction fees @@ -112,7 +113,8 @@ public class BlockChain { /** Whether to use legacy, broken RIPEMD160 implementation when converting public keys to addresses. */ private boolean useBrokenMD160ForAddresses = false; - /** Whether only one registered name is allowed per account. */ + /** This should get ignored and overwritten in the oneNamePerAccount(int blockchainHeight) method, + * because it is based on block height, not based on the genesis block.*/ private boolean oneNamePerAccount = false; /** Checkpoints */ @@ -474,8 +476,9 @@ public class BlockChain { return this.useBrokenMD160ForAddresses; } - public boolean oneNamePerAccount() { - return this.oneNamePerAccount; + public boolean oneNamePerAccount(int blockchainHeight) { + // this is not set on a simple blockchain setting, it is based on a feature trigger height + return blockchainHeight < this.getMultipleNamesPerAccountHeight(); } public List getCheckpoints() { @@ -688,6 +691,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.adminQueryFixHeight.name()).intValue(); } + public int getMultipleNamesPerAccountHeight() { + return this.featureTriggers.get(FeatureTrigger.multipleNamesPerAccountHeight.name()).intValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/java/org/qortal/transaction/BuyNameTransaction.java b/src/main/java/org/qortal/transaction/BuyNameTransaction.java index 72c15f69..b7ca1d93 100644 --- a/src/main/java/org/qortal/transaction/BuyNameTransaction.java +++ b/src/main/java/org/qortal/transaction/BuyNameTransaction.java @@ -79,7 +79,7 @@ public class BuyNameTransaction extends Transaction { return ValidationResult.BUYER_ALREADY_OWNER; // If accounts are only allowed one registered name then check for this - if (BlockChain.getInstance().oneNamePerAccount() + if (BlockChain.getInstance().oneNamePerAccount(this.repository.getBlockRepository().getBlockchainHeight()) && !this.repository.getNameRepository().getNamesByOwner(buyer.getAddress()).isEmpty()) return ValidationResult.MULTIPLE_NAMES_FORBIDDEN; diff --git a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java index a89e60c0..c4520fbf 100644 --- a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java +++ b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java @@ -94,7 +94,7 @@ public class RegisterNameTransaction extends Transaction { return ValidationResult.NAME_ALREADY_REGISTERED; // If accounts are only allowed one registered name then check for this - if (BlockChain.getInstance().oneNamePerAccount() + if (BlockChain.getInstance().oneNamePerAccount(this.repository.getBlockRepository().getBlockchainHeight()) && !this.repository.getNameRepository().getNamesByOwner(getRegistrant().getAddress()).isEmpty()) return ValidationResult.MULTIPLE_NAMES_FORBIDDEN; diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 3264b670..14ba924b 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -119,7 +119,8 @@ "adminsReplaceFoundersHeight": 2012800, "nullGroupMembershipHeight": 2012800, "ignoreLevelForRewardShareHeight": 2012800, - "adminQueryFixHeight": 2012800 + "adminQueryFixHeight": 2012800, + "multipleNamesPerAccountHeight": 9999999 }, "checkpoints": [ { "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" } diff --git a/src/test/java/org/qortal/test/naming/BuySellTests.java b/src/test/java/org/qortal/test/naming/BuySellTests.java index a1c644fc..ddf9d913 100644 --- a/src/test/java/org/qortal/test/naming/BuySellTests.java +++ b/src/test/java/org/qortal/test/naming/BuySellTests.java @@ -4,6 +4,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.block.BlockChain; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.BuyNameTransactionData; import org.qortal.data.transaction.CancelSellNameTransactionData; @@ -17,6 +18,7 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.transaction.Transaction; import org.qortal.utils.Amounts; import java.util.Random; @@ -84,6 +86,46 @@ public class BuySellTests extends Common { assertTrue(repository.getNameRepository().nameExists(name)); } + @Test + public void testRegisterNameMultiple() throws DataException { + // register name 1 + RegisterNameTransactionData transactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData1, alice); + + String name1 = transactionData1.getName(); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name1)); + + // register another name, second registered name should fail before the feature trigger + final String name2 = "another name"; + RegisterNameTransactionData transactionData2 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult resultBeforeFeatureTrigger = TransactionUtils.signAndImport(repository, transactionData2, alice); + + // check that that multiple names is forbidden + assertTrue(Transaction.ValidationResult.MULTIPLE_NAMES_FORBIDDEN.equals(resultBeforeFeatureTrigger)); + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // register again, now that we are passed the feature trigger + RegisterNameTransactionData transactionData3 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult resultAfterFeatureTrigger = TransactionUtils.signAndImport(repository, transactionData3, alice); + + // check that multiple names is ok + assertTrue(Transaction.ValidationResult.OK.equals(resultAfterFeatureTrigger)); + + // mint block, confirm transaction + BlockUtils.mintBlock(repository); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name2)); + + // check that there are 2 names for one account + assertEquals(2, repository.getNameRepository().getNamesByOwner(alice.getAddress(), 0, 0, false).size() ); + } + @Test public void testSellName() throws DataException { // Register-name diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 5395116f..e410aae4 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -114,7 +114,8 @@ "adminsReplaceFoundersHeight": 9999999999999, "ignoreLevelForRewardShareHeight": 9999999999999, "nullGroupMembershipHeight": 20, - "adminQueryFixHeight": 9999999999999 + "adminQueryFixHeight": 9999999999999, + "multipleNamesPerAccountHeight": 10 }, "genesisInfo": { "version": 4, From 88d009c9795f2ab9ef840d4c311597b679b32f7a Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 6 May 2025 15:26:24 -0700 Subject: [PATCH 06/42] multiple registered names for single accounts API call now returns ordered by time of registration, earliest to latest --- .../qortal/repository/hsqldb/HSQLDBNameRepository.java | 2 +- src/test/java/org/qortal/test/naming/BuySellTests.java | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java index 06e41663..7bcdebda 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java @@ -267,7 +267,7 @@ public class HSQLDBNameRepository implements NameRepository { StringBuilder sql = new StringBuilder(512); sql.append("SELECT name, reduced_name, data, registered_when, updated_when, " - + "is_for_sale, sale_price, reference, creation_group_id FROM Names WHERE owner = ? ORDER BY name"); + + "is_for_sale, sale_price, reference, creation_group_id FROM Names WHERE owner = ? ORDER BY registered_when"); if (reverse != null && reverse) sql.append(" DESC"); diff --git a/src/test/java/org/qortal/test/naming/BuySellTests.java b/src/test/java/org/qortal/test/naming/BuySellTests.java index ddf9d913..2283404c 100644 --- a/src/test/java/org/qortal/test/naming/BuySellTests.java +++ b/src/test/java/org/qortal/test/naming/BuySellTests.java @@ -21,6 +21,7 @@ import org.qortal.transaction.RegisterNameTransaction; import org.qortal.transaction.Transaction; import org.qortal.utils.Amounts; +import java.util.List; import java.util.Random; import static org.junit.Assert.*; @@ -123,7 +124,13 @@ public class BuySellTests extends Common { assertTrue(repository.getNameRepository().nameExists(name2)); // check that there are 2 names for one account - assertEquals(2, repository.getNameRepository().getNamesByOwner(alice.getAddress(), 0, 0, false).size() ); + List namesByOwner = repository.getNameRepository().getNamesByOwner(alice.getAddress(), 0, 0, false); + + assertEquals(2, namesByOwner.size() ); + + // check that the order is correct + assertEquals(name1, namesByOwner.get(0).getName()); + } @Test From 70ae122f5ccedc87447d86b588e52f4c951fa56d Mon Sep 17 00:00:00 2001 From: PhilReact Date: Sat, 10 May 2025 22:21:13 +0300 Subject: [PATCH 07/42] pass ui lang to qapps --- src/main/java/org/qortal/api/HTMLParser.java | 13 ++++++--- .../resource/DevProxyServerResource.java | 12 +++++++- .../restricted/resource/RenderResource.java | 29 ++++++++++--------- .../arbitrary/ArbitraryDataRenderer.java | 6 +++- src/main/resources/q-apps/q-apps.js | 8 +++-- 5 files changed, 47 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java index 4887cf84..f5af7138 100644 --- a/src/main/java/org/qortal/api/HTMLParser.java +++ b/src/main/java/org/qortal/api/HTMLParser.java @@ -1,14 +1,13 @@ package org.qortal.api; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; import org.qortal.arbitrary.misc.Service; - import java.util.Objects; - public class HTMLParser { private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class); @@ -22,10 +21,11 @@ public class HTMLParser { private String identifier; private String path; private String theme; + private String lang; private boolean usingCustomRouting; public HTMLParser(String resourceId, String inPath, String prefix, boolean includeResourceIdInPrefix, byte[] data, - String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting) { + String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting, String lang) { String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : String.format("/%s",inPath); this.qdnBase = includeResourceIdInPrefix ? String.format("%s/%s", prefix, resourceId) : prefix; this.qdnBaseWithPath = includeResourceIdInPrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : String.format("%s%s", prefix, inPathWithoutFilename); @@ -36,6 +36,7 @@ public class HTMLParser { this.identifier = identifier; this.path = inPath; this.theme = theme; + this.lang = lang; this.usingCustomRouting = usingCustomRouting; } @@ -61,9 +62,13 @@ public class HTMLParser { String identifier = this.identifier != null ? this.identifier.replace("\\", "").replace("\"","\\\"") : ""; String path = this.path != null ? this.path.replace("\\", "").replace("\"","\\\"") : ""; String theme = this.theme != null ? this.theme.replace("\\", "").replace("\"","\\\"") : ""; + String lang = this.lang != null ? this.lang.replace("\\", "").replace("\"", "\\\"") : ""; String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\\", "").replace("\"","\\\"") : ""; String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\\", "").replace("\"","\\\"") : ""; - String qdnContextVar = String.format("", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath); + String qdnContextVar = String.format( + "", + qdnContext, theme, lang, service, name, identifier, path, qdnBase, qdnBaseWithPath + ); head.get(0).prepend(qdnContextVar); // Add base href tag diff --git a/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java b/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java index 7972c551..bdaf1ced 100644 --- a/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java +++ b/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java @@ -142,10 +142,20 @@ public class DevProxyServerResource { } } + String lang = request.getParameter("lang"); + if (lang == null || lang.isBlank()) { + lang = "en"; // fallback + } + + String theme = request.getParameter("theme"); + if (theme == null || theme.isBlank()) { + theme = "light"; + } + // Parse and modify output if needed if (HTMLParser.isHtmlFile(filename)) { // HTML file - needs to be parsed - HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true); + HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, theme , true, lang); htmlParser.addAdditionalHeaderTags(); response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;"); response.setContentType(con.getContentType()); diff --git a/src/main/java/org/qortal/api/restricted/resource/RenderResource.java b/src/main/java/org/qortal/api/restricted/resource/RenderResource.java index 5eb169a4..3bab7bf3 100644 --- a/src/main/java/org/qortal/api/restricted/resource/RenderResource.java +++ b/src/main/java/org/qortal/api/restricted/resource/RenderResource.java @@ -71,33 +71,33 @@ public class RenderResource { @Path("/signature/{signature}") @SecurityRequirement(name = "apiKey") public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null); - return this.get(signature, ResourceIdType.SIGNATURE, null, null, "/", null, "/render/signature", true, true, theme); + return this.get(signature, ResourceIdType.SIGNATURE, null, null, "/", null, "/render/signature", true, true, theme, lang); } @GET @Path("/signature/{signature}/{path:.*}") @SecurityRequirement(name = "apiKey") public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null); - return this.get(signature, ResourceIdType.SIGNATURE, null, null, inPath,null, "/render/signature", true, true, theme); + return this.get(signature, ResourceIdType.SIGNATURE, null, null, inPath,null, "/render/signature", true, true, theme, lang); } @GET @Path("/hash/{hash}") @SecurityRequirement(name = "apiKey") public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null); - return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, "/", secret58, "/render/hash", true, false, theme); + return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, "/", secret58, "/render/hash", true, false, theme, lang); } @GET @@ -105,11 +105,11 @@ public class RenderResource { @SecurityRequirement(name = "apiKey") public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath, @QueryParam("secret") String secret58, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null); - return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, inPath, secret58, "/render/hash", true, false, theme); + return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, inPath, secret58, "/render/hash", true, false, theme, lang); } @GET @@ -119,12 +119,12 @@ public class RenderResource { @PathParam("name") String name, @PathParam("path") String inPath, @QueryParam("identifier") String identifier, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, name, service, null); String prefix = String.format("/render/%s", service); - return this.get(name, ResourceIdType.NAME, service, identifier, inPath, null, prefix, true, true, theme); + return this.get(name, ResourceIdType.NAME, service, identifier, inPath, null, prefix, true, true, theme, lang); } @GET @@ -133,18 +133,18 @@ public class RenderResource { public HttpServletResponse getIndexByName(@PathParam("service") Service service, @PathParam("name") String name, @QueryParam("identifier") String identifier, - @QueryParam("theme") String theme) { + @QueryParam("theme") String theme, @QueryParam("lang") String lang) { if (!Settings.getInstance().isQDNAuthBypassEnabled()) Security.requirePriorAuthorization(request, name, service, null); String prefix = String.format("/render/%s", service); - return this.get(name, ResourceIdType.NAME, service, identifier, "/", null, prefix, true, true, theme); + return this.get(name, ResourceIdType.NAME, service, identifier, "/", null, prefix, true, true, theme, lang); } private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String identifier, - String inPath, String secret58, String prefix, boolean includeResourceIdInPrefix, boolean async, String theme) { + String inPath, String secret58, String prefix, boolean includeResourceIdInPrefix, boolean async, String theme, String lang) { ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, identifier, inPath, secret58, prefix, includeResourceIdInPrefix, async, "render", request, response, context); @@ -152,6 +152,9 @@ public class RenderResource { if (theme != null) { renderer.setTheme(theme); } + if (lang != null) { + renderer.setLang(lang); + } return renderer.render(); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java index eb51e8a4..72cd4097 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java @@ -37,6 +37,7 @@ public class ArbitraryDataRenderer { private final Service service; private final String identifier; private String theme = "light"; + private String lang = "en"; private String inPath; private final String secret58; private final String prefix; @@ -166,7 +167,7 @@ public class ArbitraryDataRenderer { if (HTMLParser.isHtmlFile(filename)) { // HTML file - needs to be parsed byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory - HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, includeResourceIdInPrefix, data, qdnContext, service, identifier, theme, usingCustomRouting); + HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, includeResourceIdInPrefix, data, qdnContext, service, identifier, theme, usingCustomRouting, lang); htmlParser.addAdditionalHeaderTags(); response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' wss:;"); response.setContentType(context.getMimeType(filename)); @@ -256,5 +257,8 @@ public class ArbitraryDataRenderer { public void setTheme(String theme) { this.theme = theme; } + public void setLang(String lang) { + this.lang = lang; + } } diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js index d7222750..486c6543 100644 --- a/src/main/resources/q-apps/q-apps.js +++ b/src/main/resources/q-apps/q-apps.js @@ -45,6 +45,7 @@ function parseUrl(url) { // Remove theme, identifier, and time queries if they exist parsedUrl.searchParams.delete("theme"); + parsedUrl.searchParams.delete("lang"); parsedUrl.searchParams.delete("identifier"); parsedUrl.searchParams.delete("time"); parsedUrl.searchParams.delete("isManualNavigation"); @@ -213,8 +214,11 @@ function buildResourceUrl(service, name, identifier, path, isLink) { if (path != null) url = url.concat((path.startsWith("/") ? "" : "/") + path); } - if (isLink) url = url.concat((url.includes("?") ? "" : "?") + "&theme=" + _qdnTheme); - + if (isLink) { + const hasQuery = url.includes("?"); + const queryPrefix = hasQuery ? "&" : "?"; + url += queryPrefix + "theme=" + _qdnTheme + "&lang=" + _qdnLang; + } return url; } From 93dab1a3e3ada98181346774450e548aa2e41512 Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 13 May 2025 11:13:55 -0700 Subject: [PATCH 08/42] detailed test case for the invite orphan vulnerability patch that was committed in 2/1/25 --- .../qortal/test/group/DevGroupAdminTests.java | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/src/test/java/org/qortal/test/group/DevGroupAdminTests.java b/src/test/java/org/qortal/test/group/DevGroupAdminTests.java index 38ad0c53..925e2f3e 100644 --- a/src/test/java/org/qortal/test/group/DevGroupAdminTests.java +++ b/src/test/java/org/qortal/test/group/DevGroupAdminTests.java @@ -1,6 +1,7 @@ package org.qortal.test.group; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; @@ -369,6 +370,105 @@ public class DevGroupAdminTests extends Common { } } + @Test + public void testOrphanSecondInviteApproval() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + + Block block = BlockUtils.mintBlocks(repository, NULL_GROUP_MEMBERSHIP_HEIGHT); + assertEquals(NULL_GROUP_MEMBERSHIP_HEIGHT + 1, block.getBlockData().getHeight().intValue()); + + // establish accounts + PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE); + PrivateKeyAccount bob = Common.getTestAccount(repository, BOB); + PrivateKeyAccount chloe = Common.getTestAccount(repository, CHLOE); + PrivateKeyAccount dilbert = Common.getTestAccount(repository, DILBERT); + + // assert admin statuses + assertEquals(2, repository.getGroupRepository().countGroupAdmins(DEV_GROUP_ID).intValue()); + assertTrue(isAdmin(repository, Group.NULL_OWNER_ADDRESS, DEV_GROUP_ID)); + assertTrue(isAdmin(repository, alice.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, chloe.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, dilbert.getAddress(), DEV_GROUP_ID)); + + // confirm Bob is not a member + assertFalse(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + + // alice invites bob, alice signs which is 50% approval while 40% is needed + TransactionData createInviteTransactionData = createGroupInviteForGroupApproval(repository, alice, DEV_GROUP_ID, bob.getAddress(), 3600); + Transaction.ApprovalStatus bobsInviteStatus = signForGroupApproval(repository, createInviteTransactionData, List.of(alice)); + + // assert approval + assertEquals(Transaction.ApprovalStatus.APPROVED, bobsInviteStatus); + + // bob joins + joinGroup(repository, bob, DEV_GROUP_ID); + + // confirm Bob is a member now, but still not an admin + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + assertFalse(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + + // bob creates transaction to add himself as an admin + TransactionData addGroupAdminTransactionData1 = addGroupAdmin(repository, bob, DEV_GROUP_ID, bob.getAddress()); + + // bob creates add admin transaction for himself, alice signs which is 50% approval while 40% is needed + signForGroupApproval(repository, addGroupAdminTransactionData1, List.of(alice)); + + // assert 3 admins in group and bob is an admin now + assertEquals(3, repository.getGroupRepository().countGroupAdmins(DEV_GROUP_ID).intValue()); + assertTrue(isAdmin(repository, bob.getAddress(), DEV_GROUP_ID)); + + // bob invites chloe, bob signs which is 33% approval while 40% is needed + TransactionData chloeInvite1 = createGroupInviteForGroupApproval(repository, bob, DEV_GROUP_ID, chloe.getAddress(), 3600); + Transaction.ApprovalStatus chloeInvite1Status = signForGroupApproval(repository, chloeInvite1, List.of(bob)); + + // assert invite 1 pending + assertEquals(Transaction.ApprovalStatus.PENDING, chloeInvite1Status); + + // bob invites chloe again, bob signs which is 33% approval while 40% is needed + // since chloe is not a member yet, this invite is valie + TransactionData chloeInvite2 = createGroupInviteForGroupApproval(repository, bob, DEV_GROUP_ID, chloe.getAddress(), 3600); + Transaction.ApprovalStatus chloeInvite2Status = signForGroupApproval(repository, chloeInvite2, List.of(bob)); + + // assert invite 2 is pending + assertEquals(Transaction.ApprovalStatus.PENDING, chloeInvite2Status); + + // alice signs which is 66% approval while 40% is needed + chloeInvite1Status = signForGroupApproval(repository, chloeInvite1, List.of(alice)); + + // assert invite 1 approval + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeInvite1Status); + + // chloe joins + joinGroup(repository, chloe, DEV_GROUP_ID); + + // assert chloe is in the group + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + + // alice signs invite 2 which is 66% approval while 40% is needed + chloeInvite2Status = signForGroupApproval(repository, chloeInvite2, List.of(alice)); + + // assert invite 2 approval + assertEquals(Transaction.ApprovalStatus.APPROVED, chloeInvite2Status); + + boolean exceptionThrown = false; + + try { + // confront the bug by orphaning the block of the second approval, approval after the join + // prior to the fix, this would raise an exception + BlockUtils.orphanLastBlock(repository); + } catch (DataException e) { + exceptionThrown = true; + } + + Assert.assertFalse(exceptionThrown); + + // assert chloe is still a member + assertTrue(isMember(repository, bob.getAddress(), DEV_GROUP_ID)); + } + } + @Test public void testNullOwnershipMembership() throws DataException{ try (final Repository repository = RepositoryManager.getRepository()) { From 8c811ef1efda57563b4875747305aec9b778691d Mon Sep 17 00:00:00 2001 From: PhilReact Date: Wed, 14 May 2025 20:00:04 +0300 Subject: [PATCH 09/42] initial --- pom.xml | 5 + src/main/java/org/qortal/api/ApiService.java | 1 + .../api/resource/ArbitraryResource.java | 473 +++++++++++++++--- .../qortal/arbitrary/ArbitraryDataFile.java | 2 +- .../ArbitraryDataTransactionBuilder.java | 5 +- src/main/java/org/qortal/crypto/AES.java | 4 +- src/main/java/org/qortal/utils/ZipUtils.java | 57 ++- 7 files changed, 460 insertions(+), 87 deletions(-) diff --git a/pom.xml b/pom.xml index 13ad4807..c4236d24 100644 --- a/pom.xml +++ b/pom.xml @@ -796,5 +796,10 @@ jaxb-runtime ${jaxb-runtime.version} + +org.apache.tika +tika-core +3.1.0 + diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java index 00ab29e0..1cfab1da 100644 --- a/src/main/java/org/qortal/api/ApiService.java +++ b/src/main/java/org/qortal/api/ApiService.java @@ -46,6 +46,7 @@ public class ApiService { private ApiService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.restricted.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index a6f44373..a49dc7f5 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -3,6 +3,7 @@ package org.qortal.api.resource; import com.google.common.primitives.Bytes; import com.j256.simplemagic.ContentInfo; import com.j256.simplemagic.ContentInfoUtil; + import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -12,6 +13,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; @@ -63,14 +65,19 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; + import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.FileNameMap; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -79,6 +86,14 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.tika.Tika; +import org.apache.tika.mime.MimeTypeException; +import org.apache.tika.mime.MimeTypes; + +import javax.ws.rs.core.Response; + +import org.glassfish.jersey.media.multipart.FormDataParam; + @Path("/arbitrary") @Tag(name = "Arbitrary") public class ArbitraryResource { @@ -878,6 +893,230 @@ public class ArbitraryResource { } + @GET + @Path("/check-tmp-space") + @Produces(MediaType.TEXT_PLAIN) + @Operation( + summary = "Check if the disk has enough disk space for an upcoming upload", + responses = { + @ApiResponse(description = "OK if sufficient space", responseCode = "200"), + @ApiResponse(description = "Insufficient space", responseCode = "507") // 507 = Insufficient Storage + } + ) + @SecurityRequirement(name = "apiKey") + public Response checkUploadSpace(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @QueryParam("totalSize") Long totalSize) { + Security.checkApiCallAllowed(request); + + if (totalSize == null || totalSize <= 0) { + return Response.status(Response.Status.BAD_REQUEST) + .entity("Missing or invalid totalSize parameter").build(); + } + + File baseTmp = new File(System.getProperty("java.io.tmpdir")); + long usableSpace = baseTmp.getUsableSpace(); + long requiredSpace = totalSize * 2; // chunked + merged file estimate + + if (usableSpace < requiredSpace) { + return Response.status(507).entity("Insufficient disk space").build(); // 507 = Insufficient Storage + } + + return Response.ok("Sufficient disk space").build(); + } + + @POST +@Path("/{service}/{name}/{identifier}/chunk") +@Consumes(MediaType.MULTIPART_FORM_DATA) +@Operation( + summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.MULTIPART_FORM_DATA, + schema = @Schema( + implementation = Object.class + ) + ) + ), + responses = { + @ApiResponse( + description = "Chunk uploaded successfully", + responseCode = "200" + ), + @ApiResponse( + description = "Error writing chunk", + responseCode = "500" + ) + } +) +@SecurityRequirement(name = "apiKey") +public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @FormDataParam("chunk") InputStream chunkStream, + @FormDataParam("index") int index) { + Security.checkApiCallAllowed(request); + + try { + java.nio.file.Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + Files.createDirectories(tempDir); + + java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); + Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING); + + return Response.ok("Chunk " + index + " received").build(); + } catch (IOException e) { + return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); + } +} + +@POST +@Path("/{service}/{name}/{identifier}/finalize") +@Produces(MediaType.TEXT_PLAIN) +@Operation( + summary = "Finalize a chunked upload and build a raw, unsigned, ARBITRARY transaction", + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content(mediaType = MediaType.TEXT_PLAIN) + ) + } +) +@SecurityRequirement(name = "apiKey") +public String finalizeUpload( + @HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") List tags, + @QueryParam("category") Category category, + @QueryParam("filename") String filename, + @QueryParam("fee") Long fee, + @QueryParam("preview") Boolean preview +) { + Security.checkApiCallAllowed(request); + java.nio.file.Path tempFile = null; + java.nio.file.Path tempDir = null; + java.nio.file.Path chunkDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + + try { + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); + } + + // Step 1: Determine a safe filename for disk temp file (regardless of extension correctness) + String safeFilename = filename; + if (filename == null || filename.isBlank()) { + safeFilename = "qortal-" + NTP.getTime(); + } + + tempDir = Files.createTempDirectory("qortal-"); + tempFile = tempDir.resolve(safeFilename); + + // Step 2: Merge chunks + + try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + byte[] buffer = new byte[65536]; + for (java.nio.file.Path chunk : Files.list(chunkDir) + .filter(path -> path.getFileName().toString().startsWith("chunk_")) + .sorted(Comparator.comparingInt(path -> { + String name2 = path.getFileName().toString(); + String numberPart = name2.substring("chunk_".length()); + return Integer.parseInt(numberPart); + })).collect(Collectors.toList())) { + try (InputStream in = Files.newInputStream(chunk)) { + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } + } + + + // Step 3: Determine correct extension + String detectedExtension = ""; + String uploadFilename = null; + boolean extensionIsValid = false; + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + if (lastDot > 0 && lastDot < filename.length() - 1) { + extensionIsValid = true; + uploadFilename = filename; + } + } + + if (!extensionIsValid) { + Tika tika = new Tika(); + String mimeType = tika.detect(tempFile.toFile()); + try { + MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); + org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType); + detectedExtension = mime.getExtension(); + } catch (MimeTypeException e) { + LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e); + } + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename; + uploadFilename = baseName + (detectedExtension != null ? detectedExtension : ""); + } else { + uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : ""); + } + } + + + + return this.upload( + Service.valueOf(serviceString), + name, + identifier, + tempFile.toString(), + null, + null, + false, + fee, + uploadFilename, + title, + description, + tags, + category, + preview + ); + + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); + } finally { + if (tempDir != null) { + try { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete temp directory: {}", tempDir, e); + } + } + + try { + Files.walk(chunkDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e); + } + } +} + + + + // Upload base64-encoded data @@ -1409,6 +1648,7 @@ public class ArbitraryResource { ); transactionBuilder.build(); + // Don't compute nonce - this is done by the client (or via POST /arbitrary/compute) ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData(); return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData)); @@ -1424,23 +1664,127 @@ public class ArbitraryResource { } } - private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + // private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + // try { + // ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); + + // int attempts = 0; + // if (maxAttempts == null) { + // maxAttempts = 5; + // } + + // // Loop until we have data + // if (async) { + // // Asynchronous + // arbitraryDataReader.loadAsynchronously(false, 1); + // } + // else { + // // Synchronous + // while (!Controller.isStopping()) { + // attempts++; + // if (!arbitraryDataReader.isBuilding()) { + // try { + // arbitraryDataReader.loadSynchronously(rebuild); + // break; + // } catch (MissingDataException e) { + // if (attempts > maxAttempts) { + // // Give up after 5 attempts + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); + // } + // } + // } + // Thread.sleep(3000L); + // } + // } + + // java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); + // if (outputPath == null) { + // // Assume the resource doesn't exist + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); + // } + + // if (filepath == null || filepath.isEmpty()) { + // // No file path supplied - so check if this is a single file resource + // String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); + // if (files != null && files.length == 1) { + // // This is a single file resource + // filepath = files[0]; + // } + // else { + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, + // "filepath is required for resources containing more than one file"); + // } + // } + + // java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); + // if (!Files.exists(path)) { + // String message = String.format("No file exists at filepath: %s", filepath); + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); + // } + + // byte[] data; + // int fileSize = (int)path.toFile().length(); + // int length = fileSize; + + // // Parse "Range" header + // Integer rangeStart = null; + // Integer rangeEnd = null; + // String range = request.getHeader("Range"); + // if (range != null) { + // range = range.replace("bytes=", ""); + // String[] parts = range.split("-"); + // rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null; + // rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize; + // } + + // if (rangeStart != null && rangeEnd != null) { + // // We have a range, so update the requested length + // length = rangeEnd - rangeStart; + // } + + // if (length < fileSize && encoding == null) { + // // Partial content requested, and not encoding the data + // response.setStatus(206); + // response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize)); + // data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length); + // } + // else { + // // Full content requested (or encoded data) + // response.setStatus(200); + // data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory + // } + + // // Encode the data if requested + // if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) { + // data = Base64.encode(data); + // } + + // response.addHeader("Accept-Ranges", "bytes"); + // response.setContentType(context.getMimeType(path.toString())); + // response.setContentLength(data.length); + // response.getOutputStream().write(data); + + // return response; + // } catch (Exception e) { + // LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); + // } + // } + + private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { try { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); - + int attempts = 0; if (maxAttempts == null) { maxAttempts = 5; } - - // Loop until we have data + + // Load the file if (async) { - // Asynchronous arbitraryDataReader.loadAsynchronously(false, 1); - } - else { - // Synchronous + } else { while (!Controller.isStopping()) { attempts++; if (!arbitraryDataReader.isBuilding()) { @@ -1449,7 +1793,6 @@ public class ArbitraryResource { break; } catch (MissingDataException e) { if (attempts > maxAttempts) { - // Give up after 5 attempts throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); } } @@ -1457,80 +1800,94 @@ public class ArbitraryResource { Thread.sleep(3000L); } } - + java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); if (outputPath == null) { - // Assume the resource doesn't exist throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); } - + if (filepath == null || filepath.isEmpty()) { - // No file path supplied - so check if this is a single file resource String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); if (files != null && files.length == 1) { - // This is a single file resource filepath = files[0]; - } - else { - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, - "filepath is required for resources containing more than one file"); + } else { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "filepath is required for resources containing more than one file"); } } - + java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); if (!Files.exists(path)) { - String message = String.format("No file exists at filepath: %s", filepath); - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No file exists at filepath: " + filepath); } - - byte[] data; - int fileSize = (int)path.toFile().length(); - int length = fileSize; - - // Parse "Range" header - Integer rangeStart = null; - Integer rangeEnd = null; + + long fileSize = Files.size(path); + String mimeType = context.getMimeType(path.toString()); String range = request.getHeader("Range"); - if (range != null) { + + long rangeStart = 0; + long rangeEnd = fileSize - 1; + boolean isPartial = false; + + if (range != null && encoding == null) { range = range.replace("bytes=", ""); String[] parts = range.split("-"); - rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null; - rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize; + if (parts.length > 0 && !parts[0].isEmpty()) { + rangeStart = Long.parseLong(parts[0]); + } + if (parts.length > 1 && !parts[1].isEmpty()) { + rangeEnd = Long.parseLong(parts[1]); + } + isPartial = true; } - - if (rangeStart != null && rangeEnd != null) { - // We have a range, so update the requested length - length = rangeEnd - rangeStart; + + long contentLength = rangeEnd - rangeStart + 1; + + // Set headers + response.setContentType(mimeType); + response.setHeader("Accept-Ranges", "bytes"); + + if (isPartial) { + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); + } else { + response.setStatus(HttpServletResponse.SC_OK); } - - if (length < fileSize && encoding == null) { - // Partial content requested, and not encoding the data - response.setStatus(206); - response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize)); - data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length); + + OutputStream rawOut = response.getOutputStream(); + + if (encoding != null && "base64".equalsIgnoreCase(encoding)) { + // Stream Base64-encoded output + java.util.Base64.Encoder encoder = java.util.Base64.getEncoder(); + rawOut = encoder.wrap(rawOut); + } else { + // Set Content-Length only when not Base64 + response.setContentLength((int) contentLength); } - else { - // Full content requested (or encoded data) - response.setStatus(200); - data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory + + // Stream file content + try (InputStream inputStream = Files.newInputStream(path)) { + if (rangeStart > 0) { + inputStream.skip(rangeStart); + } + + byte[] buffer = new byte[65536]; + long bytesRemaining = contentLength; + int bytesRead; + + while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) { + rawOut.write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } } - - // Encode the data if requested - if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) { - data = Base64.encode(data); - } - - response.addHeader("Accept-Ranges", "bytes"); - response.setContentType(context.getMimeType(path.toString())); - response.setContentLength(data.length); - response.getOutputStream().write(data); - + return response; - } catch (Exception e) { + + } catch (IOException | InterruptedException | NumberFormatException | ApiException | DataException e) { LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); } } + private FileProperties getFileProperties(Service service, String name, String identifier) { try { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 6e1ca0b9..518e27d0 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -52,7 +52,7 @@ public class ArbitraryDataFile { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class); - public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB + public static final long MAX_FILE_SIZE = 3L * 1024 * 1024 * 1024; // 3 GiB protected static final int MAX_CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static final int CHUNK_SIZE = 512 * 1024; // 0.5MiB public static int SHORT_DIGEST_LENGTH = 8; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java index a77442ec..b40f72c7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java @@ -29,6 +29,7 @@ import org.qortal.utils.FilesystemUtils; import org.qortal.utils.NTP; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -197,7 +198,7 @@ public class ArbitraryDataTransactionBuilder { // We can't use PATCH for on-chain data because this requires the .qortal directory, which can't be put on chain final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(this.path, false); - final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE); + final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE); if (shouldUseOnChainData) { LOGGER.info("Data size is small enough to go on chain - using PUT"); return Method.PUT; @@ -245,7 +246,7 @@ public class ArbitraryDataTransactionBuilder { // Single file resources are handled differently, especially for very small data payloads, as these go on chain final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(path, false); - final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE); + final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE); // Use zip compression if data isn't going on chain Compression compression = shouldUseOnChainData ? Compression.NONE : Compression.ZIP; diff --git a/src/main/java/org/qortal/crypto/AES.java b/src/main/java/org/qortal/crypto/AES.java index d42e22f9..9bfa172a 100644 --- a/src/main/java/org/qortal/crypto/AES.java +++ b/src/main/java/org/qortal/crypto/AES.java @@ -100,7 +100,7 @@ public class AES { // Prepend the output stream with the 16 byte initialization vector outputStream.write(iv.getIV()); - byte[] buffer = new byte[1024]; + byte[] buffer = new byte[65536]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { byte[] output = cipher.update(buffer, 0, bytesRead); @@ -138,7 +138,7 @@ public class AES { Cipher cipher = Cipher.getInstance(algorithm); cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv)); - byte[] buffer = new byte[64]; + byte[] buffer = new byte[65536]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { byte[] output = cipher.update(buffer, 0, bytesRead); diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index c61723e7..8ef2960b 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -27,6 +27,8 @@ package org.qortal.utils; +import java.io.BufferedOutputStream; + import org.qortal.controller.Controller; import java.io.File; @@ -44,11 +46,17 @@ public class ZipUtils { File sourceFile = new File(sourcePath); boolean isSingleFile = Paths.get(sourcePath).toFile().isFile(); FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); + + // 🔧 Use best speed compression level ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); + zipOutputStream.setLevel(java.util.zip.Deflater.BEST_SPEED); + ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile); + zipOutputStream.close(); fileOutputStream.close(); } + public static void zip(final File fileToZip, final String enclosingFolderName, final ZipOutputStream zipOut, boolean isSingleFile) throws IOException, InterruptedException { if (Controller.isStopping()) { @@ -82,7 +90,7 @@ public class ZipUtils { final FileInputStream fis = new FileInputStream(fileToZip); final ZipEntry zipEntry = new ZipEntry(enclosingFolderName); zipOut.putNextEntry(zipEntry); - final byte[] bytes = new byte[1024]; + final byte[] bytes = new byte[65536]; int length; while ((length = fis.read(bytes)) >= 0) { zipOut.write(bytes, 0, length); @@ -92,33 +100,34 @@ public class ZipUtils { public static void unzip(String sourcePath, String destPath) throws IOException { final File destDir = new File(destPath); - final byte[] buffer = new byte[1024]; - final ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath)); - ZipEntry zipEntry = zis.getNextEntry(); - while (zipEntry != null) { - final File newFile = ZipUtils.newFile(destDir, zipEntry); - if (zipEntry.isDirectory()) { - if (!newFile.isDirectory() && !newFile.mkdirs()) { - throw new IOException("Failed to create directory " + newFile); + final byte[] buffer = new byte[65536]; + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath))) { + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + final File newFile = ZipUtils.newFile(destDir, zipEntry); + if (zipEntry.isDirectory()) { + if (!newFile.isDirectory() && !newFile.mkdirs()) { + throw new IOException("Failed to create directory " + newFile); + } + } else { + File parent = newFile.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("Failed to create directory " + parent); + } + + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(newFile), buffer.length)) { + int len; + while ((len = zis.read(buffer)) > 0) { + bos.write(buffer, 0, len); + } + } } - } else { - File parent = newFile.getParentFile(); - if (!parent.isDirectory() && !parent.mkdirs()) { - throw new IOException("Failed to create directory " + parent); - } - - final FileOutputStream fos = new FileOutputStream(newFile); - int len; - while ((len = zis.read(buffer)) > 0) { - fos.write(buffer, 0, len); - } - fos.close(); + zipEntry = zis.getNextEntry(); } - zipEntry = zis.getNextEntry(); + zis.closeEntry(); } - zis.closeEntry(); - zis.close(); } + /** * See: https://snyk.io/research/zip-slip-vulnerability From 5780a6de7d648e71007c9a4af1a34f3ff63625bc Mon Sep 17 00:00:00 2001 From: PhilReact Date: Wed, 14 May 2025 20:21:13 +0300 Subject: [PATCH 10/42] remove zip best speed --- src/main/java/org/qortal/utils/ZipUtils.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index 8ef2960b..e747e175 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -48,9 +48,7 @@ public class ZipUtils { FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); // 🔧 Use best speed compression level - ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); - zipOutputStream.setLevel(java.util.zip.Deflater.BEST_SPEED); - + ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile); zipOutputStream.close(); From 994761a87ec97beb8799174f4e5e22af00c9db3d Mon Sep 17 00:00:00 2001 From: PhilReact Date: Thu, 15 May 2025 01:20:40 +0300 Subject: [PATCH 11/42] added missing requires --- src/main/java/org/qortal/api/DevProxyService.java | 1 + src/main/java/org/qortal/api/DomainMapService.java | 1 + src/main/java/org/qortal/api/GatewayService.java | 1 + 3 files changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/api/DevProxyService.java b/src/main/java/org/qortal/api/DevProxyService.java index e0bf02db..c0c4e224 100644 --- a/src/main/java/org/qortal/api/DevProxyService.java +++ b/src/main/java/org/qortal/api/DevProxyService.java @@ -40,6 +40,7 @@ public class DevProxyService { private DevProxyService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.proxy.resource", "org.qortal.api.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/DomainMapService.java b/src/main/java/org/qortal/api/DomainMapService.java index 8b791121..5b4a6bbe 100644 --- a/src/main/java/org/qortal/api/DomainMapService.java +++ b/src/main/java/org/qortal/api/DomainMapService.java @@ -39,6 +39,7 @@ public class DomainMapService { private DomainMapService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.domainmap.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/GatewayService.java b/src/main/java/org/qortal/api/GatewayService.java index 24a7b7c9..3ac77799 100644 --- a/src/main/java/org/qortal/api/GatewayService.java +++ b/src/main/java/org/qortal/api/GatewayService.java @@ -39,6 +39,7 @@ public class GatewayService { private GatewayService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.gateway.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); From bc4e0716db7b8c2b12c67efd81a68380dff24a9e Mon Sep 17 00:00:00 2001 From: PhilReact Date: Thu, 15 May 2025 16:56:53 +0300 Subject: [PATCH 12/42] fix streaming for base64 --- .../api/resource/ArbitraryResource.java | 158 ++++-------------- 1 file changed, 36 insertions(+), 122 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index a49dc7f5..97146b8f 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -85,6 +85,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; import org.apache.tika.Tika; import org.apache.tika.mime.MimeTypeException; @@ -701,7 +702,7 @@ public class ArbitraryResource { ) } ) - public HttpServletResponse get(@PathParam("service") Service service, + public void get(@PathParam("service") Service service, @PathParam("name") String name, @QueryParam("filepath") String filepath, @QueryParam("encoding") String encoding, @@ -714,7 +715,7 @@ public class ArbitraryResource { Security.checkApiCallAllowed(request); } - return this.download(service, name, null, filepath, encoding, rebuild, async, attempts); + this.download(service, name, null, filepath, encoding, rebuild, async, attempts); } @GET @@ -734,7 +735,7 @@ public class ArbitraryResource { ) } ) - public HttpServletResponse get(@PathParam("service") Service service, + public void get(@PathParam("service") Service service, @PathParam("name") String name, @PathParam("identifier") String identifier, @QueryParam("filepath") String filepath, @@ -748,7 +749,7 @@ public class ArbitraryResource { Security.checkApiCallAllowed(request, null); } - return this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts); + this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts); } @@ -1664,115 +1665,7 @@ public String finalizeUpload( } } - // private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { - - // try { - // ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); - - // int attempts = 0; - // if (maxAttempts == null) { - // maxAttempts = 5; - // } - - // // Loop until we have data - // if (async) { - // // Asynchronous - // arbitraryDataReader.loadAsynchronously(false, 1); - // } - // else { - // // Synchronous - // while (!Controller.isStopping()) { - // attempts++; - // if (!arbitraryDataReader.isBuilding()) { - // try { - // arbitraryDataReader.loadSynchronously(rebuild); - // break; - // } catch (MissingDataException e) { - // if (attempts > maxAttempts) { - // // Give up after 5 attempts - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); - // } - // } - // } - // Thread.sleep(3000L); - // } - // } - - // java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); - // if (outputPath == null) { - // // Assume the resource doesn't exist - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); - // } - - // if (filepath == null || filepath.isEmpty()) { - // // No file path supplied - so check if this is a single file resource - // String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); - // if (files != null && files.length == 1) { - // // This is a single file resource - // filepath = files[0]; - // } - // else { - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, - // "filepath is required for resources containing more than one file"); - // } - // } - - // java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); - // if (!Files.exists(path)) { - // String message = String.format("No file exists at filepath: %s", filepath); - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); - // } - - // byte[] data; - // int fileSize = (int)path.toFile().length(); - // int length = fileSize; - - // // Parse "Range" header - // Integer rangeStart = null; - // Integer rangeEnd = null; - // String range = request.getHeader("Range"); - // if (range != null) { - // range = range.replace("bytes=", ""); - // String[] parts = range.split("-"); - // rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null; - // rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize; - // } - - // if (rangeStart != null && rangeEnd != null) { - // // We have a range, so update the requested length - // length = rangeEnd - rangeStart; - // } - - // if (length < fileSize && encoding == null) { - // // Partial content requested, and not encoding the data - // response.setStatus(206); - // response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize)); - // data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length); - // } - // else { - // // Full content requested (or encoded data) - // response.setStatus(200); - // data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory - // } - - // // Encode the data if requested - // if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) { - // data = Base64.encode(data); - // } - - // response.addHeader("Accept-Ranges", "bytes"); - // response.setContentType(context.getMimeType(path.toString())); - // response.setContentLength(data.length); - // response.getOutputStream().write(data); - - // return response; - // } catch (Exception e) { - // LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); - // } - // } - - private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { try { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); @@ -1843,7 +1736,6 @@ public String finalizeUpload( long contentLength = rangeEnd - rangeStart + 1; // Set headers - response.setContentType(mimeType); response.setHeader("Accept-Ranges", "bytes"); if (isPartial) { @@ -1854,13 +1746,25 @@ public String finalizeUpload( } OutputStream rawOut = response.getOutputStream(); - + OutputStream base64Out = null; + OutputStream gzipOut = null; if (encoding != null && "base64".equalsIgnoreCase(encoding)) { - // Stream Base64-encoded output - java.util.Base64.Encoder encoder = java.util.Base64.getEncoder(); - rawOut = encoder.wrap(rawOut); + response.setContentType("text/plain"); + + String acceptEncoding = request.getHeader("Accept-Encoding"); + boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip"); + + if (wantsGzip) { + response.setHeader("Content-Encoding", "gzip"); + gzipOut = new GZIPOutputStream(rawOut); + base64Out = java.util.Base64.getEncoder().wrap(gzipOut); + } else { + base64Out = java.util.Base64.getEncoder().wrap(rawOut); + } + + rawOut = base64Out; } else { - // Set Content-Length only when not Base64 + response.setContentType(mimeType != null ? mimeType : "application/octet-stream"); response.setContentLength((int) contentLength); } @@ -1879,9 +1783,19 @@ public String finalizeUpload( bytesRemaining -= bytesRead; } } - - return response; - +// Stream finished +if (base64Out != null) { + base64Out.close(); // Also flushes and closes the wrapped gzipOut +} else if (gzipOut != null) { + gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out +} else { + rawOut.flush(); // Flush only the base output stream if nothing was wrapped +} +if (!response.isCommitted()) { + response.setStatus(HttpServletResponse.SC_OK); + response.getWriter().write(" "); +} + } catch (IOException | InterruptedException | NumberFormatException | ApiException | DataException e) { LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); From f2b5802d9c11beccf25e6196761eae0d257dc4ca Mon Sep 17 00:00:00 2001 From: PhilReact Date: Fri, 16 May 2025 01:17:01 +0300 Subject: [PATCH 13/42] change to streaming --- .../org/qortal/utils/FilesystemUtils.java | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index e9921561..5a0f676c 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -6,6 +6,7 @@ import org.qortal.settings.Settings; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.RandomAccessFile; import java.nio.charset.StandardCharsets; import java.nio.file.*; @@ -232,31 +233,37 @@ public class FilesystemUtils { } public static byte[] getSingleFileContents(Path path, Integer maxLength) throws IOException { - byte[] data = null; - // TODO: limit the file size that can be loaded into memory - - // If the path is a file, read the contents directly - if (path.toFile().isFile()) { - int fileSize = (int)path.toFile().length(); - maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize; - data = FilesystemUtils.readFromFile(path.toString(), 0, maxLength); - } - - // Or if it's a directory, only load file contents if there is a single file inside it - else if (path.toFile().isDirectory()) { + Path filePath = null; + + if (Files.isRegularFile(path)) { + filePath = path; + } else if (Files.isDirectory(path)) { String[] files = ArrayUtils.removeElement(path.toFile().list(), ".qortal"); if (files.length == 1) { - Path filePath = Paths.get(path.toString(), files[0]); - if (filePath.toFile().isFile()) { - int fileSize = (int)filePath.toFile().length(); - maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize; - data = FilesystemUtils.readFromFile(filePath.toString(), 0, maxLength); - } + filePath = path.resolve(files[0]); } } - - return data; + + if (filePath == null || !Files.exists(filePath)) { + return null; + } + + long fileSize = Files.size(filePath); + int length = (maxLength != null) ? Math.min(maxLength, (int) Math.min(fileSize, Integer.MAX_VALUE)) : (int) Math.min(fileSize, Integer.MAX_VALUE); + + try (InputStream in = Files.newInputStream(filePath)) { + byte[] buffer = new byte[length]; + int bytesRead = in.read(buffer); + if (bytesRead < length) { + // Resize buffer to actual read size + byte[] trimmed = new byte[bytesRead]; + System.arraycopy(buffer, 0, trimmed, 0, bytesRead); + return trimmed; + } + return buffer; + } } + /** * isSingleFileResource From 2cd5f9e4cd5ee5f8df7336703160fc0a9ced2c2a Mon Sep 17 00:00:00 2001 From: PhilReact Date: Fri, 16 May 2025 01:18:02 +0300 Subject: [PATCH 14/42] change limit --- src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 518e27d0..96bdcdb5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -52,7 +52,7 @@ public class ArbitraryDataFile { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class); - public static final long MAX_FILE_SIZE = 3L * 1024 * 1024 * 1024; // 3 GiB + public static final long MAX_FILE_SIZE = 2L * 1024 * 1024 * 1024; // 2 GiB protected static final int MAX_CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static final int CHUNK_SIZE = 512 * 1024; // 0.5MiB public static int SHORT_DIGEST_LENGTH = 8; From 1c52c18d3208ae1d25ace32293981e028c8cbfd1 Mon Sep 17 00:00:00 2001 From: PhilReact Date: Fri, 16 May 2025 15:49:47 +0300 Subject: [PATCH 15/42] added endpoints --- .../api/resource/ArbitraryResource.java | 186 +++++++++++++++++- 1 file changed, 184 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 97146b8f..3ca3ce75 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -925,6 +925,187 @@ public class ArbitraryResource { return Response.ok("Sufficient disk space").build(); } + @POST +@Path("/{service}/{name}/chunk") +@Consumes(MediaType.MULTIPART_FORM_DATA) +@Operation( + summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource (no identifier)", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.MULTIPART_FORM_DATA, + schema = @Schema( + implementation = Object.class + ) + ) + ), + responses = { + @ApiResponse( + description = "Chunk uploaded successfully", + responseCode = "200" + ), + @ApiResponse( + description = "Error writing chunk", + responseCode = "500" + ) + } +) +@SecurityRequirement(name = "apiKey") +public Response uploadChunkNoIdentifier(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @FormDataParam("chunk") InputStream chunkStream, + @FormDataParam("index") int index) { + Security.checkApiCallAllowed(request); + + try { + java.nio.file.Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name); + Files.createDirectories(tempDir); + + java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); + Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING); + + return Response.ok("Chunk " + index + " received").build(); + } catch (IOException e) { + return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); + } +} + +@POST +@Path("/{service}/{name}/finalize") +@Produces(MediaType.TEXT_PLAIN) +@Operation( + summary = "Finalize a chunked upload (no identifier) and build a raw, unsigned, ARBITRARY transaction", + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content(mediaType = MediaType.TEXT_PLAIN) + ) + } +) +@SecurityRequirement(name = "apiKey") +public String finalizeUploadNoIdentifier( + @HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") List tags, + @QueryParam("category") Category category, + @QueryParam("filename") String filename, + @QueryParam("fee") Long fee, + @QueryParam("preview") Boolean preview, + @QueryParam("isZip") Boolean isZip +) { + Security.checkApiCallAllowed(request); + java.nio.file.Path tempFile = null; + java.nio.file.Path tempDir = null; + java.nio.file.Path chunkDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name); + + try { + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); + } + + String safeFilename = (filename == null || filename.isBlank()) ? "qortal-" + NTP.getTime() : filename; + tempDir = Files.createTempDirectory("qortal-"); + tempFile = tempDir.resolve(safeFilename); + + try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + byte[] buffer = new byte[65536]; + for (java.nio.file.Path chunk : Files.list(chunkDir) + .filter(path -> path.getFileName().toString().startsWith("chunk_")) + .sorted(Comparator.comparingInt(path -> { + String name2 = path.getFileName().toString(); + String numberPart = name2.substring("chunk_".length()); + return Integer.parseInt(numberPart); + })).collect(Collectors.toList())) { + try (InputStream in = Files.newInputStream(chunk)) { + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } + } + + String detectedExtension = ""; + String uploadFilename = null; + boolean extensionIsValid = false; + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + if (lastDot > 0 && lastDot < filename.length() - 1) { + extensionIsValid = true; + uploadFilename = filename; + } + } + + if (!extensionIsValid) { + Tika tika = new Tika(); + String mimeType = tika.detect(tempFile.toFile()); + try { + MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); + org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType); + detectedExtension = mime.getExtension(); + } catch (MimeTypeException e) { + LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e); + } + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename; + uploadFilename = baseName + (detectedExtension != null ? detectedExtension : ""); + } else { + uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : ""); + } + } + + // ✅ Call upload with `null` as identifier + return this.upload( + Service.valueOf(serviceString), + name, + null, // no identifier + tempFile.toString(), + null, + null, + isZip, + fee, + uploadFilename, + title, + description, + tags, + category, + preview + ); + + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); + } finally { + if (tempDir != null) { + try { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete temp directory: {}", tempDir, e); + } + } + + try { + Files.walk(chunkDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e); + } + } +} + + + @POST @Path("/{service}/{name}/{identifier}/chunk") @Consumes(MediaType.MULTIPART_FORM_DATA) @@ -996,7 +1177,8 @@ public String finalizeUpload( @QueryParam("category") Category category, @QueryParam("filename") String filename, @QueryParam("fee") Long fee, - @QueryParam("preview") Boolean preview + @QueryParam("preview") Boolean preview, + @QueryParam("isZip") Boolean isZip ) { Security.checkApiCallAllowed(request); java.nio.file.Path tempFile = null; @@ -1080,7 +1262,7 @@ public String finalizeUpload( tempFile.toString(), null, null, - false, + isZip, fee, uploadFilename, title, From e1ea8d65f8638af8e794f2dd920fbefc41d2339c Mon Sep 17 00:00:00 2001 From: PhilReact Date: Fri, 16 May 2025 23:39:32 +0300 Subject: [PATCH 16/42] fix blank filename issue --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 3ca3ce75..a187aeb5 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1765,7 +1765,7 @@ public String finalizeUpload( if (path == null) { // See if we have a string instead if (string != null) { - if (filename == null) { + if (filename == null || filename.isBlank()) { // Use current time as filename filename = String.format("qortal-%d", NTP.getTime()); } @@ -1780,7 +1780,7 @@ public String finalizeUpload( } // ... or base64 encoded raw data else if (base64 != null) { - if (filename == null) { + if (filename == null || filename.isBlank()) { // Use current time as filename filename = String.format("qortal-%d", NTP.getTime()); } From 58ab02c4f0c81bec88ed401c5ad4d92b9370085e Mon Sep 17 00:00:00 2001 From: PhilReact Date: Sun, 18 May 2025 23:21:49 +0300 Subject: [PATCH 17/42] fix to temp dir --- .../api/resource/ArbitraryResource.java | 85 +++++++++++++++---- 1 file changed, 69 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index a187aeb5..42621582 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -94,6 +94,7 @@ import org.apache.tika.mime.MimeTypes; import javax.ws.rs.core.Response; import org.glassfish.jersey.media.multipart.FormDataParam; +import static org.qortal.api.ApiError.REPOSITORY_ISSUE; @Path("/arbitrary") @Tag(name = "Arbitrary") @@ -914,14 +915,18 @@ public class ArbitraryResource { .entity("Missing or invalid totalSize parameter").build(); } - File baseTmp = new File(System.getProperty("java.io.tmpdir")); - long usableSpace = baseTmp.getUsableSpace(); - long requiredSpace = totalSize * 2; // chunked + merged file estimate - - if (usableSpace < requiredSpace) { - return Response.status(507).entity("Insufficient disk space").build(); // 507 = Insufficient Storage + File uploadDir = new File("uploads-temp"); + if (!uploadDir.exists()) { + uploadDir.mkdirs(); // ensure the folder exists } - + + long usableSpace = uploadDir.getUsableSpace(); + long requiredSpace = totalSize * 2; // estimate for chunks + merge + + if (usableSpace < requiredSpace) { + return Response.status(507).entity("Insufficient disk space").build(); + } + return Response.ok("Sufficient disk space").build(); } @@ -959,14 +964,20 @@ public Response uploadChunkNoIdentifier(@HeaderParam(Security.API_KEY_HEADER) St Security.checkApiCallAllowed(request); try { - java.nio.file.Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name); - Files.createDirectories(tempDir); + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + + java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName); + Files.createDirectories(tempDir); + + java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING); return Response.ok("Chunk " + index + " received").build(); } catch (IOException e) { + LOGGER.error("Failed to write chunk {} for service '{}' and name '{}'", index, serviceString, name, e); return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); } } @@ -1000,16 +1011,23 @@ public String finalizeUploadNoIdentifier( Security.checkApiCallAllowed(request); java.nio.file.Path tempFile = null; java.nio.file.Path tempDir = null; - java.nio.file.Path chunkDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name); + java.nio.file.Path chunkDir = null; + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + + try { + chunkDir = Paths.get("uploads-temp", safeService, safeName); + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); } String safeFilename = (filename == null || filename.isBlank()) ? "qortal-" + NTP.getTime() : filename; tempDir = Files.createTempDirectory("qortal-"); - tempFile = tempDir.resolve(safeFilename); + String sanitizedFilename = Paths.get(safeFilename).getFileName().toString(); + tempFile = tempDir.resolve(sanitizedFilename); try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { byte[] buffer = new byte[65536]; @@ -1061,6 +1079,13 @@ public String finalizeUploadNoIdentifier( } } + Boolean isZipBoolean = false; + + if (isZip != null && isZip) { + isZipBoolean = true; + } + + // ✅ Call upload with `null` as identifier return this.upload( Service.valueOf(serviceString), @@ -1069,7 +1094,7 @@ public String finalizeUploadNoIdentifier( tempFile.toString(), null, null, - isZip, + isZipBoolean, fee, uploadFilename, title, @@ -1080,6 +1105,8 @@ public String finalizeUploadNoIdentifier( ); } catch (IOException e) { + LOGGER.error("Failed to merge chunks for service='{}', name='{}'", serviceString, name, e); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); } finally { if (tempDir != null) { @@ -1141,7 +1168,12 @@ public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey, Security.checkApiCallAllowed(request); try { - java.nio.file.Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + String safeIdentifier = Paths.get(identifier).getFileName().toString(); + + java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName, safeIdentifier); + Files.createDirectories(tempDir); java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); @@ -1149,6 +1181,7 @@ public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey, return Response.ok("Chunk " + index + " received").build(); } catch (IOException e) { + LOGGER.error("Failed to write chunk {} for service='{}', name='{}', identifier='{}'", index, serviceString, name, identifier, e); return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); } } @@ -1183,9 +1216,19 @@ public String finalizeUpload( Security.checkApiCallAllowed(request); java.nio.file.Path tempFile = null; java.nio.file.Path tempDir = null; - java.nio.file.Path chunkDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + java.nio.file.Path chunkDir = null; + + + + try { + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + String safeIdentifier = Paths.get(identifier).getFileName().toString(); + java.nio.file.Path baseUploadsDir = Paths.get("uploads-temp"); // relative to Qortal working dir + chunkDir = baseUploadsDir.resolve(safeService).resolve(safeName).resolve(safeIdentifier); + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); } @@ -1197,7 +1240,9 @@ public String finalizeUpload( } tempDir = Files.createTempDirectory("qortal-"); - tempFile = tempDir.resolve(safeFilename); + String sanitizedFilename = Paths.get(safeFilename).getFileName().toString(); + tempFile = tempDir.resolve(sanitizedFilename); + // Step 2: Merge chunks @@ -1253,6 +1298,12 @@ public String finalizeUpload( } } + + Boolean isZipBoolean = false; + + if (isZip != null && isZip) { + isZipBoolean = true; + } return this.upload( @@ -1262,7 +1313,7 @@ public String finalizeUpload( tempFile.toString(), null, null, - isZip, + isZipBoolean, fee, uploadFilename, title, @@ -1273,6 +1324,8 @@ public String finalizeUpload( ); } catch (IOException e) { + LOGGER.error("Unexpected error in finalizeUpload for service='{}', name='{}', name='{}'", serviceString, name, identifier, e); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); } finally { if (tempDir != null) { From ca88cb1f887a899b34d541ea6000cfa07507b867 Mon Sep 17 00:00:00 2001 From: PhilReact Date: Mon, 19 May 2025 16:55:12 +0300 Subject: [PATCH 18/42] allow downloads --- .../api/resource/ArbitraryResource.java | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 42621582..2b912e0b 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -709,14 +709,14 @@ public class ArbitraryResource { @QueryParam("encoding") String encoding, @QueryParam("rebuild") boolean rebuild, @QueryParam("async") boolean async, - @QueryParam("attempts") Integer attempts) { + @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) { // Authentication can be bypassed in the settings, for those running public QDN nodes if (!Settings.getInstance().isQDNAuthBypassEnabled()) { Security.checkApiCallAllowed(request); } - this.download(service, name, null, filepath, encoding, rebuild, async, attempts); + this.download(service, name, null, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename); } @GET @@ -743,14 +743,14 @@ public class ArbitraryResource { @QueryParam("encoding") String encoding, @QueryParam("rebuild") boolean rebuild, @QueryParam("async") boolean async, - @QueryParam("attempts") Integer attempts) { + @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) { // Authentication can be bypassed in the settings, for those running public QDN nodes if (!Settings.getInstance().isQDNAuthBypassEnabled()) { Security.checkApiCallAllowed(request, null); } - this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts); + this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename); } @@ -1900,8 +1900,9 @@ public String finalizeUpload( } } - private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts, boolean attachment, String attachmentFilename) { try { + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); int attempts = 0; @@ -1948,6 +1949,33 @@ public String finalizeUpload( throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No file exists at filepath: " + filepath); } + if (attachment) { + String rawFilename; + + if (attachmentFilename != null && !attachmentFilename.isEmpty()) { + // 1. Sanitize first + String safeAttachmentFilename = attachmentFilename.replaceAll("[\\\\/:*?\"<>|]", "_"); + + // 2. Check for a valid extension (3–5 alphanumeric chars) + if (!safeAttachmentFilename.matches(".*\\.[a-zA-Z0-9]{2,5}$")) { + safeAttachmentFilename += ".bin"; + } + + rawFilename = safeAttachmentFilename; + } else { + // Fallback if no filename is provided + String baseFilename = (identifier != null && !identifier.isEmpty()) + ? name + "-" + identifier + : name; + rawFilename = baseFilename.replaceAll("[\\\\/:*?\"<>|]", "_") + ".bin"; + } + + // Optional: trim length + rawFilename = rawFilename.length() > 100 ? rawFilename.substring(0, 100) : rawFilename; + + // 3. Set Content-Disposition header + response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\""); + } long fileSize = Files.size(path); String mimeType = context.getMimeType(path.toString()); String range = request.getHeader("Range"); From 88fe3b0af661ac242487056effc93b6d67837623 Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 23 May 2025 17:49:26 -0700 Subject: [PATCH 19/42] primary names implementation --- src/main/java/org/qortal/account/Account.java | 142 ++++++++++++++ .../qortal/api/resource/NamesResource.java | 40 ++++ src/main/java/org/qortal/block/Block.java | 4 + .../org/qortal/block/PrimaryNamesBlock.java | 47 +++++ .../java/org/qortal/data/naming/NameData.java | 5 + .../org/qortal/repository/NameRepository.java | 10 +- .../hsqldb/HSQLDBChatRepository.java | 22 ++- .../hsqldb/HSQLDBDatabaseUpdates.java | 6 + .../hsqldb/HSQLDBNameRepository.java | 50 +++++ .../transaction/BuyNameTransaction.java | 40 +++- .../transaction/RegisterNameTransaction.java | 22 +++ .../transaction/UpdateNameTransaction.java | 27 +++ .../org/qortal/test/naming/BuySellTests.java | 63 +++++++ .../qortal/test/naming/IntegrityTests.java | 12 +- .../org/qortal/test/naming/MiscTests.java | 173 +++++++++++++++++- .../org/qortal/test/naming/UpdateTests.java | 121 ++++++++++++ 16 files changed, 778 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/qortal/block/PrimaryNamesBlock.java diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index 722e70da..f741b166 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -2,12 +2,14 @@ package org.qortal.account; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.api.resource.TransactionsResource; import org.qortal.block.BlockChain; import org.qortal.controller.LiteNode; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.naming.NameData; +import org.qortal.data.transaction.TransactionData; import org.qortal.repository.DataException; import org.qortal.repository.GroupRepository; import org.qortal.repository.NameRepository; @@ -19,7 +21,11 @@ import org.qortal.utils.Groups; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Comparator; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Optional; import static org.qortal.utils.Amounts.prettyAmount; @@ -361,6 +367,142 @@ public class Account { return accountData.getLevel(); } + /** + * Get Primary Name + * + * @return the primary name for this address if present, otherwise empty + * + * @throws DataException + */ + public Optional getPrimaryName() throws DataException { + + return this.repository.getNameRepository().getPrimaryName(this.address); + } + + /** + * Remove Primary Name + * + * @throws DataException + */ + public void removePrimaryName() throws DataException { + this.repository.getNameRepository().removePrimaryName(this.address); + } + + /** + * Reset Primary Name + * + * Set primary name based on the names (and their history) this account owns. + * + * @param confirmationStatus the status of the transactions for the determining the primary name + * + * @return the primary name, empty if their isn't one + * + * @throws DataException + */ + public Optional resetPrimaryName(TransactionsResource.ConfirmationStatus confirmationStatus) throws DataException { + Optional primaryName = determinePrimaryName(confirmationStatus); + + if(primaryName.isPresent()) { + return setPrimaryName(primaryName.get()); + } + else { + return primaryName; + } + } + + /** + * Determine Primary Name + * + * Determine primary name based on a list of registered names. + * + * @param confirmationStatus the status of the transactions for this determination + * + * @return the primary name, empty if there is no primary name + * + * @throws DataException + */ + public Optional determinePrimaryName(TransactionsResource.ConfirmationStatus confirmationStatus) throws DataException { + + // all registered names for the owner + List names = this.repository.getNameRepository().getNamesByOwner(this.address); + + Optional primaryName; + + // if no registered names, the no primary name possible + if (names.isEmpty()) { + primaryName = Optional.empty(); + } + // if names + else { + // if one name, then that is the primary name + if (names.size() == 1) { + primaryName = Optional.of( names.get(0).getName() ); + } + // if more than one name, then seek the earliest name acquisition that was never released + else { + Map txByName = new HashMap<>(names.size()); + + // for each name, get the latest transaction + for (NameData nameData : names) { + + // since the name is currently registered to the owner, + // we assume the latest transaction involving this name was the transaction that the acquired + // name through registration, purchase or update + Optional latestTransaction + = this.repository + .getTransactionRepository() + .getTransactionsInvolvingName( + nameData.getName(), + confirmationStatus + ) + .stream() + .sorted(Comparator.comparing( + TransactionData::getTimestamp).reversed() + ) + .findFirst(); // first is the last, since it was reversed + + // if there is a latest transaction, expected for all registered names + if (latestTransaction.isPresent()) { + txByName.put(nameData.getName(), latestTransaction.get()); + } + // if there is no latest transaction, then + else { + LOGGER.warn("No matching transaction for name: " + nameData.getName()); + } + } + + // get the first name aqcuistion for this address + Optional> firstNameEntry + = txByName.entrySet().stream().sorted(Comparator.comparing(entry -> entry.getValue().getTimestamp())).findFirst(); + + // if their is a name acquisition, then the first one is the primary name + if (firstNameEntry.isPresent()) { + primaryName = Optional.of( firstNameEntry.get().getKey() ); + } + // if there is no nameacquistion, then there is no primary name + else { + primaryName = Optional.empty(); + } + } + } + return primaryName; + } + + /** + * Set Primary Name + * + * @param primaryName the primary to set to this address + * + * @return the primary name if successful, empty if unsuccessful + * + * @throws DataException + */ + public Optional setPrimaryName( String primaryName ) throws DataException { + int changed = this.repository.getNameRepository().setPrimaryName(this.address, primaryName); + + return changed > 0 ? Optional.of(primaryName) : Optional.empty(); + } + /** * Returns reward-share minting address, or unknown if reward-share does not exist. * diff --git a/src/main/java/org/qortal/api/resource/NamesResource.java b/src/main/java/org/qortal/api/resource/NamesResource.java index c7d4a425..a28615fd 100644 --- a/src/main/java/org/qortal/api/resource/NamesResource.java +++ b/src/main/java/org/qortal/api/resource/NamesResource.java @@ -33,6 +33,7 @@ import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; @Path("/names") @@ -104,6 +105,45 @@ public class NamesResource { } } + @GET + @Path("/primary/{address}") + @Operation( + summary = "primary name owned by address", + responses = { + @ApiResponse( + description = "registered primary name info", + content = @Content( + mediaType = MediaType.APPLICATION_JSON, + schema = @Schema(implementation = NameSummary.class) + ) + ) + } + ) + @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE, ApiError.UNAUTHORIZED}) + public NameSummary getPrimaryNameByAddress(@PathParam("address") String address) { + if (!Crypto.isValidAddress(address)) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); + + try (final Repository repository = RepositoryManager.getRepository()) { + + if (Settings.getInstance().isLite()) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED); + } + else { + Optional primaryName = repository.getNameRepository().getPrimaryName(address); + + if(primaryName.isPresent()) { + return new NameSummary(new NameData(primaryName.get(), address)); + } + else { + return new NameSummary((new NameData(null, address))); + } + } + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + @GET @Path("/{name}") @Operation( diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 67e6dd43..753b5dfa 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1640,6 +1640,8 @@ public class Block { SelfSponsorshipAlgoV2Block.processAccountPenalties(this); } else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) { SelfSponsorshipAlgoV3Block.processAccountPenalties(this); + } else if (this.blockData.getHeight() == BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + PrimaryNamesBlock.processNames(this.repository); } } } @@ -1952,6 +1954,8 @@ public class Block { SelfSponsorshipAlgoV2Block.orphanAccountPenalties(this); } else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) { SelfSponsorshipAlgoV3Block.orphanAccountPenalties(this); + } else if (this.blockData.getHeight() == BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + PrimaryNamesBlock.orphanNames( this.repository ); } } diff --git a/src/main/java/org/qortal/block/PrimaryNamesBlock.java b/src/main/java/org/qortal/block/PrimaryNamesBlock.java new file mode 100644 index 00000000..3dd21344 --- /dev/null +++ b/src/main/java/org/qortal/block/PrimaryNamesBlock.java @@ -0,0 +1,47 @@ +package org.qortal.block; + +import org.qortal.account.Account; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.data.naming.NameData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class PrimaryNamesBlock + */ +public class PrimaryNamesBlock { + + /** + * Process Primary Names + * + * @param repository + * @throws DataException + */ + public static void processNames(Repository repository) throws DataException { + + Set addressesWithNames + = repository.getNameRepository().getAllNames().stream() + .map(NameData::getOwner).collect(Collectors.toSet()); + + // for each address with a name, set primary name to the address + for( String address : addressesWithNames ) { + + Account account = new Account(repository, address); + account.resetPrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED); + } + } + + /** + * Orphan the Primary Names Block + * + * @param repository + * @throws DataException + */ + public static void orphanNames(Repository repository) throws DataException { + + repository.getNameRepository().clearPrimaryNames(); + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/data/naming/NameData.java b/src/main/java/org/qortal/data/naming/NameData.java index 16e490a2..c76b7d48 100644 --- a/src/main/java/org/qortal/data/naming/NameData.java +++ b/src/main/java/org/qortal/data/naming/NameData.java @@ -67,6 +67,11 @@ public class NameData { this(name, reducedName, owner, data, registered, null, false, null, reference, creationGroupId); } + // Typically used for name summsry + public NameData(String name, String owner) { + this(name, null, owner, null, 0L, null, false, null, null, 0); + } + // Getters / setters public String getName() { diff --git a/src/main/java/org/qortal/repository/NameRepository.java b/src/main/java/org/qortal/repository/NameRepository.java index c49d5d18..c61bb96f 100644 --- a/src/main/java/org/qortal/repository/NameRepository.java +++ b/src/main/java/org/qortal/repository/NameRepository.java @@ -3,6 +3,7 @@ package org.qortal.repository; import org.qortal.data.naming.NameData; import java.util.List; +import java.util.Optional; public interface NameRepository { @@ -34,10 +35,17 @@ public interface NameRepository { return getNamesByOwner(address, null, null, null); } + public int setPrimaryName(String address, String primaryName) throws DataException; + + public void removePrimaryName(String address) throws DataException; + + public Optional getPrimaryName(String address) throws DataException; + + public int clearPrimaryNames() throws DataException; + public List getRecentNames(long startTimestamp) throws DataException; public void save(NameData nameData) throws DataException; public void delete(String name) throws DataException; - } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java index 80865739..48262dee 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java @@ -1,5 +1,8 @@ package org.qortal.repository.hsqldb; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.BlockChain; import org.qortal.data.chat.ActiveChats; import org.qortal.data.chat.ActiveChats.DirectChat; import org.qortal.data.chat.ActiveChats.GroupChat; @@ -18,6 +21,8 @@ import static org.qortal.data.chat.ChatMessage.Encoding; public class HSQLDBChatRepository implements ChatRepository { + private static final Logger LOGGER = LogManager.getLogger(HSQLDBChatRepository.class); + protected HSQLDBRepository repository; public HSQLDBChatRepository(HSQLDBRepository repository) { @@ -142,10 +147,23 @@ public class HSQLDBChatRepository implements ChatRepository { @Override public ChatMessage toChatMessage(ChatTransactionData chatTransactionData, Encoding encoding) throws DataException { + + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.info("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.info("using Names for chat transactions"); + tableName = "Names"; + } + String sql = "SELECT SenderNames.name, RecipientNames.name " + "FROM ChatTransactions " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " - + "LEFT OUTER JOIN Names AS RecipientNames ON RecipientNames.owner = recipient " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS RecipientNames ON RecipientNames.owner = recipient " + "WHERE signature = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, chatTransactionData.getSignature())) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index ca55f3a8..69a06b6a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -1053,6 +1053,12 @@ public class HSQLDBDatabaseUpdates { stmt.execute("UPDATE Accounts SET blocks_minted_penalty = -5000000 WHERE blocks_minted_penalty < 0"); break; + case 50: + // Primary name for a Qortal Address, 0-1 for any address + stmt.execute("CREATE TABLE PrimaryNames (owner QortalAddress, name RegisteredName, " + + "PRIMARY KEY (owner), FOREIGN KEY (name) REFERENCES Names (name) ON DELETE CASCADE)"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java index 7bcdebda..fba1e83d 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java @@ -8,6 +8,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; public class HSQLDBNameRepository implements NameRepository { @@ -333,6 +334,55 @@ public class HSQLDBNameRepository implements NameRepository { } } + @Override + public void removePrimaryName(String address) throws DataException { + try { + this.repository.delete("PrimaryNames", "owner = ?", address); + } catch (SQLException e) { + throw new DataException("Unable to delete primary name from repository", e); + } + } + + @Override + public Optional getPrimaryName(String address) throws DataException { + String sql = "SELECT name FROM PrimaryNames WHERE owner = ?"; + + List names = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql, address)) { + if (resultSet == null) + return Optional.empty(); + + String name = resultSet.getString(1); + + return Optional.of(name); + } catch (SQLException e) { + throw new DataException("Unable to fetch recent names from repository", e); + } + } + + @Override + public int setPrimaryName(String address, String primaryName) throws DataException { + + String sql = "INSERT INTO PrimaryNames (owner, name) VALUES (?, ?) ON DUPLICATE KEY UPDATE name = ?"; + + try{ + return this.repository.executeCheckedUpdate(sql, address, primaryName, primaryName); + } catch (SQLException e) { + throw new DataException("Unable to set primary name", e); + } + } + + @Override + public int clearPrimaryNames() throws DataException { + + try { + return this.repository.delete("PrimaryNames"); + } catch (SQLException e) { + throw new DataException("Unable to clear primary names from repository", e); + } + } + @Override public void save(NameData nameData) throws DataException { HSQLDBSaver saveHelper = new HSQLDBSaver("Names"); diff --git a/src/main/java/org/qortal/transaction/BuyNameTransaction.java b/src/main/java/org/qortal/transaction/BuyNameTransaction.java index b7ca1d93..6c390085 100644 --- a/src/main/java/org/qortal/transaction/BuyNameTransaction.java +++ b/src/main/java/org/qortal/transaction/BuyNameTransaction.java @@ -16,6 +16,7 @@ import org.qortal.utils.Unicode; import java.util.Collections; import java.util.List; +import java.util.Optional; public class BuyNameTransaction extends Transaction { @@ -117,6 +118,25 @@ public class BuyNameTransaction extends Transaction { // Save transaction with updated "name reference" pointing to previous transaction that changed name this.repository.getTransactionRepository().save(this.buyNameTransactionData); + + // if multiple names feature is activated, then check the buyer and seller's primary name status + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account seller = new Account(this.repository, this.buyNameTransactionData.getSeller()); + Optional sellerPrimaryName = seller.getPrimaryName(); + + // if the seller sold their primary name, then remove their primary name + if (sellerPrimaryName.isPresent() && sellerPrimaryName.get().equals(buyNameTransactionData.getName())) { + seller.removePrimaryName(); + } + + Account buyer = new Account(this.repository, this.getBuyer().getAddress()); + + // if the buyer had no primary name, then set the primary name to the name bought + if( buyer.getPrimaryName().isEmpty() ) { + buyer.setPrimaryName(this.buyNameTransactionData.getName()); + } + } } @Override @@ -127,6 +147,24 @@ public class BuyNameTransaction extends Transaction { // Save this transaction, with previous "name reference" this.repository.getTransactionRepository().save(this.buyNameTransactionData); - } + // if multiple names feature is activated, then check the buyer and seller's primary name status + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account seller = new Account(this.repository, this.buyNameTransactionData.getSeller()); + + // if the seller lost their primary name, then set their primary name back + if (seller.getPrimaryName().isEmpty()) { + seller.setPrimaryName(this.buyNameTransactionData.getName()); + } + + Account buyer = new Account(this.repository, this.getBuyer().getAddress()); + Optional buyerPrimaryName = buyer.getPrimaryName(); + + // if the buyer bought their primary, then remove it + if( buyerPrimaryName.isPresent() && this.buyNameTransactionData.getName().equals(buyerPrimaryName.get()) ) { + buyer.removePrimaryName(); + } + } + } } diff --git a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java index c4520fbf..8e8e2fcc 100644 --- a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java +++ b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java @@ -2,10 +2,12 @@ package org.qortal.transaction; import com.google.common.base.Utf8; import org.qortal.account.Account; +import org.qortal.api.resource.TransactionsResource; import org.qortal.asset.Asset; import org.qortal.block.BlockChain; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; +import org.qortal.data.naming.NameData; import org.qortal.data.transaction.RegisterNameTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.naming.Name; @@ -15,6 +17,7 @@ import org.qortal.utils.Unicode; import java.util.Collections; import java.util.List; +import java.util.Optional; public class RegisterNameTransaction extends Transaction { @@ -54,6 +57,15 @@ public class RegisterNameTransaction extends Transaction { Account registrant = getRegistrant(); String name = this.registerNameTransactionData.getName(); + Optional registrantPrimaryName = registrant.getPrimaryName(); + if( registrantPrimaryName.isPresent() ) { + + NameData nameData = repository.getNameRepository().fromName(registrantPrimaryName.get()); + if (nameData.isForSale()) { + return ValidationResult.NOT_SUPPORTED; + } + } + int blockchainHeight = this.repository.getBlockRepository().getBlockchainHeight(); final int start = BlockChain.getInstance().getSelfSponsorshipAlgoV2Height() - 1180; final int end = BlockChain.getInstance().getSelfSponsorshipAlgoV3Height(); @@ -117,6 +129,16 @@ public class RegisterNameTransaction extends Transaction { // Register Name Name name = new Name(this.repository, this.registerNameTransactionData); name.register(); + + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account account = new Account(this.repository, this.getCreator().getAddress()); + + // if there is no primary name established, then the new registered name is the primary name + if (account.getPrimaryName().isEmpty()) { + account.setPrimaryName(this.registerNameTransactionData.getName()); + } + } } @Override diff --git a/src/main/java/org/qortal/transaction/UpdateNameTransaction.java b/src/main/java/org/qortal/transaction/UpdateNameTransaction.java index 8d42207c..bf0d12dc 100644 --- a/src/main/java/org/qortal/transaction/UpdateNameTransaction.java +++ b/src/main/java/org/qortal/transaction/UpdateNameTransaction.java @@ -3,6 +3,7 @@ package org.qortal.transaction; import com.google.common.base.Utf8; import org.qortal.account.Account; import org.qortal.asset.Asset; +import org.qortal.block.BlockChain; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.crypto.Crypto; import org.qortal.data.naming.NameData; @@ -49,6 +50,12 @@ public class UpdateNameTransaction extends Transaction { public ValidationResult isValid() throws DataException { String name = this.updateNameTransactionData.getName(); + // if the account has more than one name, then they cannot update their primary name + if( this.repository.getNameRepository().getNamesByOwner(this.getOwner().getAddress()).size() > 1 && + this.getOwner().getPrimaryName().get().equals(name) ) { + return ValidationResult.NOT_SUPPORTED; + } + // Check name size bounds int nameLength = Utf8.encodedLength(name); if (nameLength < Name.MIN_NAME_SIZE || nameLength > Name.MAX_NAME_SIZE) @@ -152,6 +159,16 @@ public class UpdateNameTransaction extends Transaction { // Save this transaction, now with updated "name reference" to previous transaction that changed name this.repository.getTransactionRepository().save(this.updateNameTransactionData); + + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account account = new Account(this.repository, this.getCreator().getAddress()); + + // if updating the primary name, then set primary name to new name + if( account.getPrimaryName().isEmpty() || account.getPrimaryName().get().equals(this.updateNameTransactionData.getName())) { + account.setPrimaryName(this.updateNameTransactionData.getNewName()); + } + } } @Override @@ -167,6 +184,16 @@ public class UpdateNameTransaction extends Transaction { // Save this transaction, with previous "name reference" this.repository.getTransactionRepository().save(this.updateNameTransactionData); + + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + + Account account = new Account(this.repository, this.getCreator().getAddress()); + + // if the primary name is the new updated name, then it needs to be set back to the previous name + if (account.getPrimaryName().isPresent() && account.getPrimaryName().get().equals(this.updateNameTransactionData.getNewName())) { + account.setPrimaryName(this.updateNameTransactionData.getName()); + } + } } } diff --git a/src/test/java/org/qortal/test/naming/BuySellTests.java b/src/test/java/org/qortal/test/naming/BuySellTests.java index 2283404c..9788bd7c 100644 --- a/src/test/java/org/qortal/test/naming/BuySellTests.java +++ b/src/test/java/org/qortal/test/naming/BuySellTests.java @@ -4,6 +4,7 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.resource.TransactionsResource; import org.qortal.block.BlockChain; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.BuyNameTransactionData; @@ -22,6 +23,7 @@ import org.qortal.transaction.Transaction; import org.qortal.utils.Amounts; import java.util.List; +import java.util.Optional; import java.util.Random; import static org.junit.Assert.*; @@ -135,13 +137,26 @@ public class BuySellTests extends Common { @Test public void testSellName() throws DataException { + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Register-name testRegisterName(); + // assert primary name for alice + Optional alicePrimaryName1 = alice.getPrimaryName(); + assertTrue(alicePrimaryName1.isPresent()); + assertTrue(alicePrimaryName1.get().equals(name)); + // Sell-name SellNameTransactionData transactionData = new SellNameTransactionData(TestTransaction.generateBase(alice), name, price); TransactionUtils.signAndMint(repository, transactionData, alice); + // assert primary name for alice + Optional alicePrimaryName2 = alice.getPrimaryName(); + assertTrue(alicePrimaryName2.isPresent()); + assertTrue(alicePrimaryName2.get().equals(name)); + NameData nameData; // Check name is for sale @@ -149,6 +164,14 @@ public class BuySellTests extends Common { assertTrue(nameData.isForSale()); assertEquals("price incorrect", price, nameData.getSalePrice()); + // assert alice cannot register another name while primary name is for sale + final String name2 = "another name"; + RegisterNameTransactionData registerSecondNameData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult registrationResult = TransactionUtils.signAndImport(repository, registerSecondNameData, alice); + + // check that registering is not supported while primary name is for sale + assertTrue(Transaction.ValidationResult.NOT_SUPPORTED.equals(registrationResult)); + // Orphan sell-name BlockUtils.orphanLastBlock(repository); @@ -168,6 +191,10 @@ public class BuySellTests extends Common { // Orphan sell-name and register-name BlockUtils.orphanBlocks(repository, 2); + // assert primary name for alice + Optional alicePrimaryName3 = alice.getPrimaryName(); + assertTrue(alicePrimaryName3.isEmpty()); + // Check name no longer exists assertFalse(repository.getNameRepository().nameExists(name)); nameData = repository.getNameRepository().fromName(name); @@ -261,15 +288,36 @@ public class BuySellTests extends Common { @Test public void testBuyName() throws DataException { + // move passed primary initiation + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Register-name and sell-name testSellName(); String seller = alice.getAddress(); + // assert alice has the name as primary + Optional alicePrimaryName1 = alice.getPrimaryName(); + assertTrue(alicePrimaryName1.isPresent()); + assertEquals(name, alicePrimaryName1.get()); + + // assert bob does not have a primary name + Optional bobPrimaryName1 = bob.getPrimaryName(); + assertTrue(bobPrimaryName1.isEmpty()); + // Buy-name BuyNameTransactionData transactionData = new BuyNameTransactionData(TestTransaction.generateBase(bob), name, price, seller); TransactionUtils.signAndMint(repository, transactionData, bob); + // assert alice does not have a primary name anymore + Optional alicePrimaryName2 = alice.getPrimaryName(); + assertTrue(alicePrimaryName2.isEmpty()); + + // assert bob does have the name as primary + Optional bobPrimaryName2 = bob.getPrimaryName(); + assertTrue(bobPrimaryName2.isPresent()); + assertEquals(name, bobPrimaryName2.get()); + NameData nameData; // Check name is sold @@ -280,6 +328,15 @@ public class BuySellTests extends Common { // Orphan buy-name BlockUtils.orphanLastBlock(repository); + // assert alice has the name as primary + Optional alicePrimaryNameOrphaned = alice.getPrimaryName(); + assertTrue(alicePrimaryNameOrphaned.isPresent()); + assertEquals(name, alicePrimaryNameOrphaned.get()); + + // assert bob does not have a primary name + Optional bobPrimaryNameOrphaned = bob.getPrimaryName(); + assertTrue(bobPrimaryNameOrphaned.isEmpty()); + // Check name is for sale (not sold) nameData = repository.getNameRepository().fromName(name); assertTrue(nameData.isForSale()); @@ -314,6 +371,9 @@ public class BuySellTests extends Common { assertFalse(nameData.isForSale()); // Not concerned about price assertEquals(bob.getAddress(), nameData.getOwner()); + + assertEquals(alice.getPrimaryName(), alice.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); + assertEquals(bob.getPrimaryName(), bob.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); } @Test @@ -373,6 +433,9 @@ public class BuySellTests extends Common { assertTrue(nameData.isForSale()); assertEquals("price incorrect", newPrice, nameData.getSalePrice()); assertEquals(bob.getAddress(), nameData.getOwner()); + + assertEquals(alice.getPrimaryName(), alice.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); + assertEquals(bob.getPrimaryName(), bob.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); } } diff --git a/src/test/java/org/qortal/test/naming/IntegrityTests.java b/src/test/java/org/qortal/test/naming/IntegrityTests.java index 767ea388..14a6891b 100644 --- a/src/test/java/org/qortal/test/naming/IntegrityTests.java +++ b/src/test/java/org/qortal/test/naming/IntegrityTests.java @@ -4,6 +4,8 @@ import org.junit.Before; import org.junit.Ignore; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.block.BlockChain; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.data.naming.NameData; import org.qortal.data.transaction.*; @@ -13,6 +15,7 @@ import org.qortal.repository.RepositoryFactory; import org.qortal.repository.RepositoryManager; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; +import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; @@ -385,6 +388,8 @@ public class IntegrityTests extends Common { @Test public void testUpdateToMissingName() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Register-name PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String initialName = "test-name"; @@ -422,7 +427,12 @@ public class IntegrityTests extends Common { // Therefore the name that we are trying to rename TO already exists Transaction.ValidationResult result = transaction.importAsUnconfirmed(); assertTrue("Transaction should be invalid", Transaction.ValidationResult.OK != result); - assertTrue("Destination name should already exist", Transaction.ValidationResult.NAME_ALREADY_REGISTERED == result); + + // this assertion has been updated, because the primary name logic now comes into play and you cannot update a primary name when there + // is other names registered and if your try a NOT SUPPORTED result will be given + assertTrue("Destination name should already exist", Transaction.ValidationResult.NOT_SUPPORTED == result); + + assertEquals(alice.getPrimaryName(), alice.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); } } diff --git a/src/test/java/org/qortal/test/naming/MiscTests.java b/src/test/java/org/qortal/test/naming/MiscTests.java index 324cdf12..b85ca95c 100644 --- a/src/test/java/org/qortal/test/naming/MiscTests.java +++ b/src/test/java/org/qortal/test/naming/MiscTests.java @@ -1,6 +1,7 @@ package org.qortal.test.naming; import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; @@ -8,6 +9,7 @@ import org.qortal.api.AmountTypeAdapter; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.UnitFeesByTimestamp; import org.qortal.controller.BlockMinter; +import org.qortal.data.naming.NameData; import org.qortal.data.transaction.PaymentTransactionData; import org.qortal.data.transaction.RegisterNameTransactionData; import org.qortal.data.transaction.TransactionData; @@ -28,6 +30,7 @@ import org.qortal.utils.NTP; import java.util.Arrays; import java.util.List; +import java.util.Optional; import static org.junit.Assert.*; @@ -121,6 +124,8 @@ public class MiscTests extends Common { transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); TransactionUtils.signAndMint(repository, transactionData, alice); + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Register another name that we will later attempt to rename to first name (above) String otherName = "new-name"; String otherData = ""; @@ -335,6 +340,8 @@ public class MiscTests extends Common { public void testRegisterNameFeeIncrease() throws Exception { try (final Repository repository = RepositoryManager.getRepository()) { + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + // Add original fee to nameRegistrationUnitFees UnitFeesByTimestamp originalFee = new UnitFeesByTimestamp(); originalFee.timestamp = 0L; @@ -517,4 +524,168 @@ public class MiscTests extends Common { } } -} + @Test + public void testPrimaryNameEmpty() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + Optional primaryName = repository.getNameRepository().getPrimaryName(alice.getAddress()); + + Assert.assertNotNull(primaryName); + Assert.assertTrue(primaryName.isEmpty()); + } + } + + @Test + public void testPrimaryNameSingle() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + String name = "alice 1"; + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + + // register name 1 + RegisterNameTransactionData transactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData1, alice); + + String name1 = transactionData1.getName(); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name1)); + + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight() + 1); + + Optional primaryName = repository.getNameRepository().getPrimaryName(alice.getAddress()); + + Assert.assertNotNull(primaryName); + Assert.assertTrue(primaryName.isPresent()); + Assert.assertEquals(name, primaryName.get()); + } + } + + @Test + public void testPrimaryNameSingleAfterFeature() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + String name = "alice 1"; + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // register name 1 + RegisterNameTransactionData transactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData1, alice); + + String name1 = transactionData1.getName(); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name1)); + + + Optional primaryName = repository.getNameRepository().getPrimaryName(alice.getAddress()); + + Assert.assertNotNull(primaryName); + Assert.assertTrue(primaryName.isPresent()); + Assert.assertEquals(name, primaryName.get()); + + BlockUtils.orphanLastBlock(repository); + + Optional primaryNameOrpaned = repository.getNameRepository().getPrimaryName(alice.getAddress()); + + Assert.assertNotNull(primaryNameOrpaned); + Assert.assertTrue(primaryNameOrpaned.isEmpty()); + } + } + + @Test + public void testUpdateNameMultiple() throws DataException { + + try (final Repository repository = RepositoryManager.getRepository()) { + String name = "alice 1"; + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + + // register name 1 + RegisterNameTransactionData transactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData1, alice); + + String name1 = transactionData1.getName(); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name1)); + + // register another name, second registered name should fail before the feature trigger + final String name2 = "another name"; + RegisterNameTransactionData transactionData2 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult resultBeforeFeatureTrigger = TransactionUtils.signAndImport(repository, transactionData2, alice); + + // check that that multiple names is forbidden + assertTrue(Transaction.ValidationResult.MULTIPLE_NAMES_FORBIDDEN.equals(resultBeforeFeatureTrigger)); + + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // register again, now that we are passed the feature trigger + RegisterNameTransactionData transactionData3 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult resultAfterFeatureTrigger = TransactionUtils.signAndImport(repository, transactionData3, alice); + + // check that multiple names is ok + assertTrue(Transaction.ValidationResult.OK.equals(resultAfterFeatureTrigger)); + + // mint block, confirm transaction + BlockUtils.mintBlock(repository); + + // check name does exist + assertTrue(repository.getNameRepository().nameExists(name2)); + + // check that there are 2 names for one account + List namesByOwner = repository.getNameRepository().getNamesByOwner(alice.getAddress(), 0, 0, false); + + assertEquals(2, namesByOwner.size()); + + // check that the order is correct + assertEquals(name1, namesByOwner.get(0).getName()); + + String newestName = "newest-name"; + String newestReducedName = "newest-name"; + String newestData = "newest-data"; + TransactionData newestTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name2, newestName, newestData); + TransactionUtils.signAndMint(repository, newestTransactionData, alice); + + // Check previous name no longer exists + assertFalse(repository.getNameRepository().nameExists(name2)); + + // Check newest name exists + assertTrue(repository.getNameRepository().nameExists(newestName)); + + Optional alicePrimaryName1 = alice.getPrimaryName(); + + assertTrue( alicePrimaryName1.isPresent() ); + assertEquals( name1, alicePrimaryName1.get() ); + + // orphan and recheck + BlockUtils.orphanLastBlock(repository); + + Optional alicePrimaryName2 = alice.getPrimaryName(); + + assertTrue( alicePrimaryName2.isPresent() ); + assertEquals( name1, alicePrimaryName2.get() ); + + // Check newest name no longer exists + assertFalse(repository.getNameRepository().nameExists(newestName)); + assertNull(repository.getNameRepository().fromReducedName(newestReducedName)); + + // Check previous name exists again + assertTrue(repository.getNameRepository().nameExists(name2)); + } + } +} \ No newline at end of file diff --git a/src/test/java/org/qortal/test/naming/UpdateTests.java b/src/test/java/org/qortal/test/naming/UpdateTests.java index 8e54eb96..01b9a2ee 100644 --- a/src/test/java/org/qortal/test/naming/UpdateTests.java +++ b/src/test/java/org/qortal/test/naming/UpdateTests.java @@ -3,8 +3,12 @@ package org.qortal.test.naming; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.api.resource.TransactionsResource; +import org.qortal.block.BlockChain; import org.qortal.data.naming.NameData; +import org.qortal.data.transaction.BuyNameTransactionData; import org.qortal.data.transaction.RegisterNameTransactionData; +import org.qortal.data.transaction.SellNameTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.UpdateNameTransactionData; import org.qortal.repository.DataException; @@ -15,6 +19,9 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.transaction.Transaction; + +import java.util.Optional; import static org.junit.Assert.*; @@ -395,6 +402,13 @@ public class UpdateTests extends Common { assertTrue(repository.getNameRepository().nameExists(initialName)); assertNotNull(repository.getNameRepository().fromReducedName(initialReducedName)); + // move passed primary initiation + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // check primary name + assertTrue(alice.getPrimaryName().isPresent()); + assertEquals(initialName, alice.getPrimaryName().get()); + // Update data String middleName = "middle-name"; String middleReducedName = "midd1e-name"; @@ -402,6 +416,11 @@ public class UpdateTests extends Common { transactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), initialName, middleName, middleData); TransactionUtils.signAndMint(repository, transactionData, alice); + // check primary name + Optional alicePrimaryName1 = alice.getPrimaryName(); + assertTrue(alicePrimaryName1.isPresent()); + assertEquals(middleName, alicePrimaryName1.get()); + // Check data is correct assertEquals(middleData, repository.getNameRepository().fromName(middleName).getData()); @@ -414,6 +433,11 @@ public class UpdateTests extends Common { // Check data is correct assertEquals(newestData, repository.getNameRepository().fromName(newestName).getData()); + // check primary name + Optional alicePrimaryName2 = alice.getPrimaryName(); + assertTrue(alicePrimaryName2.isPresent()); + assertEquals(newestName, alicePrimaryName2.get()); + // Check initial name no longer exists assertFalse(repository.getNameRepository().nameExists(initialName)); assertNull(repository.getNameRepository().fromReducedName(initialReducedName)); @@ -516,4 +540,101 @@ public class UpdateTests extends Common { } } + @Test + public void testUpdatePrimaryName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + + // register name 1 + String initialName = "initial-name"; + RegisterNameTransactionData registerNameTransactionData1 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, "{}"); + registerNameTransactionData1.setFee(new RegisterNameTransaction(null, null).getUnitFee(registerNameTransactionData1.getTimestamp())); + TransactionUtils.signAndMint(repository, registerNameTransactionData1, alice); + + // assert name 1 registration, assert primary name + assertTrue(repository.getNameRepository().nameExists(initialName)); + + Optional primaryNameOptional = alice.getPrimaryName(); + assertTrue(primaryNameOptional.isPresent()); + assertEquals(initialName, primaryNameOptional.get()); + + // register name 2 + String secondName = "second-name"; + RegisterNameTransactionData registerNameTransactionData2 = new RegisterNameTransactionData(TestTransaction.generateBase(alice), secondName, "{}"); + registerNameTransactionData2.setFee(new RegisterNameTransaction(null, null).getUnitFee(registerNameTransactionData2.getTimestamp())); + TransactionUtils.signAndMint(repository, registerNameTransactionData2, alice); + + // assert name 2 registration, assert primary has not changed + assertTrue(repository.getNameRepository().nameExists(secondName)); + + // the name alice is trying to update to + String newName = "updated-name"; + + // update name, assert invalid + updateName(repository, initialName, newName, Transaction.ValidationResult.NOT_SUPPORTED, alice); + + // check primary name did not update + // check primary name update + Optional primaryNameNotUpdateOptional = alice.getPrimaryName(); + assertTrue(primaryNameNotUpdateOptional.isPresent()); + assertEquals(initialName, primaryNameNotUpdateOptional.get()); + + // sell name 2, assert valid + Long amount = 1000000L; + SellNameTransactionData transactionData = new SellNameTransactionData(TestTransaction.generateBase(alice), secondName, amount); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Check name is for sale + NameData nameData = repository.getNameRepository().fromName(secondName); + assertTrue(nameData.isForSale()); + assertEquals("price incorrect", amount, nameData.getSalePrice()); + + // bob buys name 2, assert + BuyNameTransactionData bobBuysName2Data = new BuyNameTransactionData(TestTransaction.generateBase(bob), secondName, amount, alice.getAddress()); + TransactionUtils.signAndMint(repository, bobBuysName2Data, bob); + + // update name, assert valid, assert primary name change + updateName(repository, initialName, newName, Transaction.ValidationResult.OK, alice); + + // check primary name update + Optional primaryNameUpdateOptional = alice.getPrimaryName(); + assertTrue(primaryNameUpdateOptional.isPresent()); + assertEquals(newName, primaryNameUpdateOptional.get()); + + assertEquals(alice.getPrimaryName(), alice.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); + assertEquals(bob.getPrimaryName(), bob.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); + } + } + + /** + * Update Name + * + * @param repository + * @param initialName the name before the update + * @param newName the name after the update + * @param expectedValidationResult the validation result expected from the update + * @param account the account for the update + * + * @throws DataException + */ + private static void updateName(Repository repository, String initialName, String newName, Transaction.ValidationResult expectedValidationResult, PrivateKeyAccount account) throws DataException { + TransactionData data = new UpdateNameTransactionData(TestTransaction.generateBase(account), initialName, newName, "{}"); + Transaction.ValidationResult result = TransactionUtils.signAndImport(repository,data, account); + + assertEquals("Transaction invalid", expectedValidationResult, result); + + BlockUtils.mintBlock(repository); + + if( Transaction.ValidationResult.OK.equals(expectedValidationResult) ) { + assertTrue(repository.getNameRepository().nameExists(newName)); + } + else { + // the new name should not exist, because the update was invalid + assertFalse(repository.getNameRepository().nameExists(newName)); + } + } } From 9e4925c8dda3f2a1fe41ae8f99ca6d6371b2d7b0 Mon Sep 17 00:00:00 2001 From: PhilReact Date: Sat, 24 May 2025 19:15:36 +0300 Subject: [PATCH 20/42] added back comments --- .../java/org/qortal/api/resource/ArbitraryResource.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 2b912e0b..6adb9e90 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1910,10 +1910,12 @@ public String finalizeUpload( maxAttempts = 5; } - // Load the file + // Loop until we have data if (async) { + // Asynchronous arbitraryDataReader.loadAsynchronously(false, 1); } else { + // Synchronous while (!Controller.isStopping()) { attempts++; if (!arbitraryDataReader.isBuilding()) { @@ -1932,12 +1934,15 @@ public String finalizeUpload( java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); if (outputPath == null) { + // Assume the resource doesn't exist throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); } if (filepath == null || filepath.isEmpty()) { + // No file path supplied - so check if this is a single file resource String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); if (files != null && files.length == 1) { + // This is a single file resource filepath = files[0]; } else { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "filepath is required for resources containing more than one file"); From 140d86e209c1edb84a7158a8e5ec691694d0c92c Mon Sep 17 00:00:00 2001 From: PhilReact Date: Sat, 24 May 2025 22:29:33 +0300 Subject: [PATCH 21/42] added comments --- .../api/resource/ArbitraryResource.java | 136 ++++++++++-------- 1 file changed, 79 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 6adb9e90..c453d7b0 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -896,7 +896,7 @@ public class ArbitraryResource { @GET - @Path("/check-tmp-space") + @Path("/check/tmp") @Produces(MediaType.TEXT_PLAIN) @Operation( summary = "Check if the disk has enough disk space for an upcoming upload", @@ -921,7 +921,7 @@ public class ArbitraryResource { } long usableSpace = uploadDir.getUsableSpace(); - long requiredSpace = totalSize * 2; // estimate for chunks + merge + long requiredSpace = (long)(((double)totalSize) * 2.2); // estimate for chunks + merge if (usableSpace < requiredSpace) { return Response.status(507).entity("Insufficient disk space").build(); @@ -1981,61 +1981,79 @@ public String finalizeUpload( // 3. Set Content-Disposition header response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\""); } - long fileSize = Files.size(path); - String mimeType = context.getMimeType(path.toString()); - String range = request.getHeader("Range"); - - long rangeStart = 0; - long rangeEnd = fileSize - 1; - boolean isPartial = false; - - if (range != null && encoding == null) { - range = range.replace("bytes=", ""); - String[] parts = range.split("-"); - if (parts.length > 0 && !parts[0].isEmpty()) { - rangeStart = Long.parseLong(parts[0]); - } - if (parts.length > 1 && !parts[1].isEmpty()) { - rangeEnd = Long.parseLong(parts[1]); - } - isPartial = true; - } - - long contentLength = rangeEnd - rangeStart + 1; - - // Set headers - response.setHeader("Accept-Ranges", "bytes"); - - if (isPartial) { - response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); - response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); - } else { - response.setStatus(HttpServletResponse.SC_OK); - } - - OutputStream rawOut = response.getOutputStream(); - OutputStream base64Out = null; - OutputStream gzipOut = null; - if (encoding != null && "base64".equalsIgnoreCase(encoding)) { - response.setContentType("text/plain"); + // Determine the total size of the requested file + long fileSize = Files.size(path); + String mimeType = context.getMimeType(path.toString()); - String acceptEncoding = request.getHeader("Accept-Encoding"); - boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip"); - - if (wantsGzip) { - response.setHeader("Content-Encoding", "gzip"); - gzipOut = new GZIPOutputStream(rawOut); - base64Out = java.util.Base64.getEncoder().wrap(gzipOut); - } else { - base64Out = java.util.Base64.getEncoder().wrap(rawOut); + // Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads) + String range = request.getHeader("Range"); + + long rangeStart = 0; + long rangeEnd = fileSize - 1; + boolean isPartial = false; + + // If a Range header is present and no base64 encoding is requested, parse the range values + if (range != null && encoding == null) { + range = range.replace("bytes=", ""); // Remove the "bytes=" prefix + String[] parts = range.split("-"); // Split the range into start and end + + // Parse range start + if (parts.length > 0 && !parts[0].isEmpty()) { + rangeStart = Long.parseLong(parts[0]); + } + + // Parse range end, if present + if (parts.length > 1 && !parts[1].isEmpty()) { + rangeEnd = Long.parseLong(parts[1]); + } + + isPartial = true; // Indicate that this is a partial content request + } + + // Calculate how many bytes should be sent in the response + long contentLength = rangeEnd - rangeStart + 1; + + // Inform the client that byte ranges are supported + response.setHeader("Accept-Ranges", "bytes"); + + if (isPartial) { + // If partial content was requested, return 206 Partial Content with appropriate headers + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); + } else { + // Otherwise, return the entire file with status 200 OK + response.setStatus(HttpServletResponse.SC_OK); + } + + // Initialize output streams for writing the file to the response + OutputStream rawOut = response.getOutputStream(); + OutputStream base64Out = null; + OutputStream gzipOut = null; + + if (encoding != null && "base64".equalsIgnoreCase(encoding)) { + // If base64 encoding is requested, override content type + response.setContentType("text/plain"); + + // Check if the client accepts gzip encoding + String acceptEncoding = request.getHeader("Accept-Encoding"); + boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip"); + + if (wantsGzip) { + // Wrap output in GZIP and Base64 streams if gzip is accepted + response.setHeader("Content-Encoding", "gzip"); + gzipOut = new GZIPOutputStream(rawOut); + base64Out = java.util.Base64.getEncoder().wrap(gzipOut); + } else { + // Wrap output in Base64 only + base64Out = java.util.Base64.getEncoder().wrap(rawOut); + } + + rawOut = base64Out; // Use the wrapped stream for writing + } else { + // For raw binary output, set the content type and length + response.setContentType(mimeType != null ? mimeType : "application/octet-stream"); + response.setContentLength((int) contentLength); } - - rawOut = base64Out; - } else { - response.setContentType(mimeType != null ? mimeType : "application/octet-stream"); - response.setContentLength((int) contentLength); - } - // Stream file content try (InputStream inputStream = Files.newInputStream(path)) { if (rangeStart > 0) { @@ -2064,10 +2082,14 @@ if (!response.isCommitted()) { response.getWriter().write(" "); } - } catch (IOException | InterruptedException | NumberFormatException | ApiException | DataException e) { - LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); + } catch (IOException | InterruptedException | ApiException | DataException e) { + LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); } + catch ( NumberFormatException e) { + LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); + } } From 02e10e9de98e913923ceb2a7937252ff995b7fa6 Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 27 May 2025 08:15:50 -0700 Subject: [PATCH 22/42] invalidated name buys and sales that violate primary names --- .../transaction/BuyNameTransaction.java | 9 +++ .../transaction/SellNameTransaction.java | 6 ++ .../org/qortal/test/naming/BuySellTests.java | 61 +++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/src/main/java/org/qortal/transaction/BuyNameTransaction.java b/src/main/java/org/qortal/transaction/BuyNameTransaction.java index 6c390085..370e770a 100644 --- a/src/main/java/org/qortal/transaction/BuyNameTransaction.java +++ b/src/main/java/org/qortal/transaction/BuyNameTransaction.java @@ -49,6 +49,15 @@ public class BuyNameTransaction extends Transaction { @Override public ValidationResult isValid() throws DataException { + Optional buyerPrimaryName = this.getBuyer().getPrimaryName(); + if( buyerPrimaryName.isPresent() ) { + + NameData nameData = repository.getNameRepository().fromName(buyerPrimaryName.get()); + if (nameData.isForSale()) { + return ValidationResult.NOT_SUPPORTED; + } + } + String name = this.buyNameTransactionData.getName(); // Check seller address is valid diff --git a/src/main/java/org/qortal/transaction/SellNameTransaction.java b/src/main/java/org/qortal/transaction/SellNameTransaction.java index 3b6bf5df..8dcf9bd2 100644 --- a/src/main/java/org/qortal/transaction/SellNameTransaction.java +++ b/src/main/java/org/qortal/transaction/SellNameTransaction.java @@ -45,6 +45,12 @@ public class SellNameTransaction extends Transaction { public ValidationResult isValid() throws DataException { String name = this.sellNameTransactionData.getName(); + // if the account has more than one name, then they cannot sell their primary name + if( this.repository.getNameRepository().getNamesByOwner(this.getOwner().getAddress()).size() > 1 && + this.getOwner().getPrimaryName().get().equals(name) ) { + return ValidationResult.NOT_SUPPORTED; + } + // Check name size bounds int nameLength = Utf8.encodedLength(name); if (nameLength < 1 || nameLength > Name.MAX_NAME_SIZE) diff --git a/src/test/java/org/qortal/test/naming/BuySellTests.java b/src/test/java/org/qortal/test/naming/BuySellTests.java index 9788bd7c..6f807265 100644 --- a/src/test/java/org/qortal/test/naming/BuySellTests.java +++ b/src/test/java/org/qortal/test/naming/BuySellTests.java @@ -133,6 +133,11 @@ public class BuySellTests extends Common { // check that the order is correct assertEquals(name1, namesByOwner.get(0).getName()); + SellNameTransactionData sellPrimaryNameData = new SellNameTransactionData(TestTransaction.generateBase(alice), name, price); + Transaction.ValidationResult sellPrimaryNameResult = TransactionUtils.signAndImport(repository, sellPrimaryNameData, alice); + + // check that selling primary name is not supported while owning multiple addresses + assertTrue(Transaction.ValidationResult.NOT_SUPPORTED.equals(sellPrimaryNameResult)); } @Test @@ -438,4 +443,60 @@ public class BuySellTests extends Common { assertEquals(bob.getPrimaryName(), bob.determinePrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED)); } + @Test + public void testBuyInvalidationDuringPrimaryNameSale() throws DataException { + // mint passed the feature trigger block + BlockUtils.mintBlocks(repository, BlockChain.getInstance().getMultipleNamesPerAccountHeight()); + + // Register-name + testRegisterName(); + + // assert primary name for alice + Optional alicePrimaryName1 = alice.getPrimaryName(); + assertTrue(alicePrimaryName1.isPresent()); + assertTrue(alicePrimaryName1.get().equals(name)); + + // Sell-name + SellNameTransactionData transactionData = new SellNameTransactionData(TestTransaction.generateBase(alice), name, price); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // assert primary name for alice + Optional alicePrimaryName2 = alice.getPrimaryName(); + assertTrue(alicePrimaryName2.isPresent()); + assertTrue(alicePrimaryName2.get().equals(name)); + + NameData nameData; + + // Check name is for sale + nameData = repository.getNameRepository().fromName(name); + assertTrue(nameData.isForSale()); + assertEquals("price incorrect", price, nameData.getSalePrice()); + + // assert alice cannot register another name while primary name is for sale + final String name2 = "another name"; + RegisterNameTransactionData registerSecondNameData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, "{}"); + Transaction.ValidationResult registrationResult = TransactionUtils.signAndImport(repository, registerSecondNameData, alice); + + // check that registering is not supported while primary name is for sale + assertTrue(Transaction.ValidationResult.NOT_SUPPORTED.equals(registrationResult)); + + String bobName = "bob"; + RegisterNameTransactionData bobRegisterData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(bobRegisterData.getTimestamp())); + TransactionUtils.signAndMint(repository, bobRegisterData, bob); + + Optional bobPrimaryName = bob.getPrimaryName(); + + assertTrue(bobPrimaryName.isPresent()); + assertEquals(bobName, bobPrimaryName.get()); + + SellNameTransactionData bobSellData = new SellNameTransactionData(TestTransaction.generateBase(bob), bobName, price); + TransactionUtils.signAndMint(repository, bobSellData, bob); + + BuyNameTransactionData aliceBuyData = new BuyNameTransactionData(TestTransaction.generateBase(alice), bobName, price, bob.getAddress()); + Transaction.ValidationResult aliceBuyResult = TransactionUtils.signAndImport(repository, aliceBuyData, alice); + + // check that buying is not supported while primary name is for sale + assertTrue(Transaction.ValidationResult.NOT_SUPPORTED.equals(aliceBuyResult)); + } } From d66616f3755f0d8f7508f41c9993a96f52174381 Mon Sep 17 00:00:00 2001 From: PhilReact Date: Wed, 28 May 2025 16:29:52 +0300 Subject: [PATCH 23/42] fix issue of not breaking when file is complete --- .../api/resource/ArbitraryResource.java | 188 ++++++++++-------- .../ArbitraryDataFileListManager.java | 2 +- .../arbitrary/ArbitraryDataFileManager.java | 14 +- 3 files changed, 111 insertions(+), 93 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index c453d7b0..00c4be0d 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1902,7 +1902,6 @@ public String finalizeUpload( private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts, boolean attachment, String attachmentFilename) { try { - ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); int attempts = 0; @@ -1928,7 +1927,6 @@ public String finalizeUpload( } } } - Thread.sleep(3000L); } } @@ -1956,16 +1954,16 @@ public String finalizeUpload( if (attachment) { String rawFilename; - + if (attachmentFilename != null && !attachmentFilename.isEmpty()) { // 1. Sanitize first String safeAttachmentFilename = attachmentFilename.replaceAll("[\\\\/:*?\"<>|]", "_"); - + // 2. Check for a valid extension (3–5 alphanumeric chars) if (!safeAttachmentFilename.matches(".*\\.[a-zA-Z0-9]{2,5}$")) { safeAttachmentFilename += ".bin"; } - + rawFilename = safeAttachmentFilename; } else { // Fallback if no filename is provided @@ -1974,70 +1972,74 @@ public String finalizeUpload( : name; rawFilename = baseFilename.replaceAll("[\\\\/:*?\"<>|]", "_") + ".bin"; } - + // Optional: trim length rawFilename = rawFilename.length() > 100 ? rawFilename.substring(0, 100) : rawFilename; - + // 3. Set Content-Disposition header response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\""); } + // Determine the total size of the requested file - long fileSize = Files.size(path); - String mimeType = context.getMimeType(path.toString()); - - // Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads) - String range = request.getHeader("Range"); - - long rangeStart = 0; - long rangeEnd = fileSize - 1; - boolean isPartial = false; - - // If a Range header is present and no base64 encoding is requested, parse the range values - if (range != null && encoding == null) { - range = range.replace("bytes=", ""); // Remove the "bytes=" prefix - String[] parts = range.split("-"); // Split the range into start and end - - // Parse range start - if (parts.length > 0 && !parts[0].isEmpty()) { - rangeStart = Long.parseLong(parts[0]); - } - - // Parse range end, if present - if (parts.length > 1 && !parts[1].isEmpty()) { - rangeEnd = Long.parseLong(parts[1]); - } - - isPartial = true; // Indicate that this is a partial content request + long fileSize = Files.size(path); + String mimeType = context.getMimeType(path.toString()); + + // Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads) + String range = request.getHeader("Range"); + + long rangeStart = 0; + long rangeEnd = fileSize - 1; + boolean isPartial = false; + + // If a Range header is present and no base64 encoding is requested, parse the range values + if (range != null && encoding == null) { + range = range.replace("bytes=", ""); // Remove the "bytes=" prefix + String[] parts = range.split("-"); // Split the range into start and end + + // Parse range start + if (parts.length > 0 && !parts[0].isEmpty()) { + rangeStart = Long.parseLong(parts[0]); } - - // Calculate how many bytes should be sent in the response - long contentLength = rangeEnd - rangeStart + 1; - - // Inform the client that byte ranges are supported - response.setHeader("Accept-Ranges", "bytes"); - - if (isPartial) { - // If partial content was requested, return 206 Partial Content with appropriate headers - response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); - response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); - } else { - // Otherwise, return the entire file with status 200 OK - response.setStatus(HttpServletResponse.SC_OK); + + // Parse range end, if present + if (parts.length > 1 && !parts[1].isEmpty()) { + rangeEnd = Long.parseLong(parts[1]); } - - // Initialize output streams for writing the file to the response - OutputStream rawOut = response.getOutputStream(); - OutputStream base64Out = null; - OutputStream gzipOut = null; - + + isPartial = true; // Indicate that this is a partial content request + } + + // Calculate how many bytes should be sent in the response + long contentLength = rangeEnd - rangeStart + 1; + + // Inform the client that byte ranges are supported + response.setHeader("Accept-Ranges", "bytes"); + + if (isPartial) { + // If partial content was requested, return 206 Partial Content with appropriate headers + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); + } else { + // Otherwise, return the entire file with status 200 OK + response.setStatus(HttpServletResponse.SC_OK); + } + + // Initialize output streams for writing the file to the response + OutputStream rawOut = null; + OutputStream base64Out = null; + OutputStream gzipOut = null; + + try { + rawOut = response.getOutputStream(); + if (encoding != null && "base64".equalsIgnoreCase(encoding)) { // If base64 encoding is requested, override content type response.setContentType("text/plain"); - + // Check if the client accepts gzip encoding String acceptEncoding = request.getHeader("Accept-Encoding"); boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip"); - + if (wantsGzip) { // Wrap output in GZIP and Base64 streams if gzip is accepted response.setHeader("Content-Encoding", "gzip"); @@ -2047,51 +2049,63 @@ public String finalizeUpload( // Wrap output in Base64 only base64Out = java.util.Base64.getEncoder().wrap(rawOut); } - + rawOut = base64Out; // Use the wrapped stream for writing } else { // For raw binary output, set the content type and length response.setContentType(mimeType != null ? mimeType : "application/octet-stream"); response.setContentLength((int) contentLength); } - // Stream file content - try (InputStream inputStream = Files.newInputStream(path)) { - if (rangeStart > 0) { - inputStream.skip(rangeStart); + + // Stream file content + try (InputStream inputStream = Files.newInputStream(path)) { + if (rangeStart > 0) { + inputStream.skip(rangeStart); + } + + byte[] buffer = new byte[65536]; + long bytesRemaining = contentLength; + int bytesRead; + + while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) { + rawOut.write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } } - byte[] buffer = new byte[65536]; - long bytesRemaining = contentLength; - int bytesRead; - - while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) { - rawOut.write(buffer, 0, bytesRead); - bytesRemaining -= bytesRead; + // Stream finished + if (base64Out != null) { + base64Out.close(); // Also flushes and closes the wrapped gzipOut + } else if (gzipOut != null) { + gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out + } else { + rawOut.flush(); // Flush only the base output stream if nothing was wrapped } + + if (!response.isCommitted()) { + response.setStatus(HttpServletResponse.SC_OK); + response.getWriter().write(" "); + } + + } catch (IOException e) { + // Streaming errors should not rethrow — just log + LOGGER.warn(String.format("Streaming error for %s %s: %s", service, name, e.getMessage()), e); + } + + } catch (IOException | ApiException | DataException e) { + LOGGER.warn(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); + if (!response.isCommitted()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); + } + } catch (NumberFormatException e) { + LOGGER.warn(String.format("Invalid range for %s %s: %s", service, name, e.getMessage()), e); + if (!response.isCommitted()) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); } -// Stream finished -if (base64Out != null) { - base64Out.close(); // Also flushes and closes the wrapped gzipOut -} else if (gzipOut != null) { - gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out -} else { - rawOut.flush(); // Flush only the base output stream if nothing was wrapped -} -if (!response.isCommitted()) { - response.setStatus(HttpServletResponse.SC_OK); - response.getWriter().write(" "); -} - - } catch (IOException | InterruptedException | ApiException | DataException e) { - LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); - } - catch ( NumberFormatException e) { - LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); } } + private FileProperties getFileProperties(Service service, String name, String identifier) { try { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 1d5e4149..fd5fc50a 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -400,7 +400,7 @@ public class ArbitraryDataFileListManager { String signature58 = Base58.encode(signature); for (Iterator>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) { Map.Entry> entry = it.next(); - if (entry == null || entry.getKey() == null || entry.getValue() != null) { + if (entry == null || entry.getKey() == null || entry.getValue() == null) { continue; } if (Objects.equals(entry.getValue().getA(), signature58)) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 5836dcd8..d5bbcfb6 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -212,8 +212,7 @@ public class ArbitraryDataFileManager extends Thread { arbitraryDataFileRequests.remove(hash58); LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); - // We may need to remove the file list request, if we have all the files for this transaction - this.handleFileListRequests(signature); + if (response == null) { LOGGER.debug("Received null response from peer {}", peer); @@ -258,6 +257,9 @@ public class ArbitraryDataFileManager extends Thread { } } + // We may need to remove the file list request, if we have all the files for this transaction + this.handleFileListRequests(signature); + return arbitraryDataFile; } @@ -270,10 +272,12 @@ public class ArbitraryDataFileManager extends Thread { return; } - boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData); + boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData); - if (allChunksExist) { - // Update requests map to reflect that we've received all chunks + if (completeFileExists) { + String signature58 = Base58.encode(arbitraryTransactionData.getSignature()); + LOGGER.info("All chunks or complete file exist for transaction {}", signature58); + ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature); } From 517f7b92d59bf14c88d05b3114ba461a289eb49e Mon Sep 17 00:00:00 2001 From: PhilReact Date: Sun, 1 Jun 2025 20:31:36 +0300 Subject: [PATCH 24/42] in memory to stream --- .../qortal/arbitrary/ArbitraryDataDigest.java | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java index b07fd396..004bda6e 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java @@ -4,6 +4,7 @@ import org.qortal.repository.DataException; import org.qortal.utils.Base58; import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; @@ -25,47 +26,53 @@ public class ArbitraryDataDigest { } public void compute() throws IOException, DataException { - List allPaths = Files.walk(path).filter(Files::isRegularFile).sorted().collect(Collectors.toList()); + List allPaths = Files.walk(path) + .filter(Files::isRegularFile) + .sorted() + .collect(Collectors.toList()); + Path basePathAbsolute = this.path.toAbsolutePath(); - + MessageDigest sha256; try { sha256 = MessageDigest.getInstance("SHA-256"); } catch (NoSuchAlgorithmException e) { throw new DataException("SHA-256 hashing algorithm unavailable"); } - + for (Path path : allPaths) { // We need to work with paths relative to the base path, to ensure the same hash // is generated on different systems Path relativePath = basePathAbsolute.relativize(path.toAbsolutePath()); - + // Exclude Qortal folder since it can be different each time // We only care about hashing the actual user data if (relativePath.startsWith(".qortal/")) { continue; } - + // Account for \ VS / : Linux VS Windows String pathString = relativePath.toString(); - - if(relativePath.getFileSystem().toString().contains("Windows")) { - pathString = pathString.replace("\\","/"); + if (relativePath.getFileSystem().toString().contains("Windows")) { + pathString = pathString.replace("\\", "/"); } - + // Hash path byte[] filePathBytes = pathString.getBytes(StandardCharsets.UTF_8); - System.out.printf("Path: %s \n", pathString); - System.out.printf("Path Byte Array: %s \n", Arrays.toString(filePathBytes)); sha256.update(filePathBytes); - - // Hash contents - byte[] fileContent = Files.readAllBytes(path); - System.out.printf("File Content: %s \n", Arrays.toString(fileContent)); - sha256.update(fileContent); + + try (InputStream in = Files.newInputStream(path)) { + byte[] buffer = new byte[65536]; // 64 KB + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + sha256.update(buffer, 0, bytesRead); + } + } } + this.hash = sha256.digest(); } + public boolean isHashValid(byte[] hash) { return Arrays.equals(hash, this.hash); From 6145db5357cdffbaf0116311d810570dccdc62ca Mon Sep 17 00:00:00 2001 From: PhilReact Date: Tue, 3 Jun 2025 03:33:35 +0300 Subject: [PATCH 25/42] add cleanup of chunks at startup --- .../org/qortal/controller/Controller.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 33c49bde..17d2fcbf 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -46,6 +46,7 @@ import org.qortal.utils.*; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; + import java.awt.TrayIcon.MessageType; import java.io.File; import java.io.FileNotFoundException; @@ -53,6 +54,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.InetSocketAddress; import java.net.UnknownHostException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.security.SecureRandom; @@ -70,6 +72,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import java.util.stream.Stream; public class Controller extends Thread { @@ -396,6 +399,9 @@ public class Controller extends Thread { Controller.newInstance(args); + + cleanChunkUploadTempDir(); // cleanup leftover chunks from streaming to disk + LOGGER.info("Starting NTP"); Long ntpOffset = Settings.getInstance().getTestNtpOffset(); if (ntpOffset != null) @@ -2174,6 +2180,24 @@ public class Controller extends Thread { return now - offset; } + private static void cleanChunkUploadTempDir() { + Path uploadsTemp = Paths.get("uploads-temp"); + if (!Files.exists(uploadsTemp)) { + return; + } + + try (Stream paths = Files.walk(uploadsTemp)) { + paths.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + + LOGGER.info("Cleaned up all temporary uploads in {}", uploadsTemp); + } catch (IOException e) { + LOGGER.warn("Failed to clean up uploads-temp directory", e); + } + } + + public StatsSnapshot getStatsSnapshot() { return this.stats; } From 8e0e455d411b1def0f25c4339e878b119aa720eb Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 6 Jun 2025 19:01:09 -0700 Subject: [PATCH 26/42] blocks minted adjustments removal is a new feature trigger primary names are now used throughout the chat repository numerous message handlers have been optimized, many message handlers are now getting added to a list and scheduled for processing and when they get processed, the database gets queried significantly less, because the message requests and responses are getting batched together for database access rather than querying the database one by one, the thread limits for these message types have been significantly increased, because each individual thread coming in does very little, all it does is add the message to a list to be scheduled at a later time --- .../api/resource/ArbitraryResource.java | 4 +- .../qortal/api/resource/CrossChainUtils.java | 48 ++ src/main/java/org/qortal/block/Block.java | 20 +- .../java/org/qortal/block/BlockChain.java | 7 +- .../controller/TransactionImporter.java | 217 +++++-- .../ArbitraryDataFileListManager.java | 582 +++++++++++------- .../arbitrary/ArbitraryDataFileManager.java | 58 +- .../ArbitraryDataFileRequestThread.java | 207 ++++--- .../arbitrary/ArbitraryMetadataManager.java | 247 +++++--- .../controller/arbitrary/PeerMessage.java | 22 + .../qortal/controller/tradebot/TradeBot.java | 287 ++++++--- src/main/java/org/qortal/crosschain/ACCT.java | 7 + .../org/qortal/crosschain/BitcoinACCTv1.java | 25 +- .../org/qortal/crosschain/BitcoinACCTv3.java | 25 +- .../org/qortal/crosschain/DigibyteACCTv3.java | 25 +- .../org/qortal/crosschain/DogecoinACCTv1.java | 25 +- .../org/qortal/crosschain/DogecoinACCTv3.java | 25 +- .../org/qortal/crosschain/LitecoinACCTv1.java | 25 +- .../org/qortal/crosschain/LitecoinACCTv3.java | 25 +- .../org/qortal/crosschain/PirateChain.java | 7 +- .../qortal/crosschain/PirateChainACCTv3.java | 25 +- .../org/qortal/crosschain/PirateWallet.java | 5 +- .../qortal/crosschain/RavencoinACCTv3.java | 25 +- src/main/java/org/qortal/network/Network.java | 2 +- .../org/qortal/repository/ATRepository.java | 4 + .../qortal/repository/AccountRepository.java | 2 + .../repository/TransactionRepository.java | 3 + .../repository/hsqldb/HSQLDBATRepository.java | 111 ++++ .../hsqldb/HSQLDBAccountRepository.java | 33 + .../hsqldb/HSQLDBChatRepository.java | 52 +- .../HSQLDBTransactionRepository.java | 52 ++ .../java/org/qortal/settings/Settings.java | 12 +- .../org/qortal/utils/ArbitraryIndexUtils.java | 9 - .../utils/ArbitraryTransactionUtils.java | 18 + src/main/resources/blockchain.json | 3 +- src/test/resources/test-chain-v2.json | 3 +- 36 files changed, 1598 insertions(+), 649 deletions(-) create mode 100644 src/main/java/org/qortal/controller/arbitrary/PeerMessage.java diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 00c4be0d..79fb8528 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -2093,12 +2093,12 @@ public String finalizeUpload( } } catch (IOException | ApiException | DataException e) { - LOGGER.warn(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); + LOGGER.warn(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); if (!response.isCommitted()) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); } } catch (NumberFormatException e) { - LOGGER.warn(String.format("Invalid range for %s %s: %s", service, name, e.getMessage()), e); + LOGGER.warn(String.format("Invalid range for %s %s: %s", service, name, e.getMessage())); if (!response.isCommitted()) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); } diff --git a/src/main/java/org/qortal/api/resource/CrossChainUtils.java b/src/main/java/org/qortal/api/resource/CrossChainUtils.java index c923850f..eec784e7 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainUtils.java +++ b/src/main/java/org/qortal/api/resource/CrossChainUtils.java @@ -12,7 +12,9 @@ import org.bouncycastle.util.Strings; import org.json.simple.JSONObject; import org.qortal.api.model.CrossChainTradeLedgerEntry; import org.qortal.api.model.crosschain.BitcoinyTBDRequest; +import org.qortal.asset.Asset; import org.qortal.crosschain.*; +import org.qortal.data.account.AccountBalanceData; import org.qortal.data.at.ATData; import org.qortal.data.at.ATStateData; import org.qortal.data.crosschain.*; @@ -30,6 +32,7 @@ import java.io.Writer; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; @@ -234,6 +237,9 @@ public class CrossChainUtils { return bitcoiny.getBlockchainProvider().removeServer(server); } + public static ChainableServer getCurrentServer( Bitcoiny bitcoiny ) { + return bitcoiny.getBlockchainProvider().getCurrentServer(); + } /** * Set Current Server * @@ -773,4 +779,46 @@ public class CrossChainUtils { entries.add(ledgerEntry); } } + + public static List populateTradeDataList(Repository repository, ACCT acct, List atDataList) throws DataException { + + if(atDataList.isEmpty()) return new ArrayList<>(0); + + List latestATStates + = repository.getATRepository() + .getLatestATStates( + atDataList.stream() + .map(ATData::getATAddress) + .collect(Collectors.toList()) + ); + + Map atStateDataByAtAddress + = latestATStates.stream().collect(Collectors.toMap(ATStateData::getATAddress, Function.identity())); + + Map atDataByAtAddress + = atDataList.stream().collect(Collectors.toMap(ATData::getATAddress, Function.identity())); + + Map balanceByAtAddress + = repository + .getAccountRepository() + .getBalances(new ArrayList<>(atDataByAtAddress.keySet()), Asset.QORT) + .stream().collect(Collectors.toMap(AccountBalanceData::getAddress, AccountBalanceData::getBalance)); + + List crossChainTradeDataList = new ArrayList<>(latestATStates.size()); + + for( ATStateData atStateData : latestATStates ) { + ATData atData = atDataByAtAddress.get(atStateData.getATAddress()); + crossChainTradeDataList.add( + acct.populateTradeData( + repository, + atData.getCreatorPublicKey(), + atData.getCreation(), + atStateData, + OptionalLong.of(balanceByAtAddress.get(atStateData.getATAddress())) + ) + ); + } + + return crossChainTradeDataList; + } } \ No newline at end of file diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 753b5dfa..f2291910 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1723,7 +1723,15 @@ public class Block { accountData.setBlocksMinted(accountData.getBlocksMinted() + 1); LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); - final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment() + accountData.getBlocksMintedPenalty(); + int blocksMintedAdjustment + = + (this.blockData.getHeight() > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight()) + ? + 0 + : + accountData.getBlocksMintedAdjustment(); + + final int effectiveBlocksMinted = accountData.getBlocksMinted() + blocksMintedAdjustment + accountData.getBlocksMintedPenalty(); for (int newLevel = maximumLevel; newLevel >= 0; --newLevel) if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) { @@ -2131,7 +2139,15 @@ public class Block { accountData.setBlocksMinted(accountData.getBlocksMinted() - 1); LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); - final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment() + accountData.getBlocksMintedPenalty(); + int blocksMintedAdjustment + = + (this.blockData.getHeight() > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight()) + ? + 0 + : + accountData.getBlocksMintedAdjustment(); + + final int effectiveBlocksMinted = accountData.getBlocksMinted() + blocksMintedAdjustment + accountData.getBlocksMintedPenalty(); for (int newLevel = maximumLevel; newLevel >= 0; --newLevel) if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) { diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index d7e405ed..1349383d 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -93,7 +93,8 @@ public class BlockChain { nullGroupMembershipHeight, ignoreLevelForRewardShareHeight, adminQueryFixHeight, - multipleNamesPerAccountHeight + multipleNamesPerAccountHeight, + mintedBlocksAdjustmentRemovalHeight } // Custom transaction fees @@ -695,6 +696,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.multipleNamesPerAccountHeight.name()).intValue(); } + public int getMintedBlocksAdjustmentRemovalHeight() { + return this.featureTriggers.get(FeatureTrigger.mintedBlocksAdjustmentRemovalHeight.name()).intValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/java/org/qortal/controller/TransactionImporter.java b/src/main/java/org/qortal/controller/TransactionImporter.java index 6c846f3b..c1ace5fb 100644 --- a/src/main/java/org/qortal/controller/TransactionImporter.java +++ b/src/main/java/org/qortal/controller/TransactionImporter.java @@ -2,6 +2,7 @@ package org.qortal.controller; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.arbitrary.PeerMessage; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; @@ -20,7 +21,11 @@ import org.qortal.utils.Base58; import org.qortal.utils.NTP; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; import java.util.stream.Collectors; public class TransactionImporter extends Thread { @@ -50,6 +55,10 @@ public class TransactionImporter extends Thread { /** Cached list of unconfirmed transactions, used when counting per creator. This is replaced regularly */ public static List unconfirmedTransactionsCache = null; + public TransactionImporter() { + signatureMessageScheduler.scheduleAtFixedRate(this::processNetworkTransactionSignaturesMessage, 60, 1, TimeUnit.SECONDS); + getTransactionMessageScheduler.scheduleAtFixedRate(this::processNetworkGetTransactionMessages, 60, 1, TimeUnit.SECONDS); + } public static synchronized TransactionImporter getInstance() { if (instance == null) { @@ -371,36 +380,104 @@ public class TransactionImporter extends Thread { } } + // List to collect messages + private final List getTransactionMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object getTransactionMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService getTransactionMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkGetTransactionMessage(Peer peer, Message message) { - GetTransactionMessage getTransactionMessage = (GetTransactionMessage) message; - byte[] signature = getTransactionMessage.getSignature(); - try (final Repository repository = RepositoryManager.getRepository()) { + synchronized (getTransactionMessageLock) { + getTransactionMessageList.add(new PeerMessage(peer, message)); + } + } + + private void processNetworkGetTransactionMessages() { + + try { + List messagesToProcess; + synchronized (getTransactionMessageLock) { + messagesToProcess = new ArrayList<>(getTransactionMessageList); + getTransactionMessageList.clear(); + } + + if( messagesToProcess.isEmpty() ) return; + + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); + + for( PeerMessage peerMessage : messagesToProcess ) { + GetTransactionMessage getTransactionMessage = (GetTransactionMessage) peerMessage.getMessage(); + byte[] signature = getTransactionMessage.getSignature(); + + peerMessageBySignature58.put(Base58.encode(signature), peerMessage); + } + // Firstly check the sig-valid transactions that are currently queued for import - TransactionData transactionData = this.getCachedSigValidTransactions().stream() - .filter(t -> Arrays.equals(signature, t.getSignature())) - .findFirst().orElse(null); + Map transactionsCachedBySignature58 + = this.getCachedSigValidTransactions().stream() + .collect(Collectors.toMap(t -> Base58.encode(t.getSignature()), Function.identity())); - if (transactionData == null) { + Map>> transactionsCachedBySignature58Partition + = peerMessageBySignature58.entrySet().stream() + .collect(Collectors.partitioningBy(entry -> transactionsCachedBySignature58.containsKey(entry.getKey()))); + + List signaturesNeeded + = transactionsCachedBySignature58Partition.get(false).stream() + .map(Map.Entry::getValue) + .map(PeerMessage::getMessage) + .map(message -> (GetTransactionMessage) message) + .map(GetTransactionMessage::getSignature) + .collect(Collectors.toList()); + + // transaction found in the import queue + Map transactionsToSendBySignature58 = new HashMap<>(messagesToProcess.size()); + for( Map.Entry entry : transactionsCachedBySignature58Partition.get(true)) { + transactionsToSendBySignature58.put(entry.getKey(), transactionsCachedBySignature58.get(entry.getKey())); + } + + if( !signaturesNeeded.isEmpty() ) { // Not found in import queue, so try the database - transactionData = repository.getTransactionRepository().fromSignature(signature); + try (final Repository repository = RepositoryManager.getRepository()) { + transactionsToSendBySignature58.putAll( + repository.getTransactionRepository().fromSignatures(signaturesNeeded).stream() + .collect(Collectors.toMap(transactionData -> Base58.encode(transactionData.getSignature()), Function.identity())) + ); + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + } } - if (transactionData == null) { - // Still not found - so we don't have this transaction - LOGGER.debug(() -> String.format("Ignoring GET_TRANSACTION request from peer %s for unknown transaction %s", peer, Base58.encode(signature))); - // Send no response at all??? - return; - } + for( final Map.Entry entry : transactionsToSendBySignature58.entrySet() ) { - Message transactionMessage = new TransactionMessage(transactionData); + PeerMessage peerMessage = peerMessageBySignature58.get(entry.getKey()); + final Message message = peerMessage.getMessage(); + final Peer peer = peerMessage.getPeer(); + + Runnable sendTransactionMessageRunner = () -> sendTransactionMessage(entry.getKey(), entry.getValue(), message, peer); + Thread sendTransactionMessageThread = new Thread(sendTransactionMessageRunner); + sendTransactionMessageThread.start(); + } + } catch (Exception e) { + LOGGER.error(e.getMessage(),e); + } + } + + private static void sendTransactionMessage(String signature58, TransactionData data, Message message, Peer peer) { + try { + Message transactionMessage = new TransactionMessage(data); transactionMessage.setId(message.getId()); + if (!peer.sendMessage(transactionMessage)) peer.disconnect("failed to send transaction"); - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); - } catch (TransformationException e) { - LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); + } + catch (TransformationException e) { + LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", signature58, peer), e); + } + catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } @@ -421,44 +498,86 @@ public class TransactionImporter extends Thread { } } + // List to collect messages + private final List signatureMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object signatureMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService signatureMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkTransactionSignaturesMessage(Peer peer, Message message) { - TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) message; - List signatures = transactionSignaturesMessage.getSignatures(); + synchronized (signatureMessageLock) { + signatureMessageList.add(new PeerMessage(peer, message)); + } + } - try (final Repository repository = RepositoryManager.getRepository()) { - for (byte[] signature : signatures) { - String signature58 = Base58.encode(signature); - if (invalidUnconfirmedTransactions.containsKey(signature58)) { - // Previously invalid transaction - don't keep requesting it - // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks - continue; - } + public void processNetworkTransactionSignaturesMessage() { - // Ignore if this transaction is in the queue - if (incomingTransactionQueueContains(signature)) { - LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer)); - continue; - } + try { + List messagesToProcess; + synchronized (signatureMessageLock) { + messagesToProcess = new ArrayList<>(signatureMessageList); + signatureMessageList.clear(); + } - // Do we have it already? (Before requesting transaction data itself) - if (repository.getTransactionRepository().exists(signature)) { - LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer)); - continue; - } + Map signatureBySignature58 = new HashMap<>(messagesToProcess.size() * 10); + Map peerBySignature58 = new HashMap<>( messagesToProcess.size() * 10 ); - // Check isInterrupted() here and exit fast - if (Thread.currentThread().isInterrupted()) - return; + for( PeerMessage peerMessage : messagesToProcess ) { - // Fetch actual transaction data from peer - Message getTransactionMessage = new GetTransactionMessage(signature); - if (!peer.sendMessage(getTransactionMessage)) { - peer.disconnect("failed to request transaction"); - return; + TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) peerMessage.getMessage(); + List signatures = transactionSignaturesMessage.getSignatures(); + + for (byte[] signature : signatures) { + String signature58 = Base58.encode(signature); + if (invalidUnconfirmedTransactions.containsKey(signature58)) { + // Previously invalid transaction - don't keep requesting it + // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks + continue; + } + + // Ignore if this transaction is in the queue + if (incomingTransactionQueueContains(signature)) { + LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peerMessage.getPeer())); + continue; + } + + signatureBySignature58.put(signature58, signature); + peerBySignature58.put(signature58, peerMessage.getPeer()); } } - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer %s", peer), e); + + if( !signatureBySignature58.isEmpty() ) { + try (final Repository repository = RepositoryManager.getRepository()) { + + // remove signatures in db already + repository.getTransactionRepository() + .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream() + .map(TransactionData::getSignature) + .map(signature -> Base58.encode(signature)) + .forEach(signature58 -> signatureBySignature58.remove(signature58)); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer"), e); + } + } + + // Check isInterrupted() here and exit fast + if (Thread.currentThread().isInterrupted()) + return; + + for (Map.Entry entry : signatureBySignature58.entrySet()) { + + Peer peer = peerBySignature58.get(entry.getKey()); + + // Fetch actual transaction data from peer + Message getTransactionMessage = new GetTransactionMessage(entry.getValue()); + if (peer != null && !peer.sendMessage(getTransactionMessage)) { + peer.disconnect("failed to request transaction"); + } + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index fd5fc50a..ee37dbec 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -25,6 +25,10 @@ import org.qortal.utils.NTP; import org.qortal.utils.Triple; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.qortal.controller.arbitrary.ArbitraryDataFileManager.MAX_FILE_HASH_RESPONSES; @@ -73,6 +77,8 @@ public class ArbitraryDataFileListManager { private ArbitraryDataFileListManager() { + getArbitraryDataFileListMessageScheduler.scheduleAtFixedRate(this::processNetworkGetArbitraryDataFileListMessage, 60, 1, TimeUnit.SECONDS); + arbitraryDataFileListMessageScheduler.scheduleAtFixedRate(this::processNetworkArbitraryDataFileListMessage, 60, 1, TimeUnit.SECONDS); } public static ArbitraryDataFileListManager getInstance() { @@ -413,70 +419,116 @@ public class ArbitraryDataFileListManager { // Network handlers + // List to collect messages + private final List arbitraryDataFileListMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object arbitraryDataFileListMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService arbitraryDataFileListMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) { // Don't process if QDN is disabled if (!Settings.getInstance().isQdnEnabled()) { return; } - ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; - LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); - - if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) { - long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime(); - LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}", - totalRequestTime, arbitraryDataFileListMessage.getRequestHops(), - arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + synchronized (arbitraryDataFileListMessageLock) { + arbitraryDataFileListMessageList.add(new PeerMessage(peer, message)); } + } - // Do we have a pending request for this data? - Triple request = arbitraryDataFileListRequests.get(message.getId()); - if (request == null || request.getA() == null) { - return; - } - boolean isRelayRequest = (request.getB() != null); + private void processNetworkArbitraryDataFileListMessage() { - // Does this message's signature match what we're expecting? - byte[] signature = arbitraryDataFileListMessage.getSignature(); - String signature58 = Base58.encode(signature); - if (!request.getA().equals(signature58)) { - return; - } + try { + List messagesToProcess; + synchronized (arbitraryDataFileListMessageLock) { + messagesToProcess = new ArrayList<>(arbitraryDataFileListMessageList); + arbitraryDataFileListMessageList.clear(); + } - List hashes = arbitraryDataFileListMessage.getHashes(); - if (hashes == null || hashes.isEmpty()) { - return; - } + if (messagesToProcess.isEmpty()) return; - ArbitraryTransactionData arbitraryTransactionData = null; + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); + Map signatureBySignature58 = new HashMap<>(messagesToProcess.size()); + Map isRelayRequestBySignature58 = new HashMap<>(messagesToProcess.size()); + Map> hashesBySignature58 = new HashMap<>(messagesToProcess.size()); + Map> requestBySignature58 = new HashMap<>(messagesToProcess.size()); - // Check transaction exists and hashes are correct - try (final Repository repository = RepositoryManager.getRepository()) { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) + for (PeerMessage peerMessage : messagesToProcess) { + Peer peer = peerMessage.getPeer(); + Message message = peerMessage.getMessage(); + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; + LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size()); + + if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) { + long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime(); + LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}", + totalRequestTime, arbitraryDataFileListMessage.getRequestHops(), + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + } + + // Do we have a pending request for this data? + Triple request = arbitraryDataFileListRequests.get(message.getId()); + if (request == null || request.getA() == null) { + continue; + } + boolean isRelayRequest = (request.getB() != null); + + // Does this message's signature match what we're expecting? + byte[] signature = arbitraryDataFileListMessage.getSignature(); + String signature58 = Base58.encode(signature); + if (!request.getA().equals(signature58)) { + continue; + } + + List hashes = arbitraryDataFileListMessage.getHashes(); + if (hashes == null || hashes.isEmpty()) { + continue; + } + + peerMessageBySignature58.put(signature58, peerMessage); + signatureBySignature58.put(signature58, signature); + isRelayRequestBySignature58.put(signature58, isRelayRequest); + hashesBySignature58.put(signature58, hashes); + requestBySignature58.put(signature58, request); + } + + if (signatureBySignature58.isEmpty()) return; + + List arbitraryTransactionDataList; + + // Check transaction exists and hashes are correct + try (final Repository repository = RepositoryManager.getRepository()) { + arbitraryTransactionDataList + = repository.getTransactionRepository() + .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream() + .filter(data -> data instanceof ArbitraryTransactionData) + .map(data -> (ArbitraryTransactionData) data) + .collect(Collectors.toList()); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list"), e); return; + } - arbitraryTransactionData = (ArbitraryTransactionData) transactionData; + for (ArbitraryTransactionData arbitraryTransactionData : arbitraryTransactionDataList) { -// // Load data file(s) -// ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData); -// -// // Check all hashes exist -// for (byte[] hash : hashes) { -// //LOGGER.debug("Received hash {}", Base58.encode(hash)); -// if (!arbitraryDataFile.containsChunk(hash)) { -// // Check the hash against the complete file -// if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) { -// LOGGER.info("Received non-matching chunk hash {} for signature {}. This could happen if we haven't obtained the metadata file yet.", Base58.encode(hash), signature58); -// return; -// } -// } -// } + byte[] signature = arbitraryTransactionData.getSignature(); + String signature58 = Base58.encode(signature); - if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { - Long now = NTP.getTime(); + List hashes = hashesBySignature58.get(signature58); + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + Peer peer = peerMessage.getPeer(); + Message message = peerMessage.getMessage(); + + ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message; + + Boolean isRelayRequest = isRelayRequestBySignature58.get(signature58); + if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { + Long now = NTP.getTime(); - if (ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.size() < MAX_FILE_HASH_RESPONSES) { // Keep track of the hashes this peer reports to have access to for (byte[] hash : hashes) { String hash58 = Base58.encode(hash); @@ -487,233 +539,303 @@ public class ArbitraryDataFileListManager { ArbitraryFileListResponseInfo responseInfo = new ArbitraryFileListResponseInfo(hash58, signature58, peer, now, arbitraryDataFileListMessage.getRequestTime(), requestHops); - ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.add(responseInfo); + ArbitraryDataFileManager.getInstance().addResponse(responseInfo); + } + + // Keep track of the source peer, for direct connections + if (arbitraryDataFileListMessage.getPeerAddress() != null) { + ArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique( + new ArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now)); } } - // Keep track of the source peer, for direct connections - if (arbitraryDataFileListMessage.getPeerAddress() != null) { - ArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique( - new ArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now)); - } - } + // Forwarding + if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); - } + boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); + if (!isBlocked) { + Triple request = requestBySignature58.get(signature58); + Peer requestingPeer = request.getB(); + if (requestingPeer != null) { + Long requestTime = arbitraryDataFileListMessage.getRequestTime(); + Integer requestHops = arbitraryDataFileListMessage.getRequestHops(); - // Forwarding - if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) { - boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName())); - if (!isBlocked) { - Peer requestingPeer = request.getB(); - if (requestingPeer != null) { - Long requestTime = arbitraryDataFileListMessage.getRequestTime(); - Integer requestHops = arbitraryDataFileListMessage.getRequestHops(); + // Add each hash to our local mapping so we know who to ask later + Long now = NTP.getTime(); + for (byte[] hash : hashes) { + String hash58 = Base58.encode(hash); + ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); + ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo); + } - // Add each hash to our local mapping so we know who to ask later - Long now = NTP.getTime(); - for (byte[] hash : hashes) { - String hash58 = Base58.encode(hash); - ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops); - ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo); - } + // Bump requestHops if it exists + if (requestHops != null) { + requestHops++; + } - // Bump requestHops if it exists - if (requestHops != null) { - requestHops++; - } + ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage; - ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage; + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); + } + forwardArbitraryDataFileListMessage.setId(message.getId()); - // Remove optional parameters if the requesting peer doesn't support it yet - // A message with less statistical data is better than no message at all - if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); - } else { - forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, - arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); - } - forwardArbitraryDataFileListMessage.setId(message.getId()); - - // Forward to requesting peer - LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); - if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) { - requestingPeer.disconnect("failed to forward arbitrary data file list"); + // Forward to requesting peer + LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); + if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) { + requestingPeer.disconnect("failed to forward arbitrary data file list"); + } + } } } } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } + // List to collect messages + private final List getArbitraryDataFileListMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object getArbitraryDataFileListMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService getArbitraryDataFileListMessageScheduler = Executors.newScheduledThreadPool(1); + public void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) { // Don't respond if QDN is disabled if (!Settings.getInstance().isQdnEnabled()) { return; } - Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); - - GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; - byte[] signature = getArbitraryDataFileListMessage.getSignature(); - String signature58 = Base58.encode(signature); - Long now = NTP.getTime(); - Triple newEntry = new Triple<>(signature58, peer, now); - - // If we've seen this request recently, then ignore - if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) { - LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58); - return; + synchronized (getArbitraryDataFileListMessageLock) { + getArbitraryDataFileListMessageList.add(new PeerMessage(peer, message)); } + } - List requestedHashes = getArbitraryDataFileListMessage.getHashes(); - int hashCount = requestedHashes != null ? requestedHashes.size() : 0; - String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer(); + private void processNetworkGetArbitraryDataFileListMessage() { - if (requestingPeer != null) { - LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58); - } - else { - LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58); - } + try { + List messagesToProcess; + synchronized (getArbitraryDataFileListMessageLock) { + messagesToProcess = new ArrayList<>(getArbitraryDataFileListMessageList); + getArbitraryDataFileListMessageList.clear(); + } - List hashes = new ArrayList<>(); - ArbitraryTransactionData transactionData = null; - boolean allChunksExist = false; - boolean hasMetadata = false; + if (messagesToProcess.isEmpty()) return; - try (final Repository repository = RepositoryManager.getRepository()) { + Map signatureBySignature58 = new HashMap<>(messagesToProcess.size()); + Map> requestedHashesBySignature58 = new HashMap<>(messagesToProcess.size()); + Map requestingPeerBySignature58 = new HashMap<>(messagesToProcess.size()); + Map nowBySignature58 = new HashMap<>((messagesToProcess.size())); + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); - // Firstly we need to lookup this file on chain to get a list of its hashes - transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); - if (transactionData instanceof ArbitraryTransactionData) { + for (PeerMessage messagePeer : messagesToProcess) { + Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet(); + + Message message = messagePeer.message; + Peer peer = messagePeer.peer; + + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; + byte[] signature = getArbitraryDataFileListMessage.getSignature(); + String signature58 = Base58.encode(signature); + Long now = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peer, now); + + // If we've seen this request recently, then ignore + if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) { + LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58); + continue; + } + + List requestedHashes = getArbitraryDataFileListMessage.getHashes(); + int hashCount = requestedHashes != null ? requestedHashes.size() : 0; + String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer(); + + if (requestingPeer != null) { + LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58); + } else { + LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58); + } + + signatureBySignature58.put(signature58, signature); + requestedHashesBySignature58.put(signature58, requestedHashes); + requestingPeerBySignature58.put(signature58, requestingPeer); + nowBySignature58.put(signature58, now); + peerMessageBySignature58.put(signature58, messagePeer); + } + + if (signatureBySignature58.isEmpty()) { + return; + } + + List hashes = new ArrayList<>(); + boolean allChunksExist = false; + boolean hasMetadata = false; + + List transactionDataList; + try (final Repository repository = RepositoryManager.getRepository()) { + + // Firstly we need to lookup this file on chain to get a list of its hashes + transactionDataList + = repository.getTransactionRepository() + .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream() + .filter(data -> data instanceof ArbitraryTransactionData) + .map(data -> (ArbitraryTransactionData) data) + .collect(Collectors.toList()); + + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer"), e); + return; + } + + for (ArbitraryTransactionData transactionData : transactionDataList) { + byte[] signature = transactionData.getSignature(); + String signature58 = Base58.encode(signature); + List requestedHashes = requestedHashesBySignature58.get(signature58); // Check if we're even allowed to serve data for this transaction if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { - // Load file(s) and add any that exist to the list of hashes - ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData); + try { + // Load file(s) and add any that exist to the list of hashes + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData); - // If the peer didn't supply a hash list, we need to return all hashes for this transaction - if (requestedHashes == null || requestedHashes.isEmpty()) { - requestedHashes = new ArrayList<>(); + // If the peer didn't supply a hash list, we need to return all hashes for this transaction + if (requestedHashes == null || requestedHashes.isEmpty()) { + requestedHashes = new ArrayList<>(); - // Add the metadata file - if (arbitraryDataFile.getMetadataHash() != null) { - requestedHashes.add(arbitraryDataFile.getMetadataHash()); - hasMetadata = true; + // Add the metadata file + if (arbitraryDataFile.getMetadataHash() != null) { + requestedHashes.add(arbitraryDataFile.getMetadataHash()); + hasMetadata = true; + } + + // Add the chunk hashes + if (!arbitraryDataFile.getChunkHashes().isEmpty()) { + requestedHashes.addAll(arbitraryDataFile.getChunkHashes()); + } + // Add complete file if there are no hashes + else { + requestedHashes.add(arbitraryDataFile.getHash()); + } } - // Add the chunk hashes - if (!arbitraryDataFile.getChunkHashes().isEmpty()) { - requestedHashes.addAll(arbitraryDataFile.getChunkHashes()); - } - // Add complete file if there are no hashes - else { - requestedHashes.add(arbitraryDataFile.getHash()); + + // Assume all chunks exists, unless one can't be found below + allChunksExist = true; + + for (byte[] requestedHash : requestedHashes) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature); + if (chunk.exists()) { + hashes.add(chunk.getHash()); + //LOGGER.trace("Added hash {}", chunk.getHash58()); + } else { + LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); + allChunksExist = false; + } } + } catch (DataException e) { + LOGGER.error(e.getMessage(), e); + } + } + + // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that, + // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam. + if (hasMetadata && hashes.size() == 1) { + hashes.clear(); + } + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + Peer peer = peerMessage.getPeer(); + Message message = peerMessage.getMessage(); + + Long now = nowBySignature58.get(signature58); + + // We should only respond if we have at least one hash + String requestingPeer = requestingPeerBySignature58.get(signature58); + if (!hashes.isEmpty()) { + + // Firstly we should keep track of the requesting peer, to allow for potential direct connections later + ArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer); + + // We have all the chunks, so update requests map to reflect that we've sent it + // There is no need to keep track of the request, as we can serve all the chunks + if (allChunksExist) { + Triple newEntry = new Triple<>(null, null, now); + arbitraryDataFileListRequests.put(message.getId(), newEntry); } - // Assume all chunks exists, unless one can't be found below - allChunksExist = true; + String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort(); + ArbitraryDataFileListMessage arbitraryDataFileListMessage; + + // Remove optional parameters if the requesting peer doesn't support it yet + // A message with less statistical data is better than no message at all + if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, + hashes, NTP.getTime(), 0, ourAddress, true); + } + + arbitraryDataFileListMessage.setId(message.getId()); + + if (!peer.sendMessage(arbitraryDataFileListMessage)) { + LOGGER.debug("Couldn't send list of hashes"); + peer.disconnect("failed to send list of hashes"); + continue; + } + + if (allChunksExist) { + // Nothing left to do, so return to prevent any unnecessary forwarding from occurring + LOGGER.debug("No need for any forwarding because file list request is fully served"); + continue; + } + + } + + // We may need to forward this request on + boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName())); + if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { + // In relay mode - so ask our other peers if they have it + + + GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; + + long requestTime = getArbitraryDataFileListMessage.getRequestTime(); + int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1; + long totalRequestTime = now - requestTime; + + if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { + // Relay request hasn't timed out yet, so can potentially be rebroadcast + if (requestHops < RELAY_REQUEST_MAX_HOPS) { + // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + + Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer); + relayGetArbitraryDataFileListMessage.setId(message.getId()); + + LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); + Network.getInstance().broadcast( + broadcastPeer -> + !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : + broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage + ); - for (byte[] requestedHash : requestedHashes) { - ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature); - if (chunk.exists()) { - hashes.add(chunk.getHash()); - //LOGGER.trace("Added hash {}", chunk.getHash58()); } else { - LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); - allChunksExist = false; + // This relay request has reached the maximum number of allowed hops } + } else { + // This relay request has timed out } } } - - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e); - } - - // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that, - // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam. - if (hasMetadata && hashes.size() == 1) { - hashes.clear(); - } - - // We should only respond if we have at least one hash - if (!hashes.isEmpty()) { - - // Firstly we should keep track of the requesting peer, to allow for potential direct connections later - ArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer); - - // We have all the chunks, so update requests map to reflect that we've sent it - // There is no need to keep track of the request, as we can serve all the chunks - if (allChunksExist) { - newEntry = new Triple<>(null, null, now); - arbitraryDataFileListRequests.put(message.getId(), newEntry); - } - - String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort(); - ArbitraryDataFileListMessage arbitraryDataFileListMessage; - - // Remove optional parameters if the requesting peer doesn't support it yet - // A message with less statistical data is better than no message at all - if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); - } else { - arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, - hashes, NTP.getTime(), 0, ourAddress, true); - } - - arbitraryDataFileListMessage.setId(message.getId()); - - if (!peer.sendMessage(arbitraryDataFileListMessage)) { - LOGGER.debug("Couldn't send list of hashes"); - peer.disconnect("failed to send list of hashes"); - return; - } - LOGGER.debug("Sent list of hashes (count: {})", hashes.size()); - - if (allChunksExist) { - // Nothing left to do, so return to prevent any unnecessary forwarding from occurring - LOGGER.debug("No need for any forwarding because file list request is fully served"); - return; - } - - } - - // We may need to forward this request on - boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName())); - if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { - // In relay mode - so ask our other peers if they have it - - long requestTime = getArbitraryDataFileListMessage.getRequestTime(); - int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1; - long totalRequestTime = now - requestTime; - - if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { - // Relay request hasn't timed out yet, so can potentially be rebroadcast - if (requestHops < RELAY_REQUEST_MAX_HOPS) { - // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast - - Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer); - relayGetArbitraryDataFileListMessage.setId(message.getId()); - - LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); - Network.getInstance().broadcast( - broadcastPeer -> - !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : - broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage - ); - - } - else { - // This relay request has reached the maximum number of allowed hops - } - } - else { - // This relay request has timed out - } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index d5bbcfb6..a4034596 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -25,6 +25,8 @@ import java.security.SecureRandom; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; public class ArbitraryDataFileManager extends Thread { @@ -48,7 +50,7 @@ public class ArbitraryDataFileManager extends Thread { /** * List to keep track of any arbitrary data file hash responses */ - public final List arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>()); + private final List arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>()); /** * List to keep track of peers potentially available for direct connections, based on recent requests @@ -67,6 +69,7 @@ public class ArbitraryDataFileManager extends Thread { private ArbitraryDataFileManager() { + this.arbitraryDataFileHashResponseScheduler.scheduleAtFixedRate( this::processResponses, 60, 1, TimeUnit.SECONDS); } public static ArbitraryDataFileManager getInstance() { @@ -81,13 +84,6 @@ public class ArbitraryDataFileManager extends Thread { Thread.currentThread().setName("Arbitrary Data File Manager"); try { - // Use a fixed thread pool to execute the arbitrary data file requests - int threadCount = 5; - ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount); - for (int i = 0; i < threadCount; i++) { - arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread()); - } - while (!isStopping) { // Nothing to do yet Thread.sleep(1000); @@ -112,7 +108,6 @@ public class ArbitraryDataFileManager extends Thread { final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT; arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp); - arbitraryDataFileHashResponses.removeIf(entry -> entry.getTimestamp() < relayMinimumTimestamp); final long directConnectionInfoMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_DIRECT_CONNECTION_INFO_TIMEOUT; directConnectionInfo.removeIf(entry -> entry.getTimestamp() < directConnectionInfoMinimumTimestamp); @@ -125,8 +120,7 @@ public class ArbitraryDataFileManager extends Thread { // Fetch data files by hash - public boolean fetchArbitraryDataFiles(Repository repository, - Peer peer, + public boolean fetchArbitraryDataFiles(Peer peer, byte[] signature, ArbitraryTransactionData arbitraryTransactionData, List hashes) throws DataException { @@ -151,16 +145,10 @@ public class ArbitraryDataFileManager extends Thread { if (receivedArbitraryDataFile != null) { LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFile.getHash58(), peer, (endTime-startTime)); receivedAtLeastOneFile = true; - - // Remove this hash from arbitraryDataFileHashResponses now that we have received it - arbitraryDataFileHashResponses.remove(hash58); } else { LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime)); - // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it - arbitraryDataFileHashResponses.remove(hash58); - // Stop asking for files from this peer break; } @@ -169,10 +157,6 @@ public class ArbitraryDataFileManager extends Thread { LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer); } } - else { - // Remove this hash from arbitraryDataFileHashResponses because we have a local copy - arbitraryDataFileHashResponses.remove(hash58); - } } if (receivedAtLeastOneFile) { @@ -191,6 +175,38 @@ public class ArbitraryDataFileManager extends Thread { return receivedAtLeastOneFile; } + // Lock to synchronize access to the list + private final Object arbitraryDataFileHashResponseLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService arbitraryDataFileHashResponseScheduler = Executors.newScheduledThreadPool(1); + + + public void addResponse( ArbitraryFileListResponseInfo responseInfo ) { + + synchronized (arbitraryDataFileHashResponseLock) { + this.arbitraryDataFileHashResponses.add(responseInfo); + } + } + + private void processResponses() { + try { + List responsesToProcess; + synchronized (arbitraryDataFileHashResponseLock) { + responsesToProcess = new ArrayList<>(arbitraryDataFileHashResponses); + arbitraryDataFileHashResponses.clear(); + } + + if (responsesToProcess.isEmpty()) return; + + Long now = NTP.getTime(); + + ArbitraryDataFileRequestThread.getInstance().processFileHashes(now, responsesToProcess, this); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash, Message originalMessage) throws DataException { ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature); boolean fileAlreadyExists = existingFile.exists(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index b8285052..1872898f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -4,127 +4,172 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo; +import org.qortal.data.arbitrary.ArbitraryResourceData; import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.event.DataMonitorEvent; -import org.qortal.event.EventBus; import org.qortal.network.Peer; +import org.qortal.network.message.MessageType; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.utils.ArbitraryTransactionUtils; import org.qortal.utils.Base58; import org.qortal.utils.NTP; +import java.net.http.HttpResponse; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import static java.lang.Thread.NORM_PRIORITY; -public class ArbitraryDataFileRequestThread implements Runnable { +public class ArbitraryDataFileRequestThread { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class); - public ArbitraryDataFileRequestThread() { + private ConcurrentHashMap executorByPeer = new ConcurrentHashMap<>(); + private ArbitraryDataFileRequestThread() { + cleanupExecutorByPeerScheduler.scheduleAtFixedRate(this::cleanupExecutorsByPeer, 1, 1, TimeUnit.MINUTES); } - @Override - public void run() { - Thread.currentThread().setName("Arbitrary Data File Request Thread"); - Thread.currentThread().setPriority(NORM_PRIORITY); + private static ArbitraryDataFileRequestThread instance = null; + + public static ArbitraryDataFileRequestThread getInstance() { + + if( instance == null ) { + instance = new ArbitraryDataFileRequestThread(); + } + + return instance; + } + + private final ScheduledExecutorService cleanupExecutorByPeerScheduler = Executors.newScheduledThreadPool(1); + + private void cleanupExecutorsByPeer() { try { - while (!Controller.isStopping()) { - Long now = NTP.getTime(); - this.processFileHashes(now); - } - } catch (InterruptedException e) { - // Fall-through to exit thread... + this.executorByPeer.forEach((key, value) -> { + if (value instanceof ThreadPoolExecutor) { + ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) value; + if (threadPoolExecutor.getActiveCount() == 0) { + if (this.executorByPeer.computeIfPresent(key, (k, v) -> null) == null) { + LOGGER.info("removed executor: peer = " + key); + } + } + } else { + LOGGER.warn("casting issue in cleanup"); + } + }); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } - - private void processFileHashes(Long now) throws InterruptedException { + public void processFileHashes(Long now, List responseInfos, ArbitraryDataFileManager arbitraryDataFileManager) { if (Controller.isStopping()) { return; } - ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); - String signature58 = null; - String hash58 = null; - Peer peer = null; - boolean shouldProcess = false; + List toProcess = new ArrayList<>(responseInfos.size()); - synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) { - if (!arbitraryDataFileManager.arbitraryDataFileHashResponses.isEmpty()) { + Map responseInfoByHash58 = new HashMap<>(responseInfos.size()); + Map signatureBySignature58 = new HashMap<>(toProcess.size()); + Map> responseInfoBySignature58 = new HashMap<>(); - // Sort by lowest number of node hops first - Comparator lowestHopsFirstComparator = - Comparator.comparingInt(ArbitraryFileListResponseInfo::getRequestHops); - arbitraryDataFileManager.arbitraryDataFileHashResponses.sort(lowestHopsFirstComparator); + for( ArbitraryFileListResponseInfo responseInfo : responseInfos) { - Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.iterator(); - while (iterator.hasNext()) { - if (Controller.isStopping()) { - return; - } + if( responseInfo == null ) continue; - ArbitraryFileListResponseInfo responseInfo = (ArbitraryFileListResponseInfo) iterator.next(); - if (responseInfo == null) { - iterator.remove(); - continue; - } - - hash58 = responseInfo.getHash58(); - peer = responseInfo.getPeer(); - signature58 = responseInfo.getSignature58(); - Long timestamp = responseInfo.getTimestamp(); - - if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { - // Ignore - to be deleted - iterator.remove(); - continue; - } - - // Skip if already requesting, but don't remove, as we might want to retry later - if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) { - // Already requesting - leave this attempt for later - continue; - } - - // We want to process this file - shouldProcess = true; - iterator.remove(); - break; - } + if (Controller.isStopping()) { + return; } + + Peer peer = responseInfo.getPeer(); + + // if relay timeout, then move on + if (now - responseInfo.getTimestamp() >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || responseInfo.getSignature58() == null || peer == null) { + continue; + } + + // Skip if already requesting, but don't remove, as we might want to retry later + if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(responseInfo.getHash58())) { + // Already requesting - leave this attempt for later + arbitraryDataFileManager.addResponse(responseInfo); // don't remove -> adding back, beacause it was removed already above + continue; + } + + + byte[] hash = Base58.decode(responseInfo.getHash58()); + byte[] signature = Base58.decode(responseInfo.getSignature58()); + + // check for null + if (signature == null || hash == null || peer == null) { + continue; + } + + // We want to process this file, store and map data to process later + toProcess.add(responseInfo); + responseInfoByHash58.put(responseInfo.getHash58(), responseInfo); + signatureBySignature58.put(responseInfo.getSignature58(), signature); + responseInfoBySignature58 + .computeIfAbsent(responseInfo.getSignature58(), signature58 -> new ArrayList<>()) + .add(responseInfo); } - if (!shouldProcess) { - // Nothing to do - Thread.sleep(1000L); - return; - } + // if there are no signatures, then there is nothing to process and nothing query the database + if( signatureBySignature58.isEmpty() ) return; - byte[] hash = Base58.decode(hash58); - byte[] signature = Base58.decode(signature58); + List arbitraryTransactionDataList = new ArrayList<>(); // Fetch the transaction data try (final Repository repository = RepositoryManager.getRepository()) { - ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); - if (arbitraryTransactionData == null) { - return; - } - - if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) { - return; - } - - LOGGER.trace("Fetching file {} from peer {} via request thread...", hash58, peer); - arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); - + arbitraryTransactionDataList.addAll( + ArbitraryTransactionUtils.fetchTransactionDataList(repository, new ArrayList<>(signatureBySignature58.values()))); } catch (DataException e) { - LOGGER.debug("Unable to process file hashes: {}", e.getMessage()); + LOGGER.warn("Unable to fetch transaction data: {}", e.getMessage()); + } + + if( !arbitraryTransactionDataList.isEmpty() ) { + long start = System.currentTimeMillis(); + + for(ArbitraryTransactionData data : arbitraryTransactionDataList ) { + String signature58 = Base58.encode(data.getSignature()); + for( ArbitraryFileListResponseInfo responseInfo : responseInfoBySignature58.get(signature58)) { + Runnable fetcher = () -> arbitraryDataFileFetcher(arbitraryDataFileManager, responseInfo, data); + this.executorByPeer + .computeIfAbsent( + responseInfo.getPeer().toString(), + peer -> Executors.newFixedThreadPool( + Settings.getInstance().getMaxThreadsForMessageType(MessageType.GET_ARBITRARY_DATA_FILE)) + ) + .execute(fetcher); + } + } + long timeLapse = System.currentTimeMillis() - start; } } -} + + private void arbitraryDataFileFetcher(ArbitraryDataFileManager arbitraryDataFileManager, ArbitraryFileListResponseInfo responseInfo, ArbitraryTransactionData arbitraryTransactionData) { + try { + arbitraryDataFileManager.fetchArbitraryDataFiles( + responseInfo.getPeer(), + arbitraryTransactionData.getSignature(), + arbitraryTransactionData, + Arrays.asList(Base58.decode(responseInfo.getHash58())) + ); + } catch (DataException e) { + LOGGER.warn("Unable to process file hashes: {}", e.getMessage()); + } + } +} \ No newline at end of file diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index 993a2b72..d38d329f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -24,6 +24,11 @@ import org.qortal.utils.Triple; import java.io.IOException; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; import static org.qortal.controller.arbitrary.ArbitraryDataFileListManager.*; @@ -61,6 +66,7 @@ public class ArbitraryMetadataManager { private ArbitraryMetadataManager() { + scheduler.scheduleAtFixedRate(this::processNetworkGetArbitraryMetadataMessage, 60, 1, TimeUnit.SECONDS); } public static ArbitraryMetadataManager getInstance() { @@ -371,107 +377,160 @@ public class ArbitraryMetadataManager { } } + // List to collect messages + private final List messageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object lock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + public void onNetworkGetArbitraryMetadataMessage(Peer peer, Message message) { + // Don't respond if QDN is disabled if (!Settings.getInstance().isQdnEnabled()) { return; } - Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet(); - - GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) message; - byte[] signature = getArbitraryMetadataMessage.getSignature(); - String signature58 = Base58.encode(signature); - Long now = NTP.getTime(); - Triple newEntry = new Triple<>(signature58, peer, now); - - // If we've seen this request recently, then ignore - if (arbitraryMetadataRequests.putIfAbsent(message.getId(), newEntry) != null) { - LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peer, signature58); - return; - } - - LOGGER.debug("Received metadata request from peer {} for signature {}", peer, signature58); - - ArbitraryTransactionData transactionData = null; - ArbitraryDataFile metadataFile = null; - - try (final Repository repository = RepositoryManager.getRepository()) { - - // Firstly we need to lookup this file on chain to get its metadata hash - transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature); - if (transactionData instanceof ArbitraryTransactionData) { - - // Check if we're even allowed to serve metadata for this transaction - if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { - - byte[] metadataHash = transactionData.getMetadataHash(); - if (metadataHash != null) { - - // Load metadata file - metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature); - } - } - } - - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while fetching arbitrary metadata for peer %s", peer), e); - } - - // We should only respond if we have the metadata file - if (metadataFile != null && metadataFile.exists()) { - - // We have the metadata file, so update requests map to reflect that we've sent it - newEntry = new Triple<>(null, null, now); - arbitraryMetadataRequests.put(message.getId(), newEntry); - - ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, metadataFile); - arbitraryMetadataMessage.setId(message.getId()); - if (!peer.sendMessage(arbitraryMetadataMessage)) { - LOGGER.debug("Couldn't send metadata"); - peer.disconnect("failed to send metadata"); - return; - } - LOGGER.debug("Sent metadata"); - - // Nothing left to do, so return to prevent any unnecessary forwarding from occurring - LOGGER.debug("No need for any forwarding because metadata request is fully served"); - return; - - } - - // We may need to forward this request on - boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName())); - if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { - // In relay mode - so ask our other peers if they have it - - long requestTime = getArbitraryMetadataMessage.getRequestTime(); - int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1; - long totalRequestTime = now - requestTime; - - if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { - // Relay request hasn't timed out yet, so can potentially be rebroadcast - if (requestHops < RELAY_REQUEST_MAX_HOPS) { - // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast - - Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops); - relayGetArbitraryMetadataMessage.setId(message.getId()); - - LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); - Network.getInstance().broadcast( - broadcastPeer -> - !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : - broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage); - - } - else { - // This relay request has reached the maximum number of allowed hops - } - } - else { - // This relay request has timed out - } + synchronized (lock) { + messageList.add(new PeerMessage(peer, message)); } } + private void processNetworkGetArbitraryMetadataMessage() { + + try { + List messagesToProcess; + synchronized (lock) { + messagesToProcess = new ArrayList<>(messageList); + messageList.clear(); + } + + Map signatureBySignature58 = new HashMap<>((messagesToProcess.size())); + Map nowBySignature58 = new HashMap<>(messagesToProcess.size()); + Map peerMessageBySignature58 = new HashMap<>(messagesToProcess.size()); + + for( PeerMessage peerMessage : messagesToProcess) { + Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet(); + + GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) peerMessage.message; + byte[] signature = getArbitraryMetadataMessage.getSignature(); + String signature58 = Base58.encode(signature); + Long now = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peerMessage.peer, now); + + // If we've seen this request recently, then ignore + if (arbitraryMetadataRequests.putIfAbsent(peerMessage.message.getId(), newEntry) != null) { + LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peerMessage.peer, signature58); + continue; + } + + LOGGER.debug("Received metadata request from peer {} for signature {}", peerMessage.peer, signature58); + + signatureBySignature58.put(signature58, signature); + nowBySignature58.put(signature58, now); + peerMessageBySignature58.put(signature58, peerMessage); + } + + if( signatureBySignature58.isEmpty() ) return; + + List transactionDataList; + try (final Repository repository = RepositoryManager.getRepository()) { + + // Firstly we need to lookup this file on chain to get its metadata hash + transactionDataList = repository.getTransactionRepository().fromSignatures(new ArrayList(signatureBySignature58.values())); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary transactions"), e); + return; + } + + Map dataBySignature58 + = transactionDataList.stream() + .filter(data -> data instanceof ArbitraryTransactionData) + .map(ArbitraryTransactionData.class::cast) + .collect(Collectors.toMap(data -> Base58.encode(data.getSignature()), Function.identity())); + + for(Map.Entry entry : dataBySignature58.entrySet()) { + String signature58 = entry.getKey(); + ArbitraryTransactionData transactionData = entry.getValue(); + + try { + + // Check if we're even allowed to serve metadata for this transaction + if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) { + + byte[] metadataHash = transactionData.getMetadataHash(); + if (metadataHash != null) { + + // Load metadata file + ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, transactionData.getSignature()); + // We should only respond if we have the metadata file + if (metadataFile != null && metadataFile.exists()) { + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + Message message = peerMessage.message; + Peer peer = peerMessage.peer; + + // We have the metadata file, so update requests map to reflect that we've sent it + Triple newEntry = new Triple<>(null, null, nowBySignature58.get(signature58)); + arbitraryMetadataRequests.put(message.getId(), newEntry); + + ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(entry.getValue().getSignature(), metadataFile); + arbitraryMetadataMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryMetadataMessage)) { + LOGGER.debug("Couldn't send metadata"); + peer.disconnect("failed to send metadata"); + continue; + } + LOGGER.debug("Sent metadata"); + + // Nothing left to do, so return to prevent any unnecessary forwarding from occurring + LOGGER.debug("No need for any forwarding because metadata request is fully served"); + } + + } + } + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary metadata"), e); + } + + // We may need to forward this request on + boolean isBlocked = (transactionDataList == null || ListUtils.isNameBlocked(transactionData.getName())); + if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) { + // In relay mode - so ask our other peers if they have it + + PeerMessage peerMessage = peerMessageBySignature58.get(signature58); + GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) peerMessage.message; + long requestTime = getArbitraryMetadataMessage.getRequestTime(); + int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1; + long totalRequestTime = nowBySignature58.get(signature58) - requestTime; + + if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { + // Relay request hasn't timed out yet, so can potentially be rebroadcast + if (requestHops < RELAY_REQUEST_MAX_HOPS) { + // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + + byte[] signature = signatureBySignature58.get(signature58); + Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops); + relayGetArbitraryMetadataMessage.setId(getArbitraryMetadataMessage.getId()); + + Peer peer = peerMessage.peer; + LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); + Network.getInstance().broadcast( + broadcastPeer -> + !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null : + broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage); + + } else { + // This relay request has reached the maximum number of allowed hops + } + } else { + // This relay request has timed out + } + } + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } } diff --git a/src/main/java/org/qortal/controller/arbitrary/PeerMessage.java b/src/main/java/org/qortal/controller/arbitrary/PeerMessage.java new file mode 100644 index 00000000..e77eca4b --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/PeerMessage.java @@ -0,0 +1,22 @@ +package org.qortal.controller.arbitrary; + +import org.qortal.network.Peer; +import org.qortal.network.message.Message; + +public class PeerMessage { + Peer peer; + Message message; + + public PeerMessage(Peer peer, Message message) { + this.peer = peer; + this.message = message; + } + + public Peer getPeer() { + return peer; + } + + public Message getMessage() { + return message; + } +} diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 654513f2..c17e5758 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -8,6 +8,7 @@ import org.qortal.account.PrivateKeyAccount; import org.qortal.api.model.crosschain.TradeBotCreateRequest; import org.qortal.controller.Controller; import org.qortal.controller.Synchronizer; +import org.qortal.controller.arbitrary.PeerMessage; import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult; import org.qortal.crosschain.*; import org.qortal.crypto.Crypto; @@ -37,7 +38,12 @@ import org.qortal.utils.NTP; import java.awt.TrayIcon.MessageType; import java.security.SecureRandom; import java.util.*; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; /** * Performing cross-chain trading steps on behalf of user. @@ -118,6 +124,9 @@ public class TradeBot implements Listener { private Map validTrades = new HashMap<>(); private TradeBot() { + + tradePresenceMessageScheduler.scheduleAtFixedRate( this::processTradePresencesMessages, 60, 1, TimeUnit.SECONDS); + EventBus.INSTANCE.addListener(event -> TradeBot.getInstance().listen(event)); } @@ -551,77 +560,139 @@ public class TradeBot implements Listener { } } + // List to collect messages + private final List tradePresenceMessageList = new ArrayList<>(); + // Lock to synchronize access to the list + private final Object tradePresenceMessageLock = new Object(); + + // Scheduled executor service to process messages every second + private final ScheduledExecutorService tradePresenceMessageScheduler = Executors.newScheduledThreadPool(1); + public void onTradePresencesMessage(Peer peer, Message message) { - TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) message; - List peersTradePresences = tradePresencesMessage.getTradePresences(); + synchronized (tradePresenceMessageLock) { + tradePresenceMessageList.add(new PeerMessage(peer, message)); + } + } - long now = NTP.getTime(); - // Timestamps before this are too far into the past - long pastThreshold = now; - // Timestamps after this are too far into the future - long futureThreshold = now + PRESENCE_LIFETIME; + public void processTradePresencesMessages() { - Map> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap(); + try { + List messagesToProcess; + synchronized (tradePresenceMessageLock) { + messagesToProcess = new ArrayList<>(tradePresenceMessageList); + tradePresenceMessageList.clear(); + } - int newCount = 0; + if( messagesToProcess.isEmpty() ) return; - try (final Repository repository = RepositoryManager.getRepository()) { - for (TradePresenceData peersTradePresence : peersTradePresences) { - long timestamp = peersTradePresence.getTimestamp(); + Map> tradePresencesByPeer = new HashMap<>(messagesToProcess.size()); - // Ignore if timestamp is out of bounds - if (timestamp < pastThreshold || timestamp > futureThreshold) { - if (timestamp < pastThreshold) - LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}", - peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold - ); - else - LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}", - peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold + // map all trade presences from the messages to their peer + for( PeerMessage peerMessage : messagesToProcess ) { + TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) peerMessage.getMessage(); + + List peersTradePresences = tradePresencesMessage.getTradePresences(); + + tradePresencesByPeer.put(peerMessage.getPeer(), peersTradePresences); + } + + long now = NTP.getTime(); + // Timestamps before this are too far into the past + long pastThreshold = now; + // Timestamps after this are too far into the future + long futureThreshold = now + PRESENCE_LIFETIME; + + Map> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap(); + + int newCount = 0; + + Map> peersByAtAddress = new HashMap<>(tradePresencesByPeer.size()); + Map tradePresenceByAtAddress = new HashMap<>(tradePresencesByPeer.size()); + + // for each batch of trade presence data from a peer, validate and populate the maps declared above + for ( Map.Entry> entry: tradePresencesByPeer.entrySet()) { + + Peer peer = entry.getKey(); + + for( TradePresenceData peersTradePresence : entry.getValue() ) { + // TradePresenceData peersTradePresence + long timestamp = peersTradePresence.getTimestamp(); + + // Ignore if timestamp is out of bounds + if (timestamp < pastThreshold || timestamp > futureThreshold) { + if (timestamp < pastThreshold) + LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}", + peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold + ); + else + LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}", + peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold + ); + + continue; + } + + ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey()); + + // Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older + TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray); + if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) { + if (timestamp == existingTradeData.getTimestamp()) + LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before", + peersTradePresence.getAtAddress(), peer, timestamp + ); + else + LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}", + peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp() + ); + + continue; + } + + // Check timestamp signature + byte[] timestampSignature = peersTradePresence.getSignature(); + byte[] timestampBytes = Longs.toByteArray(timestamp); + byte[] publicKey = peersTradePresence.getPublicKey(); + if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) { + LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify", + peersTradePresence.getAtAddress(), peer ); - continue; + continue; + } + + peersByAtAddress.computeIfAbsent(peersTradePresence.getAtAddress(), address -> new ArrayList<>()).add(peer); + tradePresenceByAtAddress.put(peersTradePresence.getAtAddress(), peersTradePresence); } + } - ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey()); + if( tradePresenceByAtAddress.isEmpty() ) return; - // Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older - TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray); - if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) { - if (timestamp == existingTradeData.getTimestamp()) - LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before", - peersTradePresence.getAtAddress(), peer, timestamp - ); - else - LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}", - peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp() - ); + List atDataList; + try (final Repository repository = RepositoryManager.getRepository()) { + atDataList = repository.getATRepository().fromATAddresses( new ArrayList<>(tradePresenceByAtAddress.keySet()) ); + } catch (DataException e) { + LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e); + return; + } - continue; - } + Map> supplierByAtAddress = new HashMap<>(atDataList.size()); - // Check timestamp signature - byte[] timestampSignature = peersTradePresence.getSignature(); - byte[] timestampBytes = Longs.toByteArray(timestamp); - byte[] publicKey = peersTradePresence.getPublicKey(); - if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) { - LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify", - peersTradePresence.getAtAddress(), peer - ); + List validatedAtDataList = new ArrayList<>(atDataList.size()); - continue; - } + // for each trade + for( ATData atData : atDataList ) { - ATData atData = repository.getATRepository().fromATAddress(peersTradePresence.getAtAddress()); + TradePresenceData peersTradePresence = tradePresenceByAtAddress.get(atData.getATAddress()); if (atData == null || atData.getIsFrozen() || atData.getIsFinished()) { if (atData == null) - LOGGER.trace("Ignoring trade presence {} from peer {} as AT doesn't exist", - peersTradePresence.getAtAddress(), peer + LOGGER.trace("Ignoring trade presence {} from peer as AT doesn't exist", + peersTradePresence.getAtAddress() ); else - LOGGER.trace("Ignoring trade presence {} from peer {} as AT is frozen or finished", - peersTradePresence.getAtAddress(), peer + LOGGER.trace("Ignoring trade presence {} from peer as AT is frozen or finished", + peersTradePresence.getAtAddress() ); continue; @@ -630,51 +701,87 @@ public class TradeBot implements Listener { ByteArray atCodeHash = ByteArray.wrap(atData.getCodeHash()); Supplier acctSupplier = acctSuppliersByCodeHash.get(atCodeHash); if (acctSupplier == null) { - LOGGER.trace("Ignoring trade presence {} from peer {} as AT isn't a known ACCT?", - peersTradePresence.getAtAddress(), peer + LOGGER.trace("Ignoring trade presence {} from peer as AT isn't a known ACCT?", + peersTradePresence.getAtAddress() ); continue; } - - CrossChainTradeData tradeData = acctSupplier.get().populateTradeData(repository, atData); - if (tradeData == null) { - LOGGER.trace("Ignoring trade presence {} from peer {} as trade data not found?", - peersTradePresence.getAtAddress(), peer - ); - - continue; - } - - // Convert signer's public key to address form - String signerAddress = peersTradePresence.getTradeAddress(); - - // Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form) - if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) { - LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?", - peersTradePresence.getAtAddress(), peer - ); - - continue; - } - - // This is new to us - this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence); - ++newCount; - - LOGGER.trace("Added trade presence {} from peer {} with timestamp {}", - peersTradePresence.getAtAddress(), peer, timestamp - ); - - EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence)); + validatedAtDataList.add(atData); } - } catch (DataException e) { - LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e); - } - if (newCount > 0) { - LOGGER.debug("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size()); - rebuildSafeAllTradePresences(); + // populated data for each trade + List crossChainTradeDataList; + + // validated trade data grouped by code (cross chain coin) + Map> atDataByCodeHash + = validatedAtDataList.stream().collect( + Collectors.groupingBy(data -> ByteArray.wrap(data.getCodeHash()))); + + try (final Repository repository = RepositoryManager.getRepository()) { + + crossChainTradeDataList = new ArrayList<>(); + + // for each code (cross chain coin), get each trade, then populate trade data + for( Map.Entry> entry : atDataByCodeHash.entrySet() ) { + + Supplier acctSupplier = acctSuppliersByCodeHash.get(entry.getKey()); + + crossChainTradeDataList.addAll( + acctSupplier.get().populateTradeDataList( + repository, + entry.getValue() + ) + .stream().filter( data -> data != null ) + .collect(Collectors.toList()) + ); + } + } catch (DataException e) { + LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e); + return; + } + + // for each populated trade data, validate and fire event + for( CrossChainTradeData tradeData : crossChainTradeDataList ) { + + List peers = peersByAtAddress.get(tradeData.qortalAtAddress); + + for( Peer peer : peers ) { + + TradePresenceData peersTradePresence = tradePresenceByAtAddress.get(tradeData.qortalAtAddress); + + // Convert signer's public key to address form + String signerAddress = peersTradePresence.getTradeAddress(); + + // Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form) + if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) { + LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?", + peersTradePresence.getAtAddress(), peer + ); + + continue; + } + + ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey()); + + // This is new to us + this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence); + ++newCount; + + LOGGER.trace("Added trade presence {} from peer {} with timestamp {}", + peersTradePresence.getAtAddress(), peer, tradeData.creationTimestamp + ); + + EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence)); + } + } + + if (newCount > 0) { + LOGGER.info("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size()); + rebuildSafeAllTradePresences(); + } + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); } } diff --git a/src/main/java/org/qortal/crosschain/ACCT.java b/src/main/java/org/qortal/crosschain/ACCT.java index de28cfce..83b453e6 100644 --- a/src/main/java/org/qortal/crosschain/ACCT.java +++ b/src/main/java/org/qortal/crosschain/ACCT.java @@ -6,6 +6,9 @@ import org.qortal.data.crosschain.CrossChainTradeData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import java.util.List; +import java.util.OptionalLong; + public interface ACCT { public byte[] getCodeBytesHash(); @@ -16,8 +19,12 @@ public interface ACCT { public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException; + public List populateTradeDataList(Repository respository, List atDataList) throws DataException; + public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException; + CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException; + public byte[] buildCancelMessage(String creatorQortalAddress); public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException; diff --git a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java index cb855466..d7b8bb77 100644 --- a/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java +++ b/src/main/java/org/qortal/crosschain/BitcoinACCTv1.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -608,7 +610,14 @@ public class BitcoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -617,13 +626,14 @@ public class BitcoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -636,8 +646,13 @@ public class BitcoinACCTv1 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java b/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java index ecf768ed..efef0959 100644 --- a/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/BitcoinACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class BitcoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class BitcoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class BitcoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java b/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java index 9fa67592..5dec80f0 100644 --- a/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java +++ b/src/main/java/org/qortal/crosschain/DigibyteACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class DigibyteACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class DigibyteACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class DigibyteACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java index a5ec6f1f..a0caeb80 100644 --- a/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java +++ b/src/main/java/org/qortal/crosschain/DogecoinACCTv1.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -566,7 +568,14 @@ public class DogecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -575,13 +584,14 @@ public class DogecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -594,8 +604,13 @@ public class DogecoinACCTv1 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java b/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java index 06b04705..18581b2c 100644 --- a/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/DogecoinACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class DogecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class DogecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class DogecoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java index 6a828981..b91e8e65 100644 --- a/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv1.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -559,7 +561,14 @@ public class LitecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -568,13 +577,14 @@ public class LitecoinACCTv1 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -587,8 +597,13 @@ public class LitecoinACCTv1 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java b/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java index 4a533b4b..97222317 100644 --- a/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/LitecoinACCTv3.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -562,7 +564,14 @@ public class LitecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -571,13 +580,14 @@ public class LitecoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -590,8 +600,13 @@ public class LitecoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/PirateChain.java b/src/main/java/org/qortal/crosschain/PirateChain.java index 48178f28..53a6fa9c 100644 --- a/src/main/java/org/qortal/crosschain/PirateChain.java +++ b/src/main/java/org/qortal/crosschain/PirateChain.java @@ -52,12 +52,7 @@ public class PirateChain extends Bitcoiny { public Collection getServers() { return Arrays.asList( // Servers chosen on NO BASIS WHATSOEVER from various sources! - new Server("lightd.pirate.black", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr1.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr2.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr3.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr4.qortal.online", Server.ConnectionType.SSL, 443), - new Server("wallet-arrr5.qortal.online", Server.ConnectionType.SSL, 443) + new Server("lightd.pirate.black", Server.ConnectionType.SSL, 443) ); } diff --git a/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java b/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java index 8873eeab..4c2893ea 100644 --- a/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java +++ b/src/main/java/org/qortal/crosschain/PirateChainACCTv3.java @@ -4,6 +4,7 @@ import com.google.common.hash.HashCode; import com.google.common.primitives.Bytes; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -19,6 +20,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -580,7 +582,14 @@ public class PirateChainACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -589,13 +598,14 @@ public class PirateChainACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -608,8 +618,13 @@ public class PirateChainACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/crosschain/PirateWallet.java b/src/main/java/org/qortal/crosschain/PirateWallet.java index d8fdc351..9ecfc2c0 100644 --- a/src/main/java/org/qortal/crosschain/PirateWallet.java +++ b/src/main/java/org/qortal/crosschain/PirateWallet.java @@ -8,6 +8,7 @@ import org.bouncycastle.util.encoders.DecoderException; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.controller.PirateChainWalletController; import org.qortal.crypto.Crypto; import org.qortal.settings.Settings; @@ -67,8 +68,8 @@ public class PirateWallet { } // Pick a random server - PirateLightClient.Server server = this.getRandomServer(); - String serverUri = String.format("https://%s:%d/", server.hostname, server.port); + ChainableServer server = PirateChain.getInstance().blockchainProvider.getCurrentServer(); + String serverUri = String.format("https://%s:%d/", server.getHostName(), server.getPort()); // Pirate library uses base64 encoding String entropy64 = Base64.toBase64String(this.entropyBytes); diff --git a/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java b/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java index f027e9ca..b880f831 100644 --- a/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java +++ b/src/main/java/org/qortal/crosschain/RavencoinACCTv3.java @@ -6,6 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.ciyam.at.*; import org.qortal.account.Account; +import org.qortal.api.resource.CrossChainUtils; import org.qortal.asset.Asset; import org.qortal.at.QortalFunctionCode; import org.qortal.crypto.Crypto; @@ -21,6 +22,7 @@ import org.qortal.utils.BitTwiddling; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.OptionalLong; import static org.ciyam.at.OpCode.calcOffset; @@ -569,7 +571,14 @@ public class RavencoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException { ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); + } + + @Override + public List populateTradeDataList(Repository repository, List atDataList) throws DataException { + List crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList); + + return crossChainTradeDataList; } /** @@ -578,13 +587,14 @@ public class RavencoinACCTv3 implements ACCT { @Override public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException { ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress()); - return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData); + return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty()); } /** * Returns CrossChainTradeData with useful info extracted from AT. */ - public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException { + @Override + public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException { byte[] addressBytes = new byte[25]; // for general use String atAddress = atStateData.getATAddress(); @@ -597,8 +607,13 @@ public class RavencoinACCTv3 implements ACCT { tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey); tradeData.creationTimestamp = creationTimestamp; - Account atAccount = new Account(repository, atAddress); - tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + if(optionalBalance.isPresent()) { + tradeData.qortBalance = optionalBalance.getAsLong(); + } + else { + Account atAccount = new Account(repository, atAddress); + tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT); + } byte[] stateData = atStateData.getStateData(); ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData); diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index f500b2e8..3737852f 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -982,7 +982,7 @@ public class Network { if (maxThreadsForMessageType != null) { Integer threadCount = threadsPerMessageType.get(message.getType()); if (threadCount != null && threadCount >= maxThreadsForMessageType) { - LOGGER.trace("Discarding {} message as there are already {} active threads", message.getType().name(), threadCount); + LOGGER.warn("Discarding {} message as there are already {} active threads", message.getType().name(), threadCount); return; } } diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 2b653ab5..7e6c16ff 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -14,6 +14,8 @@ public interface ATRepository { /** Returns ATData using AT's address or null if none found */ public ATData fromATAddress(String atAddress) throws DataException; + public List fromATAddresses(List atAddresses) throws DataException; + /** Returns where AT with passed address exists in repository */ public boolean exists(String atAddress) throws DataException; @@ -62,6 +64,8 @@ public interface ATRepository { */ public ATStateData getLatestATState(String atAddress) throws DataException; + public List getLatestATStates(List collect) throws DataException; + /** * Returns final ATStateData for ATs matching codeHash (required) * and specific data segment value (optional). diff --git a/src/main/java/org/qortal/repository/AccountRepository.java b/src/main/java/org/qortal/repository/AccountRepository.java index f68fe8eb..daac1e02 100644 --- a/src/main/java/org/qortal/repository/AccountRepository.java +++ b/src/main/java/org/qortal/repository/AccountRepository.java @@ -130,6 +130,8 @@ public interface AccountRepository { */ public AccountBalanceData getBalance(String address, long assetId) throws DataException; + public List getBalances(List addresses, long assetId) throws DataException; + /** Returns all account balances for given assetID, optionally excluding zero balances. */ public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException; diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index d4517485..79d55c9a 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -18,6 +18,8 @@ public interface TransactionRepository { public TransactionData fromSignature(byte[] signature) throws DataException; + public List fromSignatures(List signatures) throws DataException; + public TransactionData fromReference(byte[] reference) throws DataException; public TransactionData fromHeightAndSequence(int height, int sequence) throws DataException; @@ -351,4 +353,5 @@ public interface TransactionRepository { public void delete(TransactionData transactionData) throws DataException; + } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 6310ec02..c941e15a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -15,8 +15,12 @@ import org.qortal.utils.ByteArray; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.qortal.data.account.AccountData; @@ -76,6 +80,63 @@ public class HSQLDBATRepository implements ATRepository { } } + @Override + public List fromATAddresses(List atAddresses) throws DataException { + String sql = "SELECT creator, created_when, version, asset_id, code_bytes, code_hash, " + + "is_sleeping, sleep_until_height, is_finished, had_fatal_error, " + + "is_frozen, frozen_balance, sleep_until_message_timestamp, AT_address " + + "FROM ATs " + + "WHERE AT_address IN (" + + String.join(", ", Collections.nCopies(atAddresses.size(), "?")) + + ")" + ; + + List list; + try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddresses.toArray(new String[atAddresses.size()]))) { + if (resultSet == null) { + return new ArrayList<>(0); + } + + list = new ArrayList<>(atAddresses.size()); + + do { + byte[] creatorPublicKey = resultSet.getBytes(1); + long created = resultSet.getLong(2); + int version = resultSet.getInt(3); + long assetId = resultSet.getLong(4); + byte[] codeBytes = resultSet.getBytes(5); // Actually BLOB + byte[] codeHash = resultSet.getBytes(6); + boolean isSleeping = resultSet.getBoolean(7); + + Integer sleepUntilHeight = resultSet.getInt(8); + if (sleepUntilHeight == 0 && resultSet.wasNull()) + sleepUntilHeight = null; + + boolean isFinished = resultSet.getBoolean(9); + boolean hadFatalError = resultSet.getBoolean(10); + boolean isFrozen = resultSet.getBoolean(11); + + Long frozenBalance = resultSet.getLong(12); + if (frozenBalance == 0 && resultSet.wasNull()) + frozenBalance = null; + + Long sleepUntilMessageTimestamp = resultSet.getLong(13); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + + String atAddress = resultSet.getString(14); + + list.add(new ATData(atAddress, creatorPublicKey, created, version, assetId, codeBytes, codeHash, + isSleeping, sleepUntilHeight, isFinished, hadFatalError, isFrozen, frozenBalance, + sleepUntilMessageTimestamp)); + } while ( resultSet.next()); + + return list; + } catch (SQLException e) { + throw new DataException("Unable to fetch AT from repository", e); + } + } + @Override public boolean exists(String atAddress) throws DataException { try { @@ -403,6 +464,56 @@ public class HSQLDBATRepository implements ATRepository { } } + @Override + public List getLatestATStates(List atAddresses) throws DataException{ + String sql = "SELECT height, state_data, state_hash, fees, is_initial, sleep_until_message_timestamp, AT_address " + + "FROM ATStates " + + "JOIN ATStatesData USING (AT_address, height) " + + "WHERE ATStates.AT_address IN (" + + String.join(", ", Collections.nCopies(atAddresses.size(), "?")) + + ")"; + + List stateDataList; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddresses.toArray(new String[atAddresses.size()]))) { + if (resultSet == null) + return new ArrayList<>(0); + + stateDataList = new ArrayList<>(); + + do { + int height = resultSet.getInt(1); + byte[] stateData = resultSet.getBytes(2); // Actually BLOB + byte[] stateHash = resultSet.getBytes(3); + long fees = resultSet.getLong(4); + boolean isInitial = resultSet.getBoolean(5); + + Long sleepUntilMessageTimestamp = resultSet.getLong(6); + if (sleepUntilMessageTimestamp == 0 && resultSet.wasNull()) + sleepUntilMessageTimestamp = null; + + String atAddress = resultSet.getString(7); + stateDataList.add(new ATStateData(atAddress, height, stateData, stateHash, fees, isInitial, sleepUntilMessageTimestamp)); + } while( resultSet.next()); + } catch (SQLException e) { + throw new DataException("Unable to fetch latest AT state from repository", e); + } + + Map> stateDataByAtAddress + = stateDataList.stream() + .collect(Collectors.groupingBy(ATStateData::getATAddress)); + + List latestForEachAtAddress + = stateDataByAtAddress.values().stream() + .map(list -> list.stream() + .max(Comparator.comparing(ATStateData::getHeight)) + .orElse(null)) + .filter(obj -> obj != null) + .collect(Collectors.toList()); + + return latestForEachAtAddress; + } + @Override public List getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished, Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight, diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java index 9cec85b2..7a0e486c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java @@ -407,6 +407,39 @@ public class HSQLDBAccountRepository implements AccountRepository { } } + @Override + public List getBalances(List addresses, long assetId) throws DataException { + + StringBuffer sql = new StringBuffer(); + sql.append("SELECT balance, account, asset_id FROM AccountBalances "); + sql.append("WHERE account IN ("); + sql.append(String.join(", ", Collections.nCopies(addresses.size(), "?"))); + sql.append(")"); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), addresses.toArray(new String[addresses.size()]))) { + if (resultSet == null) + return new ArrayList<>(0); + + List balances = new ArrayList<>(addresses.size()); + do { + long balance = resultSet.getLong(1); + String address = resultSet.getString(2); + Long assetIdResult = resultSet.getLong(3); + + if( assetIdResult != assetId ) { + LOGGER.warn("assetIdResult = " + assetIdResult); + continue; + } + + balances.add(new AccountBalanceData(address, assetId, balance) ); + } while( resultSet.next()); + + return balances; + } catch (SQLException e) { + throw new DataException("Unable to fetch account balance from repository", e); + } + } + @Override public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException { StringBuilder sql = new StringBuilder(1024); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java index 48262dee..535c3ed6 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java @@ -40,13 +40,25 @@ public class HSQLDBChatRepository implements ChatRepository { StringBuilder sql = new StringBuilder(1024); + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + sql.append("SELECT created_when, tx_group_id, Transactions.reference, creator, " + "sender, SenderNames.name, recipient, RecipientNames.name, " + "chat_reference, data, is_text, is_encrypted, signature " + "FROM ChatTransactions " + "JOIN Transactions USING (signature) " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " - + "LEFT OUTER JOIN Names AS RecipientNames ON RecipientNames.owner = recipient "); + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS RecipientNames ON RecipientNames.owner = recipient "); // WHERE clauses @@ -152,11 +164,11 @@ public class HSQLDBChatRepository implements ChatRepository { // if the PrimaryTable is available, then use it if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { - LOGGER.info("using PrimaryNames for chat transactions"); + LOGGER.debug("using PrimaryNames for chat transactions"); tableName = "PrimaryNames"; } else { - LOGGER.info("using Names for chat transactions"); + LOGGER.debug("using Names for chat transactions"); tableName = "Names"; } @@ -202,6 +214,18 @@ public class HSQLDBChatRepository implements ChatRepository { } private List getActiveGroupChats(String address, Encoding encoding, Boolean hasChatReference) throws DataException { + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + // Find groups where address is a member and potential latest message details String groupsSql = "SELECT group_id, group_name, latest_timestamp, sender, sender_name, signature, data " + "FROM GroupMembers " @@ -210,7 +234,7 @@ public class HSQLDBChatRepository implements ChatRepository { + "SELECT created_when AS latest_timestamp, sender, name AS sender_name, signature, data " + "FROM ChatTransactions " + "JOIN Transactions USING (signature) " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " // NOTE: We need to qualify "Groups.group_id" here to avoid "General error" bug in HSQLDB v2.5.0 + "WHERE tx_group_id = Groups.group_id AND type = " + TransactionType.CHAT.value + " "; @@ -254,7 +278,7 @@ public class HSQLDBChatRepository implements ChatRepository { String grouplessSql = "SELECT created_when, sender, SenderNames.name, signature, data " + "FROM ChatTransactions " + "JOIN Transactions USING (signature) " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + "WHERE tx_group_id = 0 " + "AND recipient IS NULL "; @@ -294,6 +318,18 @@ public class HSQLDBChatRepository implements ChatRepository { } private List getActiveDirectChats(String address, Boolean hasChatReference) throws DataException { + String tableName; + + // if the PrimaryTable is available, then use it + if( this.repository.getBlockRepository().getBlockchainHeight() > BlockChain.getInstance().getMultipleNamesPerAccountHeight()) { + LOGGER.debug("using PrimaryNames for chat transactions"); + tableName = "PrimaryNames"; + } + else { + LOGGER.debug("using Names for chat transactions"); + tableName = "Names"; + } + // Find chat messages involving address String directSql = "SELECT other_address, name, latest_timestamp, sender, sender_name " + "FROM (" @@ -307,7 +343,7 @@ public class HSQLDBChatRepository implements ChatRepository { + "SELECT created_when AS latest_timestamp, sender, name AS sender_name " + "FROM ChatTransactions " + "NATURAL JOIN Transactions " - + "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender " + + "LEFT OUTER JOIN " + tableName + " AS SenderNames ON SenderNames.owner = sender " + "WHERE (sender = other_address AND recipient = ?) " + "OR (sender = ? AND recipient = other_address) "; @@ -323,7 +359,7 @@ public class HSQLDBChatRepository implements ChatRepository { directSql += "ORDER BY created_when DESC " + "LIMIT 1" + ") AS LatestMessages " - + "LEFT OUTER JOIN Names ON owner = other_address"; + + "LEFT OUTER JOIN " + tableName + " ON owner = other_address"; Object[] bindParams = new Object[] { address, address, address, address }; diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index fe0b4d0b..cd646cb9 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -155,6 +155,58 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } } + @Override + public List fromSignatures(List signatures) throws DataException { + StringBuffer sql = new StringBuffer(); + + sql.append("SELECT type, reference, creator, created_when, fee, tx_group_id, block_height, approval_status, approval_height, signature "); + sql.append("FROM Transactions WHERE signature IN ("); + sql.append(String.join(", ", Collections.nCopies(signatures.size(), "?"))); + sql.append(")"); + + List list; + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), signatures.toArray(new byte[0][]))) { + if (resultSet == null) { + return new ArrayList<>(0); + } + + list = new ArrayList<>(signatures.size()); + + do { + TransactionType type = TransactionType.valueOf(resultSet.getInt(1)); + + byte[] reference = resultSet.getBytes(2); + byte[] creatorPublicKey = resultSet.getBytes(3); + long timestamp = resultSet.getLong(4); + + Long fee = resultSet.getLong(5); + if (fee == 0 && resultSet.wasNull()) + fee = null; + + int txGroupId = resultSet.getInt(6); + + Integer blockHeight = resultSet.getInt(7); + if (blockHeight == 0 && resultSet.wasNull()) + blockHeight = null; + + ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(8)); + Integer approvalHeight = resultSet.getInt(9); + if (approvalHeight == 0 && resultSet.wasNull()) + approvalHeight = null; + + byte[] signature = resultSet.getBytes(10); + + BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature); + + list.add( fromBase(type, baseTransactionData) ); + } while( resultSet.next()); + + return list; + } catch (SQLException e) { + throw new DataException("Unable to fetch transactions from repository", e); + } + } + @Override public TransactionData fromReference(byte[] reference) throws DataException { String sql = "SELECT type, signature, creator, created_when, fee, tx_group_id, block_height, approval_status, approval_height " diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 3e82655b..3123ae96 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -759,14 +759,14 @@ public class Settings { maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA", 5)); maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE_LIST", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE_LIST", 5)); + maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE_LIST", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE_LIST", 50)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_SIGNATURES", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_METADATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_TRANSACTION", 10)); - maxThreadsPerMessageType.add(new ThreadLimit("TRANSACTION_SIGNATURES", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("TRADE_PRESENCES", 5)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_TRANSACTION", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("TRANSACTION_SIGNATURES", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("TRADE_PRESENCES", 50)); } // Getters / setters diff --git a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java index 17c966fe..2ebd3b0e 100644 --- a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java @@ -131,16 +131,12 @@ public class ArbitraryIndexUtils { ) ); - LOGGER.info("processed indices by term: count = " + indicesByTerm.size()); - // lock, clear old, load new synchronized( IndexCache.getInstance().getIndicesByTerm() ) { IndexCache.getInstance().getIndicesByTerm().clear(); IndexCache.getInstance().getIndicesByTerm().putAll(indicesByTerm); } - LOGGER.info("loaded indices by term"); - LOGGER.debug("processing indices by issuer ..."); Map> indicesByIssuer = indexDetails.stream().collect( @@ -154,15 +150,11 @@ public class ArbitraryIndexUtils { ) ); - LOGGER.info("processed indices by issuer: count = " + indicesByIssuer.size()); - // lock, clear old, load new synchronized( IndexCache.getInstance().getIndicesByIssuer() ) { IndexCache.getInstance().getIndicesByIssuer().clear(); IndexCache.getInstance().getIndicesByIssuer().putAll(indicesByIssuer); } - - LOGGER.info("loaded indices by issuer"); } } @@ -221,7 +213,6 @@ public class ArbitraryIndexUtils { } } } - Thread.sleep(3000L); } java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index c860a034..1d4cab18 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -48,6 +48,24 @@ public class ArbitraryTransactionUtils { } } + public static List fetchTransactionDataList(final Repository repository, final List signature) { + try { + List transactions = repository.getTransactionRepository().fromSignatures(signature); + + List list + = transactions.stream() + .filter( transaction -> transaction instanceof ArbitraryTransactionData ) + .map( transactionData -> (ArbitraryTransactionData) transactionData) + .collect(Collectors.toList()); + + return list; + + } catch (DataException e) { + LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + return null; + } + } + public static ArbitraryTransactionData fetchLatestPut(Repository repository, ArbitraryTransactionData arbitraryTransactionData) { if (arbitraryTransactionData == null) { return null; diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 14ba924b..2f347d18 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -120,7 +120,8 @@ "nullGroupMembershipHeight": 2012800, "ignoreLevelForRewardShareHeight": 2012800, "adminQueryFixHeight": 2012800, - "multipleNamesPerAccountHeight": 9999999 + "multipleNamesPerAccountHeight": 9999999, + "mintedBlocksAdjustmentRemovalHeight": 9999999 }, "checkpoints": [ { "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" } diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index e410aae4..3bf89ab5 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -115,7 +115,8 @@ "ignoreLevelForRewardShareHeight": 9999999999999, "nullGroupMembershipHeight": 20, "adminQueryFixHeight": 9999999999999, - "multipleNamesPerAccountHeight": 10 + "multipleNamesPerAccountHeight": 10, + "mintedBlocksAdjustmentRemovalHeight": 9999999999999 }, "genesisInfo": { "version": 4, From ccb59559d6d2fd2fee9b1d5b083f7b2e3172e8e8 Mon Sep 17 00:00:00 2001 From: kennycud Date: Mon, 9 Jun 2025 18:25:43 -0700 Subject: [PATCH 27/42] the bootstrapper was resetting the database configuration that the db cache was dependent on, so that dependency was changed --- .../restricted/resource/AdminResource.java | 21 ------------------- .../org/qortal/controller/Controller.java | 6 ++---- .../org/qortal/repository/Repository.java | 2 ++ .../repository/hsqldb/HSQLDBCacheUtils.java | 18 ++++++++-------- .../repository/hsqldb/HSQLDBRepository.java | 5 +++++ 5 files changed, 18 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/qortal/api/restricted/resource/AdminResource.java b/src/main/java/org/qortal/api/restricted/resource/AdminResource.java index 439904eb..8c075d7e 100644 --- a/src/main/java/org/qortal/api/restricted/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/restricted/resource/AdminResource.java @@ -1092,25 +1092,4 @@ public class AdminResource { return info; } - - @GET - @Path("/dbstates") - @Operation( - summary = "Get DB States", - description = "Get DB States", - responses = { - @ApiResponse( - content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = DbConnectionInfo.class))) - ) - } - ) - public List getDbConnectionsStates() { - - try { - return Controller.REPOSITORY_FACTORY.getDbConnectionsStates(); - } catch (Exception e) { - LOGGER.error(e.getMessage(), e); - return new ArrayList<>(0); - } - } } \ No newline at end of file diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 33c49bde..826395ac 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -73,8 +73,6 @@ import java.util.stream.Collectors; public class Controller extends Thread { - public static HSQLDBRepositoryFactory REPOSITORY_FACTORY; - static { // This must go before any calls to LogManager/Logger System.setProperty("log4j2.formatMsgNoLookups", "true"); @@ -405,8 +403,8 @@ public class Controller extends Thread { LOGGER.info("Starting repository"); try { - REPOSITORY_FACTORY = new HSQLDBRepositoryFactory(getRepositoryUrl()); - RepositoryManager.setRepositoryFactory(REPOSITORY_FACTORY); + HSQLDBRepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); + RepositoryManager.setRepositoryFactory(repositoryFactory); RepositoryManager.setRequestedCheckpoint(Boolean.TRUE); try (final Repository repository = RepositoryManager.getRepository()) { diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index c0bdb0d9..a361ee95 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -1,6 +1,7 @@ package org.qortal.repository; import java.io.IOException; +import java.sql.Connection; import java.util.concurrent.TimeoutException; public interface Repository extends AutoCloseable { @@ -62,4 +63,5 @@ public interface Repository extends AutoCloseable { public static void attemptRecovery(String connectionUrl, String name) throws DataException {} + public Connection getConnection(); } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java index 46cd7cab..bee629b8 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java @@ -468,7 +468,7 @@ public class HSQLDBCacheUtils { Thread.currentThread().setName(DB_CACHE_TIMER_TASK); - try (final HSQLDBRepository respository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) { + try (final Repository respository = RepositoryManager.getRepository()) { fillCache(ArbitraryResourceCache.getInstance(), respository); } catch( DataException e ) { @@ -611,7 +611,7 @@ public class HSQLDBCacheUtils { private static int recordCurrentBalances(ConcurrentHashMap> balancesByHeight) { int currentHeight; - try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) { + try (final Repository repository = RepositoryManager.getRepository()) { // get current balances List accountBalances = getAccountBalances(repository); @@ -675,7 +675,7 @@ public class HSQLDBCacheUtils { * @param cache the cache to fill * @param repository the data source to fill the cache with */ - public static void fillCache(ArbitraryResourceCache cache, HSQLDBRepository repository) { + public static void fillCache(ArbitraryResourceCache cache, Repository repository) { try { // ensure all data is committed in, before we query it @@ -713,7 +713,7 @@ public class HSQLDBCacheUtils { * * @throws SQLException */ - private static void fillNamepMap(ConcurrentHashMap levelByName, HSQLDBRepository repository ) throws SQLException { + private static void fillNamepMap(ConcurrentHashMap levelByName, Repository repository ) throws SQLException { StringBuilder sql = new StringBuilder(512); @@ -721,7 +721,7 @@ public class HSQLDBCacheUtils { sql.append("FROM NAMES "); sql.append("INNER JOIN ACCOUNTS on owner = account "); - Statement statement = repository.connection.createStatement(); + Statement statement = repository.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql.toString()); @@ -744,7 +744,7 @@ public class HSQLDBCacheUtils { * @return the resources * @throws SQLException */ - private static List getResources( HSQLDBRepository repository) throws SQLException { + private static List getResources( Repository repository) throws SQLException { List resources = new ArrayList<>(); @@ -756,7 +756,7 @@ public class HSQLDBCacheUtils { sql.append("LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) WHERE name IS NOT NULL"); List arbitraryResources = new ArrayList<>(); - Statement statement = repository.connection.createStatement(); + Statement statement = repository.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql.toString()); @@ -822,7 +822,7 @@ public class HSQLDBCacheUtils { return resources; } - public static List getAccountBalances(HSQLDBRepository repository) { + public static List getAccountBalances(Repository repository) { StringBuilder sql = new StringBuilder(); @@ -836,7 +836,7 @@ public class HSQLDBCacheUtils { LOGGER.info( "Getting account balances ..."); try { - Statement statement = repository.connection.createStatement(); + Statement statement = repository.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql.toString()); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 4a41ed68..2bf88657 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -174,6 +174,11 @@ public class HSQLDBRepository implements Repository { // Transaction COMMIT / ROLLBACK / savepoints + @Override + public Connection getConnection() { + return this.connection; + } + @Override public void saveChanges() throws DataException { long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis(); From 92077f291212498eb66dbaf001ca6ebfed3be7fb Mon Sep 17 00:00:00 2001 From: Ice Date: Wed, 11 Jun 2025 15:45:08 -0400 Subject: [PATCH 28/42] Logging for Failed Respository Connections on Optional Runs --- src/main/java/org/qortal/network/Network.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 3737852f..d6d662dc 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -714,6 +714,7 @@ public class Network { // We can't block here so use tryRepository(). We don't NEED to connect a new peer. try (Repository repository = RepositoryManager.tryRepository()) { if (repository == null) { + LOGGER.warn("Unable to get repository connection : Network.getConnectablePeer()"); return null; } @@ -1499,6 +1500,7 @@ public class Network { // Pruning peers isn't critical so no need to block for a repository instance. try (Repository repository = RepositoryManager.tryRepository()) { if (repository == null) { + LOGGER.warn("Unable to get repository connection : Network.prunePeers()"); return; } @@ -1567,6 +1569,7 @@ public class Network { // Merging peers isn't critical so don't block for a repository instance. try (Repository repository = RepositoryManager.tryRepository()) { if (repository == null) { + LOGGER.warn("Unable to get repository connection : Network.opportunisticMergePeers()"); return; } From 47e313067fa9cc6f4951e6e554d8bec9767464c9 Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 13 Jun 2025 12:13:52 -0700 Subject: [PATCH 29/42] fixed a flaw in the blocks minted adjustment removal feature, instead of increasing or decreasing the level we need to reset the level when it is incorrect --- src/main/java/org/qortal/block/Block.java | 6 +- .../test/minting/BlocksMintedCountTests.java | 123 +++++++++++++++++- src/test/resources/test-chain-v2.json | 2 +- 3 files changed, 124 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index f2291910..1e07f2e7 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1735,7 +1735,7 @@ public class Block { for (int newLevel = maximumLevel; newLevel >= 0; --newLevel) if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) { - if (newLevel > accountData.getLevel()) { + if (newLevel != accountData.getLevel()) { // Account has increased in level! accountData.setLevel(newLevel); bumpedAccounts.put(accountData.getAddress(), newLevel); @@ -2141,7 +2141,7 @@ public class Block { int blocksMintedAdjustment = - (this.blockData.getHeight() > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight()) + (this.blockData.getHeight() -1 > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight()) ? 0 : @@ -2151,7 +2151,7 @@ public class Block { for (int newLevel = maximumLevel; newLevel >= 0; --newLevel) if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) { - if (newLevel < accountData.getLevel()) { + if (newLevel != accountData.getLevel()) { // Account has decreased in level! accountData.setLevel(newLevel); repository.getAccountRepository().setLevel(accountData); diff --git a/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java b/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java index 65a616b0..16ff354f 100644 --- a/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java +++ b/src/test/java/org/qortal/test/minting/BlocksMintedCountTests.java @@ -4,8 +4,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.block.BlockChain; import org.qortal.controller.BlockMinter; import org.qortal.controller.OnlineAccountsManager; +import org.qortal.data.account.AccountData; import org.qortal.data.account.RewardShareData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -15,8 +17,9 @@ import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TestAccount; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import java.util.List; + +import static org.junit.Assert.*; public class BlocksMintedCountTests extends Common { @@ -85,6 +88,121 @@ public class BlocksMintedCountTests extends Common { } } + @Test + public void testLevelSetting() { + + boolean exceptionThrown = false; + + try (final Repository repository = RepositoryManager.getRepository()) { + + // get the Alice's reward share account + PrivateKeyAccount aliceMintingAccount = Common.getTestAccount(repository, "alice-reward-share"); + + // give Alice an 8 blocks minted adjustment + int blocksMintedAdjustmentForAlice = 8; + adjustMintingData(repository, "alice", blocksMintedAdjustmentForAlice); + + // Confirm reward-share exists + RewardShareData aliceRewardShareData = repository.getAccountRepository().getRewardShare(aliceMintingAccount.getPublicKey()); + assertNotNull(aliceRewardShareData); + + // mint 40 blocks + for( int i = 0; i < 40; i++ ) { + // Create signed timestamps + OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(aliceMintingAccount); + + // Mint another block + BlockMinter.mintTestingBlockRetainingTimestamps(repository, aliceMintingAccount); + + // assert Alice's minting data after another block minted + assertMintingData(repository, "alice", blocksMintedAdjustmentForAlice); + + // orphan the block + BlockUtils.orphanLastBlock(repository); + + // assert the orphaning + assertMintingData(repository, "alice", blocksMintedAdjustmentForAlice); + + // mint another block to reverse the orpaning + BlockMinter.mintTestingBlockRetainingTimestamps(repository, aliceMintingAccount); + } + } + catch (DataException e) { + exceptionThrown = true; + } + + assertFalse(exceptionThrown); + } + + /** + * Assert Minting Data + * + * @param repository the data repository + * @param name the name of the minting account + * @param adjustment the blocks minted adjustment + * + * @throws DataException + */ + private static void assertMintingData(Repository repository, String name, int adjustment ) throws DataException { + + // get the test account data + TestAccount testAccount = Common.getTestAccount(repository, name); + AccountData testAccountData = repository.getAccountRepository().getAccount(testAccount.getAddress()); + + List blocksNeededByLevel = BlockChain.getInstance().getBlocksNeededByLevel(); + + // determine current height and adjustment ability + int height = repository.getBlockRepository().getBlockchainHeight(); + int adjustmentRemovalHeight = BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight(); + boolean isAdjustingEnabled = height <= adjustmentRemovalHeight; + + // initialize loop variables + int blocksLeft; + + if( isAdjustingEnabled ) + blocksLeft = testAccountData.getBlocksMinted() + adjustment; + else + blocksLeft = testAccountData.getBlocksMinted(); + + int index = 0; + int expectedLevel = 0; + + // update expected level based on the blocks needed by level list entries + while( blocksNeededByLevel.size() > index ) { + + Integer blocksNeededByThisLevel = blocksNeededByLevel.get(index); + if( blocksNeededByThisLevel <= blocksLeft ) { + expectedLevel++; + blocksLeft -= blocksNeededByThisLevel; + } + else { + break; + } + + index++; + } + + // print and assert the expected and derived numbers + System.out.println(String.format("height = %s,expectedLevel = %s, adjustment = %s, blocksMinted = %s", height, expectedLevel, adjustment, testAccountData.getBlocksMinted()) ); + assertEquals( expectedLevel, testAccountData.getLevel() ); + } + + /** + * Adjust Minting Data + * + * @param repository the data repository + * @param name the name of the account to adjust + * @param blocksMintedAdjustment the number of blocks to adjust + * + * @throws DataException + */ + private static void adjustMintingData(Repository repository, String name, int blocksMintedAdjustment) throws DataException { + TestAccount testAccount = Common.getTestAccount(repository, name); + AccountData testAccountData = repository.getAccountRepository().getAccount(testAccount.getAddress()); + testAccountData.setBlocksMintedAdjustment(blocksMintedAdjustment); + repository.getAccountRepository().setBlocksMintedAdjustment(testAccountData); + } + private void testRewardShare(Repository repository, PrivateKeyAccount testRewardShareAccount, int aliceDelta, int bobDelta) throws DataException { // Create signed timestamps OnlineAccountsManager.getInstance().ensureTestingAccountsOnline(testRewardShareAccount); @@ -124,5 +242,4 @@ public class BlocksMintedCountTests extends Common { TestAccount testAccount = Common.getTestAccount(repository, name); return repository.getAccountRepository().getAccount(testAccount.getAddress()).getBlocksMinted(); } - } diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 3bf89ab5..d7f7ea13 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -116,7 +116,7 @@ "nullGroupMembershipHeight": 20, "adminQueryFixHeight": 9999999999999, "multipleNamesPerAccountHeight": 10, - "mintedBlocksAdjustmentRemovalHeight": 9999999999999 + "mintedBlocksAdjustmentRemovalHeight": 27 }, "genesisInfo": { "version": 4, From 89236d6504628f5efeb2ad0b95fa9449ada813b5 Mon Sep 17 00:00:00 2001 From: kennycud Date: Sat, 14 Jun 2025 13:11:19 -0700 Subject: [PATCH 30/42] no longer repackaging missing data exceptions as io exceptions when loading json data for indices --- src/main/java/org/qortal/utils/ArbitraryIndexUtils.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java index 2ebd3b0e..4a539ca5 100644 --- a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java @@ -111,6 +111,8 @@ public class ArbitraryIndexUtils { indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1), indexResource.identifier )); } + } catch (MissingDataException e) { + LOGGER.warn( e.getMessage() ); } catch (InvalidFormatException e) { LOGGER.debug("invalid format, skipping: " + indexResource); } catch (UnrecognizedPropertyException e) { @@ -191,7 +193,7 @@ public class ArbitraryIndexUtils { } } - public static String getJson(String name, String identifier) throws IOException { + public static String getJson(String name, String identifier) throws IOException, MissingDataException { try { ArbitraryDataReader arbitraryDataReader @@ -209,7 +211,7 @@ public class ArbitraryIndexUtils { } catch (MissingDataException e) { if (attempts > maxAttempts) { // Give up after 5 attempts - throw new IOException("Data unavailable. Please try again later."); + throw e; } } } From 5b402e0bcaee7f867d56e689d267f58fb9803e62 Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 17 Jun 2025 15:08:20 -0700 Subject: [PATCH 31/42] validate name buyer's balance relative to the amount of the name purchase in addition to the fee --- src/main/java/org/qortal/transaction/BuyNameTransaction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/transaction/BuyNameTransaction.java b/src/main/java/org/qortal/transaction/BuyNameTransaction.java index 370e770a..fc5bf6fc 100644 --- a/src/main/java/org/qortal/transaction/BuyNameTransaction.java +++ b/src/main/java/org/qortal/transaction/BuyNameTransaction.java @@ -102,7 +102,7 @@ public class BuyNameTransaction extends Transaction { return ValidationResult.INVALID_AMOUNT; // Check buyer has enough funds - if (buyer.getConfirmedBalance(Asset.QORT) < this.buyNameTransactionData.getFee()) + if (buyer.getConfirmedBalance(Asset.QORT) < this.buyNameTransactionData.getFee() + this.buyNameTransactionData.getAmount()) return ValidationResult.NO_BALANCE; return ValidationResult.OK; From a3bb6638bf745088e97a17d138166df7da89b238 Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 17 Jun 2025 15:09:11 -0700 Subject: [PATCH 32/42] added support for single file websites --- src/main/java/org/qortal/arbitrary/misc/Service.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java index e154f001..21c027c4 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Service.java +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -62,7 +62,17 @@ public enum Service { // Custom validation function to require an index HTML file in the root directory List fileNames = ArbitraryDataRenderer.indexFiles(); - String[] files = path.toFile().list(); + List files; + + // single files are paackaged differently + if( path.toFile().isFile() ) { + files = new ArrayList<>(1); + files.add(path.getFileName().toString()); + } + else { + files = new ArrayList<>(Arrays.asList(path.toFile().list())); + } + if (files != null) { for (String file : files) { Path fileName = Paths.get(file).getFileName(); From 940c641759d9365a277e0442caf4150ab39d693f Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 17 Jun 2025 15:10:37 -0700 Subject: [PATCH 33/42] removed stack trace from streaming error warnings --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 79fb8528..bf69fda0 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -2089,7 +2089,7 @@ public String finalizeUpload( } catch (IOException e) { // Streaming errors should not rethrow — just log - LOGGER.warn(String.format("Streaming error for %s %s: %s", service, name, e.getMessage()), e); + LOGGER.warn(String.format("Streaming error for %s %s: %s", service, name, e.getMessage())); } } catch (IOException | ApiException | DataException e) { From 7a7f0e53ac325a133611fc8bbc735a28ba8ca8e0 Mon Sep 17 00:00:00 2001 From: kennycud Date: Tue, 17 Jun 2025 15:56:04 -0700 Subject: [PATCH 34/42] reduced index caching errors to warnings, because it is only an error if it continually happens --- src/main/java/org/qortal/utils/ArbitraryIndexUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java index 4a539ca5..156948a9 100644 --- a/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryIndexUtils.java @@ -53,7 +53,7 @@ public class ArbitraryIndexUtils { try { fillCache(IndexCache.getInstance()); } catch (IOException | DataException e) { - LOGGER.error(e.getMessage(), e); + LOGGER.warn(e.getMessage()); } } }; From 60b3bacd159b599b9b0aafa3255b0735426b1493 Mon Sep 17 00:00:00 2001 From: kennycud Date: Wed, 18 Jun 2025 17:55:30 -0700 Subject: [PATCH 35/42] reduced arbitrary data storage addition and deletion thresholds from 98% and 90% to 90% and 80% --- .../controller/arbitrary/ArbitraryDataStorageManager.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index c54a1e12..ab16605f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -47,15 +47,15 @@ public class ArbitraryDataStorageManager extends Thread { private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes - /** Treat storage as full at 90% usage, to reduce risk of going over the limit. + /** Treat storage as full at 80% usage, to reduce risk of going over the limit. * This is necessary because we don't calculate total storage values before every write. * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. * This must be lower than DELETION_THRESHOLD. */ - private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90% + private static final double STORAGE_FULL_THRESHOLD = 0.8f; // 80% - /** Start deleting files once we reach 98% usage. + /** Start deleting files once we reach 90% usage. * This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */ - public static final double DELETION_THRESHOLD = 0.98f; // 98% + public static final double DELETION_THRESHOLD = 0.9f; // 90% private static final long PER_NAME_STORAGE_MULTIPLIER = 4L; From 3687455c62100f5d66c55956ce0be07c1e2a17fb Mon Sep 17 00:00:00 2001 From: kennycud Date: Wed, 18 Jun 2025 17:57:18 -0700 Subject: [PATCH 36/42] increasing arbitrary data message thread limits, because the algorithms can handle it --- src/main/java/org/qortal/settings/Settings.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 3123ae96..59578ab8 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -756,14 +756,14 @@ public class Settings { private void setAdditionalDefaults() { // Populate defaults for maxThreadsPerMessageType. If any are specified in settings.json, they will take priority. maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 5)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 20)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA", 5)); maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE_LIST", 50)); maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE_LIST", 50)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_SIGNATURES", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_METADATA", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 50)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_METADATA", 100)); maxThreadsPerMessageType.add(new ThreadLimit("GET_TRANSACTION", 50)); maxThreadsPerMessageType.add(new ThreadLimit("TRANSACTION_SIGNATURES", 50)); maxThreadsPerMessageType.add(new ThreadLimit("TRADE_PRESENCES", 50)); From b8e171288175f7f56d09044467617f4b1e3884a7 Mon Sep 17 00:00:00 2001 From: PhilReact Date: Mon, 23 Jun 2025 13:48:35 +0300 Subject: [PATCH 37/42] add blob: to connect-src directive --- src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java index 72cd4097..e1794edf 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java @@ -169,7 +169,7 @@ public class ArbitraryDataRenderer { byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, includeResourceIdInPrefix, data, qdnContext, service, identifier, theme, usingCustomRouting, lang); htmlParser.addAdditionalHeaderTags(); - response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' wss:;"); + response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' wss: blob:;"); response.setContentType(context.getMimeType(filename)); response.setContentLength(htmlParser.getData().length); response.getOutputStream().write(htmlParser.getData()); From b48b6b9d42f8c199aab51ef9b652172284d88d77 Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 27 Jun 2025 14:01:51 -0700 Subject: [PATCH 38/42] added test cases for single file websites --- .../test/arbitrary/ArbitraryServiceTests.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java index b4c10fac..cd7d3b7a 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java @@ -98,6 +98,45 @@ public class ArbitraryServiceTests extends Common { assertEquals(ValidationResult.MISSING_INDEX_FILE, service.validate(path)); } + @Test + public void testValidateWebsiteWithIndexFile() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateWebsiteWithoutIndexFile"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "index.html"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data1.html"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data2"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "data3"), data, StandardOpenOption.CREATE); + + Service service = Service.WEBSITE; + assertTrue(service.isValidationRequired()); + + // There is no index file in the root + assertEquals(ValidationResult.OK, service.validate(path)); + } + + @Test + public void testValidateWebsiteWithIndexFileOnly() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateWebsiteWithoutIndexFile"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "index.html"), data, StandardOpenOption.CREATE); + + Service service = Service.WEBSITE; + assertTrue(service.isValidationRequired()); + + // There is no index file in the root + assertEquals(ValidationResult.OK, service.validate(path)); + } + @Test public void testValidateWebsiteWithoutIndexFileInRoot() throws IOException { // Generate some random data From 170668ef78ae38b62b0d97fd6c4366ac9a1ea61b Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 27 Jun 2025 14:04:39 -0700 Subject: [PATCH 39/42] reduced logging levels on numerous messages --- .../qortal/controller/ForeignFeesManager.java | 94 +++++++++---------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/qortal/controller/ForeignFeesManager.java b/src/main/java/org/qortal/controller/ForeignFeesManager.java index c2ffa70e..b6bc0303 100644 --- a/src/main/java/org/qortal/controller/ForeignFeesManager.java +++ b/src/main/java/org/qortal/controller/ForeignFeesManager.java @@ -169,7 +169,7 @@ public class ForeignFeesManager implements Listener { } catch (DataException | IOException e) { - LOGGER.info("Unable to import data into foreign fees manager: {}", e.getMessage()); + LOGGER.debug("Unable to import data into foreign fees manager: {}", e.getMessage()); } catch (Exception e) { LOGGER.error(e.getMessage(), e); @@ -220,12 +220,12 @@ public class ForeignFeesManager implements Listener { */ public void addSignedFees(List signedFees) { - LOGGER.info("adding signed fees: count = " + signedFees.size()); + LOGGER.debug("adding signed fees: count = " + signedFees.size()); // for each encoided fee, decode and add to import queue for( ForeignFeeEncodedData signedFeeEncoded : signedFees ) { - LOGGER.info("adding to import queue: " + signedFeeEncoded); + LOGGER.debug("adding to import queue: " + signedFeeEncoded); // decode the fee data and add to the queue this.foreignFeesImportQueue.add( @@ -236,10 +236,10 @@ public class ForeignFeesManager implements Listener { signedFeeEncoded.getFee() ) ); - LOGGER.info("added"); + LOGGER.debug("added"); } - LOGGER.info("done adding to queue: count = " + this.foreignFeesImportQueue.size()); + LOGGER.debug("done adding to queue: count = " + this.foreignFeesImportQueue.size()); // process the fees immediately (not waiting for the fee process timer task already in place) processForeignFeesImportQueue(); @@ -274,7 +274,7 @@ public class ForeignFeesManager implements Listener { long now = nowDetermined.get(); try (final Repository repository = RepositoryManager.getRepository()) { - LOGGER.info("processing trade offer in waiting event"); + LOGGER.debug("processing trade offer in waiting event"); Optional offerOptional = getTradeOfferData(repository, data.getAtAddress()); @@ -350,16 +350,16 @@ public class ForeignFeesManager implements Listener { try { if( this.needToBackupLockingForeignFees.compareAndSet( true, false )) { - LOGGER.info("backing up locking foreign fees"); + LOGGER.debug("backing up locking foreign fees"); backupForeignFeeData( bitcoiny -> bitcoiny.getFeePerKb().value, LOCKING_FOREIGN_FEES_FILE_NAME, LOCKING_FOREIGN_FEES_TYPE); } if( this.needToBackupRequiredForeignFees.compareAndSet(true, false) ) { - LOGGER.info("backing up required foreign fees"); + LOGGER.debug("backing up required foreign fees"); backupForeignFeeData(Bitcoiny::getFeeRequired, REQUIRED_FOREIGN_FEES_FILE_NAME, REQUIRED_FOREIGN_FEES_TYPE); } if( this.needToBackupSignedForeignFees.compareAndSet( true, false ) ) { - LOGGER.info("backing up signed foreign fees"); + LOGGER.debug("backing up signed foreign fees"); backupSignedForeignFeeData(); } } catch (DataException e) { @@ -380,15 +380,15 @@ public class ForeignFeesManager implements Listener { */ private void processForeignFeesImportQueue() { - LOGGER.info("processing foreign fees import queue ..."); + LOGGER.debug("processing foreign fees import queue ..."); if (this.foreignFeesImportQueue.isEmpty()) { - LOGGER.info("foreign fees import queue is empty"); + LOGGER.debug("foreign fees import queue is empty"); return; } - LOGGER.info("Processing foreign fee import queue (size: {})", this.foreignFeesImportQueue.size()); + LOGGER.debug("Processing foreign fee import queue (size: {})", this.foreignFeesImportQueue.size()); Set foreignFeesToRemove = new HashSet<>(this.foreignFeesImportQueue.size()); @@ -403,7 +403,7 @@ public class ForeignFeesManager implements Listener { // need to get the AT address for mapping key identification purposes String atAddress = foreignFeeToImport.getAtAddress(); - LOGGER.info("foreign fee import, timestamp = " + getFormattedDateTime(foreignFeeToImport.getTimestamp())); + LOGGER.debug("foreign fee import, timestamp = " + getFormattedDateTime(foreignFeeToImport.getTimestamp())); Optional validatedForeignFeeData = this.signedByAT.getOrDefault( atAddress, Optional.empty() ); @@ -415,7 +415,7 @@ public class ForeignFeesManager implements Listener { ATData atData = repository.getATRepository().fromATAddress(atAddress); - LOGGER.info("verify signer for atAddress = " + atAddress); + LOGGER.debug("verify signer for atAddress = " + atAddress); // determine if the creator authorized the foreign fee byte[] publicKey = atData.getCreatorPublicKey(); @@ -430,7 +430,7 @@ public class ForeignFeesManager implements Listener { // if trade offer creator authorized the imported fee, // then finish the import and clear it from the unsigned mapping if( Crypto.verify(publicKey, signature, message) ) { - LOGGER.info("signer verified"); + LOGGER.debug("signer verified"); this.signedByAT.put(atAddress, Optional.of(foreignFeeToImport)); this.needToBackupSignedForeignFees.compareAndSet(false, true); this.unsignedByAT.remove(atAddress); @@ -444,8 +444,8 @@ public class ForeignFeesManager implements Listener { .findAny() .isEmpty(); - LOGGER.info("tradeOfferCreatorAddress = " + tradeOfferCreatorAddress); - LOGGER.info("allSignedForCreatorAddress = " + allSignedForCreatorAddress); + LOGGER.debug("tradeOfferCreatorAddress = " + tradeOfferCreatorAddress); + LOGGER.debug("allSignedForCreatorAddress = " + allSignedForCreatorAddress); if(allSignedForCreatorAddress) { EventBus.INSTANCE.notify(new FeeWaitingEvent(false, tradeOfferCreatorAddress)); @@ -453,11 +453,11 @@ public class ForeignFeesManager implements Listener { } // otherwise this fee will get discarded else { - LOGGER.info("invalid signature"); + LOGGER.debug("invalid signature"); } } else { - LOGGER.info( + LOGGER.debug( "skipping imported fee since the timestamp is not updated: atAddress = {}, timestamp = {}", atAddress, foreignFeeToImport.getTimestamp() @@ -470,7 +470,7 @@ public class ForeignFeesManager implements Listener { } catch (Exception e) { LOGGER.error("Repository issue while verifying foreign fees", e); } finally { - LOGGER.info("removing foreign fees from import queue: count = " + foreignFeesToRemove.size()); + LOGGER.debug("removing foreign fees from import queue: count = " + foreignFeesToRemove.size()); this.foreignFeesImportQueue.removeAll(foreignFeesToRemove); } } @@ -500,7 +500,7 @@ public class ForeignFeesManager implements Listener { */ private void maintainCrossChainOffers() { - LOGGER.info("maintaining ATs ..."); + LOGGER.debug("maintaining ATs ..."); try (final Repository repository = RepositoryManager.getRepository()) { @@ -513,14 +513,14 @@ public class ForeignFeesManager implements Listener { .map( data -> data.qortalAtAddress ) .collect(Collectors.toSet()); - LOGGER.info("foreign fees before AT removal: count = " + this.signedByAT.size() ); + LOGGER.debug("foreign fees before AT removal: count = " + this.signedByAT.size() ); // retain the fees for the current sell offers, remove all others if( retainFeeByAT(this.signedByAT, atAddresses) ) { this.needToBackupSignedForeignFees.compareAndSet(false, true); } - LOGGER.info("foreign fees after AT removal: count = " + this.signedByAT.size() ); + LOGGER.debug("foreign fees after AT removal: count = " + this.signedByAT.size() ); } catch (Exception e) { LOGGER.error(e.getMessage(), e); } @@ -649,11 +649,11 @@ public class ForeignFeesManager implements Listener { */ private void requestRemoteForeignFees() { - LOGGER.info("requesting remote foreign fees ..."); + LOGGER.debug("requesting remote foreign fees ..."); if (!isUpToDate()) return; - LOGGER.info("Requesting foreign fees via broadcast..."); + LOGGER.debug("Requesting foreign fees via broadcast..."); Message message = new GetForeignFeesMessage( @@ -664,7 +664,7 @@ public class ForeignFeesManager implements Listener { Network.getInstance().broadcast(peer -> message); - LOGGER.info("Requested foreign fees via broadcast..."); + LOGGER.debug("Requested foreign fees via broadcast..."); } /** @@ -683,7 +683,7 @@ public class ForeignFeesManager implements Listener { if (!Controller.getInstance().isUpToDate()) { - LOGGER.info("not up to date, aborting"); + LOGGER.debug("not up to date, aborting"); return false; } return true; @@ -731,7 +731,7 @@ public class ForeignFeesManager implements Listener { Set addressesThatNeedSignatures = new HashSet<>(); - LOGGER.info("processing local foreign fees ..."); + LOGGER.debug("processing local foreign fees ..."); Optional nowDetermined = determineNow(); if (nowDetermined.isEmpty()){ @@ -750,7 +750,7 @@ public class ForeignFeesManager implements Listener { .filter(d -> SupportedBlockchain.getAcctByName( d.getAcctName() ).getBlockchain().equals( bitcoiny )) .collect(Collectors.toList()); - LOGGER.info("trade offers waiting: count = " + tradeOffersWaiting.size()); + LOGGER.debug("trade offers waiting: count = " + tradeOffersWaiting.size()); // process each local trade offer waiting (listed) for (TradeBotData tradeOfferWaiting : tradeOffersWaiting) { @@ -810,7 +810,7 @@ public class ForeignFeesManager implements Listener { String foreignBlockchain = tradeOfferWaiting.getForeignBlockchain(); SupportedBlockchain supportedBlockchain = SupportedBlockchain.fromString(foreignBlockchain); - LOGGER.info("trade offer waiting: blockchain = " + foreignBlockchain); + LOGGER.debug("trade offer waiting: blockchain = " + foreignBlockchain); // if the supported blockchain is a Bitcoiny blockchain, then the fee will be available if (supportedBlockchain.getInstance() instanceof Bitcoiny) { @@ -820,7 +820,7 @@ public class ForeignFeesManager implements Listener { String atAddress = tradeOfferWaiting.getAtAddress(); int fee = Math.toIntExact(bitcoiny.getFeeRequired()); - LOGGER.info("atAddress = {}, fee = {}", atAddress, fee); + LOGGER.debug("atAddress = {}, fee = {}", atAddress, fee); // get the signed foreign fee, if it exists Optional foreignFeeDecodedData = this.signedByAT.get(atAddress); @@ -828,13 +828,13 @@ public class ForeignFeesManager implements Listener { // if the foreign fee has been signed if (foreignFeeDecodedData != null && foreignFeeDecodedData.isPresent()) { - LOGGER.info("signed available"); + LOGGER.debug("signed available"); // if the local fee is different than the fee stored in this manager, // then empty the fee in the manager and set the updated fee to unsigned data if (!foreignFeeDecodedData.get().getFee().equals(fee)) { - LOGGER.info("fee updated"); + LOGGER.debug("fee updated"); this.signedByAT.remove(atAddress); this.needToBackupSignedForeignFees.compareAndSet(false, true); @@ -842,12 +842,12 @@ public class ForeignFeesManager implements Listener { isFeeWaiting = true; } else { - LOGGER.info("fee not updated"); + LOGGER.debug("fee not updated"); } } // if the foreign fee has not been signed, then set the fee to unsigned data else { - LOGGER.info("fee not signed"); + LOGGER.debug("fee not signed"); setUnsignedData(now, atAddress, fee); isFeeWaiting = true; } @@ -878,9 +878,9 @@ public class ForeignFeesManager implements Listener { fee ); - LOGGER.info("updating unsigned"); + LOGGER.debug("updating unsigned"); this.unsignedByAT.put(atAddress, feeData); - LOGGER.info("updated unsigned = " + this.unsignedByAT); + LOGGER.debug("updated unsigned = " + this.unsignedByAT); } // Network handlers @@ -912,12 +912,12 @@ public class ForeignFeesManager implements Listener { String atAddress = entry.getKey(); - LOGGER.info("comparing signed foreign fee for get foreign fee message: atAddress = " + atAddress); + LOGGER.debug("comparing signed foreign fee for get foreign fee message: atAddress = " + atAddress); // if message contains AT address, then check timestamps if (inMessageDataByAT.containsKey(atAddress) ) { - LOGGER.info("message does contain: atAddress = " + atAddress); + LOGGER.debug("message does contain: atAddress = " + atAddress); // get data from message for AT address ForeignFeeDecodedData feeData = inMessageDataByAT.get(atAddress); @@ -929,23 +929,23 @@ public class ForeignFeesManager implements Listener { } // if the message does not contain data for this AT, then send the data out to the peer else { - LOGGER.info("message does not contain: atAddress = " + atAddress); + LOGGER.debug("message does not contain: atAddress = " + atAddress); outgoingForeignFees.add(signedForeignFeeData.get()); } } // if value is empty, then do nothing else { - LOGGER.info("unavailable signed foreign fee for get foreign fee message: atAddress = " + entry.getKey()); + LOGGER.debug("unavailable signed foreign fee for get foreign fee message: atAddress = " + entry.getKey()); } } - LOGGER.info("Sending {} foreign fees to {}", outgoingForeignFees.size(), peer); + LOGGER.debug("Sending {} foreign fees to {}", outgoingForeignFees.size(), peer); // send out to peer peer.sendMessage(new ForeignFeesMessage(outgoingForeignFees)); - LOGGER.info("Sent {} foreign fees to {}", outgoingForeignFees.size(), peer); + LOGGER.debug("Sent {} foreign fees to {}", outgoingForeignFees.size(), peer); } /** @@ -958,7 +958,7 @@ public class ForeignFeesManager implements Listener { ForeignFeesMessage onlineAccountsMessage = (ForeignFeesMessage) message; List peersForeignFees = onlineAccountsMessage.getForeignFees(); - LOGGER.info("Received {} foreign fees from {}", peersForeignFees.size(), peer); + LOGGER.debug("Received {} foreign fees from {}", peersForeignFees.size(), peer); int importCount = 0; @@ -970,7 +970,7 @@ public class ForeignFeesManager implements Listener { } if (importCount > 0) - LOGGER.info("Added {} foreign to queue", importCount); + LOGGER.debug("Added {} foreign to queue", importCount); } /** @@ -1099,7 +1099,7 @@ public class ForeignFeesManager implements Listener { throw new FileNotFoundException(String.format("Unable to read file contents: %s", filename)); } - LOGGER.info(String.format("Importing %s into foreign fees manager ...", filename)); + LOGGER.debug(String.format("Importing %s into foreign fees manager ...", filename)); String jsonString = new String(fileContents); @@ -1129,7 +1129,7 @@ public class ForeignFeesManager implements Listener { } } - LOGGER.info(String.format("Imported %s into foreign fees manager from %s", type, filename)); + LOGGER.debug(String.format("Imported %s into foreign fees manager from %s", type, filename)); } /** From b2579a457cbb271c02cf8588c572dce5c8b05373 Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 27 Jun 2025 14:09:08 -0700 Subject: [PATCH 40/42] reverting the GET_ARBITRARY_DATA_FILE thread limit, because it puts too much pressure on the peers with the previously lower limit, planning on updating this to a higher number right before the next release when all nodes are ready for it --- src/main/java/org/qortal/settings/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 59578ab8..e5efc7b4 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -756,7 +756,7 @@ public class Settings { private void setAdditionalDefaults() { // Populate defaults for maxThreadsPerMessageType. If any are specified in settings.json, they will take priority. maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE", 5)); - maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 20)); + maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA_FILE", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA", 5)); maxThreadsPerMessageType.add(new ThreadLimit("GET_ARBITRARY_DATA", 5)); maxThreadsPerMessageType.add(new ThreadLimit("ARBITRARY_DATA_FILE_LIST", 50)); From 65c014b21598ce82389821dea8f0828bee8ab957 Mon Sep 17 00:00:00 2001 From: kennycud Date: Fri, 27 Jun 2025 14:13:14 -0700 Subject: [PATCH 41/42] removed redundant data collecting, reintroduced relay timeout threshold --- .../ArbitraryDataFileRequestThread.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index 1872898f..b27eef26 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -76,15 +76,13 @@ public class ArbitraryDataFileRequestThread { LOGGER.error(e.getMessage(), e); } } + public void processFileHashes(Long now, List responseInfos, ArbitraryDataFileManager arbitraryDataFileManager) { if (Controller.isStopping()) { return; } - List toProcess = new ArrayList<>(responseInfos.size()); - - Map responseInfoByHash58 = new HashMap<>(responseInfos.size()); - Map signatureBySignature58 = new HashMap<>(toProcess.size()); + Map signatureBySignature58 = new HashMap<>(responseInfos.size()); Map> responseInfoBySignature58 = new HashMap<>(); for( ArbitraryFileListResponseInfo responseInfo : responseInfos) { @@ -119,8 +117,6 @@ public class ArbitraryDataFileRequestThread { } // We want to process this file, store and map data to process later - toProcess.add(responseInfo); - responseInfoByHash58.put(responseInfo.getHash58(), responseInfo); signatureBySignature58.put(responseInfo.getSignature58(), signature); responseInfoBySignature58 .computeIfAbsent(responseInfo.getSignature58(), signature58 -> new ArrayList<>()) @@ -162,6 +158,17 @@ public class ArbitraryDataFileRequestThread { private void arbitraryDataFileFetcher(ArbitraryDataFileManager arbitraryDataFileManager, ArbitraryFileListResponseInfo responseInfo, ArbitraryTransactionData arbitraryTransactionData) { try { + Long now = NTP.getTime(); + + if (now - responseInfo.getTimestamp() >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT ) { + + Peer peer = responseInfo.getPeer(); + String hash58 = responseInfo.getHash58(); + String signature58 = responseInfo.getSignature58(); + LOGGER.debug("Peer {} version {} didn't fetch data file {} for signature {} due to relay timeout.", peer, peer.getPeersVersionString(), hash58, signature58); + return; + } + arbitraryDataFileManager.fetchArbitraryDataFiles( responseInfo.getPeer(), arbitraryTransactionData.getSignature(), From b2c72c3927cbe83104aabc79ea0cad9b2f9442fa Mon Sep 17 00:00:00 2001 From: kennycud Date: Sun, 29 Jun 2025 11:08:42 -0700 Subject: [PATCH 42/42] null pointer solution by using an empty list instead of a null value --- src/main/java/org/qortal/controller/ForeignFeesManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/ForeignFeesManager.java b/src/main/java/org/qortal/controller/ForeignFeesManager.java index b6bc0303..3763b7b3 100644 --- a/src/main/java/org/qortal/controller/ForeignFeesManager.java +++ b/src/main/java/org/qortal/controller/ForeignFeesManager.java @@ -438,7 +438,7 @@ public class ForeignFeesManager implements Listener { String tradeOfferCreatorAddress = Crypto.toAddress(publicKey); boolean allSignedForCreatorAddress = this.offersByAddress - .get(tradeOfferCreatorAddress).stream() + .getOrDefault(tradeOfferCreatorAddress, new ArrayList<>(0)).stream() .map(data -> data.qortalAtAddress) .filter(qortalAtAddress -> this.unsignedByAT.contains(qortalAtAddress)) .findAny()